feat: Cache DSM info (#378)

* feat: Cache location

* feat: Cache location

* feat: Add support URL

* feat: Cache location

* fix: Remove files

* feat: Reset filesystem

* fix: Exit when PID is missing

* fix: Counter file

* fix: Check flags

* docs: Readme

* feat: Cleanup files

* fix: Check flags

* fix: Check flags

* fix: Initialization

* fix: Initalization

* fix: Initialization

* fix: Cleanup temp

* fix: Initialize system

* feat: Config system

* feat: Configure system

* fix: Variables

* fix: Variables

* fix: Error handling

* style: Comments

* fix: Returnvalue

* fix: Returnvalue

* fix: Returnvalue

* fix: Returnvalue

* fix: Returnvalue

* docs: Multi-disk support

* feat: Use cached location

* fix: Swap order

readme.md

@@ -17,7 +17,6 @@ Virtual DSM in a docker container.
  - Multi-platform
  - KVM acceleration
  - GPU passthrough
- - Graceful shutdowns
  - Upgrades supported
  
 ## Usage
@@ -86,6 +85,32 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti
 
     Keep in mind that this will not affect any of your existing disks, it only applies to newly created disks.
     
+  * ### How do I add multiple disks?
+
+    To mount extra volumes, modify your compose file like this:
+    
+    ```yaml
+    environment:
+      DISK2_SIZE: "32G"
+      DISK3_SIZE: "64G"
+    volumes:
+      - /home/example2:/storage2
+      - /home/example3:/storage3
+    ```
+
+    Additionally, it's also possible to passthrough raw disk devices like this:
+
+    ```yaml
+    environment:
+      DEVICE2: "/dev/vdc1"
+      DEVICE3: "/dev/vdc2"
+    devices:
+      - /dev/vdc1
+      - /dev/vdc2
+    ```    
+
+    Please beware that any pre-existing partitions and data on those devices will be wiped.
+
   * ### How do I increase the amount of CPU or RAM?
 
     By default, a single core and 512 MB of RAM are allocated to the container. To increase this, add the following environment variables:

run/check.sh

@@ -2,7 +2,11 @@
 set -u
 
 [ ! -f "/run/qemu.pid" ] && echo "QEMU not running yet.." && exit 0
-[ -f "/run/qemu.counter" ] && echo "QEMU is shutting down.." && exit 1
+[ -f "/run/qemu.count" ] && echo "QEMU is shutting down.." && exit 1
+
+file="/run/dsm.url"
+
+if [ ! -f  "$file" ]; then
 
   # Retrieve IP from guest VM for Docker healthcheck
   RESPONSE=$(curl -s -m 16 -S http://127.0.0.1:2210/read?command=10 2>&1)
@@ -36,15 +40,22 @@ IP=${rest%%\"*}
 
   [ -z "${IP}" ] && echo "Guest has not received an IP yet.." && exit 1
 
-if ! curl -m 3 -ILfSs "http://${IP}:${PORT}/" > /dev/null; then
+  echo "${IP}:${PORT}" > $file
-  echo "Failed to reach ${IP}:${PORT}"
+
+fi
+
+LOCATION=$(cat "$file")
+
+if ! curl -m 20 -ILfSs "http://${LOCATION}/" > /dev/null; then
+  rm -f $file
+  echo "Failed to reach http://${LOCATION}"
   exit 1
 fi
 
-if [[ "$IP" == "20.20"* ]]; then
+if [[ "$LOCATION" == "20.20"* ]]; then
   echo "Healthcheck OK"
 else
-  echo "Healthcheck OK ( $IP )"
+  echo "Healthcheck OK ( ${LOCATION%:*} )"
 fi
 
 exit 0

run/config.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+KVM_ERR=""
+KVM_OPTS=""
+
+if [ -e /dev/kvm ] && sh -c 'echo -n > /dev/kvm' &> /dev/null; then
+  if ! grep -q -e vmx -e svm /proc/cpuinfo; then
+    KVM_ERR="(vmx/svm disabled)"
+  fi
+else
+  [ -e /dev/kvm ] && KVM_ERR="(no write access)" || KVM_ERR="(device file missing)"
+fi
+
+if [ -n "${KVM_ERR}" ]; then
+  if [ "$ARCH" == "amd64" ]; then
+    error "KVM acceleration not detected ${KVM_ERR}, see the FAQ about this."
+    [[ "${DEBUG}" != [Yy1]* ]] && exit 88
+  fi
+else
+  KVM_OPTS=",accel=kvm -enable-kvm -cpu host"
+fi
+
+DEF_OPTS="-nographic -nodefaults -boot strict=on -display none"
+RAM_OPTS=$(echo "-m ${RAM_SIZE}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
+CPU_OPTS="-smp ${CPU_CORES},sockets=1,dies=1,cores=${CPU_CORES},threads=1"
+MAC_OPTS="-machine type=q35,usb=off,dump-guest-core=off,hpet=off${KVM_OPTS}"
+EXTRA_OPTS="-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4"
+EXTRA_OPTS="$EXTRA_OPTS -object rng-random,id=objrng0,filename=/dev/urandom"
+EXTRA_OPTS="$EXTRA_OPTS -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pcie.0,addr=0x1c"
+
+if [[ "${GPU}" == [Yy1]* ]] && [[ "$ARCH" == "amd64" ]]; then
+  DEF_OPTS="-nodefaults -boot strict=on -display egl-headless,rendernode=/dev/dri/renderD128"
+  DEF_OPTS="${DEF_OPTS} -device virtio-vga,id=video0,max_outputs=1,bus=pcie.0,addr=0x1"
+fi
+
+ARGS="${DEF_OPTS} ${CPU_OPTS} ${RAM_OPTS} ${MAC_OPTS} ${MON_OPTS} ${SERIAL_OPTS} ${NET_OPTS} ${DISK_OPTS} ${EXTRA_OPTS} ${ARGUMENTS}"
+ARGS=$(echo "$ARGS" | sed 's/\t/ /g' | tr -s ' ')
+
+return 0

run/disk.sh

@@ -237,3 +237,5 @@ if [ -n "${DEVICE3}" ]; then
     -device scsi-hd,bus=hw-userdata6.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata6,id=userdata6,rotation_rate=${DISK_ROTATION},bootindex=8"
 
 fi
+
+return 0

run/gpu.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 set -Eeuo pipefail
 
+if [[ "${GPU}" != [Yy1]* ]] || [[ "$ARCH" != "amd64" ]]; then
+  return 0
+fi
+
 [ ! -d /dev/dri ] && mkdir -m 755 /dev/dri
 
 if [ ! -c /dev/dri/card0 ]; then
@@ -14,9 +18,6 @@ fi
 chmod 666 /dev/dri/card0
 chmod 666 /dev/dri/renderD128
 
-DEF_OPTS="-nodefaults -boot strict=on -display egl-headless,rendernode=/dev/dri/renderD128"
-DEF_OPTS="${DEF_OPTS} -device virtio-vga,id=video0,max_outputs=1,bus=pcie.0,addr=0x1"
-
 if ! apt-mark showinstall | grep -q "xserver-xorg-video-intel"; then
 
   info "Installing Intel GPU drivers..."
@@ -40,3 +41,5 @@ if ! apt-mark showinstall | grep -q "qemu-system-modules-opengl"; then
   apt-get -qq --no-install-recommends -y install qemu-system-modules-opengl > /dev/null
 
 fi
+
+return 0

run/install.sh

@@ -1,6 +1,23 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
+STORAGE="/storage"
+[ ! -d "$STORAGE" ] && error "Storage folder (${STORAGE}) not found!" && exit 13
+
+if [ -f "$STORAGE"/dsm.ver ]; then
+  BASE=$(cat "${STORAGE}/dsm.ver")
+else
+  # Fallback for old installs
+  BASE="DSM_VirtualDSM_42962"
+fi
+
+[ -n "$URL" ] && BASE=$(basename "$URL" .pat)
+
+if [[ -f "$STORAGE/$BASE.boot.img" ]] && [[ -f "$STORAGE/$BASE.system.img" ]]; then
+  # Previous installation found
+  return 0
+fi
+
 # Display wait message
 /run/server.sh 5000 install &
 
@@ -31,7 +48,6 @@ rm -f "$STORAGE"/"$BASE".agent
 rm -f "$STORAGE"/"$BASE".boot.img
 rm -f "$STORAGE"/"$BASE".system.img
 
-TMP="/tmp/dsm"
 MIN_SPACE=6442450944
 FS=$(stat -f -c %T "$STORAGE")
 
@@ -39,10 +55,13 @@ if [[ "$FS" != "fat"* && "$FS" != "vfat"* && "$FS" != "exfat"* && \
         "$FS" != "ntfs"* && "$FS" != "fuse"* && "$FS" != "msdos"* ]]; then
   TMP="$STORAGE/tmp"
 else
+  TMP="/tmp/dsm"
   SPACE=$(df --output=avail -B 1 /tmp | tail -n 1)
   (( MIN_SPACE > SPACE )) && TMP="$STORAGE/tmp"
 fi
 
+rm -rf /tmp/dsm
+rm -rf "$STORAGE/tmp"
 rm -rf "$TMP" && mkdir -p "$TMP"
 
 # Check free diskspace

run/power.sh

@@ -3,14 +3,14 @@ set -Eeuo pipefail
 
 # Configure QEMU for graceful shutdown
 
-QEMU_MONPORT=7100
+QEMU_PORT=7100
-QEMU_POWERDOWN_TIMEOUT=50
+QEMU_TIMEOUT=50
 
-_QEMU_PID=/run/qemu.pid
+QEMU_PID=/run/qemu.pid
-_QEMU_SHUTDOWN_COUNTER=/run/qemu.counter
+QEMU_COUNT=/run/qemu.count
 
-rm -f "${_QEMU_PID}"
+rm -f "${QEMU_PID}"
-rm -f "${_QEMU_SHUTDOWN_COUNTER}"
+rm -f "${QEMU_COUNT}"
 
 _trap(){
     func="$1" ; shift
@@ -23,14 +23,14 @@ _graceful_shutdown() {
 
   set +e
 
-  [ ! -f "${_QEMU_PID}" ] && return
+  [ ! -f "${QEMU_PID}" ] && exit 130
-  [ -f "${_QEMU_SHUTDOWN_COUNTER}" ] && return
+  [ -f "${QEMU_COUNT}" ] && return
 
   echo && info "Received $1 signal, shutting down..."
-  echo 0 > "${_QEMU_SHUTDOWN_COUNTER}"
+  echo 0 > "${QEMU_COUNT}"
 
   # Don't send the powerdown signal because vDSM ignores ACPI signals
-  # echo 'system_powerdown' | nc -q 1 -w 1 localhost "${QEMU_MONPORT}" > /dev/null
+  # echo 'system_powerdown' | nc -q 1 -w 1 localhost "${QEMU_PORT}" > /dev/null
 
   # Send shutdown command to guest agent via serial port
   RESPONSE=$(curl -s -m 5 -S http://127.0.0.1:2210/read?command=6 2>&1)
@@ -39,22 +39,22 @@ _graceful_shutdown() {
 
     echo && error "Could not send shutdown command to the guest ($RESPONSE)"
 
-    kill -15 "$(cat "${_QEMU_PID}")"
+    kill -15 "$(cat "${QEMU_PID}")"
     pkill -f qemu-system-x86_64 || true
 
   fi
 
-  while [ "$(cat ${_QEMU_SHUTDOWN_COUNTER})" -lt "${QEMU_POWERDOWN_TIMEOUT}" ]; do
+  while [ "$(cat ${QEMU_COUNT})" -lt "${QEMU_TIMEOUT}" ]; do
 
     # Increase the counter
-    echo $(($(cat ${_QEMU_SHUTDOWN_COUNTER})+1)) > ${_QEMU_SHUTDOWN_COUNTER}
+    echo $(($(cat ${QEMU_COUNT})+1)) > ${QEMU_COUNT}
 
     # Try to connect to qemu
-    if echo 'info version'| nc -q 1 -w 1 localhost "${QEMU_MONPORT}" >/dev/null 2>&1 ; then
+    if echo 'info version'| nc -q 1 -w 1 localhost "${QEMU_PORT}" >/dev/null 2>&1 ; then
 
       sleep 1
 
-      CNT="$(cat ${_QEMU_SHUTDOWN_COUNTER})/${QEMU_POWERDOWN_TIMEOUT}"
+      CNT="$(cat ${QEMU_COUNT})/${QEMU_TIMEOUT}"
       [[ "${DEBUG}" == [Yy1]* ]] && info "Shutting down, waiting... (${CNT})"
 
     fi
@@ -62,7 +62,7 @@ _graceful_shutdown() {
   done
 
   echo && echo "❯ Quitting..."
-  echo 'quit' | nc -q 1 -w 1 localhost "${QEMU_MONPORT}" >/dev/null 2>&1 || true
+  echo 'quit' | nc -q 1 -w 1 localhost "${QEMU_PORT}" >/dev/null 2>&1 || true
 
   closeNetwork
 
@@ -71,4 +71,4 @@ _graceful_shutdown() {
 
 _trap _graceful_shutdown SIGTERM SIGHUP SIGINT SIGABRT SIGQUIT
 
-MON_OPTS="-monitor telnet:localhost:${QEMU_MONPORT},server,nowait,nodelay"
+MON_OPTS="-monitor telnet:localhost:${QEMU_PORT},server,nowait,nodelay"

run/print.sh

@@ -4,12 +4,13 @@ set -Eeuo pipefail
 info () { echo -e >&2 "\E[1;34m❯\E[1;36m $1\E[0m" ; }
 error () { echo -e >&2 "\E[1;31m❯ ERROR: $1\E[0m" ; }
 
-retry=true
+file="/run/dsm.url"
 
-while [ "$retry" = true ]
+while [ ! -f  "$file" ]
 do
 
   sleep 3
+  [ -f "$file" ] && continue
 
   # Retrieve IP from guest VM
 
@@ -46,14 +47,16 @@ do
 
   [ -z "${IP}" ] && continue
 
-  retry=false
+  echo "${IP}:${PORT}" > $file
 
 done
 
-if [[ "$IP" == "20.20"* ]]; then
+LOCATION=$(cat "$file")
-  MSG="port ${PORT}"
+
+if [[ "$LOCATION" == "20.20"* ]]; then
+  MSG="port ${LOCATION##*:}"
 else
-  MSG="http://${IP}:${PORT}"
+  MSG="http://${LOCATION}"
 fi
 
 echo "" >&2

run/reset.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+info () { echo -e "\E[1;34m❯ \E[1;36m$1\E[0m" ; }
+error () { echo -e >&2 "\E[1;31m❯ ERROR: $1\E[0m" ; }
+trap 'error "Status $? while: ${BASH_COMMAND} (line $LINENO/$BASH_LINENO)"' ERR
+
+[ ! -f "/run/run.sh" ] && error "Script must run inside Docker container!" && exit 11
+[ "$(id -u)" -ne "0" ] && error "Script must be executed with root privileges." && exit 12
+
+# Docker environment variables
+
+: ${URL:=''}            # URL of the PAT file
+: ${GPU:='N'}           # Enable GPU passthrough
+: ${DEBUG:='N'}         # Enable debugging mode
+: ${ALLOCATE:='Y'}      # Preallocate diskspace
+: ${ARGUMENTS:=''}      # Extra QEMU parameters
+: ${CPU_CORES:='1'}     # Amount of CPU cores
+: ${DISK_SIZE:='16G'}   # Initial data disk size
+: ${RAM_SIZE:='512M'}   # Maximum RAM amount
+
+# Helper variables
+
+KERNEL=$(uname -r | cut -b 1)
+MINOR=$(uname -r | cut -d '.' -f2)
+ARCH=$(dpkg --print-architecture)
+VERS=$(qemu-system-x86_64 --version | head -n 1 | cut -d '(' -f 1)
+
+# Cleanup files
+
+rm -f /run/dsm.url
+rm -f /run/qemu.pid
+rm -f /run/qemu.count
+
+return 0

run/run.sh

@@ -1,97 +1,26 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-# Docker environment variables
-
-: ${URL:=''}            # URL of the PAT file
-: ${GPU:='N'}           # Enable GPU passthrough
-: ${DEBUG:='N'}         # Enable debugging mode
-: ${ALLOCATE:='Y'}      # Preallocate diskspace
-: ${ARGUMENTS:=''}      # Extra QEMU parameters
-: ${CPU_CORES:='1'}     # Amount of CPU cores
-: ${DISK_SIZE:='16G'}   # Initial data disk size
-: ${RAM_SIZE:='512M'}   # Maximum RAM amount
-
 echo "❯ Starting Virtual DSM for Docker v${VERSION}..."
+echo "❯ For support visit https://github.com/vdsm/virtual-dsm/"
 
-info () { echo -e "\E[1;34m❯ \E[1;36m$1\E[0m" ; }
+. /run/reset.sh   # Initialize system
-error () { echo -e >&2 "\E[1;31m❯ ERROR: $1\E[0m" ; }
+. /run/install.sh   # Run installation
-trap 'error "Status $? while: ${BASH_COMMAND} (line $LINENO/$BASH_LINENO)"' ERR
-
-[ ! -f "/run/run.sh" ] && error "Script must run inside Docker container!" && exit 11
-[ "$(id -u)" -ne "0" ] && error "Script must be executed with root privileges." && exit 12
-
-STORAGE="/storage"
-KERNEL=$(uname -r | cut -b 1)
-MINOR=$(uname -r | cut -d '.' -f2)
-ARCH=$(dpkg --print-architecture)
-VERS=$(qemu-system-x86_64 --version | head -n 1 | cut -d '(' -f 1)
-
-[ ! -d "$STORAGE" ] && error "Storage folder (${STORAGE}) not found!" && exit 13
-
-if [ -f "$STORAGE"/dsm.ver ]; then
-  BASE=$(cat "${STORAGE}/dsm.ver")
-else
-  # Fallback for old installs
-  BASE="DSM_VirtualDSM_42962"
-fi
-
-[ -n "$URL" ] && BASE=$(basename "$URL" .pat)
-
-if [[ ! -f "$STORAGE/$BASE.boot.img" ]] || [[ ! -f "$STORAGE/$BASE.system.img" ]]; then
-  . /run/install.sh
-fi
-
 . /run/disk.sh     # Initialize disks
 . /run/network.sh  # Initialize network
+. /run/gpu.sh     # Initialize graphics
 . /run/serial.sh   # Initialize serialport
 . /run/power.sh    # Configure shutdown
-
+. /run/config.sh    # Configure arguments
-KVM_ERR=""
-KVM_OPTS=""
-
-if [ -e /dev/kvm ] && sh -c 'echo -n > /dev/kvm' &> /dev/null; then
-  if ! grep -q -e vmx -e svm /proc/cpuinfo; then
-    KVM_ERR="(vmx/svm disabled)"
-  fi
-else
-  [ -e /dev/kvm ] && KVM_ERR="(no write access)" || KVM_ERR="(device file missing)"
-fi
-
-if [ -n "${KVM_ERR}" ]; then
-  if [ "$ARCH" == "amd64" ]; then
-    error "KVM acceleration not detected ${KVM_ERR}, see the FAQ about this."
-    [[ "${DEBUG}" != [Yy1]* ]] && exit 88
-  fi
-else
-  KVM_OPTS=",accel=kvm -enable-kvm -cpu host"
-fi
-
-DEF_OPTS="-nographic -nodefaults -boot strict=on -display none"
-RAM_OPTS=$(echo "-m ${RAM_SIZE}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
-CPU_OPTS="-smp ${CPU_CORES},sockets=1,dies=1,cores=${CPU_CORES},threads=1"
-MAC_OPTS="-machine type=q35,usb=off,dump-guest-core=off,hpet=off${KVM_OPTS}"
-EXTRA_OPTS="-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4"
-EXTRA_OPTS="$EXTRA_OPTS -object rng-random,id=objrng0,filename=/dev/urandom"
-EXTRA_OPTS="$EXTRA_OPTS -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pcie.0,addr=0x1c"
-
-[[ "${GPU}" == [Yy1]* ]] && [[ "$ARCH" == "amd64" ]] && . /run/gpu.sh
-
-ARGS="${DEF_OPTS} ${CPU_OPTS} ${RAM_OPTS} ${MAC_OPTS} ${MON_OPTS} ${SERIAL_OPTS} ${NET_OPTS} ${DISK_OPTS} ${EXTRA_OPTS} ${ARGUMENTS}"
-ARGS=$(echo "$ARGS" | sed 's/\t/ /g' | tr -s ' ')
 
 trap - ERR
 
 set -m
 (
   [[ "${DEBUG}" == [Yy1]* ]] && info "$VERS" && set -x
-  qemu-system-x86_64 ${ARGS:+ $ARGS} & echo $! > "${_QEMU_PID}"
+  qemu-system-x86_64 ${ARGS:+ $ARGS} & echo $! > "${QEMU_PID}"
   { set +x; } 2>/dev/null
 )
 set +m
 
-#if (( KERNEL > 5 )) || ( (( KERNEL == 5 )) && (( MINOR > 2 )) ); then
+tail --pid "$(cat "${QEMU_PID}")" --follow /dev/null & wait $!
-#  pidwait -F "${_QEMU_PID}" & wait $!
-#else
-
-tail --pid "$(cat "${_QEMU_PID}")" --follow /dev/null & wait $!

run/serial.sh

@@ -51,3 +51,5 @@ SERIAL_OPTS="\
         -device isa-serial,chardev=charserial0,id=serial0 \
         -chardev socket,id=charchannel0,host=127.0.0.1,port=12345,reconnect=10 \
         -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=vchannel"
+
+return 0

run/server.sh

@@ -46,17 +46,15 @@ if [[ "$2" != "/run/ip.sh" ]]; then
 
 else
 
-  BODY="The location of DSM is <a href='http://\${IP}:\${PORT}'>http://\${IP}:\${PORT}</a><script>"
+  BODY="The location of DSM is <a href='http://\${LOCATION}'>http://\${LOCATION}</a><script>"
-  BODY="${BODY}setTimeout(function(){ window.location.assign('http://\${IP}:\${PORT}'); }, 3000);</script>"
+  BODY="${BODY}setTimeout(function(){ window.location.assign('http://\${LOCATION}'); }, 3000);</script>"
   WAIT="Please wait while discovering IP...<script>setTimeout(() => { document.location.reload(); }, 4999);</script>"
 
   HTML=$(html "xxx")
 
   { echo "#!/bin/bash"
-    echo "INFO=\$(curl -s -m 2 -S http://127.0.0.1:2210/read?command=10 2>/dev/null)"
+    echo "[ -f \"/run/dsm.url\" ] && LOCATION=\$(cat \"/run/dsm.url\")"
-    echo "rest=\${INFO#*http_port}; rest=\${rest#*:}; rest=\${rest%%,*}; PORT=\${rest%%\\\"*}"
+    echo "HTML=\"$HTML\"; [ -z \"\${LOCATION}\" ] && BODY=\"$WAIT\" || BODY=\"$BODY\"; HTML=\${HTML/xxx/\$BODY}"
-    echo "rest=\${INFO#*eth0}; rest=\${rest#*ip}; rest=\${rest#*:}; rest=\${rest#*\\\"}; IP=\${rest%%\\\"*}"
-    echo "HTML=\"$HTML\"; [ -z \"\${IP}\" ] && BODY=\"$WAIT\" || BODY=\"$BODY\"; HTML=\${HTML/xxx/\$BODY}"
     echo "printf '%b' \"HTTP/1.1 200 OK\\nContent-Length: \${#HTML}\\nConnection: close\\n\\n\$HTML\""
   } > "$TMP_FILE"