fix: Additional pass-through mounts (#722)

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml

@@ -3,6 +3,13 @@ description: Suggest an idea for improving the container
 title: "[Feature]: "
 labels: ["enhancement"]
 body:
+  - type: checkboxes
+    attributes:
+      label: Is there an existing feature request for this?
+      description: Please search to see if the feature request already exists.
+      options:
+      - label: I have searched the existing feature requests
+        required: true
   - type: textarea
     id: problem
     attributes:

.github/ISSUE_TEMPLATE/QUESTION.yml

@@ -3,11 +3,20 @@ description: General questions about the container
 title: "[Question]: "
 labels: ["question"]
 body:
-  - type: markdown
+  - type: checkboxes
     attributes:
-      value: |
-        Have a question about this container?
-        Please make sure to check the [FAQ](https://github.com/vdsm/virtual-dsm/blob/master/readme.md) first!
+      label: Is your question not already answered in the FAQ?
+      description: Please read the [FAQ](https://github.com/vdsm/virtual-dsm/blob/master/readme.md) carefully to avoid asking duplicate questions.
+      options:
+      - label: I made sure the question is not listed in the [FAQ](https://github.com/vdsm/virtual-dsm/blob/master/readme.md).
+        required: true
+  - type: checkboxes
+    attributes:
+      label: Is this a general question and not a technical issue?
+      description: For technical issues you must use the [bug report](https://github.com/vdsm/virtual-dsm/issues/new?assignees=&labels=bug&projects=&template=BUG_REPORT.yml&title=%5BBug%5D%3A+) form instead. It contains all the right fields (system info, logfiles, etc.) we need in order to be able to help you.
+      options:
+      - label: I am sure my question is not about a technical issue.
+        required: true
   - type: textarea
     id: question
     attributes:

readme.md

@@ -61,17 +61,6 @@ docker run -it --rm --name dsm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMI
   
   Enjoy your brand new machine, and don't forget to star this repo!
 
-* ### How do I change the size of the disk?
-
-  To expand the default size of 16 GB, locate the `DISK_SIZE` setting in your compose file and modify it to your preferred capacity:
-
-  ```yaml
-  environment:
-    DISK_SIZE: "128G"
-  ```
-  
-  This can also be used to resize the existing disk to a larger capacity without any data loss. 
-
 * ### How do I change the storage location?
 
   To change the storage location, include the following bind mount in your compose file:
@@ -82,6 +71,17 @@ docker run -it --rm --name dsm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMI
   ```
 
   Replace the example path `/var/dsm` with the desired storage folder.
+ 
+* ### How do I change the size of the disk?
+
+  To expand the default size of 16 GB, locate the `DISK_SIZE` setting in your compose file and modify it to your preferred capacity:
+
+  ```yaml
+  environment:
+    DISK_SIZE: "128G"
+  ```
+  
+  This can also be used to resize the existing disk to a larger capacity without any data loss.
 
 * ### How do I create a growable disk?
 
@@ -114,14 +114,13 @@ docker run -it --rm --name dsm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMI
    It is possible to pass-through disk devices directly by adding them to your compose file in this way:
 
   ```yaml
-  environment:
-    DEVICE2: "/dev/sda"
-    DEVICE3: "/dev/sdb"
   devices:
-    - /dev/sda
-    - /dev/sdb
+    - /dev/disk/by-uuid/12345-12345-12345-12345-12345:/disk2
+    - /dev/disk/by-uuid/45678-45678-45678-45678-45678:/disk3
   ```
 
+  Make sure to bind the disk via its UUID (obtainable via `lsblk -o name,uuid`) instead of its name (`/dev/sdc`), to prevent ever binding the wrong disk when the drive letters happen to change. 
+
   Please note that the device needs to be totally empty (without any partition table) otherwise DSM does not always format it into a volume.
 
   Do NOT use this feature with the goal of sharing files from the host, they will all be lost without warning when DSM creates the volume.
@@ -130,7 +129,7 @@ docker run -it --rm --name dsm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMI
 
   By default, a single CPU core and 1 GB of RAM are allocated to the container.
 
-  To increase this, add the following environment variables:
+  If there arises a need to increase this, add the following environment variables:
 
   ```yaml
   environment:
@@ -249,5 +248,5 @@ Only run this container on Synology hardware, any other use is not permitted by
 
 [Build]: https://github.com/vdsm/virtual-dsm/actions/workflows/build.yml/badge.svg
 [Size]: https://img.shields.io/docker/image-size/vdsm/virtual-dsm/latest?color=066da5&label=size
-[Pulls]: https://img.shields.io/docker/pulls/kroese/virtual-dsm.svg?style=flat&label=pulls&logo=docker
+[Pulls]: https://img.shields.io/docker/pulls/vdsm/virtual-dsm.svg?style=flat&label=pulls&logo=docker
 [Version]: https://img.shields.io/docker/v/vdsm/virtual-dsm/latest?arch=amd64&sort=semver&color=066da5

src/config.sh

@@ -2,7 +2,7 @@
 set -Eeuo pipefail
 
 DEF_OPTS="-nodefaults -boot strict=on"
-RAM_OPTS=$(echo "-m $RAM_SIZE" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
+RAM_OPTS=$(echo "-m ${RAM_SIZE^^}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
 CPU_OPTS="-cpu $CPU_FLAGS -smp $CPU_CORES,sockets=1,dies=1,cores=$CPU_CORES,threads=1"
 MAC_OPTS="-machine type=q35,usb=off,vmport=off,dump-guest-core=off,hpet=off${KVM_OPTS}"
 DEV_OPTS="-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4"
@@ -12,4 +12,18 @@ DEV_OPTS="$DEV_OPTS -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pcie.0,addr=0
 ARGS="$DEF_OPTS $CPU_OPTS $RAM_OPTS $MAC_OPTS $DISPLAY_OPTS $MON_OPTS $SERIAL_OPTS $NET_OPTS $DISK_OPTS $DEV_OPTS $ARGUMENTS"
 ARGS=$(echo "$ARGS" | sed 's/\t/ /g' | tr -s ' ')
 
+# Check available memory as the very last step
+
+RAM_AVAIL=$(free -b | grep -m 1 Mem: | awk '{print $7}')
+AVAIL_GB=$(( (RAM_AVAIL + 1073741823)/1073741824 ))
+
+if (( (RAM_WANTED + 500000000) > RAM_AVAIL )); then
+  error "Your configured RAM_SIZE of $WANTED_GB GB is higher than the $AVAIL_GB GB of memory available, please set a lower value."
+  exit 17
+fi
+
+if (( (RAM_WANTED + 1450000000) > RAM_AVAIL )); then
+  warn "your configured RAM_SIZE of $WANTED_GB GB is much too close to the $AVAIL_GB GB of memory available, please set a lower value."
+fi
+
 return 0

src/disk.sh

@@ -527,6 +527,16 @@ DISK4_FILE="/storage4/data4"
 : "${DEVICE3:=""}"
 : "${DEVICE4:=""}"
 
+[ -z "$DEVICE" ] && [ -b "/disk1" ] && DEVICE="/disk1"
+[ -z "$DEVICE2" ] && [ -b "/disk2" ] && DEVICE2="/disk2"
+[ -z "$DEVICE3" ] && [ -b "/disk3" ] && DEVICE3="/disk3"
+[ -z "$DEVICE4" ] && [ -b "/disk4" ] && DEVICE4="/disk4"
+
+[ -z "$DEVICE" ] && [ -b "/dev/disk1" ] && DEVICE="/dev/disk1"
+[ -z "$DEVICE2" ] && [ -b "/dev/disk2" ] && DEVICE2="/dev/disk2"
+[ -z "$DEVICE3" ] && [ -b "/dev/disk3" ] && DEVICE3="/dev/disk3"
+[ -z "$DEVICE4" ] && [ -b "/dev/disk4" ] && DEVICE4="/dev/disk4"
+
 if [ -n "$DEVICE" ]; then
   addDevice "$DEVICE" "device" "3" "0xc" || exit $?
 else

src/install.sh

@@ -118,14 +118,15 @@ if [ ! -s "$RDC" ]; then
   info "Install: $MSG" && html "$MSG"
 
   RD="$TMP/rd.gz"
+  SIZE=5394188
   POS="65627648-71021835"
   VERIFY="b4215a4b213ff5154db0488f92c87864"
   LOC="$DL/release/7.0.1/42218/DSM_VirtualDSM_42218.pat"
 
   rm -f "$RD"
   rm -f "$RDC"
-  /run/progress.sh "$RD" "$PRG" &
-  { curl -r "$POS" -sfk -S -o "$RD" "$LOC"; rc=$?; } || :
+  /run/progress.sh "$RD" "$SIZE" "$PRG" &
+  { curl -r "$POS" -sfk --connect-timeout 10 -S -o "$RD" "$LOC"; rc=$?; } || :
 
   fKill "progress.sh"
 
@@ -141,12 +142,14 @@ if [ ! -s "$RDC" ]; then
   if [ "$SUM" != "$VERIFY" ]; then
 
     PAT="/install.pat"
+    SIZE=379637760
+
     rm -f "$RD"
     rm -f "$PAT"
 
     html "$MSG"
-    /run/progress.sh "$PAT" "$PRG" &
-    { wget "$LOC" -O "$PAT" -q --no-check-certificate --show-progress "$PROGRESS"; rc=$?; } || :
+    /run/progress.sh "$PAT" "$SIZE" "$PRG" &
+    { wget "$LOC" -O "$PAT" -q --no-check-certificate --timeout=10 --show-progress "$PROGRESS"; rc=$?; } || :
 
     fKill "progress.sh"
     (( rc != 0 )) && error "Failed to download $LOC , reason: $rc" && exit 60
@@ -208,15 +211,18 @@ html "$MSG"
 PAT="/$BASE.pat"
 rm -f "$PAT"
 
+SIZE=0
+[[ "$URL" == *"DSM_VirtualDSM_69057.pat" ]] && SIZE=363837333
+
 if [[ "$URL" == "file://"* ]]; then
 
   cp "${URL:7}" "$PAT"
 
 else
 
-  /run/progress.sh "$PAT" "$PRG" &
+  /run/progress.sh "$PAT" "$SIZE" "$PRG" &
 
-  { wget "$URL" -O "$PAT" -q --no-check-certificate --show-progress "$PROGRESS"; rc=$?; } || :
+  { wget "$URL" -O "$PAT" -q --no-check-certificate --timeout=10 --show-progress "$PROGRESS"; rc=$?; } || :
 
   fKill "progress.sh"
   (( rc != 0 )) && error "Failed to download $URL , reason: $rc" && exit 69

src/network.sh

@@ -6,6 +6,7 @@ set -Eeuo pipefail
 : "${MAC:=""}"
 : "${DHCP:="N"}"
 : "${NETWORK:="Y"}"
+: "${HOST_PORTS:=""}"
 
 : "${VM_NET_DEV:=""}"
 : "${VM_NET_TAP:="dsm"}"
@@ -35,8 +36,8 @@ configureDHCP() {
   { ip link add link "$VM_NET_DEV" name "$VM_NET_TAP" address "$VM_NET_MAC" type macvtap mode bridge ; rc=$?; } || :
 
   if (( rc != 0 )); then
-    error "Cannot create macvtap interface. Please make sure the network type is 'macvlan' and not 'ipvlan',"
-    error "and that the NET_ADMIN capability has been added to the container: --cap-add NET_ADMIN" && exit 16
+    error "Cannot create macvtap interface. Please make sure that the network type is 'macvlan' and not 'ipvlan',"
+    error "that your kernel is recent (>4) and supports it, and that the container has the NET_ADMIN capability set." && exit 16
   fi
 
   while ! ip link set "$VM_NET_TAP" up; do
@@ -103,6 +104,21 @@ configureDNS() {
   return 0
 }
 
+getPorts() {
+
+  local list=$1
+
+  [ -z "$list" ] && echo "" && return 0
+
+  if [[ "$list" != *","* ]]; then
+    echo " ! --dport $list"
+  else
+    echo " -m multiport ! --dports $list"
+  fi
+
+  return 0
+}
+
 configureNAT() {
 
   # Create the necessary file structure for /dev/net/tun
@@ -119,12 +135,15 @@ configureNAT() {
 
   # Check port forwarding flag
   if [[ $(< /proc/sys/net/ipv4/ip_forward) -eq 0 ]]; then
-    { sysctl -w net.ipv4.ip_forward=1 ; rc=$?; } || :
-    if (( rc != 0 )); then
+    { sysctl -w net.ipv4.ip_forward=1 > /dev/null; rc=$?; } || :
+    if (( rc != 0 )) || [[ $(< /proc/sys/net/ipv4/ip_forward) -eq 0 ]]; then
       error "IP forwarding is disabled. $ADD_ERR --sysctl net.ipv4.ip_forward=1" && exit 24
     fi
   fi
 
+  local tables="The 'ip_tables' kernel module is not loaded. Try this command: sudo modprobe ip_tables iptable_nat"
+  local tuntap="The 'tun' kernel module is not available. Try this command: 'sudo modprobe tun' or run the container with 'privileged: true'."
+
   # Create a bridge with a static IP for the VM guest
 
   VM_NET_IP='20.20.20.21'
@@ -143,7 +162,9 @@ configureNAT() {
   done
 
   # QEMU Works with taps, set tap to the bridge created
-  ip tuntap add dev "$VM_NET_TAP" mode tap
+  if ! ip tuntap add dev "$VM_NET_TAP" mode tap; then
+    error "$tuntap" && exit 31
+  fi
 
   while ! ip link set "$VM_NET_TAP" up promisc on; do
     info "Waiting for TAP to become available..."
@@ -156,13 +177,19 @@ configureNAT() {
   update-alternatives --set iptables /usr/sbin/iptables-legacy > /dev/null
   update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy > /dev/null
 
-  iptables -t nat -A POSTROUTING -o "$VM_NET_DEV" -j MASQUERADE
-  iptables -t nat -A PREROUTING -i "$VM_NET_DEV" -d "$IP" -p tcp -j DNAT --to "$VM_NET_IP"
+  exclude=$(getPorts "$HOST_PORTS")
+
+  if ! iptables -t nat -A POSTROUTING -o "$VM_NET_DEV" -j MASQUERADE; then
+    error "$tables" && exit 30
+  fi
+
+  # shellcheck disable=SC2086
+  iptables -t nat -A PREROUTING -i "$VM_NET_DEV" -d "$IP" -p tcp${exclude} -j DNAT --to "$VM_NET_IP"
   iptables -t nat -A PREROUTING -i "$VM_NET_DEV" -d "$IP" -p udp  -j DNAT --to "$VM_NET_IP"
 
   if (( KERNEL > 4 )); then
     # Hack for guest VMs complaining about "bad udp checksums in 5 packets"
-    iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM --checksum-fill || true
+    iptables -A POSTROUTING -t mangle -p udp --dport bootpc -j CHECKSUM --checksum-fill > /dev/null 2>&1 || true
   fi
 
   NET_OPTS="-netdev tap,ifname=$VM_NET_TAP,script=no,downscript=no,id=hostnet0"
@@ -273,10 +300,14 @@ if [[ "$DEBUG" == [Yy1]* ]]; then
   echo
 fi
 
+if [[ "$IP" == "172.17."* ]]; then
+  warn "your container IP starts with 172.17.* which will cause conflicts when you install the Container Manager package inside DSM!"
+fi
+
 if [[ "$DHCP" == [Yy1]* ]]; then
 
-  if [[ "$GATEWAY" == "172."* ]] && [[ "$DEBUG" != [Yy1]* ]]; then
-    error "You can only enable DHCP while the container is on a macvlan network!" && exit 26
+  if [[ "$GATEWAY" == "172."* ]]; then
+    warn "your gateway IP starts with 172.* which is often a sign that you are not on a macvlan network (required for DHCP)!"
   fi
 
   # Configuration for DHCP IP

src/power.sh

@@ -10,10 +10,11 @@ API_HOST="127.0.0.1:2210"
 QEMU_TERM=""
 QEMU_PORT=7100
 QEMU_TIMEOUT=50
-QEMU_PID="/run/shm/qemu.pid"
-QEMU_LOG="/run/shm/qemu.log"
-QEMU_OUT="/run/shm/qemu.out"
-QEMU_END="/run/shm/qemu.end"
+QEMU_DIR="/run/shm"
+QEMU_PID="$QEMU_DIR/qemu.pid"
+QEMU_LOG="$QEMU_DIR/qemu.log"
+QEMU_OUT="$QEMU_DIR/qemu.out"
+QEMU_END="$QEMU_DIR/qemu.end"
 
 if [[ "$KVM" == [Nn]* ]]; then
   API_TIMEOUT=$(( API_TIMEOUT*2 ))

src/proc.sh

@@ -9,7 +9,7 @@ set -Eeuo pipefail
 : "${CPU_MODEL:=""}"
 : "${DEF_MODEL:="qemu64"}"
 
-[ "$ARCH" != "amd64" ] && KVM="N"
+[[ "${ARCH,,}" != "amd64" ]] && KVM="N"
 
 if [[ "$KVM" != [Nn]* ]]; then
 
@@ -90,7 +90,7 @@ else
 fi
 
 if [ -z "$HOST_CPU" ]; then
-  HOST_CPU=$(lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1' | sed 's# @.*##g' | sed s/"(R)"//g | sed 's/[^[:alnum:] ]\+/ /g' | sed 's/  */ /g')
+  HOST_CPU=$(lscpu | grep -m 1 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1' | sed 's# @.*##g' | sed s/"(R)"//g | sed 's/[^[:alnum:] ]\+/ /g' | sed 's/  */ /g')
 fi
 
 if [ -n "$HOST_CPU" ]; then

src/progress.sh

@@ -12,7 +12,8 @@ escape () {
 }
 
 file="$1"
-body=$(escape "$2")
+total="$2"
+body=$(escape "$3")
 info="/run/shm/msg.html"
 
 if [[ "$body" == *"..." ]]; then
@@ -24,7 +25,12 @@ do
   if [ -s "$file" ]; then
     bytes=$(du -sb "$file" | cut -f1)
     if (( bytes > 1000 )); then
-      size=$(echo "$bytes" | numfmt --to=iec --suffix=B  | sed -r 's/([A-Z])/ \1/')
+      if [ -z "$total" ] || [[ "$total" == "0" ]]; then
+        size=$(numfmt --to=iec --suffix=B  "$bytes" | sed -r 's/([A-Z])/ \1/')
+      else
+        size=$(printf '%.1f\n' "$((bytes*100*100/total))e-2")
+        size="$size%"
+      fi
       echo "${body//(\[P\])/($size)}"> "$info"
     fi
   fi

src/reset.sh

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-info () { printf "%b%s%b" "\E[1;34m❯ \E[1;36m" "$1" "\E[0m\n"; }
-error () { printf "%b%s%b" "\E[1;31m❯ " "ERROR: $1" "\E[0m\n" >&2; }
-warn () { printf "%b%s%b" "\E[1;31m❯ " "Warning: $1" "\E[0m\n" >&2; }
+info () { printf "%b%s%b" "\E[1;34m❯ \E[1;36m" "${1:-}" "\E[0m\n"; }
+error () { printf "%b%s%b" "\E[1;31m❯ " "ERROR: ${1:-}" "\E[0m\n" >&2; }
+warn () { printf "%b%s%b" "\E[1;31m❯ " "Warning: ${1:-}" "\E[0m\n" >&2; }
 
 trap 'error "Status $? while: $BASH_COMMAND (line $LINENO/$BASH_LINENO)"' ERR
 
@@ -42,8 +42,7 @@ HOST=$(hostname -s)
 KERNEL=$(echo "$SYS" | cut -b 1)
 MINOR=$(echo "$SYS" | cut -d '.' -f2)
 ARCH=$(dpkg --print-architecture)
-RAM="$(free -g | grep Mem: | awk '{print $7}')/$(free -g | grep Mem: | awk '{print $2}') GB"
-CPU=$(lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1' | sed 's# @.*##g' | sed s/"(R)"//g | sed 's/[^[:alnum:] ]\+/ /g' | sed 's/  */ /g')
+CPU=$(lscpu | grep -m 1 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1' | sed 's# @.*##g' | sed s/"(R)"//g | sed 's/[^[:alnum:] ]\+/ /g' | sed 's/  */ /g')
 
 # Check system
 
@@ -67,15 +66,31 @@ if [[ "${FS,,}" == "ecryptfs" ]] || [[ "${FS,,}" == "tmpfs" ]]; then
   DISK_CACHE="writeback"
 fi
 
+# Read memory
+RAM_AVAIL=$(free -b | grep -m 1 Mem: | awk '{print $7}')
+RAM_TOTAL=$(free -b | grep -m 1 Mem: | awk '{print $2}')
+RAM_SIZE=$(echo "${RAM_SIZE^^}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
+RAM_WANTED=$(numfmt --from=iec "$RAM_SIZE")
+AVAIL_GB=$(( (RAM_AVAIL + 1073741823)/1073741824 ))
+TOTAL_GB=$(( (RAM_TOTAL + 1073741823)/1073741824 ))
+WANTED_GB=$(( (RAM_WANTED + 1073741823)/1073741824 ))
+
 # Print system info
 SYS="${SYS/-generic/}"
 FS="${FS/ext2\/ext3/ext4}"
 SPACE=$(df --output=avail -B 1 "$STORAGE" | tail -n 1)
 SPACE_GB=$(( (SPACE + 1073741823)/1073741824 ))
 
-echo "❯ CPU: ${CPU} | RAM: ${RAM} | DISK: $SPACE_GB GB (${FS}) | HOST: ${SYS}..."
+echo "❯ CPU: ${CPU} | RAM: $AVAIL_GB/$TOTAL_GB GB | DISK: $SPACE_GB GB (${FS}) | HOST: ${SYS}..."
 echo
 
+# Check memory
+
+if (( (RAM_WANTED + 500000000) > RAM_AVAIL )); then
+  error "Your configured RAM_SIZE of $WANTED_GB GB is higher than the $AVAIL_GB GB of memory available, please set a lower value."
+  exit 17
+fi
+
 # Cleanup files
 rm -f /run/shm/qemu.*
 rm -f /run/shm/dsm.url