feat: Improve Codespaces configuration (#1079)

Reduced spare disk space threshold from 2GB to 512MB.

.devcontainer.json

@@ -1,6 +0,0 @@
-{
-  "name": "dsm",
-  "service": "dsm",
-  "forwardPorts": [5000],
-  "dockerComposeFile": "compose.yml"
-}

.devcontainer/codespaces.yml

@@ -0,0 +1,20 @@
+services:
+  dsm:
+    container_name: dsm
+    image: vdsm/virtual-dsm
+    environment:
+      ALLOCATE: "Y"
+      RAM_SIZE: "half"
+      DISK_SIZE: "max"
+      CPU_CORES: "max"
+    devices:
+      - /dev/kvm
+      - /dev/net/tun
+    cap_add:
+      - NET_ADMIN
+    ports:
+      - 5000:5000
+    volumes:
+      - ./dsm:/storage
+    restart: on-failure
+    stop_grace_period: 2m

.devcontainer/devcontainer.json

@@ -0,0 +1,18 @@
+{
+  "name": "Virtual DSM",
+  "service": "dsm",
+  "containerEnv": {
+    "ALLOCATE": "Y"
+  },  
+  "forwardPorts": [5000],  
+  "portsAttributes": {
+    "5000": {
+      "label": "Web",
+      "onAutoForward": "openBrowser"
+    }
+  },
+  "otherPortsAttributes": {
+    "onAutoForward": "silent"
+  }, 
+  "dockerComposeFile": "codespaces.yml"
+}

.dockerignore

@@ -1,4 +1,5 @@
 .dockerignore
+.devcontainer
 .git
 .github
 .gitignore

.github/workflows/check.yml

@@ -22,3 +22,8 @@ jobs:
           dockerfile: Dockerfile
           ignore: DL3008,DL3003,DL3006,DL3013
           failure-threshold: warning
+      -
+        name: Validate JSON and YML files
+        uses: GrantBirki/json-yaml-validate@v4
+        with:
+          yaml_exclude_regex: ".*\\kubernetes\\.yml$"

readme.md

@@ -124,7 +124,7 @@ kubectl apply -f https://raw.githubusercontent.com/vdsm/virtual-dsm/refs/heads/m
 
 ### How do I change the amount of CPU or RAM?
 
-  By default, the container will be allowed to use a maximum of 2 CPU cores and 2 GB of RAM.
+  By default, Virtual DSM will be allowed to use 2 CPU cores and 2 GB of RAM.
 
   If you want to adjust this, you can specify the desired amount using the following environment variables:
 

src/config.sh

@@ -12,32 +12,4 @@ DEV_OPTS+=" -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pcie.0,addr=0x1c"
 ARGS="$DEF_OPTS $CPU_OPTS $RAM_OPTS $MAC_OPTS $DISPLAY_OPTS $MON_OPTS $SERIAL_OPTS $NET_OPTS $DISK_OPTS $DEV_OPTS $ARGUMENTS"
 ARGS=$(echo "$ARGS" | sed 's/\t/ /g' | tr -s ' ')
 
-# Check available memory as the very last step
-
-if [[ "$RAM_CHECK" != [Nn]* ]]; then
-
-  RAM_AVAIL=$(free -b | grep -m 1 Mem: | awk '{print $7}')
-  AVAIL_MEM=$(formatBytes "$RAM_AVAIL")
-
-  if (( (RAM_WANTED + RAM_SPARE) > RAM_AVAIL )); then
-    msg="Your configured RAM_SIZE of ${RAM_SIZE/G/ GB} is too high for the $AVAIL_MEM of memory available, please set a lower value."
-    [[ "${FS,,}" != "zfs" ]] && error "$msg" && exit 17
-    info "$msg"
-  else
-    if (( (RAM_WANTED + (RAM_SPARE * 3)) > RAM_AVAIL )); then
-      msg="your configured RAM_SIZE of ${RAM_SIZE/G/ GB} is very close to the $AVAIL_MEM of memory available, please consider a lower value."
-      if [[ "${FS,,}" != "zfs" ]]; then
-        warn "$msg"
-      else
-        info "$msg"
-      fi
-    fi
-  fi
-
-fi
-
-if [[ "$DEBUG" == [Yy1]* ]]; then
-  printf "QEMU arguments:\n\n%s\n\n" "${ARGS// -/$'\n-'}"
-fi
-
 return 0

src/disk.sh

@@ -98,10 +98,10 @@ createDisk() {
   local FS=$5
   local DATA_SIZE DIR SPACE GB FA
 
-  DATA_SIZE=$(numfmt --from=iec "$DISK_SPACE")
-
   rm -f "$DISK_FILE"
 
+  DATA_SIZE=$(numfmt --from=iec "$DISK_SPACE")
+
   if [[ "$ALLOCATE" != [Nn]* ]]; then
 
     # Check free diskspace
@@ -424,7 +424,7 @@ addDisk () {
   local DISK_FMT=$7
   local DISK_IO=$8
   local DISK_CACHE=$9
-  local DISK_EXT DIR SPACE DATA_SIZE FS PREV_FMT PREV_EXT CUR_SIZE
+  local DISK_EXT DIR SPACE GB DATA_SIZE FS PREV_FMT PREV_EXT CUR_SIZE
 
   DISK_EXT=$(fmt2ext "$DISK_FMT")
   local DISK_FILE="$DISK_BASE.$DISK_EXT"
@@ -432,6 +432,23 @@ addDisk () {
   DIR=$(dirname "$DISK_FILE")
   [ ! -d "$DIR" ] && return 0
 
+  if [[ "${DISK_SPACE,,}" == "max" || "${DISK_SPACE,,}" == "half" ]]; then
+
+    local SPARE=2147483648
+    SPACE=$(df --output=avail -B 1 "$DIR" | tail -n 1)
+
+    if [[ "${DISK_SPACE,,}" == "max" ]]; then
+      SPACE=$((SPACE-SPARE))
+    else
+      SPACE=$(( SPACE / 2 ))
+    fi
+
+    (( SPACE < SPARE )) && SPACE="$SPARE" 
+    GB=$(( SPACE/1073741825 ))
+    DISK_SPACE="${GB}G"
+
+  fi
+
   SPACE="${DISK_SPACE// /}"
   [ -z "$SPACE" ] && SPACE="16G"
   [ -z "${SPACE//[0-9. ]}" ] && SPACE="${SPACE}G"

src/entry.sh

@@ -18,7 +18,9 @@ cd /run
 . proc.sh       # Initialize processor
 . serial.sh     # Initialize serialport
 . power.sh      # Configure shutdown
+. memory.sh     # Check available memory
 . config.sh     # Configure arguments
+. finish.sh     # Finish initialization
 
 trap - ERR
 

src/finish.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ "$DEBUG" == [Yy1]* ]]; then
+  printf "QEMU arguments:\n\n%s\n\n" "${ARGS// -/$'\n-'}"
+fi
+
+return 0

src/memory.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+RAM_AVAIL=$(free -b | grep -m 1 Mem: | awk '{print $7}')
+
+if [[ "$RAM_CHECK" != [Nn]* && "${RAM_SIZE,,}" != "max" && "${RAM_SIZE,,}" != "half" ]]; then
+
+  AVAIL_MEM=$(formatBytes "$RAM_AVAIL")
+
+  if (( (RAM_WANTED + RAM_SPARE) > RAM_AVAIL )); then
+    msg="Your configured RAM_SIZE of ${RAM_SIZE/G/ GB} is too high for the $AVAIL_MEM of memory available,"
+    if [[ "${FS,,}" == "zfs" ]]; then
+      info "$msg but since ZFS is active this will be ignored."
+    else
+      RAM_SIZE="max"
+      warn "$msg it will automatically be adjusted to a lower amount."
+    fi
+  else
+    if (( (RAM_WANTED + (RAM_SPARE * 3)) > RAM_AVAIL )); then
+      msg="your configured RAM_SIZE of ${RAM_SIZE/G/ GB} is very close to the $AVAIL_MEM of memory available,"
+      if [[ "${FS,,}" == "zfs" ]]; then
+        info "$msg but since ZFS is active this will be ignored."
+      else
+        warn "$msg please consider a lower amount."
+      fi
+    fi
+  fi
+
+fi
+
+if [[ "${RAM_SIZE,,}" == "half" ]]; then
+
+  RAM_WANTED=$(( RAM_AVAIL / 2 ))
+  RAM_WANTED=$(( RAM_WANTED / 1073741825 ))
+
+  if (( "$RAM_WANTED" < 1 )); then
+    RAM_WANTED=$(( RAM_AVAIL / 2 ))
+    RAM_WANTED=$(( RAM_WANTED / 1048577 ))
+    RAM_SIZE="${RAM_WANTED}M"
+  else
+    RAM_SIZE="${RAM_WANTED}G"
+  fi
+
+fi
+
+if [[ "${RAM_SIZE,,}" == "max" ]]; then
+
+  RAM_WANTED=$(( RAM_AVAIL - (RAM_SPARE * 3) ))
+  RAM_WANTED=$(( RAM_WANTED / 1073741825 ))
+
+  if (( "$RAM_WANTED" < 1 )); then
+
+    RAM_WANTED=$(( RAM_AVAIL - (RAM_SPARE * 2) ))
+    RAM_WANTED=$(( RAM_WANTED / 1073741825 ))
+
+    if (( "$RAM_WANTED" < 1 )); then
+
+      RAM_WANTED=$(( RAM_AVAIL - RAM_SPARE ))
+      RAM_WANTED=$(( RAM_WANTED / 1073741825 ))
+
+      if (( "$RAM_WANTED" < 1 )); then
+
+        RAM_WANTED=$(( RAM_AVAIL - RAM_SPARE ))
+        RAM_WANTED=$(( RAM_WANTED / 1048577 ))
+
+        if (( "$RAM_WANTED" < 1 )); then
+
+          RAM_WANTED=$(( RAM_AVAIL ))
+          RAM_WANTED=$(( RAM_WANTED / 1048577 ))
+
+        fi
+
+        RAM_SIZE="${RAM_WANTED}M"
+      else
+        RAM_SIZE="${RAM_WANTED}G"
+      fi
+    else
+      RAM_SIZE="${RAM_WANTED}G"
+    fi
+  else
+    RAM_SIZE="${RAM_WANTED}G"
+  fi
+
+fi
+
+return 0

src/network.sh

@@ -131,7 +131,7 @@ configureDNS() {
   rm -f "$log"
 
   case "${NETWORK,,}" in
-    "nat" | "tap" | "tun" | "tuntap" | "y" )
+    "tap" | "tun" | "tuntap" | "y" )
 
       # Create lease file for faster resolve
       echo "0 $mac $ip $host 01:$mac" > /var/lib/misc/dnsmasq.leases
@@ -185,10 +185,12 @@ configureDNS() {
 
 getHostPorts() {
 
-  local list="$1"
+  local list=""
-  list=$(echo "${list// /}" | sed 's/,*$//g')
+  list+="$MON_PORT,"
+  list+="${HOST_PORTS// /},"
 
-  [ -z "$list" ] && list="$MON_PORT" || list+=",$MON_PORT"
+  # Remove duplicates
+  list=$(echo "${list//,,/,}," | awk 'BEGIN{RS=ORS=","} !seen[$0]++' | sed 's/,*$//g')
 
   echo "$list"
   return 0
@@ -196,15 +198,43 @@ getHostPorts() {
 
 getUserPorts() {
 
-  local args=""
-  local list=$1
-  list=$(echo "${list// /}" | sed 's/,*$//g')
-
   local ssh="22"
-  local dsm="5000"
+  local dsm="5000,5001"
-  [ -z "$list" ] && list="$ssh,$dsm" || list+=",$ssh,$dsm"
 
-  echo "$list"
+  local list="$ssh,$dsm,"
+  list+="${USER_PORTS// /},"
+
+  local exclude
+  exclude=$(getHostPorts)
+
+  local ports=""
+  local userport=""
+  local hostport=""
+
+  for userport in ${list//,/ }; do
+
+    local num="${userport///tcp}"
+    num="${num///udp}"
+
+    for hostport in ${exclude//,/ }; do
+
+      local val="${hostport///tcp}"
+
+      if [[ "$num" == "${val///udp}" ]]; then
+        num=""
+        warn "Could not assign port ${val///udp} to \"USER_PORTS\" because it is already in \"HOST_PORTS\"!"
+      fi
+
+    done
+
+    [ -n "$num" ] && ports+="$userport,"
+
+  done
+
+  # Remove duplicates
+  ports=$(echo "${ports//,,/,}," | awk 'BEGIN{RS=ORS=","} !seen[$0]++' | sed 's/,*$//g')
+
+  echo "$ports"
   return 0
 }
 
@@ -214,14 +244,12 @@ getSlirp() {
   local list=""
 
   list=$(getUserPorts)
-  list="${list//,/ }"
-  list="${list## }"
-  list="${list%% }"
 
-  for port in $list; do
+  for port in ${list//,/ }; do
 
-    proto="tcp"
+    local proto="tcp"
-    num="${port%/tcp}"
+    local num="${port%/tcp}"
+    [ -z "$num" ] && continue
 
     if [[ "$port" == *"/udp" ]]; then
       proto="udp"
@@ -235,12 +263,15 @@ getSlirp() {
     args+="hostfwd=$proto::$num-$VM_NET_IP:$num,"
   done
 
+  args=$(echo "$args" | sed 's/,*$//g')
+
   echo "${args%?}"
   return 0
 }
 
 configureSlirp() {
 
+  NETWORK="slirp"
   [[ "$DEBUG" == [Yy1]* ]] && echo "Configuring slirp networking..."
 
   local ip="$IP"
@@ -255,13 +286,13 @@ configureSlirp() {
   NET_OPTS="-netdev user,id=hostnet0,ipv4=on,host=$gateway,net=${gateway%.*}.0/24,dhcpstart=$ip,${ipv6}hostname=$VM_NET_HOST"
 
   local forward=""
-  forward=$(getUserPorts "${USER_PORTS:-}")
+  forward=$(getSlirp)
   [ -n "$forward" ] && NET_OPTS+=",$forward"
 
   if [[ "${DNSMASQ_DISABLE:-}" == [Yy1]* ]]; then
     echo "$gateway" > /run/shm/qemu.gw
   else
-    cp /etc/resolv.conf /etc/resolv.dnsmasq
+    [ ! -f /etc/resolv.dnsmasq ] && cp /etc/resolv.conf /etc/resolv.dnsmasq
     configureDNS "lo" "$ip" "$VM_NET_MAC" "$VM_NET_HOST" "$VM_NET_MASK" "$gateway" || return 1
     echo -e "nameserver 127.0.0.1\nsearch .\noptions ndots:0" >/etc/resolv.conf
   fi
@@ -272,6 +303,7 @@ configureSlirp() {
 
 configurePasst() {
 
+  NETWORK="passt"
   [[ "$DEBUG" == [Yy1]* ]] && echo "Configuring user-mode networking..."
 
   local log="/var/log/passt.log"
@@ -299,7 +331,7 @@ configurePasst() {
   [ -n "$PASST_MTU" ] && PASST_OPTS+=" -m $PASST_MTU"
 
   local forward=""
-  forward=$(getUserPorts "${USER_PORTS:-}")
+  forward=$(getUserPorts)
   forward="${forward///tcp}"
   forward="${forward///udp}"
 
@@ -316,7 +348,7 @@ configurePasst() {
   PASST_OPTS+=" -q"
 
   if [[ "${DNSMASQ_DISABLE:-}" != [Yy1]* ]]; then
-    cp /etc/resolv.conf /etc/resolv.dnsmasq
+    [ ! -f /etc/resolv.dnsmasq ] && cp /etc/resolv.conf /etc/resolv.dnsmasq
     echo -e "nameserver 127.0.0.1\nsearch .\noptions ndots:0" >/etc/resolv.conf
   fi
 
@@ -331,8 +363,8 @@ configurePasst() {
 
     if (( rc != 0 )); then
       [ -f "$log" ] && cat "$log"
-      error "Failed to start passt, reason: $rc"
+      warn "failed to start passt ($rc), falling back to slirp networking!"
-      return 1
+      configureSlirp && return 0 || return 1
     fi
 
   fi
@@ -447,7 +479,7 @@ configureNAT() {
     update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy > /dev/null
   fi
 
-  exclude=$(getHostPorts "$HOST_PORTS")
+  exclude=$(getHostPorts)
 
   if [ -n "$exclude" ]; then
     if [[ "$exclude" != *","* ]]; then
@@ -755,8 +787,8 @@ else
   fi
 
   case "${NETWORK,,}" in
-    "user"* | "passt" | "slirp" ) ;;
+    "passt" | "slirp" | "user"* ) ;;
-    "nat" | "tap" | "tun" | "tuntap" | "y" )
+    "tap" | "tun" | "tuntap" | "y" )
 
       # Configure tap interface
       if ! configureNAT; then
@@ -770,11 +802,9 @@ else
 
   esac
 
-  [[ "${NETWORK,,}" == "user"* ]] && NETWORK="passt"
-
   case "${NETWORK,,}" in
-    "nat" | "tap" | "tun" | "tuntap" | "y" ) ;;
+    "tap" | "tun" | "tuntap" | "y" ) ;;
-    "passt" )
+    "passt" | "user"* )
 
       # Configure for user-mode networking (passt)
       if ! configurePasst; then

src/reset.sh

@@ -59,6 +59,8 @@ fi
 
 CPU_CORES="${CPU_CORES// /}"
 [[ "${CPU_CORES,,}" == "max" ]] && CPU_CORES="$CORES"
+[[ "${CPU_CORES,,}" == "half" ]] && CPU_CORES=$(( CORES / 2 ))
+[[ "${CPU_CORES,,}" == "0" ]] && CPU_CORES="1"
 [ -n "${CPU_CORES//[0-9 ]}" ] && error "Invalid amount of CPU_CORES: $CPU_CORES" && exit 15
 
 if [ "$CPU_CORES" -gt "$CORES" ]; then
@@ -105,20 +107,18 @@ RAM_TOTAL=$(free -b | grep -m 1 Mem: | awk '{print $2}')
 RAM_SIZE="${RAM_SIZE// /}"
 [ -z "$RAM_SIZE" ] && error "RAM_SIZE not specified!" && exit 16
 
-if [[ "${RAM_SIZE,,}" == "max" ]]; then
+if [[ "${RAM_SIZE,,}" != "max" && "${RAM_SIZE,,}" != "half" ]]; then
-  RAM_WANTED=$(( RAM_AVAIL - RAM_SPARE - RAM_SPARE ))
-  RAM_WANTED=$(( RAM_WANTED / 1073741825 ))
-  RAM_SIZE="${RAM_WANTED}G"
-fi
 
-if [ -z "${RAM_SIZE//[0-9. ]}" ]; then
+  if [ -z "${RAM_SIZE//[0-9. ]}" ]; then
     [ "${RAM_SIZE%%.*}" -lt "130" ] && RAM_SIZE="${RAM_SIZE}G" || RAM_SIZE="${RAM_SIZE}M"
-fi
+  fi
 
-RAM_SIZE=$(echo "${RAM_SIZE^^}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
+  RAM_SIZE=$(echo "${RAM_SIZE^^}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
-! numfmt --from=iec "$RAM_SIZE" &>/dev/null && error "Invalid RAM_SIZE: $RAM_SIZE" && exit 16
+  ! numfmt --from=iec "$RAM_SIZE" &>/dev/null && error "Invalid RAM_SIZE: $RAM_SIZE" && exit 16
-RAM_WANTED=$(numfmt --from=iec "$RAM_SIZE")
+  RAM_WANTED=$(numfmt --from=iec "$RAM_SIZE")
-[ "$RAM_WANTED" -lt "136314880 " ] && error "RAM_SIZE is too low: $RAM_SIZE" && exit 16
+  [ "$RAM_WANTED" -lt "136314880 " ] && error "RAM_SIZE is too low: $RAM_SIZE" && exit 16
+
+fi
 
 # Print system info
 SYS="${SYS/-generic/}"
@@ -133,15 +133,6 @@ TOTAL_MEM=$(formatBytes "$RAM_TOTAL" "up")
 echo "❯ CPU: ${CPU} | RAM: ${AVAIL_MEM/ GB/}/$TOTAL_MEM | DISK: $SPACE_GB (${FS}) | KERNEL: ${SYS}..."
 echo
 
-# Check available memory
-
-if [[ "$RAM_CHECK" != [Nn]* ]] && (( (RAM_WANTED + RAM_SPARE) > RAM_AVAIL )); then
-  AVAIL_MEM=$(formatBytes "$RAM_AVAIL")
-  msg="Your configured RAM_SIZE of ${RAM_SIZE/G/ GB} is too high for the $AVAIL_MEM of memory available, please set a lower value."
-  [[ "${FS,,}" != "zfs" ]] && error "$msg" && exit 17
-  info "$msg"
-fi
-
 # Check KVM support
 
 if [[ "${PLATFORM,,}" == "x64" ]]; then

src/socket.sh

@@ -1,16 +1,31 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
+lastmsg=""
 path="/run/shm/msg.html"
 
-if [ -f "$path" ] && [ -s "$path" ]; then
+refresh() {
-  echo -n "s: " && cat "$path"
+
-fi
+  [ ! -f "$path" ] && return 0
+  [ ! -s "$path" ] && return 0
+
+  msg=$(< "$path")
+  msg="${msg%$'\n'}"
+
+  [ -z "$msg" ] && return 0
+  [[ "$msg" == "$lastmsg" ]] && return 0
+
+  lastmsg="$msg"
+  echo "s: $msg"
+  return 0
+}
+
+refresh
 
 inotifywait -m "$path" |
   while read -r fp event fn; do
     case "${event,,}" in
-      "modify"* ) echo -n "s: " && cat "$path" ;;
+      "modify"* ) refresh ;;
       "delete_self" ) echo "c: vnc" ;;
     esac    
   done