feat: Configure sizes for multiple disks

feat: Configure sizes for multiple disks

.github/workflows/build.yml

@@ -12,7 +12,6 @@ on:
       - '.dockerignore'
       - '.github/**'
       - '.github/workflows/**'
-      - 'Dockerfile'
 
 jobs:
   shellcheck:
@@ -28,10 +27,10 @@ jobs:
     steps:
       - 
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       -
@@ -45,6 +44,8 @@ jobs:
           TAGS=()
           TAGS=("${{ github.repository }}:latest")
           TAGS+=("${{ github.repository }}:${VERSION}")
+          #TAGS+=("${{ secrets.DOCKERHUB_MIRROR }}:latest")
+          #TAGS+=("${{ secrets.DOCKERHUB_MIRROR }}:${VERSION}")
           TAGS+=("ghcr.io/${{ github.repository }}:latest")
           TAGS+=("ghcr.io/${{ github.repository }}:${VERSION}")
           
@@ -54,13 +55,13 @@ jobs:
           echo "build_date=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
       -
         name: Login into Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -87,25 +88,6 @@ jobs:
         name: Clear Docker credentials
         run: |
           rm -f ${HOME}/.docker/config.json
-      -
-        name: Get previous tag
-        id: previousTag
-        run: |
-          name=$(git --no-pager tag --sort=creatordate --merged ${{ github.ref_name }} | tail -1)
-          echo "previousTag: $name"
-          echo "previousTag=$name" >> $GITHUB_ENV
-      -
-        name: Generate changelog
-        id: changelog
-        uses: requarks/changelog-action@v1
-        with:
-          token: ${{ github.token }}
-          fromTag: ${{ github.ref_name }}
-          toTag: ${{ env.previousTag }}
-          writeToFile: false
-          reverseOrder: true
-          includeInvalidCommits: true
-          excludeTypes: "docs,build,chore"
       -
         name: Create a release
         uses: action-pack/github-release@v2
@@ -114,10 +96,6 @@ jobs:
         with:
           tag: "v${{ steps.prepare.outputs.version }}"
           title: "v${{ steps.prepare.outputs.version }}"
-          body: |
-            ${{ steps.changelog.outputs.changes }}
-
-            **Full Changelog**: https://github.com/${{ github.repository }}/compare//${{ env.previousTag }}...v${{ steps.prepare.outputs.version }}
       -
         name: Increment version variable
         uses: action-pack/bump@v2

.github/workflows/check.yml

@@ -7,8 +7,8 @@ jobs:
     name: shellcheck
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
 env:
-        SHELLCHECK_OPTS: -x -e SC2001 -e SC2002 -e SC2223 -e SC2034 -e SC2064 -e SC2317 -e SC2028
+        SHELLCHECK_OPTS: -x -e SC2001 -e SC2002 -e SC2223 -e SC2034 -e SC2064 -e SC2317 -e SC2028 -e SC2153 -e SC2004 

.github/workflows/hub.yml

@@ -12,13 +12,13 @@ jobs:
   dockerHubDescription:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - 
       name: Docker Hub Description
       uses: peter-evans/dockerhub-description@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-        repository: ${{ github.repository }}
+        repository: ${{ secrets.DOCKERHUB_REPO }}
         short-description: ${{ github.event.repository.description }}
         readme-filepath: ./readme.md

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/qemu-tools/qemu-host as builder
+FROM qemux/qemu-host as builder
 
 #  FROM golang as builder
 #  WORKDIR /
@@ -7,35 +7,33 @@ FROM ghcr.io/qemu-tools/qemu-host as builder
 #  RUN go mod download
 #  RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /qemu-host.bin .
 
-FROM debian:bookworm-slim
+FROM debian:trixie-slim
 
 ARG DEBCONF_NOWARNINGS="yes"
 ARG DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && apt-get -y upgrade && \
     apt-get --no-install-recommends -y install \
-	curl \
-	cpio \
-	wget \
-	fdisk \
-	unzip \
-	socat \	
-	procps \
-	dnsmasq \
-	xz-utils \
-	iptables \
-	iproute2 \
-	net-tools \
-	btrfs-progs \
-	netcat-openbsd \
-	ca-certificates \
-	qemu-system-x86 \
+        tini \    
+        curl \
+        cpio \
+        wget \
+        fdisk \
+        unzip \
+        socat \
+        procps \
+        xz-utils \
+        iptables \
+        iproute2 \
+        dnsmasq \
+        net-tools \
+        ca-certificates \
+        netcat-openbsd \
+        qemu-system-x86 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-    
-COPY run/*.sh /run/
-COPY agent/*.sh /agent/
 
+COPY run/*.sh /run/
 COPY --from=builder /qemu-host.bin /run/host.bin
 
 RUN chmod +x /run/*.sh && chmod +x /run/*.bin
@@ -44,7 +42,7 @@ VOLUME /storage
 
 EXPOSE 22
 EXPOSE 80
-EXPOSE 139 
+EXPOSE 139
 EXPOSE 445
 EXPOSE 5000
 
@@ -57,12 +55,15 @@ ARG BUILD_ARG=0
 ARG VERSION_ARG="0.0"
 ENV VERSION=$VERSION_ARG
 
+LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.title="Virtual DSM"
 LABEL org.opencontainers.image.created=${DATE_ARG}
 LABEL org.opencontainers.image.revision=${BUILD_ARG}
 LABEL org.opencontainers.image.version=${VERSION_ARG}
-LABEL org.opencontainers.image.source=https://github.com/kroese/virtual-dsm/
-LABEL org.opencontainers.image.url=https://hub.docker.com/r/kroese/virtual-dsm/
+LABEL org.opencontainers.image.source="https://github.com/vdsm/virtual-dsm/"
+LABEL org.opencontainers.image.url="https://hub.docker.com/r/vdsm/virtual-dsm/"
+LABEL org.opencontainers.image.description="Virtual DSM in a docker container"
 
 HEALTHCHECK --interval=60s --retries=2 CMD /run/check.sh
 
-ENTRYPOINT ["/run/run.sh"]
+ENTRYPOINT ["/usr/bin/tini", "-s", "/run/run.sh"]

agent/agent.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -u
 
-VERSION="7"
+VERSION="9"
 HEADER="VirtualDSM Agent"
 
 # Functions
@@ -38,15 +38,19 @@ function downloadUpdate {
 
   # Auto update the agent
 
-  URL="https://raw.githubusercontent.com/kroese/virtual-dsm/master/agent/agent.sh"
+  URL="https://raw.githubusercontent.com/vdsm/virtual-dsm/master/agent/agent.sh"
   
   remote_size=$(curl -sIk -m 4 "${URL}" | grep -i "content-length:" | tr -d " \t" | cut -d ':' -f 2)
   remote_size=${remote_size//$'\r'}
 
   [[ "$remote_size" == "" || "$remote_size" == "0" ]] && return
 
+  remote_size=$(($remote_size+0))
+  ((remote_size<100)) && return
+
   SCRIPT=$(readlink -f "${BASH_SOURCE[0]}")
   local_size=$(stat -c%s "$SCRIPT")
+  local_size=$(($local_size+0))
 
   [[ remote_size -eq local_size ]] && return
 
@@ -100,6 +104,8 @@ function installPackages {
 trap finish SIGINT SIGTERM
 
 ts=$(date +%s%N)
+
+echo ""
 echo "❯ Started $HEADER v$VERSION..."
 
 checkNMI
@@ -124,31 +130,6 @@ else
 
 fi
 
-delay=500
-elapsed=$((($(date +%s%N) - ts)/1000000))
-
-if [[ delay -gt elapsed ]]; then
-  difference=$((delay-elapsed))
-  float=$(echo | awk -v diff="${difference}" '{print diff * 0.001}')
-  sleep "$float"
-fi
-
-# Display message in docker log output
-
-IP=$(ip address show dev eth0 | grep inet | awk '/inet / { print $2 }' | cut -f1 -d/)
-
-if [[ "$IP" == "20.20"* ]]; then
-  MSG="port 5000"
-else
-  MSG="http://${IP}:5000"
-fi
-
-echo ""
-info "--------------------------------------------------------"
-info " You can now login to DSM at ${MSG}"
-info "--------------------------------------------------------"
-echo ""
-
 # Wait for NMI interrupt as a shutdown signal
 
 while true; do

agent/service.sh

@@ -28,7 +28,7 @@ start() {
 
   if [ ! -f "$SCRIPT" ]; then
 
-    URL="https://raw.githubusercontent.com/kroese/virtual-dsm/master/agent/agent.sh"
+    URL="https://raw.githubusercontent.com/vdsm/virtual-dsm/master/agent/agent.sh"
 
     if ! curl -sfk -m 10 -o "${SCRIPT}" "${URL}"; then
       error 'Failed to download agent script.' > /dev/ttyS0

docker-compose.yml

@@ -2,7 +2,7 @@ version: "3"
 services:
     dsm:
         container_name: dsm
-        image: kroese/virtual-dsm:latest
+        image: vdsm/virtual-dsm:latest
         environment:
             CPU_CORES: "1"
             DISK_SIZE: "16G"

readme.md

@@ -1,6 +1,6 @@
 <h1 align="center">Virtual DSM for Docker<br />
 <div align="center">
-<img src="https://github.com/kroese/virtual-dsm/raw/master/.github/screen.jpg" title="Screenshot" style="max-width:100%;" width="432" />
+<img src="https://github.com/vdsm/virtual-dsm/raw/master/.github/screen.jpg" title="Screenshot" style="max-width:100%;" width="432" />
 </div>
 <div align="center">
 
@@ -16,9 +16,10 @@ Virtual DSM in a docker container.
 
  - Multi-platform
  - KVM acceleration
- - Graceful shutdown
+ - GPU passthrough
+ - Graceful shutdowns
  - Upgrades supported
-
+ 
 ## Usage
 
 Via `docker-compose.yml`
@@ -26,28 +27,28 @@ Via `docker-compose.yml`
 ```yaml
 version: "3"
 services:
-    dsm:
-        container_name: dsm
-        image: kroese/virtual-dsm:latest
-        environment:
-            DISK_SIZE: "16G"
-        devices:
-            - /dev/kvm
-            - /dev/vhost-net
-        cap_add:
-            - NET_ADMIN                       
-        ports:
-            - 5000:5000
-        volumes:
-            - /opt/dsm:/storage
-        restart: on-failure
-        stop_grace_period: 1m
+  dsm:
+    container_name: dsm
+    image: vdsm/virtual-dsm:latest
+    environment:
+      DISK_SIZE: "16G"
+    devices:
+      - /dev/kvm
+      - /dev/vhost-net
+    cap_add:
+      - NET_ADMIN
+    ports:
+      - 5000:5000
+    volumes:
+      - /opt/dsm:/storage
+    restart: on-failure
+    stop_grace_period: 1m
 ```
 
 Via `docker run`
 
 ```bash
-docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-timeout 60 kroese/virtual-dsm:latest
+docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-timeout 60 vdsm/virtual-dsm:latest
 ```
 
 ## FAQ
@@ -58,10 +59,10 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti
 
     ```yaml
     environment:
-        DISK_SIZE: "256G"
+      DISK_SIZE: "256G"
     ```
     
-    This can also be used to resize the existing disk to a larger capacity without data loss. 
+    This can also be used to resize the existing disk to a larger capacity without any data loss. 
 
   * ### How do I change the location of the virtual disk?
 
@@ -69,7 +70,7 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti
 
     ```yaml
     volumes:
-        - /home/user/data:/storage
+      - /home/user/data:/storage
     ```
 
     Replace the example path `/home/user/data` with the desired storage folder.
@@ -80,7 +81,7 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti
 
     ```yaml
     environment:
-        ALLOCATE: "N"
+      ALLOCATE: "N"
     ```
 
     Keep in mind that this will not affect any of your existing disks, it only applies to newly created disks.
@@ -91,8 +92,8 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti
 
     ```yaml
     environment:
-        CPU_CORES: "4"
-        RAM_SIZE: "2048M"
+      CPU_CORES: "4"
+      RAM_SIZE: "2048M"
     ```
 
   * ### How do I verify if my system supports KVM?
@@ -126,16 +127,16 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti
 
     ```yaml
     services:
-        dsm:
-            container_name: dsm
-            ..<snip>..
-            networks:
-                vdsm:             
-                    ipv4_address: 192.168.0.100
+      dsm:
+        container_name: dsm
+        ..<snip>..
+        networks:
+          vdsm:
+            ipv4_address: 192.168.0.100
 
     networks:
-        vdsm:
-            external: true
+      vdsm:
+        external: true
     ```
    
     An added benefit of this approach is that you won't have to perform any port mapping anymore since all ports will be exposed by default.
@@ -150,45 +151,58 @@ docker run -it --rm -p 5000:5000 --device=/dev/kvm --cap-add NET_ADMIN --stop-ti
 
     ```yaml
     environment:
-        DHCP: "Y"
+      DHCP: "Y"
     devices:
-        - /dev/vhost-net
+      - /dev/vhost-net
     device_cgroup_rules:
-        - 'c *:* rwm'
+      - 'c *:* rwm'
     ```
 
     Please note that even if you don't need DHCP, it's still recommended to enable this feature as it prevents NAT issues and increases performance by using a `macvtap` interface.
 
   * ### How do I install a specific version of vDSM?
 
-    By default, version 7.2 will be installed, but if you prefer an older version, you can add its URL to your compose file as follows:
+    By default, version 7.2.1 will be installed, but if you prefer an older version, you can add its download URL to your compose file as follows:
 
     ```yaml
     environment:
-        URL: "https://global.synologydownload.com/download/DSM/release/7.0.1/42218/DSM_VirtualDSM_42218.pat"
+      URL: "https://global.synologydownload.com/download/DSM/release/7.0.1/42218/DSM_VirtualDSM_42218.pat"
     ```
 
-    With this method, you are able to switch between different versions while keeping your file data.
+    With this method, it is even possible to switch between different versions while keeping all your file data intact.
 
+  * ### How do I passthrough my GPU?
+
+    To passthrough your Intel GPU, add the following lines to your compose file:
+
+    ```yaml
+    environment:
+      GPU: "Y"
+    devices:
+      - /dev/dri
+    ```
+
+    This can be used to enable the facial recognition function in Synology Photos for example.
+    
   * ### What are the differences compared to the standard DSM?
 
-    There are only two minor differences: the Virtual Machine Manager package is not provided, and Surveillance Station doesn't include any free licenses.
+    There are only two minor differences: the Virtual Machine Manager package is not available, and Surveillance Station will not include any free licenses.
     
   * ### Is this project legal?
 
     Yes, this project contains only open-source code and does not distribute any copyrighted material. Neither does it try to circumvent any copyright protection measures. So under all applicable laws, this project would be considered legal. 
     
-    However, by installing Synology's Virtual DSM, you must accept their end-user license agreement, which does not permit installation on non-Synology hardware. So only run this project on an official Synology NAS via the Container Manager package, as any other use will be a violation of their terms and conditions.
+    However, by installing Synology's Virtual DSM, you must accept their end-user license agreement, which does not permit installation on non-Synology hardware. So only run this project on an official Synology NAS, as any other use will be a violation of their terms and conditions.
 
 ## Disclaimer
 
 Only run this container on Synology hardware, any other use is not permitted by their EULA. The product names, logos, brands, and other trademarks referred to within this project are the property of their respective trademark holders. This project is not affiliated, sponsored, or endorsed by Synology, Inc.
 
-[build_url]: https://github.com/kroese/virtual-dsm/
-[hub_url]: https://hub.docker.com/r/kroese/virtual-dsm
-[tag_url]: https://hub.docker.com/r/kroese/virtual-dsm/tags
+[build_url]: https://github.com/vdsm/virtual-dsm/
+[hub_url]: https://hub.docker.com/r/vdsm/virtual-dsm
+[tag_url]: https://hub.docker.com/r/vdsm/virtual-dsm/tags
 
-[Build]: https://github.com/kroese/virtual-dsm/actions/workflows/build.yml/badge.svg
-[Size]: https://img.shields.io/docker/image-size/kroese/virtual-dsm/latest?color=066da5&label=size
+[Build]: https://github.com/vdsm/virtual-dsm/actions/workflows/build.yml/badge.svg
+[Size]: https://img.shields.io/docker/image-size/vdsm/virtual-dsm/latest?color=066da5&label=size
 [Pulls]: https://img.shields.io/docker/pulls/kroese/virtual-dsm.svg?style=flat&label=pulls&logo=docker
-[Version]: https://img.shields.io/docker/v/kroese/virtual-dsm?arch=amd64&sort=date&color=066da5
+[Version]: https://img.shields.io/docker/v/vdsm/virtual-dsm/latest?arch=amd64&sort=semver&color=066da5

run/check.sh

@@ -2,6 +2,7 @@
 set -u
 
 [ ! -f "/run/qemu.pid" ] && echo "QEMU not running yet.." && exit 0
+[ -f "/run/qemu.counter" ] && echo "QEMU is shutting down.." && exit 1
 
 # Retrieve IP from guest VM for Docker healthcheck
 RESPONSE=$(curl -s -m 16 -S http://127.0.0.1:2210/read?command=10 2>&1)

run/disk.sh

@@ -21,10 +21,11 @@ if [[ ! -f "${DATA}" ]] && [[ -f "$STORAGE/data$DISK_SIZE.img" ]]; then
   DATA="$STORAGE/data$DISK_SIZE.img"
 fi
 
+MIN_SIZE=6442450944
 DISK_SIZE=$(echo "${DISK_SIZE}" | sed 's/MB/M/g;s/GB/G/g;s/TB/T/g')
 DATA_SIZE=$(numfmt --from=iec "${DISK_SIZE}")
 
-if (( DATA_SIZE < 6442450944 )); then
+if (( DATA_SIZE < MIN_SIZE )); then
   error "Please increase DISK_SIZE to at least 6 GB." && exit 83
 fi
 
@@ -47,7 +48,7 @@ if [ -f "${DATA}" ]; then
 
       REQ=$((DATA_SIZE-OLD_SIZE))
 
-      # Check free diskspace    
+      # Check free diskspace
       SPACE=$(df --output=avail -B 1 "${STORAGE}" | tail -n 1)
 
       if (( REQ > SPACE )); then
@@ -124,9 +125,6 @@ if [ ! -f "${DATA}" ]; then
     error "Virtual disk does not exist ($DATA)" && exit 88
   fi
 
-  # Format as BTRFS filesystem
-  mkfs.btrfs -q -L data -d single -m dup "${DATA}" > /dev/null
-
 fi
 
 # Check the filesize
@@ -136,13 +134,6 @@ if [[ SIZE -ne DATA_SIZE ]]; then
   error "Virtual disk has the wrong size: ${SIZE}" && exit 89
 fi
 
-AGENT="${STORAGE}/${BASE}.agent"
-[ -f "$AGENT" ] && AGENT_VERSION=$(cat "${AGENT}") || AGENT_VERSION=1
-
-if ((AGENT_VERSION < 5)); then
-  info "The installed VirtualDSM Agent v${AGENT_VERSION} is an outdated version, please upgrade it."
-fi
-
 DISK_OPTS="\
     -device virtio-scsi-pci,id=hw-synoboot,bus=pcie.0,addr=0xa \
     -drive file=${BOOT},if=none,id=drive-synoboot,format=raw,cache=${DISK_CACHE},aio=${DISK_IO},discard=${DISK_DISCARD},detect-zeroes=on \
@@ -153,3 +144,96 @@ DISK_OPTS="\
     -device virtio-scsi-pci,id=hw-userdata,bus=pcie.0,addr=0xc \
     -drive file=${DATA},if=none,id=drive-userdata,format=raw,cache=${DISK_CACHE},aio=${DISK_IO},discard=${DISK_DISCARD},detect-zeroes=on \
     -device scsi-hd,bus=hw-userdata.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata,id=userdata0,rotation_rate=${DISK_ROTATION},bootindex=3"
+
+: ${DISK2_SIZE:=''}
+EXTRA_SIZE=DISK2_SIZE
+EXTRA_DISK="/storage2/data.img"
+
+if [ -d "$(dirname "${EXTRA_DISK}")" ]; then
+
+  if [ ! -f "${EXTRA_DISK}" ]; then
+    [ -z "$EXTRA_SIZE" ] && EXTRA_SIZE="16G"
+    if ! truncate -s "${EXTRA_SIZE}" "${EXTRA_DISK}"; then
+      error "Could not create the file for the second disk." && exit 53
+    fi
+  fi
+
+  if [ -n "$EXTRA_SIZE" ]; then
+    CUR_SIZE=$(stat -c%s "${EXTRA_DISK}")
+    DATA_SIZE=$(numfmt --from=iec "${EXTRA_SIZE}")
+    if [ "$DATA_SIZE" -gt "$CUR_SIZE" ]; then
+      truncate -s "${EXTRA_SIZE}" "${EXTRA_DISK}"
+    fi
+  fi
+  
+  DISK_OPTS="${DISK_OPTS} \
+    -device virtio-scsi-pci,id=hw-userdata2,bus=pcie.0,addr=0xd \
+    -drive file=${EXTRA_DISK},if=none,id=drive-userdata2,format=raw,cache=${DISK_CACHE},aio=${DISK_IO},discard=${DISK_DISCARD},detect-zeroes=on \
+    -device scsi-hd,bus=hw-userdata2.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata2,id=userdata2,rotation_rate=${DISK_ROTATION},bootindex=4"
+
+fi
+
+: ${DISK3_SIZE:=''}
+EXTRA_SIZE=DISK3_SIZE
+EXTRA_DISK="/storage3/data.img"
+
+if [ -d "$(dirname "${EXTRA_DISK}")" ]; then
+
+  if [ ! -f "${EXTRA_DISK}" ]; then
+    [ -z "$EXTRA_SIZE" ] && EXTRA_SIZE="16G"
+    if ! truncate -s "${EXTRA_SIZE}" "${EXTRA_DISK}"; then
+      error "Could not create the file for the third disk." && exit 54
+    fi
+  fi
+
+  if [ -n "$EXTRA_SIZE" ]; then
+    CUR_SIZE=$(stat -c%s "${EXTRA_DISK}")
+    DATA_SIZE=$(numfmt --from=iec "${EXTRA_SIZE}")
+    if [ "$DATA_SIZE" -gt "$CUR_SIZE" ]; then
+      truncate -s "${EXTRA_SIZE}" "${EXTRA_DISK}"
+    fi
+  fi
+
+  DISK_OPTS="${DISK_OPTS} \
+    -device virtio-scsi-pci,id=hw-userdata3,bus=pcie.0,addr=0xe \
+    -drive file=${EXTRA_DISK},if=none,id=drive-userdata3,format=raw,cache=${DISK_CACHE},aio=${DISK_IO},discard=${DISK_DISCARD},detect-zeroes=on \
+    -device scsi-hd,bus=hw-userdata3.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata3,id=userdata3,rotation_rate=${DISK_ROTATION},bootindex=5"
+
+fi
+
+: ${DEVICE:=''}        # Docker variable to passthrough a block device, like /dev/vdc1.
+: ${DEVICE2:=''}
+: ${DEVICE3:=''}
+
+if [ -n "${DEVICE}" ]; then
+
+  [ ! -b "${DEVICE}" ] && error "Device ${DEVICE} cannot be found! Please add it to the 'devices' section of your compose file." && exit 55
+
+  DISK_OPTS="${DISK_OPTS} \
+    -device virtio-scsi-pci,id=hw-userdata4,bus=pcie.0,addr=0xf \
+    -drive file=${DEVICE},if=none,id=drive-userdata4,format=raw,cache=${DISK_CACHE},aio=${DISK_IO},discard=${DISK_DISCARD},detect-zeroes=on \
+    -device scsi-hd,bus=hw-userdata4.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata4,id=userdata4,rotation_rate=${DISK_ROTATION},bootindex=6"
+
+fi
+
+if [ -n "${DEVICE2}" ]; then
+
+  [ ! -b "${DEVICE2}" ] && error "Device ${DEVICE2} cannot be found! Please add it to the 'devices' section of your compose file." && exit 56
+
+  DISK_OPTS="${DISK_OPTS} \
+    -device virtio-scsi-pci,id=hw-userdata5,bus=pcie.0,addr=0x5 \
+    -drive file=${DEVICE2},if=none,id=drive-userdata5,format=raw,cache=${DISK_CACHE},aio=${DISK_IO},discard=${DISK_DISCARD},detect-zeroes=on \
+    -device scsi-hd,bus=hw-userdata5.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata5,id=userdata5,rotation_rate=${DISK_ROTATION},bootindex=7"
+
+fi
+
+if [ -n "${DEVICE3}" ]; then
+
+  [ ! -b "${DEVICE3}" ] && error "Device ${DEVICE3} cannot be found! Please add it to the 'devices' section of your compose file." && exit 57
+
+  DISK_OPTS="${DISK_OPTS} \
+    -device virtio-scsi-pci,id=hw-userdata6,bus=pcie.0,addr=0x6 \
+    -drive file=${DEVICE3},if=none,id=drive-userdata6,format=raw,cache=${DISK_CACHE},aio=${DISK_IO},discard=${DISK_DISCARD},detect-zeroes=on \
+    -device scsi-hd,bus=hw-userdata6.0,channel=0,scsi-id=0,lun=0,drive=drive-userdata6,id=userdata6,rotation_rate=${DISK_ROTATION},bootindex=8"
+
+fi

run/gpu.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+[ ! -d /dev/dri ] && mkdir -m 755 /dev/dri
+
+if [ ! -c /dev/dri/card0 ]; then
+  mknod /dev/dri/card0 c 226 0
+fi
+
+if [ ! -c /dev/dri/renderD128 ]; then
+  mknod /dev/dri/renderD128 c 226 128
+fi
+
+chmod 666 /dev/dri/card0
+chmod 666 /dev/dri/renderD128
+
+DEF_OPTS="-nodefaults -boot strict=on -display egl-headless,rendernode=/dev/dri/renderD128"
+DEF_OPTS="${DEF_OPTS} -device virtio-vga,id=video0,max_outputs=1,bus=pcie.0,addr=0x1"
+
+if ! apt-mark showinstall | grep -q "xserver-xorg-video-intel"; then
+
+  info "Installing Intel GPU drivers..."
+
+  export DEBCONF_NOWARNINGS="yes"
+  export DEBIAN_FRONTEND="noninteractive"
+
+  apt-get -qq update
+  apt-get -qq --no-install-recommends -y install xserver-xorg-video-intel > /dev/null
+
+fi
+
+if ! apt-mark showinstall | grep -q "qemu-system-modules-opengl"; then
+
+  info "Installing OpenGL module..."
+
+  export DEBCONF_NOWARNINGS="yes"
+  export DEBIAN_FRONTEND="noninteractive"
+
+  apt-get -qq update
+  apt-get -qq --no-install-recommends -y install qemu-system-modules-opengl > /dev/null
+
+fi

run/install.sh

@@ -10,7 +10,7 @@ DL="https://global.synologydownload.com/download/DSM"
 if [ -z "$URL" ]; then
 
   if [ "$ARCH" == "amd64" ]; then
-    URL="$DL/release/7.2/64570-1/DSM_VirtualDSM_64570.pat"
+    URL="$DL/release/7.2.1/69057-1/DSM_VirtualDSM_69057.pat"
   else
     URL="$DL/release/7.0.1/42218/DSM_VirtualDSM_42218.pat"
   fi
@@ -31,13 +31,20 @@ rm -f "$STORAGE"/"$BASE".agent
 rm -f "$STORAGE"/"$BASE".boot.img
 rm -f "$STORAGE"/"$BASE".system.img
 
-TMP="$STORAGE/tmp"
-RDC="$STORAGE/dsm.rd"
-
+TMP="/tmp/dsm"
+FS=$(stat -f -c %T "$STORAGE")
+[[ "$FS" == "ext"* ]] && TMP="$STORAGE/tmp"
 rm -rf "$TMP" && mkdir -p "$TMP"
 
+# Check free diskspace
+MIN_SPACE=5842450944
+SPACE=$(df --output=avail -B 1 "$TMP" | tail -n 1)
+(( MIN_SPACE > SPACE )) && error "Not enough free space for installation." && exit 95
+
 [[ "${DEBUG}" == [Yy1]* ]] && set -x
 
+RDC="$STORAGE/dsm.rd"
+
 if [ ! -f "${RDC}" ]; then
 
   info "Install: Downloading installer..."
@@ -95,7 +102,7 @@ if [ -f "${RDC}" ]; then
     cp "$TMP/usr/lib/libc.so.6" /lib64/
     cp "$TMP/usr/lib/libpthread.so.0" /lib64/
     cp "$TMP/usr/lib/ld-linux-x86-64.so.2" /lib64/
-  fi  
+  fi
 
   mv /run/extract/scemd /run/extract/syno_extract_system_patch
   chmod +x /run/extract/syno_extract_system_patch
@@ -120,28 +127,27 @@ if ((SIZE<250000000)); then
   error "The specified PAT file is probably an update pack as it's too small." && exit 62
 fi
 
-info "Install: Extracting downloaded image..."
-
 if { tar tf "$PAT"; } >/dev/null 2>&1; then
 
+  info "Install: Extracting downloaded image..."
   tar xpf "$PAT" -C "$TMP/."
 
 else
 
   if [ "$ARCH" != "amd64" ]; then
 
+    info "Install: Installing QEMU..."
+
     export DEBCONF_NOWARNINGS="yes"
     export DEBIAN_FRONTEND="noninteractive"
 
     apt-get -qq update
-    apt-get -qq -y upgrade
     apt-get -qq --no-install-recommends -y install qemu-user > /dev/null
 
-    export DEBIAN_FRONTEND=""
-    export DEBCONF_NOWARNINGS=""
-
   fi
 
+  info "Install: Extracting downloaded image..."
+
   export LD_LIBRARY_PATH="/run/extract"
 
   if [ "$ARCH" == "amd64" ]; then
@@ -198,14 +204,14 @@ SIZE=$(stat -c%s "${SYSTEM}")
 
 PART="$TMP/partition.fdisk"
 
-{	echo "label: dos"
-	echo "label-id: 0x6f9ee2e9"
-	echo "device: ${SYSTEM}"
-	echo "unit: sectors"
-	echo "sector-size: 512"
-	echo ""
-	echo "${SYSTEM}1 : start=        2048, size=     4980480, type=83"
-	echo "${SYSTEM}2 : start=     4982528, size=     4194304, type=82"
+{       echo "label: dos"
+        echo "label-id: 0x6f9ee2e9"
+        echo "device: ${SYSTEM}"
+        echo "unit: sectors"
+        echo "sector-size: 512"
+        echo ""
+        echo "${SYSTEM}1 : start=        2048, size=     4980480, type=83"
+        echo "${SYSTEM}2 : start=     4982528, size=     4194304, type=82"
 } > "$PART"
 
 sfdisk -q "$SYSTEM" < "$PART"
@@ -224,21 +230,6 @@ rm -f "$MOUNT/.SynoUpgradePackages/ActiveInsight-"*
 [ -f "$HDP.txz" ] && tar xpfJ "$HDP.txz" --absolute-names -C "$MOUNT/"
 [ -f "$IDB.txz" ] && tar xpfJ "$IDB.txz" --absolute-names -C "$MOUNT/usr/syno/synoman/indexdb/"
 
-# Install Agent
-
-LOC="$MOUNT/usr/local/bin"
-mkdir -p "$LOC"
-cp /agent/agent.sh "$LOC/agent.sh"
-chmod 755 "$LOC/agent.sh"
-
-LOC="$MOUNT/usr/local/etc/rc.d"
-mkdir -p "$LOC"
-cp /agent/service.sh "$LOC/agent.sh"
-chmod 755 "$LOC/agent.sh"
-
-# Store agent version
-echo "7" > "$STORAGE"/"$BASE".agent
-
 info "Install: Installing system partition..."
 
 LABEL="1.44.1-42218"
@@ -250,6 +241,11 @@ mke2fs -q -t ext4 -b 4096 -d "$MOUNT/" -L "$LABEL" -F -E "offset=$OFFSET" "$SYST
 rm -rf "$MOUNT"
 
 echo "$BASE" > "$STORAGE"/dsm.ver
+
+# Check free diskspace
+SPACE=$(df --output=avail -B 1 "$STORAGE" | tail -n 1)
+(( MIN_SPACE > SPACE )) && error "Not enough free space in storage folder." && exit 94
+
 mv -f "$PAT" "$STORAGE"/"$BASE".pat
 mv -f "$BOOT" "$STORAGE"/"$BASE".boot.img
 mv -f "$SYSTEM" "$STORAGE"/"$BASE".system.img

run/network.sh

@@ -31,7 +31,10 @@ configureDHCP() {
     error "and that the NET_ADMIN capability has been added to the container config: --cap-add NET_ADMIN" && exit 16
   fi
 
-  ip link set "${VM_NET_TAP}" up
+  while ! ip link set "${VM_NET_TAP}" up; do
+    info "Waiting for address to become available..."
+    sleep 2
+  done
 
   TAP_NR=$(</sys/class/net/"${VM_NET_TAP}"/ifindex)
   TAP_PATH="/dev/tap${TAP_NR}"
@@ -127,17 +130,28 @@ configureNAT () {
   fi
 
   ip address add ${VM_NET_IP%.*}.1/24 broadcast ${VM_NET_IP%.*}.255 dev dockerbridge
-  ip link set dockerbridge up
+
+  while ! ip link set dockerbridge up; do
+    info "Waiting for address to become available..."
+    sleep 2
+  done
 
   # QEMU Works with taps, set tap to the bridge created
   ip tuntap add dev "${VM_NET_TAP}" mode tap
-  ip link set "${VM_NET_TAP}" up promisc on
+
+  while ! ip link set "${VM_NET_TAP}" up promisc on; do
+    info "Waiting for tap to become available..."
+    sleep 2
+  done
+
   ip link set dev "${VM_NET_TAP}" master dockerbridge
 
   # Add internet connection to the VM
+  IP=$(ip address show dev "${VM_NET_DEV}" | grep inet | awk '/inet / { print $2 }' | cut -f1 -d/)
+
   iptables -t nat -A POSTROUTING -o "${VM_NET_DEV}" -j MASQUERADE
-  iptables -t nat -A PREROUTING -i "${VM_NET_DEV}" -p tcp  -j DNAT --to $VM_NET_IP
-  iptables -t nat -A PREROUTING -i "${VM_NET_DEV}" -p udp  -j DNAT --to $VM_NET_IP
+  iptables -t nat -A PREROUTING -i "${VM_NET_DEV}" -d "${IP}" -p tcp  -j DNAT --to $VM_NET_IP
+  iptables -t nat -A PREROUTING -i "${VM_NET_DEV}" -d "${IP}" -p udp  -j DNAT --to $VM_NET_IP
 
   if (( KERNEL > 4 )); then
     # Hack for guest VMs complaining about "bad udp checksums in 5 packets"
@@ -165,6 +179,24 @@ configureNAT () {
   return 0
 }
 
+closeNetwork () {
+
+  if [[ "${DHCP}" == [Yy1]* ]]; then
+
+    ip link set "${VM_NET_TAP}" down || true
+    ip link delete "${VM_NET_TAP}" || true
+
+  else
+
+    ip link set "${VM_NET_TAP}" down promisc off || true
+    ip link delete "${VM_NET_TAP}" || true
+
+    ip link set dockerbridge down || true
+    ip link delete dockerbridge || true
+
+  fi
+}
+
 # ######################################
 #  Configure Network
 # ######################################
@@ -202,7 +234,11 @@ fi
 if [[ "${DHCP}" == [Yy1]* ]]; then
 
   if [[ "$GATEWAY" == "172."* ]]; then
-    error "You can only enable DHCP while the container is on a macvlan network!" && exit 86
+    if [[ "${DEBUG}" == [Yy1]* ]]; then
+      info "Warning: Are you sure the container is on a macvlan network?"
+    else
+      error "You can only enable DHCP while the container is on a macvlan network!" && exit 86
+    fi
   fi
 
   # Configuration for DHCP IP

run/power.sh

@@ -39,28 +39,9 @@ _graceful_shutdown() {
 
     echo && error "Could not send shutdown command to the guest ($RESPONSE)"
 
-    # If we cannot shutdown the usual way, fallback to the NMI method
+    kill -15 "$(cat "${_QEMU_PID}")"
+    pkill -f qemu-system-x86_64 || true
 
-    AGENT="${STORAGE}/${BASE}.agent"
-    [ -f "$AGENT" ] && AGENT_VERSION=$(cat "${AGENT}") || AGENT_VERSION=1
-
-    if ((AGENT_VERSION > 1)); then
-
-      # Send a NMI interrupt which will be detected by the kernel
-      if ! echo 'nmi' | nc -q 1 -w 1 localhost "${QEMU_MONPORT}" > /dev/null ; then
-        AGENT_VERSION=0
-      fi
-
-    fi
-
-    if ((AGENT_VERSION < 2)); then
-
-      echo && info "Please update the VirtualDSM Agent to allow for gracefull shutdowns..."
-
-      kill -15 "$(cat "${_QEMU_PID}")"
-      pkill -f qemu-system-x86_64 || true
-
-    fi
   fi
 
   while [ "$(cat ${_QEMU_SHUTDOWN_COUNTER})" -lt "${QEMU_POWERDOWN_TIMEOUT}" ]; do
@@ -72,7 +53,9 @@ _graceful_shutdown() {
     if echo 'info version'| nc -q 1 -w 1 localhost "${QEMU_MONPORT}" >/dev/null 2>&1 ; then
 
       sleep 1
-      #info "Shutting down, waiting... ($(cat ${_QEMU_SHUTDOWN_COUNTER})/${QEMU_POWERDOWN_TIMEOUT})"
+
+      CNT="$(cat ${_QEMU_SHUTDOWN_COUNTER})/${QEMU_POWERDOWN_TIMEOUT}"
+      [[ "${DEBUG}" == [Yy1]* ]] && info "Shutting down, waiting... (${CNT})"
 
     fi
 
@@ -81,6 +64,8 @@ _graceful_shutdown() {
   echo && echo "❯ Quitting..."
   echo 'quit' | nc -q 1 -w 1 localhost "${QEMU_MONPORT}" >/dev/null 2>&1 || true
 
+  closeNetwork
+
   return
 }
 

run/print.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+info () { echo -e >&2 "\E[1;34m❯\E[1;36m $1\E[0m" ; }
+error () { echo -e >&2 "\E[1;31m❯ ERROR: $1\E[0m" ; }
+
+retry=true
+
+while [ "$retry" = true ]
+do
+
+  sleep 3
+
+  # Retrieve IP from guest VM
+
+  set +e
+  RESPONSE=$(curl -s -m 16 -S http://127.0.0.1:2210/read?command=10 2>&1)
+  set -e
+
+  if [[ ! "${RESPONSE}" =~ "\"success\"" ]] ; then
+    error "Failed to connect to guest: $RESPONSE" && continue
+  fi
+
+  # Retrieve the HTTP port number
+  if [[ ! "${RESPONSE}" =~ "\"http_port\"" ]] ; then
+    error "Failed to parse response from guest: $RESPONSE" && continue
+  fi
+
+  rest=${RESPONSE#*http_port}
+  rest=${rest#*:}
+  rest=${rest%%,*}
+  PORT=${rest%%\"*}
+
+  [ -z "${PORT}" ] && continue
+
+  # Retrieve the IP address
+  if [[ ! "${RESPONSE}" =~ "eth0" ]] ; then
+    error "Failed to parse response from guest: $RESPONSE" && continue
+  fi
+
+  rest=${RESPONSE#*eth0}
+  rest=${rest#*ip}
+  rest=${rest#*:}
+  rest=${rest#*\"}
+  IP=${rest%%\"*}
+
+  [ -z "${IP}" ] && continue
+
+  retry=false
+
+done
+
+if [[ "$IP" == "20.20"* ]]; then
+  MSG="port ${PORT}"
+else
+  MSG="http://${IP}:${PORT}"
+fi
+
+echo "" >&2
+info "--------------------------------------------------------"
+info " You can now login to DSM at ${MSG}"
+info "--------------------------------------------------------"
+echo "" >&2

run/run.sh

@@ -4,8 +4,10 @@ set -Eeuo pipefail
 # Docker environment variables
 
 : ${URL:=''}            # URL of the PAT file
-: ${DEBUG:='N'}         # Enable debug mode
+: ${GPU:='N'}           # Enable GPU passthrough
+: ${DEBUG:='N'}         # Enable debugging mode
 : ${ALLOCATE:='Y'}      # Preallocate diskspace
+: ${ARGUMENTS:=''}      # Extra QEMU parameters
 : ${CPU_CORES:='1'}     # Amount of CPU cores
 : ${DISK_SIZE:='16G'}   # Initial data disk size
 : ${RAM_SIZE:='512M'}   # Maximum RAM amount
@@ -40,17 +42,10 @@ if [[ ! -f "$STORAGE/$BASE.boot.img" ]] || [[ ! -f "$STORAGE/$BASE.system.img" ]
   . /run/install.sh
 fi
 
-# Initialize disks
-. /run/disk.sh
-
-# Initialize network
-. /run/network.sh
-
-# Initialize serialport
-. /run/serial.sh
-
-# Configure shutdown
-. /run/power.sh
+. /run/disk.sh     # Initialize disks
+. /run/network.sh  # Initialize network
+. /run/serial.sh   # Initialize serialport
+. /run/power.sh    # Configure shutdown
 
 KVM_ERR=""
 KVM_OPTS=""
@@ -80,7 +75,9 @@ EXTRA_OPTS="-device virtio-balloon-pci,id=balloon0,bus=pcie.0,addr=0x4"
 EXTRA_OPTS="$EXTRA_OPTS -object rng-random,id=objrng0,filename=/dev/urandom"
 EXTRA_OPTS="$EXTRA_OPTS -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pcie.0,addr=0x1c"
 
-ARGS="${DEF_OPTS} ${CPU_OPTS} ${RAM_OPTS} ${MAC_OPTS} ${MON_OPTS} ${SERIAL_OPTS} ${NET_OPTS} ${DISK_OPTS} ${EXTRA_OPTS}"
+[[ "${GPU}" == [Yy1]* ]] && [[ "$ARCH" == "amd64" ]] && . /run/gpu.sh
+
+ARGS="${DEF_OPTS} ${CPU_OPTS} ${RAM_OPTS} ${MAC_OPTS} ${MON_OPTS} ${SERIAL_OPTS} ${NET_OPTS} ${DISK_OPTS} ${EXTRA_OPTS} ${ARGUMENTS}"
 ARGS=$(echo "$ARGS" | sed 's/\t/ /g' | tr -s ' ')
 
 trap - ERR
@@ -93,8 +90,8 @@ set -m
 )
 set +m
 
-if (( KERNEL > 5 )) || ( (( KERNEL == 5 )) && (( MINOR > 2 )) ); then
-  pidwait -F "${_QEMU_PID}" & wait $!
-else
-  tail --pid "$(cat "${_QEMU_PID}")" --follow /dev/null & wait $!
-fi
+#if (( KERNEL > 5 )) || ( (( KERNEL == 5 )) && (( MINOR > 2 )) ); then
+#  pidwait -F "${_QEMU_PID}" & wait $!
+#else
+
+tail --pid "$(cat "${_QEMU_PID}")" --follow /dev/null & wait $!

run/serial.sh

@@ -5,13 +5,10 @@ set -Eeuo pipefail
 
 : ${HOST_CPU:=''}
 : ${HOST_MAC:=''}
-: ${HOST_BUILD:=''}
 : ${HOST_DEBUG:=''}
 : ${HOST_SERIAL:=''}
-: ${GUEST_SERIAL:=''}
 : ${HOST_MODEL:=''}
-: ${HOST_VERSION:=''}
-: ${HOST_TIMESTAMP:=''}
+: ${GUEST_SERIAL:=''}
 
 if [ -z "$HOST_CPU" ]; then
   HOST_CPU=$(lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1' | sed 's# @.*##g' | sed s/"(R)"//g | sed 's/[^[:alnum:] ]\+/ /g' | sed 's/  */ /g')
@@ -32,12 +29,9 @@ HOST_ARGS+=("-cpu=${CPU_CORES}")
 HOST_ARGS+=("-cpu_arch=${HOST_CPU}")
 
 [ -n "$HOST_MAC" ] && HOST_ARGS+=("-mac=${HOST_MAC}")
-[ -n "$HOST_BUILD" ] && HOST_ARGS+=("-build=${HOST_BUILD}")
 [ -n "$HOST_MODEL" ] && HOST_ARGS+=("-model=${HOST_MODEL}")
 [ -n "$HOST_SERIAL" ] && HOST_ARGS+=("-hostsn=${HOST_SERIAL}")
 [ -n "$GUEST_SERIAL" ] && HOST_ARGS+=("-guestsn=${GUEST_SERIAL}")
-[ -n "$HOST_VERSION" ] && HOST_ARGS+=("-version=${HOST_VERSION}")
-[ -n "$HOST_TIMESTAMP" ] && HOST_ARGS+=("-ts=${HOST_TIMESTAMP}")
 
 if [[ "${HOST_DEBUG}" == [Yy1]* ]]; then
   set -x
@@ -45,15 +39,15 @@ if [[ "${HOST_DEBUG}" == [Yy1]* ]]; then
   { set +x; } 2>/dev/null
   echo
 else
-  ./run/host.bin "${HOST_ARGS[@]}" 2> /dev/null &
+  ./run/host.bin "${HOST_ARGS[@]}" >/dev/null &
 fi
 
 # Configure serial ports
 
 SERIAL_OPTS="\
 	-serial mon:stdio \
-	-device virtio-serial-pci,id=virtio-serial0,bus=pcie.0,addr=0x3 \
-	-chardev pty,id=charserial0 \
-	-device isa-serial,chardev=charserial0,id=serial0 \
-	-chardev socket,id=charchannel0,host=127.0.0.1,port=12345,reconnect=10 \
-	-device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=vchannel"
+        -device virtio-serial-pci,id=virtio-serial0,bus=pcie.0,addr=0x3 \
+        -chardev pty,id=charserial0 \
+        -device isa-serial,chardev=charserial0,id=serial0 \
+        -chardev socket,id=charchannel0,host=127.0.0.1,port=12345,reconnect=10 \
+        -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=vchannel"