feat: New extraction method (#954)

.github/workflows/check.yml

@@ -20,5 +20,5 @@ jobs:
         uses: hadolint/hadolint-action@v3.1.0
         with:
           dockerfile: Dockerfile
-          ignore: DL3008,DL3003,DL3006
+          ignore: DL3008,DL3003,DL3006,DL3013
           failure-threshold: warning

Dockerfile

@@ -15,13 +15,12 @@ ARG DEBCONF_NOWARNINGS="yes"
 ARG DEBIAN_FRONTEND="noninteractive"
 ARG DEBCONF_NONINTERACTIVE_SEEN="true"
 
-RUN set -eu && extra="" && \
+RUN set -eu && \
     apt-get update && \
     apt-get --no-install-recommends -y install \
         jq \
         tini \
         curl \
-        cpio \
         wget \
         fdisk \
         unzip \
@@ -57,8 +56,7 @@ COPY --chmod=755 ./src /run/
 COPY --chmod=755 ./web /var/www/
 COPY --chmod=755 --from=builder /qemu-host.bin /run/host.bin
 COPY --chmod=744 ./web/conf/nginx.conf /etc/nginx/sites-enabled/web.conf
-
+ADD --chmod=775 https://raw.githubusercontent.com/sud0woodo/patology/refs/heads/main/patology.py /run/extract.py
-ADD https://raw.githubusercontent.com/sud0woodo/patology/refs/heads/main/patology.py /run/patology.py
 
 VOLUME /storage
 EXPOSE 22 139 445 5000

readme.md

@@ -58,7 +58,7 @@ kubectl apply -f https://raw.githubusercontent.com/vdsm/virtual-dsm/refs/heads/m
 
 ##### Via Github Codespaces:
 
-[`Click here to launch this container in the cloud!`](https://github.com/codespaces/new?skip_quickstart=true&machine=basicLinux32gb&repo=619260050&ref=master&devcontainer_path=.devcontainer.json)
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new?repo=619260050&machine=basicLinux32gb)
 
 ## FAQ 💬
 
@@ -123,21 +123,16 @@ kubectl apply -f https://raw.githubusercontent.com/vdsm/virtual-dsm/refs/heads/m
 
 ### How do I pass-through a disk?
 
-   It is possible to pass-through a disk device directly, which can be useful when your host is a virtual machine, as it removes an extra layer and allows for easier management. For use with physical disks this method provides little advantage and is not recommended.
+   It is possible to pass-through a disk device directly, which can be useful when your host is a virtual machine, as it removes an extra layer and allows for easier image management on the host.
    
-   You can add the virtual device to your compose file like this:
+   For use with physical disks this method provides little advantage over using an image and is not recommended. You can add the device to your compose file like this:
 
   ```yaml
   devices:
-    - /dev/disk/by-uuid/12345-12345-12345-12345-12345:/disk2
+    - /dev/sdc:/disk1
   ```
 
-  The device needs to be totally empty (without any partition table) otherwise DSM does not always format it into a volume.
+  Replace `/dev/sdc` with the name of the disk. It needs to be totally empty (without any partition table) otherwise DSM does not always format it into a volume.
-  
-  Make sure to bind the disk via its UUID (obtainable via `lsblk -o name,uuid`) instead of its name (`/dev/sdc`), to prevent ever binding the wrong disk when the drive letters happen to change. 
- 
-> [!CAUTION]
-> Do NOT use this feature with the goal of sharing files from the host, they might all get lost without warning when DSM creates the volume.
 
 ### How do I change the amount of CPU or RAM?
 
@@ -256,14 +251,14 @@ kubectl apply -f https://raw.githubusercontent.com/vdsm/virtual-dsm/refs/heads/m
 
   With this method, it is even possible to switch back and forth between versions while keeping your file data intact.
 
-  If you don't have internet access, it's also possible to skip the download by setting `URL` to:
+  Alternatively, you can also skip the download and use a local file instead, by binding it in your compose file in this way:
-
+  
   ```yaml
-  environment:
+  volumes:
-    URL: "DSM_VirtualDSM_42218.pat"
+    - ./DSM_VirtualDSM_42218.pat:/boot.pat
   ```
 
-  after placing a copy of [DSM_VirtualDSM_42218.pat](https://global.synologydownload.com/download/DSM/release/7.0.1/42218/DSM_VirtualDSM_42218.pat) in the root of your `/storage` folder.
+  Replace the example path `./DSM_VirtualDSM_42218.pat` with the filename of your desired `.pat` file. The value of `URL` will be ignored in this case.
 
 ### What are the differences compared to the standard DSM?
 

src/install.sh

@@ -12,18 +12,28 @@ else
   BASE="DSM_VirtualDSM_42962"
 fi
 
+FN="boot.pat"
+if [ -d "/$FN" ]; then
+  error "The file /$FN does not exist, please check that you mapped it to a valid path!" && exit 65
+fi
+
+FILE=$(find / -maxdepth 1 -type f -iname "$FN" | head -n 1)
+[ ! -s "$FILE" ] && FILE=$(find "$STORAGE" -maxdepth 1 -type f -iname "$FN" | head -n 1)
+[ -s "$FILE" ] && BASE="DSM_VirtualDSM" && URL="file://$FILE" 
+
 if [ -n "$URL" ]; then
-  BASE=$(basename "$URL" .pat)
+  if [ ! -s "$FILE" ]; then
-  if [ ! -s "$STORAGE/$BASE.system.img" ]; then
+    BASE=$(basename "$URL" .pat)
-    BASE=$(basename "${URL%%\?*}" .pat)
+    if [ ! -s "$STORAGE/$BASE.system.img" ]; then
-    : "${BASE//+/ }"; printf -v BASE '%b' "${_//%/\\x}"
+      BASE=$(basename "${URL%%\?*}" .pat)
-    BASE=$(echo "$BASE" | sed -e 's/[^A-Za-z0-9._-]/_/g')
+      : "${BASE//+/ }"; printf -v BASE '%b' "${_//%/\\x}"
+      BASE=$(echo "$BASE" | sed -e 's/[^A-Za-z0-9._-]/_/g')
+    fi
   fi
-  if [[ "${URL,,}" != "http"* ]]; then
+  if [[ "${URL,,}" != "http"* ]] && [[ "${URL,,}" != "file:"* ]] ; then
-    if [ -s "$STORAGE/$BASE.pat" ]; then
+    URL="file:///$BASE.pat"
-      URL="file://$STORAGE/$BASE.pat"
+    if [ ! -s "/$BASE.pat" ]; then
-    else
+      error "File '$BASE.pat' does not exist!" && exit 65
-      error "File $STORAGE/$BASE.pat does not exist!" && exit 65
     fi
   fi
 fi
@@ -51,9 +61,11 @@ if [ -z "$URL" ]; then
   URL="$DL/release/7.2.2/72806/DSM_VirtualDSM_72806.pat"
 fi
 
-BASE=$(basename "${URL%%\?*}" .pat)
+if [ ! -s "$FILE" ]; then
-: "${BASE//+/ }"; printf -v BASE '%b' "${_//%/\\x}"
+  BASE=$(basename "${URL%%\?*}" .pat)
-BASE=$(echo "$BASE" | sed -e 's/[^A-Za-z0-9._-]/_/g')
+  : "${BASE//+/ }"; printf -v BASE '%b' "${_//%/\\x}"
+  BASE=$(echo "$BASE" | sed -e 's/[^A-Za-z0-9._-]/_/g')
+fi
 
 if [[ "$URL" != "file://$STORAGE/$BASE.pat" ]]; then
   rm -f "$STORAGE/$BASE.pat"
@@ -114,10 +126,15 @@ else
   PROGRESS="--progress=dot:giga"
 fi
 
-info "Install: Downloading $BASE.pat..."
+if [[ "$URL" == "file://"* ]]; then
-
+  MSG="Copying DSM"
-MSG="Downloading DSM"
+  ERR="Failed to copy ${URL:7}"
-ERR="Failed to download $URL"
+  info "Install: Copying installation image..."
+else
+  MSG="Downloading DSM"
+  ERR="Failed to download $URL"
+  info "Install: Downloading $BASE.pat..."
+fi
 
 html "$MSG..."
 
@@ -126,6 +143,10 @@ rm -f "$PAT"
 
 if [[ "$URL" == "file://"* ]]; then
 
+  if [ ! -f "${URL:7}" ]; then
+    error "File '${URL:7}' does not exist!" && exit 65
+  fi
+
   cp "${URL:7}" "$PAT"
 
 else
@@ -156,7 +177,7 @@ if ((SIZE<250000000)); then
   error "The specified PAT file is probably an update pack as it's too small." && exit 62
 fi
 
-MSG="Extracting downloaded image..."
+MSG="Extracting installation image..."
 info "Install: $MSG" && html "$MSG"
 
 if { tar tf "$PAT"; } >/dev/null 2>&1; then
@@ -165,16 +186,18 @@ if { tar tf "$PAT"; } >/dev/null 2>&1; then
 
 else
 
-  { (cd "$TMP" && python3 /run/patology.py -i "$PAT" -d >/dev/null); rc=$?; } || :
+  { (cd "$TMP" && python3 /run/extract.py -i "$PAT" -d 2>/run/extract.log); rc=$?; } || :
-  (( rc != 0 )) && error "Failed to extract PAT file, reason $rc" && exit 63
+
+  if (( rc != 0 )); then
+    cat /run/extract.log
+    error "Failed to extract PAT file, reason $rc" && exit 63
+  fi
 
 fi
 
 MSG="Preparing system partition..."
 info "Install: $MSG" && html "$MSG"
 
-ls -lh $TMP
-
 BOOT=$(find "$TMP" -name "*.bin.zip")
 [ ! -s "$BOOT" ] && error "The PAT file contains no boot image." && exit 67
 
@@ -258,22 +281,10 @@ OFFSET="1048576"    # 2048 * 512
 NUMBLOCKS="2097152" # (16777216 * 512) / 4096
 MSG="Installing system partition..."
 
-if [[ "$ROOT" != [Nn]* ]]; then
+fakeroot -- bash -c "set -Eeu;\
-
+  tar xpfJ $HDA.txz --absolute-names --skip-old-files -C $MOUNT/;\
-  tar xpfJ "$HDA.txz" --absolute-names --skip-old-files -C "$MOUNT/"
+  printf '%b%s%b' '\E[1;34m❯ \E[1;36m' 'Install: $MSG' '\E[0m\n';\
-
+  mke2fs -q -t ext4 -b 4096 -d $MOUNT/ -L $LABEL -F -E offset=$OFFSET $SYSTEM $NUMBLOCKS"
-  info "Install: $MSG" && html "$MSG"
-
-  mke2fs -q -t ext4 -b 4096 -d "$MOUNT/" -L "$LABEL" -F -E "offset=$OFFSET" "$SYSTEM" "$NUMBLOCKS"
-
-else
-
-  fakeroot -- bash -c "set -Eeu;\
-        tar xpfJ $HDA.txz --absolute-names --skip-old-files -C $MOUNT/;\
-        printf '%b%s%b' '\E[1;34m❯ \E[1;36m' 'Install: $MSG' '\E[0m\n';\
-        mke2fs -q -t ext4 -b 4096 -d $MOUNT/ -L $LABEL -F -E offset=$OFFSET $SYSTEM $NUMBLOCKS"
-
-fi
 
 rm -rf "$MOUNT"
 echo "$BASE" > "$STORAGE/dsm.ver"