Harden ISO build workflow: verifiable upload, cleanup on cancel/exit

.gitea/workflows/build-iso.yaml

@@ -5,6 +5,10 @@ on:
     - cron: "0 0 1 * *" # Runs monthly on the 1st day of the month
   workflow_dispatch: # Allows manual triggers (multiple times a month)
 
+concurrency:
+  group: iso-build
+  cancel-in-progress: false # Never kill a build that is mid-flight
+
 jobs:
   build:
     runs-on: archlinux
@@ -17,88 +21,191 @@ jobs:
 
       - name: Setup local AUR repo
         run: |
+          set -euo pipefail
           mkdir -p /tmp/localrepo
           cp localrepo/*.pkg.tar.zst localrepo/local-aur.db* localrepo/local-aur.files* /tmp/localrepo/
           sed -i 's|file:///home/alvin/alvinarch/localrepo|file:///tmp/localrepo|' pacman.conf
 
       - name: Install archiso and dependencies
         run: |
+          set -euo pipefail
           pacman -Syu --noconfirm --needed archiso jq
 
       - name: Build Arch ISO
         run: |
-          rm -rf "$PWD/work" || true
+          set -euo pipefail
+          # Clean up immediately even if this step is interrupted (cancel/exit):
+          # unmount leftovers first, then remove the multi-GB work dir.
+          trap 'umount -R "$PWD/work" 2>/dev/null || true; rm -rf "$PWD/work"' EXIT HUP INT TERM
+          rm -rf "$PWD/work"
+          # Always start from an empty out dir so stale ISOs are never re-uploaded
+          rm -rf "$PWD/out"
+          mkdir -p "$PWD/out"
           mkarchiso -v -w "$PWD/work" -o "$PWD/out" -- "$PWD"
 
       - name: Generate Unique Dynamic Tag
         id: tag
         env:
           GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
-          SERVER_URL: ${{ github.server_url }}
+          GITEA_API: http://192.168.1.168:3000/api/v1
           REPOSITORY: ${{ github.repository }}
         run: |
+          set -euo pipefail
+          test -n "$GITEA_TOKEN"
+
+          # Pre-flight: confirm the internal Gitea API is reachable so failures are clear
+          if ! curl -sfS -o /dev/null "$GITEA_API/version"; then
+            echo "❌ Cannot reach Gitea API at $GITEA_API" >&2
+            exit 1
+          fi
+
           BASE_DATE=$(date -u +'%Y.%m.%d')
           REVISION=1
           TAG_NAME="iso-${BASE_DATE}-r${REVISION}"
 
           # Check if the tag already exists via Gitea API, increment revision loop if it does
-          while curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITEA_TOKEN" \
-            "$SERVER_URL/api/v1/repos/$REPOSITORY/releases/tags/$TAG_NAME" | grep -q "200"; do
+          while curl -sf -o /dev/null -H "Authorization: token $GITEA_TOKEN" \
+            "$GITEA_API/repos/$REPOSITORY/releases/tags/$TAG_NAME"; do
             REVISION=$((REVISION + 1))
             TAG_NAME="iso-${BASE_DATE}-r${REVISION}"
           done
 
-          echo "tag_name=${TAG_NAME}" >> $GITHUB_OUTPUT
-          echo "release_name=Arch Linux ISO (${BASE_DATE} Rev ${REVISION})" >> $GITHUB_OUTPUT
+          echo "tag_name=${TAG_NAME}" >> "$GITHUB_OUTPUT"
+          echo "release_name=Arch Linux ISO (${BASE_DATE} Rev ${REVISION})" >> "$GITHUB_OUTPUT"
 
       - name: Create Release and Upload ISO
         env:
           GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
-          SERVER_URL: ${{ github.server_url }}
           # Use internal Gitea URL to bypass traefik proxy timeout on large uploads
           GITEA_API: http://192.168.1.168:3000/api/v1
           REPOSITORY: ${{ github.repository }}
           TAG_NAME: ${{ steps.tag.outputs.tag_name }}
           RELEASE_NAME: ${{ steps.tag.outputs.release_name }}
         run: |
+          set -euo pipefail
+          shopt -s nullglob
+          test -n "$GITEA_TOKEN"
+          test -n "$TAG_NAME"
+          test -n "$RELEASE_NAME"
+
+          # Pre-flight: confirm the internal Gitea API is reachable so failures are clear
+          if ! curl -sfS -o /dev/null "$GITEA_API/version"; then
+            echo "❌ Cannot reach Gitea API at $GITEA_API" >&2
+            exit 1
+          fi
+
           # Create the release via Gitea API (internal, no traefik)
           RELEASE_ID=$(curl -sf -X POST \
             -H "Authorization: token $GITEA_TOKEN" \
             -H "Content-Type: application/json" \
             "$GITEA_API/repos/$REPOSITORY/releases" \
             -d "{\"tag_name\": \"$TAG_NAME\", \"name\": \"$RELEASE_NAME\"}" \
-            | jq -r '.id')
+            | jq -er '.id')
 
           echo "Created release ID: $RELEASE_ID"
 
-          # Upload each ISO as a release asset (internal, no traefik)
-          for iso in out/*.iso; do
-            echo "Uploading $iso ($(du -h "$iso" | cut -f1))..."
-            curl -sf --retry 3 --retry-delay 10 \
-              --max-time 1800 \
+          # Collect the ISOs produced by the build
+          isos=(out/*.iso)
+          if (( ${#isos[@]} == 0 )); then
+            echo "❌ No ISO found in out/ — nothing to upload, aborting." >&2
+            exit 1
+          fi
+
+          for iso in "${isos[@]}"; do
+            fname=$(basename "$iso")
+            size=$(stat -c %s "$iso")
+            # Sanity check: a real ISO is multi-GB; anything tiny is the wrong file
+            if (( size < 100000000 )); then
+              echo "❌ '$iso' is only $size bytes — refusing to upload (not an ISO)." >&2
+              exit 1
+            fi
+
+            echo "Uploading $fname ($(( size / 1024 / 1024 )) MiB)..."
+
+            http_code=""
+            for attempt in 1 2 3; do
+              http_code=$(curl -sS --max-time 1800 \
+                -o /dev/null -w "%{http_code}" \
                 -H "Authorization: token $GITEA_TOKEN" \
-              -F "attachment=@$iso" \
-              "$GITEA_API/repos/$REPOSITORY/releases/$RELEASE_ID/assets"
-            echo "Upload complete: $iso"
+                -F "attachment=@$iso;filename=$fname" \
+                "$GITEA_API/repos/$REPOSITORY/releases/$RELEASE_ID/assets" || true)
+
+              if [[ "$http_code" == "201" ]]; then
+                echo "  ✅ Upload OK (HTTP 201) on attempt $attempt"
+                break
+              fi
+              echo "  ⚠️  Upload attempt $attempt returned HTTP $http_code — retrying..." >&2
+              sleep 5
+            done
+
+            if [[ "$http_code" != "201" ]]; then
+              echo "❌ Failed to upload '$fname' (last HTTP code: $http_code)" >&2
+              exit 1
+            fi
+
+            # Verify the exact asset made it: correct name AND byte size
+            confirmed=$(curl -sf -H "Authorization: token $GITEA_TOKEN" \
+              "$GITEA_API/repos/$REPOSITORY/releases/$RELEASE_ID/assets" \
+              | jq -er --arg n "$fname" --argjson s "$size" \
+                'first(.[] | select(.name == $n and .size == $s) | .name) // empty')
+            if [[ -z "$confirmed" ]]; then
+              echo "❌ Verification failed: asset '$fname' ($size bytes) not found on the release." >&2
+              exit 1
+            fi
+            echo "  ✅ Verified '$confirmed' ($size bytes) attached to $TAG_NAME"
           done
 
           echo "Release $TAG_NAME created and ISOs uploaded."
 
       - name: Keep only latest 3 releases
+        if: success()
         env:
           GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
           GITEA_API: http://192.168.1.168:3000/api/v1
           REPOSITORY: ${{ github.repository }}
         run: |
-          # Fetch all releases
-          releases=$(curl -sf -H "Authorization: token $GITEA_TOKEN" \
-            "$GITEA_API/repos/$REPOSITORY/releases")
+          set -euo pipefail
+          test -n "$GITEA_TOKEN"
 
-          # Sort by creation date and delete old ones, keeping only the 3 newest releases
-          echo "$releases" | jq -r 'sort_by(.created_at) | .[].id' | head -n -3 | while read -r id; do
-            if [ -n "$id" ]; then
+          # Fetch all release ids, oldest first
+          readarray -t ids < <(curl -sf -H "Authorization: token $GITEA_TOKEN" \
+            "$GITEA_API/repos/$REPOSITORY/releases" \
+            | jq -er 'sort_by(.created_at) | .[].id')
+
+          # Delete the oldest releases, keeping only the 3 newest
+          to_delete=$(( ${#ids[@]} - 3 ))
+          if (( to_delete > 0 )); then
+            for ((i = 0; i < to_delete; i++)); do
+              id=${ids[$i]}
               echo "Deleting old release ID: $id"
               curl -sf -X DELETE -H "Authorization: token $GITEA_TOKEN" \
                 "$GITEA_API/repos/$REPOSITORY/releases/$id"
-            fi
             done
+          fi
+
+      - name: Cleanup mkarchiso mounts and workdirs
+        if: always() # Runs on success, failure, or cancellation
+        run: |
+          set -eu
+          # Steps run as root (pacman is used directly); fall back to sudo if not
+          if [ "$(id -u)" -eq 0 ]; then
+            PRIV=""
+          else
+            PRIV="sudo"
+          fi
+
+          # Unmount any leftover loop devices or bind mounts in the work directory
+          $PRIV umount -R "$PWD/work" 2>/dev/null || true
+
+          # Clean up loop devices if any remain attached to the work dir
+          $PRIV losetup -D || true
+
+          # Remove working directories with proper permissions
+          $PRIV rm -rf "$PWD/work"
+
+      - name: Remove built ISO (success or cancellation)
+        if: success() || cancelled()
+        run: |
+          set -eu
+          # On failure the ISO is kept in out/ for manual upload
+          rm -rf "$PWD/out"