From 01c7c4e1fa74fda6171b27103e6d63d14b84712b Mon Sep 17 00:00:00 2001 From: alvinlollo Date: Wed, 16 Sep 2026 16:37:25 +1000 Subject: [PATCH] Harden ISO build workflow: verifiable upload, cleanup on cancel/exit --- .gitea/workflows/build-iso.yaml | 161 ++++++++++++++++++++++++++------ 1 file changed, 134 insertions(+), 27 deletions(-) diff --git a/.gitea/workflows/build-iso.yaml b/.gitea/workflows/build-iso.yaml index 5de0e16..fbe9b97 100644 --- a/.gitea/workflows/build-iso.yaml +++ b/.gitea/workflows/build-iso.yaml @@ -5,6 +5,10 @@ on: - cron: "0 0 1 * *" # Runs monthly on the 1st day of the month workflow_dispatch: # Allows manual triggers (multiple times a month) +concurrency: + group: iso-build + cancel-in-progress: false # Never kill a build that is mid-flight + jobs: build: runs-on: archlinux @@ -17,88 +21,191 @@ jobs: - name: Setup local AUR repo run: | + set -euo pipefail mkdir -p /tmp/localrepo cp localrepo/*.pkg.tar.zst localrepo/local-aur.db* localrepo/local-aur.files* /tmp/localrepo/ sed -i 's|file:///home/alvin/alvinarch/localrepo|file:///tmp/localrepo|' pacman.conf - name: Install archiso and dependencies run: | + set -euo pipefail pacman -Syu --noconfirm --needed archiso jq - name: Build Arch ISO run: | - rm -rf "$PWD/work" || true + set -euo pipefail + # Clean up immediately even if this step is interrupted (cancel/exit): + # unmount leftovers first, then remove the multi-GB work dir. + trap 'umount -R "$PWD/work" 2>/dev/null || true; rm -rf "$PWD/work"' EXIT HUP INT TERM + rm -rf "$PWD/work" + # Always start from an empty out dir so stale ISOs are never re-uploaded + rm -rf "$PWD/out" + mkdir -p "$PWD/out" mkarchiso -v -w "$PWD/work" -o "$PWD/out" -- "$PWD" - name: Generate Unique Dynamic Tag id: tag env: GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - SERVER_URL: ${{ github.server_url }} + GITEA_API: http://192.168.1.168:3000/api/v1 REPOSITORY: ${{ github.repository }} run: | + set -euo pipefail + test -n "$GITEA_TOKEN" + + # Pre-flight: confirm the internal Gitea API is reachable so failures are clear + if ! curl -sfS -o /dev/null "$GITEA_API/version"; then + echo "❌ Cannot reach Gitea API at $GITEA_API" >&2 + exit 1 + fi + BASE_DATE=$(date -u +'%Y.%m.%d') REVISION=1 TAG_NAME="iso-${BASE_DATE}-r${REVISION}" # Check if the tag already exists via Gitea API, increment revision loop if it does - while curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITEA_TOKEN" \ - "$SERVER_URL/api/v1/repos/$REPOSITORY/releases/tags/$TAG_NAME" | grep -q "200"; do - REVISION=$((REVISION + 1)) - TAG_NAME="iso-${BASE_DATE}-r${REVISION}" + while curl -sf -o /dev/null -H "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API/repos/$REPOSITORY/releases/tags/$TAG_NAME"; do + REVISION=$((REVISION + 1)) + TAG_NAME="iso-${BASE_DATE}-r${REVISION}" done - echo "tag_name=${TAG_NAME}" >> $GITHUB_OUTPUT - echo "release_name=Arch Linux ISO (${BASE_DATE} Rev ${REVISION})" >> $GITHUB_OUTPUT + echo "tag_name=${TAG_NAME}" >> "$GITHUB_OUTPUT" + echo "release_name=Arch Linux ISO (${BASE_DATE} Rev ${REVISION})" >> "$GITHUB_OUTPUT" - name: Create Release and Upload ISO env: GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} - SERVER_URL: ${{ github.server_url }} # Use internal Gitea URL to bypass traefik proxy timeout on large uploads GITEA_API: http://192.168.1.168:3000/api/v1 REPOSITORY: ${{ github.repository }} TAG_NAME: ${{ steps.tag.outputs.tag_name }} RELEASE_NAME: ${{ steps.tag.outputs.release_name }} run: | + set -euo pipefail + shopt -s nullglob + test -n "$GITEA_TOKEN" + test -n "$TAG_NAME" + test -n "$RELEASE_NAME" + + # Pre-flight: confirm the internal Gitea API is reachable so failures are clear + if ! curl -sfS -o /dev/null "$GITEA_API/version"; then + echo "❌ Cannot reach Gitea API at $GITEA_API" >&2 + exit 1 + fi + # Create the release via Gitea API (internal, no traefik) RELEASE_ID=$(curl -sf -X POST \ -H "Authorization: token $GITEA_TOKEN" \ -H "Content-Type: application/json" \ "$GITEA_API/repos/$REPOSITORY/releases" \ -d "{\"tag_name\": \"$TAG_NAME\", \"name\": \"$RELEASE_NAME\"}" \ - | jq -r '.id') + | jq -er '.id') echo "Created release ID: $RELEASE_ID" - # Upload each ISO as a release asset (internal, no traefik) - for iso in out/*.iso; do - echo "Uploading $iso ($(du -h "$iso" | cut -f1))..." - curl -sf --retry 3 --retry-delay 10 \ - --max-time 1800 \ - -H "Authorization: token $GITEA_TOKEN" \ - -F "attachment=@$iso" \ - "$GITEA_API/repos/$REPOSITORY/releases/$RELEASE_ID/assets" - echo "Upload complete: $iso" + # Collect the ISOs produced by the build + isos=(out/*.iso) + if (( ${#isos[@]} == 0 )); then + echo "❌ No ISO found in out/ — nothing to upload, aborting." >&2 + exit 1 + fi + + for iso in "${isos[@]}"; do + fname=$(basename "$iso") + size=$(stat -c %s "$iso") + # Sanity check: a real ISO is multi-GB; anything tiny is the wrong file + if (( size < 100000000 )); then + echo "❌ '$iso' is only $size bytes — refusing to upload (not an ISO)." >&2 + exit 1 + fi + + echo "Uploading $fname ($(( size / 1024 / 1024 )) MiB)..." + + http_code="" + for attempt in 1 2 3; do + http_code=$(curl -sS --max-time 1800 \ + -o /dev/null -w "%{http_code}" \ + -H "Authorization: token $GITEA_TOKEN" \ + -F "attachment=@$iso;filename=$fname" \ + "$GITEA_API/repos/$REPOSITORY/releases/$RELEASE_ID/assets" || true) + + if [[ "$http_code" == "201" ]]; then + echo " ✅ Upload OK (HTTP 201) on attempt $attempt" + break + fi + echo " ⚠️ Upload attempt $attempt returned HTTP $http_code — retrying..." >&2 + sleep 5 + done + + if [[ "$http_code" != "201" ]]; then + echo "❌ Failed to upload '$fname' (last HTTP code: $http_code)" >&2 + exit 1 + fi + + # Verify the exact asset made it: correct name AND byte size + confirmed=$(curl -sf -H "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API/repos/$REPOSITORY/releases/$RELEASE_ID/assets" \ + | jq -er --arg n "$fname" --argjson s "$size" \ + 'first(.[] | select(.name == $n and .size == $s) | .name) // empty') + if [[ -z "$confirmed" ]]; then + echo "❌ Verification failed: asset '$fname' ($size bytes) not found on the release." >&2 + exit 1 + fi + echo " ✅ Verified '$confirmed' ($size bytes) attached to $TAG_NAME" done echo "Release $TAG_NAME created and ISOs uploaded." - name: Keep only latest 3 releases + if: success() env: GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} GITEA_API: http://192.168.1.168:3000/api/v1 REPOSITORY: ${{ github.repository }} run: | - # Fetch all releases - releases=$(curl -sf -H "Authorization: token $GITEA_TOKEN" \ - "$GITEA_API/repos/$REPOSITORY/releases") + set -euo pipefail + test -n "$GITEA_TOKEN" - # Sort by creation date and delete old ones, keeping only the 3 newest releases - echo "$releases" | jq -r 'sort_by(.created_at) | .[].id' | head -n -3 | while read -r id; do - if [ -n "$id" ]; then + # Fetch all release ids, oldest first + readarray -t ids < <(curl -sf -H "Authorization: token $GITEA_TOKEN" \ + "$GITEA_API/repos/$REPOSITORY/releases" \ + | jq -er 'sort_by(.created_at) | .[].id') + + # Delete the oldest releases, keeping only the 3 newest + to_delete=$(( ${#ids[@]} - 3 )) + if (( to_delete > 0 )); then + for ((i = 0; i < to_delete; i++)); do + id=${ids[$i]} echo "Deleting old release ID: $id" curl -sf -X DELETE -H "Authorization: token $GITEA_TOKEN" \ "$GITEA_API/repos/$REPOSITORY/releases/$id" - fi - done + done + fi + + - name: Cleanup mkarchiso mounts and workdirs + if: always() # Runs on success, failure, or cancellation + run: | + set -eu + # Steps run as root (pacman is used directly); fall back to sudo if not + if [ "$(id -u)" -eq 0 ]; then + PRIV="" + else + PRIV="sudo" + fi + + # Unmount any leftover loop devices or bind mounts in the work directory + $PRIV umount -R "$PWD/work" 2>/dev/null || true + + # Clean up loop devices if any remain attached to the work dir + $PRIV losetup -D || true + + # Remove working directories with proper permissions + $PRIV rm -rf "$PWD/work" + + - name: Remove built ISO (success or cancellation) + if: success() || cancelled() + run: | + set -eu + # On failure the ISO is kept in out/ for manual upload + rm -rf "$PWD/out" \ No newline at end of file