ci: retry pacman syncs and reset host pacman.conf before build

Run 131 failed at 'pacman -Syu archiso jq' because the gateway (192.168.1.1) intermittently drops UDP/53: cachyos.db and chaotic-aur.db lookups hit libalpm's 10s resolve timeout while core/extra survived on resolved's cache. - build-iso: reset the host /etc/pacman.conf to stock core/extra first (tests.yaml leaves [cachyos]/[chaotic-aur] behind, and mkarchiso reads the profile's pacman.conf anyway), pre-warm mirror hostnames in the resolver cache, then retry pacman -Syu 5x with backoff. - tests: wrap all three pacman -Syu calls in the same retry loop. DNS-side redundancy was added on the runner separately (networkd drop-in adds 1.1.1.1/8.8.8.8 next to the gateway).
2026-10-07 11:14:28 +11:00
parent a6c99b72d0
commit 0fbec83573
2 changed files with 101 additions and 4 deletions
@@ -41,7 +41,62 @@ jobs:
      - name: Install archiso and dependencies
        run: |
          set -euo pipefail
-          pacman -Syu --noconfirm --needed archiso jq
+          # The host's /etc/pacman.conf is shared, mutable state: the tests
+          # workflow appends [cachyos] and [chaotic-aur] to it, so without this
+          # reset the build would sync two repos the host never needs (it only
+          # installs archiso/jq from extra — mkarchiso reads the *profile's*
+          # pacman.conf). Every extra repo is another DNS lookup, and the
+          # gateway intermittently drops UDP/53 (that is what failed run 131
+          # at this very step). Reset to stock core/extra.
+          cat > /etc/pacman.conf << 'STOCKCONF'
+          [options]
+          DisableSandbox
+          HoldPkg     = pacman glibc
+          Architecture = auto
+          CheckSpace
+          ParallelDownloads = 5
+          SigLevel    = Required DatabaseOptional
+          LocalFileSigLevel = Optional
+
+          [core]
+          Include = /etc/pacman.d/mirrorlist
+
+          [extra]
+          Include = /etc/pacman.d/mirrorlist
+          STOCKCONF
+          sed -i 's/^          //' /etc/pacman.conf
+          # Drop sync databases of the custom repos so they cannot linger
+          rm -f /var/lib/pacman/sync/cachyos* /var/lib/pacman/sync/chaotic*
+
+          # Warm the resolver cache for the mirrors we are about to query.
+          # systemd-resolved serves later lookups from cache, so a gateway
+          # UDP/53 burst cannot hit them again. Failures here are not fatal —
+          # the retry loop below is the real safety net.
+          for host in mirror.aarnet.edu.au gsl-syd.mm.fcix.net au.arch.niranjan.co; do
+            for try in 1 2 3 4 5; do
+              if getent hosts "$host" >/dev/null 2>&1; then
+                break
+              fi
+              echo "  ⏳ DNS for $host failed (try $try/5), retrying in $(( try * 3 ))s..."
+              sleep $(( try * 3 ))
+            done
+          done
+
+          # The gateway drops DNS queries in bursts; one burst must not fail
+          # the whole job, so retry with backoff before giving up.
+          synced=0
+          for try in 1 2 3 4 5; do
+            if pacman -Syu --noconfirm --needed archiso jq; then
+              synced=1
+              break
+            fi
+            echo "  ⚠️ pacman -Syu failed (attempt $try/5), retrying in $(( try * 15 ))s..." >&2
+            sleep $(( try * 15 ))
+          done
+          if [ "$synced" -ne 1 ]; then
+            echo "❌ pacman -Syu failed after 5 attempts — giving up" >&2
+            exit 1
+          fi

      - name: Ensure /dev/fd symlinks exist for mkinitcpio and procsub
        run: |
@@ -36,7 +36,23 @@ jobs:
          rm -f /var/lib/pacman/sync/cachyos* /var/lib/pacman/sync/chaotic*

      - name: Install prerequisites
-        run: pacman -Syu --noconfirm --needed shellcheck yamllint
+        run: |
+          # The gateway intermittently drops UDP/53 (it failed build run 131
+          # at the equivalent step); retry with backoff instead of dying on
+          # the first bad DNS burst.
+          synced=0
+          for try in 1 2 3 4 5; do
+            if pacman -Syu --noconfirm --needed shellcheck yamllint; then
+              synced=1
+              break
+            fi
+            echo "⚠️ pacman -Syu failed (attempt $try/5), retrying in $(( try * 15 ))s..." >&2
+            sleep $(( try * 15 ))
+          done
+          if [ "$synced" -ne 1 ]; then
+            echo "❌ pacman -Syu failed after 5 attempts — giving up" >&2
+            exit 1
+          fi

      - name: Set up CachyOS repo
        run: |
@@ -54,7 +70,20 @@ jobs:
          printf '\n[cachyos]\nInclude = /etc/pacman.d/cachyos-mirrorlist\n' \
            >> /etc/pacman.conf

-          pacman -Syu --noconfirm --needed
+          # Retry: gateway drops UDP/53 in bursts (see run 131)
+          synced=0
+          for try in 1 2 3 4 5; do
+            if pacman -Syu --noconfirm --needed; then
+              synced=1
+              break
+            fi
+            echo "⚠️ pacman -Syu failed (attempt $try/5), retrying in $(( try * 15 ))s..." >&2
+            sleep $(( try * 15 ))
+          done
+          if [ "$synced" -ne 1 ]; then
+            echo "❌ pacman -Syu failed after 5 attempts — giving up" >&2
+            exit 1
+          fi

      - name: Set up chaotic-aur repo
        run: |
@@ -65,7 +94,20 @@ jobs:
          printf '\n[chaotic-aur]\nInclude = /etc/pacman.d/chaotic-mirrorlist\n' \
            >> /etc/pacman.conf

-          pacman -Syu --noconfirm --needed
+          # Retry: gateway drops UDP/53 in bursts (see run 131)
+          synced=0
+          for try in 1 2 3 4 5; do
+            if pacman -Syu --noconfirm --needed; then
+              synced=1
+              break
+            fi
+            echo "⚠️ pacman -Syu failed (attempt $try/5), retrying in $(( try * 15 ))s..." >&2
+            sleep $(( try * 15 ))
+          done
+          if [ "$synced" -ne 1 ]; then
+            echo "❌ pacman -Syu failed after 5 attempts — giving up" >&2
+            exit 1
+          fi

      - name: Lint shell scripts with ShellCheck
        run: |