From 0fbec83573fdbabb5c363a2647017cfbfdf1567d Mon Sep 17 00:00:00 2001 From: alvinlollo Date: Wed, 7 Oct 2026 11:14:28 +1100 Subject: [PATCH] ci: retry pacman syncs and reset host pacman.conf before build Run 131 failed at 'pacman -Syu archiso jq' because the gateway (192.168.1.1) intermittently drops UDP/53: cachyos.db and chaotic-aur.db lookups hit libalpm's 10s resolve timeout while core/extra survived on resolved's cache. - build-iso: reset the host /etc/pacman.conf to stock core/extra first (tests.yaml leaves [cachyos]/[chaotic-aur] behind, and mkarchiso reads the profile's pacman.conf anyway), pre-warm mirror hostnames in the resolver cache, then retry pacman -Syu 5x with backoff. - tests: wrap all three pacman -Syu calls in the same retry loop. DNS-side redundancy was added on the runner separately (networkd drop-in adds 1.1.1.1/8.8.8.8 next to the gateway). --- .gitea/workflows/build-iso.yaml | 57 ++++++++++++++++++++++++++++++++- .gitea/workflows/tests.yaml | 48 +++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/build-iso.yaml b/.gitea/workflows/build-iso.yaml index fb9c592..39269da 100644 --- a/.gitea/workflows/build-iso.yaml +++ b/.gitea/workflows/build-iso.yaml @@ -41,7 +41,62 @@ jobs: - name: Install archiso and dependencies run: | set -euo pipefail - pacman -Syu --noconfirm --needed archiso jq + # The host's /etc/pacman.conf is shared, mutable state: the tests + # workflow appends [cachyos] and [chaotic-aur] to it, so without this + # reset the build would sync two repos the host never needs (it only + # installs archiso/jq from extra — mkarchiso reads the *profile's* + # pacman.conf). Every extra repo is another DNS lookup, and the + # gateway intermittently drops UDP/53 (that is what failed run 131 + # at this very step). Reset to stock core/extra. + cat > /etc/pacman.conf << 'STOCKCONF' + [options] + DisableSandbox + HoldPkg = pacman glibc + Architecture = auto + CheckSpace + ParallelDownloads = 5 + SigLevel = Required DatabaseOptional + LocalFileSigLevel = Optional + + [core] + Include = /etc/pacman.d/mirrorlist + + [extra] + Include = /etc/pacman.d/mirrorlist + STOCKCONF + sed -i 's/^ //' /etc/pacman.conf + # Drop sync databases of the custom repos so they cannot linger + rm -f /var/lib/pacman/sync/cachyos* /var/lib/pacman/sync/chaotic* + + # Warm the resolver cache for the mirrors we are about to query. + # systemd-resolved serves later lookups from cache, so a gateway + # UDP/53 burst cannot hit them again. Failures here are not fatal — + # the retry loop below is the real safety net. + for host in mirror.aarnet.edu.au gsl-syd.mm.fcix.net au.arch.niranjan.co; do + for try in 1 2 3 4 5; do + if getent hosts "$host" >/dev/null 2>&1; then + break + fi + echo " ⏳ DNS for $host failed (try $try/5), retrying in $(( try * 3 ))s..." + sleep $(( try * 3 )) + done + done + + # The gateway drops DNS queries in bursts; one burst must not fail + # the whole job, so retry with backoff before giving up. + synced=0 + for try in 1 2 3 4 5; do + if pacman -Syu --noconfirm --needed archiso jq; then + synced=1 + break + fi + echo " ⚠️ pacman -Syu failed (attempt $try/5), retrying in $(( try * 15 ))s..." >&2 + sleep $(( try * 15 )) + done + if [ "$synced" -ne 1 ]; then + echo "❌ pacman -Syu failed after 5 attempts — giving up" >&2 + exit 1 + fi - name: Ensure /dev/fd symlinks exist for mkinitcpio and procsub run: | diff --git a/.gitea/workflows/tests.yaml b/.gitea/workflows/tests.yaml index 8c7db95..04d1d86 100644 --- a/.gitea/workflows/tests.yaml +++ b/.gitea/workflows/tests.yaml @@ -36,7 +36,23 @@ jobs: rm -f /var/lib/pacman/sync/cachyos* /var/lib/pacman/sync/chaotic* - name: Install prerequisites - run: pacman -Syu --noconfirm --needed shellcheck yamllint + run: | + # The gateway intermittently drops UDP/53 (it failed build run 131 + # at the equivalent step); retry with backoff instead of dying on + # the first bad DNS burst. + synced=0 + for try in 1 2 3 4 5; do + if pacman -Syu --noconfirm --needed shellcheck yamllint; then + synced=1 + break + fi + echo "⚠️ pacman -Syu failed (attempt $try/5), retrying in $(( try * 15 ))s..." >&2 + sleep $(( try * 15 )) + done + if [ "$synced" -ne 1 ]; then + echo "❌ pacman -Syu failed after 5 attempts — giving up" >&2 + exit 1 + fi - name: Set up CachyOS repo run: | @@ -54,7 +70,20 @@ jobs: printf '\n[cachyos]\nInclude = /etc/pacman.d/cachyos-mirrorlist\n' \ >> /etc/pacman.conf - pacman -Syu --noconfirm --needed + # Retry: gateway drops UDP/53 in bursts (see run 131) + synced=0 + for try in 1 2 3 4 5; do + if pacman -Syu --noconfirm --needed; then + synced=1 + break + fi + echo "⚠️ pacman -Syu failed (attempt $try/5), retrying in $(( try * 15 ))s..." >&2 + sleep $(( try * 15 )) + done + if [ "$synced" -ne 1 ]; then + echo "❌ pacman -Syu failed after 5 attempts — giving up" >&2 + exit 1 + fi - name: Set up chaotic-aur repo run: | @@ -65,7 +94,20 @@ jobs: printf '\n[chaotic-aur]\nInclude = /etc/pacman.d/chaotic-mirrorlist\n' \ >> /etc/pacman.conf - pacman -Syu --noconfirm --needed + # Retry: gateway drops UDP/53 in bursts (see run 131) + synced=0 + for try in 1 2 3 4 5; do + if pacman -Syu --noconfirm --needed; then + synced=1 + break + fi + echo "⚠️ pacman -Syu failed (attempt $try/5), retrying in $(( try * 15 ))s..." >&2 + sleep $(( try * 15 )) + done + if [ "$synced" -ne 1 ]; then + echo "❌ pacman -Syu failed after 5 attempts — giving up" >&2 + exit 1 + fi - name: Lint shell scripts with ShellCheck run: |