feat(ci): add retries for registry logins (#2369)

The retry function is lifted straight from our base images [1].

Shold make CI flakes a little less likely [2] as seen in [3].

[1]: 8301eaad96/justfile (L33-L53)
[2]: https://github.com/ublue-os/aurora/issues/2337
[3]: https://github.com/ublue-os/aurora/actions/runs/27338439302/job/80768647685

.github/workflows/reusable-build.yml

@@ -323,8 +323,9 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_ACTOR: ${{ github.actor }}
         run: |
-          echo "${GITHUB_TOKEN}" | podman login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin
-          echo "${GITHUB_TOKEN}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin
+          just login-registry podman ghcr.io
+          # cosign gets credentials from docker
+          just login-registry docker ghcr.io
 
       # Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR.
       # https://github.com/macbre/push-to-ghcr/issues/12
@@ -396,7 +397,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_ACTOR: ${{ github.actor }}
         run: |
-          echo "${GITHUB_TOKEN}" | oras login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin
+          just login-registry oras ghcr.io
 
       - name: Upload SBOM
         if: inputs.publish

Justfile

@@ -34,6 +34,28 @@ export PODMAN := if path_exists("/usr/bin/podman") == "true" { env("PODMAN", "/u
 export PULL_POLICY := if PODMAN =~ "docker" { "missing" } else { "newer" }
 just := just_executable()
 
+# Define a retry function for use in recipes
+retry_function := '
+retry() {
+    if [[ "${#}" -lt 3 ]]; then
+        echo "retry usage: <number of tries> <time between retries> <command> ..."
+        return 1
+    fi
+    tries="${1}"
+    sleep="${2}"
+    shift 2
+    for i in $(seq 1 ${tries}); do
+        if [[ ${i} -gt 1 ]]; then
+            # echo "[+] Command failed. Waiting for ${sleep} seconds"
+            sleep ${sleep}
+        fi
+        # echo "[+] Running (try: ${i}): ${@}"
+        "${@}" && r=0 && break || r=$?
+    done
+    return $r
+}
+'
+
 [private]
 default:
     @{{ just }} --list
@@ -815,6 +837,15 @@ push-image $image="aurora" $tag="latest" $flavor="main" $ghcr="0" $image_registr
       exit 1
     fi
 
+# Login to Container Registry
+login-registry bin="" registry="":
+    #!/bin/bash
+    set -euxo pipefail
+
+    {{ retry_function }}
+
+    echo "${GITHUB_TOKEN}" | retry 5 60 "{{ bin }}" login "{{ registry }}" -u "${GITHUB_ACTOR}" --password-stdin
+
 # # Examples:
 #   > just retag-nvidia-on-ghcr stable stable-41.20250126.3 0
 #   > just retag-nvidia-on-ghcr latest latest-41.20250228.1 0