fix(ci): be smarter about DNF package caching (#1952)

Currently we are very inefficient with our cache usage as every image
has it's own cache where most of the packages are the same across
variants anyway. See 973333f.

This should fix a couple problems around this as we are now also
considering the Fedora version in the cache key and only allowing one
blessed image to create cache. Which is why I decided to implement this
into the Justfile even tho this is only needed when running from github
as it just makes things easier to troubleshoot.

aurora-dx makes the most sense here as that is the heaviest image. We
don't need a separate nvidia image cache as the biggest RPMs exclusive
to that variant are cached in akmods anyway and don't have to be
downloaded.

Only allowing cache to be created at all on workflow_dispatch and on
schedule happens to go hand in hand with the limitation/design of the
cache action as we only build images from the main branch in those two
cases, rest is merge queue, which also happens to address the
"infighting" across stable and latest builds when they kick off at the
same time.

This should make it easier going forward especially when we take future
possible ARM, testing images or any other reason for our matrix to get
bigger into consideration.

.github/workflows/reusable-build.yml

@@ -90,15 +90,30 @@ jobs:
           echo "Default Tag: ${DEFAULT_TAG}"
           echo "DEFAULT_TAG=${DEFAULT_TAG}" >> $GITHUB_ENV
 
-      - name: Setup DNF package cache
-        uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+      - name: DNF Package Cache Setup
+        shell: bash
+        id: setup-cache
         env:
-          IMAGE_NAME: ${{ env.IMAGE_NAME }}
+          MATRIX_BASE_NAME: ${{ matrix.base_name }}
+          GITHUB_EVENT_NAME: ${{ github.event_name }}
+        run: |
+          CACHE="$(just setup-cache "${MATRIX_BASE_NAME}" \
+                            "1" \
+                            "${GITHUB_EVENT_NAME}")"
+
+          CACHE_NAME="$(echo "${CACHE}" | cut -d' ' -f 1)"
+          ALLOW_CACHE_WRITE="$(echo "${CACHE}" | cut -d' ' -f 2)"
+
+          echo "cache_name=${CACHE_NAME}" >> "$GITHUB_OUTPUT"
+          echo "allow_cache_write=${ALLOW_CACHE_WRITE}" >> "$GITHUB_OUTPUT"
+
+      - name: Restore DNF package cache
+        uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+        env:
+          CACHE_NAME: ${{ steps.setup-cache.outputs.cache_name }}
         with:
           path: /var/tmp/buildah-cache-*
-          key: ${{ runner.os }}-buildah-${{ env.IMAGE_NAME }}
-          restore-keys: |
-            ${{ runner.os }}-buildah-${{ env.IMAGE_NAME }}
+          key: ${{ runner.os }}-${{ runner.arch }}-buildah-${{ env.CACHE_NAME }}
 
       - name: Build Image
         id: build-image
@@ -116,6 +131,22 @@ jobs:
                                "${MATRIX_IMAGE_FLAVOR}" \
                                "${INPUTS_KERNEL_PIN}"
 
+      # https://github.com/actions/cache/issues/1533
+      - name: Hack around permission issue caching
+        shell: bash
+        id: cache-perms
+        run: |
+          sudo chmod 777 --recursive /var/tmp/buildah-cache-0
+
+      - name: Write new DNF package cache
+        if: steps.setup-cache.outputs.allow_cache_write == 'true'
+        uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
+        env:
+          CACHE_NAME: ${{ steps.setup-cache.outputs.cache_name }}
+        with:
+          path: /var/tmp/buildah-cache-*
+          key: ${{ runner.os }}-${{ runner.arch }}-buildah-${{ env.CACHE_NAME }}
+
       - name: Setup Syft
         id: setup-syft
         if: github.event_name != 'pull_request'
@@ -338,12 +369,6 @@ jobs:
           subject-digest: ${{ steps.push.outputs.outputs && fromJSON(steps.push.outputs.outputs).digest }}
           push-to-registry: true
 
-      # https://github.com/actions/cache/issues/1533
-      - name: Hack around permission issue caching
-        id: cache-perms
-        run: |
-          sudo chmod 777 --recursive /var/tmp/buildah-cache-0
-
   check:
     name: Check all ${{ matrix.stream_name }} builds successful
     if: always()

Justfile

@@ -706,6 +706,27 @@ tag-images image_name="" default_tag="" tags="":
     # Show Images
     ${PODMAN} images
 
+# DNF CI package cache
+[group('Utility')]
+setup-cache $image="aurora" $ghcr="0" $github_event="0":
+    #!/usr/bin/bash
+    set -eou pipefail
+
+    image_name=$({{ just }} image_name '{{ image }}')
+    fedora_version=$({{ just }} fedora_version '{{ image }}')
+
+    ALLOW_CACHE_WRITE="false"
+
+    if [[ "${image_name}" == "aurora-dx" ]] && \
+       [[ "{{ ghcr }}" == "1" ]] && \
+       [[ "${github_event}" == "workflow_dispatch" || "${github_event}" == "schedule" ]]; then
+        ALLOW_CACHE_WRITE="true"
+    fi
+
+    CACHE_NAME="${image_name}"-"${fedora_version}"
+
+    echo "${CACHE_NAME}" "${ALLOW_CACHE_WRITE}"
+
 # # Examples:
 #   > just retag-nvidia-on-ghcr stable-daily stable-daily-41.20250126.3 0
 #   > just retag-nvidia-on-ghcr latest latest-41.20250228.1 0