50ba0fcd2d
Currently we are very inefficient with our cache usage as every image
has it's own cache where most of the packages are the same across
variants anyway. See 973333f.
This should fix a couple problems around this as we are now also
considering the Fedora version in the cache key and only allowing one
blessed image to create cache. Which is why I decided to implement this
into the Justfile even tho this is only needed when running from github
as it just makes things easier to troubleshoot.
aurora-dx makes the most sense here as that is the heaviest image. We
don't need a separate nvidia image cache as the biggest RPMs exclusive
to that variant are cached in akmods anyway and don't have to be
downloaded.
Only allowing cache to be created at all on workflow_dispatch and on
schedule happens to go hand in hand with the limitation/design of the
cache action as we only build images from the main branch in those two
cases, rest is merge queue, which also happens to address the
"infighting" across stable and latest builds when they kick off at the
same time.
This should make it easier going forward especially when we take future
possible ARM, testing images or any other reason for our matrix to get
bigger into consideration.
391 lines
14 KiB
YAML
391 lines
14 KiB
YAML
name: Reusable Build and Push
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
image_flavors:
|
|
description: "JSON string of flavors to build, '[main, nvidia-open]'"
|
|
default: "['main','nvidia-open']"
|
|
type: string
|
|
brand_name:
|
|
description: "The Brand Name: aurora"
|
|
required: true
|
|
default: aurora
|
|
type: string
|
|
stream_name:
|
|
description: "The Fedora Version: stable, or latest"
|
|
required: true
|
|
type: string
|
|
kernel_pin:
|
|
description: "The full kernel version to pin"
|
|
type: string
|
|
|
|
env:
|
|
IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }}
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}-${{ inputs.brand_name}}-${{ inputs.stream_name }}
|
|
cancel-in-progress: true
|
|
|
|
permissions: {}
|
|
|
|
jobs:
|
|
build_container:
|
|
name: image
|
|
runs-on: ubuntu-24.04
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
id-token: write
|
|
attestations: write
|
|
artifact-metadata: write
|
|
|
|
continue-on-error: false
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
image_flavor: ${{ fromJson(inputs.image_flavors) }}
|
|
base_name: ["${{ inputs.brand_name }}", "${{ inputs.brand_name }}-dx"]
|
|
stream_name: ["${{ inputs.stream_name }}"]
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
|
|
# mount /mnt as /var/lib/containers
|
|
- name: Mount BTRFS for podman storage
|
|
id: container-storage-action
|
|
uses: ublue-os/container-storage-action@dc1f4c8f17b672069e921f001132f7cf98a423a6
|
|
continue-on-error: true
|
|
with:
|
|
target-dir: /var/lib/containers
|
|
mount-opts: compress-force=zstd:2
|
|
loopback-free: '1'
|
|
|
|
- name: Setup Just
|
|
uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3
|
|
|
|
- name: Check Just Syntax
|
|
shell: bash
|
|
run: |
|
|
just check
|
|
|
|
- name: Image Name
|
|
shell: bash
|
|
env:
|
|
MATRIX_BASE_NAME: ${{ matrix.base_name }}
|
|
MATRIX_STREAM_NAME: ${{ matrix.stream_name }}
|
|
MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
|
|
run: |
|
|
IMAGE_NAME="$(just image_name ${MATRIX_BASE_NAME} ${MATRIX_STREAM_NAME} ${MATRIX_IMAGE_FLAVOR})"
|
|
echo "IMAGE_NAME=${IMAGE_NAME}" >> $GITHUB_ENV
|
|
|
|
- name: Default Tag
|
|
shell: bash
|
|
env:
|
|
MATRIX_STREAM_NAME: ${{ matrix.stream_name }}
|
|
run: |
|
|
DEFAULT_TAG="$(just generate-default-tag \
|
|
"${MATRIX_STREAM_NAME}" \
|
|
"1")"
|
|
echo "Default Tag: ${DEFAULT_TAG}"
|
|
echo "DEFAULT_TAG=${DEFAULT_TAG}" >> $GITHUB_ENV
|
|
|
|
- name: DNF Package Cache Setup
|
|
shell: bash
|
|
id: setup-cache
|
|
env:
|
|
MATRIX_BASE_NAME: ${{ matrix.base_name }}
|
|
GITHUB_EVENT_NAME: ${{ github.event_name }}
|
|
run: |
|
|
CACHE="$(just setup-cache "${MATRIX_BASE_NAME}" \
|
|
"1" \
|
|
"${GITHUB_EVENT_NAME}")"
|
|
|
|
CACHE_NAME="$(echo "${CACHE}" | cut -d' ' -f 1)"
|
|
ALLOW_CACHE_WRITE="$(echo "${CACHE}" | cut -d' ' -f 2)"
|
|
|
|
echo "cache_name=${CACHE_NAME}" >> "$GITHUB_OUTPUT"
|
|
echo "allow_cache_write=${ALLOW_CACHE_WRITE}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Restore DNF package cache
|
|
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
|
|
env:
|
|
CACHE_NAME: ${{ steps.setup-cache.outputs.cache_name }}
|
|
with:
|
|
path: /var/tmp/buildah-cache-*
|
|
key: ${{ runner.os }}-${{ runner.arch }}-buildah-${{ env.CACHE_NAME }}
|
|
|
|
- name: Build Image
|
|
id: build-image
|
|
shell: bash
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
MATRIX_BASE_NAME: ${{ matrix.base_name }}
|
|
MATRIX_STREAM_NAME: ${{ matrix.stream_name }}
|
|
MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
|
|
INPUTS_KERNEL_PIN: ${{ inputs.kernel_pin }}
|
|
run: |
|
|
sudo -E $(command -v just) repo_organization="${{ github.repository_owner }}" \
|
|
build-ghcr "${MATRIX_BASE_NAME}" \
|
|
"${MATRIX_STREAM_NAME}" \
|
|
"${MATRIX_IMAGE_FLAVOR}" \
|
|
"${INPUTS_KERNEL_PIN}"
|
|
|
|
# https://github.com/actions/cache/issues/1533
|
|
- name: Hack around permission issue caching
|
|
shell: bash
|
|
id: cache-perms
|
|
run: |
|
|
sudo chmod 777 --recursive /var/tmp/buildah-cache-0
|
|
|
|
- name: Write new DNF package cache
|
|
if: steps.setup-cache.outputs.allow_cache_write == 'true'
|
|
uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
|
|
env:
|
|
CACHE_NAME: ${{ steps.setup-cache.outputs.cache_name }}
|
|
with:
|
|
path: /var/tmp/buildah-cache-*
|
|
key: ${{ runner.os }}-${{ runner.arch }}-buildah-${{ env.CACHE_NAME }}
|
|
|
|
- name: Setup Syft
|
|
id: setup-syft
|
|
if: github.event_name != 'pull_request'
|
|
uses: anchore/sbom-action/download-syft@e22c389904149dbc22b58101806040fa8d37a610 # v0
|
|
with:
|
|
syft-version: v1.39.0
|
|
|
|
- name: Generate SBOM
|
|
if: github.event_name != 'pull_request'
|
|
id: generate-sbom
|
|
env:
|
|
IMAGE: ${{ env.IMAGE_NAME }}
|
|
STREAM_NAME: ${{ matrix.stream_name }}
|
|
SYFT_CMD: ${{ steps.setup-syft.outputs.cmd }}
|
|
OCI_DIR: "/tmp/image-oci-dir"
|
|
run: |
|
|
mkdir -p ${OCI_DIR}/rootfs
|
|
sudo podman container create --replace --name "${IMAGE}" "localhost/${IMAGE}:${STREAM_NAME}"
|
|
sudo podman export "${IMAGE}" | sudo tar -C ${OCI_DIR}/rootfs -xf -
|
|
sudo podman container rm "${IMAGE}"
|
|
|
|
SBOM="$(mktemp -d)/sbom.json"
|
|
export SYFT_PARALLELISM=$(($(nproc)*2))
|
|
sudo $SYFT_CMD --source-name "${IMAGE}"-"${STREAM_NAME}" ${OCI_DIR} -o syft-json=${SBOM}
|
|
du -sh ${SBOM}
|
|
echo "SBOM=${SBOM}" >> $GITHUB_OUTPUT
|
|
sudo rm -rf ${OCI_DIR}
|
|
|
|
- name: Rechunk Image
|
|
id: rechunk-image
|
|
shell: bash
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
MATRIX_BASE_NAME: ${{ matrix.base_name }}
|
|
MATRIX_STREAM_NAME: ${{ matrix.stream_name }}
|
|
MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
|
|
run: |
|
|
sudo -E $(command -v just) rechunk "${MATRIX_BASE_NAME}" \
|
|
"${MATRIX_STREAM_NAME}" \
|
|
"${MATRIX_IMAGE_FLAVOR}" \
|
|
"1"
|
|
|
|
- name: Load Image into Podman
|
|
id: load-rechunk
|
|
shell: bash
|
|
env:
|
|
MATRIX_BASE_NAME: ${{ matrix.base_name }}
|
|
DEFAULT_TAG: ${{ env.DEFAULT_TAG }}
|
|
MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
|
|
run: |
|
|
sudo -E $(command -v just) load-rechunk "${MATRIX_BASE_NAME}" \
|
|
"${DEFAULT_TAG}" \
|
|
"${MATRIX_IMAGE_FLAVOR}"
|
|
|
|
- name: Secureboot Check
|
|
id: secureboot
|
|
shell: bash
|
|
env:
|
|
MATRIX_BASE_NAME: ${{ matrix.base_name }}
|
|
DEFAULT_TAG: ${{ env.DEFAULT_TAG }}
|
|
MATRIX_IMAGE_FLAVOR: ${{ matrix.image_flavor }}
|
|
run: |
|
|
sudo -E $(command -v just) secureboot "${MATRIX_BASE_NAME}" \
|
|
"${DEFAULT_TAG}" \
|
|
"${MATRIX_IMAGE_FLAVOR}"
|
|
|
|
- name: Generate tags
|
|
id: generate-tags
|
|
shell: bash
|
|
env:
|
|
MATRIX_BASE_NAME: "${{ matrix.base_name }}"
|
|
MATRIX_STREAM_NAME: "${{ matrix.stream_name }}"
|
|
MATRIX_IMAGE_FLAVOR: "${{ matrix.image_flavor }}"
|
|
INPUTS_KERNEL_PIN: "${{ inputs.kernel_pin }}"
|
|
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
|
DEFAULT_TAG: ${{ env.DEFAULT_TAG }}
|
|
GITHUB_EVENT_NAME: ${{ github.event_name }}
|
|
GITHUB_EVENT_NUMBER: ${{ github.event.number }}
|
|
run: |
|
|
alias_tags="$(just generate-build-tags \
|
|
"${MATRIX_BASE_NAME}" \
|
|
"${MATRIX_STREAM_NAME}" \
|
|
"${MATRIX_IMAGE_FLAVOR}" \
|
|
"${INPUTS_KERNEL_PIN}" \
|
|
"1" \
|
|
"$(podman inspect ${IMAGE_NAME}:${DEFAULT_TAG} | jq -r '.[].Config.Labels["org.opencontainers.image.version"]')" \
|
|
"${GITHUB_EVENT_NAME}" \
|
|
"${GITHUB_EVENT_NUMBER}")"
|
|
|
|
echo "Tags for this Action..."
|
|
echo "$alias_tags"
|
|
echo "alias_tags=${alias_tags}" >> $GITHUB_OUTPUT
|
|
|
|
# Tag Images
|
|
- name: Tag Images
|
|
shell: bash
|
|
env:
|
|
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
|
DEFAULT_TAG: ${{ env.DEFAULT_TAG }}
|
|
ALIAS_TAGS: ${{ steps.generate-tags.outputs.alias_tags }}
|
|
run: |
|
|
set -eoux pipefail
|
|
sudo -E $(command -v just) tag-images "${IMAGE_NAME}" \
|
|
"${DEFAULT_TAG}" \
|
|
"${ALIAS_TAGS}"
|
|
|
|
# Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR.
|
|
# https://github.com/macbre/push-to-ghcr/issues/12
|
|
- name: Lowercase Registry
|
|
id: registry_case
|
|
env:
|
|
IMAGE_REGISTRY: ${{ env.IMAGE_REGISTRY }}
|
|
run: |
|
|
echo "lowercase=${IMAGE_REGISTRY,,}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Login to GitHub Container Registry
|
|
if: github.event_name != 'pull_request'
|
|
run: |
|
|
echo ${{ secrets.GITHUB_TOKEN }} | podman login ghcr.io -u ${{ github.actor }} --password-stdin
|
|
echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin
|
|
|
|
- name: Push to GHCR
|
|
id: push
|
|
if: github.event_name != 'pull_request'
|
|
uses: Wandalen/wretry.action@e68c23e6309f2871ca8ae4763e7629b9c258e1ea # v3.8.0
|
|
env:
|
|
ALIAS_TAGS: ${{ steps.generate-tags.outputs.alias_tags }}
|
|
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
|
LOWERCASE: ${{ steps.registry_case.outputs.lowercase }}
|
|
DEFAULT_TAG: ${{ env.DEFAULT_TAG }}
|
|
with:
|
|
attempt_limit: 3
|
|
attempt_delay: 15000
|
|
command: |
|
|
set -euox pipefail
|
|
|
|
for tag in ${ALIAS_TAGS}; do
|
|
sudo -E podman push ${IMAGE_NAME}:${tag} ${LOWERCASE}/${IMAGE_NAME}:${tag}
|
|
done
|
|
|
|
digest=$(skopeo inspect docker://${LOWERCASE}/${IMAGE_NAME}:${DEFAULT_TAG} --format '{{.Digest}}')
|
|
|
|
echo "digest=${digest}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Upload OCI dir as Artifact
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
|
with:
|
|
name: ${{ env.IMAGE_NAME }}.oci
|
|
path: ${{ env.IMAGE_NAME }}_build
|
|
if-no-files-found: error
|
|
retention-days: 1
|
|
|
|
- name: PR Testing Instructions
|
|
if: github.event_name == 'pull_request'
|
|
run: |
|
|
echo "Download an image, run unzip aurora.oci.zip -d aurora" >> $GITHUB_STEP_SUMMARY
|
|
echo "Rebase to them: sudo bootc switch --transport oci /absolute/path/to/dir/aurora" >> $GITHUB_STEP_SUMMARY
|
|
echo "Go back: sudo bootc switch --enforce-container-sigpolicy ghcr.io/ublue-os/aurora" >> $GITHUB_STEP_SUMMARY
|
|
|
|
- name: Install Cosign
|
|
uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1
|
|
if: github.event_name != 'pull_request'
|
|
|
|
- name: Sign container image
|
|
if: github.event_name != 'pull_request'
|
|
run: |
|
|
cosign sign -y --key env://COSIGN_PRIVATE_KEY ${LOWERCASE}/${IMAGE_NAME}@${TAGS}
|
|
env:
|
|
TAGS: ${{ steps.push.outputs.outputs && fromJSON(steps.push.outputs.outputs).digest }}
|
|
COSIGN_EXPERIMENTAL: false
|
|
COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }}
|
|
LOWERCASE: ${{ steps.registry_case.outputs.lowercase }}
|
|
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
|
|
|
- name: Install ORAS
|
|
if: github.event_name != 'pull_request'
|
|
uses: oras-project/setup-oras@22ce207df3b08e061f537244349aac6ae1d214f6 # v1.2.4
|
|
|
|
- name: Login to GitHub Container Registry with ORAS
|
|
if: github.event_name != 'pull_request'
|
|
run: |
|
|
echo ${{ secrets.GITHUB_TOKEN }} | oras login ghcr.io -u ${{ github.actor }} --password-stdin
|
|
|
|
- name: Upload SBOM
|
|
if: github.event_name != 'pull_request'
|
|
id: upload-sbom
|
|
env:
|
|
IMAGE: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}
|
|
DIGEST: ${{ steps.push.outputs.outputs && fromJSON(steps.push.outputs.outputs).digest }}
|
|
SBOM: ${{ steps.generate-sbom.outputs.SBOM }}
|
|
run: |
|
|
cd "$(dirname "${SBOM}")"
|
|
oras attach \
|
|
--artifact-type application/vnd.spdx+json \
|
|
--annotation filename=$(basename "$SBOM") \
|
|
"${IMAGE}@${DIGEST}" \
|
|
"$(basename ${SBOM})"
|
|
|
|
sbom_digest=$(oras discover --format json "${IMAGE}@${DIGEST}" | jq -r '.referrers[] | select(.artifactType == "application/vnd.spdx+json") | .digest')
|
|
|
|
echo "sbom_digest=${sbom_digest}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Sign SBOM OCI Artifact
|
|
if: github.event_name != 'pull_request'
|
|
env:
|
|
COSIGN_EXPERIMENTAL: false
|
|
COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }}
|
|
LOWERCASE: ${{ steps.registry_case.outputs.lowercase }}
|
|
IMAGE_NAME: ${{ env.IMAGE_NAME }}
|
|
SBOM_DIGEST: ${{ steps.upload-sbom.outputs.sbom_digest }}
|
|
run: |
|
|
cosign sign -y --key env://COSIGN_PRIVATE_KEY ${LOWERCASE}/${IMAGE_NAME}@${SBOM_DIGEST}
|
|
|
|
- name: Attestation
|
|
if: github.event_name != 'pull_request'
|
|
uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 #v4.1.0
|
|
with:
|
|
subject-name: ${{ env.IMAGE_REGISTRY }}/${{ env.IMAGE_NAME }}
|
|
subject-digest: ${{ steps.push.outputs.outputs && fromJSON(steps.push.outputs.outputs).digest }}
|
|
push-to-registry: true
|
|
|
|
check:
|
|
name: Check all ${{ matrix.stream_name }} builds successful
|
|
if: always()
|
|
runs-on: ubuntu-latest
|
|
needs: [build_container]
|
|
steps:
|
|
- name: Check Jobs
|
|
env:
|
|
JOBS: ${{ toJson(needs) }}
|
|
run: |
|
|
echo "Job status:"
|
|
echo $JOBS | jq -r 'to_entries[] | " - \(.key): \(.value.result)"'
|
|
|
|
for i in $(echo $JOBS | jq -r 'to_entries[] | .value.result'); do
|
|
if [ "$i" != "success" ] && [ "$i" != "skipped" ]; then
|
|
echo ""
|
|
echo "Status check not okay!"
|
|
exit 1
|
|
fi
|
|
done
|