chromium-setup/sync.sh

#!/usr/bin/env bash
# Sync phase: clone/update depot_tools, clone the chromium-fork checkout,
# gclient sync. Run via `hetzner-runner exec -script sync.sh` against a box
# already provisioned by a separate `up` step -- see ../../.gitea/workflows
# in chromium-fork for the phase breakdown this belongs to.
set -euo pipefail

: "${GITEA_TOKEN:?GITEA_TOKEN must be set}"
: "${GITEA_REPO:=Cold-Air-Networks/chromium-fork}"
: "${GITEA_REF:=main}"
: "${MIRROR_READ_TOKEN:=$GITEA_TOKEN}"
: "${CHROMIUM_ROOT:=/root/chromium-ci}"

retry() {
  local desc="$1"; shift
  local attempt=1
  local max=5
  until "$@"; do
    if [ "$attempt" -ge "$max" ]; then
      echo "$desc failed after $attempt attempts, giving up"
      return 1
    fi
    echo "$desc failed (attempt $attempt/$max); retrying in 5s"
    attempt=$((attempt + 1))
    sleep 5
  done
}

redact() {
  sed -E 's#x-access-token:[^@]*@#x-access-token:REDACTED@#g'
}

echo "===== base packages (file, strace) ====="
# `file` is a hard requirement: install-build-deps.py's dev_list() shells
# out to it (via `file -L <path>`) to detect ELF architecture, and this
# minimal Ubuntu cloud image doesn't ship it by default (confirmed: ci run
# #61, 2026-08-05, FileNotFoundError: 'file'). strace is a diagnostic
# nicety for stall-watch.py, not a build requirement -- best-effort only.
if ! command -v file >/dev/null 2>&1 || ! command -v strace >/dev/null 2>&1; then
  apt-get update -qq
fi
command -v file >/dev/null 2>&1 || apt-get install -y file
command -v strace >/dev/null 2>&1 || apt-get install -y strace \
  || echo "strace install failed, continuing without it"

echo "===== depot_tools ====="
if [ ! -d "$HOME/depot_tools" ]; then
  retry "clone depot_tools" git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git "$HOME/depot_tools"
fi
export PATH="$HOME/depot_tools:$PATH"

# Explicit bootstrap before anything else touches depot_tools' `python3`
# wrapper. That wrapper hard-requires python3_bin_reldir.txt (see
# depot_tools/python3), which is normally created by `update_depot_tools`
# -- but update_depot_tools itself refuses to do anything as root
# ("Running depot tools as root is sad", see depot_tools/update_depot_tools)
# and we're root on this box. gclient sync still works without this
# (confirmed: ci run #54, 2026-08-04, sync completed fully) because it goes
# through vpython3, a separately-bootstrapped wrapper -- but any later
# direct python3 invocation, e.g. install-build-deps.py's shebang, fails
# with exactly this error otherwise. cipd_bin_setup and bootstrap_python3
# are the two functions update_depot_tools calls after its root check;
# calling them straight sidesteps that check entirely.
#
# set +u around this block: these are third-party scripts (depot_tools'
# own cipd/cipd_bin_setup.sh), not written to be safe under nounset --
# confirmed cipd itself references an unset var with no default and dies
# under set -u. They're sourced (not exec'd), so they inherit our shell's
# flags; restore -u immediately after.
set +u
source "$HOME/depot_tools/cipd_bin_setup.sh"
cipd_bin_setup
source "$HOME/depot_tools/bootstrap_python3"
bootstrap_python3
set -u

mkdir -p "$CHROMIUM_ROOT"
cd "$CHROMIUM_ROOT"

REPO_URL="https://x-access-token:${GITEA_TOKEN}@forge.quinefoundation.com/${GITEA_REPO}.git"

# VK-GL-CTS against chromium.googlesource.com directly was the single
# biggest source of sync flakiness in chromium-fork's original CI workflow
# (corrupted packs, silent stalls). Redirect to the org's own mirror.
VK_GL_CTS_UPSTREAM="https://chromium.googlesource.com/external/github.com/KhronosGroup/VK-GL-CTS"
git config --global \
  url."https://x-access-token:${MIRROR_READ_TOKEN}@forge.quinefoundation.com/Cold-Air-Networks/mirror-vk-gl-cts.git".insteadOf \
  "$VK_GL_CTS_UPSTREAM"

echo "===== registered url.insteadOf mappings ====="
git config --global --get-regexp '^url\.' | redact
echo "===== end registered url.insteadOf mappings ====="

cat > .gclient <<EOF
solutions = [
  {
    "name": "src",
    "url": "$REPO_URL",
    "custom_deps": {},
    "custom_vars": {},
  },
]
EOF

echo "===== fetching $GITEA_REPO@$GITEA_REF ====="
if [ ! -d src/.git ]; then
  retry "clone src" git clone "$REPO_URL" src
fi
cd src
git remote set-url origin "$REPO_URL"
retry "fetch $GITEA_REF" git fetch origin "$GITEA_REF"
git checkout --force FETCH_HEAD

# --shallow is deliberately absent: chromium-fork's original CI workflow
# traced every VK-GL-CTS stall to --shallow itself (git_cache.py turns it
# into --depth 10000, a much harder reachability problem across VK-GL-CTS's
# 150+ tags/branches than an unlimited-depth fetch of the same refs) --
# removing it is what got that workflow's sync to actually succeed.
echo "===== gclient sync ====="
# Wrapped in stall-watch.py: ci run #56 (2026-08-04) stalled here for ~3h
# with zero output (last thing seen: devtools-frontend's
# sync_rollup_libs.py hook) and left no diagnostic trail. 5 minutes of true
# silence is well outside anything a legitimate hook/clone/cipd-deploy
# should ever go without printing *something* -- if it happens again this
# dumps the process tree, network state, and a live strace sample of
# whatever's actually stuck, before killing it.
#
# ci run #57 (2026-08-05) then spent 90+ minutes cycling through this loop
# with no clear signal from the outside about which attempt it was on or
# whether stall-watch had even fired -- Gitea's log capture visibly lags
# real execution by a lot on long steps (confirmed on both runs), so
# "no new log lines" doesn't mean "no progress". Two fixes for that,
# neither of which depends on the log catching up in real time:
#   - explicit timestamped start/end/duration per attempt, so whatever log
#     *does* eventually land is unambiguous about where the time went.
#   - a hard wall-clock budget across the whole retry loop (not just per
#     attempt) -- run #57 never got a chance to hit this because it was
#     killed manually; now the loop itself will give up and report
#     instead of running indefinitely regardless of individual attempts
#     each staying under the stall threshold.
SYNC_LOOP_START=$(date +%s)
SYNC_LOOP_BUDGET=2400  # 40 minutes total across all attempts
ATTEMPT=1
MAX_ATTEMPTS=5
while true; do
  ATTEMPT_START=$(date +%s)
  echo "----- gclient sync attempt $ATTEMPT/$MAX_ATTEMPTS starting at $(date -u +%FT%TZ) -----"
  if python3 "$HOME/stall-watch.py" --stall-timeout 300 -- \
      gclient sync --with_branch_heads --with_tags -j"$(nproc)" --verbose --force 2>&1 | redact; then
    ATTEMPT_ELAPSED=$(( $(date +%s) - ATTEMPT_START ))
    echo "----- gclient sync attempt $ATTEMPT/$MAX_ATTEMPTS succeeded after ${ATTEMPT_ELAPSED}s -----"
    break
  fi
  ATTEMPT_ELAPSED=$(( $(date +%s) - ATTEMPT_START ))
  LOOP_ELAPSED=$(( $(date +%s) - SYNC_LOOP_START ))
  echo "----- gclient sync attempt $ATTEMPT/$MAX_ATTEMPTS failed after ${ATTEMPT_ELAPSED}s (${LOOP_ELAPSED}s total so far) -----"
  if [ "$ATTEMPT" -ge "$MAX_ATTEMPTS" ]; then
    echo "gclient sync failed after $ATTEMPT attempts, giving up"
    exit 1
  fi
  if [ "$LOOP_ELAPSED" -ge "$SYNC_LOOP_BUDGET" ]; then
    echo "gclient sync retry loop exceeded its ${SYNC_LOOP_BUDGET}s wall-clock budget after attempt $ATTEMPT, giving up rather than continuing indefinitely"
    exit 1
  fi
  echo "retrying ($((ATTEMPT + 1))/$MAX_ATTEMPTS)"
  ATTEMPT=$((ATTEMPT + 1))
done