Add exec subcommand; split monolithic build.sh into per-phase scripts
exec runs a command/script against an already-provisioned server (from a prior up), without provisioning or tearing down -- lets the CI workflow split a build into separately-visible steps (Sync, gn gen, Build v8, ...) that all share one box. sync.sh also fixes a real bug found in ci run #54: chromium's own gclient hooks run update_depot_tools_toggle.py --disable partway through sync, which can win a race against depot_tools' python3 wrapper's own first-use bootstrap and leave python3_bin_reldir.txt missing for the next phase's install-build-deps.py. Forcing the bootstrap before sync starts sidesteps the race.
This commit is contained in:
Philip Peterson
parent
d40351daa3
commit
a084710b65
4 changed files with 193 additions and 165 deletions
139
build.sh
139
build.sh
|
|
@ -1,139 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Chromium build script for a fresh, single-shot Hetzner box. Piped over SSH
|
||||
# and run via `bash -s` by hetzner-runner's `run -script` — see main.go.
|
||||
#
|
||||
# This is a stripped-down cousin of ../../src/.gitea/workflows/ci.yml's
|
||||
# build-linux job: same auth pattern, same gclient-sync retry/no-shallow
|
||||
# fix (that took multiple days of the existing runner's CI history to land
|
||||
# on), but with the cross-run persistence machinery removed (runner.lock
|
||||
# forensics, install-build-deps marker gating, cancelled-run lock cleanup)
|
||||
# since this box is destroyed after one run and never sees a "previous run".
|
||||
# -j is nproc instead of the existing runner's forced -j1: that cap existed
|
||||
# only because the old runner has 3.3GB RAM and no swap; this box has real
|
||||
# headroom.
|
||||
set -euo pipefail
|
||||
|
||||
: "${GITEA_TOKEN:?GITEA_TOKEN must be set}"
|
||||
: "${GITEA_REPO:=Cold-Air-Networks/chromium-fork}"
|
||||
: "${GITEA_REF:=main}"
|
||||
: "${MIRROR_READ_TOKEN:=$GITEA_TOKEN}"
|
||||
: "${CHROMIUM_ROOT:=/root/chromium-ci}"
|
||||
: "${OUT_DIR:=out/Default}"
|
||||
: "${NINJA_JOBS:=$(nproc)}"
|
||||
|
||||
retry() {
|
||||
local desc="$1"; shift
|
||||
local attempt=1
|
||||
local max=5
|
||||
until "$@"; do
|
||||
if [ "$attempt" -ge "$max" ]; then
|
||||
echo "$desc failed after $attempt attempts, giving up"
|
||||
return 1
|
||||
fi
|
||||
echo "$desc failed (attempt $attempt/$max); retrying in 5s"
|
||||
attempt=$((attempt + 1))
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
redact() {
|
||||
sed -E 's#x-access-token:[^@]*@#x-access-token:REDACTED@#g'
|
||||
}
|
||||
|
||||
echo "===== depot_tools ====="
|
||||
if [ ! -d "$HOME/depot_tools" ]; then
|
||||
retry "clone depot_tools" git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git "$HOME/depot_tools"
|
||||
fi
|
||||
export PATH="$HOME/depot_tools:$PATH"
|
||||
|
||||
mkdir -p "$CHROMIUM_ROOT"
|
||||
cd "$CHROMIUM_ROOT"
|
||||
|
||||
REPO_URL="https://x-access-token:${GITEA_TOKEN}@forge.quinefoundation.com/${GITEA_REPO}.git"
|
||||
|
||||
# Same fix as ci.yml: VK-GL-CTS against chromium.googlesource.com directly
|
||||
# was the single biggest source of sync flakiness (corrupted packs, silent
|
||||
# stalls) during that workflow's rollout. Redirect to the org's own mirror
|
||||
# instead.
|
||||
VK_GL_CTS_UPSTREAM="https://chromium.googlesource.com/external/github.com/KhronosGroup/VK-GL-CTS"
|
||||
git config --global \
|
||||
url."https://x-access-token:${MIRROR_READ_TOKEN}@forge.quinefoundation.com/Cold-Air-Networks/mirror-vk-gl-cts.git".insteadOf \
|
||||
"$VK_GL_CTS_UPSTREAM"
|
||||
|
||||
echo "===== registered url.insteadOf mappings ====="
|
||||
git config --global --get-regexp '^url\.' | redact
|
||||
echo "===== end registered url.insteadOf mappings ====="
|
||||
|
||||
cat > .gclient <<EOF
|
||||
solutions = [
|
||||
{
|
||||
"name": "src",
|
||||
"url": "$REPO_URL",
|
||||
"custom_deps": {},
|
||||
"custom_vars": {},
|
||||
},
|
||||
]
|
||||
EOF
|
||||
|
||||
echo "===== fetching $GITEA_REPO@$GITEA_REF ====="
|
||||
if [ ! -d src/.git ]; then
|
||||
retry "clone src" git clone "$REPO_URL" src
|
||||
fi
|
||||
cd src
|
||||
git remote set-url origin "$REPO_URL"
|
||||
retry "fetch $GITEA_REF" git fetch origin "$GITEA_REF"
|
||||
git checkout --force FETCH_HEAD
|
||||
|
||||
# --shallow is deliberately absent: ci.yml run #37 (2026-08-02) traced every
|
||||
# prior VK-GL-CTS stall to --shallow itself (git_cache.py turns it into
|
||||
# --depth 10000, which is a much harder reachability problem across
|
||||
# VK-GL-CTS's 150+ tags/branches than an unlimited-depth fetch of the same
|
||||
# refs) — removing it is what got Sync source to actually succeed.
|
||||
echo "===== gclient sync ====="
|
||||
ATTEMPT=1
|
||||
MAX_ATTEMPTS=5
|
||||
while true; do
|
||||
if gclient sync --with_branch_heads --with_tags -j"$(nproc)" --verbose --force 2>&1 | redact; then
|
||||
break
|
||||
fi
|
||||
if [ "$ATTEMPT" -ge "$MAX_ATTEMPTS" ]; then
|
||||
echo "gclient sync failed after $ATTEMPT attempts, giving up"
|
||||
exit 1
|
||||
fi
|
||||
echo "attempt $ATTEMPT failed; retrying ($((ATTEMPT + 1))/$MAX_ATTEMPTS)"
|
||||
ATTEMPT=$((ATTEMPT + 1))
|
||||
done
|
||||
|
||||
echo "===== install-build-deps ====="
|
||||
# Already root on this box — no sudo/no_new_privs wrinkle to work around
|
||||
# (that was specific to however the existing runner's systemd unit is set
|
||||
# up, not something inherent to install-build-deps.py).
|
||||
retry "install-build-deps" "$CHROMIUM_ROOT/src/build/install-build-deps.py" --no-prompt
|
||||
|
||||
echo "===== gn gen ====="
|
||||
mkdir -p "$OUT_DIR"
|
||||
cat > "$OUT_DIR/args.gn" <<GNEOF
|
||||
is_debug = false
|
||||
is_component_build = true
|
||||
symbol_level = 0
|
||||
use_remoteexec = false
|
||||
GNEOF
|
||||
gn gen "$OUT_DIR"
|
||||
|
||||
echo "===== build memory watchdog (background, for diagnosing OOM if it happens) ====="
|
||||
WATCHDOG_LOG="/root/build-watchdog.log"
|
||||
: > "$WATCHDOG_LOG"
|
||||
nohup bash -c "
|
||||
while true; do
|
||||
{ date -u +%FT%TZ; free -h; } >> '$WATCHDOG_LOG' 2>&1
|
||||
sleep 10
|
||||
done
|
||||
" >/dev/null 2>&1 &
|
||||
WATCHDOG_PID=$!
|
||||
trap 'kill "$WATCHDOG_PID" 2>/dev/null || true' EXIT
|
||||
|
||||
echo "===== autoninja chrome (-j$NINJA_JOBS) ====="
|
||||
autoninja -j"$NINJA_JOBS" -C "$OUT_DIR" chrome
|
||||
|
||||
test -f "$OUT_DIR/chrome"
|
||||
echo "BUILD OK: $CHROMIUM_ROOT/src/$OUT_DIR/chrome"
|
||||
25
install-deps.sh
Normal file
25
install-deps.sh
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/env bash
|
||||
# Install-build-deps phase: run via `hetzner-runner exec -script install-deps.sh`
|
||||
# against a box already synced by sync.sh in an earlier step.
|
||||
set -euo pipefail
|
||||
|
||||
: "${CHROMIUM_ROOT:=/root/chromium-ci}"
|
||||
export PATH="$HOME/depot_tools:$PATH"
|
||||
|
||||
retry() {
|
||||
local desc="$1"; shift
|
||||
local attempt=1
|
||||
local max=5
|
||||
until "$@"; do
|
||||
if [ "$attempt" -ge "$max" ]; then
|
||||
echo "$desc failed after $attempt attempts, giving up"
|
||||
return 1
|
||||
fi
|
||||
echo "$desc failed (attempt $attempt/$max); retrying in 5s"
|
||||
attempt=$((attempt + 1))
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
# Already root on this box -- no sudo/no_new_privs wrinkle to work around.
|
||||
retry "install-build-deps" "$CHROMIUM_ROOT/src/build/install-build-deps.py" --no-prompt
|
||||
90
main.go
90
main.go
|
|
@ -10,6 +10,10 @@
|
|||
// # up, pipe build.sh to `bash -s` on the box
|
||||
// # (env vars forwarded from this process's
|
||||
// # own environment, not the CLI args), down (always)
|
||||
// hetzner-runner exec -- <cmd> [args] # run against the server from a prior `up`,
|
||||
// hetzner-runner exec -script phase.sh ... # no provisioning, no teardown -- lets a CI
|
||||
// # workflow split a build into separate,
|
||||
// # separately-visible steps sharing one box
|
||||
//
|
||||
// HCLOUD_TOKEN must be set in the environment. Never pass secrets as flags
|
||||
// (shows up in process lists / CI logs) — use -env to forward them instead.
|
||||
|
|
@ -125,36 +129,38 @@ func main() {
|
|||
case "run":
|
||||
fs := flag.NewFlagSet("run", flag.ExitOnError)
|
||||
bindConfig(fs, &cfg)
|
||||
var scriptPath string
|
||||
var envNames stringSlice
|
||||
fs.StringVar(&scriptPath, "script", "", "local script to pipe to `bash -s` on the remote box, instead of a -- command")
|
||||
fs.Var(&envNames, "env", "name of an env var to read from this process's environment and forward to the remote script (repeatable)")
|
||||
scriptPath, envNames := bindPayloadFlags(fs)
|
||||
fs.Parse(os.Args[2:])
|
||||
payload := buildPayload("run", scriptPath, envNames, fs.Args())
|
||||
os.Exit(runLifecycle(client, token, cfg, payload))
|
||||
|
||||
var payload remotePayload
|
||||
if scriptPath != "" {
|
||||
script, err := os.ReadFile(scriptPath)
|
||||
if err != nil {
|
||||
fatal("run: reading -script %q: %v", scriptPath, err)
|
||||
}
|
||||
env := map[string]string{}
|
||||
for _, name := range envNames {
|
||||
v, ok := os.LookupEnv(name)
|
||||
if !ok {
|
||||
fatal("run: -env %s: not set in this process's environment", name)
|
||||
}
|
||||
env[name] = v
|
||||
}
|
||||
payload = remotePayload{script: string(script), env: env}
|
||||
} else {
|
||||
remainder := fs.Args()
|
||||
if len(remainder) == 0 {
|
||||
fatal("run: usage: hetzner-runner run [flags] -- <remote command...> (or -script <path>)")
|
||||
}
|
||||
payload = remotePayload{command: strings.Join(remainder, " ")}
|
||||
case "exec":
|
||||
// Runs a command/script against an already-provisioned server (from
|
||||
// a prior `up`), without provisioning or tearing down -- lets a CI
|
||||
// workflow split a build into separately-visible steps (Sync, gn
|
||||
// gen, Build v8, ...) that all share one box, with `up`/`down` as
|
||||
// their own explicit steps (the last one gated on `if: always()`
|
||||
// for guaranteed cleanup).
|
||||
fs := flag.NewFlagSet("exec", flag.ExitOnError)
|
||||
scriptPath, envNames := bindPayloadFlags(fs)
|
||||
fs.Parse(os.Args[2:])
|
||||
payload := buildPayload("exec", scriptPath, envNames, fs.Args())
|
||||
|
||||
st, err := loadState()
|
||||
if err != nil {
|
||||
fatal("exec: %v (run `up` first)", err)
|
||||
}
|
||||
|
||||
os.Exit(runLifecycle(client, token, cfg, payload))
|
||||
var code int
|
||||
if payload.script != "" {
|
||||
code, err = sshExecScript(st, payload.script, payload.env)
|
||||
} else {
|
||||
code, err = sshExec(st, payload.command)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "remote command failed: %v\n", err)
|
||||
}
|
||||
os.Exit(code)
|
||||
|
||||
default:
|
||||
fmt.Fprintln(os.Stderr, "usage: hetzner-runner <up|down|run> [flags]")
|
||||
|
|
@ -162,6 +168,38 @@ func main() {
|
|||
}
|
||||
}
|
||||
|
||||
// bindPayloadFlags registers -script/-env on fs and returns pointers to
|
||||
// read after fs.Parse. Shared by `run` and `exec`, which both accept either
|
||||
// a -- command or a -script.
|
||||
func bindPayloadFlags(fs *flag.FlagSet) (scriptPath *string, envNames *stringSlice) {
|
||||
scriptPath = fs.String("script", "", "local script to pipe to `bash -s` on the remote box, instead of a -- command")
|
||||
envNames = &stringSlice{}
|
||||
fs.Var(envNames, "env", "name of an env var to read from this process's environment and forward to the remote script (repeatable)")
|
||||
return scriptPath, envNames
|
||||
}
|
||||
|
||||
func buildPayload(subcmd string, scriptPath *string, envNames *stringSlice, remainder []string) remotePayload {
|
||||
if *scriptPath != "" {
|
||||
script, err := os.ReadFile(*scriptPath)
|
||||
if err != nil {
|
||||
fatal("%s: reading -script %q: %v", subcmd, *scriptPath, err)
|
||||
}
|
||||
env := map[string]string{}
|
||||
for _, name := range *envNames {
|
||||
v, ok := os.LookupEnv(name)
|
||||
if !ok {
|
||||
fatal("%s: -env %s: not set in this process's environment", subcmd, name)
|
||||
}
|
||||
env[name] = v
|
||||
}
|
||||
return remotePayload{script: string(script), env: env}
|
||||
}
|
||||
if len(remainder) == 0 {
|
||||
fatal("%s: usage: hetzner-runner %s [flags] -- <remote command...> (or -script <path>)", subcmd, subcmd)
|
||||
}
|
||||
return remotePayload{command: strings.Join(remainder, " ")}
|
||||
}
|
||||
|
||||
func bindConfig(fs *flag.FlagSet, cfg *config) {
|
||||
fs.StringVar(&cfg.serverType, "type", cfg.serverType, "Hetzner server type")
|
||||
fs.StringVar(&cfg.image, "image", cfg.image, "OS image")
|
||||
|
|
|
|||
104
sync.sh
Normal file
104
sync.sh
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
#!/usr/bin/env bash
|
||||
# Sync phase: clone/update depot_tools, clone the chromium-fork checkout,
|
||||
# gclient sync. Run via `hetzner-runner exec -script sync.sh` against a box
|
||||
# already provisioned by a separate `up` step -- see ../../.gitea/workflows
|
||||
# in chromium-fork for the phase breakdown this belongs to.
|
||||
set -euo pipefail
|
||||
|
||||
: "${GITEA_TOKEN:?GITEA_TOKEN must be set}"
|
||||
: "${GITEA_REPO:=Cold-Air-Networks/chromium-fork}"
|
||||
: "${GITEA_REF:=main}"
|
||||
: "${MIRROR_READ_TOKEN:=$GITEA_TOKEN}"
|
||||
: "${CHROMIUM_ROOT:=/root/chromium-ci}"
|
||||
|
||||
retry() {
|
||||
local desc="$1"; shift
|
||||
local attempt=1
|
||||
local max=5
|
||||
until "$@"; do
|
||||
if [ "$attempt" -ge "$max" ]; then
|
||||
echo "$desc failed after $attempt attempts, giving up"
|
||||
return 1
|
||||
fi
|
||||
echo "$desc failed (attempt $attempt/$max); retrying in 5s"
|
||||
attempt=$((attempt + 1))
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
redact() {
|
||||
sed -E 's#x-access-token:[^@]*@#x-access-token:REDACTED@#g'
|
||||
}
|
||||
|
||||
echo "===== depot_tools ====="
|
||||
if [ ! -d "$HOME/depot_tools" ]; then
|
||||
retry "clone depot_tools" git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git "$HOME/depot_tools"
|
||||
fi
|
||||
export PATH="$HOME/depot_tools:$PATH"
|
||||
|
||||
# Explicit bootstrap before anything else touches python3. depot_tools'
|
||||
# `python3` wrapper needs python3_bin_reldir.txt, normally created on first
|
||||
# use -- but chromium's own gclient hooks run
|
||||
# update_depot_tools_toggle.py --disable partway through sync (confirmed:
|
||||
# ci run #54, 2026-08-04), which can win the race and leave that file
|
||||
# missing for any *later* direct python3 invocation, e.g.
|
||||
# install-build-deps.py's shebang in the next phase. Forcing the bootstrap
|
||||
# here, before sync even starts, sidesteps the race entirely.
|
||||
"$HOME/depot_tools/python3" --version >/dev/null
|
||||
|
||||
mkdir -p "$CHROMIUM_ROOT"
|
||||
cd "$CHROMIUM_ROOT"
|
||||
|
||||
REPO_URL="https://x-access-token:${GITEA_TOKEN}@forge.quinefoundation.com/${GITEA_REPO}.git"
|
||||
|
||||
# VK-GL-CTS against chromium.googlesource.com directly was the single
|
||||
# biggest source of sync flakiness in chromium-fork's original CI workflow
|
||||
# (corrupted packs, silent stalls). Redirect to the org's own mirror.
|
||||
VK_GL_CTS_UPSTREAM="https://chromium.googlesource.com/external/github.com/KhronosGroup/VK-GL-CTS"
|
||||
git config --global \
|
||||
url."https://x-access-token:${MIRROR_READ_TOKEN}@forge.quinefoundation.com/Cold-Air-Networks/mirror-vk-gl-cts.git".insteadOf \
|
||||
"$VK_GL_CTS_UPSTREAM"
|
||||
|
||||
echo "===== registered url.insteadOf mappings ====="
|
||||
git config --global --get-regexp '^url\.' | redact
|
||||
echo "===== end registered url.insteadOf mappings ====="
|
||||
|
||||
cat > .gclient <<EOF
|
||||
solutions = [
|
||||
{
|
||||
"name": "src",
|
||||
"url": "$REPO_URL",
|
||||
"custom_deps": {},
|
||||
"custom_vars": {},
|
||||
},
|
||||
]
|
||||
EOF
|
||||
|
||||
echo "===== fetching $GITEA_REPO@$GITEA_REF ====="
|
||||
if [ ! -d src/.git ]; then
|
||||
retry "clone src" git clone "$REPO_URL" src
|
||||
fi
|
||||
cd src
|
||||
git remote set-url origin "$REPO_URL"
|
||||
retry "fetch $GITEA_REF" git fetch origin "$GITEA_REF"
|
||||
git checkout --force FETCH_HEAD
|
||||
|
||||
# --shallow is deliberately absent: chromium-fork's original CI workflow
|
||||
# traced every VK-GL-CTS stall to --shallow itself (git_cache.py turns it
|
||||
# into --depth 10000, a much harder reachability problem across VK-GL-CTS's
|
||||
# 150+ tags/branches than an unlimited-depth fetch of the same refs) --
|
||||
# removing it is what got that workflow's sync to actually succeed.
|
||||
echo "===== gclient sync ====="
|
||||
ATTEMPT=1
|
||||
MAX_ATTEMPTS=5
|
||||
while true; do
|
||||
if gclient sync --with_branch_heads --with_tags -j"$(nproc)" --verbose --force 2>&1 | redact; then
|
||||
break
|
||||
fi
|
||||
if [ "$ATTEMPT" -ge "$MAX_ATTEMPTS" ]; then
|
||||
echo "gclient sync failed after $ATTEMPT attempts, giving up"
|
||||
exit 1
|
||||
fi
|
||||
echo "attempt $ATTEMPT failed; retrying ($((ATTEMPT + 1))/$MAX_ATTEMPTS)"
|
||||
ATTEMPT=$((ATTEMPT + 1))
|
||||
done
|
||||
Loading…
Reference in a new issue