Widen self-destruct failsafe to 10h; add cleanup-on-error to exec
Two related hardenings found while investigating ci run #65's orphaned box: the "Tear down Hetzner build box" if:always() step never got a chance to run at all, because act_runner cancelling a step on its own job timeout aborts the whole job immediately rather than continuing to later steps -- SIGKILL to the exec process, no defer, no signal handler, nothing runs. - max-lifetime (cloud-init self-destruct) raised from 8h to 10h: with the job timeout doubled to 6h (separate commit, same investigation), 8h left only 2h of margin -- uncomfortably close given provisioning/retry overhead on top of the job's own 6h budget. - exec now defers a cleanup attempt for genuine Go-level errors (upload failure, SSH connection failure, panic) partway through a step -- distinct from an ordinary nonzero remote exit code, which is left alone for the workflow's own separate, later "down" step (so one failing build phase doesn't nuke a box other phases might still want). A SIGKILL still can't be caught by anything running in-process, so this doesn't close that specific gap, but it covers every other way this process could die with an error instead of a clean exit -- cases that previously fell through to nothing until the separate down step, or the cloud-init failsafe hours later, eventually caught them. Verified against a real box: a plain `exit 3` from the remote command correctly leaves the server alive (exit code 3 propagated, no teardown); an upload failure against a nonexistent local file correctly tears the server down immediately ("tore down the server after an error in this step", exit code 1).
This commit is contained in:
Philip Peterson
parent
b73550f958
commit
93516327e9
1 changed files with 93 additions and 33 deletions
126
main.go
126
main.go
|
|
@ -79,7 +79,16 @@ func defaultConfig() config {
|
|||
image: envOr("HETZNER_IMAGE", "ubuntu-24.04"),
|
||||
location: envOr("HETZNER_LOCATION", "hil"),
|
||||
namePrefix: envOr("HETZNER_NAME_PREFIX", "chromium-build"),
|
||||
maxLifetime: 8 * time.Hour,
|
||||
// Failsafe backstop for exactly the gap found in ci run #65
|
||||
// (2026-08-05): act_runner cancelling a step on its own job
|
||||
// timeout aborts the whole job immediately rather than
|
||||
// continuing to the `if: always()` teardown step, so this is
|
||||
// sometimes the *only* thing that deletes the server. Needs
|
||||
// comfortable margin beyond the job timeout (now 6h, doubled
|
||||
// from 3h the same day this was discovered) -- 8h left only 2h
|
||||
// of margin, uncomfortably close given provisioning/retry
|
||||
// overhead on top of the job's own 6h.
|
||||
maxLifetime: 10 * time.Hour,
|
||||
sshTimeout: 3 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
|
@ -186,38 +195,7 @@ func main() {
|
|||
fatal("exec: %v (run `up` first)", err)
|
||||
}
|
||||
|
||||
payloadDesc := payload.command
|
||||
if payload.script != "" {
|
||||
payloadDesc = fmt.Sprintf("-script %s (%d bytes)", *scriptPath, len(payload.script))
|
||||
}
|
||||
dlog("exec: starting against server_id=%d: %s", st.ServerID, payloadDesc)
|
||||
|
||||
for _, u := range *uploads {
|
||||
local, remote, ok := strings.Cut(u, "=")
|
||||
if !ok {
|
||||
fatal("exec: -upload %q: expected local=remote", u)
|
||||
}
|
||||
dlog("exec: uploading %s -> %s", local, remote)
|
||||
if err := uploadFile(st, local, remote); err != nil {
|
||||
dlog("exec: upload FAILED: %v", err)
|
||||
fatal("exec: uploading %s: %v", local, err)
|
||||
}
|
||||
}
|
||||
|
||||
var code int
|
||||
code, err = withHeartbeat(payloadDesc, func() (int, error) {
|
||||
if payload.script != "" {
|
||||
return sshExecScript(st, payload.script, payload.env)
|
||||
}
|
||||
return sshExec(st, payload.command)
|
||||
})
|
||||
if err != nil {
|
||||
dlog("exec: FAILED: %v", err)
|
||||
fmt.Fprintf(os.Stderr, "remote command failed: %v\n", err)
|
||||
} else {
|
||||
dlog("exec: finished, exit code=%d", code)
|
||||
}
|
||||
os.Exit(code)
|
||||
os.Exit(runExecCommand(client, st, payload, *uploads, scriptPath))
|
||||
|
||||
default:
|
||||
fmt.Fprintln(os.Stderr, "usage: hetzner-runner <up|down|run> [flags]")
|
||||
|
|
@ -307,6 +285,88 @@ func withHeartbeat(label string, fn func() (int, error)) (int, error) {
|
|||
// runLifecycle does up -> exec -> down, guaranteeing down runs even if the
|
||||
// remote command fails or this process is interrupted (Ctrl-C, CI job
|
||||
// cancellation).
|
||||
// runExecCommand runs payload against an already-provisioned server (st),
|
||||
// uploading any -upload files first.
|
||||
//
|
||||
// Unlike an ordinary step failure (a nonzero remote exit code -- left for
|
||||
// the CI workflow's own separate, later "down" step to clean up, since one
|
||||
// failing build phase shouldn't nuke a box other phases might still want),
|
||||
// this proactively tears the server down itself if it hits a genuine
|
||||
// Go-level error partway through (upload failure, SSH connection failure,
|
||||
// or a panic) -- belt-and-suspenders against the gap found in ci run #65:
|
||||
// act_runner cancelling a step on its own job timeout aborts the whole job
|
||||
// immediately, skipping the `if: always()` teardown step entirely. A
|
||||
// SIGKILL still can't be caught here either (nothing can), but this at
|
||||
// least covers every OTHER way this process could die with an error
|
||||
// instead of a clean exit, which previously fell through to nothing until
|
||||
// the separate down step (or the cloud-init failsafe, hours later) caught
|
||||
// it.
|
||||
func runExecCommand(client *hcloud.Client, st *state, payload remotePayload, uploads stringSlice, scriptPath *string) (code int) {
|
||||
succeeded := false
|
||||
defer func() {
|
||||
if succeeded {
|
||||
return
|
||||
}
|
||||
r := recover()
|
||||
dlog("exec: attempting cleanup after error (recovered_panic=%v)", r != nil)
|
||||
downCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
if err := down(downCtx, client, st); err != nil {
|
||||
dlog("exec: cleanup-on-error teardown FAILED: %v", err)
|
||||
fmt.Fprintf(os.Stderr, "WARNING: cleanup-on-error teardown failed, server %d may still be running and billing: %v\n", st.ServerID, err)
|
||||
fmt.Fprintf(os.Stderr, " manual cleanup: hcloud server delete %d\n", st.ServerID)
|
||||
} else {
|
||||
os.Remove(stateFileName)
|
||||
dlog("exec: cleanup-on-error teardown succeeded, server_id=%d", st.ServerID)
|
||||
fmt.Fprintln(os.Stderr, "note: tore down the server after an error in this step")
|
||||
}
|
||||
if r != nil {
|
||||
panic(r)
|
||||
}
|
||||
}()
|
||||
|
||||
payloadDesc := payload.command
|
||||
if payload.script != "" {
|
||||
payloadDesc = fmt.Sprintf("-script %s (%d bytes)", *scriptPath, len(payload.script))
|
||||
}
|
||||
dlog("exec: starting against server_id=%d: %s", st.ServerID, payloadDesc)
|
||||
|
||||
for _, u := range uploads {
|
||||
local, remote, ok := strings.Cut(u, "=")
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "exec: -upload %q: expected local=remote\n", u)
|
||||
return 2 // succeeded stays false: not a server-side error, but nothing ran either, so tear down for consistency
|
||||
}
|
||||
dlog("exec: uploading %s -> %s", local, remote)
|
||||
if err := uploadFile(st, local, remote); err != nil {
|
||||
dlog("exec: upload FAILED (Go-level error, cleaning up): %v", err)
|
||||
fmt.Fprintf(os.Stderr, "exec: uploading %s: %v\n", local, err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
var runErr error
|
||||
code, runErr = withHeartbeat(payloadDesc, func() (int, error) {
|
||||
if payload.script != "" {
|
||||
return sshExecScript(st, payload.script, payload.env)
|
||||
}
|
||||
return sshExec(st, payload.command)
|
||||
})
|
||||
if runErr != nil {
|
||||
dlog("exec: Go-level FAILURE (cleaning up): %v", runErr)
|
||||
fmt.Fprintf(os.Stderr, "remote command failed: %v\n", runErr)
|
||||
return 1
|
||||
}
|
||||
|
||||
// A normal nonzero exit code from the remote command is NOT a
|
||||
// Go-level error -- an ordinary step failure, left for the workflow's
|
||||
// own separate teardown step. Only Go-level errors (above) skip
|
||||
// marking succeeded, triggering cleanup here.
|
||||
succeeded = true
|
||||
dlog("exec: finished, exit code=%d", code)
|
||||
return code
|
||||
}
|
||||
|
||||
func runLifecycle(client *hcloud.Client, token string, cfg config, payload remotePayload) (exitCode int) {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
|
|
|||
Loading…
Reference in a new issue