Add download capability: exec -download remote=local

Mirror of -upload: streams a remote file straight to a local path via a
plain (non-PTY) `cat remotePath`, without buffering it in memory first --
build artifacts pulled this way can be hundreds of MB to a few GB.
Only downloads if the remote command actually succeeded (a nonzero exit
means there's nothing valid to pull), and a download failure counts as a
Go-level error for the existing cleanup-on-error path, same as upload
failures.

For retrieving Chromium build output before the ephemeral box is torn
down: hetzner-runner runs on mainframe itself, so this pulls straight to
mainframe's own disk (a mounted Hetzner Volume, /mnt/volume-hil-1) with
no intermediate storage needed.

Verified against a real box: 500KB random file round-tripped with
matching md5sum on both ends.
This commit is contained in:
Philip Peterson 2026-08-06 01:00:23 -08:00
parent ec0c63d74d
commit 64914555c7

62
main.go
View file

@ -187,6 +187,8 @@ func main() {
scriptPath, envNames := bindPayloadFlags(fs)
uploads := &stringSlice{}
fs.Var(uploads, "upload", "local=remote file to copy to the box before running (repeatable)")
downloads := &stringSlice{}
fs.Var(downloads, "download", "remote=local file to copy from the box after running, only if it succeeded (repeatable)")
fs.Parse(os.Args[2:])
payload := buildPayload("exec", scriptPath, envNames, fs.Args())
@ -195,7 +197,7 @@ func main() {
fatal("exec: %v (run `up` first)", err)
}
os.Exit(runExecCommand(client, st, payload, *uploads, scriptPath))
os.Exit(runExecCommand(client, st, payload, *uploads, *downloads, scriptPath))
default:
fmt.Fprintln(os.Stderr, "usage: hetzner-runner <up|down|run> [flags]")
@ -301,7 +303,7 @@ func withHeartbeat(label string, fn func() (int, error)) (int, error) {
// instead of a clean exit, which previously fell through to nothing until
// the separate down step (or the cloud-init failsafe, hours later) caught
// it.
func runExecCommand(client *hcloud.Client, st *state, payload remotePayload, uploads stringSlice, scriptPath *string) (code int) {
func runExecCommand(client *hcloud.Client, st *state, payload remotePayload, uploads, downloads stringSlice, scriptPath *string) (code int) {
succeeded := false
defer func() {
if succeeded {
@ -358,6 +360,25 @@ func runExecCommand(client *hcloud.Client, st *state, payload remotePayload, upl
return 1
}
// Only download if the command actually succeeded -- a nonzero exit
// (e.g. the tar step itself failing) means there's nothing valid to
// pull down.
if code == 0 {
for _, d := range downloads {
remote, local, ok := strings.Cut(d, "=")
if !ok {
fmt.Fprintf(os.Stderr, "exec: -download %q: expected remote=local\n", d)
return 2
}
dlog("exec: downloading %s -> %s", remote, local)
if err := downloadFile(st, remote, local); err != nil {
dlog("exec: download FAILED (Go-level error, cleaning up): %v", err)
fmt.Fprintf(os.Stderr, "exec: downloading %s: %v\n", remote, err)
return 1
}
}
}
// A normal nonzero exit code from the remote command is NOT a
// Go-level error -- an ordinary step failure, left for the workflow's
// own separate teardown step. Only Go-level errors (above) skip
@ -635,6 +656,43 @@ func uploadBytes(st *state, content []byte, remotePath string) error {
return nil
}
// downloadFile is uploadBytes's mirror image: streams remotePath's bytes
// (via a plain, non-PTY `cat remotePath`) straight to a local file,
// without buffering the whole thing in memory first -- build artifacts
// pulled this way can be hundreds of MB to a few GB, and hetzner-runner
// itself runs on the same host (mainframe) as the destination disk, so
// this is the direct path rather than routing through any intermediate
// storage.
func downloadFile(st *state, remotePath, localPath string) error {
session, client, err := sshSession(st, false)
if err != nil {
return err
}
defer client.Close()
defer session.Close()
if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
return fmt.Errorf("creating local directory for %s: %w", localPath, err)
}
f, err := os.Create(localPath)
if err != nil {
return err
}
defer f.Close()
var stderrBuf bytes.Buffer
session.Stdout = f
session.Stderr = &stderrBuf
if err := session.Run(fmt.Sprintf("cat %s", shellQuote(remotePath))); err != nil {
if exitErr, ok := err.(*ssh.ExitError); ok {
return fmt.Errorf("remote `cat %s` exited %d: %s", remotePath, exitErr.ExitStatus(), stderrBuf.String())
}
return err
}
return nil
}
// sshExecScript runs script on the remote box by uploading it as a file
// and executing it, rather than piping it to `bash -s` over stdin.
//