Add download capability: exec -download remote=local
Mirror of -upload: streams a remote file straight to a local path via a plain (non-PTY) `cat remotePath`, without buffering it in memory first -- build artifacts pulled this way can be hundreds of MB to a few GB. Only downloads if the remote command actually succeeded (a nonzero exit means there's nothing valid to pull), and a download failure counts as a Go-level error for the existing cleanup-on-error path, same as upload failures. For retrieving Chromium build output before the ephemeral box is torn down: hetzner-runner runs on mainframe itself, so this pulls straight to mainframe's own disk (a mounted Hetzner Volume, /mnt/volume-hil-1) with no intermediate storage needed. Verified against a real box: 500KB random file round-tripped with matching md5sum on both ends.
This commit is contained in:
parent
ec0c63d74d
commit
64914555c7
1 changed files with 60 additions and 2 deletions
62
main.go
62
main.go
|
|
@ -187,6 +187,8 @@ func main() {
|
|||
scriptPath, envNames := bindPayloadFlags(fs)
|
||||
uploads := &stringSlice{}
|
||||
fs.Var(uploads, "upload", "local=remote file to copy to the box before running (repeatable)")
|
||||
downloads := &stringSlice{}
|
||||
fs.Var(downloads, "download", "remote=local file to copy from the box after running, only if it succeeded (repeatable)")
|
||||
fs.Parse(os.Args[2:])
|
||||
payload := buildPayload("exec", scriptPath, envNames, fs.Args())
|
||||
|
||||
|
|
@ -195,7 +197,7 @@ func main() {
|
|||
fatal("exec: %v (run `up` first)", err)
|
||||
}
|
||||
|
||||
os.Exit(runExecCommand(client, st, payload, *uploads, scriptPath))
|
||||
os.Exit(runExecCommand(client, st, payload, *uploads, *downloads, scriptPath))
|
||||
|
||||
default:
|
||||
fmt.Fprintln(os.Stderr, "usage: hetzner-runner <up|down|run> [flags]")
|
||||
|
|
@ -301,7 +303,7 @@ func withHeartbeat(label string, fn func() (int, error)) (int, error) {
|
|||
// instead of a clean exit, which previously fell through to nothing until
|
||||
// the separate down step (or the cloud-init failsafe, hours later) caught
|
||||
// it.
|
||||
func runExecCommand(client *hcloud.Client, st *state, payload remotePayload, uploads stringSlice, scriptPath *string) (code int) {
|
||||
func runExecCommand(client *hcloud.Client, st *state, payload remotePayload, uploads, downloads stringSlice, scriptPath *string) (code int) {
|
||||
succeeded := false
|
||||
defer func() {
|
||||
if succeeded {
|
||||
|
|
@ -358,6 +360,25 @@ func runExecCommand(client *hcloud.Client, st *state, payload remotePayload, upl
|
|||
return 1
|
||||
}
|
||||
|
||||
// Only download if the command actually succeeded -- a nonzero exit
|
||||
// (e.g. the tar step itself failing) means there's nothing valid to
|
||||
// pull down.
|
||||
if code == 0 {
|
||||
for _, d := range downloads {
|
||||
remote, local, ok := strings.Cut(d, "=")
|
||||
if !ok {
|
||||
fmt.Fprintf(os.Stderr, "exec: -download %q: expected remote=local\n", d)
|
||||
return 2
|
||||
}
|
||||
dlog("exec: downloading %s -> %s", remote, local)
|
||||
if err := downloadFile(st, remote, local); err != nil {
|
||||
dlog("exec: download FAILED (Go-level error, cleaning up): %v", err)
|
||||
fmt.Fprintf(os.Stderr, "exec: downloading %s: %v\n", remote, err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A normal nonzero exit code from the remote command is NOT a
|
||||
// Go-level error -- an ordinary step failure, left for the workflow's
|
||||
// own separate teardown step. Only Go-level errors (above) skip
|
||||
|
|
@ -635,6 +656,43 @@ func uploadBytes(st *state, content []byte, remotePath string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// downloadFile is uploadBytes's mirror image: streams remotePath's bytes
|
||||
// (via a plain, non-PTY `cat remotePath`) straight to a local file,
|
||||
// without buffering the whole thing in memory first -- build artifacts
|
||||
// pulled this way can be hundreds of MB to a few GB, and hetzner-runner
|
||||
// itself runs on the same host (mainframe) as the destination disk, so
|
||||
// this is the direct path rather than routing through any intermediate
|
||||
// storage.
|
||||
func downloadFile(st *state, remotePath, localPath string) error {
|
||||
session, client, err := sshSession(st, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer client.Close()
|
||||
defer session.Close()
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(localPath), 0755); err != nil {
|
||||
return fmt.Errorf("creating local directory for %s: %w", localPath, err)
|
||||
}
|
||||
f, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var stderrBuf bytes.Buffer
|
||||
session.Stdout = f
|
||||
session.Stderr = &stderrBuf
|
||||
|
||||
if err := session.Run(fmt.Sprintf("cat %s", shellQuote(remotePath))); err != nil {
|
||||
if exitErr, ok := err.(*ssh.ExitError); ok {
|
||||
return fmt.Errorf("remote `cat %s` exited %d: %s", remotePath, exitErr.ExitStatus(), stderrBuf.String())
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sshExecScript runs script on the remote box by uploading it as a file
|
||||
// and executing it, rather than piping it to `bash -s` over stdin.
|
||||
//
|
||||
|
|
|
|||
Loading…
Reference in a new issue