chromium-setup/stall-watch.py

#!/usr/bin/env python3
"""Run a command; if it produces no output for --stall-timeout seconds,
dump process/network/strace diagnostics and kill it.

Exists to catch silent hangs without paying for hours of a hung, billing
Hetzner box before anyone notices -- ci run #56 (2026-08-04) stalled for
~3h with zero output after devtools-frontend's sync_rollup_libs.py hook,
and there was no diagnostic trail left to say why.

Allocates a PTY for the child (not a plain pipe) so wrapping a command
here doesn't reintroduce the block-buffering that hetzner-runner's own
SSH PTY was added to avoid -- git/gclient still think they're attached to
a terminal and line-buffer accordingly.
"""
import argparse
import os
import pty
import signal
import subprocess
import sys
import threading
import time


def run_capture(cmd, timeout=15):
    try:
        out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
        return out.stdout + out.stderr
    except Exception as e:
        return f"(failed: {e})"


def dump_diagnostics(pid):
    print("===== STALL DETECTED: dumping diagnostics =====", flush=True)

    print("--- process tree (ps -eo pid,ppid,stat,etimes,cmd, sorted by runtime) ---", flush=True)
    print(run_capture(["ps", "-eo", "pid,ppid,stat,etimes,cmd", "--sort=-etimes"]), flush=True)

    print("--- network connections (ss -tnp) ---", flush=True)
    print(run_capture(["ss", "-tnp"]), flush=True)

    print("--- processes with a cmdline, and their /proc/<pid>/wchan (kernel wait reason) ---", flush=True)
    try:
        proc_entries = sorted(os.listdir("/proc"))
    except OSError as e:
        proc_entries = []
        print(f"(/proc unavailable: {e})", flush=True)
    for entry in proc_entries:
        if not entry.isdigit():
            continue
        p = entry
        try:
            with open(f"/proc/{p}/cmdline", "rb") as f:
                cmdline = f.read().replace(b"\0", b" ").decode(errors="replace").strip()
            if not cmdline:
                continue
            wchan = ""
            try:
                with open(f"/proc/{p}/wchan") as f:
                    wchan = f.read()
            except OSError:
                pass
            print(f"pid={p} wchan={wchan} cmd={cmdline}", flush=True)
        except OSError:
            continue

    print("--- 5s strace sample of the stalled process (syscall it's actually blocked in) ---", flush=True)
    if subprocess.run(["which", "strace"], capture_output=True).returncode == 0:
        try:
            proc = subprocess.Popen(
                ["strace", "-f", "-tt", "-T", "-p", str(pid)],
                stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
            )
            time.sleep(5)
            proc.send_signal(signal.SIGINT)
            out, _ = proc.communicate(timeout=10)
            print(out, flush=True)
        except Exception as e:
            print(f"(strace sample failed: {e})", flush=True)
    else:
        print("(strace not installed -- apt-get install -y strace to get syscall-level diagnostics next time)", flush=True)

    print("===== end stall diagnostics =====", flush=True)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--stall-timeout", type=int, default=300,
                     help="seconds of zero output before treating the command as stalled")
    ap.add_argument("cmd", nargs=argparse.REMAINDER)
    args = ap.parse_args()
    cmd = args.cmd[1:] if args.cmd[:1] == ["--"] else args.cmd
    if not cmd:
        sys.exit("usage: stall-watch.py [--stall-timeout SECONDS] -- <cmd...>")

    master_fd, slave_fd = pty.openpty()
    proc = subprocess.Popen(
        cmd, stdout=slave_fd, stderr=slave_fd, stdin=subprocess.DEVNULL,
        start_new_session=True, close_fds=True,
    )
    os.close(slave_fd)

    last_output = time.time()
    lock = threading.Lock()
    stalled = threading.Event()

    def reader():
        nonlocal last_output
        while True:
            try:
                chunk = os.read(master_fd, 4096)
            except OSError:
                break
            if not chunk:
                break
            sys.stdout.buffer.write(chunk)
            sys.stdout.flush()
            with lock:
                last_output = time.time()

    t = threading.Thread(target=reader, daemon=True)
    t.start()

    while proc.poll() is None:
        time.sleep(10)
        with lock:
            elapsed = time.time() - last_output
        if elapsed >= args.stall_timeout and not stalled.is_set():
            stalled.set()
            # Diagnostics must never block the kill that follows -- a
            # missing tool, a permission error, or a process that exits
            # mid-scan must not leave a stalled build hanging forever
            # because dump_diagnostics() raised.
            try:
                dump_diagnostics(proc.pid)
            except Exception as e:
                print(f"(dump_diagnostics itself failed: {e!r} -- killing anyway)", flush=True)
            print(f"killing process group (pgid={os.getpgid(proc.pid)}) after {elapsed:.0f}s of silence", flush=True)
            try:
                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
            except ProcessLookupError:
                pass
            time.sleep(5)
            try:
                os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
            except ProcessLookupError:
                pass
            break

    t.join(timeout=5)
    code = proc.wait()
    os.close(master_fd)
    sys.exit(124 if stalled.is_set() else code)


if __name__ == "__main__":
    main()