#!/usr/bin/env python3
# SPDX-License-Identifier: AGPL-3.0-or-later
"""External supervisor for one recorded smoke run.

Long unattended runs stop depending on the controller process staying
alive: this supervisor launches exactly one ``scripts/smoke_run.py``
run as its child, owns the child's wall-clock deadline, and on
controller death (child killed, child lost, or child exit without a
settled workdir) reaps the child, kills the whole process tree, and
drives the recorded recovery path (``--resume`` under the run's
recorded authorisation) so every trial settles and no spend
reservation is left in flight.

The supervisor is the smallest slice that fails closed: the child
discipline is ``scripts/stress_run.py``'s (``run_child`` /
``ChildResult`` / the deadline bound) plus the recorded ``--resume``
recovery. No daemon, no heartbeat protocol, no credential re-issuance
beyond what ``--resume`` already does, and no new event type. The
supervisor never writes the workdir, the stores, or the ledger itself
— every effect on trial state goes through the two recorded subprocess
heads.

The expiry kill runs inside a container boundary: the child is forked
into its own cgroup v2 leaf, so every process it forks inherits the
leaf, and the kill's container half reaches what adoption moves out of
the descendant walk — a session-escaped forker's tail forks and a
predecessor-exited sleeper already adopted by init die with the
deadline because cgroup membership is fixed at fork and is not changed
by reparenting. The boundary is measured at runtime, never assumed:
with no cgroup v2 delegation the sweep stands alone.

Usage (from the repository root):

    python3 scripts/supervise_run.py --budget SECONDS

The run's environment passes through unchanged: every ``ORCHESTRATOR_*``
variable the operator sets reaches the child (and the recovery) exactly
as set — the supervisor adds none and redacts none, so the recorded
authorisation and the provider credential ride the same environment
into both heads and the secret never touches the supervisor's own
output. Two supervisor-only settings exist: the wall-clock budget
(``--budget``, seconds) and the workdir, read from the same
``ORCHESTRATOR_WORKDIR`` the child uses, so supervisor and child
cannot disagree about which workdir the recovery reads.

The child is launched as ``sys.executable scripts/smoke_run.py --json``
— the recorded head, by relative path from the supervisor's working
directory (the repository root under the documented invocation form).
Narration goes to stderr (the child's and the recovery's narration is
forwarded there; the recorded head never logs secrets); stdout carries
exactly one JSON document:

    {"workdir": ..., "code": <child-or-recovery code>,
     "timed_out": <bool>, "recovered": <bool>,
     "child_wall_seconds": <float>}

The supervisor exits 0 only when the child exited 0 with a JSON
document and no recovery ran; otherwise it exits the child's code (or
1 when the child produced none — killed at the deadline, or exited 0
without a document). A missing workdir, an unparsable budget, or a
``--resume`` that itself fails is a nonzero exit with the reason on
stderr and no JSON document claimed as success.
"""
from __future__ import annotations

import argparse
import dataclasses
import json
import math
import os
import signal
import subprocess
import sys
import tempfile
import time
from typing import Any, Dict, Optional

# The recorded head, launched exactly as the command text reads:
# ``sys.executable scripts/smoke_run.py --json``. The path is relative
# to the supervisor's working directory — the repository root under the
# documented invocation form — and the recovery is the same head with
# ``--resume``.
CHILD = "scripts/smoke_run.py"

# Misuse exits nonzero with the reason on stderr and no JSON document.
MISUSE_EXIT = 2

# The expiry kill's bounded sweep-to-quiescence budget: the frozen
# child's tree is re-captured after this many passes at most, with a
# short settle before the first capture (to let the freeze land) and
# between passes (to let a fork from the previous pass appear in
# /proc).
SWEEP_PASSES = 3
SWEEP_SETTLE = 0.1

# The container boundary: the child runs inside its own cgroup v2 leaf
# from the moment it is forked, so every process it forks inherits the
# leaf. cgroup membership is fixed at fork and is not changed by
# reparenting, so the container kill reaches what adoption moves out of
# the descendant walk: a session-escaped forker's tail forks and a
# predecessor-exited sleeper already adopted by init die with the
# deadline because they are still in the leaf. Availability is measured
# at runtime — the unified mount, this process's own cgroup, a writable
# parent's cgroup.procs, and a creatable leaf — never assumed: with no
# delegation the sweep stands alone, exactly as it stood before the
# boundary. The leaf is per supervisor process and is removed when the
# child is done, with one clean-child corner it does not cover: a
# process that outlived a clean child's exit was never in any kill's
# surface, so a leaf still occupied after the bounded removal budget is
# left standing by construction, and the deadline contract does not
# cover that path.
CONTAINER_PREFIX = "orchestrator-supervisor-"
# Removing the leaf retries: a SIGKILLed member leaves the leaf only
# once it exits, and the exit lands asynchronously.
CONTAINER_REMOVALS = 20
CONTAINER_REMOVAL_SETTLE = 0.05


def _cgroup2_mount() -> Optional[str]:
    """The cgroup2 mount point on this host, or None when the unified
    hierarchy is not mounted."""
    try:
        with open("/proc/self/mountinfo") as handle:
            for line in handle:
                left, _, right = line.partition(" - ")
                if right.split()[:1] == ["cgroup2"]:
                    return left.split()[4]
    except OSError:
        return None
    return None


def _own_cgroup() -> Optional[str]:
    """This process's cgroup v2 path — the ``0::`` line of
    ``/proc/self/cgroup`` — or None on a host that runs cgroup v1."""
    try:
        with open("/proc/self/cgroup") as handle:
            for line in handle:
                if line.startswith("0::"):
                    return line.strip().split(":")[2]
    except OSError:
        return None
    return None


def _container_boundary() -> Optional[str]:
    """The writable cgroup v2 directory this supervisor would create its
    container leaf under, or None when no delegation is available.
    Measured, not assumed: the unified mount, this process's own cgroup,
    and the nearest writable ``cgroup.procs`` at or above it. This is
    also the boundary's own availability question — where it returns
    None (no cgroup v2 in ``mountinfo``, as inside the R08 grading
    sandbox's fresh root, or no writable delegation) no process here can
    be put in a container at all, and the sweep stands alone."""
    mount = _cgroup2_mount()
    own = _own_cgroup()
    if mount is None or own is None:
        return None
    parts = [part for part in own.split("/") if part]
    for depth in range(len(parts), -1, -1):
        parent = mount if depth == 0 else f"{mount}/{'/'.join(parts[:depth])}"
        if (
            os.path.isdir(parent)
            and os.access(parent, os.W_OK)
            and os.access(f"{parent}/cgroup.procs", os.W_OK)
        ):
            return parent
    return None


def _container_open() -> Optional[str]:
    """This supervisor's container leaf, or None when no cgroup v2
    delegation is available (measured, not assumed, by
    ``_container_boundary``). The child is then never more than one
    write away from the boundary."""
    parent = _container_boundary()
    if parent is None:
        return None
    try:
        leaf = tempfile.mkdtemp(prefix=CONTAINER_PREFIX, dir=parent)
    except OSError:
        return None
    if os.access(f"{leaf}/cgroup.procs", os.W_OK):
        return leaf
    _container_close(leaf)
    return None


def _container_join(leaf: str) -> None:
    """The pre-exec hook: the child moves itself into the container leaf
    between fork and exec — before it can fork anything — so every
    descendant inherits the boundary. Defensive by construction: a write
    failure must not cost the launch, and the parent's own join after
    the launch is the fallback."""
    try:
        with open(f"{leaf}/cgroup.procs", "w") as handle:
            handle.write(str(os.getpid()))
    except OSError:
        pass


def _container_adopt(leaf: str, pid: int) -> None:
    """The parent-side join: moves the launched child into the leaf if
    the pre-exec hook could not (idempotent when it did). Best-effort —
    a failure leaves the sweep standing alone, never a crash."""
    try:
        with open(f"{leaf}/cgroup.procs", "w") as handle:
            handle.write(str(pid))
    except OSError:
        pass


def _container_kill(leaf: str) -> None:
    """Kill every process in the container leaf — the boundary half of
    each sweep pass. Membership is fixed at fork and survives
    reparenting, so this reaches what the descendant walk cannot: a tail
    fork born after the last capture, and a sleeper that is already
    init's child because its middleman exited before the deadline.
    ``cgroup.kill`` (one write, kernel 5.14 and later) is preferred; a
    leaf without the file is swept through ``cgroup.procs``, which
    lists the same membership."""
    try:
        with open(f"{leaf}/cgroup.kill", "w") as handle:
            handle.write("1")
    except OSError:
        pass
    try:
        with open(f"{leaf}/cgroup.procs") as handle:
            members = handle.read().split()
    except OSError:
        return
    for member in members:
        try:
            os.kill(int(member), signal.SIGKILL)
        except (OSError, ValueError):
            # Already dead, or not ours to signal: the kill's job is to
            # leave no survivor, not to report one.
            continue


def _container_close(leaf: str) -> None:
    """Remove the leaf. Bounded retries: a SIGKILLed member leaves the
    leaf only once it exits, and the exit lands asynchronously. A leaf
    still occupied after the budget — a process that outlived a clean
    child and was never in the kill's surface — is left standing:
    cleanup is best-effort and never waits on the recovery."""
    for _ in range(CONTAINER_REMOVALS):
        try:
            os.rmdir(leaf)
            return
        except FileNotFoundError:
            return
        except OSError:
            time.sleep(CONTAINER_REMOVAL_SETTLE)


def _descendants(root_pid: int) -> list:
    """Every live descendant of ``root_pid`` from the /proc parent
    chain.

    A grandchild that starts its own session (the sandbox shape:
    ``start_new_session``/``setsid``) leaves the child's process
    group, so ``killpg`` alone misses it; the descendant chain
    still identifies it until its parents are reaped. This is the
    supervisor's own small copy of the technique already in
    ``SandboxRunner._descendants`` — no import from the controller
    stack, so the supervisor's only heads stay the two recorded
    subprocesses."""
    children: Dict[int, list] = {}
    try:
        entries = os.listdir("/proc")
    except OSError:
        return []
    for entry in entries:
        if not entry.isdigit():
            continue
        try:
            with open(f"/proc/{entry}/stat", "rb") as handle:
                data = handle.read().decode(errors="replace")
        except OSError:
            continue
        try:
            parent = int(data.rsplit(")", 1)[1].split()[1])
        except (IndexError, ValueError):
            continue
        children.setdefault(parent, []).append(int(entry))
    found = []
    stack = [root_pid]
    while stack:
        for child in children.get(stack.pop(), ()):
            found.append(child)
            stack.append(child)
    return found


@dataclasses.dataclass
class ChildResult:
    """The reaped child (``scripts/stress_run.py``'s ``ChildResult``
    discipline, minus the workload: the supervisor runs exactly one
    child). ``exit_code`` is ``None`` when the deadline killed the
    child — it produced no code — and ``document`` is ``None`` when the
    child was lost: no JSON document on stdout."""

    exit_code: Optional[int]
    document: Optional[Dict[str, Any]]
    stderr: str
    wall_seconds: float
    timed_out: bool = False


def run_child(budget: float) -> ChildResult:
    """Launch the one child and own its wall-clock deadline.

    ``start_new_session`` puts the child in its own session — and thus
    its own process group — so the expiry kill reaches the child and
    everything it started in that group with one ``killpg``. The child
    is forked into a container leaf as well: it moves itself in between
    fork and exec (``_container_join``), so every process it forks
    inherits the leaf and the kill's container half
    (``_container_kill``) reaches what the group cannot hold and the
    walk cannot see — a tail fork born after the last capture, and a
    sleeper already adopted by init because its middleman exited before
    the deadline. Because a process forked between the chain capture
    and the group kill would otherwise be adopted out of the tree the
    moment the group kill lands, the group is frozen first (SIGSTOP) and
    the descendant sweep then loops capture-kill-sweep to quiescence —
    the walk reaches what left the group (a sandboxed grandchild started
    in its own session) as well as any fork that raced the capture — so
    the whole process tree dies with the deadline. stdout is captured
    for the JSON document; stderr is the recorded head's narration,
    returned for the supervisor to forward.
    """
    started = time.monotonic()
    container = _container_open()
    if container is None:
        # Measured, not assumed: no cgroup v2 delegation here, so no
        # container boundary exists and the expiry kill's boundary half
        # cannot run. The sweep stands alone — said once, on stderr, so
        # the degraded surface is never silent.
        print(
            "[supervise] no cgroup v2 delegation: the container boundary"
            " is unavailable, the expiry sweep stands alone",
            file=sys.stderr,
            flush=True,
        )
    process = subprocess.Popen(
        [sys.executable, CHILD, "--json"],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
        start_new_session=True,
        preexec_fn=(lambda: _container_join(container))
        if container is not None
        else None,
    )
    try:
        if container is not None:
            _container_adopt(container, process.pid)
        try:
            stdout, stderr = process.communicate(timeout=budget)
        except subprocess.TimeoutExpired:
            # The deadline owns the child: kill the whole process tree, to
            # quiescence. The descendant chain is captured (the /proc
            # parent-chain walk) so a process that left the child's group —
            # a sandboxed grandchild started in its own session — is still
            # identified after the group kill misses it. A single
            # capture-then-signal would miss a process forked between the
            # capture and the group kill, and once the group kill lands that
            # process's parent is gone and it is adopted out of the tree,
            # where no later walk reaches it. So the child's process group
            # is frozen first — SIGSTOP stops the in-group forker, so the
            # capture below is stable — and the sweep then loops
            # capture-kill-sweep until a capture comes back empty, up to a
            # small bounded number of passes. Each pass captures the
            # descendant chain, kills the group with one killpg (the fast
            # path for the in-group shape), SIGKILLs every captured
            # descendant that survived the group kill (the structural half;
            # the technique already in SandboxRunner._kill), and kills the
            # container leaf (the boundary half: cgroup membership is fixed
            # at fork and is not changed by reparenting, so the leaf still
            # holds what the walk moved past — a forker that itself escaped
            # the group, re-opening the capture window every pass, and a
            # predecessor-exited sleeper already adopted by init). The
            # settle between passes lets a fork from the previous pass
            # appear in /proc before the re-capture. The budget bounds the
            # loop, not the kill: every pass still signals every captured
            # descendant and kills the container. What lies outside both
            # surfaces is only what the child never forked — a process some
            # other session started was never in the tree and is not the
            # supervisor's to kill.
            try:
                os.killpg(process.pid, signal.SIGSTOP)
            except ProcessLookupError:
                # The group vanished between the deadline and the kill: the
                # child is already dead, and the deadline still fired.
                pass
            # Let the freeze land before the first capture, so a fork that
            # raced the signal cannot slip past the stable capture.
            time.sleep(SWEEP_SETTLE)
            for _ in range(SWEEP_PASSES):
                descendants = _descendants(process.pid)
                try:
                    os.killpg(process.pid, signal.SIGKILL)
                except ProcessLookupError:
                    pass
                for pid in descendants:
                    try:
                        os.kill(pid, signal.SIGKILL)
                    except (ProcessLookupError, PermissionError):
                        # Already dead, or not ours to signal: the sweep's
                        # job is to leave no survivor, not to report one.
                        continue
                if container is not None:
                    _container_kill(container)
                if not descendants:
                    break
                time.sleep(SWEEP_SETTLE)
            stdout, stderr = process.communicate()
            print(
                f"[supervise] child killed at the {budget:g}s wall budget",
                file=sys.stderr,
                flush=True,
            )
            return ChildResult(
                exit_code=None,
                document=None,
                stderr=stderr or "",
                wall_seconds=time.monotonic() - started,
                timed_out=True,
            )
        wall = time.monotonic() - started
        document = None
        try:
            document = json.loads(stdout)
        except (json.JSONDecodeError, TypeError):
            document = None
        return ChildResult(
            exit_code=process.returncode,
            document=document,
            stderr=stderr or "",
            wall_seconds=wall,
        )
    finally:
        if container is not None:
            _container_close(container)


def run_recovery() -> int:
    """Drive the recorded recovery exactly once, as a second subprocess
    under the same environment — the run's recorded authorisation rides
    the environment the operator set; the supervisor never invents one.
    The recovery's narration (stdout and stderr) is forwarded to this
    supervisor's stderr, so stdout stays the supervisor's alone."""
    print(
        "[supervise] recovery: scripts/smoke_run.py --resume",
        file=sys.stderr,
        flush=True,
    )
    process = subprocess.run(
        [sys.executable, CHILD, "--resume"],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
    )
    for stream in (process.stdout, process.stderr):
        if stream:
            sys.stderr.write(stream)
            sys.stderr.flush()
    return process.returncode


def parse_budget(raw: str) -> float:
    """The wall-clock budget in seconds: a positive, finite number."""
    try:
        value = float(raw)
    except ValueError:
        raise argparse.ArgumentTypeError(
            f"budget {raw!r} is not a number of seconds"
        ) from None
    if not math.isfinite(value) or value <= 0:
        raise argparse.ArgumentTypeError(
            f"budget {raw!r} is not a positive number of seconds"
        )
    return value


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--budget",
        required=True,
        type=parse_budget,
        help="the child's wall-clock budget in seconds",
    )
    args = parser.parse_args()

    # The workdir is read from the same ORCHESTRATOR_WORKDIR the child
    # uses, so supervisor and child cannot disagree about which workdir
    # the recovery reads. A missing workdir is misuse: fail closed.
    workdir = os.environ.get("ORCHESTRATOR_WORKDIR", "").strip()
    if not workdir:
        print(
            "error: ORCHESTRATOR_WORKDIR is not set: the supervisor needs"
            " the workdir the child runs in (the same one the recorded"
            " --resume recovery reads)",
            file=sys.stderr,
        )
        return MISUSE_EXIT

    child = run_child(args.budget)
    if child.stderr:
        sys.stderr.write(child.stderr)
        sys.stderr.flush()
    print(
        f"[supervise] child exit={child.exit_code}"
        f" wall={child.wall_seconds:.2f}s timed_out={child.timed_out}",
        file=sys.stderr,
        flush=True,
    )

    # Controller death: the child was killed at the deadline, was lost
    # (no JSON document on stdout), or exited nonzero. Drive the
    # recorded recovery exactly once. A clean child — exit 0 with a
    # document — runs no recovery.
    recovered = False
    code = child.exit_code
    if child.timed_out or child.exit_code != 0 or child.document is None:
        recovery_code = run_recovery()
        if recovery_code != 0:
            # The recovery itself failed: fail closed with the reason on
            # stderr and no JSON document claimed as success.
            print(
                f"error: the recorded recovery (--resume) failed with exit"
                f" code {recovery_code}; the workdir {workdir} is not"
                " settled",
                file=sys.stderr,
            )
            return recovery_code
        recovered = True
        code = recovery_code

    # The supervisor exits 0 only when the child exited 0 with a
    # document and no recovery ran; otherwise it exits the child's code
    # (or 1 when the child produced none — killed at the deadline, or
    # exited 0 without a document).
    if not recovered and child.exit_code == 0 and child.document is not None:
        supervisor_exit = 0
    elif child.timed_out:
        supervisor_exit = 1
    elif child.exit_code != 0:
        supervisor_exit = child.exit_code
    else:
        supervisor_exit = 1

    print(
        json.dumps(
            {
                "workdir": workdir,
                "code": code,
                "timed_out": child.timed_out,
                "recovered": recovered,
                "child_wall_seconds": round(child.wall_seconds, 3),
            }
        )
    )
    return supervisor_exit


if __name__ == "__main__":
    raise SystemExit(main())
