# SPDX-License-Identifier: AGPL-3.0-or-later
"""Axis 2 robustness/load regressions.

Pins the operational-envelope behaviour this milestone must keep under real
provider pressure: bounded rate-limit backoff in the client, fail-closed
handling of 5xx/timeouts/malformed replies, repair-loop retries as fresh billed
attempts that settle, no credential bytes in any store, and no unhandled
exception escaping a trial. The parallel hostile sweep itself is driven by
``scripts/stress_run.py`` (also run as a slow test here).
"""
from __future__ import annotations

import json
import os
import subprocess
import sys
import time
from typing import Sequence

import pytest

import provider_stub
import smoke_run
import stress_run
from helpers import build_run_config, experiment_authorization

from orchestrator import experiment, model_adapter, sandbox, secrets, spend
from orchestrator.controller import Controller, ExecutorFailed
from orchestrator.executor import ScriptedExecutor
from orchestrator.model_adapter import ModelAdapter
from orchestrator.store import EventStore

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SECRET = "axis2-echo-secret-value"
_MODEL_LIMITS = {
    "root_model_input_tokens": 32768,
    "root_model_output_tokens": 8192,
}


def _client(stub, scenario, **overrides):
    credential = secrets.Credential(
        credential_id="axis2-cred",
        scope="MODEL_ENDPOINT",
        secret=SECRET,
        expires_at=time.time() + 3600,
    )
    kwargs = dict(
        base_url=stub.base_url,
        path=f"/s/{scenario}",
        credential=credential,
        units="USD",
        price_input_per_mtok=0.3,
        price_output_per_mtok=1.2,
        max_input_tokens=4000,
        timeout=5.0,
        max_reply_bytes=1_048_576,
    )
    kwargs.update(overrides)
    return smoke_run.OpenAICompatibleClient(
        smoke_run.experiment.ModelProfile(
            model_id="stub-model", model_version="1", temperature=0.0, top_p=1.0,
            max_output_tokens=1024, input_tokens_per_call=4000,
            output_tokens_per_call=1024, cost_per_call=5.0,
        ),
        **kwargs,
    )


def _request(**overrides):
    request = {
        "prompt": "review this",
        "prompt_hash": "sha256:x",
        "attempt_id": "a1",
        "node_id": "n1",
        "call_id": "a1",
        "call_key": "n1",
    }
    request.update(overrides)
    return request


def test_stub_catalogues_hostile_replies_and_faults():
    for scenario in (
        "truncated", "oversized_body", "oversized_reply", "bad_verdict",
        "slow_drip", "drip_forever", "echo_secret", "prose", "empty",
        "bad_json_body", "deep_body", "overrun_unparseable", "bad_verdict_huge",
    ):
        assert scenario in provider_stub.SCENARIOS
    for scenario in ("http_429", "http_500", "http_503", "stall",
                     "flaky_5xx", "flaky_429", "reject_format_once"):
        assert scenario in provider_stub.SCENARIOS


def test_rate_limit_backoff_recovers_within_one_attempt():
    with provider_stub.StubServer() as stub:
        client = _client(stub, "flaky_429?fails=2&retry_after=0",
                         timeout=5.0, max_rate_limit_retries=2,
                         rate_limit_backoff=0.01)
        completion = client.complete(_request())
        assert completion.outputs[0].body["verdict"] == "PASS"
        # Three HTTP requests, one completion: the backoff re-sent the rejected
        # request inside the single per-call deadline.
        assert stub.state.requests == 3


def test_rate_limit_retries_are_bounded_then_fail_closed():
    with provider_stub.StubServer() as stub:
        client = _client(stub, "http_429?retry_after=0", timeout=5.0,
                         max_rate_limit_retries=1, rate_limit_backoff=0.01)
        with pytest.raises(RuntimeError):
            client.complete(_request())
        assert stub.state.requests == 2
        with provider_stub.StubServer() as bare:
            none = _client(bare, "http_429?retry_after=0", timeout=5.0,
                           max_rate_limit_retries=0)
            with pytest.raises(RuntimeError):
                none.complete(_request())
            assert bare.state.requests == 1


def test_5xx_fails_closed_with_no_client_retry():
    with provider_stub.StubServer() as stub:
        client = _client(stub, "http_500", timeout=5.0,
                         max_rate_limit_retries=2, rate_limit_backoff=0.01)
        with pytest.raises(RuntimeError):
            client.complete(_request())
        # An ambiguous, possibly billed outcome is never re-sent in secret: any
        # further call must be a fresh recorded attempt through the repair loop.
        assert stub.state.requests == 1


def test_stall_is_bounded_by_the_call_deadline():
    with provider_stub.StubServer() as stub:
        client = _client(stub, "stall", timeout=0.5)
        started = time.monotonic()
        with pytest.raises(RuntimeError):
            client.complete(_request())
        assert time.monotonic() - started < 3.0


def test_truncated_completion_fails_closed():
    with provider_stub.StubServer() as stub:
        client = _client(stub, "truncated")
        with pytest.raises(RuntimeError, match="truncated"):
            client.complete(_request())


def test_invalid_verdict_fails_closed():
    with provider_stub.StubServer() as stub:
        client = _client(stub, "bad_verdict")
        with pytest.raises(RuntimeError):
            client.complete(_request())
        client = _client(stub, "prose")
        with pytest.raises(RuntimeError):
            client.complete(_request())


def test_oversized_reply_fails_closed():
    with provider_stub.StubServer() as stub:
        client = _client(stub, "oversized_reply?size=200000",
                         max_reply_bytes=65536)
        with pytest.raises(RuntimeError):
            client.complete(_request())
        assert stub.state.requests == 1


def test_format_rejection_resends_within_the_same_deadline():
    """The one non-billed re-send (a 400 on ``response_format``) must actually
    re-issue the request with the same deadline, not crash the call."""
    with provider_stub.StubServer() as stub:
        client = _client(stub, "reject_format_once")
        completion = client.complete(_request())
        assert completion.outputs[0].body["verdict"] == "PASS"
        assert stub.state.requests == 2


# ---------------------------------------------------------------------------
# Credential redaction at the admission boundary (echo-secret hostile class)
# ---------------------------------------------------------------------------


def test_echoed_secret_is_redacted_before_admission(tmp_path):
    source = secrets.SecretsSource()
    credential = source.issue(
        "MODEL_ENDPOINT",
        credential_id="axis2-echo",
        ttl_seconds=3600,
        secret=SECRET,
    )
    with provider_stub.StubServer() as stub:
        client = smoke_run.OpenAICompatibleClient(
            smoke_run.experiment.ModelProfile(
                model_id="stub-model", model_version="1", temperature=0.0,
                top_p=1.0, max_output_tokens=1024, input_tokens_per_call=4000,
                output_tokens_per_call=1024, cost_per_call=5.0,
            ),
            base_url=stub.base_url,
            path="/s/echo_secret",
            credential=credential,
            units="USD",
            price_input_per_mtok=0.3,
            price_output_per_mtok=1.2,
            max_input_tokens=4000,
            timeout=5.0,
        )
        store = EventStore(str(tmp_path / "echo.db"))
        controller = Controller(
            store,
            ScriptedExecutor({}),
            model_executor=ModelAdapter(client, model_id="stub-model",
                                        model_version="1"),
            secrets_source=source,
            executor_environment={"base_commit": "HEAD", "timeout_seconds": 30.0},
        )
        root = controller.submit_run(
            build_run_config("RUN-ECHO", limits=_MODEL_LIMITS)
        )
        controller.run_model_review("RUN-ECHO", root, 5.0)
        events = controller.store.history("RUN-ECHO")
        store.close()
    admitted = [e for e in events if e.event_type == "MODEL_OUTPUT_ADMITTED"]
    assert admitted, "a well-formed evaluation must be admitted (untrusted)"
    rationale = admitted[0].payload["output"]["rationale"]
    assert SECRET not in rationale
    assert rationale.startswith("the key is redacted:sha256:")
    blob = (tmp_path / "echo.db").read_bytes()
    assert SECRET.encode() not in blob


# ---------------------------------------------------------------------------
# Attempt lineage: a repair-loop retry is a fresh, billed, settled attempt
# ---------------------------------------------------------------------------


def test_plan_attempt_derives_the_retry_cause_from_state(tmp_path):
    client = _FlakyClient(_completion(), failures=10 ** 6)
    store = EventStore(str(tmp_path / "cause.db"))
    controller = Controller(
        store,
        ScriptedExecutor({}),
        model_executor=ModelAdapter(client, model_id="model-x", model_version="1"),
        executor_environment={"base_commit": "HEAD", "timeout_seconds": 30.0},
    )
    try:
        root = controller.submit_run(
            build_run_config("RUN-CAUSE", limits=_MODEL_LIMITS)
        )
        first = controller.plan_attempt("RUN-CAUSE", root, 5.0, action_type="REVIEW")
        assert store.state("RUN-CAUSE").attempts[first].cause == "INITIAL"
        with pytest.raises(ExecutorFailed):
            controller.dispatch_attempt("RUN-CAUSE", first, root)
        attempt_id = controller.plan_attempt(
            "RUN-CAUSE", root, 5.0, action_type="REVIEW"
        )
        assert store.state("RUN-CAUSE").attempts[attempt_id].cause == "RETRY"
        # The cause cannot be asserted by a caller at all (Axis 2 review F6).
        with pytest.raises(TypeError):
            controller.plan_attempt(
                "RUN-CAUSE", root, 5.0, action_type="REVIEW", cause="INITIAL"
            )
    finally:
        store.close()


class _FlakyClient(model_adapter.ModelClient):
    """Fails the first completion and succeeds afterwards. Declares a maximum
    charge safely under the driver's per-attempt commitment."""

    def __init__(self, completion, failures=1):
        self.completion = completion
        self.failures = failures
        self.calls = 0

    def max_cost(self, request):
        return 0.5

    def complete(self, request):
        self.calls += 1
        if self.calls <= self.failures:
            raise RuntimeError("provider HTTP 500")
        return self.completion


def _completion():
    return model_adapter.ModelCompletion(
        outputs=(model_adapter.ModelOutput(
            "SEMANTIC_EVALUATION",
            {"subject_reference": "x", "verdict": "PASS", "rationale": "ok"},
        ),),
        cost=0.01,
        units="COST_UNITS",
        model_id="model-x",
        model_version="1",
        input_tokens=10,
        output_tokens=5,
    )


def test_repair_loop_retry_is_a_fresh_billed_settled_attempt(tmp_path):
    client = _FlakyClient(_completion(), failures=1)
    store = EventStore(str(tmp_path / "retry.db"))
    controller = Controller(
        store,
        ScriptedExecutor({}),
        model_executor=ModelAdapter(client, model_id="model-x", model_version="1"),
        executor_environment={"base_commit": "HEAD", "timeout_seconds": 30.0},
    )
    try:
        root = controller.submit_run(
            build_run_config("RUN-RETRY", limits=_MODEL_LIMITS)
        )
        with pytest.raises(ExecutorFailed):
            controller.run_model_review("RUN-RETRY", root, 5.0)
        controller.record_failure(
            "RUN-RETRY", root,
            action_type="REVIEW",
            diagnostic_category="EXECUTOR_FAILED",
            evidence_digest="sha256:executor-failed",
        )
        attempt_id, response = controller.run_model_review("RUN-RETRY", root, 5.0)
        assert response["model_outputs"]
        state = controller.store.state("RUN-RETRY")
        review = [a for a in state.attempts.values() if a.action_type == "REVIEW"]
        assert len(review) == 2
        causes = sorted(a.cause for a in review)
        assert causes == ["INITIAL", "RETRY"]
        assert review[0].attempt_id != review[1].attempt_id
        # Each retry is billed and settles: the failed call is settled
        # conservatively at its commitment and the re-issue reconciles its
        # reported actual. Nothing stays provisional or open.
        for attempt in review:
            assert attempt.accounting_state == "FINALLY_SETTLED"
            assert attempt.actual_cost is not None
        charged = state.ledger.total_charged
        assert charged == pytest.approx(5.0 + 0.01)
    finally:
        store.close()


# ---------------------------------------------------------------------------
# Trial-level: no unhandled exception escapes a trial (slow)
# ---------------------------------------------------------------------------


def _require_sandbox():
    caps = sandbox.capabilities()
    if not caps.profile_available:
        pytest.fail(f"NAMESPACE_V1 unavailable (fail-closed gate): {caps.unavailable_reason()}")


def _run_one_trial(workdir, arm, seed, client_factory):
    target = experiment.build_fixed_target(workdir)
    oracle_manifest = experiment.build_hidden_oracle(target)
    ledger = spend.open_spend_ledger(workdir, experiment_authorization())
    experiment.set_model_client_factory(client_factory)
    try:
        return experiment.run_trial(
            target,
            oracle_manifest,
            experiment.placeholder_limits(),
            experiment.MODEL_PROFILE,
            arm,
            seed,
            workdir,
            spend_ledger=ledger,
        )
    finally:
        experiment.set_model_client_factory(None)


@pytest.mark.slow
def test_concurrent_arm_survives_provider_failures_without_escaping(tmp_path):
    """A provider failure inside a *batched* review must not crash the trial:
    the batch degrades to the bounded single-leaf repair path and the trial is
    graded from its recorded outcomes (Axis 2)."""
    _require_sandbox()

    def failing_factory(profile, seed):
        return _FlakyClient(_completion(), failures=10 ** 6)

    metrics = _run_one_trial(
        str(tmp_path), experiment.ARM_RECURSIVE_CONCURRENT, 1003, failing_factory
    )
    assert metrics.classification
    # The trial settled every attempt instead of leaking one on the way out.
    trial_root = os.path.join(str(tmp_path), "trials",
                              f"{experiment.ARM_RECURSIVE_CONCURRENT}-1003")
    store = EventStore(os.path.join(trial_root, "events.db"))
    try:
        state = store.state(metrics.run_id)
    finally:
        store.close()
    assert state.attempts
    for attempt in state.attempts.values():
        assert attempt.accounting_state == "FINALLY_SETTLED"
    ledger = json.loads(
        (tmp_path / "experiment-spend.json").read_text()
    )
    assert ledger["in_flight"] == {}
    assert ledger["committed"] == pytest.approx(
        sum(record["actual"] for record in ledger["trials"].values())
    )


@pytest.mark.slow
def test_flaky_provider_retry_settles_through_the_spend_ledger(tmp_path):
    """The bounded repair loop re-issues a real call as a fresh RETRY attempt
    that is billed and settles; the experiment ledger closes clean."""
    _require_sandbox()
    source = secrets.SecretsSource()
    source.issue("MODEL_ENDPOINT", credential_id="axis2-trial-cred",
                 ttl_seconds=3600, secret=SECRET)
    experiment.set_secrets_source(source)
    try:
        with provider_stub.StubServer() as stub:
            def flaky_factory(profile, seed):
                return _client(stub, "flaky_5xx?fails=1")

            metrics = _run_one_trial(
                str(tmp_path), experiment.ARM_DIRECT, 1001, flaky_factory
            )
    finally:
        experiment.set_secrets_source(None)
    assert metrics.classification
    assert metrics.retries >= 1
    trial_root = os.path.join(str(tmp_path), "trials", f"{experiment.ARM_DIRECT}-1001")
    store = EventStore(os.path.join(trial_root, "events.db"))
    try:
        state = store.state(metrics.run_id)
        history = store.history(metrics.run_id)
    finally:
        store.close()
    planned = [
        event for event in history
        if event.event_type == "ATTEMPT_PLANNED"
        and event.payload.get("action_type") == "REVIEW"
    ]
    assert [event.payload["cause"] for event in planned] == ["INITIAL", "RETRY"]
    for attempt in state.attempts.values():
        assert attempt.accounting_state == "FINALLY_SETTLED"
        assert attempt.actual_cost is not None
    assert state.ledger.total_charged > 0
    ledger = json.loads((tmp_path / "experiment-spend.json").read_text())
    assert ledger["in_flight"] == {}
    record = ledger["trials"][metrics.run_id]
    assert record["actual"] == pytest.approx(state.ledger.total_charged)
    # No credential bytes in any of the trial's stores.
    for path in (tmp_path / "trials").rglob("*"):
        if path.is_file():
            assert SECRET.encode() not in path.read_bytes()


@pytest.mark.slow
@pytest.mark.timeout(300)
def test_axis2_stress_sweep_meets_the_acceptance_criteria(tmp_path):
    """The parallel hostile sweep: several smoke_run processes with their own
    workdir/authorization against the stub, checked against every Axis 2
    acceptance criterion."""
    _require_sandbox()
    root = tmp_path / "stress"
    report_path = tmp_path / "stress-report.json"
    process = subprocess.run(
        [
            sys.executable,
            os.path.join(ROOT, "scripts", "stress_run.py"),
            "--scale", "tiny",
            "--skip-conformance",
            "--root", str(root),
            "--json", str(report_path),
        ],
        capture_output=True,
        text=True,
        timeout=600,
    )
    assert report_path.exists(), process.stderr[-2000:]
    report = json.loads(report_path.read_text())
    failed = [check for check in report["checks"] if not check["ok"]]
    assert process.returncode == 0, json.dumps(failed, indent=2)
    assert not failed
    assert all(check["ok"] for check in report["checks"])
    names = {check["check"] for check in report["checks"]}
    assert names >= {
        "no_unhandled_exception",
        "no_pass_from_hostile",
        "failures_recorded",
        "failures_settled",
        "deadlines_bound",
        "stores_complete",
        "ledger_consistent",
        "retry_fresh_attempts",
        "transcripts_sound",
        "no_credential_bytes",
    }


# ---------------------------------------------------------------------------
# Axis 2 review regressions (F1-F6)
# ---------------------------------------------------------------------------


def _fake_workdir(root: Path, name: str, trial_names: Sequence[str]) -> Path:
    """A workdir whose spend ledger records ``trial_names`` and whose trials/
    directory holds one readable store per name in ``present``."""
    workdir = root / name
    (workdir / "trials").mkdir(parents=True)
    ledger = {
        "committed": 0.0,
        "in_flight": {},
        "trials": {f"EXP-{trial}": {"actual": 0.0, "status": "SETTLED"}
                   for trial in trial_names},
    }
    (workdir / "experiment-spend.json").write_text(json.dumps(ledger))
    return workdir


def _add_store(workdir: Path, trial: str, *, corrupt: bool = False) -> None:
    trial_dir = workdir / "trials" / trial
    trial_dir.mkdir(parents=True, exist_ok=True)
    arm, seed = trial.rsplit("-", 1)
    (trial_dir / "identity.json").write_text(json.dumps({"arm": arm, "seed": int(seed)}))
    if corrupt:
        (trial_dir / "events.db").write_bytes(b"not a sqlite database")
    else:
        EventStore(str(trial_dir / "events.db")).close()


def _child_result(name: str, **overrides):
    workload = stress_run.Workload(
        name=name, kind=stress_run.FAULT, path="/s/x", seed=1003, secret="s",
    )
    return stress_run.ChildResult(
        workload=workload, exit_code=0, document={"trials": []}, stderr="",
        wall_seconds=1.0, **overrides,
    )


def test_stores_complete_rejects_a_missing_or_unreadable_store(tmp_path):
    """Axis 2 review F1: a deleted or corrupt trial store must fail the stress
    checks instead of being silently skipped (vacuous pass)."""
    workdir = _fake_workdir(tmp_path, "w", ["DIRECT-1003", "RECURSIVE_LAZY-1003"])
    _add_store(workdir, "DIRECT-1003")
    results = [_child_result("w")]
    missing = stress_run.check_stores_complete(results, tmp_path)
    assert not missing.ok
    assert any("RECURSIVE_LAZY-1003" in finding for finding in missing.findings)
    _add_store(workdir, "RECURSIVE_LAZY-1003")
    complete = stress_run.check_stores_complete(results, tmp_path)
    assert complete.ok
    (workdir / "trials" / "DIRECT-1003" / "events.db").write_bytes(b"corrupted")
    corrupt = stress_run.check_stores_complete(results, tmp_path)
    assert not corrupt.ok
    assert any("unreadable" in finding for finding in corrupt.findings)


def test_workdir_lock_is_exclusive_per_workdir(tmp_path):
    """Axis 2 review F2: a resume must never run against a live writer."""
    with smoke_run.workdir_lock(str(tmp_path)):
        with pytest.raises(SystemExit, match="locked"):
            with smoke_run.workdir_lock(str(tmp_path)):
                pass
    with smoke_run.workdir_lock(str(tmp_path)):
        pass


def test_resume_refuses_when_the_workdir_is_locked(tmp_path, monkeypatch):
    workdir = tmp_path / "wd"
    (workdir / "trials").mkdir(parents=True)
    monkeypatch.setenv("ORCHESTRATOR_WORKDIR", str(workdir))
    monkeypatch.setenv("ORCHESTRATOR_OWNER", "axis2-test")
    monkeypatch.setenv("ORCHESTRATOR_CAP", "500")
    settings = smoke_run.load_settings()
    with smoke_run.workdir_lock(str(workdir)):
        with pytest.raises(SystemExit, match="locked"):
            smoke_run.resume(settings)


def test_deadlines_bound_reports_a_killed_child():
    """Axis 2 review F3: the kill signal must be owned by the deadline check."""
    workload = stress_run.Workload(
        name="w", kind=stress_run.FAULT, path="/s/stall", seed=1003,
        secret="s", wall_budget=10.0,
    )
    killed = stress_run.ChildResult(workload, -1, None, "killed", 10.0, timed_out=True)
    over = stress_run.ChildResult(workload, 0, {}, "", 12.0)
    ok = stress_run.ChildResult(workload, 0, {}, "", 5.0)
    result = stress_run.check_deadlines_bound([killed, over, ok])
    assert not result.ok
    assert len(result.findings) == 2
    assert stress_run.check_deadlines_bound([ok]).ok


def test_recover_settles_reserved_stragglers_on_a_live_run(tmp_path):
    """Axis 2 review F5: a PLANNED/RESERVED straggler is cancelled and settled
    by recovery regardless of the run outcome, so the sweep invariant cannot be
    left broken."""
    store = EventStore(str(tmp_path / "straggler.db"))
    try:
        controller = Controller(store, ScriptedExecutor({}))
        root = controller.submit_run(build_run_config("RUN-STRAGGLER"))
        attempt_id = controller.plan_attempt("RUN-STRAGGLER", root, 5.0)
        state = store.state("RUN-STRAGGLER")
        assert state.attempts[attempt_id].state == "RESERVED"
        assert state.attempts[attempt_id].accounting_state == "OPEN"
        assert state.outcome is None
        assert controller.recover("RUN-STRAGGLER") == [attempt_id]
        state = store.state("RUN-STRAGGLER")
        attempt = state.attempts[attempt_id]
        assert attempt.state == "CANCELLED"
        assert attempt.accounting_state == "FINALLY_SETTLED"
        assert attempt.actual_cost == 0.0
    finally:
        store.close()


class _FailNthClient(model_adapter.ModelClient):
    """Fails exactly one completion (1-based index), succeeds otherwise."""

    def __init__(self, completion, fail_on: int):
        self.completion = completion
        self.fail_on = fail_on
        self.calls = 0

    def max_cost(self, request):
        return 0.5

    def complete(self, request):
        self.calls += 1
        if self.calls == self.fail_on:
            raise RuntimeError("provider HTTP 500")
        return self.completion


@pytest.mark.slow
@pytest.mark.timeout(300)
def test_batch_degrade_does_not_drop_reviewed_siblings(tmp_path):
    """Axis 2 review F4: a sibling whose review already succeeded is executed
    by the degrade path instead of being silently dropped to READY."""
    _require_sandbox()
    client = _FailNthClient(_completion(), fail_on=2)

    def factory(profile, seed):
        return client

    metrics = _run_one_trial(
        str(tmp_path), experiment.ARM_RECURSIVE_CONCURRENT, 1001, factory
    )
    trial_root = os.path.join(str(tmp_path), "trials",
                              f"{experiment.ARM_RECURSIVE_CONCURRENT}-1001")
    store = EventStore(os.path.join(trial_root, "events.db"))
    try:
        state = store.state(metrics.run_id)
    finally:
        store.close()
    # The first reviewed sibling executed with its original review; the failed
    # node re-reviewed as a RETRY and completed too.
    for leaf in ("parse", "render", "report"):
        assert state.nodes[f"{metrics.run_id}-{leaf}"].state == "ACCEPTED"
    assert metrics.retries >= 1
