# SPDX-License-Identifier: AGPL-3.0-or-later
"""Crash-injection matrix over representative workflows.

For each workflow, the events it commits are discovered once, then the
workflow is re-run crashing before commit at every event index. After each
crash the store is reopened, ``recover`` is run, and the result must satisfy the
invariant library plus the recovery invariants: no active liability without an
attempt, no child budget without its node, and no redispatch of paid work.
Recovery must also be idempotent.

Crashes are bounded to short workflows because the matrix is quadratic in
workflow length.
"""
from __future__ import annotations

import pytest

from orchestrator.controller import Controller, WorkflowError
from orchestrator.executor import AttemptTimedOut, ScriptedExecutor
from orchestrator.model import TOLERANCE
from orchestrator.store import CrashInjected, EventStore

from helpers import build_run_config
from invariants import assert_recovery_invariants

EXPECTED = (CrashInjected, AttemptTimedOut, WorkflowError)


def _completed_script(_request):
    return {
        "self_reported_status": "COMPLETED",
        "artefacts": [{"type": "GIT_CANDIDATE", "content_hash": "sha256:matrix"}],
        "observations": [{"type": "COMMAND_RESULT", "command_id": "BUILD", "exit_code": 0}],
        "cost_report": {"committed": 5, "actual": 5, "units": "USD"},
    }


def _timeout_script(_request):
    return {"timeout": True}


class _CountingExecutor(ScriptedExecutor):
    def __init__(self, scripts):
        super().__init__(scripts)
        self.dispatch_calls = 0

    def dispatch(self, request):
        self.dispatch_calls += 1
        return super().dispatch(request)


def _accept_workflow(controller: Controller, run_id: str) -> None:
    root = controller.submit_run(build_run_config(run_id))
    attempt = controller.execute_attempt(run_id, root, 5.0)
    controller.validate_and_accept(run_id, root, {"checks": ["BUILD"]}, f"{attempt}-result")


def _decomposition_workflow(controller: Controller, run_id: str) -> None:
    config = build_run_config(run_id)
    config["root_contract"] = {
        **config["root_contract"],
        "acceptance_criteria": [{"id": "AC-001", "mandatory": True}],
    }
    root = controller.submit_run(config)
    child = f"{run_id}-child"
    controller.admit_derived_decomposition(
        run_id,
        root,
        "decomp-1",
        [
            {
                "node_id": child,
                "mandatory": True,
                "criterion_ids": ["AC-001"],
                "budget_allocation": {"MODEL_COST": 5.0},
                "contract": {
                    "description": "child",
                    "owned_scope": ["src/child/**"],
                    "acceptance_criteria": [{"id": "AC-001", "mandatory": True}],
                },
            }
        ],
        [{"criterion_id": "AC-001", "child_id": child, "mandatory_cover": True}],
    )


def _timeout_workflow(controller: Controller, run_id: str) -> None:
    root = controller.submit_run(build_run_config(run_id))
    attempt = controller.plan_attempt(run_id, root, 5.0)
    controller.dispatch_attempt(run_id, attempt, root)


def _cancel_workflow(controller: Controller, run_id: str) -> None:
    root = controller.submit_run(build_run_config(run_id))
    attempt = controller.plan_attempt(run_id, root, 5.0)
    controller.dispatch_attempt(run_id, attempt, root)
    controller.cancel_run(run_id, root)


def _cancel_after_observation_workflow(controller: Controller, run_id: str) -> None:
    root = controller.submit_run(build_run_config(run_id))
    attempt = controller.plan_attempt(run_id, root, 5.0)
    controller.dispatch_attempt(run_id, attempt, root)
    controller._emit(
        run_id,
        "EXECUTOR_OBSERVATION_ADMITTED",
        {
            "attempt_id": attempt,
            "observation": {
                "cost_report": {"committed": 5, "actual": 9.0, "units": "USD"}
            },
        },
        f"observe-{attempt}",
        source="executor",
    )
    controller._emit(
        run_id,
        "ATTEMPT_EXECUTION_ENDED_UNRECONCILED",
        {"attempt_id": attempt},
        f"end-{attempt}",
    )
    controller.cancel_run(run_id, root)


def _root_replacement_workflow(controller: Controller, run_id: str) -> None:
    config = build_run_config(run_id)
    root = controller.submit_run(config)
    controller.execute_attempt(run_id, root, 5.0)
    new_root = f"{run_id}-replacement"
    contract = dict(config["root_contract"])
    contract.update(
        {"budget_id": f"{new_root}-budget", "budgets": {"MODEL_COST": 5.0}}
    )
    controller.replace_root(
        run_id,
        root,
        new_root,
        contract,
        transferred_budgets=[
            {
                "from_budget_id": f"{run_id}-root-budget",
                "to_budget_id": f"{new_root}-budget",
                "amounts": {"MODEL_COST": 1.0},
            }
        ],
    )


def _budget_exhaustion_batch_workflow(controller: Controller, run_id: str) -> None:
    config = build_run_config(run_id)
    config["root_contract"] = {
        **config["root_contract"],
        "acceptance_criteria": [{"id": "AC-001", "mandatory": True}],
    }
    root = controller.submit_run(config)
    child = f"{run_id}-child"
    controller.admit_derived_decomposition(
        run_id,
        root,
        "decomp-1",
        [
            {
                "node_id": child,
                "mandatory": True,
                "criterion_ids": ["AC-001"],
                "budget_allocation": {"MODEL_COST": 3.0},
                "contract": {
                    "description": "child",
                    "owned_scope": ["src/child/**"],
                    "acceptance_criteria": [{"id": "AC-001", "mandatory": True}],
                },
            }
        ],
        [{"criterion_id": "AC-001", "child_id": child, "mandatory_cover": True}],
    )
    for node_id in (root, child):
        attempt = controller.plan_attempt(run_id, node_id, 2.0)
        controller.dispatch_attempt(run_id, attempt, node_id)
    for node_id in (root, child):
        controller._emit(
            run_id,
            "BUDGET_EXHAUSTION_RECORDED",
            {"node_id": node_id},
            f"exhaust-{node_id}",
        )


def _scripts_accept(run_id):
    return {f"{run_id}-root": _completed_script}


def _scripts_none(_run_id):
    return {}


def _scripts_timeout(run_id):
    return {f"{run_id}-root": _timeout_script}


def _scripts_root_and_child(run_id):
    return {f"{run_id}-root": _completed_script, f"{run_id}-child": _completed_script}


WORKFLOWS = (
    ("accept", _accept_workflow, _scripts_accept),
    ("decomposition", _decomposition_workflow, _scripts_none),
    ("timeout", _timeout_workflow, _scripts_timeout),
)

# A4: workflows the first matrix did not run. Slow tier.
EXTENDED_WORKFLOWS = (
    ("cancel", _cancel_workflow, _scripts_accept),
    ("cancel_after_observation", _cancel_after_observation_workflow, _scripts_accept),
    ("root_replacement", _root_replacement_workflow, _scripts_accept),
    ("budget_exhaustion_batch", _budget_exhaustion_batch_workflow, _scripts_root_and_child),
)


def _assert_no_laundered_actual(store: EventStore, run_id: str) -> None:
    """An attempt with a durable reported actual is finally settled at that
    actual or the commitment, never zero (SPEND review F2)."""
    state = store.state(run_id)
    for attempt in state.attempts.values():
        actual = attempt.actual_cost
        if actual is None or float(actual) <= 0.0:
            continue
        charged = state.ledger.charged.get(attempt.attempt_id, 0.0)
        commitment = (
            float(attempt.commitment) if attempt.commitment is not None else None
        )
        settles_at_actual = abs(charged - float(actual)) <= TOLERANCE
        settles_at_commitment = commitment is not None and abs(charged - commitment) <= TOLERANCE
        assert charged > 0.0 and (settles_at_actual or settles_at_commitment), (
            f"attempt {attempt.attempt_id} reported actual {actual} but was charged "
            f"{charged} (commitment {commitment})"
        )


def _discover_event_count(name, workflow, scripts_factory, tmp_path) -> int:
    run_id = f"MATRIX-{name}"
    store = EventStore(str(tmp_path / f"{name}-probe.db"))
    try:
        controller = Controller(store, ScriptedExecutor(scripts_factory(run_id)))
        try:
            workflow(controller, run_id)
        except EXPECTED:
            pass
        return len(store.history(run_id))
    finally:
        store.close()


def _crash_then_recover(name, index, workflow, scripts_factory, tmp_path) -> None:
    run_id = f"MATRIX-{name}"
    db_path = str(tmp_path / f"{name}-{index}.db")
    scripts = scripts_factory(run_id)

    store = EventStore(db_path)
    counter = {"n": 0}

    def crash(_event):
        if counter["n"] == index:
            raise CrashInjected(f"crash before event {index}")
        counter["n"] += 1

    store.crash_before_commit = crash
    crashed = False
    try:
        controller = Controller(store, ScriptedExecutor(scripts))
        try:
            workflow(controller, run_id)
        except CrashInjected:
            crashed = True
        except EXPECTED:
            pass
    finally:
        store.close()
    assert crashed, f"{name}: crash at index {index} was never reached"

    reopened = EventStore(db_path)
    recovery_executor = _CountingExecutor(scripts)
    try:
        controller = Controller(reopened, recovery_executor)
        controller.recover(run_id)
        assert_recovery_invariants(reopened.state(run_id))
        assert recovery_executor.dispatch_calls == 0, "recovery redispatched work"
        _assert_no_laundered_actual(reopened, run_id)

        before = len(reopened.history(run_id))
        controller.recover(run_id)
        after = len(reopened.history(run_id))
        assert after == before, "recovery is not idempotent"
        assert_recovery_invariants(reopened.state(run_id))
    finally:
        reopened.close()


@pytest.mark.parametrize(
    "name,workflow,scripts_factory", WORKFLOWS, ids=[entry[0] for entry in WORKFLOWS]
)
def test_crash_injection_matrix(name, workflow, scripts_factory, tmp_path):
    total = _discover_event_count(name, workflow, scripts_factory, tmp_path)
    assert total > 3, f"{name} committed only {total} events"
    for index in range(total):
        _crash_then_recover(name, index, workflow, scripts_factory, tmp_path)


@pytest.mark.slow
@pytest.mark.parametrize(
    "name,workflow,scripts_factory",
    EXTENDED_WORKFLOWS,
    ids=[entry[0] for entry in EXTENDED_WORKFLOWS],
)
def test_crash_injection_matrix_extended(name, workflow, scripts_factory, tmp_path):
    total = _discover_event_count(name, workflow, scripts_factory, tmp_path)
    assert total > 3, f"{name} committed only {total} events"
    for index in range(total):
        _crash_then_recover(name, index, workflow, scripts_factory, tmp_path)
