# SPDX-License-Identifier: AGPL-3.0-or-later
"""Crash injection at persistence and dispatch boundaries.

docs/control-plane.md#crash-recovery:
- "The controller must record ACTION_DISPATCHED atomically before sending a
  request to an executor."
- After restart: replay the log, find attempts in DISPATCHED/RUNNING/
  OBSERVATION_RECEIVED with no terminal observation, admit
  ATTEMPT_FAILED_ON_RECOVERY, transition to FAILED, and conservatively settle
  outstanding accounting.
"""

from __future__ import annotations

import pytest

from orchestrator import events
from orchestrator.controller import Controller
from orchestrator.executor import AttemptTimedOut, ScriptedExecutor
from orchestrator.store import CrashInjected, EventStore
from helpers import build_run_config

RUN_CONFIG = build_run_config("r1")


def _script(request):
    return {
        "self_reported_status": "COMPLETED",
        "artefacts": [{"type": "GIT_CANDIDATE", "content_hash": "abc123"}],
        "cost_report": {"committed": 5, "actual": 5, "units": "USD"},
    }


def test_crash_before_persistence_leaves_no_partial_event(tmp_path):
    db_path = str(tmp_path / "events.db")
    store = EventStore(db_path)
    try:
        controller = Controller(store, ScriptedExecutor({"r1-root": _script}))
        node_id = controller.submit_run(RUN_CONFIG)

        def boom(event):
            raise CrashInjected("simulated crash before commit")

        store.crash_before_commit = boom
        attempt_id = "r1-root-attempt-crash"
        event = events.make_event(
            "ATTEMPT_PLANNED",
            {"attempt_id": attempt_id, "node_id": node_id, "cause": "INITIAL"},
            idempotency_key=f"plan-{attempt_id}",
        )
        with pytest.raises(CrashInjected):
            store.admit("r1", event)
        store.crash_before_commit = None

        # Nothing was persisted: the attempt does not exist in a fresh replay.
        assert attempt_id not in store.state("r1").attempts
        replayed = store.replay_fresh("r1")
        assert attempt_id not in replayed.attempts

        # A retry with the same idempotency key after the crash succeeds cleanly.
        retry_event = events.make_event(
            "ATTEMPT_PLANNED",
            {"attempt_id": attempt_id, "node_id": node_id, "cause": "INITIAL"},
            idempotency_key=f"plan-{attempt_id}",
        )
        result = store.admit("r1", retry_event)
        assert result.duplicate is False
        assert store.state("r1").attempts[attempt_id].state == "PLANNED"
    finally:
        store.close()


def test_crash_after_dispatch_recorded_before_executor_send_fails_attempt_on_recovery(tmp_path):
    db_path = str(tmp_path / "events.db")
    store = EventStore(db_path)
    try:
        controller = Controller(store, ScriptedExecutor({"r1-root": _script}))
        node_id = controller.submit_run(RUN_CONFIG)
        attempt_id = controller.plan_attempt("r1", node_id, 5)

        # Simulate a crash between "ACTION_DISPATCHED is durable" and "the
        # request is sent to the executor": admit the dispatch event directly
        # without calling executor.dispatch(), matching what dispatch_attempt
        # does up to the crash point.
        controller.executor.issue_token(attempt_id)
        controller._emit(
            "r1",
            "ACTION_DISPATCHED",
            {"attempt_id": attempt_id, "executor": "scripted-in-process"},
            f"dispatch-{attempt_id}",
        )
        assert store.state("r1").attempts[attempt_id].state == "DISPATCHED"
        # The executor never received the request.
        assert controller.executor._responses == {}
    finally:
        store.close()

    # "Crash": drop the store and executor, reopen the store from the durable
    # log only, and run recovery.
    restarted_store = EventStore(db_path)
    try:
        restarted_controller = Controller(restarted_store, ScriptedExecutor({"r1-root": _script}))
        recovered = restarted_controller.recover("r1")
        assert recovered == [attempt_id]
        final_state = restarted_store.state("r1")
        assert final_state.attempts[attempt_id].state == "FAILED"
        assert final_state.attempts[attempt_id].accounting_state == "FINALLY_SETTLED"
        # Conservative settlement charged the full outstanding commitment.
        assert restarted_controller.get_costs("r1")["total_charged"] == 5
    finally:
        restarted_store.close()


def test_crash_before_dispatch_recorded_produces_no_dispatched_attempt(tmp_path):
    db_path = str(tmp_path / "events.db")
    store = EventStore(db_path)
    try:
        controller = Controller(store, ScriptedExecutor({"r1-root": _script}))
        node_id = controller.submit_run(RUN_CONFIG)
        attempt_id = controller.plan_attempt("r1", node_id, 5)
        # Crash happens before ACTION_DISPATCHED is ever admitted (e.g. during
        # token issuance). No dispatch event exists in the log.
        assert store.state("r1").attempts[attempt_id].state == "RESERVED"
    finally:
        store.close()

    restarted_store = EventStore(db_path)
    try:
        restarted_controller = Controller(restarted_store, ScriptedExecutor({"r1-root": _script}))
        recovered = restarted_controller.recover("r1")
        # A crash before ACTION_DISPATCHED leaves a RESERVED straggler and no
        # dispatched attempt. Nothing is failed on recovery (no dispatch ever
        # happened): the straggler is cancelled and its unused commitment is
        # reclaimed (Axis 2 review F5), so its accounting cannot survive the
        # sweep as OPEN regardless of the run's outcome.
        assert recovered == [attempt_id]
        final_state = restarted_store.state("r1")
        assert final_state.attempts[attempt_id].state == "CANCELLED"
        assert final_state.attempts[attempt_id].accounting_state == "FINALLY_SETTLED"
        # Never dispatched: the settlement is a reclamation (actual 0), not a
        # conservative charge.
        assert restarted_controller.get_costs("r1")["total_charged"] == 0
    finally:
        restarted_store.close()


def test_crash_after_observation_admission_does_not_duplicate_result_or_cost(tmp_path):
    db_path = str(tmp_path / "events.db")
    store = EventStore(db_path)
    try:
        controller = Controller(store, ScriptedExecutor({"r1-root": _script}))
        node_id = controller.submit_run(RUN_CONFIG)
        attempt_id = controller.plan_attempt("r1", node_id, 5)
        token = controller.dispatch_attempt("r1", attempt_id, node_id)
        response = controller.executor.dispatch(
            {"run_id": "r1", "attempt_id": attempt_id, "attempt_token": token, "node_id": node_id}
        )
        controller.admit_observation_and_result("r1", attempt_id, node_id, token, response)
        assert store.state("r1").attempts[attempt_id].state == "RESULT_PRODUCED"
    finally:
        store.close()

    # "Crash" and restart: recovery must find no non-terminal attempt to fail,
    # and must not re-admit or duplicate the already-produced result or its cost.
    restarted_store = EventStore(db_path)
    try:
        restarted_controller = Controller(restarted_store, ScriptedExecutor({"r1-root": _script}))
        recovered = restarted_controller.recover("r1")
        assert recovered == []
        assert restarted_controller.get_costs("r1")["total_charged"] == 5
        history = restarted_store.history("r1")
        result_events = [e for e in history if e.event_type == "RESULT_ADMITTED"]
        assert len(result_events) == 1
    finally:
        restarted_store.close()


def test_recovery_settles_terminal_but_unsettled_accounting(tmp_path):
    """R14: a timed-out attempt left PROVISIONALLY_SETTLED is settled on recovery."""
    db_path = str(tmp_path / "events.db")
    store = EventStore(db_path)
    try:
        controller = Controller(store, ScriptedExecutor({"r1-root": lambda request: {"timeout": True}}))
        node_id = controller.submit_run(RUN_CONFIG)
        attempt_id = controller.plan_attempt("r1", node_id, 5)
        with pytest.raises(AttemptTimedOut):
            controller.dispatch_attempt("r1", attempt_id, node_id)
        attempt = store.state("r1").attempts[attempt_id]
        assert attempt.state in ("TIMED_OUT", "FAILED")
        assert attempt.accounting_state in ("OPEN", "PROVISIONALLY_SETTLED")
    finally:
        store.close()

    restarted_store = EventStore(db_path)
    try:
        restarted = Controller(restarted_store, ScriptedExecutor({}))
        restarted.recover("r1")
        settled = restarted_store.state("r1").attempts[attempt_id]
        assert settled.accounting_state == "FINALLY_SETTLED"
    finally:
        restarted_store.close()


def test_recovery_completes_interrupted_decomposition(tmp_path):
    """R14: a decomposition admitted without its child events is completed."""
    db_path = str(tmp_path / "events.db")
    store = EventStore(db_path)
    try:
        controller = Controller(store, ScriptedExecutor({}))
        config = build_run_config("r1")
        config["root_contract"] = {
            **config["root_contract"],
            "acceptance_criteria": [{"id": "AC-001", "mandatory": True}],
        }
        node_id = controller.submit_run(config)
        child_id = "r1-child"

        def boom(event):
            if event.event_type == "NODE_PROPOSED" and event.payload.get("node_id") == child_id:
                raise CrashInjected("simulated crash after decomposition admission")

        store.crash_before_commit = boom
        with pytest.raises(CrashInjected):
            controller.admit_derived_decomposition(
                "r1",
                node_id,
                "decomp-1",
                [
                    {
                        "node_id": child_id,
                        "mandatory": True,
                        "criterion_ids": ["AC-001"],
                        "budget_allocation": {"MODEL_COST": 5.0},
                        "contract": {
                            "description": "child",
                            "owned_scope": ["src/child/**"],
                            "acceptance_criteria": [{"id": "AC-001", "mandatory": True}],
                        },
                    }
                ],
                [{"criterion_id": "AC-001", "child_id": child_id, "mandatory_cover": True}],
            )
        store.crash_before_commit = None
        assert child_id not in store.state("r1").nodes
    finally:
        store.close()

    restarted_store = EventStore(db_path)
    try:
        restarted = Controller(restarted_store, ScriptedExecutor({}))
        restarted.recover("r1")
        state = restarted_store.state("r1")
        assert child_id in state.nodes
        assert f"rel-childof-{child_id}" in state.relations
        # A child without declared dependencies is made READY, matching the
        # uninterrupted decomposition path.
        assert state.nodes[child_id].state == "READY"
    finally:
        restarted_store.close()


def test_recovery_reconciles_durable_actual_instead_of_conservative_charge(tmp_path):
    """R14/R02: a crash between observation admission and reconciliation must not
    discard the durably reported (overrunning) actual cost."""
    db_path = str(tmp_path / "events.db")
    store = EventStore(db_path)
    try:
        controller = Controller(
            store,
            ScriptedExecutor({"r1-root": lambda request: {"self_reported_status": "COMPLETED"}}),
        )
        node_id = controller.submit_run(RUN_CONFIG)
        attempt_id = controller.plan_attempt("r1", node_id, 5)
        controller.dispatch_attempt("r1", attempt_id, node_id)
        # Crash point: the observation (actual 9 against a 5 commitment) is
        # durable, but reconciliation never ran.
        controller._emit(
            "r1",
            "EXECUTOR_OBSERVATION_ADMITTED",
            {"attempt_id": attempt_id, "observation": {"cost_report": {"committed": 5, "actual": 9.0, "units": "USD"}}},
            f"observe-{attempt_id}",
            source="executor",
        )
        controller._emit(
            "r1",
            "ATTEMPT_EXECUTION_ENDED_UNRECONCILED",
            {"attempt_id": attempt_id},
            f"end-{attempt_id}",
        )
        assert store.state("r1").ledger.total_charged == 0.0
    finally:
        store.close()

    restarted_store = EventStore(db_path)
    try:
        restarted = Controller(restarted_store, ScriptedExecutor({}))
        restarted.recover("r1")
        state = restarted_store.state("r1")
        assert state.attempts[attempt_id].accounting_state == "FINALLY_SETTLED"
        assert state.ledger.total_charged == 9.0
        assert state.ledger.total_overrun == 4.0
    finally:
        restarted_store.close()
