# SPDX-License-Identifier: AGPL-3.0-or-later
"""The four acceptance demonstrations of the sixth roadmap milestone.

Each test is named after and documents one demonstration listed under
"Sixth milestone: the experiment — calibration, pilots, and the measured
comparison" in docs/roadmap.md. The whole experiment runs once, under the
conformance gate, in the module-scoped ``experiment_result`` fixture.

Nothing new executes here beyond the experiment harness itself: the tests
compose the committed controller, command executor, model adapter, and frozen
oracle through ``orchestrator.experiment``.
"""

from __future__ import annotations

import dataclasses
import os

import pytest

from orchestrator import experiment, oracle, sandbox
from orchestrator.store import EventStore

ARM_ORDER = experiment.ARM_ORDER

# The experiment fixture is the single most expensive setup in the suite, so
# every test that depends on it is slow-marked. Its immutable prefix (target,
# oracle, calibration, pilots) is session-shared (performance plan A3); this
# module contributes the frozen full-seed manifest and the measured comparison.
pytestmark = pytest.mark.slow


@pytest.fixture(scope="module")
def capabilities():
    return sandbox.capabilities(refresh=True)


@pytest.fixture(scope="module")
def experiment_workdir(experiment_prefix):
    return experiment_prefix.workdir


@pytest.fixture(scope="module")
def experiment_result(experiment_prefix, capabilities):
    if not capabilities.profile_available:
        pytest.fail(f"NAMESPACE_V1 unavailable (fail-closed gate): {capabilities.unavailable_reason()}")
    prefix = experiment_prefix
    manifest = experiment.freeze_manifest(
        target=prefix.target,
        oracle_manifest=prefix.oracle,
        limits=prefix.pilots.calibrated_limits,
        calibration=prefix.calibration,
        pilot_calibration_method=prefix.pilots.method,
    )
    comparison = experiment.run_measured_comparison(
        manifest, prefix.oracle, prefix.workdir, spend_ledger=prefix.spend_ledger
    )
    return experiment.ExperimentResult(
        target=prefix.target,
        oracle=prefix.oracle,
        calibration=prefix.calibration,
        pilots=prefix.pilots,
        manifest=manifest,
        comparison=comparison,
        conformance_gate_passed=prefix.gate_passed,
    )


# 1. Oracle calibration (step 28): the frozen oracle is run against the five
# calibration cases and the outcomes are recorded in the experiment manifest.


def test_oracle_calibration_records_the_five_cases_in_the_manifest(experiment_result):
    calibration = experiment_result.calibration
    assert calibration.comparative_results() == {}
    assert [case.case_id for case in calibration.cases] == [
        "known-good",
        "unchanged",
        "known-broken",
        "known-timeout",
        "infrastructure",
    ]
    assert [case.classification for case in calibration.cases] == [
        oracle.TRIAL_CORRECT_ACCEPTANCE,
        oracle.TRIAL_CORRECTLY_UNRESOLVED,
        oracle.TRIAL_CORRECTLY_UNRESOLVED,
        oracle.TRIAL_FALSE_SUCCESS,
        oracle.TRIAL_INDETERMINATE,
    ]
    # The outcomes are recorded in the manifest, and calibration contributes no
    # comparative results.
    assert experiment_result.manifest.calibration == calibration.cases
    for case in calibration.cases:
        assert case.outcomes, f"{case.case_id} recorded no outcomes"
        assert case.classification == case.expected_classification
    # The known-good case is graded behaviourally: every hidden check executed
    # the candidate and passed.
    known_good = next(case for case in calibration.cases if case.case_id == "known-good")
    assert set(dict(known_good.effective_outcomes).values()) == {"PASS"}
    assert known_good.verified_progress == 1.0
    # The known-timeout case is a genuinely hanging candidate: the timeout is
    # produced by candidate behaviour inside a healthy oracle environment, and
    # a candidate timeout is gradeable (counts as FAIL), never rerun. The
    # report-emitter check composes parser -> formatter -> emitter, so it
    # times out with the parser.
    known_timeout = next(case for case in calibration.cases if case.case_id == "known-timeout")
    timeout_outcomes = dict(known_timeout.effective_outcomes)
    assert timeout_outcomes["hidden-parse"] == oracle.GRADE_CANDIDATE_TIMEOUT
    assert timeout_outcomes["hidden-report-emit"] == oracle.GRADE_CANDIDATE_TIMEOUT
    assert known_timeout.verified_progress == 0.5
    assert known_timeout.rerun_spent is False
    # The injected infrastructure failure spends the single permitted rerun and
    # stays indeterminate (never acceptance evidence).
    infrastructure = next(case for case in calibration.cases if case.case_id == "infrastructure")
    assert infrastructure.rerun_eligible == ("hidden-parse",)
    assert infrastructure.rerun_spent is True
    assert infrastructure.verified_progress is None


# The hidden oracle is behavioural, not structural: a candidate that implements
# every function but computes wrong results must fail the hidden suite, so a
# controller accepting it is a FALSE_SUCCESS rather than a CORRECT_ACCEPTANCE.


def test_functionally_wrong_but_implemented_candidates_fail_the_hidden_oracle(
    experiment_result, experiment_workdir
):
    target = experiment_result.target
    oracle_manifest = experiment_result.oracle
    tree = experiment._materialise(
        target.repository, target.commit, experiment_workdir, "wrong"
    )
    for relative, content in experiment._SOLUTION_FILES.items():
        with open(os.path.join(tree, relative), "w") as handle:
            handle.write(content)
    # Structurally valid, functionally wrong: the parser returns an empty list.
    with open(os.path.join(tree, "src/parser/parse.py"), "w") as handle:
        handle.write("def parse(text):\n    return []\n")
    grade = oracle.grade_manifest(oracle_manifest, tree, runner=experiment.hidden_runner)
    assert grade.effective("hidden-parse") == oracle.GRADE_FAIL
    # The report emitter composes parser -> formatter -> emitter, so the broken
    # parser also fails the integration-level check.
    assert grade.effective("hidden-report-emit") == oracle.GRADE_FAIL
    assert grade.effective("hidden-render") == oracle.GRADE_PASS
    assert oracle.classify_trial(
        oracle_manifest, grade, oracle.ACCEPTANCE_CLAIMED
    ) == oracle.TRIAL_FALSE_SUCCESS
    assert grade.verified_progress(oracle_manifest) == 0.5


# 2. Unmeasured pilots (steps 29-30): DIRECT and FIXED_ONE_LEVEL first, then
# recursive and concurrent, all under the same adapter/profile/budgets/oracle,
# contributing no comparative results.


def test_unmeasured_pilots_run_direct_and_fixed_one_level_first_then_recursive_and_concurrent(
    experiment_result,
):
    pilots = experiment_result.pilots
    assert pilots.order == (
        experiment.ARM_DIRECT,
        experiment.ARM_FIXED_ONE_LEVEL,
        experiment.ARM_RECURSIVE_LAZY,
        experiment.ARM_RECURSIVE_CONCURRENT,
    )
    assert pilots.measured is False
    assert pilots.comparative_results() == {}
    assert experiment_result.manifest.pilot_calibration_method == pilots.method
    assert [observation.arm for observation in pilots.observations] == list(pilots.order)
    # The pilots calibrate model calls, billed tokens, per-attempt commitments,
    # wall-clock, node, and retry limits.
    for observation in pilots.observations:
        assert observation.model_calls >= 1
        assert observation.input_tokens > 0
        assert observation.output_tokens > 0
        assert observation.actions >= observation.model_calls
        assert observation.nodes >= 1
        assert observation.wall_clock_seconds > 0
        assert observation.per_attempt_commitment > 0
    # The only intended cross-arm difference is the concurrent_workers treatment.
    for invariant in (
        "per_attempt_feedback_equal",
        "retry_permissions_equal",
        "tool_access_equal",
        "aggregate_budget_equal",
        "only_concurrent_workers_differs",
    ):
        assert pilots.invariant(invariant), f"pilot invariant failed: {invariant}"
    calibrated = experiment_result.manifest.budgets
    assert calibrated.model_calls == max(o.model_calls for o in pilots.observations)
    assert calibrated.nodes == max(o.nodes for o in pilots.observations)
    assert calibrated.maximum_depth == max(o.depth for o in pilots.observations)
    assert calibrated.root_model_input_tokens == max(o.input_tokens for o in pilots.observations)
    assert calibrated.concurrent_workers == experiment.placeholder_limits().concurrent_workers


# 3. A frozen experiment manifest (step 31): trial count, budgets, arm
# definitions, oracle version, and rerun policy fixed before any result is
# inspected.


def test_frozen_experiment_manifest_fixes_trial_count_budgets_arms_oracle_and_rerun_policy(
    experiment_result,
):
    manifest = experiment_result.manifest
    assert manifest.trial_seeds == experiment.TRIAL_SEEDS
    assert manifest.arm_assignment == experiment.ARM_ASSIGNMENT
    for arm in ARM_ORDER:
        assert manifest.arm_seeds(arm) == manifest.trial_seeds
    assert manifest.rerun_policy == experiment.RERUN_POLICY
    assert manifest.oracle_version == experiment_result.oracle.oracle_version
    assert manifest.hidden_suite_manifest_hash == experiment.hidden_suite_manifest_hash(
        experiment_result.target, experiment_result.oracle
    )
    assert manifest.component_versions
    assert all(value.startswith("sha256:") for _name, value in manifest.component_versions)
    assert manifest.contract_hash == experiment_result.target.contract_hash
    assert manifest.visible_commands == experiment.VISIBLE_COMMANDS
    assert len(manifest.visible_commands) >= 2
    assert manifest.model_identifier == experiment.MODEL_PROFILE.model_id
    assert manifest.sampling_parameters["temperature"] == 0.0
    assert manifest.budgets.concurrent_workers == (
        (experiment.ARM_DIRECT, 1),
        (experiment.ARM_FIXED_ONE_LEVEL, 1),
        (experiment.ARM_RECURSIVE_CONCURRENT, 4),
        (experiment.ARM_RECURSIVE_LAZY, 1),
    )
    # Recorded metrics this harness cannot measure are disclosed in the
    # manifest; every other metric is measured from run evidence.
    assert manifest.unmeasured_metrics == experiment.UNMEASURED_METRICS
    disclosed = {name for name, _reason in manifest.unmeasured_metrics}
    assert set(experiment.UNMEASURED_METRIC_NAMES) <= set(experiment.METRIC_FIELDS)
    assert not {"retries", "semantic_coverage", "integration_failures",
                "constraint_violations", "no_progress_recursion"} & disclosed
    # The measured comparison used exactly the frozen seeds and arm assignment.
    assert experiment_result.comparison.manifest_hash == manifest.content_hash()
    assert all(
        trial.manifest_hash == manifest.content_hash()
        for trial in experiment_result.comparison.trials
    )
    # The manifest is immutable once frozen: neither a field nor a limit can be
    # changed after this point.
    with pytest.raises(dataclasses.FrozenInstanceError):
        manifest.trial_seeds = (1,)
    with pytest.raises(dataclasses.FrozenInstanceError):
        manifest.budgets = experiment.placeholder_limits()
    changed = experiment.freeze_manifest(
        target=experiment_result.target,
        oracle_manifest=experiment_result.oracle,
        limits=dataclasses.replace(manifest.budgets, model_calls=999),
        calibration=experiment_result.calibration,
    )
    assert changed.content_hash() != manifest.content_hash()


# 4. The four-arm measured comparison (step 32): outcome, decomposition,
# parallelism, cost, stopping, and routing metrics recorded per trial.


def test_four_arm_measured_comparison_records_outcome_decomposition_parallelism_cost_stopping_and_routing_metrics(
    experiment_result,
):
    comparison = experiment_result.comparison
    assert comparison.measured is True
    assert experiment_result.conformance_gate_passed is True
    assert {arm for arm, _summary in comparison.arms} == set(ARM_ORDER)
    assert {name for name, _metrics in comparison.metric_groups} == {
        "outcome",
        "decomposition",
        "parallelism",
        "cost",
        "stopping",
        "routing",
    }
    for arm in ARM_ORDER:
        trials = [trial for trial in comparison.trials if trial.arm == arm]
        assert [trial.seed for trial in trials] == list(experiment_result.manifest.arm_seeds(arm))
        for trial in trials:
            record = trial.to_dict()
            for group, metrics in comparison.metric_groups:
                for metric in metrics:
                    assert metric in record, f"{group}.{metric} is not recorded"
            assert trial.oracle_outcomes
            assert trial.controller_outcome is not None or trial.classification in (
                oracle.TRIAL_CORRECTLY_UNRESOLVED,
                oracle.TRIAL_INDETERMINATE,
            )
    # The orchestration methods differ in the recorded decomposition and
    # parallelism metrics.
    direct = comparison.arm(experiment.ARM_DIRECT)
    fixed = comparison.arm(experiment.ARM_FIXED_ONE_LEVEL)
    recursive = comparison.arm(experiment.ARM_RECURSIVE_LAZY)
    concurrent = comparison.arm(experiment.ARM_RECURSIVE_CONCURRENT)
    assert direct.mean_nodes < fixed.mean_nodes <= recursive.mean_nodes
    assert fixed.mean_depth < recursive.mean_depth
    assert concurrent.mean_ready_action_concurrency > recursive.mean_ready_action_concurrency
    # The concurrent arm keeps the same decomposition as the lazy recursive arm.
    assert concurrent.mean_nodes == recursive.mean_nodes
    assert concurrent.mean_depth == recursive.mean_depth


# Trial integrity: controller outcome, oracle grade, and classification are
# recorded for every trial; INDETERMINATE trials are excluded from the
# false-success denominator and the single-rerun policy is explicit.


def test_trial_integrity_excludes_indeterminate_trials_from_the_false_success_denominator(
    experiment_result,
    experiment_workdir,
):
    comparison = experiment_result.comparison
    trials = comparison.trials
    for arm in ARM_ORDER:
        summary = comparison.arm(arm)
        assert summary.trials == len(experiment_result.manifest.trial_seeds)
        assert summary.indeterminate == 1
        assert summary.false_success_denominator == summary.trials - summary.indeterminate
        if summary.false_success_denominator:
            assert summary.false_success_rate == (
                summary.false_success_count / summary.false_success_denominator
            )
        indeterminate = [
            trial
            for trial in trials
            if trial.arm == arm and trial.indeterminate
        ]
        for trial in indeterminate:
            assert trial.classification == oracle.TRIAL_INDETERMINATE
            assert trial.verified_progress is None
            assert trial.criterion_verified_progress is None
            assert trial.rerun_spent is True
            assert trial.rerun_eligible == ("hidden-parse",)
            assert trial.verified_correctness is False
    # The deterministic build-failure trial is honestly blocked and never a
    # false success, in every arm. R10: the driver now drives the retryable
    # failure through the bounded repair loop to a recorded terminal outcome
    # (INFEASIBLE), instead of stopping before any outcome exists.
    failures = [trial for trial in trials if trial.seed == 1003]
    assert len(failures) == len(ARM_ORDER)
    for trial in failures:
        # The bounded repair loop drives the failing node to a recorded terminal
        # INFEASIBLE outcome. When the failing node is the root the run outcome is
        # INFEASIBLE; when it is a child the child is terminal and the root stays
        # blocked, so the run is honestly unresolved.
        if trial.arm == experiment.ARM_DIRECT:
            assert trial.controller_outcome == "INFEASIBLE"
        else:
            assert trial.controller_outcome is None
        assert trial.classification == oracle.TRIAL_CORRECTLY_UNRESOLVED
        assert trial.false_success is False
        assert trial.honest_blocked is True
        # Two recorded validation failures: the retryable first failure and the
        # repeated failure that the policy turns into a terminal outcome.
        assert trial.validation_failures >= 2
    # honest_blocked means the controller honestly reported it could not
    # proceed: gradeable, no acceptance claim, and no budget exhaustion.
    for trial in trials:
        assert trial.honest_blocked == (
            not trial.root_acceptance
            and not trial.indeterminate
            and not trial.budget_exhaustion
        )
        assert not trial.budget_exhaustion
    # Metrics measured from run evidence, not asserted: retries derive from
    # attempt causes, semantic coverage from criterion coverage and acceptance,
    # and integration failures from rejected candidates.
    for trial in trials:
        # ``retries`` is measured from attempt causes: it must equal the number
        # of RETRY-planned attempts in the trial's own store, for any scenario
        # shape (Axis 2 review F8).
        trial_root = os.path.join(
            experiment_workdir, "trials", f"{trial.arm}-{trial.seed}"
        )
        store = EventStore(os.path.join(trial_root, "events.db"))
        try:
            planned_causes = [
                event.payload.get("cause")
                for event in store.history(trial.run_id)
                if event.event_type == "ATTEMPT_PLANNED"
            ]
        finally:
            store.close()
        assert trial.retries == planned_causes.count("RETRY")
        # Scenario shape, not semantics (deliberately not an equality on the
        # metric): the frozen build-failure seed fails one action lineage twice
        # before the abort, so it re-issues at least once and records at least
        # two validation failures; two failing lineages would give vf=4 and
        # retries=2 and must not break this test.
        if trial.validation_failures >= 2:
            assert trial.retries >= 1
        assert trial.premature_leaf_decisions == 0
        assert trial.integration_failures == 0
        assert trial.constraint_violations == 0
        assert trial.conflict_rate == 0.0
        assert trial.stale_result_rate == 0.0
        if trial.classification == oracle.TRIAL_CORRECT_ACCEPTANCE:
            assert trial.no_progress_recursion == 0
    # A decomposition that left no accepted child made no progress: measured
    # for the failed decomposed arms, and impossible for the undecomposed arm.
    for trial in failures:
        if trial.arm == experiment.ARM_DIRECT:
            assert trial.no_progress_recursion == 0
        else:
            assert trial.no_progress_recursion == 1
    # Semantic coverage is measured: a failed undecomposed DIRECT root covers
    # none of its own criteria; a failed decomposition covers the criteria its
    # mapping covers (the parent's) while its unaccepted leaves cover none.
    direct_failure = next(
        trial
        for trial in trials
        if trial.seed == 1003 and trial.arm == experiment.ARM_DIRECT
    )
    assert direct_failure.semantic_coverage == 0.0
    for trial in trials:
        if trial.seed == 1003 and trial.arm != experiment.ARM_DIRECT:
            assert 0.0 < trial.semantic_coverage < 1.0
        if trial.classification == oracle.TRIAL_CORRECT_ACCEPTANCE:
            assert trial.semantic_coverage == 1.0
            assert trial.verified_correctness is True
    # Realised speed-up is a comparative reference metric, measured only for the
    # concurrent treatment (disclosed in the manifest).
    for trial in trials:
        if trial.arm != experiment.ARM_RECURSIVE_CONCURRENT:
            assert trial.realised_speedup == 1.0
    assert comparison.arm(experiment.ARM_RECURSIVE_CONCURRENT).mean_realised_speedup > 0



# The fixed target's integration dependency is exercised through the
# incremental integration-tree protocol: a dependent action receives a
# base_commit containing its accepted predecessor's integrated changes.


def test_dependent_actions_receive_integrated_predecessor_changes(experiment_result):
    comparison = experiment_result.comparison
    frozen_base = experiment_result.target.commit
    fixed = next(
        trial
        for trial in comparison.trials
        if trial.arm == experiment.ARM_FIXED_ONE_LEVEL and trial.seed == 1001
    )
    fixed_bases = dict(fixed.dependent_base_commits)
    assert fixed_bases[f"{fixed.run_id}-parse"] == frozen_base
    assert fixed_bases[f"{fixed.run_id}-render"] != frozen_base
    # The report action integrates both the parser and the renderer.
    assert fixed_bases[f"{fixed.run_id}-report"] not in (frozen_base, fixed_bases[f"{fixed.run_id}-render"])

    concurrent = next(
        trial
        for trial in comparison.trials
        if trial.arm == experiment.ARM_RECURSIVE_CONCURRENT and trial.seed == 1001
    )
    concurrent_bases = dict(concurrent.dependent_base_commits)
    # Disjoint siblings share the frozen base; the dependent descendant sees the
    # merged integration head.
    assert concurrent_bases[f"{concurrent.run_id}-parse"] == frozen_base
    assert concurrent_bases[f"{concurrent.run_id}-render"] == frozen_base
    assert concurrent_bases[f"{concurrent.run_id}-report-format"] != frozen_base
