# SPDX-License-Identifier: AGPL-3.0-or-later
"""Golden metric definitions.

Metrics are asserted against hand-computed numerators and denominators, so a
metric that silently changes meaning fails. The recommendation's example is
pinned directly: two trials costing ten each with one acceptance report twenty,
not ten. Metric group completeness is asserted so a metric cannot be dropped
from the disclosure.

Defects found by these golden cases are left unfixed and reported by failing
tests at the bottom of the file.
"""
from __future__ import annotations

import dataclasses

from orchestrator import experiment

from orchestrator.experiment import ArmSummary, TrialMetrics


def _default(field: dataclasses.Field):
    declared = field.type
    if isinstance(declared, str):
        if "Optional" in declared:
            return None
        if declared.startswith("Tuple"):
            return ()
        if "bool" in declared:
            return False
        if "str" in declared:
            return ""
        if "float" in declared:
            return 0.0
        if "int" in declared:
            return 0
    return None


def _trial(**overrides) -> TrialMetrics:
    values = {field.name: _default(field) for field in dataclasses.fields(TrialMetrics)}
    values.update(overrides)
    return TrialMetrics(**values)


def _summarise(trials) -> ArmSummary:
    return experiment._summarise("A", trials)


def test_two_trials_costing_ten_with_one_acceptance_report_twenty():
    summary = _summarise(
        [
            _trial(root_acceptance=True, cost_per_accepted_root=10.0),
            _trial(root_acceptance=False, cost_per_accepted_root=10.0),
        ]
    )
    assert summary.mean_cost_per_accepted_root == 20.0
    assert summary.trials == 2
    assert summary.root_acceptance_rate == 0.5


def test_failed_trial_spend_is_never_discarded():
    summary = _summarise(
        [
            _trial(root_acceptance=True, cost_per_accepted_root=5.0),
            _trial(root_acceptance=False, cost_per_accepted_root=5.0),
            _trial(root_acceptance=False, cost_per_accepted_root=5.0),
        ]
    )
    assert summary.mean_cost_per_accepted_root == 15.0


def test_indeterminate_trials_are_excluded_from_the_false_success_denominator():
    summary = _summarise(
        [
            _trial(indeterminate=False, root_acceptance=True),
            _trial(indeterminate=True),
        ]
    )
    assert summary.trials == 2
    assert summary.indeterminate == 1
    assert summary.false_success_denominator == 1


def test_false_success_rate_is_undefined_with_no_gradeable_trials():
    summary = _summarise([_trial(indeterminate=True)])
    assert summary.false_success_denominator == 0
    assert summary.false_success_rate is None


def test_means_are_taken_over_every_trial_of_the_arm():
    summary = _summarise([_trial(model_calls=1), _trial(model_calls=3)])
    assert summary.mean_model_calls == 2.0


GROUPED_METRIC_FLOOR = 52


def test_disclosed_metrics_are_unique_known_and_do_not_shrink():
    flat = [metric for _, metrics in experiment.METRIC_GROUPS for metric in metrics]
    assert len(flat) == len(set(flat)), "a metric is disclosed in more than one group"
    unknown = [metric for metric in flat if metric not in experiment.METRIC_FIELDS]
    assert not unknown, f"disclosed metrics are not recorded fields: {unknown}"
    assert len(flat) >= GROUPED_METRIC_FLOOR, (
        f"only {len(flat)} metrics are disclosed; floor is {GROUPED_METRIC_FLOOR}"
    )


# -- defects pinned by the golden definitions -------------------------------


def test_bug_cost_per_accepted_root_hides_spend_when_nothing_is_accepted():
    """BUG: an arm that spent ten and accepted nothing reports 0.0 rather than
    an undefined (None) cost per accepted root."""
    summary = _summarise([_trial(root_acceptance=False, cost_per_accepted_root=10.0)])
    assert summary.mean_cost_per_accepted_root is None


def test_bug_mean_realised_speedup_includes_failed_trials():
    """BUG: speed-up must be computed only over a comparable successful cohort,
    but a failed trial's fallback 1.0 is averaged in."""
    summary = _summarise(
        [
            _trial(realised_speedup=3.0, gradeable=True, root_acceptance=True),
            _trial(realised_speedup=1.0, gradeable=True, root_acceptance=False),
        ]
    )
    assert summary.mean_realised_speedup == 3.0
