evaluate.py

#!/usr/bin/env python3
"""Inspect the routing eval suite or grade structured trial results."""

from __future__ import annotations

import argparse
import json
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]


def load(path: Path) -> dict:
    return json.loads(path.read_text(encoding="utf-8"))


def grade(cases: list[dict], results: list[dict]) -> int:
    expected = {case["id"]: case for case in cases}
    actual = {result["id"]: result for result in results}
    failures: list[str] = []

    for case_id, case in expected.items():
        result = actual.get(case_id)
        if result is None:
            failures.append(f"{case_id}: missing result")
            continue
        should_trigger = case.get("should_trigger", True)
        if bool(result.get("triggered")) != should_trigger:
            failures.append(f"{case_id}: trigger mismatch")
            continue
        if not should_trigger:
            continue
        mode = result.get("selected_mode", result.get("mode"))
        if mode != case.get("expected_mode"):
            failures.append(f"{case_id}: expected {case.get('expected_mode')}, got {mode}")
        guidance = result.get("loaded_guidance", result.get("guidance", []))
        missing = set(case.get("required_guidance", [])) - set(guidance)
        if missing:
            failures.append(f"{case_id}: missing guidance {sorted(missing)}")
        forbidden = set(case.get("forbidden_modes", [])) & set(result.get("active_modes", []))
        if forbidden:
            failures.append(f"{case_id}: blended forbidden modes {sorted(forbidden)}")
        transition = case.get("expected_transition")
        actual_transition = result.get("expected_transition", result.get("transition"))
        if transition and actual_transition != transition:
            failures.append(f"{case_id}: expected transition {transition}")

    passed = len(expected) - len({failure.split(":", 1)[0] for failure in failures})
    print(f"passed {passed}/{len(expected)} cases")
    for failure in failures:
        print(f"FAIL: {failure}")
    return 1 if failures else 0


def stats(cases: list[dict]) -> None:
    total = len(cases)
    positive = sum(1 for c in cases if c.get("should_trigger", True))
    negative = sum(1 for c in cases if not c.get("should_trigger", True))
    failure = sum(1 for c in cases if c.get("failure_mode") is True)
    transitions = sum(1 for c in cases if "expected_transition" in c)
    security = sum(1 for c in cases if "security-autonomy" in c.get("required_guidance", []))
    percentage = lambda count: count * 100 // total if total else 0

    print(f"Total cases: {total}")
    print(f"Positive (should trigger): {positive} ({percentage(positive)}%)")
    print(f"Negative (should not trigger): {negative} ({percentage(negative)}%)")
    print(f"Failure mode cases: {failure} ({percentage(failure)}%)")
    print(f"Transition cases: {transitions}")
    print(f"Security-sensitive cases: {security}")
    print()

    modes = {}
    for case in cases:
        if case.get("should_trigger", True):
            mode = case.get("expected_mode")
            modes[mode] = modes.get(mode, 0) + 1

    print("Mode coverage:")
    for mode in sorted(modes.keys()):
        print(f"  {mode}: {modes[mode]} cases")


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--cases", type=Path, default=ROOT / "evals/routing.json")
    parser.add_argument("--results", type=Path)
    parser.add_argument("--references", action="store_true")
    parser.add_argument("--stats", action="store_true")
    args = parser.parse_args()
    cases = load(args.cases)["cases"]

    if args.stats:
        stats(cases)
        return 0
    if args.references:
        for case in cases:
            trigger = case.get("should_trigger", True)
            mode = case.get("expected_mode", "none")
            print(f"{case['id']}: trigger={str(trigger).lower()} mode={mode}")
        return 0
    if args.results:
        return grade(cases, load(args.results)["results"])
    parser.error("choose --references, --stats, or --results")
    return 2


if __name__ == "__main__":
    raise SystemExit(main())