Eval harness: regression tests for LLM apps and agents in CI

import json
import sys
from pathlib import Path

TOLERANCE = 0.02  # allowed drop in pass rate before the build fails

current = json.loads(Path("reports/current.json").read_text(encoding="utf-8"))
baseline = json.loads(Path("reports/baseline.json").read_text(encoding="utf-8"))

problems = []
for suite, result in current["suites"].items():
    base = baseline["suites"].get(suite)
    if base is None:
        continue
    if base["pass_rate"] - result["pass_rate"] > TOLERANCE:
        problems.append(f"{suite}: pass rate {result['pass_rate']:.1%} (baseline {base['pass_rate']:.1%})")
    new_critical = set(result["failed_critical"]) - set(base["failed_critical"])
    if new_critical:
        problems.append(f"{suite}: new critical failures {sorted(new_critical)}")

if problems:
    print("\n".join(problems))
    sys.exit(1)
print("No regressions against the baseline.")