#!/usr/bin/env python3
"""Score the WS1 live pilot against the blind adjudication (plan section F.2). Orchestrator only, after labelling.

  python score.py            -> results.json + a short printout

Primary labels come from a1..a3 (each of the 110 packet ids is in exactly one of them); a4 is the 30-claim
reliability overlap and is used only for agreement. Joins labels to WS1's shadow log through adjudication_key.json.
Outputs, all with Wilson 95% intervals: the fail share of live claims; Jev's router miss rate (adjudicated fails it
auto-accepted) and useful accept share, per evidence arm and threshold; Jev's checker view; the production gates'
catches (ground.py flag, step 03 match); a strict variant (any non-`none` flag counts as a fail); Opus-vs-Opus
agreement on the overlap. Vendors are reported as Vendor 1..5 (alphabetical by slug, as in WS1's charts).
"""
import json
import math
from collections import Counter, defaultdict
from pathlib import Path

ADJ = Path(__file__).resolve().parent
WS1 = ADJ.parent / "ws1"
THRESHOLDS = [0.5, 0.6, 0.7, 0.8, 0.9, 0.95]
FAIL_FLAGS = {"dim_disagreement_major", "contradicted_fact", "fabricated_or_unsupported_claim"}


def jl(p):
    return [json.loads(x) for x in p.read_text(encoding="utf-8").split("\n") if x.strip()] if p.exists() else []


def wilson(k, n, z=1.959964):
    if n == 0:
        return {"k": k, "n": n, "p": None, "lo": None, "hi": None}
    p = k / n
    d = 1 + z * z / n
    c = (p + z * z / (2 * n)) / d
    h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
    return {"k": k, "n": n, "p": round(100 * p, 1), "lo": round(100 * max(0, c - h), 1), "hi": round(100 * min(1, c + h), 1)}


def labels(agent):
    return {x["packet_id"]: x for x in jl(ADJ / agent / "labels.jsonl") if not x.get("superseded")}


def accept(reps, t):
    return bool(reps) and all(r.get("flag") == "none" and (r.get("confidence") or 0) >= t for r in reps)


def kappa(pairs):
    n = len(pairs)
    if not n:
        return None
    po = sum(a == b for a, b in pairs) / n
    ca, cb = Counter(a for a, _ in pairs), Counter(b for _, b in pairs)
    pe = sum(ca[k] * cb[k] for k in set(ca) | set(cb)) / (n * n)
    return round((po - pe) / (1 - pe), 3) if pe < 1 else None


def main():
    key = json.load(open(WS1 / "adjudication_key.json", encoding="utf-8"))["key"]
    shadow = {(r["run_id"], r["claim_id"]): r for r in jl(WS1 / "shadow_claims.jsonl") if r["scope"] == "in_scope"}
    prim = {}
    for a in ("a1", "a2", "a3"):
        prim.update(labels(a))
    slugs = sorted({k["slug"] for k in key.values()})
    anon = {s: f"Vendor {i + 1}" for i, s in enumerate(slugs)}
    rows = []
    for pid, lab in prim.items():
        k = key[pid]
        s = shadow[(k["run_id"], k["claim_id"])]
        assert s["claim_sha256"] == k["claim_sha256"]
        strict_fail = lab["flag"] != "none"
        rows.append({"pid": pid, "vendor": anon[k["slug"]], "section": s["section"], "fail": lab["verdict"] == "fail",
                     "strict_fail": strict_fail, "flag": lab["flag"], "true_per_web": lab["true_per_web"],
                     "confidence": lab["confidence"], "has_receipt": bool(s.get("receipt")),
                     "jev_A": s["jev_A"], "jev_B": s["jev_B"],
                     "checker_A": bool(s["fail_view_A_any"]), "checker_B": bool(s["fail_view_B_any"]),
                     "ground_flag": bool(s["ground_flag"]), "step03_flag": bool((s.get("step03") or {}).get("match", {}).get("any"))})
    n = len(rows)
    fails = [r for r in rows if r["fail"]]
    passes = [r for r in rows if not r["fail"]]
    out = {"n_labelled": n, "n_expected": len(key),
           "fail_share": wilson(len(fails), n),
           "flags": Counter(r["flag"] for r in rows),
           "fail_share_by_section": {sec: wilson(sum(r["fail"] for r in rows if r["section"] == sec), sum(r["section"] == sec for r in rows))
                                     for sec in sorted({r["section"] for r in rows})},
           "fail_share_by_vendor": {v: wilson(sum(r["fail"] for r in rows if r["vendor"] == v), sum(r["vendor"] == v for r in rows))
                                    for v in sorted({r["vendor"] for r in rows})},
           "false_per_web": sum(r["true_per_web"] == "false" for r in rows),
           "low_confidence_labels": sum(r["confidence"] == "low" for r in rows)}
    curve = {}
    for variant, is_fail in (("primary", lambda r: r["fail"]), ("strict", lambda r: r["strict_fail"])):
        F = [r for r in rows if is_fail(r)]
        P = [r for r in rows if not is_fail(r)]
        for arm in ("A", "B"):
            curve[f"{variant}_arm{arm}"] = [{"t": t, "miss": wilson(sum(accept(r[f"jev_{arm}"], t) for r in F), len(F)),
                                              "useful_accept": wilson(sum(accept(r[f"jev_{arm}"], t) for r in P), len(P))}
                                             for t in THRESHOLDS]
    out["jev_router_curve"] = curve
    out["jev_checker_view"] = {arm: {"fails_caught": wilson(sum(r[f"checker_{arm}"] for r in fails), len(fails)),
                                     "passes_flagged": wilson(sum(r[f"checker_{arm}"] for r in passes), len(passes))}
                               for arm in ("A", "B")}
    out["production_gates"] = {"ground_py_caught": wilson(sum(r["ground_flag"] for r in fails), len(fails)),
                               "step03_caught": wilson(sum(r["step03_flag"] for r in fails), len(fails)),
                               "either_caught": wilson(sum(r["ground_flag"] or r["step03_flag"] for r in fails), len(fails))}
    out["receipt_split_armB_accept_at_0.8"] = {h: wilson(sum(accept(r["jev_B"], 0.8) for r in rows if r["has_receipt"] == h),
                                                         sum(r["has_receipt"] == h for r in rows)) for h in (True, False)}
    a4 = labels("a4")
    pairs = [(prim[p]["verdict"], a4[p]["verdict"]) for p in a4 if p in prim]
    fpairs = [(prim[p]["flag"], a4[p]["flag"]) for p in a4 if p in prim]
    out["reliability_a4_vs_primary"] = {"n": len(pairs), "verdict_agreement": wilson(sum(a == b for a, b in pairs), len(pairs)),
                                        "verdict_kappa": kappa(pairs), "flag_agreement": wilson(sum(a == b for a, b in fpairs), len(fpairs)),
                                        "disagreements": [p for p in a4 if p in prim and prim[p]["verdict"] != a4[p]["verdict"]]}
    (ADJ / "results.json").write_text(json.dumps(out, indent=1, default=dict), encoding="utf-8")
    print(f"labelled {n}/{len(key)}; fail share {out['fail_share']}; low-confidence {out['low_confidence_labels']}")
    for k2 in ("primary_armA", "primary_armB"):
        r8 = next(c for c in curve[k2] if c["t"] == 0.8)
        print(f"{k2} @0.8: miss {r8['miss']}  useful accept {r8['useful_accept']}")
    print("gates:", out["production_gates"])
    print("reliability:", {k3: v for k3, v in out["reliability_a4_vs_primary"].items() if k3 != "disagreements"})


if __name__ == "__main__":
    main()
