#!/usr/bin/env python3
"""Score the F4 scale-up against the blind adjudication (plan section F.4). Orchestrator only, after labelling.

  python score.py            -> results.json + a short printout

Primary labels come from b1..b4 (each of the 148 packet ids is in exactly one); b5 is the 30-claim reliability
overlap and is used only for agreement. Joins labels to f4/shadow_claims.jsonl through f4/adjudication_key.json.
Endpoints follow f4/PREREG.md: primary = bad share of arm-B accepts (Wilson 95%); secondary 1-5 as registered.
Added after seeing the build report, labelled as such: a vendor-clustered bootstrap interval on the primary share
(three vendors give 33 of the 111 accepts) and a strict view (any non-`none` flag counts as bad). Vendors are
reported as Vendor 1..N (alphabetical by slug).
"""
import json
import math
import random
from collections import Counter, defaultdict
from pathlib import Path

ADJ = Path(__file__).resolve().parent
F4 = ADJ.parent
FAIL_FLAGS = {"dim_disagreement_major", "contradicted_fact", "fabricated_or_unsupported_claim"}
PRIMARY_AGENTS = ("b1", "b2", "b3", "b4")
SEED = 20260922
B = 2000


def jl(p):
    return [json.loads(x) for x in p.read_text(encoding="utf-8").split("\n") if x.strip()] if p.exists() else []


def wilson(k, n, z=1.96):
    if n == 0:
        return [None, None]
    p = k / n
    d = 1 + z * z / n
    c = (p + z * z / (2 * n)) / d
    m = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
    return [round(max(0.0, c - m), 4), round(min(1.0, c + m), 4)]


def share(k, n):
    return {"k": k, "n": n, "share": round(k / n, 4) if n else None, "wilson95": wilson(k, n)}


def labels(agent):
    out = {}
    for r in jl(ADJ / agent / "labels.jsonl"):
        if r.get("superseded"):
            continue
        out[r["packet_id"]] = r
    return out


def is_bad(lab):
    return lab["flag"] in FAIL_FLAGS


def step03_matched(s03):
    m = (s03 or {}).get("match") or {}
    return any(bool(v) for sub in m.values() for v in (sub.values() if isinstance(sub, dict) else [sub]))


def kappa(a, b):
    n = len(a)
    po = sum(x == y for x, y in zip(a, b)) / n
    ca, cb = Counter(a), Counter(b)
    pe = sum(ca[c] * cb[c] for c in set(a) | set(b)) / (n * n)
    return round((po - pe) / (1 - pe), 3) if pe < 1 else None


def main():
    key = json.loads((F4 / "adjudication_key.json").read_text(encoding="utf-8"))
    kk = key["key"]
    shadow = {(r["run_id"], r["unit_index"]): r for r in jl(F4 / "shadow_claims.jsonl")}
    prim = {}
    for a in PRIMARY_AGENTS:
        for pid, lab in labels(a).items():
            assert pid not in prim, pid
            prim[pid] = lab
    missing = sorted(set(kk) - set(prim))
    assert not missing, f"unlabelled packet ids: {missing}"

    rows = []
    for pid, k in kk.items():
        s = shadow[(k["run_id"], k["unit_index"])]
        assert s["claim_sha256"] == k["claim_sha256"], pid
        lab = prim[pid]
        rows.append({"pid": pid, "slug": k["slug"], "sel": k["selected_because"], "section": s["section"],
                     "accB": bool(s["accept_B_conservative"]), "accA": bool(s["accept_A_conservative"]),
                     "bad": is_bad(lab), "flag": lab["flag"], "strict_bad": lab["flag"] != "none",
                     "true_per_web": lab.get("true_per_web"), "supported": lab.get("supported_by_cited_source"),
                     "ground": bool(s["ground_flag"]), "s03": step03_matched(s["step03"]),
                     "confidence": lab.get("confidence")})
    for r in rows:
        assert r["accB"] == (r["sel"] in ("accept_B", "accept_AB")), r["pid"]
        assert r["accA"] == (r["sel"] in ("accept_A", "accept_AB")), r["pid"]

    vendors = sorted({r["slug"] for r in rows} | {s["slug"] for s in shadow.values() if s["scope"] == "in_scope"})
    vname = {s: f"Vendor {i + 1}" for i, s in enumerate(vendors)}
    accB = [r for r in rows if r["accB"]]
    accA = [r for r in rows if r["accA"]]
    samp = [r for r in rows if r["sel"] == "random_sample"]
    in_scope = [s for s in shadow.values() if s["scope"] == "in_scope"]
    rest_by_slug = Counter(s["slug"] for s in in_scope if not s["accept_B_conservative"] and not s["accept_A_conservative"])
    n_rest = sum(rest_by_slug.values())
    assert n_rest == key["N_rest"], (n_rest, key["N_rest"])

    res = {"n_packet": len(rows), "n_in_scope": len(in_scope), "n_rest": n_rest, "n_sample": len(samp)}
    res["primary_bad_share_armB_accepts"] = share(sum(r["bad"] for r in accB), len(accB))
    res["strict_bad_share_armB_accepts"] = share(sum(r["strict_bad"] for r in accB), len(accB))
    res["armA_bad_share"] = share(sum(r["bad"] for r in accA), len(accA))
    res["sample_fail_share"] = share(sum(r["bad"] for r in samp), len(samp))
    res["sample_strict_share"] = share(sum(r["strict_bad"] for r in samp), len(samp))
    res["flags_armB_accepts"] = dict(Counter(r["flag"] for r in accB))
    res["flags_sample"] = dict(Counter(r["flag"] for r in samp))
    res["bad_armB_accepts"] = [{"packet_id": r["pid"], "vendor": vname[r["slug"]], "section": r["section"],
                                "flag": r["flag"], "true_per_web": r["true_per_web"], "supported": r["supported"],
                                "confidence": r["confidence"]} for r in accB if r["bad"]]
    res["bad_sample"] = [{"packet_id": r["pid"], "vendor": vname[r["slug"]], "section": r["section"],
                          "flag": r["flag"], "true_per_web": r["true_per_web"]} for r in samp if r["bad"]]
    by_sec = defaultdict(lambda: [0, 0])
    for r in accB:
        by_sec[r["section"]][0] += r["bad"]
        by_sec[r["section"]][1] += 1
    res["armB_accepts_by_section"] = {k: {"bad": v[0], "n": v[1]} for k, v in sorted(by_sec.items())}
    by_v = defaultdict(lambda: [0, 0])
    for r in accB:
        by_v[vname[r["slug"]]][0] += r["bad"]
        by_v[vname[r["slug"]]][1] += 1
    res["armB_accepts_by_vendor"] = {k: {"bad": v[0], "n": v[1]} for k, v in sorted(by_v.items(), key=lambda x: -x[1][1])}

    # Secondary 3: share of all bad in-scope claims that arm B (and arm A) accepts, vendor-clustered bootstrap.
    reviewed_acc = [r for r in rows if r["accA"] or r["accB"]]

    def est(slugs, arm):
        cnt = Counter(slugs)
        bad_arm = sum(cnt[r["slug"]] for r in rows if r[arm] and r["bad"])
        bad_acc = sum(cnt[r["slug"]] for r in reviewed_acc if r["bad"])
        ns = sum(cnt[r["slug"]] for r in samp)
        ks = sum(cnt[r["slug"]] for r in samp if r["bad"])
        nr = sum(cnt[s] * rest_by_slug[s] for s in cnt)
        denom = bad_acc + (nr * ks / ns if ns else 0.0)
        return bad_arm / denom if denom > 0 else None

    rng = random.Random(SEED)
    all_slugs = sorted({s["slug"] for s in in_scope})
    for arm, name in (("accB", "armB"), ("accA", "armA")):
        point = est(all_slugs, arm)
        boots = [est([rng.choice(all_slugs) for _ in all_slugs], arm) for _ in range(B)]
        boots = sorted(x for x in boots if x is not None)
        res[f"share_of_bad_accepted_{name}"] = {
            "point": round(point, 4) if point is not None else None,
            "bootstrap95": [round(boots[int(0.025 * len(boots))], 4), round(boots[int(0.975 * len(boots)) - 1], 4)] if boots else None,
            "defined_resamples": len(boots)}

    # Added: vendor-clustered bootstrap on the primary share.
    rng2 = random.Random(SEED + 1)
    acc_slugs = sorted({r["slug"] for r in accB})
    bs = []
    for _ in range(B):
        pick = Counter(rng2.choice(acc_slugs) for _ in acc_slugs)
        n = sum(pick[r["slug"]] for r in accB)
        k = sum(pick[r["slug"]] for r in accB if r["bad"])
        bs.append(k / n)
    bs.sort()
    res["primary_vendor_clustered_bootstrap95"] = [round(bs[int(0.025 * B)], 4), round(bs[int(0.975 * B) - 1], 4)]

    # Secondary 4: existing checks on reviewed fails.
    fails = [r for r in rows if r["bad"]]
    res["existing_checks_on_reviewed_fails"] = {
        "n_fails": len(fails), "ground_flagged": sum(r["ground"] for r in fails),
        "step03_matched": sum(r["s03"] for r in fails),
        "sample_fails_weighted": round(sum(r["bad"] for r in samp) * key["rest_weight"], 1)}

    # Reliability: b5 against the primary labels.
    b5 = labels("b5")
    ids = sorted(b5)
    va = [is_bad(prim[i]) for i in ids]
    vb = [is_bad(b5[i]) for i in ids]
    fa = [prim[i]["flag"] for i in ids]
    fb = [b5[i]["flag"] for i in ids]
    res["reliability_b5"] = {"n": len(ids), "verdict_agree": sum(x == y for x, y in zip(va, vb)),
                             "verdict_kappa": kappa(va, vb), "flag_agree": sum(x == y for x, y in zip(fa, fb)),
                             "verdict_disagreements": [i for i, x, y in zip(ids, va, vb) if x != y]}
    res["low_confidence_primary"] = sum(1 for r in rows if r["confidence"] == "low")
    res["vendor_names_note"] = "Vendor N = alphabetical by slug over the in-scope drafts; mapping not published."
    (ADJ / "results.json").write_text(json.dumps(res, indent=1), encoding="utf-8")
    p = res["primary_bad_share_armB_accepts"]
    print("primary arm-B bad share:", p, "clustered:", res["primary_vendor_clustered_bootstrap95"])
    print("strict:", res["strict_bad_share_armB_accepts"])
    print("armA:", res["armA_bad_share"], "sample:", res["sample_fail_share"], "sample strict:", res["sample_strict_share"])
    print("share of bad accepted B:", res["share_of_bad_accepted_armB"], "A:", res["share_of_bad_accepted_armA"])
    print("checks:", res["existing_checks_on_reviewed_fails"])
    print("reliability:", res["reliability_b5"])
    print("flags B:", res["flags_armB_accepts"], "| sample:", res["flags_sample"])
    print("bad B:", res["bad_armB_accepts"])
    print("bad sample:", res["bad_sample"])


if __name__ == "__main__":
    main()
