#!/usr/bin/env python3
"""WS2: Jev confidence calibration, candidate accept scores under vendor-grouped CV, trade-off curves, sample sizes.

Implements PREREG.md (this folder). No API calls. Reads only out/claims_200/{results_dev,results_test,claims_200,
label_provenance}.json. Writes ws2_results.json next to this script.

  python ws2_analysis.py
"""
import hashlib
import json
import math
import os
import sys
from math import comb

import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
SPIKE = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..", ".."))
sys.path.insert(0, SPIKE)
from claims_probe import FLAGS, wilson_upper  # noqa: E402  (read-only import)

DATA = os.path.abspath(os.path.join(HERE, "..", "data"))
FOLD_SEED = "ws2-2026-09-21"
BOOT_SEED = 20260921
N_BOOT = 2000
K_FOLDS = 5
Z = 1.96
FAIL_FLAGS = [f for f, (_, fails) in FLAGS.items() if fails]
assert sorted(FAIL_FLAGS) == sorted(["dim_disagreement_major", "contradicted_fact", "fabricated_or_unsupported_claim"])


# ---------------------------------------------------------------- stats helpers
def wilson(k, n, z=Z):
    """Wilson 95% interval; k may be non-integer (used for planning)."""
    if n == 0:
        return (float("nan"), float("nan"), float("nan"))
    p = k / n
    d = 1 + z * z / n
    c = p + z * z / (2 * n)
    m = z * math.sqrt(max(p * (1 - p) / n + z * z / (4 * n * n), 0.0))
    return (p, max(0.0, (c - m) / d), min(1.0, (c + m) / d))


def prop(k, n):
    p, lo, hi = wilson(k, n)
    out = {"k": int(k), "n": int(n), "p": round(p, 4) if n else None, "lo": round(lo, 4) if n else None,
           "hi": round(hi, 4) if n else None}
    if n and n < 10:
        out["too_small"] = True
    return out


def mcnemar_exact(b, c):
    """Two-sided exact McNemar: binomial(b+c, 0.5) on the discordant pairs."""
    n = b + c
    if n == 0:
        return 1.0
    k = min(b, c)
    tail = sum(comb(n, i) for i in range(k + 1)) / 2 ** n
    return min(1.0, 2 * tail)


def fold_of(vendor):
    return int(hashlib.sha256((FOLD_SEED + vendor).encode()).hexdigest(), 16) % K_FOLDS


# ---------------------------------------------------------------- data
def load():
    claims = json.load(open(os.path.join(DATA, "claims_200.json"), encoding="utf-8"))["claims"]
    prov = json.load(open(os.path.join(DATA, "label_provenance.json"), encoding="utf-8"))
    key = {c["id"]: c for c in claims}
    out = {}
    notes = {"key_mismatch": [], "excluded_error_calls": 0, "excluded_missing_rep": [], "excluded_bad_verdict": []}
    for split in ("dev", "test"):
        rows = json.load(open(os.path.join(DATA, f"results_{split}.json"), encoding="utf-8"))["rows"]
        by = {}
        for r in rows:
            if r.get("error"):
                notes["excluded_error_calls"] += 1
                continue
            by.setdefault(r["id"], []).append(r)
        recs = []
        for cid, rs in by.items():
            c = key[cid]
            v = c["internal_verdict"]
            if v not in ("pass", "fail"):
                notes["excluded_bad_verdict"].append(cid)
                continue
            if any(r["key_verdict"] != v for r in rs):
                notes["key_mismatch"].append(cid)
            rs = sorted(rs, key=lambda r: r["rep"])
            reps = [{"flag": r["flag"], "conf": float(r["confidence"]), "probs": r["probs"]} for r in rs]
            if len(reps) != 2:
                notes["excluded_missing_rep"].append(cid)
            recs.append({"id": cid, "vendor": c["vendor"], "split": split, "fail": v == "fail",
                         "prov_only": bool(rs[0]["prov_only"]), "verbatim": bool(c.get("evidence_is_verbatim")),
                         "producer": (prov.get(cid) or {}).get("producer", "unknown"), "reps": reps})
        out[split] = recs
    return out, notes


def cn(rep):
    return rep["conf"] if rep["flag"] == "none" else 0.0


def pn(rep):
    return float(rep["probs"].get("none", 0.0))


def pnf(rep):
    return 1.0 - sum(float(rep["probs"].get(f, 0.0)) for f in FAIL_FLAGS)


SCORES = {
    "C0": ("min over 2 repeats of confidence-if-none (current rule family)", lambda r: min(cn(x) for x in r)),
    "K1": ("confidence-if-none, repeat 0 only", lambda r: cn(r[0])),
    "K2": ("min over 2 repeats of P(none)", lambda r: min(pn(x) for x in r)),
    "K3": ("mean over 2 repeats of P(none)", lambda r: sum(pn(x) for x in r) / len(r)),
    "K4": ("P(none), repeat 0 only", lambda r: pn(r[0])),
    "K5": ("min over 2 repeats of 1 - P(fail flags)", lambda r: min(pnf(x) for x in r)),
    "K6": ("mean over 2 repeats of 1 - P(fail flags)", lambda r: sum(pnf(x) for x in r) / len(r)),
    "K7": ("1 - P(fail flags), repeat 0 only", lambda r: pnf(r[0])),
}
CANDIDATES = ["K1", "K2", "K3", "K4", "K5", "K6", "K7"]


# ---------------------------------------------------------------- calibration
def calib(conf, y, bins=10):
    conf = np.asarray(conf, float)
    y = np.asarray(y, float)
    N = len(conf)
    if N == 0:
        return None
    idx = np.minimum((conf * bins).astype(int), bins - 1)
    table, ece = [], 0.0
    for b in range(bins):
        m = idx == b
        nb = int(m.sum())
        if nb == 0:
            continue
        mc = float(conf[m].mean())
        k = int(y[m].sum())
        ece += nb / N * abs(k / nb - mc)
        row = {"bin": f"[{b / bins:.1f},{(b + 1) / bins:.1f}{']' if b == bins - 1 else ')'}", "mean_conf": round(mc, 4)}
        row.update(prop(k, nb))
        table.append(row)
    gap = float(conf.mean() - y.mean())
    return {"n_calls": N, "ece": round(ece, 4), "signed_gap_conf_minus_observed": round(gap, 4),
            "mean_conf": round(float(conf.mean()), 4), "observed_share": round(float(y.mean()), 4), "table": table}


def calls(recs, which, conf_fn=lambda rep: rep["conf"], reps=(0, 1)):
    """which: 'none' -> outcome recorded pass; 'failflag' -> outcome recorded fail; 'minor' -> outcome recorded pass."""
    out = []
    for c in recs:
        for i in reps:
            rep = c["reps"][i]
            f = rep["flag"]
            if which == "none" and f == "none":
                out.append((c["vendor"], conf_fn(rep), 0 if c["fail"] else 1))
            elif which == "failflag" and f in FAIL_FLAGS:
                out.append((c["vendor"], conf_fn(rep), 1 if c["fail"] else 0))
            elif which == "minor" and f != "none" and f not in FAIL_FLAGS:
                out.append((c["vendor"], conf_fn(rep), 0 if c["fail"] else 1))
    return out


def calib_with_ci(cl, rng):
    if not cl:
        return None
    res = calib([x[1] for x in cl], [x[2] for x in cl])
    vendors = sorted({x[0] for x in cl})
    byv = {v: [x for x in cl if x[0] == v] for v in vendors}
    eces, gaps = [], []
    for _ in range(N_BOOT):
        pick = rng.choice(len(vendors), len(vendors), replace=True)
        s = [x for j in pick for x in byv[vendors[j]]]
        r = calib([x[1] for x in s], [x[2] for x in s])
        eces.append(r["ece"])
        gaps.append(r["signed_gap_conf_minus_observed"])
    res["ece_ci95"] = [round(float(np.percentile(eces, 2.5)), 4), round(float(np.percentile(eces, 97.5)), 4)]
    res["gap_ci95"] = [round(float(np.percentile(gaps, 2.5)), 4), round(float(np.percentile(gaps, 97.5)), 4)]
    res["n_vendors"] = len(vendors)
    return res


# ---------------------------------------------------------------- thresholds and CV
def cuts_for(scores):
    u = np.unique(np.asarray(scores, float))
    mids = list((u[:-1] + u[1:]) / 2) if len(u) > 1 else []
    return [float(u[0]) - 1.0] + [float(m) for m in mids] + [math.inf]


def pick_cut(s, isfail, cap, count_fail=None):
    """Most permissive cut (ascending) whose training miss rate <= cap. count_fail masks which fails count."""
    s = np.asarray(s, float)
    f = np.asarray(isfail, bool) if count_fail is None else np.asarray(count_fail, bool)
    nf = int(f.sum())
    for t in cuts_for(s):
        misses = int((f & (s >= t)).sum())
        if (nf == 0 and misses == 0) or (nf and misses / nf <= cap + 1e-12):
            return t
    return math.inf


def cv_run(recs, score_ids, cap, exclude_prov=False, nested=True):
    """Returns per-claim out-of-fold accept decisions and per-fold selections."""
    folds = np.array([fold_of(c["vendor"]) for c in recs])
    isfail = np.array([c["fail"] for c in recs])
    countf = np.array([c["fail"] and not (exclude_prov and c["prov_only"]) for c in recs])
    S = {k: np.array([SCORES[k][1](c["reps"]) for c in recs]) for k in score_ids}
    accept = np.zeros(len(recs), bool)
    chosen = []
    for k in range(K_FOLDS):
        tr, te = folds != k, folds == k
        best = None
        for sid in score_ids:
            t = pick_cut(S[sid][tr], isfail[tr], cap, countf[tr])
            passes = tr & ~isfail
            a = float((S[sid][passes] >= t).mean()) if passes.any() else 0.0
            if best is None or a > best[2] + 1e-12:
                best = (sid, t, a)
            if not nested:
                break
        sid, t, a = best
        accept[te] = S[sid][te] >= t
        chosen.append({"fold": k, "score": sid, "cut": None if math.isinf(t) else round(t, 5), "train_accept_share_pass": round(a, 4),
                       "n_test_claims": int(te.sum()), "n_test_vendors": len({recs[i]["vendor"] for i in np.where(te)[0]})})
    return accept, chosen


def summarize_accept(recs, accept, exclude_prov=False):
    isfail = np.array([c["fail"] for c in recs])
    countf = np.array([c["fail"] and not (exclude_prov and c["prov_only"]) for c in recs])
    passes = ~isfail
    return {"accept_share_recorded_pass": prop(int((accept & passes).sum()), int(passes.sum())),
            "misses": prop(int((accept & countf).sum()), int(countf.sum())),
            "accept_share_all_claims": prop(int(accept.sum()), len(recs)),
            "precision_of_accepts_recorded_pass": prop(int((accept & passes).sum()), int(accept.sum()))}


def paired(recs, a_cand, a_base, rng):
    isfail = np.array([c["fail"] for c in recs])
    passes = ~isfail
    out = {}
    for name, m in (("recorded_pass", passes), ("recorded_fail", isfail)):
        b = int((a_cand & ~a_base & m).sum())
        c = int((~a_cand & a_base & m).sum())
        out[f"mcnemar_{name}"] = {"cand_only_accepts": b, "base_only_accepts": c, "p_exact": round(mcnemar_exact(b, c), 4)}
    vendors = sorted({c["vendor"] for c in recs})
    vid = {v: i for i, v in enumerate(vendors)}
    cv = np.array([vid[c["vendor"]] for c in recs])
    diffs = []
    for _ in range(N_BOOT):
        pick = rng.choice(len(vendors), len(vendors), replace=True)
        w = np.bincount(pick, minlength=len(vendors))[cv]
        wp = w * passes
        if wp.sum() == 0:
            continue
        diffs.append(float((wp * a_cand).sum() / wp.sum() - (wp * a_base).sum() / wp.sum()))
    d0 = float(a_cand[passes].mean() - a_base[passes].mean())
    out["diff_accept_share_recorded_pass"] = {"point": round(d0, 4), "ci95": [round(float(np.percentile(diffs, 2.5)), 4),
                                                                                round(float(np.percentile(diffs, 97.5)), 4)],
                                              "n_boot": len(diffs)}
    return out


# ---------------------------------------------------------------- curves and sample sizes
def curve(recs, sid):
    s = np.array([SCORES[sid][1](c["reps"]) for c in recs])
    isfail = np.array([c["fail"] for c in recs])
    pts = []
    for t in cuts_for(s):
        acc = s >= t
        m = prop(int((acc & isfail).sum()), int(isfail.sum()))
        a = prop(int((acc & ~isfail).sum()), int((~isfail).sum()))
        pts.append({"cut": None if math.isinf(t) else round(t, 5), "miss_rate": m, "accept_share_recorded_pass": a,
                    "accept_share_all": round(float(acc.mean()), 4)})
    return pts


def n_normal(p, d):
    return math.ceil(Z * Z * p * (1 - p) / (d * d))


def n_wilson(p, d, nmax=200000):
    for n in range(1, nmax):
        _, lo, hi = wilson(p * n, n)
        if max(hi - p, p - lo) <= d:
            return n
    return None


def main():
    data, notes = load()
    rng = np.random.default_rng(BOOT_SEED)
    dev, test = data["dev"], data["test"]
    R = {"notes": notes, "counts": {}, "folds": {}}
    for sp, recs in data.items():
        R["counts"][sp] = {"claims": len(recs), "vendors": len({c["vendor"] for c in recs}),
                           "recorded_fail": sum(c["fail"] for c in recs), "recorded_pass": sum(not c["fail"] for c in recs),
                           "prov_only_fails": sum(c["fail"] and c["prov_only"] for c in recs)}
    for k in range(K_FOLDS):
        fr = [c for c in dev if fold_of(c["vendor"]) == k]
        R["folds"][k] = {"claims": len(fr), "vendors": len({c["vendor"] for c in fr}), "recorded_fail": sum(c["fail"] for c in fr)}

    # sanity: our Wilson upper equals claims_probe.wilson_upper
    assert abs(wilson(3, 87)[2] - wilson_upper(3, 87)) < 1e-12

    # ---------------- 2.1 calibration
    c21 = {}
    for sp, recs in (("dev", dev), ("test", test)):
        c21[sp] = {
            "none_conf_pooled": calib_with_ci(calls(recs, "none"), rng),
            "failflag_conf_pooled": calib_with_ci(calls(recs, "failflag"), rng),
            "none_conf_rep0": calib_with_ci(calls(recs, "none", reps=(0,)), rng),
        }
    c21["dev"]["exploratory"] = {
        "none_pnone_pooled": calib_with_ci(calls(dev, "none", conf_fn=pn), rng),
        "minor_flags_conf_outcome_pass": calib(*zip(*[(x[1], x[2]) for x in calls(dev, "minor")])),
        "none_conf_verbatim": calib_with_ci(calls([c for c in dev if c["verbatim"]], "none"), rng),
        "none_conf_paraphrased": calib_with_ci(calls([c for c in dev if not c["verbatim"]], "none"), rng),
        "none_conf_opus_labels": calib_with_ci(calls([c for c in dev if c["producer"] == "claude_opus_validator"], "none"), rng),
        "none_conf_other_labels": calib_with_ci(calls([c for c in dev if c["producer"] != "claude_opus_validator"], "none"), rng),
    }
    R["q2_1"] = c21

    # existing rule (C0 >= 0.8), dev and held-out, descriptive
    er = {}
    for sp, recs in (("dev", dev), ("test", test)):
        s = np.array([SCORES["C0"][1](c["reps"]) for c in recs])
        s1 = np.array([SCORES["K1"][1](c["reps"]) for c in recs])
        er[sp] = {"two_repeat": summarize_accept(recs, s >= 0.8), "one_repeat_rep0": summarize_accept(recs, s1 >= 0.8),
                  "two_repeat_excl_prov": summarize_accept(recs, s >= 0.8, exclude_prov=True)}
    R["existing_rule_0p8"] = er

    # ---------------- 2.2 CV
    q22 = {}
    for label, cap, excl in (("zero_miss", 0.0, False), ("cap_5pct", 0.05, False), ("cap_10pct", 0.10, False),
                             ("zero_miss_excl_prov", 0.0, True)):
        a_c0, ch_c0 = cv_run(dev, ["C0"], cap, excl, nested=False)
        a_nc, ch_nc = cv_run(dev, CANDIDATES, cap, excl, nested=True)
        per = {}
        for sid in CANDIDATES:
            a, ch = cv_run(dev, [sid], cap, excl, nested=False)
            per[sid] = {"summary": summarize_accept(dev, a, excl), "folds": ch}
        q22[label] = {"C0": {"summary": summarize_accept(dev, a_c0, excl), "folds": ch_c0},
                      "nested_candidate": {"summary": summarize_accept(dev, a_nc, excl), "folds": ch_nc},
                      "paired_candidate_vs_C0": paired(dev, a_nc, a_c0, rng),
                      "per_score_exploratory": per}
    R["q2_2"] = q22

    sel = [f["score"] for f in q22["zero_miss"]["nested_candidate"]["folds"]]
    counts = {s: sel.count(s) for s in CANDIDATES}
    top = max(counts.values())
    tied = [s for s in CANDIDATES if counts[s] == top]
    if len(tied) > 1:
        shares = {s: q22["zero_miss"]["per_score_exploratory"][s]["summary"]["accept_share_recorded_pass"]["p"] for s in tied}
        best_share = max(shares.values())
        tied = [s for s in tied if shares[s] == best_share]
    best = tied[0]
    R["best_candidate"] = {"id": best, "desc": SCORES[best][0], "fold_selections": sel}

    # ---------------- 2.3 curves, operating points, sample sizes
    R["q2_3"] = {"curves": {sid: curve(dev, sid) for sid in ("C0", best)}}
    base_rate = R["counts"]["dev"]["recorded_fail"] / R["counts"]["dev"]["claims"]
    ops = []
    for sid in ("C0", best):
        for label, cap in (("cv_zero_miss", 0.0), ("cv_cap_5pct", 0.05), ("cv_cap_10pct", 0.10)):
            a, ch = cv_run(dev, [sid], cap, nested=False)
            ops.append({"score": sid, "point": label, "summary": summarize_accept(dev, a), "cuts": [f["cut"] for f in ch]})
    s0 = np.array([SCORES["C0"][1](c["reps"]) for c in dev])
    ops.append({"score": "C0", "point": "deployed_0.8_in_sample", "summary": summarize_accept(dev, s0 >= 0.8), "cuts": [0.8]})
    for op in ops:
        m = op["summary"]["misses"]
        rows = []
        for assume, p in (("oof_point", m["p"]), ("oof_wilson_upper", m["hi"])):
            for d in (0.03, 0.05):
                nn = n_normal(p, d)
                nw = n_wilson(p, d)
                rows.append({"assumed": assume, "p": p, "half_width": d, "n_fails_normal": nn,
                             "normal_degenerate": p == 0, "n_fails_wilson": nw,
                             "claims_to_adjudicate_wilson": {f"base_{r}": math.ceil(nw / r) for r in (round(base_rate, 4), 0.2, 0.1)},
                             "claims_to_adjudicate_normal": {f"base_{r}": (math.ceil(nn / r) if nn else 0) for r in (round(base_rate, 4), 0.2, 0.1)}})
        op["sample_sizes"] = rows
    R["q2_3"]["operating_points"] = ops
    R["q2_3"]["dev_base_rate"] = round(base_rate, 4)

    with open(os.path.join(HERE, "ws2_results.json"), "w", encoding="utf-8") as fh:
        json.dump(R, fh, indent=1)
    print("wrote ws2_results.json; best candidate", best, "selections", sel)


if __name__ == "__main__":
    main()
