"""WS4 analysis: catches per unit per arm, registered primary, DV-P, DV-N, Q4.2, Q4.3, Opus secondary.
Writes raw/unit_catches.json and results.json (all numbers the report quotes)."""
from __future__ import annotations
import os

import json
import math
import random
import re
from collections import Counter, defaultdict

import ws4lib as L

RAW = L.WS4 / "raw"
SPIKE = L.WS4.parents[2]
B = 2000


def jl(p):
    return [json.loads(x) for x in p.read_text(encoding="utf-8").splitlines() if x.strip()] if p.exists() else []


def wilson(k, n, z=1.959964):
    if n == 0:
        return (None, None, None)
    p = k / n
    d = 1 + z * z / n
    c = (p + z * z / (2 * n)) / d
    h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
    return (round(p, 4), round(max(0, c - h), 4), round(min(1, c + h), 4))


def mcnemar(b, c):
    n = b + c
    if n == 0:
        return 1.0
    k = min(b, c)
    s = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
    return round(min(1.0, 2 * s), 6)


def cluster_boot(pairs, seed=L.SEED):
    """pairs: list of (vendor, x_bool, y_bool). Returns diff y-x and 95% percentile CI over vendor resamples."""
    byv = defaultdict(list)
    for v, x, y in pairs:
        byv[v].append((x, y))
    vs = sorted(byv)
    if not vs:
        return None, None, None
    rng = random.Random(seed)
    diffs = []
    for _ in range(B):
        s = [rng.choice(vs) for _ in vs]
        xs = ys = n = 0
        for v in s:
            for x, y in byv[v]:
                xs += x
                ys += y
                n += 1
        if n:
            diffs.append((ys - xs) / n)
    diffs.sort()
    n = len(pairs)
    d = (sum(y for _, _, y in pairs) - sum(x for _, x, _ in pairs)) / n
    return round(d, 4), round(diffs[int(0.025 * len(diffs))], 4), round(diffs[int(0.975 * len(diffs)) - 1], 4)


def compute(dset):
    """-> units (with per-arm catch results), call summary rows, dossier-level verdicts."""
    calls = {}
    for c in jl(RAW / "calls.jsonl"):
        if c.get("ok"):
            calls[(c["prompt_sha"], c["rep"])] = c
    jobs = [j for j in jl(RAW / "jobs.jsonl") if j["dset"] == dset]
    slugs = sorted({j["slug"] for j in jobs})
    units_all, dossier = [], {}
    ties_tot = Counter()
    for slug in slugs:
        units, _, _ = L.classify_units(dset, slug)
        for u in units:
            u["key"] = (slug, u["pos"])
        upos = {u["pos"]: u for u in units}
        res = {u["pos"]: defaultdict(dict) for u in units}  # pos -> rule -> rep -> bool
        dv = defaultdict(lambda: defaultdict(list))  # arm -> rep -> verdicts
        sj = [j for j in jobs if j["slug"] == slug]
        arms_present = {j["arm"] for j in sj}
        for j in sj:
            c = calls.get((j["prompt_sha"], j["rep"]))
            arm, rep = j["arm"], j["rep"]
            if c is None or not c.get("parsed"):
                dv[arm][rep].append("error")
                continue
            v = c["verdict"]
            dv[arm][rep].append(v)
            cand = [upos[p] for p in j["unit_pos"] if p in upos]
            cred, ties = L.credit(L.flagged_strings(c["answer"] or {}), cand)
            for u in cand:
                res[u["pos"]][f"{arm}_flag"][rep] = u["key"] in cred
                res[u["pos"]][f"{arm}_verdict"][rep] = v != "pass"
                pass
            ties_tot[arm] += ties
        for u in units:
            r = res[u["pos"]]
            u["catch"] = {}
            for rule, reps in r.items():
                if rule == "_ties":
                    continue
                u["catch"][rule] = any(reps.values()) if reps else None
                u["catch"][rule + "_r1"] = reps.get(1)
                u["catch"][rule + "_r2"] = reps.get(2)
            units_all.append(u)
        dossier[slug] = {arm: {rep: vs for rep, vs in dv[arm].items()} for arm in arms_present}
    dossier["_tie_credits"] = dict(ties_tot)
    return units_all, dossier


PRIMARY_RULE = {"a": "a_flag", "b": "b_flag", "c": "c_verdict"}
SENS_RULE = {"a": "a_verdict", "b": "b_verdict", "c": "c_flag"}


def arm_table(units, sel, rules=PRIMARY_RULE):
    out = {}
    for arm, rule in rules.items():
        xs = [u for u in units if sel(u) and u["catch"].get(rule) is not None]
        k = sum(u["catch"][rule] for u in xs)
        out[arm] = {"rule": rule, "n": len(xs), "k": k, "share_ci": wilson(k, len(xs)),
                    "n_vendors": len({u["slug"] for u in xs})}
    return out


def paired(units, sel, r_x, r_y):
    xs = [u for u in units if sel(u) and u["catch"].get(r_x) is not None and u["catch"].get(r_y) is not None]
    b = sum(1 for u in xs if u["catch"][r_y] and not u["catch"][r_x])
    c = sum(1 for u in xs if u["catch"][r_x] and not u["catch"][r_y])
    d, lo, hi = cluster_boot([(u["slug"], u["catch"][r_x], u["catch"][r_y]) for u in xs])
    return {"x": r_x, "y": r_y, "n": len(xs), "n_vendors": len({u["slug"] for u in xs}),
            "x_share": wilson(sum(u["catch"][r_x] for u in xs), len(xs)),
            "y_share": wilson(sum(u["catch"][r_y] for u in xs), len(xs)),
            "diff_y_minus_x": d, "boot95": [lo, hi], "y_only": b, "x_only": c, "mcnemar_p": mcnemar(b, c)}


def dossier_summary(dossier):
    out = {}
    for arm in ("a", "b", "c"):
        rows = []
        for slug, d in dossier.items():
            if slug.startswith("_") or arm not in d:
                continue
            per = {}
            for rep, vs in d[arm].items():
                ans = [v for v in vs if v != "error"]
                per[rep] = None if not ans else ("fail" if any(v != "pass" for v in ans) else "pass")
            rows.append((slug, per))
        n = len(rows)
        r1 = sum(1 for _, p in rows if p.get(1) == "fail")
        r2 = sum(1 for _, p in rows if p.get(2) == "fail")
        anyr = sum(1 for _, p in rows if "fail" in p.values())
        out[arm] = {"n_dossiers": n, "fail_rep1": r1, "fail_rep2": r2, "fail_any": anyr,
                    "fail_any_share": wilson(anyr, n),
                    "per_vendor": {s: p for s, p in rows}}
    out["tie_credits"] = dossier.get("_tie_credits")
    return out


def cost_summary():
    calls = [c for c in jl(RAW / "calls.jsonl")]
    jobs = jl(RAW / "jobs.jsonl")
    out = {}
    for dset in ("incumbent", "planted", "glm"):
        for arm in ("a", "b", "c"):
            cs = [c for c in calls if c["dset"] == dset and c["arm"] == arm]
            ok = [c for c in cs if c.get("ok")]
            if not cs:
                continue
            slugs = {c["slug"] for c in cs}
            njobs = len([j for j in jobs if j["dset"] == dset and j["arm"] == arm])
            out[f"{dset}/{arm}"] = {
                "calls_made": len(cs), "calls_ok": len(ok), "parsed": sum(1 for c in ok if c.get("parsed")),
                "jobs_incl_reused": njobs, "vendors": len(slugs),
                "cost_usd": round(sum(c.get("cost") or 0 for c in cs), 5),
                "prompt_tokens": sum(c.get("prompt_tokens") or 0 for c in ok),
                "completion_tokens": sum(c.get("completion_tokens") or 0 for c in ok),
                "mean_latency_s": round(sum(c["latency_s"] for c in ok) / max(1, len(ok)), 2),
                "sum_latency_s": round(sum(c["latency_s"] for c in ok), 1),
                "cost_per_dossier_usd": round(sum(c.get("cost") or 0 for c in cs) / max(1, len(slugs)), 5),
                "providers": dict(Counter(c.get("provider") for c in cs)),
                "served_models": dict(Counter(c.get("served_model") for c in cs)),
                "verdicts": dict(Counter(c.get("verdict") for c in ok)),
                "retried": sum(1 for c in cs if (c.get("attempts") or 1) > 1),
                "failed_after_retries": len(cs) - len(ok),
            }
    return out


def opus_secondary(units):
    d = json.loads((SPIKE / "out" / "claims_200" / "claims_200.json").read_text(encoding="utf-8"))
    cl = [c for c in d["claims"] if c["split"] == "dev" and c["vendor"].lower() in tuple(os.environ.get("WS4_SECONDARY_VENDORS", "").split(","))]
    out = []
    for c in cl:
        slug = c["vendor"].lower()
        us = [u for u in units if u["slug"] == slug]
        tc = L.T(c["claim"])
        best, bu = -1, None
        for u in us:
            sc_ = len(tc & L.T(u["claim_text"]))
            eu = L.Nurl(c.get("evidence_url") or "")
            if eu and len(eu) >= 12 and any(L.Nurl(x) == eu for x in u["urls"]):
                sc_ += 3
            if sc_ > best:
                best, bu = sc_, u
        ov = len(tc & L.T(bu["claim_text"])) if bu else 0
        located = bu is not None and ov >= 4 and ov / max(1, len(tc)) >= 0.4
        row = {"id": c["id"], "vendor": slug, "internal_verdict": c["internal_verdict"], "located": located,
               "overlap": ov, "n_claim_tokens": len(tc)}
        if located:
            row.update({"unit_pos": bu["pos"], "ground_cat": bu["cat"],
                        **{a: bu["catch"].get(r) for a, r in PRIMARY_RULE.items()}})
        out.append(row)
    return out


def main():
    R = {}
    U = {}
    for dset in ("incumbent", "planted", "glm"):
        if not any(j["dset"] == dset for j in jl(RAW / "jobs.jsonl")):
            continue
        units, dossier = compute(dset)
        U[dset] = units
        R[f"dossier_{dset}"] = dossier_summary(dossier)
    defect = lambda u: u["cat"] in ("ungrounded", "mistagged", "unresolved", "banned")  # noqa: E731
    clean = lambda u: u["cat"] == "clean"  # noqa: E731
    natural = lambda u: u["cat"] in ("ungrounded", "mistagged", "unresolved", "banned", "auto_fixable")  # noqa: E731
    inc = U.get("incumbent", [])
    R["unit_counts"] = {ds: dict(Counter(u["cat"] for u in us)) for ds, us in U.items()}
    # registered primary
    R["registered_primary"] = {"arms": arm_table(inc, defect),
                               "c_vs_a": paired(inc, defect, "a_flag", "c_verdict")}
    # Q4.3 unneeded flags on clean units (registered)
    R["q43_clean_incumbent"] = {"arms": arm_table(inc, clean),
                                "arms_sensitivity": arm_table(inc, clean, SENS_RULE),
                                "c_vs_a": paired(inc, clean, "a_flag", "c_verdict"),
                                "b_vs_a": paired(inc, clean, "a_flag", "b_flag"),
                                "cflag_vs_a": paired(inc, clean, "a_flag", "c_flag")}
    # DV-P planted
    if "planted" in U:
        plants = json.loads((RAW / "plants.json").read_text(encoding="utf-8"))
        pk = {(p["slug"], p["planted_pos"]): p for p in plants if p.get("confirmed")}
        pl = U["planted"]
        for u in pl:
            p = pk.get((u["slug"], u["pos"]))
            u["plant_type"] = p["type"] if p else None
        isp = lambda u: u.get("plant_type") is not None  # noqa: E731
        R["dvp"] = {"n_confirmed": len(pk),
                    "arms": arm_table(pl, isp), "arms_sensitivity": arm_table(pl, isp, SENS_RULE),
                    "c_vs_a": paired(pl, isp, "a_flag", "c_verdict"),
                    "b_vs_a": paired(pl, isp, "a_flag", "b_flag"),
                    "c_vs_b": paired(pl, isp, "b_flag", "c_verdict"),
                    "cflag_vs_a": paired(pl, isp, "a_flag", "c_flag"),
                    "by_type": {t: arm_table(pl, lambda u, t=t: u.get("plant_type") == t) for t in ("R", "U", "T")},
                    "clean_in_planted": arm_table(pl, clean)}
    # DV-N natural incl auto-fixable
    dvn = {}
    for ds in ("incumbent", "glm"):
        if ds in U:
            dvn[ds] = {"arms": arm_table(U[ds], natural), "c_vs_a": paired(U[ds], natural, "a_flag", "c_verdict"),
                       "cflag_vs_a": paired(U[ds], natural, "a_flag", "c_flag"),
                       "mistag_only": arm_table(U[ds], lambda u: u["cat"] in ("mistagged", "auto_fixable")),
                       "non_mistag": arm_table(U[ds], lambda u: u["cat"] in ("ungrounded", "unresolved", "banned"))}
    pooled = [u for ds in ("incumbent", "glm") for u in U.get(ds, [])]
    for u in pooled:
        u["vkey"] = u["dset"] + ":" + u["slug"]
    if pooled:
        pp = [dict(u, slug=u["slug"]) for u in pooled]  # cluster by vendor across drafts
        dvn["pooled"] = {"arms": arm_table(pp, natural), "c_vs_a": paired(pp, natural, "a_flag", "c_verdict"),
                         "cflag_vs_a": paired(pp, natural, "a_flag", "c_flag")}
    R["dvn"] = dvn
    if "planted" in U:
        idx = {(u["dset"], u["slug"], u["pos"]): u for ds in U for u in U[ds]}
        cnt = Counter()
        rows = []
        for p in (x for x in plants if x.get("confirmed")):
            a = idx[("incumbent", p["slug"], p["orig_pos"])]["catch"].get("c_verdict")
            b = idx[("planted", p["slug"], p["planted_pos"])]["catch"].get("c_verdict")
            cnt[f"{'fail' if a else 'pass'} -> {'fail' if b else 'pass'}"] += 1
            rows.append((a, b))
        order = ["pass -> fail", "fail -> fail", "pass -> pass", "fail -> pass"]
        R["dvp_before_after"] = {"n": len(rows), "counts": {k: cnt.get(k, 0) for k in order},
                                 "mcnemar_p": mcnemar(cnt.get("pass -> fail", 0), cnt.get("fail -> pass", 0))}
    if "glm" in U:
        R["glm_registered"] = {"arms": arm_table(U["glm"], defect),
                               "clean": arm_table(U["glm"], clean),
                               "c_vs_a_clean": paired(U["glm"], clean, "a_flag", "c_verdict"),
                               "vendors": sorted({u["slug"] for u in U["glm"]})}
    R["cost"] = cost_summary()
    R["opus_secondary"] = opus_secondary(inc) if inc else []
    # multi-unit table rows in (c)
    R["c_multi_unit_calls"] = {ds: sum(1 for u in us if u["c_group"].startswith("row") and u["n_units_on_line"] > 1)
                               for ds, us in U.items()}
    (L.WS4 / "results.json").write_text(json.dumps(R, indent=1, default=str), encoding="utf-8")
    slim = [{k: u.get(k) for k in ("dset", "slug", "pos", "cat", "why", "tags", "section", "c_group",
                                   "n_units_on_line", "plant_type", "catch")} for ds in U for u in U[ds]]
    (RAW / "unit_catches.json").write_text(json.dumps(slim, default=str), encoding="utf-8")
    print("ok")


if __name__ == "__main__":
    main()
