#!/usr/bin/env python3
"""WS1 analysis (pre-registered in PREREG.md). Reads shadow_claims.jsonl, shadow_runs.jsonl, driver_runs.jsonl and the
run folders' usage.json. Writes analysis.json. No model calls."""
import collections
import json
import math
import random
from pathlib import Path

from scipy.stats import binomtest

WS1 = Path("out/exp/ws1")
SEED, B = 20260921, 2000
Z = 1.959964


def wilson(k, n):
    if n == 0:
        return None
    p = k / n
    d = 1 + Z * Z / n
    c = (p + Z * Z / (2 * n)) / d
    m = Z * math.sqrt(p * (1 - p) / n + Z * Z / (4 * n * n)) / d
    return {"k": k, "n": n, "p": round(p, 4), "lo": round(max(0, c - m), 4), "hi": round(min(1, c + m), 4),
            "too_small": n < 10}


def mcnemar(pairs):
    b = sum(1 for a, bb in pairs if not a and bb)   # A reject, B accept
    c = sum(1 for a, bb in pairs if a and not bb)   # A accept, B reject
    p = binomtest(min(b, c), b + c, 0.5).pvalue if b + c else 1.0
    return {"b_only_B_accepts": b, "c_only_A_accepts": c, "discordant": b + c, "p_exact_two_sided": round(p, 6)}


def cluster_boot_diff(rows, fa, fb):
    by = collections.defaultdict(list)
    for r in rows:
        by[r["slug"]].append(r)
    vs = sorted(by)
    rng = random.Random(SEED)
    ds = []
    for _ in range(B):
        samp = [r for v in (rng.choice(vs) for _ in vs) for r in by[v]]
        ds.append(sum(fb(r) for r in samp) / len(samp) - sum(fa(r) for r in samp) / len(samp))
    ds.sort()
    return {"lo": round(ds[int(0.025 * B)], 4), "hi": round(ds[int(0.975 * B) - 1], 4), "resamples": B, "seed": SEED,
            "clusters": len(vs)}


def single_accept(x):
    return x["error"] is None and x["flag"] == "none" and x["confidence"] >= 0.8


FAILS = {"dim_disagreement_major", "contradicted_fact", "fabricated_or_unsupported_claim"}


def main():
    rows = [json.loads(l) for l in open(WS1 / "shadow_claims.jsonl", encoding="utf-8")]
    runs = [json.loads(l) for l in open(WS1 / "shadow_runs.jsonl", encoding="utf-8")]
    druns = [json.loads(l) for l in open(WS1 / "driver_runs.jsonl", encoding="utf-8")]
    prim = [r for r in rows if r["set"] == "primary"]
    ins = [r for r in prim if r["scope"] == "in_scope"]
    out = {"n_units": len(prim), "n_in_scope": len(ins), "vendors": sorted({r["slug"] for r in prim})}

    # ---------------- Q1.1
    per = {}
    for s in out["vendors"]:
        u = [r for r in prim if r["slug"] == s]
        i = [r for r in u if r["scope"] == "in_scope"]
        run = next(x for x in runs if x["slug"] == s and x["set"] == "primary")
        per[s] = {"units": len(u), "in_scope": len(i), "skipped": len(u) - len(i),
                  "ground_flag_in_scope": sum(r["ground_flag"] for r in i),
                  "ground_flag_all_units": sum(r["ground_flag"] for r in u),
                  "step03_any_match_in_scope": sum(bool(r["step03"] and r["step03"]["match"]["any"]) for r in i),
                  "step03_url_match_in_scope": sum(bool(r["step03"] and any(r["step03"]["match"][k]["url"] for k in ("flags", "ungrounded_or_mistagged_claims"))) for r in i),
                  "step03_text_or_receipt_match_in_scope": sum(bool(r["step03"] and any(r["step03"]["match"][k]["text"] or r["step03"]["match"][k]["receipt"] for k in ("flags", "ungrounded_or_mistagged_claims"))) for r in i),
                  "step03_verdict": run["step03_verdict"], "step03_confidence": run["step03_confidence"],
                  "driver_exit": run["driver_exit"], "no_receipt_in_scope": sum(r["arm_a_kind"] == "no_receipt" for r in i)}
    counts = [p["in_scope"] for p in per.values()]
    out["q11"] = {"per_dossier": per, "in_scope_mean": round(sum(counts) / len(counts), 2), "in_scope_min": min(counts),
                  "in_scope_max": max(counts),
                  "skip_reasons": dict(collections.Counter(r["skip_reason"] for r in prim if r["scope"] == "skipped")),
                  "ground_status_in_scope": dict(collections.Counter(r["ground_status"] for r in ins)),
                  "ground_flag_share_in_scope": wilson(sum(r["ground_flag"] for r in ins), len(ins)),
                  "step03_any_match_share_in_scope": wilson(sum(p["step03_any_match_in_scope"] for p in per.values()), len(ins)),
                  "step03_url_match_share": wilson(sum(p["step03_url_match_in_scope"] for p in per.values()), len(ins)),
                  "step03_text_match_share": wilson(sum(p["step03_text_or_receipt_match_in_scope"] for p in per.values()), len(ins)),
                  "union_ground_or_step03": wilson(sum(bool(r["ground_flag"] or (r["step03"] and r["step03"]["match"]["any"])) for r in ins), len(ins)),
                  "dedup_in_scope_units": len({(r["slug"], r["claim_sha256"], tuple(r["urls"])) for r in ins})}

    # ---------------- Q1.2
    ok = [r for r in ins if r["accept_A_conservative"] is not None and r["accept_B_conservative"] is not None]
    excl = len(ins) - len(ok)

    def block(sub, label):
        if not sub:
            return {"label": label, "n": 0}
        a = sum(r["accept_A_conservative"] for r in sub)
        b = sum(r["accept_B_conservative"] for r in sub)
        res = {"label": label, "n": len(sub), "vendors": len({r["slug"] for r in sub}),
               "accept_A": wilson(a, len(sub)), "accept_B": wilson(b, len(sub)),
               "diff_B_minus_A": round((b - a) / len(sub), 4), "mcnemar": mcnemar([(r["accept_A_conservative"], r["accept_B_conservative"]) for r in sub])}
        if len({r["slug"] for r in sub}) > 1:
            res["diff_ci_vendor_boot"] = cluster_boot_diff(sub, lambda r: r["accept_A_conservative"], lambda r: r["accept_B_conservative"])
        res["checker_fail_A_any_repeat"] = wilson(sum(r["fail_view_A_any"] for r in sub), len(sub))
        res["checker_fail_B_any_repeat"] = wilson(sum(r["fail_view_B_any"] for r in sub), len(sub))
        return res

    q12 = {"primary": block(ok, "all in-scope, conservative 2-repeat, threshold 0.8"), "excluded_for_errors": excl}
    q12["by_armB_kind"] = {k: block([r for r in ok if r["arm_b_kind"] == k], k) for k in sorted({r["arm_b_kind"] for r in ok})}
    q12["by_armA_kind"] = {k: block([r for r in ok if r["arm_a_kind"] == k], k) for k in sorted({r["arm_a_kind"] for r in ok})}
    q12["receipt_and_verbatim"] = block([r for r in ok if r["arm_a_kind"] == "receipt" and r["arm_b_kind"] == "verbatim_window"], "receipt present and located verbatim")
    q12["by_tag"] = {t: block([r for r in ok if t in r["tags"]], t) for t in ("VENDOR-CLAIMED", "THIRD-PARTY")}
    q12["by_vendor"] = {s: block([r for r in ok if r["slug"] == s], s) for s in out["vendors"]}
    # per-call (single repeat) accept and flag distribution, repeat agreement
    for arm in "AB":
        calls = [x for r in ok for x in r[f"jev_{arm}"]]
        q12[f"per_call_accept_{arm}"] = wilson(sum(single_accept(x) for x in calls), len(calls))
        q12[f"flags_{arm}"] = dict(collections.Counter(x["flag"] for x in calls))
        q12[f"repeat_flag_agreement_{arm}"] = wilson(sum(r[f"jev_{arm}"][0]["flag"] == r[f"jev_{arm}"][1]["flag"] for r in ok), len(ok))
        q12[f"repeat_accept_agreement_{arm}"] = wilson(sum(single_accept(r[f"jev_{arm}"][0]) == single_accept(r[f"jev_{arm}"][1]) for r in ok), len(ok))
        confs = sorted(x["confidence"] for x in calls if x["flag"] == "none")
        q12[f"none_conf_quartiles_{arm}"] = [confs[int(q * (len(confs) - 1))] for q in (0, .25, .5, .75, 1)] if confs else None
    # threshold sweep (exploratory; descriptive only, no rule is chosen here)
    q12["threshold_sweep_conservative"] = {}
    for t in (0.5, 0.6, 0.7, 0.8, 0.9, 0.95):
        q12["threshold_sweep_conservative"][str(t)] = {
            arm: round(sum(all(x["flag"] == "none" and x["confidence"] >= t for x in r[f"jev_{arm}"]) for r in ok) / len(ok), 4) for arm in "AB"}
    out["q12"] = q12

    # ---------------- Q1.3
    stages = collections.defaultdict(list)
    per_run = {}
    for s in out["vendors"]:
        u = json.load(open(WS1 / "onboarding-io" / "runs" / s / "stages" / "usage.json", encoding="utf-8"))
        per_run[s] = {k: {"cost_usd": v["cost_usd"], "prompt_tokens": v["prompt_tokens"], "completion_tokens": v["completion_tokens"],
                          "reasoning_tokens": v.get("reasoning_tokens"), "served_model": v.get("model")} for k, v in u["steps"].items()}
        per_run[s]["total_cost_usd"] = u["total_cost_usd"]
        per_run[s]["pricing_snapshot"] = u.get("pricing_snapshot")
        dr = [d for d in druns if d["slug"] == s]
        per_run[s]["wall_s"] = dr[-1]["wall_s"]
        sc = next(x for x in runs if x["slug"] == s and x["set"] == "primary")
        per_run[s]["sidecar_cost_usd"] = sc["sidecar_cost_usd"]
        per_run[s]["sidecar_wall_s"] = sc["sidecar_wall_s"]
        per_run[s]["sidecar_calls"] = sc["jev_calls"]
        per_run[s]["sidecar_cost_per_in_scope_claim"] = round(sc["sidecar_cost_usd"] / sc["in_scope"], 6) if sc["in_scope"] else None
        for k, v in u["steps"].items():
            stages[k].append(v["cost_usd"])

    def mean(xs):
        return round(sum(xs) / len(xs), 6)
    out["q13"] = {"per_run": per_run,
                  "mean_stage_cost": {k: mean(v) for k, v in stages.items()},
                  "mean_redraft_cost": mean([p["total_cost_usd"] for p in per_run.values()]),
                  "mean_redraft_wall_s": mean([p["wall_s"] for p in per_run.values()]),
                  "mean_sidecar_cost": mean([p["sidecar_cost_usd"] for p in per_run.values()]),
                  "mean_sidecar_wall_s": mean([p["sidecar_wall_s"] for p in per_run.values()]),
                  "fresh_retrieval_runs": [d for d in druns if d["slug"] not in out["vendors"]]}

    # ---------------- Q1.4
    m = out["q11"]["in_scope_mean"]
    proxies = {"union_ground_or_step03 (primary)": out["q11"]["union_ground_or_step03"],
               "ground_only": out["q11"]["ground_flag_share_in_scope"],
               "step03_only": out["q11"]["step03_any_match_share_in_scope"],
               "jev_armB_checker_any_repeat": q12["primary"]["checker_fail_B_any_repeat"]}
    proj = {}
    for name, w in proxies.items():
        proj[name] = {"f": w}
        for h in (0.05, 0.03):
            for p in (0.05, 0.10, 0.20, 0.50):
                nf = math.ceil(Z * Z * p * (1 - p) / (h * h))
                for deff in (1.0, 1.5, 2.0):
                    key = f"h{int(h*100)}_p{int(p*100)}_deff{deff}"
                    need = math.ceil(nf * deff)
                    cell = {"n_fail": need}
                    for fname, fv in (("f_point", w["p"]), ("f_wilson_lo", w["lo"]), ("f_wilson_hi", w["hi"])):
                        if fv and fv > 0:
                            claims = math.ceil(need / fv)
                            cell[fname] = {"claims_to_adjudicate": claims, "dossiers": math.ceil(claims / m)}
                        else:
                            cell[fname] = {"claims_to_adjudicate": None, "dossiers": None, "note": "f = 0: not estimable"}
                    proj[name][key] = cell
    # exploratory ICC (one-way ANOVA estimator) of the Jev arm-B checker flag across dossiers
    groups = [[1 if r["fail_view_B_any"] else 0 for r in ok if r["slug"] == s] for s in out["vendors"]]
    k, N = len(groups), sum(len(g) for g in groups)
    gm = sum(sum(g) for g in groups) / N
    msb = sum(len(g) * (sum(g) / len(g) - gm) ** 2 for g in groups) / (k - 1)
    msw = sum(sum((x - sum(g) / len(g)) ** 2 for x in g) for g in groups) / (N - k)
    n0 = (N - sum(len(g) ** 2 for g in groups) / N) / (k - 1)
    icc = (msb - msw) / (msb + (n0 - 1) * msw) if (msb + (n0 - 1) * msw) > 0 else None
    out["q14"] = {"in_scope_per_dossier_mean": m, "z": Z, "projections": proj,
                  "exploratory_icc_jev_armB_checker": round(icc, 4) if icc is not None else None,
                  "note": "n_fail = ceil(z^2 p(1-p)/h^2 * deff); claims = n_fail/f; dossiers = claims / mean in-scope per dossier"}
    json.dump(out, open(WS1 / "analysis.json", "w", encoding="utf-8"), indent=1)
    print(json.dumps({"n_in_scope": len(ins), "excluded_errors": excl, "q11": {k: out["q11"][k] for k in ("in_scope_mean", "ground_flag_share_in_scope", "step03_any_match_share_in_scope", "union_ground_or_step03")},
                      "q12_primary": q12["primary"]}, indent=1))


if __name__ == "__main__":
    main()
