import os
#!/usr/bin/env python3
"""T2 (2026-09-21): rebuild the evidence for claims whose frozen source page exists, using the RAW page text.

  python t2_raw_excerpt.py [--split dev] [--window small|large] --out out/claims_200/t2_raw_<window>_dev.json

Only claims whose `evidence_url` is present in a frozen bundle (`sources_index.json`) are kept; nothing is reconstructed.
Bundles: ../ys-eval-bundle/io/runs/<slug>/bundle/ (frozen 2026-06-12/13 by retrieve_bundle.py).
Excerpt selection is deterministic and was fixed before any run: split the page into sentences, score each sentence
3 x (numbers shared with the claim) + 1 x (distinct content words shared), take the best sentence plus N neighbours
either side (small: N=2, cap 1200 chars; large: N=8, cap 4000 chars). A best score of 0 means no excerpt; the claim keeps
an empty evidence string, which Jev must escalate (fail-closed). Output is a claims file claims_probe.py can run.
"""
import argparse
import json
import re

RUNS = "../ys-eval-bundle/io/runs"
SLUGS = json.load(open(os.environ.get("T2_SLUGS_FILE", "slugs.json"), encoding="utf-8"))  # release: display-name -> slug map loaded from data, not hard-coded
WIN = {"small": (2, 1200), "large": (8, 4000)}
STOP = set("that this with from have their they which were been about into more than also over such only other these those "
           "most very your will what when where while some each after before under between through".split())


def norm_url(u):
    u = re.sub(r"^https?://(www\.)?", "", (u or "").lower().strip())
    return u.split("#")[0].rstrip("/")


def tokens(s):
    nums = set(re.findall(r"\d[\d,.]*", s))
    nums = {n.rstrip(".,").replace(",", "") for n in nums if n.rstrip(".,")}
    words = {w for w in re.findall(r"[a-z][a-z\-]{3,}", s.lower()) if w not in STOP}
    return nums, words


def sentences(text):
    parts = re.split(r"(?<=[.!?])\s+|\n+", text)
    return [p.strip() for p in parts if len(p.strip()) > 2]


def select(claim, page, n, cap):
    cn, cw = tokens(claim)
    sents = sentences(page)
    best, bi = 0, -1
    for i, s in enumerate(sents):
        sn, sw = tokens(s)
        sc = 3 * len(cn & sn) + len(cw & sw)
        if sc > best:
            best, bi = sc, i
    if bi < 0:
        return "", 0
    lo, hi = max(0, bi - n), min(len(sents), bi + n + 1)
    ex = " ".join(sents[lo:hi])
    if len(ex) > cap:  # trim around the best sentence
        core = sents[bi]
        start = max(0, ex.find(core) - (cap - len(core)) // 2)
        ex = ex[start:start + cap]
    return ex, best


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--split", default="dev", choices=["dev", "test", "all"])
    ap.add_argument("--window", default="small", choices=list(WIN))
    ap.add_argument("--out", required=True)
    a = ap.parse_args()
    D = json.load(open("out/claims_200/claims_200.json", encoding="utf-8"))
    n, cap = WIN[a.window]
    out, dropped, stats = [], [], []
    for c in D["claims"]:
        if c["vendor"] not in SLUGS or (a.split != "all" and c["split"] != a.split):
            continue
        b = f"{RUNS}/{SLUGS[c['vendor']]}/bundle"
        idx = json.load(open(f"{b}/sources_index.json", encoding="utf-8"))
        hit = [s for s in idx if norm_url(s["url"]) == norm_url(str(c.get("evidence_url") or "")) and s.get("ok")]
        if not hit:
            dropped.append(c["id"])
            continue
        page = open(f"{b}/{hit[0]['path'].replace(chr(92), '/')}", encoding="utf-8", errors="replace").read()
        ex, score = select(c["claim"], page, n, cap)
        old = c["evidence"]
        q = re.findall(r"\"([^\"]{12,})\"", old)
        quote_in_window = any(x[:40].lower() in ex.lower() for x in q) if q else None
        d = dict(c, evidence=ex, evidence_kind=f"raw_bundle_window_{a.window}", evidence_is_verbatim=True,
                 t2_old_evidence_kind=c["evidence_kind"], t2_old_evidence=old[:600], t2_select_score=score,
                 t2_quote_in_window=quote_in_window, t2_source=hit[0]["url"], t2_page_bytes=hit[0]["bytes"])
        out.append(d)
        stats.append((c["id"], score, len(ex), quote_in_window))
    json.dump({"as_of": D["as_of"], "seed": D.get("seed"), "note": f"T2 raw bundle excerpts, window={a.window}", "dropped_no_bundle_page": dropped,
               "claims": out}, open(a.out, "w", encoding="utf-8"), indent=1)
    print(f"kept {len(out)} dropped {len(dropped)} {dropped}")
    print("no excerpt (score 0):", [s[0] for s in stats if s[1] == 0])
    print("median excerpt chars:", sorted(s[2] for s in stats)[len(stats) // 2] if stats else None)
    qs = [s[3] for s in stats if s[3] is not None]
    print(f"old quoted text found in the selected window: {sum(qs)}/{len(qs)}")


if __name__ == "__main__":
    main()
