#!/usr/bin/env python3
"""WS1 sidecar: Jev shadow check of one onboarding run folder (runs after the driver exits).

  python jev_shadow.py <run_dir> --exit <driver_exit_code> [--set primary|secondary] [--dry-run]

Reads stages/dossier.md, stages/defects.json, stages/03.json (optional), bundle/. Claim units come from
ground.parse_claim_units imported from the scratch worktree. In scope = a sourced tag (VENDOR-CLAIMED / THIRD-PARTY)
plus a cited URL whose page is in bundle/sources_index.json with ok=true. Everything else is logged as skipped.
Arm A evidence = the dossier's own receipt text. Arm B = the receipt located in the saved page plus 2 sentences either
side (cap 4000 chars, trimmed around the match), else the t2_raw_excerpt large keyword window (keyword_window).
Jev: claims_probe.ask request shape unchanged, 2 repeats per arm, current rule (flag none and conf >= 0.8) frozen.
Appends one JSON line per claim unit to out/exp/ws1/shadow_claims.jsonl and one run summary to shadow_runs.jsonl.
Reads OPENROUTER_API_KEY from the environment only; never prints it.
"""
import argparse
import hashlib
import json
import os
import re
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

SPIKE = Path("spike/")
WS1 = SPIKE / "out" / "exp" / "ws1"
WT = Path("../ys-jev-shadow-2026-09-21")
sys.path.insert(0, str(WT / "scripts" / "agent" / "onboarding"))
sys.path.insert(0, str(SPIKE))
import ground  # noqa: E402  (worktree copy, read-only import)
import claims_probe as cp  # noqa: E402  (spike harness, read-only import)
from t2_raw_excerpt import select as t2_select, sentences as t2_sentences  # noqa: E402

MIN_CONF = 0.8
REPEATS = 2
NO_RECEIPT = "(The dossier gives no receipt text for this claim.)"
FAIL_FLAGS = {k for k, v in cp.FLAGS.items() if v[1]}


def sha(s):
    return hashlib.sha256((s or "").encode("utf-8")).hexdigest()


def strip_md(s):
    s = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", s)
    s = re.sub(r"\*+|`+", "", s)
    s = re.sub(r"^\s*(?:[-+>]|\d+\.)\s+", "", s)
    return re.sub(r"\s+", " ", s).strip(" |:;-\u2014")


def claim_text(text, pos, ctx):
    ls = text.rfind("\n", 0, pos) + 1
    line = text[ls:pos]
    if line.lstrip().startswith("|"):
        cells = [strip_md(c) for c in line.strip().strip("|").split("|")]
        cells = [c for c in cells if c and not re.fullmatch(r"-+", c)]
        t = " | ".join(cells)
    else:
        prev = line.rfind("]]")
        t = strip_md(line[prev + 2:] if prev != -1 else line)
    if len(t) < 12:
        t2 = strip_md(ctx or "")
        if len(t2) > len(t):
            return t2, "context_fallback"
    return t, "line"


# ---- page tokenisation that mirrors validate_dossier._normalize_for_match, with raw offsets ----
ENT = re.compile(r"&[a-z]+;|&#\d+;|&#x[0-9a-f]+;", re.I)


def tok_page(raw):
    low = ENT.sub(lambda m: " " * len(m.group(0)), raw.lower())
    return [(m.group(0), m.start(), m.end()) for m in re.finditer(r"[a-z0-9]+", low)]


def locate(receipt, raw):
    """Return (start, end) raw offsets of the receipt in the page, or None."""
    rt = re.findall(r"[a-z0-9]+", ENT.sub(" ", receipt.lower()))
    if len(rt) < 2:
        return None
    pt = tok_page(raw)
    words = [w for w, _, _ in pt]
    joined = " " + " ".join(words) + " "
    needle = " " + " ".join(rt) + " "
    i = joined.find(needle)
    if i < 0:
        return None
    k0 = joined[:i + 1].count(" ") - 1
    k1 = k0 + len(rt) - 1
    return pt[k0][1], pt[k1][2]


def sentence_spans(raw):
    spans, last = [], 0
    for m in re.finditer(r"(?<=[.!?])\s+|\n+", raw):
        spans.append((last, m.start()))
        last = m.end()
    spans.append((last, len(raw)))
    return [(a, b) for a, b in spans if len(raw[a:b].strip()) > 2]


def verbatim_window(raw, s0, s1, n=2, cap=4000):
    sp = sentence_spans(raw)
    lo = next((i for i, (a, b) in enumerate(sp) if b > s0), 0)
    hi = next((i for i, (a, b) in enumerate(sp) if b >= s1), len(sp) - 1)
    a, b = sp[max(0, lo - n)][0], sp[min(len(sp) - 1, hi + n)][1]
    a, b = min(a, s0), max(b, s1)
    trimmed = False
    if b - a > cap:
        core = s1 - s0
        pad = max(0, (cap - core) // 2)
        a, b = max(a, s0 - pad), min(b, s1 + pad)
        trimmed = True
    return re.sub(r"\s+", " ", raw[a:b]).strip(), trimmed


def load_bundle(bdir):
    idx = json.loads((bdir / "sources_index.json").read_text(encoding="utf-8"))
    pages = {}
    for r in idx:
        nu = ground._norm_url(r["url"])
        pages.setdefault(nu, r)
    return pages


def page_text(bdir, rec):
    raw = (bdir / rec["path"].replace("\\", "/")).read_text(encoding="utf-8", errors="replace")
    return re.sub(r"^URL: [^\n]*\n\n", "", raw, count=1)


def ground_status(u, defects):
    res = defects.get("result", {})
    for e in res.get("ungrounded", []):
        if e.get("pos") == u["pos"]:
            return "ungrounded", e.get("why")
    for e in res.get("mistagged", []):
        if e.get("pos") == u["pos"]:
            return "mistagged", e.get("why")
    for e in res.get("auto_fixable", []):
        if e.get("section") == u["section"] and e.get("tags") == u["tags"] and e.get("urls") == u["urls"][:2]:
            return "auto_fixable_mistag", e.get("why_mistag")
    tags = u["tags"]
    if not any(t in ground.SOURCED_TAGS for t in tags) and "UNTAGGED" not in tags:
        return "hedge_not_checked", None
    return "grounded_ok", None


def norm_s(s):
    return re.sub(r"\s+", " ", (s or "").lower()).strip()


def step03_match(ctext, urls, receipt, v03):
    items = {"ungrounded_or_mistagged_claims": [str(x) for x in (v03.get("ungrounded_or_mistagged_claims") or [])],
             "flags": [str(x) for x in (v03.get("flags") or [])]}
    out = {}
    nc, nr = norm_s(ctext), norm_s(receipt)
    for k, lst in items.items():
        url_hit = any(u and u in it for it in lst for u in urls)
        text_hit = any((nc and len(nc) >= 12 and nc in norm_s(it)) or (len(norm_s(it)) >= 20 and norm_s(it) in nc)
                       for it in lst)
        rec_hit = bool(nr) and any(nr in norm_s(it) or (len(norm_s(it)) >= 20 and norm_s(it) in nr) for it in lst)
        out[k] = {"url": url_hit, "text": text_hit, "receipt": rec_hit}
    out["any"] = any(v["url"] or v["text"] or v["receipt"] for v in out.values() if isinstance(v, dict))
    return out


def jev_call(key, claim, evidence, section, as_of):
    c = {"claim": claim, "evidence": evidence, "section": section}
    t0 = time.time()
    try:
        res = cp.ask(key, c, as_of)
        ans = res["answers"]["finding"]
        flag = ans["choice"]
        if flag not in cp.FLAGS:
            raise ValueError(f"unknown flag {flag!r}")
        use = res.get("usage") or {}
        meta = {k: v for k, v in res.items() if k not in ("answers", "usage") and not isinstance(v, (dict, list))}
        return {"flag": flag, "confidence": float(ans.get("confidence") or 0.0),
                "probs": ans.get("probabilities") or {}, "cost": float(use.get("cost") or 0.0),
                "prompt_tokens": int(use.get("prompt_tokens") or use.get("input_tokens") or 0),
                "served": meta, "ms": round((time.time() - t0) * 1000), "error": None}
    except Exception as e:  # never counted as an answer
        return {"flag": None, "confidence": None, "probs": {}, "cost": 0.0, "prompt_tokens": 0, "served": {},
                "ms": round((time.time() - t0) * 1000), "error": f"{type(e).__name__}: {str(e)[:160]}"}


def accept(r):
    return r["error"] is None and r["flag"] == "none" and r["confidence"] >= MIN_CONF


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("run_dir")
    ap.add_argument("--exit", type=int, required=True)
    ap.add_argument("--set", default="primary", choices=["primary", "secondary"])
    ap.add_argument("--as-of", default="2026-09-21")
    ap.add_argument("--dry-run", action="store_true")
    a = ap.parse_args()
    rd = Path(a.run_dir)
    slug = rd.name
    bdir, sdir = rd / "bundle", rd / "stages"
    text = (sdir / "dossier.md").read_text(encoding="utf-8")
    defects = json.loads((sdir / "defects.json").read_text(encoding="utf-8"))
    v03 = json.loads((sdir / "03.json").read_text(encoding="utf-8")) if (sdir / "03.json").exists() else None
    usage = json.loads((sdir / "usage.json").read_text(encoding="utf-8")) if (sdir / "usage.json").exists() else {}
    commit = subprocess.run(["git", "-C", str(WT), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip()
    pages = load_bundle(bdir)
    units = ground.parse_claim_units(text)
    run_id = f"ws1-{slug}-{sha(text)[:10]}"
    sec_idx = {}
    rows = []
    for gi, u in enumerate(units):
        sec_idx[u["section"]] = sec_idx.get(u["section"], -1) + 1
        ctext, ctext_src = claim_text(text, u["pos"], u["context"])
        gst, gwhy = ground_status(u, defects)
        row = {"run_id": run_id, "set": a.set, "slug": slug, "date": a.as_of, "worktree_commit": commit,
               "driver_exit": a.exit, "claim_id": f"{u['section']}#{sec_idx[u['section']]}", "unit_index": gi,
               "section": u["section"], "claim": ctext, "claim_text_source": ctext_src, "claim_sha256": sha(ctext),
               "tags": u["tags"], "urls": u["urls"], "receipt": u["receipt"],
               "ground_status": gst, "ground_why": gwhy, "ground_flag": gst in ("ungrounded", "mistagged")}
        sourced = any(t in ground.SOURCED_TAGS for t in u["tags"])
        cited = []
        for url in u["urls"]:
            rec = pages.get(ground._norm_url(url)) or pages.get(ground._norm_url(ground._unwrap_wayback(url)))
            cited.append((url, rec))
        ok_pages = [(url, rec) for url, rec in cited if rec and rec.get("ok") and rec.get("path")]
        if not sourced:
            row["scope"], row["skip_reason"] = "skipped", "no sourced tag (" + ",".join(u["tags"]) + ")"
        elif not u["urls"]:
            row["scope"], row["skip_reason"] = "skipped", "sourced tag with no URL"
        elif not any(rec for _, rec in cited):
            row["scope"], row["skip_reason"] = "skipped", "cited URL not in bundle sources_index"
        elif not ok_pages:
            row["scope"], row["skip_reason"] = "skipped", "cited page in bundle but not ok"
        else:
            row["scope"], row["skip_reason"] = "in_scope", None
            ev_a = u["receipt"] if u["receipt"] else NO_RECEIPT
            ev_b, kind, src, trimmed = None, None, None, False
            if u["receipt"]:
                for url, rec in ok_pages:
                    raw = page_text(bdir, rec)
                    loc = locate(u["receipt"], raw)
                    if loc:
                        ev_b, trimmed = verbatim_window(raw, *loc)
                        kind, src = "verbatim_window", (url, rec)
                        break
            if ev_b is None:
                best = (-1, "", None)
                for url, rec in ok_pages:
                    ex, score = t2_select(ctext, page_text(bdir, rec), 8, 4000)
                    if score > best[0]:
                        best = (score, ex, (url, rec))
                ev_b, src = best[1], best[2]
                kind = "keyword_window" if ev_b else "none_found"
                row["keyword_score"] = best[0]
            row.update({"arm_a_evidence": ev_a, "arm_a_kind": "receipt" if u["receipt"] else "no_receipt",
                        "arm_a_sha256": sha(ev_a), "arm_b_evidence": ev_b, "arm_b_kind": kind,
                        "arm_b_sha256": sha(ev_b), "arm_b_trimmed": trimmed,
                        "source_url": src[0], "source_idx": src[1].get("idx"), "source_tier": src[1].get("tier"),
                        "saved_page_path": str((bdir / src[1]["path"].replace("\\", "/")).resolve())})
        if v03 is not None:
            row["step03"] = {"dossier_verdict": v03.get("verdict"), "confidence": v03.get("confidence"),
                             "match": step03_match(ctext, u["urls"], u["receipt"], v03)}
        else:
            row["step03"] = None
        rows.append(row)

    todo = [(i, arm, rep) for i, r in enumerate(rows) if r["scope"] == "in_scope" for arm in "AB" for rep in range(REPEATS)]
    t0 = time.time()
    if not a.dry_run and todo:
        key = os.environ.get("OPENROUTER_API_KEY")
        if not key:
            raise SystemExit("OPENROUTER_API_KEY not in environment")

        def work(t):
            i, arm, rep = t
            r = rows[i]
            ev = r["arm_a_evidence"] if arm == "A" else r["arm_b_evidence"]
            return t, jev_call(key, r["claim"], ev, r["section"], a.as_of)

        with ThreadPoolExecutor(max_workers=3) as ex:
            for (i, arm, rep), res in ex.map(work, todo):
                rows[i].setdefault(f"jev_{arm}", [None] * REPEATS)[rep] = res
        for r in rows:
            if r["scope"] != "in_scope":
                continue
            for arm in "AB":
                reps = r[f"jev_{arm}"]
                r[f"accept_{arm}_conservative"] = None if any(x["error"] for x in reps) else all(accept(x) for x in reps)
                r[f"fail_view_{arm}_any"] = None if any(x["error"] for x in reps) else any(x["flag"] in FAIL_FLAGS for x in reps)
    wall = time.time() - t0
    cost = sum(x["cost"] for r in rows for arm in "AB" for x in (r.get(f"jev_{arm}") or []) if x)
    errs = sum(1 for r in rows for arm in "AB" for x in (r.get(f"jev_{arm}") or []) if x and x["error"])
    outc = WS1 / ("shadow_claims.jsonl" if not a.dry_run else "shadow_claims_dryrun.jsonl")
    with open(outc, "a", encoding="utf-8") as f:
        for r in rows:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    summ = {"run_id": run_id, "set": a.set, "slug": slug, "driver_exit": a.exit, "dry_run": a.dry_run,
            "units": len(rows), "in_scope": sum(r["scope"] == "in_scope" for r in rows),
            "skipped": sum(r["scope"] == "skipped" for r in rows), "jev_calls": len(todo) if not a.dry_run else 0,
            "jev_errors": errs, "sidecar_cost_usd": round(cost, 6), "sidecar_wall_s": round(wall, 1),
            "step03_verdict": (v03 or {}).get("verdict"), "step03_confidence": (v03 or {}).get("confidence"),
            "pipeline_cost_usd": usage.get("total_cost_usd"), "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}
    with open(WS1 / ("shadow_runs.jsonl" if not a.dry_run else "shadow_runs_dryrun.jsonl"), "a", encoding="utf-8") as f:
        f.write(json.dumps(summ) + "\n")
    print(json.dumps(summ))


if __name__ == "__main__":
    main()
