"""WS4 shared library: claim units, labels, arm texts, prompt building, matching, model call.

Production code is imported READ-ONLY from the WS4 worktree (origin/main cedc26f1). Nothing is written there
(run with PYTHONDONTWRITEBYTECODE=1). All outputs go under out/exp/ws4/.
"""
from __future__ import annotations

import hashlib
import json
import os
import random
import re
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path

sys.dont_write_bytecode = True
WS4 = Path(__file__).resolve().parents[1]
WT_ONB = Path("../ys-jev-granularity-2026-09-21/scripts/agent/onboarding")
sys.path.insert(0, str(WT_ONB))
import runner_common as sc  # noqa: E402
import run_stages as rs  # noqa: E402
import ground  # noqa: E402

MODEL = "mistralai/mistral-small-3.2-24b-instruct"
SEED = 20260921
HELD_OUT = set(filter(None, os.environ.get("HELD_OUT_VENDORS", "").split(",")))


def sha(s: str) -> str:
    return hashlib.sha256(s.encode("utf-8")).hexdigest()


def seeded_order(slugs):
    s = sorted(slugs)
    random.Random(SEED).shuffle(s)
    return s


def io_root(dset: str) -> Path:
    return WS4 / f"io_{dset}"


def slugs_for(dset: str):
    return sorted(p.name for p in (io_root(dset) / "runs").iterdir()
                  if p.is_dir() and p.name not in HELD_OUT)


# ---------------------------------------------------------------- bundle / prompt
def load_bundle(dset: str, slug: str):
    b = io_root(dset) / "runs" / slug / "bundle"
    meta = json.loads((b / "meta.json").read_text(encoding="utf-8"))
    rubric = (b / "rubric.md").read_text(encoding="utf-8")
    evidence = (b / "evidence.md").read_text(encoding="utf-8")
    return meta, rubric, evidence


def load_draft(dset: str, slug: str) -> str:
    return (io_root(dset) / "runs" / slug / "stages" / "dossier.md").read_text(encoding="utf-8")


def build_user(meta, rubric, evidence, dossier_text: str) -> str:
    return rs.user_03(meta, rubric, evidence, dossier_text)


SYS = rs.SYS_03


# ---------------------------------------------------------------- units + labels
def _line_spans(text: str):
    spans, off = [], 0
    for ln in text.split("\n"):
        spans.append((off, off + len(ln), ln))
        off += len(ln) + 1
    return spans


def _line_of(spans, pos):
    for i, (a, b, _) in enumerate(spans):
        if a <= pos <= b:
            return i
    raise ValueError(pos)


def _title_line(text: str) -> str:
    for ln in text.split("\n"):
        if re.match(r"^#\s+\S", ln):
            return ln.strip()
    return ""


def _sections(text: str):
    """[(heading_line, start, end)] for `## ` sections; preamble as ('', 0, first)."""
    hs = [(m.start(), m.group(0).strip()) for m in re.finditer(r"^##\s+.+$", text, flags=re.MULTILINE)]
    out = []
    first = hs[0][0] if hs else len(text)
    if "[[" in text[:first]:
        out.append(("", 0, first))
    for i, (st, h) in enumerate(hs):
        en = hs[i + 1][0] if i + 1 < len(hs) else len(text)
        out.append((h, st, en))
    return out


def classify_units(dset: str, slug: str, text: str | None = None):
    """Every ground.py claim unit with its label and the texts for arms b and c."""
    text = text if text is not None else load_draft(dset, slug)
    dj = json.loads((io_root(dset) / "runs" / slug / "stages" / "defects.json").read_text(encoding="utf-8"))
    res, domain = dj["result"], dj["domain"]
    units = ground.parse_claim_units(text)
    lab = {}
    for k in ("ungrounded", "mistagged", "unresolved"):
        for e in res[k]:
            lab[e["pos"]] = (k, e.get("why", ""))
    banned = set(res.get("banned_urls") or [])
    spans = _line_spans(text)
    title = _title_line(text)
    secs = _sections(text)
    by_line = {}
    for u in units:
        by_line.setdefault(_line_of(spans, u["pos"]), []).append(u["pos"])
    out = []
    n_auto = 0
    for u in units:
        pos = u["pos"]
        cat, why = lab.get(pos, (None, ""))
        if cat is None:
            # replicate ground's auto_fixable rule (label flips; excluded from escapes)
            tags = u["tags"]
            sourced = any(t in ground.SOURCED_TAGS for t in tags)
            if sourced and u["urls"] and "THIRD-PARTY" in tags:
                tiers, dual = [], []
                for url in u["urls"]:
                    un = ground._unwrap_wayback(url)
                    t = ground.vd.classify_url(un, domain)
                    tiers.append(t)
                    dual.append((un != url and t in ("own", "vendor_pr")) or ground._is_linkedin_company(un))
                has_ind = any(t in ("tier1", "tier2", "unknown") for t in tiers) or any(dual)
                if not has_ind and tiers and all(t in ("own", "vendor_pr") for t in tiers):
                    cat, why = "auto_fixable", "THIRD-PARTY tag but every cited source is vendor-own"
                    n_auto += 1
        if cat is None and banned and any(url in banned for url in u["urls"]):
            cat, why = "banned", "cites a banned source"
        cat = cat or "clean"
        li = _line_of(spans, pos)
        a, b, line = spans[li]
        is_table = line.lstrip().startswith("|")
        same = by_line[li]
        k = same.index(pos)
        if is_table:
            j = li
            while j - 1 >= 0 and spans[j - 1][2].lstrip().startswith("|"):
                j -= 1
            hdr = [spans[j][2]]
            if j + 1 < li and re.match(r"^\s*\|[\s\-:|]+\|?\s*$", spans[j + 1][2]):
                hdr.append(spans[j + 1][2])
            claim = "\n".join(hdr + [line]) if j < li else line
            call_group = f"row{li}"
        else:
            rel = pos - a
            if k == 0:
                st = 0
            else:
                prev = same[k - 1] - a
                e = line.find("]]", prev)
                st = e + 2 if e != -1 and e < rel else prev
            if k == len(same) - 1:
                en = len(line)
            else:
                e = line.find("]]", rel)
                nxt = same[k + 1] - a
                en = e + 2 if (e != -1 and e < nxt) else nxt
            claim = line[st:en].strip()
            call_group = f"u{pos}"
        heading = ""
        sec_idx = None
        for si, (h, s0, s1) in enumerate(secs):
            if s0 <= pos < s1:
                heading, sec_idx = h, si
        c_text = "\n\n".join(x for x in (title, heading, claim) if x)
        out.append({
            "slug": slug, "dset": dset, "pos": pos, "cat": cat, "why": why,
            "tags": u["tags"], "urls": u["urls"], "receipt": u["receipt"], "section": u["section"],
            "line_idx": li, "line": line, "n_units_on_line": len(same), "is_table": is_table,
            "claim_text": claim, "c_group": call_group, "c_text": c_text, "sec_idx": sec_idx,
        })
    # (b) texts
    b_texts = []
    for si, (h, s0, s1) in enumerate(secs):
        body = text[s0:s1].rstrip()
        if "[[" not in body:
            continue
        b_texts.append({"sec_idx": si, "heading": h,
                        "b_text": (title + "\n\n" + body) if title else body})
    meta_counts = {"auto_fixable_expected": res.get("auto_fixable_mistags", 0), "auto_fixable_found": n_auto,
                   "claims_total": res["claims_total"], "units_parsed": len(units)}
    return out, b_texts, meta_counts


# ---------------------------------------------------------------- matching (PREREG catch rules)
STOP = set("""this that with from have been were their there which about into also only more than they them
these those such will would could should does dossier claim claims vendor evidence bundle source sources
provenance tagged tagged tag tags page pages listed cited cites citing mentioned states stated""".split())


def N(x: str) -> str:
    x = x.lower()
    x = re.sub(r"web\.archive\.org/web/\d+[a-z_]*/", "", x)
    x = re.sub(r"https?://", "", x)
    x = x.replace("www.", "")
    x = re.sub(r"\s+", " ", x)
    return x.strip()


def Nurl(u: str) -> str:
    return N(u).rstrip("/")


def T(x: str) -> set:
    toks = re.findall(r"[a-z0-9][a-z0-9.,%$]*", x.lower())
    out = set()
    for t in toks:
        t = t.strip(".,")
        if not t:
            continue
        if re.search(r"\d", t):
            out.add(t)
        elif len(t) >= 4 and t.isalpha() and t not in STOP:
            out.add(t)
    return out


QUOTE_RE = re.compile(r"[\"\u201c\u201d']([^\"\u201c\u201d']{15,})[\"\u201c\u201d']")


def match_score(s: str, u: dict):
    ns = N(s)
    m1 = any(len(Nurl(url)) >= 12 and Nurl(url) in ns for url in u["urls"])
    m2 = False
    if u.get("receipt"):
        nr = N(u["receipt"])
        if nr[:30] and nr[: min(30, len(nr))] in ns:
            m2 = True
    if not m2:
        nl = N(u["claim_text"])
        nr = N(u["receipt"]) if u.get("receipt") else ""
        for q in QUOTE_RE.findall(s):
            nq = N(q)
            if len(nq) >= 15 and ((nr and nq in nr) or nq in nl):
                m2 = True
                break
    ts, tl = T(s), T(u["claim_text"])
    inter = len(ts & tl)
    m3 = inter >= 4 and ts and inter / len(ts) >= 0.5
    if not (m1 or m2 or m3):
        return None
    return 1000 * m1 + 1000 * m2 + inter


def flagged_strings(j: dict):
    out = []
    for k in ("ungrounded_or_mistagged_claims", "flags"):
        v = j.get(k) or []
        if not isinstance(v, list):
            v = [v]
        for e in v:
            out.append(e if isinstance(e, str) else json.dumps(e, ensure_ascii=False))
    return out


def credit(flags, cand_units):
    """-> (set of credited unit keys, n tie credits). cand_units: list of unit dicts with 'key'."""
    credited, ties = set(), 0
    for s in flags:
        scored = [(match_score(s, u), u["key"]) for u in cand_units]
        scored = [(sc_, k) for sc_, k in scored if sc_ is not None]
        if not scored:
            continue
        best = max(x[0] for x in scored)
        top = [k for sc_, k in scored if sc_ == best]
        if len(top) > 1:
            ties += len(top) - 1
        credited.update(top)
    return credited, ties


# ---------------------------------------------------------------- model call
class Stop(Exception):
    pass


def call_model(system: str, user: str, max_tries: int = 4):
    key = os.environ.get("OPENROUTER_API_KEY", "")
    if not key:
        raise Stop("OPENROUTER_API_KEY not in environment")
    payload = {"model": MODEL, "messages": [{"role": "system", "content": system},
                                           {"role": "user", "content": user}],
               "temperature": 0.0, "response_format": {"type": "json_object"},
               "usage": {"include": True}}
    budget = 8000
    extra_cost = 0.0
    errs = []
    t_start = time.time()
    for attempt in range(1, max_tries + 1):
        payload["max_tokens"] = budget
        req = urllib.request.Request(sc.OPENROUTER_ENDPOINT, data=json.dumps(payload).encode("utf-8"),
                                     method="POST", headers={
                                         "Authorization": f"Bearer {key}", "Content-Type": "application/json",
                                         "User-Agent": sc._BROWSER_UA,
                                         "HTTP-Referer": "https://yardstickresearch.app",
                                         "X-Title": "Yardstick WS4 granularity experiment"})
        t0 = time.time()
        try:
            with urllib.request.urlopen(req, timeout=180) as resp:
                data = json.loads(resp.read().decode("utf-8", "ignore"))
        except urllib.error.HTTPError as e:
            detail = e.read().decode("utf-8", "ignore")[:300]
            errs.append(f"HTTP {e.code}: {detail}")
            if e.code == 402:
                raise Stop("402 out of credits")
            if attempt < max_tries:
                time.sleep(2 ** attempt + random.random() * 2)
                continue
            break
        except Exception as e:  # noqa: BLE001
            errs.append(f"net: {str(e)[:200]}")
            if attempt < max_tries:
                time.sleep(2 ** attempt + random.random() * 2)
                continue
            break
        if "error" in data and not data.get("choices"):
            errs.append(f"api error: {json.dumps(data.get('error'))[:300]}")
            if attempt < max_tries:
                time.sleep(2 ** attempt + random.random() * 2)
                continue
            break
        ch = (data.get("choices") or [{}])[0]
        content = (ch.get("message") or {}).get("content") or ""
        finish = ch.get("finish_reason")
        u = data.get("usage") or {}
        rec = {"ok": True, "content": content, "finish": finish, "provider": data.get("provider"),
               "served_model": data.get("model"), "prompt_tokens": u.get("prompt_tokens", 0),
               "completion_tokens": u.get("completion_tokens", 0), "cost": u.get("cost"),
               "latency_s": round(time.time() - t0, 2), "total_s": round(time.time() - t_start, 2),
               "attempts": attempt, "errors": errs, "max_tokens": budget}
        rec["cost"] = (u.get("cost") or 0) + extra_cost
        if not content.strip():
            errs.append(f"empty content finish={finish} cost={u.get('cost')}")
            extra_cost += u.get("cost") or 0
            errs_cost = extra_cost
            if attempt < max_tries:
                if finish == "length":
                    budget = min(budget * 2, 32000)
                else:
                    time.sleep(2 ** attempt)
                continue
            return {"ok": False, "errors": errs, "attempts": attempt, "cost": errs_cost,
                    "provider": data.get("provider")}
        return rec
    return {"ok": False, "errors": errs, "attempts": len(errs), "cost": 0}


def parse_answer(content: str):
    try:
        j = sc.extract_json(content)
        v = str(j.get("verdict", "missing")).strip().lower()
        return j, v, True
    except Exception:  # noqa: BLE001
        return None, "error", False
