"""Render every Jev claim-triage chart spec to PNG (200 dpi) and SVG (plan step F.1).

Reads ws1..ws5/charts.json, ws10/charts.json and charts/specs/{adj,f4}.json, in that order, and
writes <id>.png and <id>.svg into out/exp/charts/, where id is <source>-<n> in spec order.
Everything outside out/exp/charts/ is read-only. No model calls.

The renderer draws the numbers as given. The only departures from a spec are listed in the
OVERRIDES table below (layout aids, intervals that exist in the results file but were left out of
the spec, missing required fields filled from the workstream's REPORT.md, and vendor-name
redaction). Every override is printed in the footnote and logged to _work/render_log.json, which
QA.md summarises.

Content fixes from the F.1 QA pass (a title that disagreed with its data, a point no run produced, a
value never measured) live in charts/specs/overrides.json, keyed by chart id, and are applied on top of
the workstream spec before rendering; the ws*/charts.json files are never edited. Each such chart gets a
"Changed in F.1 QA" footnote line, and QA.md records why.

Run: python render_all.py            (all 29 charts)
     python render_all.py ws1-2 ...  (only the named charts; the render log keeps the rest)
"""
import json
import math
import os
import re
import textwrap

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt  # noqa: E402
from matplotlib.lines import Line2D  # noqa: E402
from matplotlib.patches import Patch  # noqa: E402

HERE = os.path.dirname(os.path.abspath(__file__))
EXP = os.path.dirname(HERE)
Z = 1.959964

SOURCES = [
    ("ws1", "ws1/charts.json"), ("ws2", "ws2/charts.json"), ("ws3", "ws3/charts.json"),
    ("ws4", "ws4/charts.json"), ("ws5", "ws5/charts.json"), ("ws10", "ws10/charts.json"),
    ("adj", "charts/specs/adj.json"), ("f4", "charts/specs/f4.json"),
    ("jev", "charts/specs/jev.json"),  # 2026-09-26: Jev-only Finding 6 chart (build_jev_spec.py); appended last so no id moves
]

# Okabe-Ito, yellow dropped (too light on white). Validated with the dataviz palette checker:
# all checks pass, CVD separation in the 6-8 band, so every series also gets a marker, dash or hatch.
PALETTE = ["#0072B2", "#D55E00", "#009E73", "#CC79A7", "#E69F00", "#56B4E9"]
REF_COLOR = "#595959"
INK, INK2, GRID = "#1a1a1a", "#404040", "#d9d9d9"
MARKERS = {"circle": "o", "square": "s", "triangle": "^", "diamond": "D", "star": "*", "none": None, None: None}
DASHES = {"solid": "-", "dashed": "--", "dash": "--", "dotted": ":", "dot": ":", "dashdot": "-.", "none": "",
          None: "-"}
HATCHES = {"solid": "", "hatched": "///", "diagonal-hatch": "///", "diagonal_hatch": "///",
           "cross-hatch": "xx", "dots": "..", None: None}
HATCH_ORDER = ["", "///", "xx", "..", "\\\\\\"]
MARKER_ORDER = ["o", "s", "^", "D", "v"]
DASH_ORDER = ["-", "--", ":", "-."]


def load(rel):
    with open(os.path.join(EXP, rel), encoding="utf-8") as f:
        return json.load(f)


def wilson(k, n):
    p = k / n
    den = 1 + Z * Z / n
    c = (p + Z * Z / (2 * n)) / den
    h = Z * math.sqrt(p * (1 - p) / n + Z * Z / (4 * n * n)) / den
    return round(max(0.0, c - h), 3), round(min(1.0, c + h), 3)


# ------------------------------------------------------------------ intervals that exist in results
def ws3_intervals():
    """Wilson intervals for ws3-1 and ws3-4, read from ws3/rekey120/run/tables.json (q31.pooled.*).

    Source moved on 2026-09-22 from ws3/tables.json to the re-key working copy, so the error bars sit on the
    same 120-fail answer key as the bars (ws3/REKEY-120-REPORT.md, charts/REKEY-120-CHARTS.md).
    """
    T = load("ws3/rekey120/run/tables.json")["q31"]["pooled"]
    models = ["jev_checker_miss", "mistral_vocab_miss"]
    types = ["funding_company", "score_rubric", "customers_logos", "integration"]
    small = ["capability_feature", "certification_compliance", "date_event", "numeric_fact", "other", "pricing"]
    out1 = []
    for m in models:
        lo, hi = [], []
        for t in types:
            d = T["type"][t]["fails"][m]
            lo.append(d["lo"]), hi.append(d["hi"])
        k = sum(T["type"][t]["fails"][m]["k"] for t in small)
        n = sum(T["type"][t]["fails"][m]["n"] for t in small)
        a, b = wilson(k, n)  # combined category is not stored in tables.json: computed from summed k, n
        lo.append(a), hi.append(b)
        d = T["overall"]["fails"][m]
        lo.append(d["lo"]), hi.append(d["hi"])
        out1.append((lo, hi))
    out4 = []
    for m in models:
        lo = [T["verbatim"][v]["fails"][m]["lo"] for v in ("True", "False")]
        hi = [T["verbatim"][v]["fails"][m]["hi"] for v in ("True", "False")]
        out4.append((lo, hi))
    return out1, out4


def ws10_3_intervals():
    fs = load("ws10/results.json")["flip_share"]["any"]
    arms = [["N_end", "N_quote"], ["I_end", "I_quote"], ["F_I_end", "F_I_quote"]]
    return [([round(100 * fs[a]["wilson95"][0], 1) for a in row], [round(100 * fs[a]["wilson95"][1], 1) for a in row])
            for row in arms]


WS10_FILL = {
    "split": "dev: the 23 WS4 development vendors, held-out vendors not used",
    "n_recorded_fails": 22,
}
OVERRIDES = {
    "ws1-1": {"vline": (0.8, "Current rule (threshold 0.8)")},
    "adj-1": {"vline": (0.8, "Current rule (threshold 0.8)"), "nudge": 0.008},
    "ws2-1": {"zoom_y": 30},
    "ws2-2": {"reference_series": ["Perfect calibration"], "hollow_note": True},
    "ws3-2": {"reference_series": ["If the two models missed independently"]},
    "ws10-1": {"fill": dict(WS10_FILL, n_claims=22, n_claims_unit="must-fail dossiers", direction="lower is safer")},
    "ws10-2": {"fill": dict(WS10_FILL, n_claims=22, n_claims_unit="must-fail dossiers", direction="lower is safer")},
    "ws10-3": {"fill": dict(WS10_FILL, n_claims=22, n_claims_unit="must-fail dossiers", direction="lower is safer")},
    "ws10-4": {"fill": dict(WS10_FILL, n_claims=22, n_claims_unit="must-fail dossiers (44 calls per arm)",
                            direction="lower is safer")},
    "ws10-5": {"fill": dict(WS10_FILL, n_claims=23, n_claims_unit="dossiers screened",
                            direction="not applicable: this chart counts usable test cases (per the spec note)")},
}
REDACT = [(re.compile(os.environ.get("HELD_OUT_CAPTION_REGEX", "(?!x)x")), "2 named vendors excluded")]


# ------------------------------------------------------------------ QA overrides (specs/overrides.json)
def file_overrides():
    """Chart-content fixes from the F.1 QA pass, keyed by chart id (see the file's _doc and QA.md)."""
    path = os.path.join(HERE, "specs", "overrides.json")
    if not os.path.exists(path):
        return {}
    with open(path, encoding="utf-8") as f:
        d = json.load(f)
    return {k: v for k, v in d.items() if not k.startswith("_")}


def get_path(obj, dotted):
    for part in dotted.split("."):
        obj = obj[part]
    return obj


def projection_values(op, xs):
    """ws1-3: the WS1 half-width formula z*sqrt(p(1-p)/(n*m*f)), with f read from its source file."""
    fs = load(op["fail_share"]["file"])
    f = get_path(fs, op["fail_share"]["k"]) / get_path(fs, op["fail_share"]["n"])
    z = get_path(load(op["z"]["file"]), op["z"]["field"])
    m = get_path(load(op["per_dossier"]["file"]), op["per_dossier"]["field"])
    p = op["miss_rate"]
    return [round(100 * z * math.sqrt(p * (1 - p) / (float(n) * m * f)), 2) for n in xs]


def apply_file_overrides(cid, spec, ov, fo):
    """Apply one chart's overrides.json entry to a private spec copy and layout dict. Returns the op names."""
    e = fo.get(cid)
    if not e:
        return []
    ops = []
    for k, v in e.get("set", {}).items():
        spec[k] = v
    if e.get("set"):
        ops.append("set " + ", ".join(e["set"]))
    for idx, fields in e.get("series", {}).items():
        spec["series"][int(idx)].update(fields)
        ops.append(f"series[{idx}] " + ", ".join(fields))
    rel = e.get("relabel_x", {})
    if rel:
        for key in ("categories", "x"):
            if isinstance(spec.get(key), list):
                spec[key] = [rel.get(c, c) if isinstance(c, str) else c for c in spec[key]]
        if isinstance(spec.get("x_axis"), dict) and spec["x_axis"].get("categories"):
            spec["x_axis"]["categories"] = [rel.get(c, c) for c in spec["x_axis"]["categories"]]
        for s in spec["series"]:
            for v in s["values"]:
                if isinstance(v, dict) and v.get("x") in rel:
                    v["x"] = rel[v["x"]]
        ops.append("relabel_x")
    for d in e.get("drop_points", []):
        for i in d["series"]:
            s = spec["series"][i]
            before = len(s["values"])
            s["values"] = [v for v in s["values"]
                           if not all(isinstance(v, dict) and v.get(k) == w for k, w in d["where"].items())]
            ops.append(f"drop_points series[{i}] {before - len(s['values'])} point(s) where {d['where']}")
    if "add_projection" in e:
        op = e["add_projection"]
        spec["series"].append({"name": op["name"], "values": projection_values(op, spec["x"]),
                               "dash": op.get("dash"), "marker": op.get("marker")})
        ops.append("add_projection")
    for k, v in e.get("renderer", {}).items():
        if v is None:
            ov.pop(k, None)
        else:
            ov[k] = v
        ops.append(f"renderer {k}={v}")
    return ops


def vendor_slugs():
    slugs = set(load("ws10/raw/vendor_map.json").keys())
    slugs |= {r["vendor"] for r in load("ws3/per_claim.json").values() if isinstance(r.get("vendor"), str)}
    for rel in ("ws1/shadow_claims.jsonl", "f4/shadow_claims.jsonl"):
        with open(os.path.join(EXP, rel), encoding="utf-8") as f:
            slugs |= {json.loads(l)["slug"] for l in f.read().split("\n") if l.strip()}
    slugs |= set(filter(None, os.environ.get("EXTRA_VENDOR_SLUGS", "").split(",")))
    return {s.lower() for s in slugs if s and not s.lower().startswith("vendor")}


# ------------------------------------------------------------------ spec normalisation
def fmt(v):
    if v is None:
        return ""
    if isinstance(v, int) or float(v).is_integer():
        return str(int(v))
    return repr(float(v))


def is_pct(spec):
    t = (spec.get("y_title") or (spec.get("y_axis") or {}).get("title") or "") + " " + \
        str((spec.get("y_axis") or {}).get("unit", ""))
    return "%" in t or "percent" in t.lower()


def categories(spec):
    if spec.get("categories"):
        return list(spec["categories"])
    if isinstance(spec.get("x_axis"), dict) and spec["x_axis"].get("categories"):
        return list(spec["x_axis"]["categories"])
    if isinstance(spec.get("x"), list):
        return list(spec["x"])
    cats = []
    for s in spec["series"]:
        for v in s["values"]:
            if isinstance(v, dict) and v["x"] not in cats:
                cats.append(v["x"])
    return cats


def norm_series(spec, s):
    vals = s["values"]
    out = {"name": s.get("legend") or s["name"], "raw": s, "lo": None, "hi": None, "tags": None,
           "hollow": None, "plabels": None}
    if vals and isinstance(vals[0], dict):
        out["x"] = [v["x"] for v in vals]
        out["y"] = [v["y"] for v in vals]
        if any("y_lo" in v for v in vals):
            out["lo"] = [v.get("y_lo") for v in vals]
            out["hi"] = [v.get("y_hi") for v in vals]
        elif any("ci95" in v for v in vals):
            out["lo"] = [v["ci95"][0] if v.get("ci95") else None for v in vals]
            out["hi"] = [v["ci95"][1] if v.get("ci95") else None for v in vals]
        tags = []
        for v in vals:
            t = []
            if "n" in v and not v.get("too_small") is None and False:
                pass
            if "oof_misses" in v:
                t.append(f"misses {v['oof_misses']}")
            elif "n" in v and spec.get("type", "").startswith(("bar", "grouped")):
                t.append(f"n={v['n']}")
            tags.append(", ".join(t))
        out["tags"] = tags if any(tags) else None
        if any("too_small" in v for v in vals):
            out["hollow"] = [bool(v.get("too_small")) for v in vals]
        if any("label" in v for v in vals):
            out["plabels"] = [v.get("label") for v in vals]
    else:
        out["x"] = categories(spec)
        out["y"] = list(vals)
        if "ci_low" in s:
            out["lo"], out["hi"] = list(s["ci_low"]), list(s["ci_high"])
        elif "ci95" in s:
            out["lo"] = [c[0] for c in s["ci95"]]
            out["hi"] = [c[1] for c in s["ci95"]]
        if "counts" in s:
            out["tags"] = list(s["counts"])
        elif "n" in s:
            out["tags"] = [f"n={n}" for n in s["n"]]
        elif spec.get("labels_n"):
            out["tags"] = [f"n={n}" for n in spec["labels_n"]]
    return out


# ------------------------------------------------------------------ drawing
def style_axes(ax):
    for side in ("top", "right"):
        ax.spines[side].set_visible(False)
    for side in ("left", "bottom"):
        ax.spines[side].set_color("#8c8c8c")
    ax.tick_params(colors=INK2, labelsize=9)
    ax.grid(axis="y", color=GRID, linewidth=0.7)
    ax.set_axisbelow(True)


def draw_line(ax, spec, series, ov, handles, zoom=False):
    nudge = ov.get("nudge", 0)
    ref = set(ov.get("reference_series", []))
    ci = 0
    for i, s in enumerate(series):
        raw = s["raw"]
        is_ref = s["raw"]["name"] in ref
        color = REF_COLOR if is_ref else PALETTE[ci % len(PALETTE)]
        mk = MARKERS.get(raw.get("marker"), None) if "marker" in raw else MARKER_ORDER[ci % 5]
        ls = DASHES.get(raw.get("dash"), "-") if "dash" in raw else DASH_ORDER[ci % 4]
        if not is_ref:
            ci += 1
        k = len(series)
        off = (i - (k - 1) / 2) * nudge if nudge else 0
        xs = [float(x) + off for x in s["x"]]
        ys = [float("nan") if y is None else y for y in s["y"]]
        z = 3 + 0.1 * (k - i)  # earlier series on top, so a dense later curve does not bury it
        ax.plot(xs, ys, linestyle=ls if ls else "none", color=color, linewidth=2 if not is_ref else 1.4,
                zorder=z, label=None)
        if mk:
            hollow = s["hollow"] or [False] * len(xs)
            fx = [x for x, h in zip(xs, hollow) if not h]
            fy = [y for y, h in zip(ys, hollow) if not h]
            hx = [x for x, h in zip(xs, hollow) if h]
            hy = [y for y, h in zip(ys, hollow) if h]
            many = len(xs) > 25
            msz = 5 if many else (11 if mk == "*" else 7)
            ax.plot(fx, fy, linestyle="none", marker=mk, markersize=msz, color=color,
                    markeredgecolor="white", markeredgewidth=0.8, zorder=z + 1,
                    markevery=max(1, len(fx) // 12) if many else None)
            if hx:
                ax.plot(hx, hy, linestyle="none", marker=mk, markersize=msz, markerfacecolor="white",
                        markeredgecolor=color, markeredgewidth=1.5, zorder=4)
        if s["lo"] is not None:
            lo = [y - l if (l is not None and y is not None) else 0 for y, l in zip(s["y"], s["lo"])]
            hi = [h - y if (h is not None and y is not None) else 0 for y, h in zip(s["y"], s["hi"])]
            many = len(xs) > 25
            ax.errorbar(xs, ys, yerr=[lo, hi], fmt="none", ecolor=color, elinewidth=0.8 if many else 1.2,
                        alpha=0.3 if many else 0.9, capsize=0 if many else 3, zorder=2)
        # point labels: on the zoom panel when there is one (room to spread them), else on the main axes
        if s["plabels"] and (zoom or "zoom_y" not in ov):
            for j, (x, y, lab) in enumerate(zip(xs, ys, s["plabels"])):
                if lab:
                    ov.setdefault("_nlab", 0)
                    n = ov["_nlab"]
                    ov["_nlab"] += 1
                    ax.annotate(lab, (x, y), textcoords="offset points", xytext=(-40 + 10 * n, 40 + 28 * n),
                                fontsize=8, color=INK, arrowprops=dict(arrowstyle="-", color=INK2, lw=0.7),
                                bbox=dict(boxstyle="round,pad=0.2", facecolor="white", edgecolor=GRID), zorder=7)
        if not zoom:
            handles.append((Line2D([0], [0], color=color, linestyle=ls if ls else "none", marker=mk,
                                   markersize=7, linewidth=2), s["name"]))
    if ov.get("hollow_note") and not zoom:
        handles.append((Line2D([0], [0], linestyle="none", marker="o", markerfacecolor="white",
                               markeredgecolor=INK2, markersize=7), "Hollow marker: bin has fewer than 10 answers"))
    if "vline" in ov:
        x, lab = ov["vline"]
        ax.axvline(x, color=REF_COLOR, linestyle="-.", linewidth=1.2, zorder=1)
        if not zoom:
            handles.append((Line2D([0], [0], color=REF_COLOR, linestyle="-.", linewidth=1.2), lab))
    xs_all = [float(x) for s in series for x in s["x"]]
    return xs_all


def bar_label(ax, x, y, text, pct):
    ax.annotate(text, (x, y), textcoords="offset points", xytext=(0, 3), ha="center", va="bottom",
                fontsize=7.5, color=INK, zorder=6)


def draw_bars(ax, spec, series, ov, handles, pct):
    cats = categories(spec)
    ref = set(ov.get("reference_series", []))
    bars = [s for s in series if s["raw"]["name"] not in ref]
    refs = [s for s in series if s["raw"]["name"] in ref]
    stacked = spec.get("type") == "stacked_bar"
    used_hatch = []
    styles = []
    for i, s in enumerate(bars):
        raw = s["raw"]
        h = HATCHES.get(raw.get("pattern")) if raw.get("pattern") in HATCHES else None
        if h is None or h in used_hatch:
            h = next(x for x in HATCH_ORDER if x not in used_hatch)
        used_hatch.append(h)
        styles.append((PALETTE[i % len(PALETTE)], h))
    width_total = 0.8
    bottoms = [0.0] * len(cats)
    for i, s in enumerate(bars):
        color, h = styles[i]
        for j, (cx, y) in enumerate(zip(s["x"], s["y"])):
            c = cats.index(cx)
            if stacked:
                w, xpos = 0.6, c
            else:
                present = [b for b in bars if cx in b["x"]]
                k = len(present)
                w = width_total / max(k, 1)
                xpos = c - width_total / 2 + w * (present.index(s) + 0.5)
            tag = s["tags"][j] if s["tags"] else ""
            if y is None:
                ax.annotate(tag or "n/a", (xpos, 0), textcoords="offset points", xytext=(0, 3), ha="center",
                            va="bottom", fontsize=7.5, color=INK2, rotation=90)
                continue
            b0 = bottoms[c] if stacked else 0
            if y == 0 and not stacked:  # zero-height bar: a coloured stub on the baseline keeps it visible
                ax.hlines(0, xpos - w * 0.46, xpos + w * 0.46, colors=color, linewidth=3.5, zorder=4)
            ax.bar(xpos, y, width=w * 0.92, bottom=b0, color=color, hatch=h, edgecolor="white", linewidth=0.8,
                   zorder=3)
            if s["lo"] is not None and s["lo"][j] is not None:
                ax.errorbar(xpos, y, yerr=[[y - s["lo"][j]], [s["hi"][j] - y]], fmt="none", ecolor=INK,
                            elinewidth=1.1, capsize=3, zorder=4)
                top = s["hi"][j]
            else:
                top = y + b0
            if stacked:
                if y:
                    ax.text(xpos, b0 + y / 2, fmt(y), ha="center", va="center", fontsize=8, color="white",
                            fontweight="bold", zorder=6,
                            bbox=dict(boxstyle="round,pad=0.15", facecolor=color, edgecolor="none"))
                bottoms[c] += y
            else:
                label = fmt(y) + ("%" if pct else "")
                if tag:
                    label += "\n" + tag
                bar_label(ax, xpos, top, label, pct)
        handles.append((Patch(facecolor=color, hatch=h, edgecolor="white"), s["name"]))
    if stacked:
        for c, tot in enumerate(bottoms):
            ax.annotate(f"total {fmt(tot)}", (c, tot), textcoords="offset points", xytext=(0, 3), ha="center",
                        fontsize=8, color=INK)
    for s in refs:
        ls = DASHES.get(s["raw"].get("dash"), "--")
        for cx, y in zip(s["x"], s["y"]):
            c = cats.index(cx)
            ax.hlines(y, c - 0.45, c + 0.45, colors=REF_COLOR, linestyles=ls, linewidth=1.8, zorder=5)
            ax.annotate(fmt(y), (c + 0.45, y), textcoords="offset points", xytext=(3, 0), va="center", fontsize=7.5,
                        color=REF_COLOR)
        handles.append((Line2D([0], [0], color=REF_COLOR, linestyle=ls, linewidth=1.8), s["name"]))
    for j, rl in enumerate(spec.get("reference_lines", [])):
        ls = ["--", ":"][j % 2]
        ax.axhline(rl["value"], color=[REF_COLOR, INK][j % 2], linestyle=ls, linewidth=1.8, zorder=5)
        handles.append((Line2D([0], [0], color=[REF_COLOR, INK][j % 2], linestyle=ls, linewidth=1.8),
                        f"{rl.get('legend', rl['name'])}: {fmt(rl['value'])}{'%' if pct else ''}"))
    ax.set_xticks(range(len(cats)))
    wrapw = max(12, int(110 / max(len(cats), 1)))
    ax.set_xticklabels([textwrap.fill(str(c), wrapw) for c in cats], fontsize=8.5)
    ax.set_xlim(-0.6, len(cats) - 0.4)


def draw_points(ax, spec, series, ov, handles, pct):
    cats = categories(spec)
    for i, s in enumerate(series):
        color = PALETTE[i % len(PALETTE)]
        mk = MARKERS.get(s["raw"].get("marker"), "o") or "o"
        xs = [cats.index(x) for x in s["x"]]
        ax.plot(xs, s["y"], linestyle="none", marker=mk, markersize=9, color=color, zorder=4)
        if s["lo"] is not None:
            ax.errorbar(xs, s["y"], yerr=[[y - l for y, l in zip(s["y"], s["lo"])],
                                          [h - y for y, h in zip(s["y"], s["hi"])]],
                        fmt="none", ecolor=color, elinewidth=1.6, capsize=6, zorder=3)
            for x, y, h, t in zip(xs, s["y"], s["hi"], s["tags"] or [""] * len(xs)):
                ax.annotate(f"{t}: {fmt(y)}%, upper bound {fmt(h)}%" if pct else f"{t}", (x, h),
                            textcoords="offset points", xytext=(0, 6), ha="center", fontsize=8.5, color=INK)
        handles.append((Line2D([0], [0], color=color, linestyle="none", marker=mk, markersize=8), s["name"]))
    ax.set_xticks(range(len(cats)))
    ax.set_xticklabels([textwrap.fill(str(c), 26) for c in cats], fontsize=9)
    ax.set_xlim(-0.6, len(cats) - 0.4)


# ------------------------------------------------------------------ one chart
def render(cid, spec, slugs, log, fo=None):
    ov = dict(OVERRIDES.get(cid, {}))  # copy: file overrides and draw helpers may change it
    notes_applied = []
    spec = json.loads(json.dumps(spec))  # private copy
    fo = file_overrides() if fo is None else fo
    qa_ops = apply_file_overrides(cid, spec, ov, fo)
    fill = ov.get("fill", {})
    filled = []
    for k, v in fill.items():
        if k in ("n_claims_unit",):
            continue
        if spec.get(k) in (None, ""):
            spec[k] = v
            filled.append(k)
    title = spec["title"]
    x_title = spec.get("x_title") or (spec.get("x_axis") or {}).get("title") or ""
    y_title = spec.get("y_title") or (spec.get("y_axis") or {}).get("title") or ""
    if not spec.get("x_title") and spec.get("x_axis"):
        u = spec["x_axis"].get("unit")
        notes_applied.append("axis titles read from x_axis / y_axis objects")
    pct = is_pct(spec)
    if cid == "ws1-2":
        pct = True

    # interval additions
    if cid in ("ws3-1", "ws3-4"):
        i1, i4 = ws3_intervals()
        for s, (lo, hi) in zip(spec["series"], i1 if cid == "ws3-1" else i4):
            s["ci_low"], s["ci_high"] = lo, hi
        notes_applied.append("Wilson 95% intervals added from ws3/rekey120/run/tables.json (q31.pooled); not in the spec"
                             + ("; the combined smaller-types interval is computed from summed k and n" if cid == "ws3-1" else ""))
    if cid == "ws10-3":
        for s, (lo, hi) in zip(spec["series"], ws10_3_intervals()):
            s["ci_low"], s["ci_high"] = lo, hi
        notes_applied.append("Wilson 95% intervals added from ws10/results.json flip_share.any.<arm>.wilson95; not in the spec")

    series = [norm_series(spec, s) for s in spec["series"]]
    kind = spec.get("type", "bar")
    kind = "line" if kind.startswith("line") else ("points" if kind == "point_ci" else "bars")
    has_ci = any(s["lo"] is not None for s in series)

    # ---- text blocks
    n_c, n_f = spec.get("n_claims"), spec.get("n_recorded_fails")
    if isinstance(n_c, dict):
        n_c_txt = ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in n_c.items())
    elif n_c is None:
        n_c_txt = "n claims not stated"
    else:
        unit = fill.get("n_claims_unit") if "n_claims" in filled else "claims"
        n_c_txt = spec["n_claims_note"] if spec.get("n_claims_note") else f"{n_c} {unit}"
    if n_f is None:
        n_f_txt = "recorded fails: " + (spec.get("n_recorded_fails_note") or "not stated")
    else:
        n_f_txt = f"{n_f} recorded fail{'' if n_f == 1 else 's'}" + (f" ({spec['n_recorded_fails_note']})" if spec.get("n_recorded_fails_note") else "")
    dag = lambda k: "†" if k in filled else ""  # noqa: E731
    split = spec.get("split") or "not stated"
    for rx, rep in REDACT:
        if rx.search(split):
            split = rx.sub(rep, split)
            notes_applied.append("two vendor names in the spec's split text replaced with '2 named vendors excluded'")
    sub1 = f"n = {n_c_txt}{dag('n_claims')}; {n_f_txt}{dag('n_recorded_fails')}"
    sub2 = f"Split{dag('split')}: {split}"
    sub3 = f"Direction{dag('direction')}: {spec.get('direction') or 'not stated'}"
    subtitle = spec.get("subtitle")

    foot = []
    if has_ci:
        eb = spec.get("error_bars") or "Wilson 95%"
        foot.append(f"Error bars: {eb} intervals" + (" (drawn faint on the dense curves)" if kind == "line" and
                                                    any(len(s["x"]) > 25 for s in series) else "") + ".")
    for key in ("note", "assumptions"):
        if spec.get(key):
            foot.append(("Assumptions: " if key == "assumptions" else "Note: ") + spec[key])
    for a in spec.get("annotations", []):
        foot.append("Note: " + a)
    if filled:
        foot.append("† Not in the workstream spec; filled by the F.1 renderer from the workstream's REPORT.md "
                    "(see QA.md).")
    for n in notes_applied:
        foot.append("Renderer: " + n + ".")
    if qa_ops and fo.get(cid, {}).get("footnote"):
        fn = fo[cid]["footnote"]
        foot.append("Changed in F.1 QA (charts/specs/overrides.json, see QA.md): " + fn[0].upper() + fn[1:])
    if "zoom_y" in ov:
        foot.append(f"Renderer: the right panel repeats the same data with the y axis cut at {ov['zoom_y']}% so the "
                    "low range can be read; no value is changed.")
    src = dict(SOURCES)[cid.rsplit("-", 1)[0]]
    foot.append(f"Spec: {src}, chart {cid.rsplit('-', 1)[1]}. Source fields for every number: charts/INDEX.md.")

    # ---- figure
    W = 11.0
    tl = textwrap.wrap(title, 92)
    sl = []
    if subtitle:
        sl += textwrap.wrap(subtitle, 125)
    for s in (sub1, sub2, sub3):
        sl += textwrap.wrap(s, 125)
    AX_H = 3.9
    top_block = 0.25 + 0.27 * len(tl) + 0.2 * len(sl) + 0.3
    H = top_block + AX_H + 0.2
    fig = plt.figure(figsize=(W, H))
    fig.patch.set_facecolor("white")
    ax_bottom = 0.2 / H
    zoom = "zoom_y" in ov
    if zoom:
        ax = fig.add_axes([0.08, ax_bottom, 0.5, AX_H / H])
        ax2 = fig.add_axes([0.66, ax_bottom, 0.32, AX_H / H])
    else:
        ax = fig.add_axes([0.08, ax_bottom, 0.9, AX_H / H])
        ax2 = None
    y = 1 - 0.25 / H
    for line in tl:
        fig.text(0.02, y, line, fontsize=13.5, fontweight="bold", color=INK, va="top", ha="left")
        y -= 0.27 / H
    y -= 0.05 / H
    for line in sl:
        fig.text(0.02, y, line, fontsize=9.5, color=INK2, va="top", ha="left")
        y -= 0.2 / H

    handles = []
    for a in ([ax] + ([ax2] if ax2 else [])):
        style_axes(a)
    if kind == "line":
        draw_line(ax, spec, series, ov, handles)
        if ax2:
            draw_line(ax2, spec, series, ov, [], zoom=True)
            ax2.set_ylim(-0.03 * ov["zoom_y"], ov["zoom_y"])
            ax2.set_title(f"Same data, y axis cut at {ov['zoom_y']}%", fontsize=9, color=INK2)
    elif kind == "points":
        draw_points(ax, spec, series, ov, handles, pct)
    else:
        draw_bars(ax, spec, series, ov, handles, pct)

    # y range: data plus headroom for labels
    vals = [v for s in series for v in (s["y"] + (s["hi"] or [])) if isinstance(v, (int, float))]
    ya = spec.get("y_axis") or {}
    ymax = max(vals) if vals else 1
    if kind == "bars" and spec.get("type") == "stacked_bar":
        ymax = max(ymax, max(sum((s["y"][j] or 0) for s in series) for j in range(len(series[0]["y"]))))
    if "max" in ya:
        top = ya["max"]
    else:
        top = ymax * (1.22 if kind != "line" else 1.08) if ymax > 0 else 1
        if pct and ymax <= 100:
            top = min(top, 112 if kind != "line" else 105)
        if kind == "points":
            top = ymax * 1.3
    if kind == "line":
        # a little room below zero so markers sitting at 0 are not cut by the axis (layout only)
        ax.set_ylim(min(0, min(vals) if vals else 0) - 0.03 * top, top)
    elif kind == "points":
        ax.set_ylim(-0.04 * top, top)  # markers at 0 stay whole (layout only)
    else:
        ax.set_ylim(ya.get("min", 0), top)
    if kind == "line":
        lo_x = min(float(x) for s in series for x in s["x"])
        hi_x = max(float(x) for s in series for x in s["x"])
        pad = (hi_x - lo_x) * 0.04
        ax.set_xlim(lo_x - pad, hi_x + pad)
        if ax2:
            ax2.set_xlim(lo_x - pad, hi_x + pad)

    ax.set_ylabel(textwrap.fill(y_title, 48), fontsize=9.5, color=INK)
    ax.set_xlabel(textwrap.fill(x_title, 110), fontsize=9.5, color=INK, labelpad=6)
    if ax2:
        ax2.set_xlabel(textwrap.fill(x_title, 50), fontsize=8.5, color=INK, labelpad=6)

    # legend under the axes, footnote under the legend
    fig.canvas.draw()
    r = fig.canvas.get_renderer()
    bb = [ax.get_tightbbox(r)] + ([ax2.get_tightbbox(r)] if ax2 else [])
    y0 = min(b.y0 for b in bb)
    inv = fig.transFigure.inverted()
    yleg = inv.transform((0, y0))[1] - 0.12 / H
    names = [h[1] for h in handles]
    ncol = 1 if max(len(n) for n in names) > 55 else (2 if len(names) > 1 else 1)
    leg = fig.legend([h[0] for h in handles], [textwrap.fill(n, 95 if ncol == 1 else 55) for n in names],
                     loc="upper left", bbox_to_anchor=(0.02, yleg), ncol=ncol, frameon=True, fontsize=9,
                     title="Legend", title_fontsize=9, alignment="left", edgecolor=GRID, handlelength=3,
                     handleheight=1.4)
    fig.canvas.draw()
    lb = leg.get_window_extent(r)
    yf = inv.transform((0, lb.y0))[1] - 0.12 / H
    foot_lines = []
    for f in foot:
        foot_lines += textwrap.wrap(f, 150)
    fig.text(0.02, yf, "\n".join(foot_lines), fontsize=8.2, color=INK2, va="top", ha="left", linespacing=1.35)

    # vendor-name scan over every piece of text on the figure
    texts = [t.get_text() for t in fig.findobj(matplotlib.text.Text)]
    blob = " ".join(texts).lower()
    hits = sorted(s for s in slugs if re.search(r"(?<![a-z0-9])" + re.escape(s).replace(r"\-", "[- ]") + r"(?![a-z0-9])", blob))

    for ext in ("png", "svg"):
        fig.savefig(os.path.join(HERE, f"{cid}.{ext}"), dpi=200, bbox_inches="tight", pad_inches=0.3,
                    facecolor="white")
    plt.close(fig)
    log[cid] = {"title": title, "filled": filled, "renderer_notes": notes_applied + (["zoom panel"] if zoom else []),
                "vendor_hits": hits, "type": spec.get("type"), "has_ci": has_ci, "qa_overrides": qa_ops}


def main():
    """python render_all.py [chart-id ...]: no ids renders all 29; ids re-render only those, and the render
    log keeps the other charts' entries."""
    import sys
    only = set(sys.argv[1:])
    slugs = vendor_slugs()
    fo = file_overrides()
    log, errors = {}, {}
    log_path = os.path.join(HERE, "_work", "render_log.json")
    if only and os.path.exists(log_path):
        with open(log_path, encoding="utf-8") as f:
            prev = json.load(f)
        log, errors = prev.get("charts", {}), prev.get("errors", {})
    for key, rel in SOURCES:
        specs = load(rel)
        for i, spec in enumerate(specs, 1):
            cid = f"{key}-{i}"
            if only and cid not in only:
                continue
            errors.pop(cid, None)
            try:
                render(cid, spec, slugs, log, fo)
                print("ok  ", cid)
            except Exception as e:  # keep going; failures go to QA.md
                import traceback
                errors[cid] = f"{type(e).__name__}: {e}"
                print("FAIL", cid, errors[cid])
                traceback.print_exc(limit=2)
    os.makedirs(os.path.join(HERE, "_work"), exist_ok=True)
    with open(os.path.join(HERE, "_work", "render_log.json"), "w", encoding="utf-8") as f:
        json.dump({"charts": log, "errors": errors}, f, indent=1, ensure_ascii=False)
    print(len(log), "rendered,", len(errors), "failed; vendor hits:",
          {k: v["vendor_hits"] for k, v in log.items() if v["vendor_hits"]})


if __name__ == "__main__":
    main()
