[
 {
  "title": "Both checkers miss about one recorded fail in three to four on company, score and customer claims, and fewer on integration claims",
  "type": "grouped_bar",
  "x_title": "Claim type (tagged blind from claim text)",
  "y_title": "Miss rate on recorded fails (share, 0 to 1)",
  "categories": [
   "funding company",
   "score rubric",
   "customers logos",
   "integration",
   "six smaller types combined",
   "all claims"
  ],
  "n_recorded_fails_per_category": [
   40,
   24,
   19,
   19,
   23,
   125
  ],
  "series": [
   {
    "name": "Jev, checker view (2 repeats, fail if any repeat fails)",
    "values": [
     0.325,
     0.333,
     0.316,
     0.158,
     0.304,
     0.296
    ],
    "pattern": "solid"
   },
   {
    "name": "Mistral step 03 per claim, vocab prompt (2 repeats)",
    "values": [
     0.2,
     0.292,
     0.263,
     0.158,
     0.261,
     0.232
    ],
    "pattern": "diagonal-hatch"
   }
  ],
  "n_claims": 125,
  "n_recorded_fails": 125,
  "split": "dev + held-out pooled (held-out descriptive only)",
  "direction": "lower is safer",
  "note": "Wilson 95% intervals in tables.md; every category above has at least 19 fails except the combined smaller types (23). Intervals overlap for all types."
 },
 {
  "title": "When Jev misses a recorded fail, Mistral misses it too about half the time, versus about one time in nine when Jev catches it",
  "type": "bar",
  "x_title": "Jev's result on the recorded fail",
  "y_title": "Share Mistral also misses (0 to 1)",
  "categories": [
   "Jev missed it (n=37)",
   "Jev caught it (n=88)"
  ],
  "series": [
   {
    "name": "Observed",
    "values": [
     0.514,
     0.114
    ],
    "pattern": "solid",
    "ci95": [
     [
      0.359,
      0.666
     ],
     [
      0.063,
      0.197
     ]
    ]
   },
   {
    "name": "If the two models missed independently",
    "values": [
     0.232,
     0.232
    ],
    "dash": "dashed",
    "marker": "none"
   }
  ],
  "n_claims": 125,
  "n_recorded_fails": 125,
  "split": "dev + held-out pooled",
  "direction": "lower is safer",
  "note": "Odds ratio 8.233 (vendor-clustered bootstrap 95% 3.273 to 24.439); 19 fails missed by both vs 8.6 expected if independent."
 },
 {
  "title": "Almost half of the misses are about citations, labels or evidence the model never saw, not a wrong fact in the claim",
  "type": "stacked_bar",
  "x_title": "Cause of the miss (one code per claim, pre-registered list)",
  "y_title": "Recorded fails missed (count)",
  "categories": [
   "provenance or citation only",
   "rubric or score judgment",
   "unsupported qualifier",
   "questionable label",
   "defect not in supplied evidence",
   "stale fact",
   "plain miss, defect visible",
   "V017 over-precision"
  ],
  "series": [
   {
    "name": "Jev only",
    "values": [
     7,
     6,
     3,
     0,
     0,
     1,
     0,
     1
    ],
    "pattern": "solid"
   },
   {
    "name": "Both models",
    "values": [
     4,
     3,
     2,
     5,
     3,
     1,
     1,
     0
    ],
    "pattern": "cross-hatch"
   },
   {
    "name": "Mistral only",
    "values": [
     2,
     4,
     1,
     0,
     1,
     1,
     1,
     0
    ],
    "pattern": "dots"
   }
  ],
  "n_claims": 47,
  "n_recorded_fails": 47,
  "split": "dev + held-out pooled",
  "direction": "lower is safer",
  "note": "Coded by one reader after seeing labels and outputs; see REPORT section 2.3 for every line."
 },
 {
  "title": "Misses are far more common when the evidence is a verbatim page quote than when it is a paraphrase, because paraphrased evidence often states the defect",
  "type": "grouped_bar",
  "x_title": "Form of the supplied evidence",
  "y_title": "Miss rate on recorded fails (share, 0 to 1)",
  "categories": [
   "verbatim quote (58 fails)",
   "paraphrase or transcription (67 fails)"
  ],
  "series": [
   {
    "name": "Jev, checker view",
    "values": [
     0.483,
     0.134
    ],
    "pattern": "solid"
   },
   {
    "name": "Mistral step 03 per claim, vocab",
    "values": [
     0.379,
     0.104
    ],
    "pattern": "diagonal-hatch"
   }
  ],
  "n_claims": 125,
  "n_recorded_fails": 125,
  "split": "dev + held-out pooled",
  "direction": "lower is safer",
  "note": "50 of 67 paraphrased fail-evidence texts contain a negation cue (\"not\", \"no\", ...) vs 14 of 58 verbatim ones: the paraphrase is often the validator describing the defect. Verbatim rates are closer to live use."
 }
]