[
 {
  "title": "Judging one claim per call caught 23% of planted defects against 0% for the one-call check, but it also failed 18% of clean claims",
  "type": "grouped_bar_with_ci",
  "x_title": "Validation arm",
  "y_title": "Share of claims (%)",
  "series": [
   {
    "name": "Planted defects caught (%)",
    "marker": "circle",
    "dash": "solid",
    "values": [
     {
      "x": "(a) whole dossier, 1 call",
      "y": 0.0,
      "ci95": [
       0,
       11.0
      ],
      "n": 31
     },
     {
      "x": "(b) one section per call",
      "y": 3.2,
      "ci95": [
       0.6,
       16.2
      ],
      "n": 31
     },
     {
      "x": "(c) one claim per call",
      "y": 22.6,
      "ci95": [
       11.4,
       39.8
      ],
      "n": 31
     }
    ]
   },
   {
    "name": "Clean claims flagged (%)",
    "marker": "square",
    "dash": "dot",
    "values": [
     {
      "x": "(a) whole dossier, 1 call",
      "y": 1.0,
      "ci95": [
       0.5,
       2.2
      ],
      "n": 599
     },
     {
      "x": "(b) one section per call",
      "y": 1.0,
      "ci95": [
       0.5,
       2.2
      ],
      "n": 599
     },
     {
      "x": "(c) one claim per call",
      "y": 18.4,
      "ci95": [
       15.5,
       21.7
      ],
      "n": 599
     }
    ]
   }
  ],
  "n_claims": {
   "planted_defects": 31,
   "clean_claims": 599
  },
  "n_recorded_fails": 31,
  "split": "dev-side only: 23 ys-glm-bench vendors (V113, V132 excluded); planted defects on incumbent drafts; clean claims on unmodified incumbent drafts",
  "direction": "higher is better for defects caught; lower is safer for clean claims flagged",
  "note": "(c) counts a claim as caught when its call's verdict is not pass; (a),(b) need the flagged text to point at the claim (PREREG match rule). Conservative 2-repeat rule."
 },
 {
  "title": "Counting only flags that point at the defective claim, no arm caught more than 1 in 10 planted defects",
  "type": "grouped_bar_with_ci",
  "x_title": "Validation arm (all arms scored by whether the flagged text points at the claim)",
  "y_title": "Share of claims (%)",
  "series": [
   {
    "name": "Planted defects caught (%)",
    "marker": "circle",
    "dash": "solid",
    "values": [
     {
      "x": "(a) whole dossier, 1 call",
      "y": 0.0,
      "ci95": [
       0,
       11.0
      ],
      "n": 31
     },
     {
      "x": "(b) one section per call",
      "y": 3.2,
      "ci95": [
       0.6,
       16.2
      ],
      "n": 31
     },
     {
      "x": "(c) one claim per call [flag match]",
      "y": 9.7,
      "ci95": [
       3.4,
       24.9
      ],
      "n": 31
     }
    ]
   },
   {
    "name": "Clean claims flagged (%)",
    "marker": "square",
    "dash": "dot",
    "values": [
     {
      "x": "(a) whole dossier, 1 call",
      "y": 1.0,
      "ci95": [
       0.5,
       2.2
      ],
      "n": 599
     },
     {
      "x": "(b) one section per call",
      "y": 1.0,
      "ci95": [
       0.5,
       2.2
      ],
      "n": 599
     },
     {
      "x": "(c) one claim per call [flag match]",
      "y": 4.7,
      "ci95": [
       3.2,
       6.7
      ],
      "n": 599
     }
    ]
   }
  ],
  "n_claims": {
   "planted_defects": 31,
   "clean_claims": 599
  },
  "n_recorded_fails": 31,
  "split": "dev-side only: 23 ys-glm-bench vendors (V113, V132 excluded); planted defects on incumbent drafts; clean claims on unmodified incumbent drafts",
  "direction": "higher is better for defects caught; lower is safer for clean claims flagged"
 },
 {
  "title": "Judging one claim per call cost about 22 times more per dossier than the one-call check",
  "type": "bar",
  "x_title": "Validation arm",
  "y_title": "Cost per dossier (US cents, both repeats)",
  "series": [
   {
    "name": "Cost per dossier (US cents, 2 repeats)",
    "pattern": "solid",
    "marker": "none",
    "values": [
     {
      "x": "(a) whole dossier, 1 call",
      "y": 0.304
     },
     {
      "x": "(b) one section per call",
      "y": 1.088
     },
     {
      "x": "(c) one claim per call",
      "y": 6.677
     }
    ]
   }
  ],
  "n_claims": 609,
  "n_recorded_fails": 1,
  "split": "23 incumbent drafts (ys-glm-bench, V113 and V132 excluded)",
  "direction": "lower is better"
 },
 {
  "title": "Planting a defect turned a passing per-claim verdict into a fail for only 3 of 31 claims",
  "type": "bar",
  "x_title": "Per-claim verdict on the same claim: before planting -> after planting",
  "y_title": "Planted claims (count)",
  "series": [
   {
    "name": "Planted claims",
    "pattern": "solid",
    "marker": "none",
    "values": [
     {
      "x": "pass -> fail",
      "y": 3
     },
     {
      "x": "fail -> fail",
      "y": 4
     },
     {
      "x": "pass -> pass",
      "y": 23
     },
     {
      "x": "fail -> pass",
      "y": 1
     }
    ]
   }
  ],
  "n_claims": 31,
  "n_recorded_fails": 31,
  "split": "31 ground.py-confirmed planted defects in 22 of the 23 incumbent drafts",
  "direction": "higher is better for pass -> fail; the other bars are background"
 }
]