{
 "dossier_incumbent": {
  "a": {
   "n_dossiers": 23,
   "fail_rep1": 2,
   "fail_rep2": 1,
   "fail_any": 2,
   "fail_any_share": [
    0.087,
    0.0242,
    0.268
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "pass",
     "2": "pass"
    },
    "V107": {
     "1": "pass",
     "2": "pass"
    },
    "V058": {
     "1": "pass",
     "2": "pass"
    },
    "V153": {
     "1": "fail",
     "2": "fail"
    },
    "V064": {
     "1": "pass",
     "2": "pass"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V136": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "pass",
     "2": "pass"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "pass",
     "2": "pass"
    },
    "V022": {
     "1": "pass",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "pass",
     "2": "pass"
    },
    "V106": {
     "1": "pass",
     "2": "pass"
    },
    "V108": {
     "1": "fail",
     "2": "pass"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "pass",
     "2": "pass"
    },
    "V070": {
     "1": "pass",
     "2": "pass"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "b": {
   "n_dossiers": 23,
   "fail_rep1": 4,
   "fail_rep2": 4,
   "fail_any": 4,
   "fail_any_share": [
    0.1739,
    0.0698,
    0.3714
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "pass",
     "2": "pass"
    },
    "V107": {
     "1": "fail",
     "2": "fail"
    },
    "V058": {
     "1": "pass",
     "2": "pass"
    },
    "V153": {
     "1": "fail",
     "2": "fail"
    },
    "V064": {
     "1": "pass",
     "2": "pass"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V136": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "pass",
     "2": "pass"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "pass",
     "2": "pass"
    },
    "V022": {
     "1": "pass",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "fail",
     "2": "fail"
    },
    "V106": {
     "1": "pass",
     "2": "pass"
    },
    "V108": {
     "1": "pass",
     "2": "pass"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "pass",
     "2": "pass"
    },
    "V070": {
     "1": "fail",
     "2": "fail"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "c": {
   "n_dossiers": 23,
   "fail_rep1": 9,
   "fail_rep2": 10,
   "fail_any": 11,
   "fail_any_share": [
    0.4783,
    0.2924,
    0.6704
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "fail",
     "2": "fail"
    },
    "V107": {
     "1": "fail",
     "2": "fail"
    },
    "V058": {
     "1": "pass",
     "2": "fail"
    },
    "V153": {
     "1": "fail",
     "2": "fail"
    },
    "V064": {
     "1": "pass",
     "2": "fail"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V136": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "fail",
     "2": "fail"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "pass",
     "2": "pass"
    },
    "V022": {
     "1": "fail",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "fail",
     "2": "fail"
    },
    "V106": {
     "1": "fail",
     "2": "fail"
    },
    "V108": {
     "1": "fail",
     "2": "fail"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "pass",
     "2": "pass"
    },
    "V070": {
     "1": "fail",
     "2": "fail"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "tie_credits": {
   "a": 1,
   "b": 0,
   "c": 11
  }
 },
 "dossier_planted": {
  "a": {
   "n_dossiers": 23,
   "fail_rep1": 4,
   "fail_rep2": 2,
   "fail_any": 4,
   "fail_any_share": [
    0.1739,
    0.0698,
    0.3714
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "pass",
     "2": "pass"
    },
    "V107": {
     "1": "fail",
     "2": "pass"
    },
    "V058": {
     "1": "fail",
     "2": "fail"
    },
    "V153": {
     "1": "fail",
     "2": null
    },
    "V064": {
     "1": "pass",
     "2": "pass"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V136": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "pass",
     "2": "pass"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "pass",
     "2": "pass"
    },
    "V022": {
     "1": "pass",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "pass",
     "2": "pass"
    },
    "V106": {
     "1": "pass",
     "2": "pass"
    },
    "V108": {
     "1": "fail",
     "2": "fail"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "pass",
     "2": "pass"
    },
    "V070": {
     "1": "pass",
     "2": "pass"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "b": {
   "n_dossiers": 23,
   "fail_rep1": 4,
   "fail_rep2": 4,
   "fail_any": 4,
   "fail_any_share": [
    0.1739,
    0.0698,
    0.3714
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "pass",
     "2": "pass"
    },
    "V107": {
     "1": "fail",
     "2": "fail"
    },
    "V058": {
     "1": "pass",
     "2": "pass"
    },
    "V153": {
     "1": "fail",
     "2": "fail"
    },
    "V064": {
     "1": "pass",
     "2": "pass"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V136": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "pass",
     "2": "pass"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "pass",
     "2": "pass"
    },
    "V022": {
     "1": "pass",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "fail",
     "2": "fail"
    },
    "V106": {
     "1": "pass",
     "2": "pass"
    },
    "V108": {
     "1": "pass",
     "2": "pass"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "pass",
     "2": "pass"
    },
    "V070": {
     "1": "fail",
     "2": "fail"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "c": {
   "n_dossiers": 23,
   "fail_rep1": 9,
   "fail_rep2": 11,
   "fail_any": 12,
   "fail_any_share": [
    0.5217,
    0.3296,
    0.7076
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "fail",
     "2": "fail"
    },
    "V107": {
     "1": "fail",
     "2": "fail"
    },
    "V058": {
     "1": "pass",
     "2": "fail"
    },
    "V153": {
     "1": "fail",
     "2": "fail"
    },
    "V064": {
     "1": "pass",
     "2": "fail"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V136": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "fail",
     "2": "fail"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "pass",
     "2": "fail"
    },
    "V022": {
     "1": "fail",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "fail",
     "2": "fail"
    },
    "V106": {
     "1": "fail",
     "2": "fail"
    },
    "V108": {
     "1": "fail",
     "2": "fail"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "pass",
     "2": "pass"
    },
    "V070": {
     "1": "fail",
     "2": "fail"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "tie_credits": {
   "a": 0,
   "b": 0,
   "c": 11
  }
 },
 "dossier_glm": {
  "a": {
   "n_dossiers": 22,
   "fail_rep1": 1,
   "fail_rep2": 1,
   "fail_any": 1,
   "fail_any_share": [
    0.0455,
    0.0081,
    0.218
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "pass",
     "2": "pass"
    },
    "V107": {
     "1": "pass",
     "2": "pass"
    },
    "V058": {
     "1": "pass",
     "2": "pass"
    },
    "V153": {
     "1": "pass",
     "2": "pass"
    },
    "V064": {
     "1": "pass",
     "2": "pass"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "pass",
     "2": "pass"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "pass",
     "2": "pass"
    },
    "V022": {
     "1": "pass",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "pass",
     "2": "pass"
    },
    "V106": {
     "1": "pass",
     "2": "pass"
    },
    "V108": {
     "1": "pass",
     "2": "pass"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "fail",
     "2": "fail"
    },
    "V070": {
     "1": "pass",
     "2": "pass"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "b": {
   "n_dossiers": 22,
   "fail_rep1": 4,
   "fail_rep2": 4,
   "fail_any": 4,
   "fail_any_share": [
    0.1818,
    0.0731,
    0.3852
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "pass",
     "2": "pass"
    },
    "V107": {
     "1": "fail",
     "2": "fail"
    },
    "V058": {
     "1": "pass",
     "2": "pass"
    },
    "V153": {
     "1": "fail",
     "2": "fail"
    },
    "V064": {
     "1": "pass",
     "2": "pass"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "pass",
     "2": "pass"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "pass",
     "2": "pass"
    },
    "V022": {
     "1": "pass",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "fail",
     "2": "fail"
    },
    "V106": {
     "1": "pass",
     "2": "pass"
    },
    "V108": {
     "1": "pass",
     "2": "pass"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "pass",
     "2": "pass"
    },
    "V070": {
     "1": "fail",
     "2": "fail"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "c": {
   "n_dossiers": 22,
   "fail_rep1": 8,
   "fail_rep2": 8,
   "fail_any": 8,
   "fail_any_share": [
    0.3636,
    0.1973,
    0.5705
   ],
   "per_vendor": {
    "V021": {
     "1": "pass",
     "2": "pass"
    },
    "V145": {
     "1": "fail",
     "2": "fail"
    },
    "V107": {
     "1": "fail",
     "2": "fail"
    },
    "V058": {
     "1": "fail",
     "2": "fail"
    },
    "V153": {
     "1": "fail",
     "2": "fail"
    },
    "V064": {
     "1": "pass",
     "2": "pass"
    },
    "V062": {
     "1": "pass",
     "2": "pass"
    },
    "V131": {
     "1": "pass",
     "2": "pass"
    },
    "V009": {
     "1": "fail",
     "2": "fail"
    },
    "V002": {
     "1": "pass",
     "2": "pass"
    },
    "V014": {
     "1": "pass",
     "2": "pass"
    },
    "V119": {
     "1": "fail",
     "2": "fail"
    },
    "V022": {
     "1": "pass",
     "2": "pass"
    },
    "V020": {
     "1": "pass",
     "2": "pass"
    },
    "V075": {
     "1": "fail",
     "2": "fail"
    },
    "V106": {
     "1": "pass",
     "2": "pass"
    },
    "V108": {
     "1": "pass",
     "2": "pass"
    },
    "V054": {
     "1": "pass",
     "2": "pass"
    },
    "V040": {
     "1": "pass",
     "2": "pass"
    },
    "V070": {
     "1": "fail",
     "2": "fail"
    },
    "V029": {
     "1": "pass",
     "2": "pass"
    },
    "V083": {
     "1": "pass",
     "2": "pass"
    }
   }
  },
  "tie_credits": {
   "a": 45,
   "b": 3,
   "c": 23
  }
 },
 "unit_counts": {
  "incumbent": {
   "clean": 599,
   "ungrounded": 1,
   "auto_fixable": 9
  },
  "planted": {
   "clean": 563,
   "ungrounded": 23,
   "auto_fixable": 14,
   "mistagged": 9
  },
  "glm": {
   "clean": 977,
   "auto_fixable": 38,
   "mistagged": 2,
   "ungrounded": 2
  }
 },
 "registered_primary": {
  "arms": {
   "a": {
    "rule": "a_flag",
    "n": 1,
    "k": 0,
    "share_ci": [
     0.0,
     0,
     0.7935
    ],
    "n_vendors": 1
   },
   "b": {
    "rule": "b_flag",
    "n": 1,
    "k": 0,
    "share_ci": [
     0.0,
     0,
     0.7935
    ],
    "n_vendors": 1
   },
   "c": {
    "rule": "c_verdict",
    "n": 1,
    "k": 0,
    "share_ci": [
     0.0,
     0,
     0.7935
    ],
    "n_vendors": 1
   }
  },
  "c_vs_a": {
   "x": "a_flag",
   "y": "c_verdict",
   "n": 1,
   "n_vendors": 1,
   "x_share": [
    0.0,
    0,
    0.7935
   ],
   "y_share": [
    0.0,
    0,
    0.7935
   ],
   "diff_y_minus_x": 0.0,
   "boot95": [
    0.0,
    0.0
   ],
   "y_only": 0,
   "x_only": 0,
   "mcnemar_p": 1.0
  }
 },
 "q43_clean_incumbent": {
  "arms": {
   "a": {
    "rule": "a_flag",
    "n": 599,
    "k": 6,
    "share_ci": [
     0.01,
     0.0046,
     0.0217
    ],
    "n_vendors": 23
   },
   "b": {
    "rule": "b_flag",
    "n": 599,
    "k": 6,
    "share_ci": [
     0.01,
     0.0046,
     0.0217
    ],
    "n_vendors": 23
   },
   "c": {
    "rule": "c_verdict",
    "n": 599,
    "k": 110,
    "share_ci": [
     0.1836,
     0.1547,
     0.2166
    ],
    "n_vendors": 23
   }
  },
  "arms_sensitivity": {
   "a": {
    "rule": "a_verdict",
    "n": 599,
    "k": 46,
    "share_ci": [
     0.0768,
     0.0581,
     0.1009
    ],
    "n_vendors": 23
   },
   "b": {
    "rule": "b_verdict",
    "n": 599,
    "k": 65,
    "share_ci": [
     0.1085,
     0.0861,
     0.136
    ],
    "n_vendors": 23
   },
   "c": {
    "rule": "c_flag",
    "n": 599,
    "k": 28,
    "share_ci": [
     0.0467,
     0.0325,
     0.0667
    ],
    "n_vendors": 23
   }
  },
  "c_vs_a": {
   "x": "a_flag",
   "y": "c_verdict",
   "n": 599,
   "n_vendors": 23,
   "x_share": [
    0.01,
    0.0046,
    0.0217
   ],
   "y_share": [
    0.1836,
    0.1547,
    0.2166
   ],
   "diff_y_minus_x": 0.1736,
   "boot95": [
    0.0565,
    0.295
   ],
   "y_only": 108,
   "x_only": 4,
   "mcnemar_p": 0.0
  },
  "b_vs_a": {
   "x": "a_flag",
   "y": "b_flag",
   "n": 599,
   "n_vendors": 23,
   "x_share": [
    0.01,
    0.0046,
    0.0217
   ],
   "y_share": [
    0.01,
    0.0046,
    0.0217
   ],
   "diff_y_minus_x": 0.0,
   "boot95": [
    -0.0147,
    0.009
   ],
   "y_only": 4,
   "x_only": 4,
   "mcnemar_p": 1.0
  },
  "cflag_vs_a": {
   "x": "a_flag",
   "y": "c_flag",
   "n": 599,
   "n_vendors": 23,
   "x_share": [
    0.01,
    0.0046,
    0.0217
   ],
   "y_share": [
    0.0467,
    0.0325,
    0.0667
   ],
   "diff_y_minus_x": 0.0367,
   "boot95": [
    0.0101,
    0.0686
   ],
   "y_only": 26,
   "x_only": 4,
   "mcnemar_p": 5.9e-05
  }
 },
 "dvp": {
  "n_confirmed": 31,
  "arms": {
   "a": {
    "rule": "a_flag",
    "n": 31,
    "k": 0,
    "share_ci": [
     0.0,
     0,
     0.1103
    ],
    "n_vendors": 22
   },
   "b": {
    "rule": "b_flag",
    "n": 31,
    "k": 1,
    "share_ci": [
     0.0323,
     0.0057,
     0.1619
    ],
    "n_vendors": 22
   },
   "c": {
    "rule": "c_verdict",
    "n": 31,
    "k": 7,
    "share_ci": [
     0.2258,
     0.114,
     0.3981
    ],
    "n_vendors": 22
   }
  },
  "arms_sensitivity": {
   "a": {
    "rule": "a_verdict",
    "n": 31,
    "k": 6,
    "share_ci": [
     0.1935,
     0.0919,
     0.3628
    ],
    "n_vendors": 22
   },
   "b": {
    "rule": "b_verdict",
    "n": 31,
    "k": 5,
    "share_ci": [
     0.1613,
     0.0709,
     0.3263
    ],
    "n_vendors": 22
   },
   "c": {
    "rule": "c_flag",
    "n": 31,
    "k": 3,
    "share_ci": [
     0.0968,
     0.0335,
     0.249
    ],
    "n_vendors": 22
   }
  },
  "c_vs_a": {
   "x": "a_flag",
   "y": "c_verdict",
   "n": 31,
   "n_vendors": 22,
   "x_share": [
    0.0,
    0,
    0.1103
   ],
   "y_share": [
    0.2258,
    0.114,
    0.3981
   ],
   "diff_y_minus_x": 0.2258,
   "boot95": [
    0.0645,
    0.4062
   ],
   "y_only": 7,
   "x_only": 0,
   "mcnemar_p": 0.015625
  },
  "b_vs_a": {
   "x": "a_flag",
   "y": "b_flag",
   "n": 31,
   "n_vendors": 22,
   "x_share": [
    0.0,
    0,
    0.1103
   ],
   "y_share": [
    0.0323,
    0.0057,
    0.1619
   ],
   "diff_y_minus_x": 0.0323,
   "boot95": [
    0.0,
    0.1
   ],
   "y_only": 1,
   "x_only": 0,
   "mcnemar_p": 1.0
  },
  "c_vs_b": {
   "x": "b_flag",
   "y": "c_verdict",
   "n": 31,
   "n_vendors": 22,
   "x_share": [
    0.0323,
    0.0057,
    0.1619
   ],
   "y_share": [
    0.2258,
    0.114,
    0.3981
   ],
   "diff_y_minus_x": 0.1935,
   "boot95": [
    0.0,
    0.3939
   ],
   "y_only": 7,
   "x_only": 1,
   "mcnemar_p": 0.070312
  },
  "cflag_vs_a": {
   "x": "a_flag",
   "y": "c_flag",
   "n": 31,
   "n_vendors": 22,
   "x_share": [
    0.0,
    0,
    0.1103
   ],
   "y_share": [
    0.0968,
    0.0335,
    0.249
   ],
   "diff_y_minus_x": 0.0968,
   "boot95": [
    0.0,
    0.2069
   ],
   "y_only": 3,
   "x_only": 0,
   "mcnemar_p": 0.25
  },
  "by_type": {
   "R": {
    "a": {
     "rule": "a_flag",
     "n": 4,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.4899
     ],
     "n_vendors": 4
    },
    "b": {
     "rule": "b_flag",
     "n": 4,
     "k": 1,
     "share_ci": [
      0.25,
      0.0456,
      0.6994
     ],
     "n_vendors": 4
    },
    "c": {
     "rule": "c_verdict",
     "n": 4,
     "k": 1,
     "share_ci": [
      0.25,
      0.0456,
      0.6994
     ],
     "n_vendors": 4
    }
   },
   "U": {
    "a": {
     "rule": "a_flag",
     "n": 18,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.1759
     ],
     "n_vendors": 18
    },
    "b": {
     "rule": "b_flag",
     "n": 18,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.1759
     ],
     "n_vendors": 18
    },
    "c": {
     "rule": "c_verdict",
     "n": 18,
     "k": 3,
     "share_ci": [
      0.1667,
      0.0584,
      0.3922
     ],
     "n_vendors": 18
    }
   },
   "T": {
    "a": {
     "rule": "a_flag",
     "n": 9,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.2991
     ],
     "n_vendors": 9
    },
    "b": {
     "rule": "b_flag",
     "n": 9,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.2991
     ],
     "n_vendors": 9
    },
    "c": {
     "rule": "c_verdict",
     "n": 9,
     "k": 3,
     "share_ci": [
      0.3333,
      0.1206,
      0.6458
     ],
     "n_vendors": 9
    }
   }
  },
  "clean_in_planted": {
   "a": {
    "rule": "a_flag",
    "n": 563,
    "k": 5,
    "share_ci": [
     0.0089,
     0.0038,
     0.0206
    ],
    "n_vendors": 23
   },
   "b": {
    "rule": "b_flag",
    "n": 563,
    "k": 8,
    "share_ci": [
     0.0142,
     0.0072,
     0.0278
    ],
    "n_vendors": 23
   },
   "c": {
    "rule": "c_verdict",
    "n": 563,
    "k": 102,
    "share_ci": [
     0.1812,
     0.1516,
     0.2151
    ],
    "n_vendors": 23
   }
  }
 },
 "dvn": {
  "incumbent": {
   "arms": {
    "a": {
     "rule": "a_flag",
     "n": 10,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.2775
     ],
     "n_vendors": 6
    },
    "b": {
     "rule": "b_flag",
     "n": 10,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.2775
     ],
     "n_vendors": 6
    },
    "c": {
     "rule": "c_verdict",
     "n": 10,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.2775
     ],
     "n_vendors": 6
    }
   },
   "c_vs_a": {
    "x": "a_flag",
    "y": "c_verdict",
    "n": 10,
    "n_vendors": 6,
    "x_share": [
     0.0,
     0,
     0.2775
    ],
    "y_share": [
     0.0,
     0,
     0.2775
    ],
    "diff_y_minus_x": 0.0,
    "boot95": [
     0.0,
     0.0
    ],
    "y_only": 0,
    "x_only": 0,
    "mcnemar_p": 1.0
   },
   "cflag_vs_a": {
    "x": "a_flag",
    "y": "c_flag",
    "n": 10,
    "n_vendors": 6,
    "x_share": [
     0.0,
     0,
     0.2775
    ],
    "y_share": [
     0.0,
     0,
     0.2775
    ],
    "diff_y_minus_x": 0.0,
    "boot95": [
     0.0,
     0.0
    ],
    "y_only": 0,
    "x_only": 0,
    "mcnemar_p": 1.0
   },
   "mistag_only": {
    "a": {
     "rule": "a_flag",
     "n": 9,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.2991
     ],
     "n_vendors": 5
    },
    "b": {
     "rule": "b_flag",
     "n": 9,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.2991
     ],
     "n_vendors": 5
    },
    "c": {
     "rule": "c_verdict",
     "n": 9,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.2991
     ],
     "n_vendors": 5
    }
   },
   "non_mistag": {
    "a": {
     "rule": "a_flag",
     "n": 1,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.7935
     ],
     "n_vendors": 1
    },
    "b": {
     "rule": "b_flag",
     "n": 1,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.7935
     ],
     "n_vendors": 1
    },
    "c": {
     "rule": "c_verdict",
     "n": 1,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.7935
     ],
     "n_vendors": 1
    }
   }
  },
  "glm": {
   "arms": {
    "a": {
     "rule": "a_flag",
     "n": 42,
     "k": 1,
     "share_ci": [
      0.0238,
      0.0042,
      0.1232
     ],
     "n_vendors": 13
    },
    "b": {
     "rule": "b_flag",
     "n": 42,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.0838
     ],
     "n_vendors": 13
    },
    "c": {
     "rule": "c_verdict",
     "n": 42,
     "k": 1,
     "share_ci": [
      0.0238,
      0.0042,
      0.1232
     ],
     "n_vendors": 13
    }
   },
   "c_vs_a": {
    "x": "a_flag",
    "y": "c_verdict",
    "n": 42,
    "n_vendors": 13,
    "x_share": [
     0.0238,
     0.0042,
     0.1232
    ],
    "y_share": [
     0.0238,
     0.0042,
     0.1232
    ],
    "diff_y_minus_x": 0.0,
    "boot95": [
     -0.0714,
     0.0789
    ],
    "y_only": 1,
    "x_only": 1,
    "mcnemar_p": 1.0
   },
   "cflag_vs_a": {
    "x": "a_flag",
    "y": "c_flag",
    "n": 42,
    "n_vendors": 13,
    "x_share": [
     0.0238,
     0.0042,
     0.1232
    ],
    "y_share": [
     0.0238,
     0.0042,
     0.1232
    ],
    "diff_y_minus_x": 0.0,
    "boot95": [
     -0.0714,
     0.0789
    ],
    "y_only": 1,
    "x_only": 1,
    "mcnemar_p": 1.0
   },
   "mistag_only": {
    "a": {
     "rule": "a_flag",
     "n": 40,
     "k": 1,
     "share_ci": [
      0.025,
      0.0044,
      0.1288
     ],
     "n_vendors": 12
    },
    "b": {
     "rule": "b_flag",
     "n": 40,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.0876
     ],
     "n_vendors": 12
    },
    "c": {
     "rule": "c_verdict",
     "n": 40,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.0876
     ],
     "n_vendors": 12
    }
   },
   "non_mistag": {
    "a": {
     "rule": "a_flag",
     "n": 2,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.6576
     ],
     "n_vendors": 2
    },
    "b": {
     "rule": "b_flag",
     "n": 2,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.6576
     ],
     "n_vendors": 2
    },
    "c": {
     "rule": "c_verdict",
     "n": 2,
     "k": 1,
     "share_ci": [
      0.5,
      0.0945,
      0.9055
     ],
     "n_vendors": 2
    }
   }
  },
  "pooled": {
   "arms": {
    "a": {
     "rule": "a_flag",
     "n": 52,
     "k": 1,
     "share_ci": [
      0.0192,
      0.0034,
      0.1012
     ],
     "n_vendors": 14
    },
    "b": {
     "rule": "b_flag",
     "n": 52,
     "k": 0,
     "share_ci": [
      0.0,
      0,
      0.0688
     ],
     "n_vendors": 14
    },
    "c": {
     "rule": "c_verdict",
     "n": 52,
     "k": 1,
     "share_ci": [
      0.0192,
      0.0034,
      0.1012
     ],
     "n_vendors": 14
    }
   },
   "c_vs_a": {
    "x": "a_flag",
    "y": "c_verdict",
    "n": 52,
    "n_vendors": 14,
    "x_share": [
     0.0192,
     0.0034,
     0.1012
    ],
    "y_share": [
     0.0192,
     0.0034,
     0.1012
    ],
    "diff_y_minus_x": 0.0,
    "boot95": [
     -0.0588,
     0.0652
    ],
    "y_only": 1,
    "x_only": 1,
    "mcnemar_p": 1.0
   },
   "cflag_vs_a": {
    "x": "a_flag",
    "y": "c_flag",
    "n": 52,
    "n_vendors": 14,
    "x_share": [
     0.0192,
     0.0034,
     0.1012
    ],
    "y_share": [
     0.0192,
     0.0034,
     0.1012
    ],
    "diff_y_minus_x": 0.0,
    "boot95": [
     -0.0588,
     0.0652
    ],
    "y_only": 1,
    "x_only": 1,
    "mcnemar_p": 1.0
   }
  }
 },
 "dvp_before_after": {
  "n": 31,
  "counts": {
   "pass -> fail": 3,
   "fail -> fail": 4,
   "pass -> pass": 23,
   "fail -> pass": 1
  },
  "mcnemar_p": 0.625
 },
 "glm_registered": {
  "arms": {
   "a": {
    "rule": "a_flag",
    "n": 4,
    "k": 0,
    "share_ci": [
     0.0,
     0,
     0.4899
    ],
    "n_vendors": 3
   },
   "b": {
    "rule": "b_flag",
    "n": 4,
    "k": 0,
    "share_ci": [
     0.0,
     0,
     0.4899
    ],
    "n_vendors": 3
   },
   "c": {
    "rule": "c_verdict",
    "n": 4,
    "k": 1,
    "share_ci": [
     0.25,
     0.0456,
     0.6994
    ],
    "n_vendors": 3
   }
  },
  "clean": {
   "a": {
    "rule": "a_flag",
    "n": 977,
    "k": 41,
    "share_ci": [
     0.042,
     0.0311,
     0.0564
    ],
    "n_vendors": 22
   },
   "b": {
    "rule": "b_flag",
    "n": 977,
    "k": 6,
    "share_ci": [
     0.0061,
     0.0028,
     0.0133
    ],
    "n_vendors": 22
   },
   "c": {
    "rule": "c_verdict",
    "n": 976,
    "k": 134,
    "share_ci": [
     0.1373,
     0.1171,
     0.1603
    ],
    "n_vendors": 22
   }
  },
  "c_vs_a_clean": {
   "x": "a_flag",
   "y": "c_verdict",
   "n": 976,
   "n_vendors": 22,
   "x_share": [
    0.042,
    0.0311,
    0.0565
   ],
   "y_share": [
    0.1373,
    0.1171,
    0.1603
   ],
   "diff_y_minus_x": 0.0953,
   "boot95": [
    -0.0067,
    0.213
   ],
   "y_only": 126,
   "x_only": 33,
   "mcnemar_p": 0.0
  },
  "vendors": [
   "V021",
   "V145",
   "V107",
   "V058",
   "V153",
   "V064",
   "V062",
   "V131",
   "V009",
   "V002",
   "V014",
   "V119",
   "V022",
   "V020",
   "V075",
   "V106",
   "V108",
   "V054",
   "V040",
   "V070",
   "V029",
   "V083"
  ]
 },
 "cost": {
  "incumbent/a": {
   "calls_made": 46,
   "calls_ok": 46,
   "parsed": 46,
   "jobs_incl_reused": 46,
   "vendors": 23,
   "cost_usd": 0.06994,
   "prompt_tokens": 705094,
   "completion_tokens": 16007,
   "mean_latency_s": 11.54,
   "sum_latency_s": 531.0,
   "cost_per_dossier_usd": 0.00304,
   "providers": {
    "Venice": 44,
    "Parasail": 1,
    "DeepInfra": 1
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 46
   },
   "verdicts": {
    "pass": 43,
    "fail": 3
   },
   "retried": 0,
   "failed_after_retries": 0
  },
  "incumbent/b": {
   "calls_made": 184,
   "calls_ok": 184,
   "parsed": 184,
   "jobs_incl_reused": 184,
   "vendors": 23,
   "cost_usd": 0.25027,
   "prompt_tokens": 2504796,
   "completion_tokens": 65349,
   "mean_latency_s": 11.06,
   "sum_latency_s": 2035.8,
   "cost_per_dossier_usd": 0.01088,
   "providers": {
    "Venice": 179,
    "Parasail": 2,
    "DeepInfra": 3
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 184
   },
   "verdicts": {
    "pass": 163,
    "fail": 21
   },
   "retried": 1,
   "failed_after_retries": 0
  },
  "incumbent/c": {
   "calls_made": 1124,
   "calls_ok": 1124,
   "parsed": 1124,
   "jobs_incl_reused": 1124,
   "vendors": 23,
   "cost_usd": 1.53573,
   "prompt_tokens": 15549280,
   "completion_tokens": 345181,
   "mean_latency_s": 8.98,
   "sum_latency_s": 10095.0,
   "cost_per_dossier_usd": 0.06677,
   "providers": {
    "Venice": 1084,
    "DeepInfra": 10,
    "Parasail": 30
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 1124
   },
   "verdicts": {
    "pass": 950,
    "fail": 174
   },
   "retried": 17,
   "failed_after_retries": 0
  },
  "planted/a": {
   "calls_made": 46,
   "calls_ok": 46,
   "parsed": 45,
   "jobs_incl_reused": 46,
   "vendors": 23,
   "cost_usd": 0.07214,
   "prompt_tokens": 705724,
   "completion_tokens": 23929,
   "mean_latency_s": 14.67,
   "sum_latency_s": 675.0,
   "cost_per_dossier_usd": 0.00314,
   "providers": {
    "Venice": 46
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 46
   },
   "verdicts": {
    "pass": 39,
    "fail": 6,
    "error": 1
   },
   "retried": 0,
   "failed_after_retries": 0
  },
  "planted/b": {
   "calls_made": 58,
   "calls_ok": 58,
   "parsed": 58,
   "jobs_incl_reused": 184,
   "vendors": 23,
   "cost_usd": 0.07501,
   "prompt_tokens": 744554,
   "completion_tokens": 21394,
   "mean_latency_s": 10.95,
   "sum_latency_s": 635.1,
   "cost_per_dossier_usd": 0.00326,
   "providers": {
    "Venice": 57,
    "DeepInfra": 1
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 58
   },
   "verdicts": {
    "pass": 47,
    "fail": 11
   },
   "retried": 1,
   "failed_after_retries": 0
  },
  "planted/c": {
   "calls_made": 72,
   "calls_ok": 72,
   "parsed": 72,
   "jobs_incl_reused": 1124,
   "vendors": 23,
   "cost_usd": 0.09137,
   "prompt_tokens": 913604,
   "completion_tokens": 22928,
   "mean_latency_s": 9.29,
   "sum_latency_s": 668.8,
   "cost_per_dossier_usd": 0.00397,
   "providers": {
    "Venice": 71,
    "Parasail": 1
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 72
   },
   "verdicts": {
    "pass": 58,
    "fail": 14
   },
   "retried": 1,
   "failed_after_retries": 0
  },
  "glm/a": {
   "calls_made": 44,
   "calls_ok": 44,
   "parsed": 44,
   "jobs_incl_reused": 44,
   "vendors": 22,
   "cost_usd": 0.07573,
   "prompt_tokens": 770616,
   "completion_tokens": 15718,
   "mean_latency_s": 12.75,
   "sum_latency_s": 561.0,
   "cost_per_dossier_usd": 0.00344,
   "providers": {
    "Venice": 41,
    "DeepInfra": 2,
    "Parasail": 1
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 44
   },
   "verdicts": {
    "pass": 42,
    "fail": 2
   },
   "retried": 0,
   "failed_after_retries": 0
  },
  "glm/b": {
   "calls_made": 212,
   "calls_ok": 212,
   "parsed": 212,
   "jobs_incl_reused": 212,
   "vendors": 22,
   "cost_usd": 0.27641,
   "prompt_tokens": 2780330,
   "completion_tokens": 74911,
   "mean_latency_s": 11.42,
   "sum_latency_s": 2420.4,
   "cost_per_dossier_usd": 0.01256,
   "providers": {
    "Venice": 196,
    "DeepInfra": 12,
    "Parasail": 4
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 212
   },
   "verdicts": {
    "pass": 190,
    "fail": 22
   },
   "retried": 2,
   "failed_after_retries": 0
  },
  "glm/c": {
   "calls_made": 1706,
   "calls_ok": 1703,
   "parsed": 1702,
   "jobs_incl_reused": 1718,
   "vendors": 22,
   "cost_usd": 2.23081,
   "prompt_tokens": 22595635,
   "completion_tokens": 570484,
   "mean_latency_s": 14.95,
   "sum_latency_s": 25453.0,
   "cost_per_dossier_usd": 0.1014,
   "providers": {
    "Venice": 1552,
    "DeepInfra": 82,
    "Parasail": 69,
    "null": 3
   },
   "served_models": {
    "mistralai/mistral-small-3.2-24b-instruct": 1703,
    "null": 3
   },
   "verdicts": {
    "pass": 1487,
    "fail": 215,
    "error": 1
   },
   "retried": 18,
   "failed_after_retries": 3
  }
 },
 "opus_secondary": [
  {
   "id": "S1-053",
   "vendor": "V153",
   "internal_verdict": "fail",
   "located": false,
   "overlap": 1,
   "n_claim_tokens": 10
  },
  {
   "id": "S1-054",
   "vendor": "V153",
   "internal_verdict": "pass",
   "located": false,
   "overlap": 2,
   "n_claim_tokens": 6
  },
  {
   "id": "S6-013",
   "vendor": "V153",
   "internal_verdict": "pass",
   "located": true,
   "overlap": 5,
   "n_claim_tokens": 12,
   "unit_pos": 839,
   "ground_cat": "clean",
   "a": false,
   "b": false,
   "c": true
  },
  {
   "id": "S6-014",
   "vendor": "V153",
   "internal_verdict": "pass",
   "located": false,
   "overlap": 2,
   "n_claim_tokens": 5
  },
  {
   "id": "S6-015",
   "vendor": "V153",
   "internal_verdict": "pass",
   "located": false,
   "overlap": 1,
   "n_claim_tokens": 7
  },
  {
   "id": "S6-016",
   "vendor": "V136",
   "internal_verdict": "pass",
   "located": true,
   "overlap": 4,
   "n_claim_tokens": 7,
   "unit_pos": 775,
   "ground_cat": "clean",
   "a": false,
   "b": false,
   "c": false
  },
  {
   "id": "S6-038",
   "vendor": "V153",
   "internal_verdict": "pass",
   "located": false,
   "overlap": 1,
   "n_claim_tokens": 10
  },
  {
   "id": "S6-039",
   "vendor": "V153",
   "internal_verdict": "pass",
   "located": false,
   "overlap": 2,
   "n_claim_tokens": 6
  }
 ],
 "c_multi_unit_calls": {
  "incumbent": 90,
  "planted": 90,
  "glm": 257
 }
}