{
  "dev": {
    "baseline": {
      "n_scenarios": 30,
      "n_cause": 24,
      "n_no_cause": 6,
      "hit1_strict": {
        "value": 0.4583333333333333,
        "k": 11,
        "n": 24,
        "ci95": [
          0.2789133373121099,
          0.6492513464108051
        ]
      },
      "hit3_strict": {
        "value": 0.5833333333333334,
        "k": 14,
        "n": 24,
        "ci95": [
          0.3883466586522876,
          0.7553239739018824
        ]
      },
      "hit1_loose": {
        "value": 0.75,
        "k": 18,
        "n": 24,
        "ci95": [
          0.55100555994846,
          0.8800063377140499
        ]
      },
      "hit3_loose": {
        "value": 0.875,
        "k": 21,
        "n": 24,
        "ci95": [
          0.6899611872949993,
          0.9565566591987655
        ]
      },
      "mrr_strict": {
        "value": 0.5138888888888888,
        "n": 24
      },
      "mrr_loose": {
        "value": 0.8055555555555555,
        "n": 24
      },
      "false_alarm": {
        "value": 1.0,
        "k": 6,
        "n": 6,
        "ci95": [
          0.6096657120978346,
          1.0
        ]
      },
      "miss_rate": {
        "value": 0.0,
        "k": 0,
        "n": 24,
        "ci95": [
          1.3877787807814457e-17,
          0.13797620467498017
        ]
      },
      "onset_accuracy": {
        "value": 0.75,
        "k": 6,
        "n": 8,
        "ci95": [
          0.40927543031016883,
          0.9285207872478909
        ]
      },
      "f0b_explained": {
        "value": 0.0,
        "k": 0,
        "n": 3,
        "ci95": [
          5.551115123125783e-17,
          0.5614970317550454
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 30,
        "ci95": [
          0.0,
          0.11351339317396876
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.0,
        "k": 0,
        "n": 6,
        "ci95": [
          2.7755575615628914e-17,
          0.3903342879021653
        ]
      },
      "seconds_per_scenario": {
        "mean": 0.3424,
        "max": 0.408,
        "n": 30
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": null
        },
        {
          "value": 0.4583333333333333,
          "k": 11,
          "n": 24,
          "ci95": [
            0.2789133373121099,
            0.6492513464108051
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 1.0
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 3,
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        },
        "F0b": {
          "n": 3,
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        },
        "F1": {
          "n": 7,
          "hit1_strict": {
            "value": 0.7142857142857143,
            "k": 5,
            "n": 7,
            "ci95": [
              0.35893445183261935,
              0.9177810759959433
            ]
          },
          "hit3_strict": {
            "value": 0.8571428571428571,
            "k": 6,
            "n": 7,
            "ci95": [
              0.4868721707256811,
              0.9743203756552565
            ]
          },
          "hit3_loose": {
            "value": 0.8571428571428571,
            "k": 6,
            "n": 7,
            "ci95": [
              0.4868721707256811,
              0.9743203756552565
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 7,
            "ci95": [
              5.551115123125783e-17,
              0.35433043506668743
            ]
          }
        },
        "F2": {
          "n": 6,
          "hit1_strict": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          },
          "hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        },
        "F3": {
          "n": 5,
          "hit1_strict": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          },
          "hit3_strict": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 5,
            "ci95": [
              0.0,
              0.43448246478317476
            ]
          }
        },
        "F4": {
          "n": 6,
          "hit1_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 4,
            "n": 6,
            "ci95": [
              0.299993315138392,
              0.9032285888942195
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 6,
            "n": 6,
            "ci95": [
              0.6096657120978346,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 8,
          "hit1_strict": {
            "value": 0.625,
            "k": 5,
            "n": 8,
            "ci95": [
              0.30574239460262737,
              0.8631557141764026
            ]
          },
          "hit3_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 8,
            "n": 8,
            "ci95": [
              0.6755924351161198,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        },
        "medium": {
          "n": 8,
          "hit1_strict": {
            "value": 0.5,
            "k": 4,
            "n": 8,
            "ci95": [
              0.21521606221387757,
              0.7847839377861224
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 4,
            "n": 8,
            "ci95": [
              0.21521606221387757,
              0.7847839377861224
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 8,
            "n": 8,
            "ci95": [
              0.6755924351161198,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        },
        "small": {
          "n": 8,
          "hit1_strict": {
            "value": 0.25,
            "k": 2,
            "n": 8,
            "ci95": [
              0.071479212752109,
              0.5907245696898311
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 4,
            "n": 8,
            "ci95": [
              0.21521606221387757,
              0.7847839377861224
            ]
          },
          "hit3_loose": {
            "value": 0.625,
            "k": 5,
            "n": 8,
            "ci95": [
              0.30574239460262737,
              0.8631557141764026
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 15,
          "hit1_strict": {
            "value": 0.5,
            "k": 6,
            "n": 12,
            "ci95": [
              0.2537815976337061,
              0.7462184023662939
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 6,
            "n": 12,
            "ci95": [
              0.2537815976337061,
              0.7462184023662939
            ]
          },
          "hit3_loose": {
            "value": 0.75,
            "k": 9,
            "n": 12,
            "ci95": [
              0.46769466506643426,
              0.9110583316059453
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        },
        "low": {
          "n": 15,
          "hit1_strict": {
            "value": 0.4166666666666667,
            "k": 5,
            "n": 12,
            "ci95": [
              0.19326031365875665,
              0.6804886874504503
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 17,
          "hit1_strict": {
            "value": 0.38461538461538464,
            "k": 5,
            "n": 13,
            "ci95": [
              0.17709707797762575,
              0.6447710848733431
            ]
          },
          "hit3_strict": {
            "value": 0.46153846153846156,
            "k": 6,
            "n": 13,
            "ci95": [
              0.23206067809538303,
              0.7085620428549398
            ]
          },
          "hit3_loose": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 13,
            "ci95": [
              0.0,
              0.22809537235419838
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 4,
            "ci95": [
              0.0,
              0.4898908364545973
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 4,
            "n": 4,
            "ci95": [
              0.5101091635454027,
              1.0
            ]
          }
        },
        "warning": {
          "n": 13,
          "hit1_strict": {
            "value": 0.5454545454545454,
            "k": 6,
            "n": 11,
            "ci95": [
              0.280091537409322,
              0.7872872837754024
            ]
          },
          "hit3_strict": {
            "value": 0.7272727272727273,
            "k": 8,
            "n": 11,
            "ci95": [
              0.43435469882387084,
              0.902539407099751
            ]
          },
          "hit3_loose": {
            "value": 0.9090909090909091,
            "k": 10,
            "n": 11,
            "ci95": [
              0.6226415635484043,
              0.9837678271141151
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 11,
            "ci95": [
              0.0,
              0.2588329669680317
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 2,
            "n": 2,
            "ci95": [
              0.34238022750665303,
              1.0
            ]
          }
        }
      }
    },
    "baseline_v2": {
      "n_scenarios": 30,
      "n_cause": 24,
      "n_no_cause": 6,
      "hit1_strict": {
        "value": 0.5416666666666666,
        "k": 13,
        "n": 24,
        "ci95": [
          0.3507486535891948,
          0.72108666268789
        ]
      },
      "hit3_strict": {
        "value": 0.75,
        "k": 18,
        "n": 24,
        "ci95": [
          0.55100555994846,
          0.8800063377140499
        ]
      },
      "hit1_loose": {
        "value": 0.5416666666666666,
        "k": 13,
        "n": 24,
        "ci95": [
          0.3507486535891948,
          0.72108666268789
        ]
      },
      "hit3_loose": {
        "value": 0.75,
        "k": 18,
        "n": 24,
        "ci95": [
          0.55100555994846,
          0.8800063377140499
        ]
      },
      "mrr_strict": {
        "value": 0.6319444444444444,
        "n": 24
      },
      "mrr_loose": {
        "value": 0.6319444444444444,
        "n": 24
      },
      "false_alarm": {
        "value": 0.8333333333333334,
        "k": 5,
        "n": 6,
        "ci95": [
          0.43649717781352987,
          0.9699466302516935
        ]
      },
      "miss_rate": {
        "value": 0.08333333333333333,
        "k": 2,
        "n": 24,
        "ci95": [
          0.023158815297043306,
          0.25848802193210685
        ]
      },
      "onset_accuracy": {
        "value": 0.8571428571428571,
        "k": 6,
        "n": 7,
        "ci95": [
          0.4868721707256811,
          0.9743203756552565
        ]
      },
      "f0b_explained": {
        "value": 1.0,
        "k": 3,
        "n": 3,
        "ci95": [
          0.4385029682449546,
          1.0
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 30,
        "ci95": [
          0.0,
          0.11351339317396876
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.0,
        "k": 0,
        "n": 6,
        "ci95": [
          2.7755575615628914e-17,
          0.3903342879021653
        ]
      },
      "seconds_per_scenario": {
        "mean": 3.8167999999999997,
        "max": 4.088,
        "n": 30
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": 0.3333333333333333,
          "k": 3,
          "n": 9,
          "ci95": [
            0.1205838183869109,
            0.6457978644196036
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": 0.6
        },
        {
          "value": 0.7692307692307693,
          "k": 10,
          "n": 13,
          "ci95": [
            0.49743624053272495,
            0.9182047128150143
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.9000000000000001
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 3,
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        },
        "F0b": {
          "n": 3,
          "false_alarm": {
            "value": 0.6666666666666666,
            "k": 2,
            "n": 3,
            "ci95": [
              0.2076596008020477,
              0.9385080552796037
            ]
          }
        },
        "F1": {
          "n": 7,
          "hit1_strict": {
            "value": 0.7142857142857143,
            "k": 5,
            "n": 7,
            "ci95": [
              0.35893445183261935,
              0.9177810759959433
            ]
          },
          "hit3_strict": {
            "value": 0.8571428571428571,
            "k": 6,
            "n": 7,
            "ci95": [
              0.4868721707256811,
              0.9743203756552565
            ]
          },
          "hit3_loose": {
            "value": 0.8571428571428571,
            "k": 6,
            "n": 7,
            "ci95": [
              0.4868721707256811,
              0.9743203756552565
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 7,
            "ci95": [
              5.551115123125783e-17,
              0.35433043506668743
            ]
          }
        },
        "F2": {
          "n": 6,
          "hit1_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 4,
            "n": 6,
            "ci95": [
              0.299993315138392,
              0.9032285888942195
            ]
          },
          "hit3_loose": {
            "value": 0.6666666666666666,
            "k": 4,
            "n": 6,
            "ci95": [
              0.299993315138392,
              0.9032285888942195
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        },
        "F3": {
          "n": 5,
          "hit1_strict": {
            "value": 0.4,
            "k": 2,
            "n": 5,
            "ci95": [
              0.11762077423264783,
              0.769275718723987
            ]
          },
          "hit3_strict": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          },
          "hit3_loose": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          },
          "miss_rate": {
            "value": 0.2,
            "k": 1,
            "n": 5,
            "ci95": [
              0.03622410863243014,
              0.6244653702374747
            ]
          }
        },
        "F4": {
          "n": 6,
          "hit1_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.16666666666666666,
            "k": 1,
            "n": 6,
            "ci95": [
              0.030053369748306635,
              0.5635028221864702
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 8,
          "hit1_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_strict": {
            "value": 1.0,
            "k": 8,
            "n": 8,
            "ci95": [
              0.6755924351161198,
              1.0
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 8,
            "n": 8,
            "ci95": [
              0.6755924351161198,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        },
        "medium": {
          "n": 8,
          "hit1_strict": {
            "value": 0.5,
            "k": 4,
            "n": 8,
            "ci95": [
              0.21521606221387757,
              0.7847839377861224
            ]
          },
          "hit3_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_loose": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        },
        "small": {
          "n": 8,
          "hit1_strict": {
            "value": 0.25,
            "k": 2,
            "n": 8,
            "ci95": [
              0.071479212752109,
              0.5907245696898311
            ]
          },
          "hit3_strict": {
            "value": 0.375,
            "k": 3,
            "n": 8,
            "ci95": [
              0.13684428582359742,
              0.6942576053973727
            ]
          },
          "hit3_loose": {
            "value": 0.375,
            "k": 3,
            "n": 8,
            "ci95": [
              0.13684428582359742,
              0.6942576053973727
            ]
          },
          "miss_rate": {
            "value": 0.25,
            "k": 2,
            "n": 8,
            "ci95": [
              0.071479212752109,
              0.5907245696898311
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 15,
          "hit1_strict": {
            "value": 0.4166666666666667,
            "k": 5,
            "n": 12,
            "ci95": [
              0.19326031365875665,
              0.6804886874504503
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "hit3_loose": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "miss_rate": {
            "value": 0.08333333333333333,
            "k": 1,
            "n": 12,
            "ci95": [
              0.014865094404917095,
              0.35387991114111694
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 0.6666666666666666,
            "k": 2,
            "n": 3,
            "ci95": [
              0.2076596008020477,
              0.9385080552796037
            ]
          }
        },
        "low": {
          "n": 15,
          "hit1_strict": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 10,
            "n": 12,
            "ci95": [
              0.5519691377470266,
              0.9530348578161463
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 10,
            "n": 12,
            "ci95": [
              0.5519691377470266,
              0.9530348578161463
            ]
          },
          "miss_rate": {
            "value": 0.08333333333333333,
            "k": 1,
            "n": 12,
            "ci95": [
              0.014865094404917095,
              0.35387991114111694
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 17,
          "hit1_strict": {
            "value": 0.5384615384615384,
            "k": 7,
            "n": 13,
            "ci95": [
              0.29143795714506004,
              0.7679393219046169
            ]
          },
          "hit3_strict": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "hit3_loose": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 13,
            "ci95": [
              0.0,
              0.22809537235419838
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 4,
            "ci95": [
              0.0,
              0.4898908364545973
            ]
          },
          "false_alarm": {
            "value": 0.75,
            "k": 3,
            "n": 4,
            "ci95": [
              0.30064184258240184,
              0.9544127391902995
            ]
          }
        },
        "warning": {
          "n": 13,
          "hit1_strict": {
            "value": 0.5454545454545454,
            "k": 6,
            "n": 11,
            "ci95": [
              0.280091537409322,
              0.7872872837754024
            ]
          },
          "hit3_strict": {
            "value": 0.6363636363636364,
            "k": 7,
            "n": 11,
            "ci95": [
              0.35380117450784887,
              0.8483352890463243
            ]
          },
          "hit3_loose": {
            "value": 0.6363636363636364,
            "k": 7,
            "n": 11,
            "ci95": [
              0.35380117450784887,
              0.8483352890463243
            ]
          },
          "miss_rate": {
            "value": 0.18181818181818182,
            "k": 2,
            "n": 11,
            "ci95": [
              0.05136768974608513,
              0.4769805619608442
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 2,
            "n": 2,
            "ci95": [
              0.34238022750665303,
              1.0
            ]
          }
        }
      }
    },
    "agent_r1": {
      "n_scenarios": 30,
      "n_cause": 24,
      "n_no_cause": 6,
      "hit1_strict": {
        "value": 0.6666666666666666,
        "k": 16,
        "n": 24,
        "ci95": [
          0.4670631683813174,
          0.8202780967270225
        ]
      },
      "hit3_strict": {
        "value": 0.7083333333333334,
        "k": 17,
        "n": 24,
        "ci95": [
          0.5083230633550337,
          0.8508535180303913
        ]
      },
      "hit1_loose": {
        "value": 0.8333333333333334,
        "k": 20,
        "n": 24,
        "ci95": [
          0.6414692935030095,
          0.9332132367136705
        ]
      },
      "hit3_loose": {
        "value": 0.8333333333333334,
        "k": 20,
        "n": 24,
        "ci95": [
          0.6414692935030095,
          0.9332132367136705
        ]
      },
      "mrr_strict": {
        "value": 0.6875,
        "n": 24
      },
      "mrr_loose": {
        "value": 0.8333333333333334,
        "n": 24
      },
      "false_alarm": {
        "value": 1.0,
        "k": 6,
        "n": 6,
        "ci95": [
          0.6096657120978346,
          1.0
        ]
      },
      "miss_rate": {
        "value": 0.0,
        "k": 0,
        "n": 24,
        "ci95": [
          1.3877787807814457e-17,
          0.13797620467498017
        ]
      },
      "onset_accuracy": {
        "value": 0.8571428571428571,
        "k": 6,
        "n": 7,
        "ci95": [
          0.4868721707256811,
          0.9743203756552565
        ]
      },
      "f0b_explained": {
        "value": 0.6666666666666666,
        "k": 2,
        "n": 3,
        "ci95": [
          0.2076596008020477,
          0.9385080552796037
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 30,
        "ci95": [
          0.0,
          0.11351339317396876
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.16666666666666666,
        "k": 1,
        "n": 6,
        "ci95": [
          0.030053369748306635,
          0.5635028221864702
        ]
      },
      "numcheck_bad_ratio": {
        "value": 0.0,
        "k": 0,
        "n": 30,
        "ci95": [
          0.0,
          0.11351339317396876
        ]
      },
      "seconds_per_scenario": {
        "mean": 8.660666666666666,
        "max": 17.41,
        "n": 30
      },
      "check_effect": {
        "requested": {
          "value": 0.0,
          "k": 0,
          "n": 30,
          "ci95": [
            0.0,
            0.11351339317396876
          ]
        },
        "top_changed_given_requested": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_hit3_strict_prelim": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_hit3_strict_final": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_false_alarm_prelim": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_false_alarm_final": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        }
      },
      "requests_per_scenario": {
        "mean": 1.1
      },
      "input_tokens_per_scenario": {
        "mean": 3513.3
      },
      "output_tokens_per_scenario": {
        "mean": 1031.3666666666666
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": null
        },
        {
          "value": 0.6666666666666666,
          "k": 16,
          "n": 24,
          "ci95": [
            0.4670631683813174,
            0.8202780967270225
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.91
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 3,
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        },
        "F0b": {
          "n": 3,
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        },
        "F1": {
          "n": 7,
          "hit1_strict": {
            "value": 0.8571428571428571,
            "k": 6,
            "n": 7,
            "ci95": [
              0.4868721707256811,
              0.9743203756552565
            ]
          },
          "hit3_strict": {
            "value": 0.8571428571428571,
            "k": 6,
            "n": 7,
            "ci95": [
              0.4868721707256811,
              0.9743203756552565
            ]
          },
          "hit3_loose": {
            "value": 0.8571428571428571,
            "k": 6,
            "n": 7,
            "ci95": [
              0.4868721707256811,
              0.9743203756552565
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 7,
            "ci95": [
              5.551115123125783e-17,
              0.35433043506668743
            ]
          }
        },
        "F2": {
          "n": 6,
          "hit1_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        },
        "F3": {
          "n": 5,
          "hit1_strict": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          },
          "hit3_strict": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 5,
            "ci95": [
              0.0,
              0.43448246478317476
            ]
          }
        },
        "F4": {
          "n": 6,
          "hit1_strict": {
            "value": 0.6666666666666666,
            "k": 4,
            "n": 6,
            "ci95": [
              0.299993315138392,
              0.9032285888942195
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          },
          "both_hit3_strict": {
            "value": 0.16666666666666666,
            "k": 1,
            "n": 6,
            "ci95": [
              0.030053369748306635,
              0.5635028221864702
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 8,
          "hit1_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 8,
            "n": 8,
            "ci95": [
              0.6755924351161198,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 1,
            "n": 2,
            "ci95": [
              0.09453120573423074,
              0.9054687942657693
            ]
          }
        },
        "medium": {
          "n": 8,
          "hit1_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 8,
            "n": 8,
            "ci95": [
              0.6755924351161198,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        },
        "small": {
          "n": 8,
          "hit1_strict": {
            "value": 0.5,
            "k": 4,
            "n": 8,
            "ci95": [
              0.21521606221387757,
              0.7847839377861224
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 4,
            "n": 8,
            "ci95": [
              0.21521606221387757,
              0.7847839377861224
            ]
          },
          "hit3_loose": {
            "value": 0.5,
            "k": 4,
            "n": 8,
            "ci95": [
              0.21521606221387757,
              0.7847839377861224
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 15,
          "hit1_strict": {
            "value": 0.4166666666666667,
            "k": 5,
            "n": 12,
            "ci95": [
              0.19326031365875665,
              0.6804886874504503
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 6,
            "n": 12,
            "ci95": [
              0.2537815976337061,
              0.7462184023662939
            ]
          },
          "hit3_loose": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        },
        "low": {
          "n": 15,
          "hit1_strict": {
            "value": 0.9166666666666666,
            "k": 11,
            "n": 12,
            "ci95": [
              0.646120088858883,
              0.9851349055950829
            ]
          },
          "hit3_strict": {
            "value": 0.9166666666666666,
            "k": 11,
            "n": 12,
            "ci95": [
              0.646120088858883,
              0.9851349055950829
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 17,
          "hit1_strict": {
            "value": 0.5384615384615384,
            "k": 7,
            "n": 13,
            "ci95": [
              0.29143795714506004,
              0.7679393219046169
            ]
          },
          "hit3_strict": {
            "value": 0.6153846153846154,
            "k": 8,
            "n": 13,
            "ci95": [
              0.355228915126657,
              0.8229029220223743
            ]
          },
          "hit3_loose": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 13,
            "ci95": [
              0.0,
              0.22809537235419838
            ]
          },
          "both_hit3_strict": {
            "value": 0.25,
            "k": 1,
            "n": 4,
            "ci95": [
              0.04558726080970055,
              0.6993581574175981
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 4,
            "n": 4,
            "ci95": [
              0.5101091635454027,
              1.0
            ]
          }
        },
        "warning": {
          "n": 13,
          "hit1_strict": {
            "value": 0.8181818181818182,
            "k": 9,
            "n": 11,
            "ci95": [
              0.523019438039156,
              0.9486323102539149
            ]
          },
          "hit3_strict": {
            "value": 0.8181818181818182,
            "k": 9,
            "n": 11,
            "ci95": [
              0.523019438039156,
              0.9486323102539149
            ]
          },
          "hit3_loose": {
            "value": 0.8181818181818182,
            "k": 9,
            "n": 11,
            "ci95": [
              0.523019438039156,
              0.9486323102539149
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 11,
            "ci95": [
              0.0,
              0.2588329669680317
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 2,
            "n": 2,
            "ci95": [
              0.34238022750665303,
              1.0
            ]
          }
        }
      },
      "model": "gemini-3.5-flash-lite",
      "prompt_hash": "eccd0400c0951a94e765695f6048df03073b92fe3491965a777d86c8c28a050a",
      "usage": {
        "logged_requests": 33,
        "cost_usd_total": 0.0
      }
    },
    "agent_r2": {
      "n_scenarios": 30,
      "n_cause": 24,
      "n_no_cause": 6,
      "hit1_strict": {
        "value": 0.6666666666666666,
        "k": 16,
        "n": 24,
        "ci95": [
          0.4670631683813174,
          0.8202780967270225
        ]
      },
      "hit3_strict": {
        "value": 0.7083333333333334,
        "k": 17,
        "n": 24,
        "ci95": [
          0.5083230633550337,
          0.8508535180303913
        ]
      },
      "hit1_loose": {
        "value": 0.8333333333333334,
        "k": 20,
        "n": 24,
        "ci95": [
          0.6414692935030095,
          0.9332132367136705
        ]
      },
      "hit3_loose": {
        "value": 0.8333333333333334,
        "k": 20,
        "n": 24,
        "ci95": [
          0.6414692935030095,
          0.9332132367136705
        ]
      },
      "mrr_strict": {
        "value": 0.6875,
        "n": 24
      },
      "mrr_loose": {
        "value": 0.8333333333333334,
        "n": 24
      },
      "false_alarm": {
        "value": 0.8333333333333334,
        "k": 5,
        "n": 6,
        "ci95": [
          0.43649717781352987,
          0.9699466302516935
        ]
      },
      "miss_rate": {
        "value": 0.0,
        "k": 0,
        "n": 24,
        "ci95": [
          1.3877787807814457e-17,
          0.13797620467498017
        ]
      },
      "onset_accuracy": {
        "value": 0.7142857142857143,
        "k": 5,
        "n": 7,
        "ci95": [
          0.35893445183261935,
          0.9177810759959433
        ]
      },
      "f0b_explained": {
        "value": 1.0,
        "k": 3,
        "n": 3,
        "ci95": [
          0.4385029682449546,
          1.0
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 30,
        "ci95": [
          0.0,
          0.11351339317396876
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.16666666666666666,
        "k": 1,
        "n": 6,
        "ci95": [
          0.030053369748306635,
          0.5635028221864702
        ]
      },
      "numcheck_bad_ratio": {
        "value": 0.0,
        "k": 0,
        "n": 30,
        "ci95": [
          0.0,
          0.11351339317396876
        ]
      },
      "seconds_per_scenario": {
        "mean": 8.244333333333334,
        "max": 15.86,
        "n": 30
      },
      "check_effect": {
        "requested": {
          "value": 0.0,
          "k": 0,
          "n": 30,
          "ci95": [
            0.0,
            0.11351339317396876
          ]
        },
        "top_changed_given_requested": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_hit3_strict_prelim": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_hit3_strict_final": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_false_alarm_prelim": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_false_alarm_final": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        }
      },
      "requests_per_scenario": {
        "mean": 1.0666666666666667
      },
      "input_tokens_per_scenario": {
        "mean": 3459.0
      },
      "output_tokens_per_scenario": {
        "mean": 951.2333333333333
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": null
        },
        {
          "value": 0.6666666666666666,
          "k": 16,
          "n": 24,
          "ci95": [
            0.4670631683813174,
            0.8202780967270225
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.90625
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 3,
          "false_alarm": {
            "value": 0.6666666666666666,
            "k": 2,
            "n": 3,
            "ci95": [
              0.2076596008020477,
              0.9385080552796037
            ]
          }
        },
        "F0b": {
          "n": 3,
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        },
        "F1": {
          "n": 7,
          "hit1_strict": {
            "value": 0.7142857142857143,
            "k": 5,
            "n": 7,
            "ci95": [
              0.35893445183261935,
              0.9177810759959433
            ]
          },
          "hit3_strict": {
            "value": 0.7142857142857143,
            "k": 5,
            "n": 7,
            "ci95": [
              0.35893445183261935,
              0.9177810759959433
            ]
          },
          "hit3_loose": {
            "value": 0.7142857142857143,
            "k": 5,
            "n": 7,
            "ci95": [
              0.35893445183261935,
              0.9177810759959433
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 7,
            "ci95": [
              5.551115123125783e-17,
              0.35433043506668743
            ]
          }
        },
        "F2": {
          "n": 6,
          "hit1_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        },
        "F3": {
          "n": 5,
          "hit1_strict": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "hit3_strict": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 5,
            "ci95": [
              0.0,
              0.43448246478317476
            ]
          }
        },
        "F4": {
          "n": 6,
          "hit1_strict": {
            "value": 0.6666666666666666,
            "k": 4,
            "n": 6,
            "ci95": [
              0.299993315138392,
              0.9032285888942195
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 6,
            "n": 6,
            "ci95": [
              0.6096657120978346,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          },
          "both_hit3_strict": {
            "value": 0.16666666666666666,
            "k": 1,
            "n": 6,
            "ci95": [
              0.030053369748306635,
              0.5635028221864702
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 8,
          "hit1_strict": {
            "value": 0.625,
            "k": 5,
            "n": 8,
            "ci95": [
              0.30574239460262737,
              0.8631557141764026
            ]
          },
          "hit3_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_loose": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 1,
            "n": 2,
            "ci95": [
              0.09453120573423074,
              0.9054687942657693
            ]
          }
        },
        "medium": {
          "n": 8,
          "hit1_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 8,
            "n": 8,
            "ci95": [
              0.6755924351161198,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        },
        "small": {
          "n": 8,
          "hit1_strict": {
            "value": 0.625,
            "k": 5,
            "n": 8,
            "ci95": [
              0.30574239460262737,
              0.8631557141764026
            ]
          },
          "hit3_strict": {
            "value": 0.625,
            "k": 5,
            "n": 8,
            "ci95": [
              0.30574239460262737,
              0.8631557141764026
            ]
          },
          "hit3_loose": {
            "value": 0.625,
            "k": 5,
            "n": 8,
            "ci95": [
              0.30574239460262737,
              0.8631557141764026
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 15,
          "hit1_strict": {
            "value": 0.3333333333333333,
            "k": 4,
            "n": 12,
            "ci95": [
              0.13812009109121315,
              0.6093779111272004
            ]
          },
          "hit3_strict": {
            "value": 0.4166666666666667,
            "k": 5,
            "n": 12,
            "ci95": [
              0.19326031365875665,
              0.6804886874504503
            ]
          },
          "hit3_loose": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 0.6666666666666666,
            "k": 2,
            "n": 3,
            "ci95": [
              0.2076596008020477,
              0.9385080552796037
            ]
          }
        },
        "low": {
          "n": 15,
          "hit1_strict": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          },
          "hit3_strict": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 17,
          "hit1_strict": {
            "value": 0.6153846153846154,
            "k": 8,
            "n": 13,
            "ci95": [
              0.355228915126657,
              0.8229029220223743
            ]
          },
          "hit3_strict": {
            "value": 0.6923076923076923,
            "k": 9,
            "n": 13,
            "ci95": [
              0.42369343180549035,
              0.8731929634428948
            ]
          },
          "hit3_loose": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 13,
            "ci95": [
              0.0,
              0.22809537235419838
            ]
          },
          "both_hit3_strict": {
            "value": 0.25,
            "k": 1,
            "n": 4,
            "ci95": [
              0.04558726080970055,
              0.6993581574175981
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 4,
            "n": 4,
            "ci95": [
              0.5101091635454027,
              1.0
            ]
          }
        },
        "warning": {
          "n": 13,
          "hit1_strict": {
            "value": 0.7272727272727273,
            "k": 8,
            "n": 11,
            "ci95": [
              0.43435469882387084,
              0.902539407099751
            ]
          },
          "hit3_strict": {
            "value": 0.7272727272727273,
            "k": 8,
            "n": 11,
            "ci95": [
              0.43435469882387084,
              0.902539407099751
            ]
          },
          "hit3_loose": {
            "value": 0.8181818181818182,
            "k": 9,
            "n": 11,
            "ci95": [
              0.523019438039156,
              0.9486323102539149
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 11,
            "ci95": [
              0.0,
              0.2588329669680317
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          },
          "false_alarm": {
            "value": 0.5,
            "k": 1,
            "n": 2,
            "ci95": [
              0.09453120573423074,
              0.9054687942657693
            ]
          }
        }
      },
      "model": "gemini-3.5-flash-lite",
      "prompt_hash": "87ff5056387f923867dc777626ce148d39b9b0107e2f243333e6c5c5835df502",
      "usage": {
        "logged_requests": 32,
        "cost_usd_total": 0.0
      }
    },
    "agent_v2_r1": {
      "n_scenarios": 30,
      "n_cause": 24,
      "n_no_cause": 6,
      "hit1_strict": {
        "value": 0.7916666666666666,
        "k": 19,
        "n": 24,
        "ci95": [
          0.5952954674681133,
          0.9075517464714814
        ]
      },
      "hit3_strict": {
        "value": 0.7916666666666666,
        "k": 19,
        "n": 24,
        "ci95": [
          0.5952954674681133,
          0.9075517464714814
        ]
      },
      "hit1_loose": {
        "value": 0.875,
        "k": 21,
        "n": 24,
        "ci95": [
          0.6899611872949993,
          0.9565566591987655
        ]
      },
      "hit3_loose": {
        "value": 0.875,
        "k": 21,
        "n": 24,
        "ci95": [
          0.6899611872949993,
          0.9565566591987655
        ]
      },
      "mrr_strict": {
        "value": 0.7916666666666666,
        "n": 24
      },
      "mrr_loose": {
        "value": 0.875,
        "n": 24
      },
      "false_alarm": {
        "value": 0.6666666666666666,
        "k": 4,
        "n": 6,
        "ci95": [
          0.299993315138392,
          0.9032285888942195
        ]
      },
      "miss_rate": {
        "value": 0.0,
        "k": 0,
        "n": 24,
        "ci95": [
          1.3877787807814457e-17,
          0.13797620467498017
        ]
      },
      "onset_accuracy": {
        "value": 0.6666666666666666,
        "k": 6,
        "n": 9,
        "ci95": [
          0.3542021355803963,
          0.879416181613089
        ]
      },
      "f0b_explained": {
        "value": 0.3333333333333333,
        "k": 1,
        "n": 3,
        "ci95": [
          0.06149194472039621,
          0.7923403991979522
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 30,
        "ci95": [
          0.0,
          0.11351339317396876
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.16666666666666666,
        "k": 1,
        "n": 6,
        "ci95": [
          0.030053369748306635,
          0.5635028221864702
        ]
      },
      "numcheck_bad_ratio": {
        "value": 0.03333333333333333,
        "k": 1,
        "n": 30,
        "ci95": [
          0.005908590381612455,
          0.16670390991409173
        ]
      },
      "seconds_per_scenario": {
        "mean": 15.337,
        "max": 17.44,
        "n": 30
      },
      "id_check": {
        "normalized_scenarios": {
          "value": 0.0,
          "k": 0,
          "n": 30,
          "ci95": [
            0.0,
            0.11351339317396876
          ]
        },
        "normalized_ids_total": 0,
        "retry": {
          "value": 0.0,
          "k": 0,
          "n": 30,
          "ci95": [
            0.0,
            0.11351339317396876
          ]
        },
        "unresolved_after_retry": {
          "value": 0.0,
          "k": 0,
          "n": 30,
          "ci95": [
            0.0,
            0.11351339317396876
          ]
        }
      },
      "check_effect": {
        "requested": {
          "value": 1.0,
          "k": 30,
          "n": 30,
          "ci95": [
            0.8864866068260312,
            0.9999999999999999
          ]
        },
        "top_changed_given_requested": {
          "value": 0.06666666666666667,
          "k": 2,
          "n": 30,
          "ci95": [
            0.018477023791270378,
            0.2132345836261692
          ]
        },
        "changed_hit3_strict_prelim": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_hit3_strict_final": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_false_alarm_prelim": {
          "value": 1.0,
          "k": 2,
          "n": 2,
          "ci95": [
            0.34238022750665303,
            1.0
          ]
        },
        "changed_false_alarm_final": {
          "value": 1.0,
          "k": 2,
          "n": 2,
          "ci95": [
            0.34238022750665303,
            1.0
          ]
        }
      },
      "requests_per_scenario": {
        "mean": 2.0
      },
      "input_tokens_per_scenario": {
        "mean": 9387.966666666667
      },
      "output_tokens_per_scenario": {
        "mean": 1912.7
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": null
        },
        {
          "value": 0.7916666666666666,
          "k": 19,
          "n": 24,
          "ci95": [
            0.5952954674681133,
            0.9075517464714814
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.8833333333333334
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 3,
          "false_alarm": {
            "value": 0.6666666666666666,
            "k": 2,
            "n": 3,
            "ci95": [
              0.2076596008020477,
              0.9385080552796037
            ]
          }
        },
        "F0b": {
          "n": 3,
          "false_alarm": {
            "value": 0.6666666666666666,
            "k": 2,
            "n": 3,
            "ci95": [
              0.2076596008020477,
              0.9385080552796037
            ]
          }
        },
        "F1": {
          "n": 7,
          "hit1_strict": {
            "value": 1.0,
            "k": 7,
            "n": 7,
            "ci95": [
              0.6456695649333126,
              1.0
            ]
          },
          "hit3_strict": {
            "value": 1.0,
            "k": 7,
            "n": 7,
            "ci95": [
              0.6456695649333126,
              1.0
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 7,
            "n": 7,
            "ci95": [
              0.6456695649333126,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 7,
            "ci95": [
              5.551115123125783e-17,
              0.35433043506668743
            ]
          }
        },
        "F2": {
          "n": 6,
          "hit1_strict": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        },
        "F3": {
          "n": 5,
          "hit1_strict": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "hit3_strict": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 4,
            "n": 5,
            "ci95": [
              0.37553462976252533,
              0.9637758913675698
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 5,
            "ci95": [
              0.0,
              0.43448246478317476
            ]
          }
        },
        "F4": {
          "n": 6,
          "hit1_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          },
          "both_hit3_strict": {
            "value": 0.16666666666666666,
            "k": 1,
            "n": 6,
            "ci95": [
              0.030053369748306635,
              0.5635028221864702
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 8,
          "hit1_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_loose": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 1,
            "n": 2,
            "ci95": [
              0.09453120573423074,
              0.9054687942657693
            ]
          }
        },
        "medium": {
          "n": 8,
          "hit1_strict": {
            "value": 0.625,
            "k": 5,
            "n": 8,
            "ci95": [
              0.30574239460262737,
              0.8631557141764026
            ]
          },
          "hit3_strict": {
            "value": 0.625,
            "k": 5,
            "n": 8,
            "ci95": [
              0.30574239460262737,
              0.8631557141764026
            ]
          },
          "hit3_loose": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        },
        "small": {
          "n": 8,
          "hit1_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_loose": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 15,
          "hit1_strict": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "hit3_loose": {
            "value": 0.75,
            "k": 9,
            "n": 12,
            "ci95": [
              0.46769466506643426,
              0.9110583316059453
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          }
        },
        "low": {
          "n": 15,
          "hit1_strict": {
            "value": 0.9166666666666666,
            "k": 11,
            "n": 12,
            "ci95": [
              0.646120088858883,
              0.9851349055950829
            ]
          },
          "hit3_strict": {
            "value": 0.9166666666666666,
            "k": 11,
            "n": 12,
            "ci95": [
              0.646120088858883,
              0.9851349055950829
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 3,
            "n": 3,
            "ci95": [
              0.4385029682449546,
              1.0
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 17,
          "hit1_strict": {
            "value": 0.7692307692307693,
            "k": 10,
            "n": 13,
            "ci95": [
              0.49743624053272495,
              0.9182047128150143
            ]
          },
          "hit3_strict": {
            "value": 0.7692307692307693,
            "k": 10,
            "n": 13,
            "ci95": [
              0.49743624053272495,
              0.9182047128150143
            ]
          },
          "hit3_loose": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 13,
            "ci95": [
              0.0,
              0.22809537235419838
            ]
          },
          "both_hit3_strict": {
            "value": 0.25,
            "k": 1,
            "n": 4,
            "ci95": [
              0.04558726080970055,
              0.6993581574175981
            ]
          },
          "false_alarm": {
            "value": 0.75,
            "k": 3,
            "n": 4,
            "ci95": [
              0.30064184258240184,
              0.9544127391902995
            ]
          }
        },
        "warning": {
          "n": 13,
          "hit1_strict": {
            "value": 0.8181818181818182,
            "k": 9,
            "n": 11,
            "ci95": [
              0.523019438039156,
              0.9486323102539149
            ]
          },
          "hit3_strict": {
            "value": 0.8181818181818182,
            "k": 9,
            "n": 11,
            "ci95": [
              0.523019438039156,
              0.9486323102539149
            ]
          },
          "hit3_loose": {
            "value": 0.9090909090909091,
            "k": 10,
            "n": 11,
            "ci95": [
              0.6226415635484043,
              0.9837678271141151
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 11,
            "ci95": [
              0.0,
              0.2588329669680317
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          },
          "false_alarm": {
            "value": 0.5,
            "k": 1,
            "n": 2,
            "ci95": [
              0.09453120573423074,
              0.9054687942657693
            ]
          }
        }
      },
      "model": "gemini-3.5-flash-lite",
      "prompt_hash": "b82de2919ab98d5a2793e15b7644ea305d6fb3beecd21366b673d2df8456c413",
      "usage": {
        "logged_requests": 60,
        "cost_usd_total": 0.0
      }
    },
    "agent_v2_r2": {
      "n_scenarios": 30,
      "n_cause": 24,
      "n_no_cause": 6,
      "hit1_strict": {
        "value": 0.7916666666666666,
        "k": 19,
        "n": 24,
        "ci95": [
          0.5952954674681133,
          0.9075517464714814
        ]
      },
      "hit3_strict": {
        "value": 0.7916666666666666,
        "k": 19,
        "n": 24,
        "ci95": [
          0.5952954674681133,
          0.9075517464714814
        ]
      },
      "hit1_loose": {
        "value": 0.8333333333333334,
        "k": 20,
        "n": 24,
        "ci95": [
          0.6414692935030095,
          0.9332132367136705
        ]
      },
      "hit3_loose": {
        "value": 0.8333333333333334,
        "k": 20,
        "n": 24,
        "ci95": [
          0.6414692935030095,
          0.9332132367136705
        ]
      },
      "mrr_strict": {
        "value": 0.7916666666666666,
        "n": 24
      },
      "mrr_loose": {
        "value": 0.8333333333333334,
        "n": 24
      },
      "false_alarm": {
        "value": 0.3333333333333333,
        "k": 2,
        "n": 6,
        "ci95": [
          0.09677141110578041,
          0.700006684861608
        ]
      },
      "miss_rate": {
        "value": 0.041666666666666664,
        "k": 1,
        "n": 24,
        "ci95": [
          0.0073934561655057685,
          0.20241806478655938
        ]
      },
      "onset_accuracy": {
        "value": 0.6666666666666666,
        "k": 6,
        "n": 9,
        "ci95": [
          0.3542021355803963,
          0.879416181613089
        ]
      },
      "f0b_explained": {
        "value": 0.6666666666666666,
        "k": 2,
        "n": 3,
        "ci95": [
          0.2076596008020477,
          0.9385080552796037
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 30,
        "ci95": [
          0.0,
          0.11351339317396876
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.16666666666666666,
        "k": 1,
        "n": 6,
        "ci95": [
          0.030053369748306635,
          0.5635028221864702
        ]
      },
      "numcheck_bad_ratio": {
        "value": 0.13333333333333333,
        "k": 4,
        "n": 30,
        "ci95": [
          0.053096554840547455,
          0.296813266820363
        ]
      },
      "seconds_per_scenario": {
        "mean": 15.452333333333335,
        "max": 23.4,
        "n": 30
      },
      "id_check": {
        "normalized_scenarios": {
          "value": 0.0,
          "k": 0,
          "n": 30,
          "ci95": [
            0.0,
            0.11351339317396876
          ]
        },
        "normalized_ids_total": 0,
        "retry": {
          "value": 0.0,
          "k": 0,
          "n": 30,
          "ci95": [
            0.0,
            0.11351339317396876
          ]
        },
        "unresolved_after_retry": {
          "value": 0.0,
          "k": 0,
          "n": 30,
          "ci95": [
            0.0,
            0.11351339317396876
          ]
        }
      },
      "check_effect": {
        "requested": {
          "value": 1.0,
          "k": 30,
          "n": 30,
          "ci95": [
            0.8864866068260312,
            0.9999999999999999
          ]
        },
        "top_changed_given_requested": {
          "value": 0.06666666666666667,
          "k": 2,
          "n": 30,
          "ci95": [
            0.018477023791270378,
            0.2132345836261692
          ]
        },
        "changed_hit3_strict_prelim": {
          "value": 0.0,
          "k": 0,
          "n": 1,
          "ci95": [
            0.0,
            0.7934506856227626
          ]
        },
        "changed_hit3_strict_final": {
          "value": 1.0,
          "k": 1,
          "n": 1,
          "ci95": [
            0.20654931437723745,
            1.0
          ]
        },
        "changed_false_alarm_prelim": {
          "value": 1.0,
          "k": 1,
          "n": 1,
          "ci95": [
            0.20654931437723745,
            1.0
          ]
        },
        "changed_false_alarm_final": {
          "value": 0.0,
          "k": 0,
          "n": 1,
          "ci95": [
            0.0,
            0.7934506856227626
          ]
        }
      },
      "requests_per_scenario": {
        "mean": 2.033333333333333
      },
      "input_tokens_per_scenario": {
        "mean": 9516.466666666667
      },
      "output_tokens_per_scenario": {
        "mean": 1736.9333333333334
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": 1.0,
          "k": 1,
          "n": 1,
          "ci95": [
            0.20654931437723745,
            1.0
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": 0.5
        },
        {
          "value": 0.8,
          "k": 4,
          "n": 5,
          "ci95": [
            0.37553462976252533,
            0.9637758913675698
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": 0.73
        },
        {
          "value": 0.8235294117647058,
          "k": 14,
          "n": 17,
          "ci95": [
            0.5897054143116588,
            0.9380887336237901
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.8511764705882353
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 3,
          "false_alarm": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          }
        },
        "F0b": {
          "n": 3,
          "false_alarm": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          }
        },
        "F1": {
          "n": 7,
          "hit1_strict": {
            "value": 1.0,
            "k": 7,
            "n": 7,
            "ci95": [
              0.6456695649333126,
              1.0
            ]
          },
          "hit3_strict": {
            "value": 1.0,
            "k": 7,
            "n": 7,
            "ci95": [
              0.6456695649333126,
              1.0
            ]
          },
          "hit3_loose": {
            "value": 1.0,
            "k": 7,
            "n": 7,
            "ci95": [
              0.6456695649333126,
              1.0
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 7,
            "ci95": [
              5.551115123125783e-17,
              0.35433043506668743
            ]
          }
        },
        "F2": {
          "n": 6,
          "hit1_strict": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        },
        "F3": {
          "n": 5,
          "hit1_strict": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          },
          "hit3_strict": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          },
          "hit3_loose": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          },
          "miss_rate": {
            "value": 0.2,
            "k": 1,
            "n": 5,
            "ci95": [
              0.03622410863243014,
              0.6244653702374747
            ]
          }
        },
        "F4": {
          "n": 6,
          "hit1_strict": {
            "value": 0.6666666666666666,
            "k": 4,
            "n": 6,
            "ci95": [
              0.299993315138392,
              0.9032285888942195
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 4,
            "n": 6,
            "ci95": [
              0.299993315138392,
              0.9032285888942195
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 5,
            "n": 6,
            "ci95": [
              0.43649717781352987,
              0.9699466302516935
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          },
          "both_hit3_strict": {
            "value": 0.16666666666666666,
            "k": 1,
            "n": 6,
            "ci95": [
              0.030053369748306635,
              0.5635028221864702
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 8,
          "hit1_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_strict": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "hit3_loose": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 1,
            "n": 2,
            "ci95": [
              0.09453120573423074,
              0.9054687942657693
            ]
          }
        },
        "medium": {
          "n": 8,
          "hit1_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_loose": {
            "value": 0.875,
            "k": 7,
            "n": 8,
            "ci95": [
              0.5291118177871464,
              0.9775825085499433
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        },
        "small": {
          "n": 8,
          "hit1_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_strict": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "hit3_loose": {
            "value": 0.75,
            "k": 6,
            "n": 8,
            "ci95": [
              0.40927543031016883,
              0.9285207872478909
            ]
          },
          "miss_rate": {
            "value": 0.125,
            "k": 1,
            "n": 8,
            "ci95": [
              0.02241749145005667,
              0.4708881822128535
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 15,
          "hit1_strict": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          },
          "hit3_loose": {
            "value": 0.75,
            "k": 9,
            "n": 12,
            "ci95": [
              0.46769466506643426,
              0.9110583316059453
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 12,
            "ci95": [
              2.7755575615628914e-17,
              0.24249400665524085
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          }
        },
        "low": {
          "n": 15,
          "hit1_strict": {
            "value": 0.9166666666666666,
            "k": 11,
            "n": 12,
            "ci95": [
              0.646120088858883,
              0.9851349055950829
            ]
          },
          "hit3_strict": {
            "value": 0.9166666666666666,
            "k": 11,
            "n": 12,
            "ci95": [
              0.646120088858883,
              0.9851349055950829
            ]
          },
          "hit3_loose": {
            "value": 0.9166666666666666,
            "k": 11,
            "n": 12,
            "ci95": [
              0.646120088858883,
              0.9851349055950829
            ]
          },
          "miss_rate": {
            "value": 0.08333333333333333,
            "k": 1,
            "n": 12,
            "ci95": [
              0.014865094404917095,
              0.35387991114111694
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          },
          "false_alarm": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 17,
          "hit1_strict": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "hit3_strict": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "hit3_loose": {
            "value": 0.8461538461538461,
            "k": 11,
            "n": 13,
            "ci95": [
              0.5776536898051745,
              0.9567418216419186
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 13,
            "ci95": [
              0.0,
              0.22809537235419838
            ]
          },
          "both_hit3_strict": {
            "value": 0.25,
            "k": 1,
            "n": 4,
            "ci95": [
              0.04558726080970055,
              0.6993581574175981
            ]
          },
          "false_alarm": {
            "value": 0.25,
            "k": 1,
            "n": 4,
            "ci95": [
              0.04558726080970055,
              0.6993581574175981
            ]
          }
        },
        "warning": {
          "n": 13,
          "hit1_strict": {
            "value": 0.7272727272727273,
            "k": 8,
            "n": 11,
            "ci95": [
              0.43435469882387084,
              0.902539407099751
            ]
          },
          "hit3_strict": {
            "value": 0.7272727272727273,
            "k": 8,
            "n": 11,
            "ci95": [
              0.43435469882387084,
              0.902539407099751
            ]
          },
          "hit3_loose": {
            "value": 0.8181818181818182,
            "k": 9,
            "n": 11,
            "ci95": [
              0.523019438039156,
              0.9486323102539149
            ]
          },
          "miss_rate": {
            "value": 0.09090909090909091,
            "k": 1,
            "n": 11,
            "ci95": [
              0.016232172885884855,
              0.37735843645159567
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 2,
            "ci95": [
              0.0,
              0.6576197724933469
            ]
          },
          "false_alarm": {
            "value": 0.5,
            "k": 1,
            "n": 2,
            "ci95": [
              0.09453120573423074,
              0.9054687942657693
            ]
          }
        }
      },
      "model": "gemini-3.5-flash-lite",
      "prompt_hash": "c2ed508330ddae861696c5012944c609f5ba8351987c9b3791f10535f394be5b",
      "usage": {
        "logged_requests": 61,
        "cost_usd_total": 0.0
      }
    },
    "chance": {
      "hit3_strict": 0.025958333333333333,
      "hit3_loose": 0.10418749999999999,
      "n_cause": 24,
      "n_draws": 2000
    },
    "compare": {
      "baseline_vs_baseline_v2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 11,
          "only_a": 3,
          "only_b": 7,
          "neither": 3,
          "p": 0.34375
        }
      },
      "baseline_vs_agent_r1": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 12,
          "only_a": 2,
          "only_b": 5,
          "neither": 5,
          "p": 0.453125
        }
      },
      "baseline_vs_agent_r2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 13,
          "only_a": 1,
          "only_b": 4,
          "neither": 6,
          "p": 0.375
        }
      },
      "baseline_vs_agent_v2_r1": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 12,
          "only_a": 2,
          "only_b": 7,
          "neither": 3,
          "p": 0.1796875
        }
      },
      "baseline_vs_agent_v2_r2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 11,
          "only_a": 3,
          "only_b": 8,
          "neither": 2,
          "p": 0.2265625
        }
      },
      "baseline_v2_vs_agent_r1": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 14,
          "only_a": 4,
          "only_b": 3,
          "neither": 3,
          "p": 1.0
        }
      },
      "baseline_v2_vs_agent_r2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 13,
          "only_a": 5,
          "only_b": 4,
          "neither": 2,
          "p": 1.0
        }
      },
      "baseline_v2_vs_agent_v2_r1": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 15,
          "only_a": 3,
          "only_b": 4,
          "neither": 2,
          "p": 1.0
        }
      },
      "baseline_v2_vs_agent_v2_r2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 16,
          "only_a": 2,
          "only_b": 3,
          "neither": 3,
          "p": 1.0
        }
      },
      "agent_r1_vs_agent_r2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 15,
          "only_a": 2,
          "only_b": 2,
          "neither": 5,
          "p": 1.0
        }
      },
      "agent_r1_vs_agent_v2_r1": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 13,
          "only_a": 4,
          "only_b": 6,
          "neither": 1,
          "p": 0.75390625
        }
      },
      "agent_r1_vs_agent_v2_r2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 13,
          "only_a": 4,
          "only_b": 6,
          "neither": 1,
          "p": 0.75390625
        }
      },
      "agent_r2_vs_agent_v2_r1": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 14,
          "only_a": 3,
          "only_b": 5,
          "neither": 2,
          "p": 0.7265625
        }
      },
      "agent_r2_vs_agent_v2_r2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 14,
          "only_a": 3,
          "only_b": 5,
          "neither": 2,
          "p": 0.7265625
        }
      },
      "agent_v2_r1_vs_agent_v2_r2": {
        "hit3_strict_mcnemar": {
          "n": 24,
          "both": 18,
          "only_a": 1,
          "only_b": 1,
          "neither": 4,
          "p": 1.0
        }
      }
    },
    "info": {
      "n_scenarios": 30,
      "fault_mix": {
        "F0a": 3,
        "F0b": 3,
        "F1": 7,
        "F2": 6,
        "F3": 5,
        "F4": 6
      },
      "regen_total": 0,
      "regen_by_fault": {
        "F0a": 0,
        "F0b": 0,
        "F1": 0,
        "F2": 0,
        "F3": 0,
        "F4": 0
      },
      "tau": 0.5,
      "methods": {
        "baseline": "runs/baseline/dev",
        "baseline_v2": "runs/baseline_v2/dev",
        "agent_r1": "runs/2026-09-26-gemini-3.5-flash-lite-eccd0400",
        "agent_r2": "runs/2026-09-26-gemini-3.5-flash-lite-87ff5056",
        "agent_v2_r1": "runs/2026-09-29-gemini-3.5-flash-lite-v2-b82de291",
        "agent_v2_r2": "runs/2026-09-29-gemini-3.5-flash-lite-v2-c2ed5083"
      },
      "evaluated_at": "2026-09-29T21:32:42"
    }
  },
  "test": {
    "baseline": {
      "n_scenarios": 120,
      "n_cause": 96,
      "n_no_cause": 24,
      "hit1_strict": {
        "value": 0.46875,
        "k": 45,
        "n": 96,
        "ci95": [
          0.3720607443853214,
          0.5678439798557835
        ]
      },
      "hit3_strict": {
        "value": 0.5208333333333334,
        "k": 50,
        "n": 96,
        "ci95": [
          0.4220376770680893,
          0.6180258401045079
        ]
      },
      "hit1_loose": {
        "value": 0.65625,
        "k": 63,
        "n": 96,
        "ci95": [
          0.5568803111840908,
          0.7435960676103857
        ]
      },
      "hit3_loose": {
        "value": 0.7395833333333334,
        "k": 71,
        "n": 96,
        "ci95": [
          0.6437894471878516,
          0.8169410002970126
        ]
      },
      "mrr_strict": {
        "value": 0.4913194444444444,
        "n": 96
      },
      "mrr_loose": {
        "value": 0.6909722222222222,
        "n": 96
      },
      "false_alarm": {
        "value": 1.0,
        "k": 24,
        "n": 24,
        "ci95": [
          0.8620237953250198,
          1.0
        ]
      },
      "miss_rate": {
        "value": 0.0,
        "k": 0,
        "n": 96,
        "ci95": [
          3.469446951953614e-18,
          0.038475587857675686
        ]
      },
      "onset_accuracy": {
        "value": 0.9285714285714286,
        "k": 26,
        "n": 28,
        "ci95": [
          0.7735463743160095,
          0.9801879358846942
        ]
      },
      "f0b_explained": {
        "value": 0.0,
        "k": 0,
        "n": 12,
        "ci95": [
          2.7755575615628914e-17,
          0.24249400665524085
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 120,
        "ci95": [
          0.0,
          0.031019166418703486
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.2222222222222222,
        "k": 4,
        "n": 18,
        "ci95": [
          0.09000928108601694,
          0.45214584316212625
        ]
      },
      "seconds_per_scenario": {
        "mean": 0.350175,
        "max": 0.423,
        "n": 120
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": null
        },
        {
          "value": 0.46875,
          "k": 45,
          "n": 96,
          "ci95": [
            0.3720607443853214,
            0.5678439798557835
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.9999999999999997
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 12,
          "false_alarm": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          }
        },
        "F0b": {
          "n": 12,
          "false_alarm": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          }
        },
        "F1": {
          "n": 24,
          "hit1_strict": {
            "value": 0.7916666666666666,
            "k": 19,
            "n": 24,
            "ci95": [
              0.5952954674681133,
              0.9075517464714814
            ]
          },
          "hit3_strict": {
            "value": 0.875,
            "k": 21,
            "n": 24,
            "ci95": [
              0.6899611872949993,
              0.9565566591987655
            ]
          },
          "hit3_loose": {
            "value": 0.9166666666666666,
            "k": 22,
            "n": 24,
            "ci95": [
              0.741511978067893,
              0.9768411847029567
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 24,
            "ci95": [
              1.3877787807814457e-17,
              0.13797620467498017
            ]
          }
        },
        "F2": {
          "n": 18,
          "hit1_strict": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "hit3_loose": {
            "value": 0.8888888888888888,
            "k": 16,
            "n": 18,
            "ci95": [
              0.672002348698212,
              0.9689804773543876
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          }
        },
        "F3": {
          "n": 18,
          "hit1_strict": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "hit3_loose": {
            "value": 0.8888888888888888,
            "k": 16,
            "n": 18,
            "ci95": [
              0.672002348698212,
              0.9689804773543876
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          }
        },
        "F4": {
          "n": 18,
          "hit1_strict": {
            "value": 0.6111111111111112,
            "k": 11,
            "n": 18,
            "ci95": [
              0.3861904160223791,
              0.7969475342783638
            ]
          },
          "hit3_strict": {
            "value": 0.7777777777777778,
            "k": 14,
            "n": 18,
            "ci95": [
              0.5478541568378739,
              0.9099907189139831
            ]
          },
          "hit3_loose": {
            "value": 0.8888888888888888,
            "k": 16,
            "n": 18,
            "ci95": [
              0.672002348698212,
              0.9689804773543876
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "both_hit3_strict": {
            "value": 0.2222222222222222,
            "k": 4,
            "n": 18,
            "ci95": [
              0.09000928108601694,
              0.45214584316212625
            ]
          }
        },
        "F5": {
          "n": 18,
          "hit1_strict": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "hit3_loose": {
            "value": 0.05555555555555555,
            "k": 1,
            "n": 18,
            "ci95": [
              0.009875190978303097,
              0.25757300781872594
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 32,
          "hit1_strict": {
            "value": 0.5,
            "k": 16,
            "n": 32,
            "ci95": [
              0.33630882869327483,
              0.6636911713067252
            ]
          },
          "hit3_strict": {
            "value": 0.53125,
            "k": 17,
            "n": 32,
            "ci95": [
              0.3644951711897464,
              0.6913061289193116
            ]
          },
          "hit3_loose": {
            "value": 0.78125,
            "k": 25,
            "n": 32,
            "ci95": [
              0.6124500635075647,
              0.8897616374739581
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 32,
            "ci95": [
              0.0,
              0.1071791982550706
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          }
        },
        "medium": {
          "n": 32,
          "hit1_strict": {
            "value": 0.59375,
            "k": 19,
            "n": 32,
            "ci95": [
              0.42260024875382873,
              0.7448036515733456
            ]
          },
          "hit3_strict": {
            "value": 0.625,
            "k": 20,
            "n": 32,
            "ci95": [
              0.4525440735307745,
              0.7706611269054578
            ]
          },
          "hit3_loose": {
            "value": 0.8125,
            "k": 26,
            "n": 32,
            "ci95": [
              0.6469084479862404,
              0.9111045531043405
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 32,
            "ci95": [
              0.0,
              0.1071791982550706
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        },
        "small": {
          "n": 32,
          "hit1_strict": {
            "value": 0.3125,
            "k": 10,
            "n": 32,
            "ci95": [
              0.17952541633609734,
              0.4856667830095541
            ]
          },
          "hit3_strict": {
            "value": 0.40625,
            "k": 13,
            "n": 32,
            "ci95": [
              0.2551963484266544,
              0.5773997512461713
            ]
          },
          "hit3_loose": {
            "value": 0.625,
            "k": 20,
            "n": 32,
            "ci95": [
              0.4525440735307745,
              0.7706611269054578
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 32,
            "ci95": [
              0.0,
              0.1071791982550706
            ]
          },
          "both_hit3_strict": {
            "value": 0.16666666666666666,
            "k": 1,
            "n": 6,
            "ci95": [
              0.030053369748306635,
              0.5635028221864702
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 60,
          "hit1_strict": {
            "value": 0.4791666666666667,
            "k": 23,
            "n": 48,
            "ci95": [
              0.34471316176951616,
              0.616707676966572
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 24,
            "n": 48,
            "ci95": [
              0.36389330502679385,
              0.6361066949732062
            ]
          },
          "hit3_loose": {
            "value": 0.6875,
            "k": 33,
            "n": 48,
            "ci95": [
              0.5466692858410841,
              0.8005431655341219
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 48,
            "ci95": [
              6.938893903907228e-18,
              0.07410012966611748
            ]
          },
          "both_hit3_strict": {
            "value": 0.1111111111111111,
            "k": 1,
            "n": 9,
            "ci95": [
              0.019890887638544907,
              0.434999705576656
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          }
        },
        "low": {
          "n": 60,
          "hit1_strict": {
            "value": 0.4583333333333333,
            "k": 22,
            "n": 48,
            "ci95": [
              0.32575242292867446,
              0.5970892545435019
            ]
          },
          "hit3_strict": {
            "value": 0.5416666666666666,
            "k": 26,
            "n": 48,
            "ci95": [
              0.40291074545649797,
              0.6742475770713254
            ]
          },
          "hit3_loose": {
            "value": 0.7916666666666666,
            "k": 38,
            "n": 48,
            "ci95": [
              0.6574109872738206,
              0.882697270420944
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 48,
            "ci95": [
              6.938893903907228e-18,
              0.07410012966611748
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 3,
            "n": 9,
            "ci95": [
              0.1205838183869109,
              0.6457978644196036
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 96,
          "hit1_strict": {
            "value": 0.4567901234567901,
            "k": 37,
            "n": 81,
            "ci95": [
              0.352733326843646,
              0.5647598410332293
            ]
          },
          "hit3_strict": {
            "value": 0.48148148148148145,
            "k": 39,
            "n": 81,
            "ci95": [
              0.37599641612884654,
              0.5886435129612427
            ]
          },
          "hit3_loose": {
            "value": 0.7160493827160493,
            "k": 58,
            "n": 81,
            "ci95": [
              0.609821476977911,
              0.802712683637712
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 81,
            "ci95": [
              0.0,
              0.045278085432415204
            ]
          },
          "both_hit3_strict": {
            "value": 0.26666666666666666,
            "k": 4,
            "n": 15,
            "ci95": [
              0.10897453325692374,
              0.5195043405598057
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 15,
            "n": 15,
            "ci95": [
              0.7961166989641515,
              1.0
            ]
          }
        },
        "warning": {
          "n": 24,
          "hit1_strict": {
            "value": 0.5333333333333333,
            "k": 8,
            "n": 15,
            "ci95": [
              0.30116980025498397,
              0.7519046463426261
            ]
          },
          "hit3_strict": {
            "value": 0.7333333333333333,
            "k": 11,
            "n": 15,
            "ci95": [
              0.4804956594401943,
              0.8910254667430761
            ]
          },
          "hit3_loose": {
            "value": 0.8666666666666667,
            "k": 13,
            "n": 15,
            "ci95": [
              0.6211801716767764,
              0.9626387408969346
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 9,
            "n": 9,
            "ci95": [
              0.7008549515804559,
              1.0
            ]
          }
        }
      }
    },
    "agent": {
      "n_scenarios": 120,
      "n_cause": 96,
      "n_no_cause": 24,
      "hit1_strict": {
        "value": 0.46875,
        "k": 45,
        "n": 96,
        "ci95": [
          0.3720607443853214,
          0.5678439798557835
        ]
      },
      "hit3_strict": {
        "value": 0.5208333333333334,
        "k": 50,
        "n": 96,
        "ci95": [
          0.4220376770680893,
          0.6180258401045079
        ]
      },
      "hit1_loose": {
        "value": 0.5625,
        "k": 54,
        "n": 96,
        "ci95": [
          0.46275882627554077,
          0.6574317252422498
        ]
      },
      "hit3_loose": {
        "value": 0.59375,
        "k": 57,
        "n": 96,
        "ci95": [
          0.49373884894170256,
          0.6865469783349833
        ]
      },
      "mrr_strict": {
        "value": 0.4930555555555556,
        "n": 96
      },
      "mrr_loose": {
        "value": 0.576388888888889,
        "n": 96
      },
      "false_alarm": {
        "value": 0.7083333333333334,
        "k": 17,
        "n": 24,
        "ci95": [
          0.5083230633550337,
          0.8508535180303913
        ]
      },
      "miss_rate": {
        "value": 0.052083333333333336,
        "k": 5,
        "n": 96,
        "ci95": [
          0.02244960047424143,
          0.11618478031492638
        ]
      },
      "onset_accuracy": {
        "value": 0.9047619047619048,
        "k": 19,
        "n": 21,
        "ci95": [
          0.7108586092608176,
          0.9734812325619255
        ]
      },
      "f0b_explained": {
        "value": 0.8333333333333334,
        "k": 10,
        "n": 12,
        "ci95": [
          0.5519691377470266,
          0.9530348578161463
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 120,
        "ci95": [
          0.0,
          0.031019166418703486
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.2777777777777778,
        "k": 5,
        "n": 18,
        "ci95": [
          0.12499753379553993,
          0.5087265656029746
        ]
      },
      "numcheck_bad_ratio": {
        "value": 0.041666666666666664,
        "k": 5,
        "n": 120,
        "ci95": [
          0.017926716880109515,
          0.09384085233703535
        ]
      },
      "seconds_per_scenario": {
        "mean": 8.977916666666667,
        "max": 23.65,
        "n": 120
      },
      "check_effect": {
        "requested": {
          "value": 0.0,
          "k": 0,
          "n": 120,
          "ci95": [
            0.0,
            0.031019166418703486
          ]
        },
        "top_changed_given_requested": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_hit3_strict_prelim": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_hit3_strict_final": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_false_alarm_prelim": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_false_alarm_final": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        }
      },
      "requests_per_scenario": {
        "mean": 1.1333333333333333
      },
      "input_tokens_per_scenario": {
        "mean": 3751.5416666666665
      },
      "output_tokens_per_scenario": {
        "mean": 965.3416666666667
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": null
        },
        {
          "value": 0.4945054945054945,
          "k": 45,
          "n": 91,
          "ci95": [
            0.39410584110376173,
            0.5953502468515007
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.8985714285714286
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 12,
          "false_alarm": {
            "value": 0.9166666666666666,
            "k": 11,
            "n": 12,
            "ci95": [
              0.646120088858883,
              0.9851349055950829
            ]
          }
        },
        "F0b": {
          "n": 12,
          "false_alarm": {
            "value": 0.5,
            "k": 6,
            "n": 12,
            "ci95": [
              0.2537815976337061,
              0.7462184023662939
            ]
          }
        },
        "F1": {
          "n": 24,
          "hit1_strict": {
            "value": 0.5416666666666666,
            "k": 13,
            "n": 24,
            "ci95": [
              0.3507486535891948,
              0.72108666268789
            ]
          },
          "hit3_strict": {
            "value": 0.5416666666666666,
            "k": 13,
            "n": 24,
            "ci95": [
              0.3507486535891948,
              0.72108666268789
            ]
          },
          "hit3_loose": {
            "value": 0.5416666666666666,
            "k": 13,
            "n": 24,
            "ci95": [
              0.3507486535891948,
              0.72108666268789
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 24,
            "ci95": [
              1.3877787807814457e-17,
              0.13797620467498017
            ]
          }
        },
        "F2": {
          "n": 18,
          "hit1_strict": {
            "value": 0.3333333333333333,
            "k": 6,
            "n": 18,
            "ci95": [
              0.1627877475083369,
              0.5625053270405489
            ]
          },
          "hit3_strict": {
            "value": 0.3333333333333333,
            "k": 6,
            "n": 18,
            "ci95": [
              0.1627877475083369,
              0.5625053270405489
            ]
          },
          "hit3_loose": {
            "value": 0.7222222222222222,
            "k": 13,
            "n": 18,
            "ci95": [
              0.4912734343970254,
              0.8750024662044601
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          }
        },
        "F3": {
          "n": 18,
          "hit1_strict": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          }
        },
        "F4": {
          "n": 18,
          "hit1_strict": {
            "value": 0.6111111111111112,
            "k": 11,
            "n": 18,
            "ci95": [
              0.3861904160223791,
              0.7969475342783638
            ]
          },
          "hit3_strict": {
            "value": 0.8888888888888888,
            "k": 16,
            "n": 18,
            "ci95": [
              0.672002348698212,
              0.9689804773543876
            ]
          },
          "hit3_loose": {
            "value": 0.8888888888888888,
            "k": 16,
            "n": 18,
            "ci95": [
              0.672002348698212,
              0.9689804773543876
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "both_hit3_strict": {
            "value": 0.2777777777777778,
            "k": 5,
            "n": 18,
            "ci95": [
              0.12499753379553993,
              0.5087265656029746
            ]
          }
        },
        "F5": {
          "n": 18,
          "hit1_strict": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "hit3_loose": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "miss_rate": {
            "value": 0.2777777777777778,
            "k": 5,
            "n": 18,
            "ci95": [
              0.12499753379553993,
              0.5087265656029746
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 32,
          "hit1_strict": {
            "value": 0.5,
            "k": 16,
            "n": 32,
            "ci95": [
              0.33630882869327483,
              0.6636911713067252
            ]
          },
          "hit3_strict": {
            "value": 0.5625,
            "k": 18,
            "n": 32,
            "ci95": [
              0.3932559113685198,
              0.7183466888495963
            ]
          },
          "hit3_loose": {
            "value": 0.59375,
            "k": 19,
            "n": 32,
            "ci95": [
              0.42260024875382873,
              0.7448036515733456
            ]
          },
          "miss_rate": {
            "value": 0.09375,
            "k": 3,
            "n": 32,
            "ci95": [
              0.032401551298366194,
              0.24218154728387864
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 3,
            "n": 6,
            "ci95": [
              0.1876163064826506,
              0.8123836935173494
            ]
          }
        },
        "medium": {
          "n": 32,
          "hit1_strict": {
            "value": 0.5,
            "k": 16,
            "n": 32,
            "ci95": [
              0.33630882869327483,
              0.6636911713067252
            ]
          },
          "hit3_strict": {
            "value": 0.5625,
            "k": 18,
            "n": 32,
            "ci95": [
              0.3932559113685198,
              0.7183466888495963
            ]
          },
          "hit3_loose": {
            "value": 0.6875,
            "k": 22,
            "n": 32,
            "ci95": [
              0.5143332169904459,
              0.8204745836639026
            ]
          },
          "miss_rate": {
            "value": 0.0625,
            "k": 2,
            "n": 32,
            "ci95": [
              0.017310567031640967,
              0.2014712314415458
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 6,
            "ci95": [
              2.7755575615628914e-17,
              0.3903342879021653
            ]
          }
        },
        "small": {
          "n": 32,
          "hit1_strict": {
            "value": 0.40625,
            "k": 13,
            "n": 32,
            "ci95": [
              0.2551963484266544,
              0.5773997512461713
            ]
          },
          "hit3_strict": {
            "value": 0.4375,
            "k": 14,
            "n": 32,
            "ci95": [
              0.2816533111504036,
              0.6067440886314801
            ]
          },
          "hit3_loose": {
            "value": 0.5,
            "k": 16,
            "n": 32,
            "ci95": [
              0.33630882869327483,
              0.6636911713067252
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 32,
            "ci95": [
              0.0,
              0.1071791982550706
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 2,
            "n": 6,
            "ci95": [
              0.09677141110578041,
              0.700006684861608
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 60,
          "hit1_strict": {
            "value": 0.4375,
            "k": 21,
            "n": 48,
            "ci95": [
              0.30701269040654244,
              0.5772498258017222
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 24,
            "n": 48,
            "ci95": [
              0.36389330502679385,
              0.6361066949732062
            ]
          },
          "hit3_loose": {
            "value": 0.6041666666666666,
            "k": 29,
            "n": 48,
            "ci95": [
              0.4631040849254725,
              0.7297917213940862
            ]
          },
          "miss_rate": {
            "value": 0.0625,
            "k": 3,
            "n": 48,
            "ci95": [
              0.021483232908389285,
              0.1683543805494635
            ]
          },
          "both_hit3_strict": {
            "value": 0.1111111111111111,
            "k": 1,
            "n": 9,
            "ci95": [
              0.019890887638544907,
              0.434999705576656
            ]
          },
          "false_alarm": {
            "value": 0.5833333333333334,
            "k": 7,
            "n": 12,
            "ci95": [
              0.31951131254954973,
              0.8067396863412435
            ]
          }
        },
        "low": {
          "n": 60,
          "hit1_strict": {
            "value": 0.5,
            "k": 24,
            "n": 48,
            "ci95": [
              0.36389330502679385,
              0.6361066949732062
            ]
          },
          "hit3_strict": {
            "value": 0.5416666666666666,
            "k": 26,
            "n": 48,
            "ci95": [
              0.40291074545649797,
              0.6742475770713254
            ]
          },
          "hit3_loose": {
            "value": 0.5833333333333334,
            "k": 28,
            "n": 48,
            "ci95": [
              0.4428133228397628,
              0.7115033222158844
            ]
          },
          "miss_rate": {
            "value": 0.041666666666666664,
            "k": 2,
            "n": 48,
            "ci95": [
              0.011501893489386586,
              0.13975655870455445
            ]
          },
          "both_hit3_strict": {
            "value": 0.4444444444444444,
            "k": 4,
            "n": 9,
            "ci95": [
              0.18877852109766452,
              0.733348706504507
            ]
          },
          "false_alarm": {
            "value": 0.8333333333333334,
            "k": 10,
            "n": 12,
            "ci95": [
              0.5519691377470266,
              0.9530348578161463
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 96,
          "hit1_strict": {
            "value": 0.4444444444444444,
            "k": 36,
            "n": 81,
            "ci95": [
              0.3411955250345785,
              0.5527242622356897
            ]
          },
          "hit3_strict": {
            "value": 0.49382716049382713,
            "k": 40,
            "n": 81,
            "ci95": [
              0.38772115676987573,
              0.6004921529268207
            ]
          },
          "hit3_loose": {
            "value": 0.5679012345679012,
            "k": 46,
            "n": 81,
            "ci95": [
              0.45937436474974946,
              0.6702792285865891
            ]
          },
          "miss_rate": {
            "value": 0.06172839506172839,
            "k": 5,
            "n": 81,
            "ci95": [
              0.02665251209702916,
              0.13649247636842118
            ]
          },
          "both_hit3_strict": {
            "value": 0.26666666666666666,
            "k": 4,
            "n": 15,
            "ci95": [
              0.10897453325692374,
              0.5195043405598057
            ]
          },
          "false_alarm": {
            "value": 0.6,
            "k": 9,
            "n": 15,
            "ci95": [
              0.357468301206246,
              0.8017550385865841
            ]
          }
        },
        "warning": {
          "n": 24,
          "hit1_strict": {
            "value": 0.6,
            "k": 9,
            "n": 15,
            "ci95": [
              0.357468301206246,
              0.8017550385865841
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 10,
            "n": 15,
            "ci95": [
              0.41713547738519774,
              0.8482367556028526
            ]
          },
          "hit3_loose": {
            "value": 0.7333333333333333,
            "k": 11,
            "n": 15,
            "ci95": [
              0.4804956594401943,
              0.8910254667430761
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          },
          "false_alarm": {
            "value": 0.8888888888888888,
            "k": 8,
            "n": 9,
            "ci95": [
              0.5650002944233439,
              0.9801091123614551
            ]
          }
        }
      },
      "model": "gemini-3.5-flash-lite",
      "prompt_hash": "87ff5056387f923867dc777626ce148d39b9b0107e2f243333e6c5c5835df502",
      "usage": {
        "logged_requests": 136,
        "cost_usd_total": 0.0
      },
      "repeatability": {
        "value": 0.7,
        "k": 14,
        "n": 20,
        "ci95": [
          0.4810271816464765,
          0.8545227551323956
        ],
        "runs_per_scenario": 3,
        "run_dirs": [
          "runs/2026-09-26-gemini-3.5-flash-lite-87ff5056",
          "runs/2026-09-26-gemini-3.5-flash-lite-87ff5056-rep1",
          "runs/2026-09-26-gemini-3.5-flash-lite-87ff5056-rep2"
        ],
        "scenarios": [
          "scn_5001",
          "scn_5002",
          "scn_5005",
          "scn_5017",
          "scn_5022",
          "scn_5026",
          "scn_5030",
          "scn_5054",
          "scn_5055",
          "scn_5056",
          "scn_5068",
          "scn_5069",
          "scn_5076",
          "scn_5086",
          "scn_5092",
          "scn_5093",
          "scn_5097",
          "scn_5108",
          "scn_5111",
          "scn_5116"
        ],
        "missing": []
      }
    },
    "chance": {
      "hit3_strict": 0.030395833333333334,
      "hit3_loose": 0.087078125,
      "n_cause": 96,
      "n_draws": 2000
    },
    "compare": {
      "baseline_vs_agent": {
        "hit3_strict_mcnemar": {
          "n": 96,
          "both": 40,
          "only_a": 10,
          "only_b": 10,
          "neither": 36,
          "p": 1.0
        }
      },
      "mcnemar_p": 1.0
    },
    "info": {
      "n_scenarios": 120,
      "fault_mix": {
        "F0a": 12,
        "F0b": 12,
        "F1": 24,
        "F2": 18,
        "F3": 18,
        "F4": 18,
        "F5": 18
      },
      "regen_total": 6,
      "regen_by_fault": {
        "F0a": 4,
        "F0b": 0,
        "F1": 2,
        "F2": 0,
        "F3": 0,
        "F4": 0,
        "F5": 0
      },
      "tau": 0.5,
      "methods": {
        "baseline": "runs/baseline/test",
        "agent": "runs/2026-09-26-gemini-3.5-flash-lite-87ff5056"
      },
      "evaluated_at": "2026-09-29T20:43:16"
    }
  },
  "test2": {
    "baseline": {
      "n_scenarios": 120,
      "n_cause": 96,
      "n_no_cause": 24,
      "hit1_strict": {
        "value": 0.3541666666666667,
        "k": 34,
        "n": 96,
        "ci95": [
          0.2657984032143951,
          0.4537569765774271
        ]
      },
      "hit3_strict": {
        "value": 0.40625,
        "k": 39,
        "n": 96,
        "ci95": [
          0.31345302166501676,
          0.5062611510582975
        ]
      },
      "hit1_loose": {
        "value": 0.53125,
        "k": 51,
        "n": 96,
        "ci95": [
          0.4321560201442166,
          0.6279392556146787
        ]
      },
      "hit3_loose": {
        "value": 0.6041666666666666,
        "k": 58,
        "n": 96,
        "ci95": [
          0.5041511204834945,
          0.6961664653794899
        ]
      },
      "mrr_strict": {
        "value": 0.375,
        "n": 96
      },
      "mrr_loose": {
        "value": 0.5642361111111112,
        "n": 96
      },
      "false_alarm": {
        "value": 1.0,
        "k": 24,
        "n": 24,
        "ci95": [
          0.8620237953250198,
          1.0
        ]
      },
      "miss_rate": {
        "value": 0.0,
        "k": 0,
        "n": 96,
        "ci95": [
          3.469446951953614e-18,
          0.038475587857675686
        ]
      },
      "onset_accuracy": {
        "value": 0.8333333333333334,
        "k": 20,
        "n": 24,
        "ci95": [
          0.6414692935030095,
          0.9332132367136705
        ]
      },
      "f0b_explained": {
        "value": 0.0,
        "k": 0,
        "n": 12,
        "ci95": [
          2.7755575615628914e-17,
          0.24249400665524085
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 120,
        "ci95": [
          0.0,
          0.031019166418703486
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.2,
        "k": 3,
        "n": 15,
        "ci95": [
          0.07047549346981563,
          0.45185448715169363
        ]
      },
      "seconds_per_scenario": {
        "mean": 0.3703416666666667,
        "max": 0.433,
        "n": 120
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": null
        },
        {
          "value": 0.3541666666666667,
          "k": 34,
          "n": 96,
          "ci95": [
            0.2657984032143951,
            0.4537569765774271
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.9999999999995062
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 12,
          "false_alarm": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          }
        },
        "F0b": {
          "n": 12,
          "false_alarm": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          }
        },
        "F1": {
          "n": 18,
          "hit1_strict": {
            "value": 0.7222222222222222,
            "k": 13,
            "n": 18,
            "ci95": [
              0.4912734343970254,
              0.8750024662044601
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          }
        },
        "F2": {
          "n": 15,
          "hit1_strict": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          }
        },
        "F3": {
          "n": 15,
          "hit1_strict": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "hit3_strict": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          }
        },
        "F4": {
          "n": 15,
          "hit1_strict": {
            "value": 0.5333333333333333,
            "k": 8,
            "n": 15,
            "ci95": [
              0.30116980025498397,
              0.7519046463426261
            ]
          },
          "hit3_strict": {
            "value": 0.6,
            "k": 9,
            "n": 15,
            "ci95": [
              0.357468301206246,
              0.8017550385865841
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "both_hit3_strict": {
            "value": 0.2,
            "k": 3,
            "n": 15,
            "ci95": [
              0.07047549346981563,
              0.45185448715169363
            ]
          }
        },
        "F5": {
          "n": 15,
          "hit1_strict": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "hit3_loose": {
            "value": 0.26666666666666666,
            "k": 4,
            "n": 15,
            "ci95": [
              0.10897453325692374,
              0.5195043405598057
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          }
        },
        "F6": {
          "n": 18,
          "hit1_strict": {
            "value": 0.05555555555555555,
            "k": 1,
            "n": 18,
            "ci95": [
              0.009875190978303097,
              0.25757300781872594
            ]
          },
          "hit3_strict": {
            "value": 0.16666666666666666,
            "k": 3,
            "n": 18,
            "ci95": [
              0.05836576805861446,
              0.39222038103915724
            ]
          },
          "hit3_loose": {
            "value": 0.16666666666666666,
            "k": 3,
            "n": 18,
            "ci95": [
              0.05836576805861446,
              0.39222038103915724
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 32,
          "hit1_strict": {
            "value": 0.40625,
            "k": 13,
            "n": 32,
            "ci95": [
              0.2551963484266544,
              0.5773997512461713
            ]
          },
          "hit3_strict": {
            "value": 0.40625,
            "k": 13,
            "n": 32,
            "ci95": [
              0.2551963484266544,
              0.5773997512461713
            ]
          },
          "hit3_loose": {
            "value": 0.625,
            "k": 20,
            "n": 32,
            "ci95": [
              0.4525440735307745,
              0.7706611269054578
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 32,
            "ci95": [
              0.0,
              0.1071791982550706
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 4,
            "ci95": [
              0.0,
              0.4898908364545973
            ]
          }
        },
        "medium": {
          "n": 32,
          "hit1_strict": {
            "value": 0.40625,
            "k": 13,
            "n": 32,
            "ci95": [
              0.2551963484266544,
              0.5773997512461713
            ]
          },
          "hit3_strict": {
            "value": 0.5,
            "k": 16,
            "n": 32,
            "ci95": [
              0.33630882869327483,
              0.6636911713067252
            ]
          },
          "hit3_loose": {
            "value": 0.6875,
            "k": 22,
            "n": 32,
            "ci95": [
              0.5143332169904459,
              0.8204745836639026
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 32,
            "ci95": [
              0.0,
              0.1071791982550706
            ]
          },
          "both_hit3_strict": {
            "value": 0.2,
            "k": 1,
            "n": 5,
            "ci95": [
              0.03622410863243014,
              0.6244653702374747
            ]
          }
        },
        "small": {
          "n": 32,
          "hit1_strict": {
            "value": 0.25,
            "k": 8,
            "n": 32,
            "ci95": [
              0.1325240091850904,
              0.4210655899424448
            ]
          },
          "hit3_strict": {
            "value": 0.3125,
            "k": 10,
            "n": 32,
            "ci95": [
              0.17952541633609734,
              0.4856667830095541
            ]
          },
          "hit3_loose": {
            "value": 0.5,
            "k": 16,
            "n": 32,
            "ci95": [
              0.33630882869327483,
              0.6636911713067252
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 32,
            "ci95": [
              0.0,
              0.1071791982550706
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 2,
            "n": 6,
            "ci95": [
              0.09677141110578041,
              0.700006684861608
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 60,
          "hit1_strict": {
            "value": 0.2916666666666667,
            "k": 14,
            "n": 48,
            "ci95": [
              0.18241578926516536,
              0.431792598095717
            ]
          },
          "hit3_strict": {
            "value": 0.3958333333333333,
            "k": 19,
            "n": 48,
            "ci95": [
              0.2702082786059137,
              0.5368959150745274
            ]
          },
          "hit3_loose": {
            "value": 0.5833333333333334,
            "k": 28,
            "n": 48,
            "ci95": [
              0.4428133228397628,
              0.7115033222158844
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 48,
            "ci95": [
              6.938893903907228e-18,
              0.07410012966611748
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 8,
            "ci95": [
              0.0,
              0.32440756488388023
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          }
        },
        "low": {
          "n": 60,
          "hit1_strict": {
            "value": 0.4166666666666667,
            "k": 20,
            "n": 48,
            "ci95": [
              0.2884966777841157,
              0.5571866771602373
            ]
          },
          "hit3_strict": {
            "value": 0.4166666666666667,
            "k": 20,
            "n": 48,
            "ci95": [
              0.2884966777841157,
              0.5571866771602373
            ]
          },
          "hit3_loose": {
            "value": 0.625,
            "k": 30,
            "n": 48,
            "ci95": [
              0.4836276351921527,
              0.747847332391318
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 48,
            "ci95": [
              6.938893903907228e-18,
              0.07410012966611748
            ]
          },
          "both_hit3_strict": {
            "value": 0.42857142857142855,
            "k": 3,
            "n": 7,
            "ci95": [
              0.15821985525146975,
              0.7495416354723428
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 12,
            "n": 12,
            "ci95": [
              0.7575059933447592,
              1.0
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 86,
          "hit1_strict": {
            "value": 0.3972602739726027,
            "k": 29,
            "n": 73,
            "ci95": [
              0.29286701857646835,
              0.5119258597705996
            ]
          },
          "hit3_strict": {
            "value": 0.4246575342465753,
            "k": 31,
            "n": 73,
            "ci95": [
              0.31784191542578866,
              0.5390061953620611
            ]
          },
          "hit3_loose": {
            "value": 0.6438356164383562,
            "k": 47,
            "n": 73,
            "ci95": [
              0.5293349749798597,
              0.743954995334245
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 73,
            "ci95": [
              0.0,
              0.0499920079557311
            ]
          },
          "both_hit3_strict": {
            "value": 0.16666666666666666,
            "k": 2,
            "n": 12,
            "ci95": [
              0.04696514218385378,
              0.4480308622529734
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 13,
            "n": 13,
            "ci95": [
              0.7719046276458016,
              0.9999999999999999
            ]
          }
        },
        "warning": {
          "n": 34,
          "hit1_strict": {
            "value": 0.21739130434782608,
            "k": 5,
            "n": 23,
            "ci95": [
              0.09663978026586204,
              0.4190348301626401
            ]
          },
          "hit3_strict": {
            "value": 0.34782608695652173,
            "k": 8,
            "n": 23,
            "ci95": [
              0.1881127593879773,
              0.55109664622737
            ]
          },
          "hit3_loose": {
            "value": 0.4782608695652174,
            "k": 11,
            "n": 23,
            "ci95": [
              0.29237157416646653,
              0.6703726266357259
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 23,
            "ci95": [
              0.0,
              0.14311661845042686
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          },
          "false_alarm": {
            "value": 1.0,
            "k": 11,
            "n": 11,
            "ci95": [
              0.7411670330319684,
              1.0
            ]
          }
        }
      }
    },
    "baseline_v2": {
      "n_scenarios": 120,
      "n_cause": 96,
      "n_no_cause": 24,
      "hit1_strict": {
        "value": 0.4479166666666667,
        "k": 43,
        "n": 96,
        "ci95": [
          0.3523575930459819,
          0.5474836140225261
        ]
      },
      "hit3_strict": {
        "value": 0.625,
        "k": 60,
        "n": 96,
        "ci95": [
          0.5251071844189874,
          0.7152739186165937
        ]
      },
      "hit1_loose": {
        "value": 0.4583333333333333,
        "k": 44,
        "n": 96,
        "ci95": [
          0.36218849483208426,
          0.5576844708227221
        ]
      },
      "hit3_loose": {
        "value": 0.6458333333333334,
        "k": 62,
        "n": 96,
        "ci95": [
          0.5462430234225731,
          0.7342015967856051
        ]
      },
      "mrr_strict": {
        "value": 0.53125,
        "n": 96
      },
      "mrr_loose": {
        "value": 0.545138888888889,
        "n": 96
      },
      "false_alarm": {
        "value": 0.5,
        "k": 12,
        "n": 24,
        "ci95": [
          0.3142742581957335,
          0.6857257418042665
        ]
      },
      "miss_rate": {
        "value": 0.10416666666666667,
        "k": 10,
        "n": 96,
        "ci95": [
          0.057571532417443415,
          0.18122164130321652
        ]
      },
      "onset_accuracy": {
        "value": 0.9459459459459459,
        "k": 35,
        "n": 37,
        "ci95": [
          0.822953456491869,
          0.9850490231668423
        ]
      },
      "f0b_explained": {
        "value": 1.0,
        "k": 12,
        "n": 12,
        "ci95": [
          0.7575059933447592,
          1.0
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 120,
        "ci95": [
          0.0,
          0.031019166418703486
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.3333333333333333,
        "k": 5,
        "n": 15,
        "ci95": [
          0.1517632443971473,
          0.5828645226148023
        ]
      },
      "seconds_per_scenario": {
        "mean": 3.8671750000000005,
        "max": 4.075,
        "n": 120
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": null
        },
        {
          "value": 0.14285714285714285,
          "k": 6,
          "n": 42,
          "ci95": [
            0.06715967015962647,
            0.27841089474444775
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": 0.6000000000000001
        },
        {
          "value": 0.8409090909090909,
          "k": 37,
          "n": 44,
          "ci95": [
            0.7063379493775939,
            0.9207332298779876
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.8999999999999999
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 12,
          "false_alarm": {
            "value": 0.5,
            "k": 6,
            "n": 12,
            "ci95": [
              0.2537815976337061,
              0.7462184023662939
            ]
          }
        },
        "F0b": {
          "n": 12,
          "false_alarm": {
            "value": 0.5,
            "k": 6,
            "n": 12,
            "ci95": [
              0.2537815976337061,
              0.7462184023662939
            ]
          }
        },
        "F1": {
          "n": 18,
          "hit1_strict": {
            "value": 0.5555555555555556,
            "k": 10,
            "n": 18,
            "ci95": [
              0.33716415642042774,
              0.7544048187299437
            ]
          },
          "hit3_strict": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "hit3_loose": {
            "value": 0.8333333333333334,
            "k": 15,
            "n": 18,
            "ci95": [
              0.6077796189608428,
              0.9416342319413856
            ]
          },
          "miss_rate": {
            "value": 0.05555555555555555,
            "k": 1,
            "n": 18,
            "ci95": [
              0.009875190978303097,
              0.25757300781872594
            ]
          }
        },
        "F2": {
          "n": 15,
          "hit1_strict": {
            "value": 0.6,
            "k": 9,
            "n": 15,
            "ci95": [
              0.357468301206246,
              0.8017550385865841
            ]
          },
          "hit3_strict": {
            "value": 0.7333333333333333,
            "k": 11,
            "n": 15,
            "ci95": [
              0.4804956594401943,
              0.8910254667430761
            ]
          },
          "hit3_loose": {
            "value": 0.7333333333333333,
            "k": 11,
            "n": 15,
            "ci95": [
              0.4804956594401943,
              0.8910254667430761
            ]
          },
          "miss_rate": {
            "value": 0.13333333333333333,
            "k": 2,
            "n": 15,
            "ci95": [
              0.03736125910306537,
              0.37881982832322364
            ]
          }
        },
        "F3": {
          "n": 15,
          "hit1_strict": {
            "value": 0.3333333333333333,
            "k": 5,
            "n": 15,
            "ci95": [
              0.1517632443971473,
              0.5828645226148023
            ]
          },
          "hit3_strict": {
            "value": 0.6,
            "k": 9,
            "n": 15,
            "ci95": [
              0.357468301206246,
              0.8017550385865841
            ]
          },
          "hit3_loose": {
            "value": 0.6,
            "k": 9,
            "n": 15,
            "ci95": [
              0.357468301206246,
              0.8017550385865841
            ]
          },
          "miss_rate": {
            "value": 0.06666666666666667,
            "k": 1,
            "n": 15,
            "ci95": [
              0.011866895493268553,
              0.2981652987378003
            ]
          }
        },
        "F4": {
          "n": 15,
          "hit1_strict": {
            "value": 0.5333333333333333,
            "k": 8,
            "n": 15,
            "ci95": [
              0.30116980025498397,
              0.7519046463426261
            ]
          },
          "hit3_strict": {
            "value": 0.7333333333333333,
            "k": 11,
            "n": 15,
            "ci95": [
              0.4804956594401943,
              0.8910254667430761
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 5,
            "n": 15,
            "ci95": [
              0.1517632443971473,
              0.5828645226148023
            ]
          }
        },
        "F5": {
          "n": 15,
          "hit1_strict": {
            "value": 0.7333333333333333,
            "k": 11,
            "n": 15,
            "ci95": [
              0.4804956594401943,
              0.8910254667430761
            ]
          },
          "hit3_strict": {
            "value": 0.9333333333333333,
            "k": 14,
            "n": 15,
            "ci95": [
              0.7018347012621998,
              0.9881331045067315
            ]
          },
          "hit3_loose": {
            "value": 0.9333333333333333,
            "k": 14,
            "n": 15,
            "ci95": [
              0.7018347012621998,
              0.9881331045067315
            ]
          },
          "miss_rate": {
            "value": 0.06666666666666667,
            "k": 1,
            "n": 15,
            "ci95": [
              0.011866895493268553,
              0.2981652987378003
            ]
          }
        },
        "F6": {
          "n": 18,
          "hit1_strict": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 18,
            "ci95": [
              0.0,
              0.17587922364665765
            ]
          },
          "hit3_loose": {
            "value": 0.05555555555555555,
            "k": 1,
            "n": 18,
            "ci95": [
              0.009875190978303097,
              0.25757300781872594
            ]
          },
          "miss_rate": {
            "value": 0.2777777777777778,
            "k": 5,
            "n": 18,
            "ci95": [
              0.12499753379553993,
              0.5087265656029746
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 32,
          "hit1_strict": {
            "value": 0.6875,
            "k": 22,
            "n": 32,
            "ci95": [
              0.5143332169904459,
              0.8204745836639026
            ]
          },
          "hit3_strict": {
            "value": 0.75,
            "k": 24,
            "n": 32,
            "ci95": [
              0.5789344100575552,
              0.8674759908149096
            ]
          },
          "hit3_loose": {
            "value": 0.75,
            "k": 24,
            "n": 32,
            "ci95": [
              0.5789344100575552,
              0.8674759908149096
            ]
          },
          "miss_rate": {
            "value": 0.0625,
            "k": 2,
            "n": 32,
            "ci95": [
              0.017310567031640967,
              0.2014712314415458
            ]
          },
          "both_hit3_strict": {
            "value": 0.25,
            "k": 1,
            "n": 4,
            "ci95": [
              0.04558726080970055,
              0.6993581574175981
            ]
          }
        },
        "medium": {
          "n": 32,
          "hit1_strict": {
            "value": 0.53125,
            "k": 17,
            "n": 32,
            "ci95": [
              0.3644951711897464,
              0.6913061289193116
            ]
          },
          "hit3_strict": {
            "value": 0.78125,
            "k": 25,
            "n": 32,
            "ci95": [
              0.6124500635075647,
              0.8897616374739581
            ]
          },
          "hit3_loose": {
            "value": 0.78125,
            "k": 25,
            "n": 32,
            "ci95": [
              0.6124500635075647,
              0.8897616374739581
            ]
          },
          "miss_rate": {
            "value": 0.03125,
            "k": 1,
            "n": 32,
            "ci95": [
              0.005537860164003122,
              0.15744263820012558
            ]
          },
          "both_hit3_strict": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          }
        },
        "small": {
          "n": 32,
          "hit1_strict": {
            "value": 0.125,
            "k": 4,
            "n": 32,
            "ci95": [
              0.04970134387480073,
              0.2806830548165022
            ]
          },
          "hit3_strict": {
            "value": 0.34375,
            "k": 11,
            "n": 32,
            "ci95": [
              0.20410437924433172,
              0.5168891202103778
            ]
          },
          "hit3_loose": {
            "value": 0.40625,
            "k": 13,
            "n": 32,
            "ci95": [
              0.2551963484266544,
              0.5773997512461713
            ]
          },
          "miss_rate": {
            "value": 0.21875,
            "k": 7,
            "n": 32,
            "ci95": [
              0.11023836252604186,
              0.3875499364924353
            ]
          },
          "both_hit3_strict": {
            "value": 0.16666666666666666,
            "k": 1,
            "n": 6,
            "ci95": [
              0.030053369748306635,
              0.5635028221864702
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 60,
          "hit1_strict": {
            "value": 0.3541666666666667,
            "k": 17,
            "n": 48,
            "ci95": [
              0.23433644125604858,
              0.4956094298965691
            ]
          },
          "hit3_strict": {
            "value": 0.5833333333333334,
            "k": 28,
            "n": 48,
            "ci95": [
              0.4428133228397628,
              0.7115033222158844
            ]
          },
          "hit3_loose": {
            "value": 0.6041666666666666,
            "k": 29,
            "n": 48,
            "ci95": [
              0.4631040849254725,
              0.7297917213940862
            ]
          },
          "miss_rate": {
            "value": 0.125,
            "k": 6,
            "n": 48,
            "ci95": [
              0.058570514385719794,
              0.24700458286386834
            ]
          },
          "both_hit3_strict": {
            "value": 0.25,
            "k": 2,
            "n": 8,
            "ci95": [
              0.071479212752109,
              0.5907245696898311
            ]
          },
          "false_alarm": {
            "value": 0.3333333333333333,
            "k": 4,
            "n": 12,
            "ci95": [
              0.13812009109121315,
              0.6093779111272004
            ]
          }
        },
        "low": {
          "n": 60,
          "hit1_strict": {
            "value": 0.5416666666666666,
            "k": 26,
            "n": 48,
            "ci95": [
              0.40291074545649797,
              0.6742475770713254
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 32,
            "n": 48,
            "ci95": [
              0.5254010970594798,
              0.7832321930518142
            ]
          },
          "hit3_loose": {
            "value": 0.6875,
            "k": 33,
            "n": 48,
            "ci95": [
              0.5466692858410841,
              0.8005431655341219
            ]
          },
          "miss_rate": {
            "value": 0.08333333333333333,
            "k": 4,
            "n": 48,
            "ci95": [
              0.032883756818091775,
              0.1955330179036728
            ]
          },
          "both_hit3_strict": {
            "value": 0.42857142857142855,
            "k": 3,
            "n": 7,
            "ci95": [
              0.15821985525146975,
              0.7495416354723428
            ]
          },
          "false_alarm": {
            "value": 0.6666666666666666,
            "k": 8,
            "n": 12,
            "ci95": [
              0.39062208887279953,
              0.8618799089087867
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 86,
          "hit1_strict": {
            "value": 0.5342465753424658,
            "k": 39,
            "n": 73,
            "ci95": [
              0.42098945719045805,
              0.6440795833605193
            ]
          },
          "hit3_strict": {
            "value": 0.726027397260274,
            "k": 53,
            "n": 73,
            "ci95": [
              0.6143701175830993,
              0.8150855500533509
            ]
          },
          "hit3_loose": {
            "value": 0.726027397260274,
            "k": 53,
            "n": 73,
            "ci95": [
              0.6143701175830993,
              0.8150855500533509
            ]
          },
          "miss_rate": {
            "value": 0.0684931506849315,
            "k": 5,
            "n": 73,
            "ci95": [
              0.029609038651838046,
              0.15052105040584768
            ]
          },
          "both_hit3_strict": {
            "value": 0.4166666666666667,
            "k": 5,
            "n": 12,
            "ci95": [
              0.19326031365875665,
              0.6804886874504503
            ]
          },
          "false_alarm": {
            "value": 0.46153846153846156,
            "k": 6,
            "n": 13,
            "ci95": [
              0.23206067809538303,
              0.7085620428549398
            ]
          }
        },
        "warning": {
          "n": 34,
          "hit1_strict": {
            "value": 0.17391304347826086,
            "k": 4,
            "n": 23,
            "ci95": [
              0.06978653563062565,
              0.3713764764022614
            ]
          },
          "hit3_strict": {
            "value": 0.30434782608695654,
            "k": 7,
            "n": 23,
            "ci95": [
              0.15604024453214022,
              0.508657562687592
            ]
          },
          "hit3_loose": {
            "value": 0.391304347826087,
            "k": 9,
            "n": 23,
            "ci95": [
              0.22157622792477952,
              0.592144776086183
            ]
          },
          "miss_rate": {
            "value": 0.21739130434782608,
            "k": 5,
            "n": 23,
            "ci95": [
              0.09663978026586204,
              0.4190348301626401
            ]
          },
          "both_hit3_strict": {
            "value": 0.0,
            "k": 0,
            "n": 3,
            "ci95": [
              5.551115123125783e-17,
              0.5614970317550454
            ]
          },
          "false_alarm": {
            "value": 0.5454545454545454,
            "k": 6,
            "n": 11,
            "ci95": [
              0.280091537409322,
              0.7872872837754024
            ]
          }
        }
      }
    },
    "agent_v2": {
      "n_scenarios": 120,
      "n_cause": 96,
      "n_no_cause": 24,
      "hit1_strict": {
        "value": 0.5208333333333334,
        "k": 50,
        "n": 96,
        "ci95": [
          0.4220376770680893,
          0.6180258401045079
        ]
      },
      "hit3_strict": {
        "value": 0.5625,
        "k": 54,
        "n": 96,
        "ci95": [
          0.46275882627554077,
          0.6574317252422498
        ]
      },
      "hit1_loose": {
        "value": 0.5625,
        "k": 54,
        "n": 96,
        "ci95": [
          0.46275882627554077,
          0.6574317252422498
        ]
      },
      "hit3_loose": {
        "value": 0.6145833333333334,
        "k": 59,
        "n": 96,
        "ci95": [
          0.5146070303586874,
          0.7057423140905956
        ]
      },
      "mrr_strict": {
        "value": 0.5381944444444444,
        "n": 96
      },
      "mrr_loose": {
        "value": 0.5833333333333334,
        "n": 96
      },
      "false_alarm": {
        "value": 0.4166666666666667,
        "k": 10,
        "n": 24,
        "ci95": [
          0.24467602609811764,
          0.6116533413477124
        ]
      },
      "miss_rate": {
        "value": 0.22916666666666666,
        "k": 22,
        "n": 96,
        "ci95": [
          0.1564891714560027,
          0.3226851053002383
        ]
      },
      "onset_accuracy": {
        "value": 0.8666666666666667,
        "k": 26,
        "n": 30,
        "ci95": [
          0.703186733179637,
          0.9469034451594526
        ]
      },
      "f0b_explained": {
        "value": 0.5,
        "k": 6,
        "n": 12,
        "ci95": [
          0.2537815976337061,
          0.7462184023662939
        ]
      },
      "invalid_rate": {
        "value": 0.0,
        "k": 0,
        "n": 120,
        "ci95": [
          0.0,
          0.031019166418703486
        ]
      },
      "f4_both_hit3_strict": {
        "value": 0.4666666666666667,
        "k": 7,
        "n": 15,
        "ci95": [
          0.24809535365737387,
          0.698830199745016
        ]
      },
      "numcheck_bad_ratio": {
        "value": 0.041666666666666664,
        "k": 5,
        "n": 120,
        "ci95": [
          0.017926716880109515,
          0.09384085233703535
        ]
      },
      "seconds_per_scenario": {
        "mean": 19.051,
        "max": 83.28,
        "n": 120
      },
      "id_check": {
        "normalized_scenarios": {
          "value": 0.0,
          "k": 0,
          "n": 120,
          "ci95": [
            0.0,
            0.031019166418703486
          ]
        },
        "normalized_ids_total": 0,
        "retry": {
          "value": 0.0,
          "k": 0,
          "n": 120,
          "ci95": [
            0.0,
            0.031019166418703486
          ]
        },
        "unresolved_after_retry": {
          "value": 0.0,
          "k": 0,
          "n": 120,
          "ci95": [
            0.0,
            0.031019166418703486
          ]
        }
      },
      "check_effect": {
        "requested": {
          "value": 1.0,
          "k": 120,
          "n": 120,
          "ci95": [
            0.9689808335812966,
            1.0
          ]
        },
        "top_changed_given_requested": {
          "value": 0.075,
          "k": 9,
          "n": 120,
          "ci95": [
            0.03995711539596028,
            0.1364091760599377
          ]
        },
        "changed_hit3_strict_prelim": {
          "value": 0.2222222222222222,
          "k": 2,
          "n": 9,
          "ci95": [
            0.06322510711784671,
            0.5474110308930111
          ]
        },
        "changed_hit3_strict_final": {
          "value": 0.2222222222222222,
          "k": 2,
          "n": 9,
          "ci95": [
            0.06322510711784671,
            0.5474110308930111
          ]
        },
        "changed_false_alarm_prelim": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        },
        "changed_false_alarm_final": {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ]
        }
      },
      "requests_per_scenario": {
        "mean": 2.0166666666666666
      },
      "input_tokens_per_scenario": {
        "mean": 9444.366666666667
      },
      "output_tokens_per_scenario": {
        "mean": 1618.0916666666667
      },
      "calibration": [
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.0,
            0.2
          ],
          "mean_conf": null
        },
        {
          "value": null,
          "k": 0,
          "n": 0,
          "ci95": [
            null,
            null
          ],
          "bin": [
            0.2,
            0.4
          ],
          "mean_conf": null
        },
        {
          "value": 0.0,
          "k": 0,
          "n": 1,
          "ci95": [
            0.0,
            0.7934506856227626
          ],
          "bin": [
            0.4,
            0.6
          ],
          "mean_conf": 0.5
        },
        {
          "value": 0.3333333333333333,
          "k": 9,
          "n": 27,
          "ci95": [
            0.18643261498715644,
            0.5217523949267991
          ],
          "bin": [
            0.6,
            0.8
          ],
          "mean_conf": 0.7103703703703703
        },
        {
          "value": 0.8913043478260869,
          "k": 41,
          "n": 46,
          "ci95": [
            0.7696217135574916,
            0.9526685414190106
          ],
          "bin": [
            0.8,
            1.0
          ],
          "mean_conf": 0.856086956521739
        }
      ],
      "by_fault": {
        "F0a": {
          "n": 12,
          "false_alarm": {
            "value": 0.3333333333333333,
            "k": 4,
            "n": 12,
            "ci95": [
              0.13812009109121315,
              0.6093779111272004
            ]
          }
        },
        "F0b": {
          "n": 12,
          "false_alarm": {
            "value": 0.5,
            "k": 6,
            "n": 12,
            "ci95": [
              0.2537815976337061,
              0.7462184023662939
            ]
          }
        },
        "F1": {
          "n": 18,
          "hit1_strict": {
            "value": 0.8888888888888888,
            "k": 16,
            "n": 18,
            "ci95": [
              0.672002348698212,
              0.9689804773543876
            ]
          },
          "hit3_strict": {
            "value": 0.8888888888888888,
            "k": 16,
            "n": 18,
            "ci95": [
              0.672002348698212,
              0.9689804773543876
            ]
          },
          "hit3_loose": {
            "value": 0.8888888888888888,
            "k": 16,
            "n": 18,
            "ci95": [
              0.672002348698212,
              0.9689804773543876
            ]
          },
          "miss_rate": {
            "value": 0.1111111111111111,
            "k": 2,
            "n": 18,
            "ci95": [
              0.03101952264561239,
              0.32799765130178804
            ]
          }
        },
        "F2": {
          "n": 15,
          "hit1_strict": {
            "value": 0.6,
            "k": 9,
            "n": 15,
            "ci95": [
              0.357468301206246,
              0.8017550385865841
            ]
          },
          "hit3_strict": {
            "value": 0.6666666666666666,
            "k": 10,
            "n": 15,
            "ci95": [
              0.41713547738519774,
              0.8482367556028526
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "miss_rate": {
            "value": 0.13333333333333333,
            "k": 2,
            "n": 15,
            "ci95": [
              0.03736125910306537,
              0.37881982832322364
            ]
          }
        },
        "F3": {
          "n": 15,
          "hit1_strict": {
            "value": 0.6666666666666666,
            "k": 10,
            "n": 15,
            "ci95": [
              0.41713547738519774,
              0.8482367556028526
            ]
          },
          "hit3_strict": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "hit3_loose": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "miss_rate": {
            "value": 0.06666666666666667,
            "k": 1,
            "n": 15,
            "ci95": [
              0.011866895493268553,
              0.2981652987378003
            ]
          }
        },
        "F4": {
          "n": 15,
          "hit1_strict": {
            "value": 0.7333333333333333,
            "k": 11,
            "n": 15,
            "ci95": [
              0.4804956594401943,
              0.8910254667430761
            ]
          },
          "hit3_strict": {
            "value": 0.8,
            "k": 12,
            "n": 15,
            "ci95": [
              0.5481455128483065,
              0.9295245065301844
            ]
          },
          "hit3_loose": {
            "value": 0.8666666666666667,
            "k": 13,
            "n": 15,
            "ci95": [
              0.6211801716767764,
              0.9626387408969346
            ]
          },
          "miss_rate": {
            "value": 0.0,
            "k": 0,
            "n": 15,
            "ci95": [
              0.0,
              0.20388330103584862
            ]
          },
          "both_hit3_strict": {
            "value": 0.4666666666666667,
            "k": 7,
            "n": 15,
            "ci95": [
              0.24809535365737387,
              0.698830199745016
            ]
          }
        },
        "F5": {
          "n": 15,
          "hit1_strict": {
            "value": 0.06666666666666667,
            "k": 1,
            "n": 15,
            "ci95": [
              0.011866895493268553,
              0.2981652987378003
            ]
          },
          "hit3_strict": {
            "value": 0.06666666666666667,
            "k": 1,
            "n": 15,
            "ci95": [
              0.011866895493268553,
              0.2981652987378003
            ]
          },
          "hit3_loose": {
            "value": 0.13333333333333333,
            "k": 2,
            "n": 15,
            "ci95": [
              0.03736125910306537,
              0.37881982832322364
            ]
          },
          "miss_rate": {
            "value": 0.4666666666666667,
            "k": 7,
            "n": 15,
            "ci95": [
              0.24809535365737387,
              0.698830199745016
            ]
          }
        },
        "F6": {
          "n": 18,
          "hit1_strict": {
            "value": 0.16666666666666666,
            "k": 3,
            "n": 18,
            "ci95": [
              0.05836576805861446,
              0.39222038103915724
            ]
          },
          "hit3_strict": {
            "value": 0.16666666666666666,
            "k": 3,
            "n": 18,
            "ci95": [
              0.05836576805861446,
              0.39222038103915724
            ]
          },
          "hit3_loose": {
            "value": 0.2222222222222222,
            "k": 4,
            "n": 18,
            "ci95": [
              0.09000928108601694,
              0.45214584316212625
            ]
          },
          "miss_rate": {
            "value": 0.5555555555555556,
            "k": 10,
            "n": 18,
            "ci95": [
              0.33716415642042774,
              0.7544048187299437
            ]
          }
        }
      },
      "by_effect": {
        "large": {
          "n": 32,
          "hit1_strict": {
            "value": 0.625,
            "k": 20,
            "n": 32,
            "ci95": [
              0.4525440735307745,
              0.7706611269054578
            ]
          },
          "hit3_strict": {
            "value": 0.625,
            "k": 20,
            "n": 32,
            "ci95": [
              0.4525440735307745,
              0.7706611269054578
            ]
          },
          "hit3_loose": {
            "value": 0.65625,
            "k": 21,
            "n": 32,
            "ci95": [
              0.4831108797896222,
              0.7958956207556682
            ]
          },
          "miss_rate": {
            "value": 0.1875,
            "k": 6,
            "n": 32,
            "ci95": [
              0.08889544689565954,
              0.3530915520137596
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 2,
            "n": 4,
            "ci95": [
              0.15003898915214947,
              0.8499610108478506
            ]
          }
        },
        "medium": {
          "n": 32,
          "hit1_strict": {
            "value": 0.625,
            "k": 20,
            "n": 32,
            "ci95": [
              0.4525440735307745,
              0.7706611269054578
            ]
          },
          "hit3_strict": {
            "value": 0.65625,
            "k": 21,
            "n": 32,
            "ci95": [
              0.4831108797896222,
              0.7958956207556682
            ]
          },
          "hit3_loose": {
            "value": 0.71875,
            "k": 23,
            "n": 32,
            "ci95": [
              0.5462549057408342,
              0.8443541950225723
            ]
          },
          "miss_rate": {
            "value": 0.1875,
            "k": 6,
            "n": 32,
            "ci95": [
              0.08889544689565954,
              0.3530915520137596
            ]
          },
          "both_hit3_strict": {
            "value": 0.6,
            "k": 3,
            "n": 5,
            "ci95": [
              0.2307242812760128,
              0.882379225767352
            ]
          }
        },
        "small": {
          "n": 32,
          "hit1_strict": {
            "value": 0.3125,
            "k": 10,
            "n": 32,
            "ci95": [
              0.17952541633609734,
              0.4856667830095541
            ]
          },
          "hit3_strict": {
            "value": 0.40625,
            "k": 13,
            "n": 32,
            "ci95": [
              0.2551963484266544,
              0.5773997512461713
            ]
          },
          "hit3_loose": {
            "value": 0.46875,
            "k": 15,
            "n": 32,
            "ci95": [
              0.30869387108068835,
              0.6355048288102536
            ]
          },
          "miss_rate": {
            "value": 0.3125,
            "k": 10,
            "n": 32,
            "ci95": [
              0.17952541633609734,
              0.4856667830095541
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 2,
            "n": 6,
            "ci95": [
              0.09677141110578041,
              0.700006684861608
            ]
          }
        }
      },
      "by_stickiness": {
        "high": {
          "n": 60,
          "hit1_strict": {
            "value": 0.4166666666666667,
            "k": 20,
            "n": 48,
            "ci95": [
              0.2884966777841157,
              0.5571866771602373
            ]
          },
          "hit3_strict": {
            "value": 0.4791666666666667,
            "k": 23,
            "n": 48,
            "ci95": [
              0.34471316176951616,
              0.616707676966572
            ]
          },
          "hit3_loose": {
            "value": 0.5625,
            "k": 27,
            "n": 48,
            "ci95": [
              0.42275017419827776,
              0.6929873095934576
            ]
          },
          "miss_rate": {
            "value": 0.25,
            "k": 12,
            "n": 48,
            "ci95": [
              0.1492063367379791,
              0.3878437280950796
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 4,
            "n": 8,
            "ci95": [
              0.21521606221387757,
              0.7847839377861224
            ]
          },
          "false_alarm": {
            "value": 0.25,
            "k": 3,
            "n": 12,
            "ci95": [
              0.08894166839405471,
              0.5323053349335657
            ]
          }
        },
        "low": {
          "n": 60,
          "hit1_strict": {
            "value": 0.625,
            "k": 30,
            "n": 48,
            "ci95": [
              0.4836276351921527,
              0.747847332391318
            ]
          },
          "hit3_strict": {
            "value": 0.6458333333333334,
            "k": 31,
            "n": 48,
            "ci95": [
              0.504390570103431,
              0.7656635587439515
            ]
          },
          "hit3_loose": {
            "value": 0.6666666666666666,
            "k": 32,
            "n": 48,
            "ci95": [
              0.5254010970594798,
              0.7832321930518142
            ]
          },
          "miss_rate": {
            "value": 0.20833333333333334,
            "k": 10,
            "n": 48,
            "ci95": [
              0.11730272957905595,
              0.34258901272617925
            ]
          },
          "both_hit3_strict": {
            "value": 0.42857142857142855,
            "k": 3,
            "n": 7,
            "ci95": [
              0.15821985525146975,
              0.7495416354723428
            ]
          },
          "false_alarm": {
            "value": 0.5833333333333334,
            "k": 7,
            "n": 12,
            "ci95": [
              0.31951131254954973,
              0.8067396863412435
            ]
          }
        }
      },
      "by_alert_level": {
        "alarm": {
          "n": 86,
          "hit1_strict": {
            "value": 0.547945205479452,
            "k": 40,
            "n": 73,
            "ci95": [
              0.43424321509808306,
              0.6568534416732852
            ]
          },
          "hit3_strict": {
            "value": 0.589041095890411,
            "k": 43,
            "n": 73,
            "ci95": [
              0.4744922324418476,
              0.6946872729906935
            ]
          },
          "hit3_loose": {
            "value": 0.6438356164383562,
            "k": 47,
            "n": 73,
            "ci95": [
              0.5293349749798597,
              0.743954995334245
            ]
          },
          "miss_rate": {
            "value": 0.2054794520547945,
            "k": 15,
            "n": 73,
            "ci95": [
              0.1286697211101815,
              0.31173653015141345
            ]
          },
          "both_hit3_strict": {
            "value": 0.5,
            "k": 6,
            "n": 12,
            "ci95": [
              0.2537815976337061,
              0.7462184023662939
            ]
          },
          "false_alarm": {
            "value": 0.46153846153846156,
            "k": 6,
            "n": 13,
            "ci95": [
              0.23206067809538303,
              0.7085620428549398
            ]
          }
        },
        "warning": {
          "n": 34,
          "hit1_strict": {
            "value": 0.43478260869565216,
            "k": 10,
            "n": 23,
            "ci95": [
              0.25634635325674215,
              0.6318862491498354
            ]
          },
          "hit3_strict": {
            "value": 0.4782608695652174,
            "k": 11,
            "n": 23,
            "ci95": [
              0.29237157416646653,
              0.6703726266357259
            ]
          },
          "hit3_loose": {
            "value": 0.5217391304347826,
            "k": 12,
            "n": 23,
            "ci95": [
              0.32962737336427406,
              0.7076284258335335
            ]
          },
          "miss_rate": {
            "value": 0.30434782608695654,
            "k": 7,
            "n": 23,
            "ci95": [
              0.15604024453214022,
              0.508657562687592
            ]
          },
          "both_hit3_strict": {
            "value": 0.3333333333333333,
            "k": 1,
            "n": 3,
            "ci95": [
              0.06149194472039621,
              0.7923403991979522
            ]
          },
          "false_alarm": {
            "value": 0.36363636363636365,
            "k": 4,
            "n": 11,
            "ci95": [
              0.15166471095367573,
              0.6461988254921511
            ]
          }
        }
      },
      "model": "gemini-3.5-flash-lite",
      "prompt_hash": "c2ed508330ddae861696c5012944c609f5ba8351987c9b3791f10535f394be5b",
      "usage": {
        "logged_requests": 243,
        "cost_usd_total": 0.0
      },
      "repeatability": {
        "value": 0.5,
        "k": 5,
        "n": 10,
        "ci95": [
          0.236593090512564,
          0.7634069094874361
        ],
        "runs_per_scenario": 3,
        "run_dirs": [
          "runs/2026-09-29-gemini-3.5-flash-lite-v2-c2ed5083",
          "runs/2026-09-29-gemini-3.5-flash-lite-v2-c2ed5083-rep1",
          "runs/2026-09-29-gemini-3.5-flash-lite-v2-c2ed5083-rep2"
        ],
        "scenarios": [
          "scn_7033",
          "scn_7034",
          "scn_7044",
          "scn_7067",
          "scn_7081",
          "scn_7084",
          "scn_7099",
          "scn_7104",
          "scn_7110",
          "scn_7119"
        ],
        "missing": []
      }
    },
    "chance": {
      "hit3_strict": 0.02619270833333333,
      "hit3_loose": 0.08589583333333334,
      "n_cause": 96,
      "n_draws": 2000
    },
    "compare": {
      "baseline_vs_baseline_v2": {
        "hit3_strict_mcnemar": {
          "n": 96,
          "both": 30,
          "only_a": 9,
          "only_b": 30,
          "neither": 27,
          "p": 0.0010650196345523
        }
      },
      "baseline_vs_agent_v2": {
        "hit3_strict_mcnemar": {
          "n": 96,
          "both": 33,
          "only_a": 6,
          "only_b": 21,
          "neither": 36,
          "p": 0.005924612283706665
        }
      },
      "baseline_v2_vs_agent_v2": {
        "hit3_strict_mcnemar": {
          "n": 96,
          "both": 44,
          "only_a": 16,
          "only_b": 10,
          "neither": 26,
          "p": 0.32693958282470703
        }
      },
      "mcnemar_p": 0.32693958282470703,
      "mcnemar_pair": "baseline_v2_vs_agent_v2"
    },
    "info": {
      "n_scenarios": 120,
      "fault_mix": {
        "F0a": 12,
        "F0b": 12,
        "F1": 18,
        "F2": 15,
        "F3": 15,
        "F4": 15,
        "F5": 15,
        "F6": 18
      },
      "regen_total": 4,
      "regen_by_fault": {
        "F0a": 1,
        "F0b": 0,
        "F1": 0,
        "F2": 0,
        "F3": 1,
        "F4": 0,
        "F5": 0,
        "F6": 2
      },
      "tau": 0.5,
      "methods": {
        "baseline": "runs/baseline/test2",
        "baseline_v2": "runs/baseline_v2/test2",
        "agent_v2": "runs/2026-09-29-gemini-3.5-flash-lite-v2-c2ed5083"
      },
      "evaluated_at": "2026-09-29T22:20:04"
    }
  }
}