{ "live_api_run": false, "kev": [ { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-decision-v2/report.json", "source_sha256": "5a93ae03370fc845c21f7ce8e2a31cb75928f16af05c20ef2724afae0e2fecba", "matching_panels": [ "kev/decision-v2/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.7521542021806406, "paired_flip": { "pairs": 80, "flip_rate": 0.7875, "both_correct_rate": 0.7625 }, "clean": { "n": 1200, "nll": 0.7995445079897864, "acc": 0.8325, "ece": 0.07350328282828288, "brier": 0.2567267699384416, "mean_conf": 0.9039366161616161, "score_mae": 0.44738425925925923, "ranked_probability_score": 0.0803466461436758 }, "tasks": { "agnews": { "n": 80, "nll": 1.7331030754235985, "acc": 0.8125, "ece": 0.15062499999999995, "brier": 0.3441975, "mean_conf": 0.961875 }, "agnews_yn": { "n": 160, "nll": 0.3788220635008283, "acc": 0.875, "ece": 0.06643750000000001, "brier": 0.20035374999999997, "mean_conf": 0.9375625000000001 }, "amazon": { "n": 80, "nll": 2.5691063403334633, "acc": 0.5875, "ece": 0.21087500000000003, "brier": 0.587905, "mean_conf": 0.785625, "score_mae": 0.519, "ranked_probability_score": 0.094865 }, "banking77": { "n": 80, "nll": 1.6822267105484368, "acc": 0.825, "ece": 0.15072853535353525, "brier": 0.3261299352106928, "mean_conf": 0.9161527777777778 }, "boolq": { "n": 80, "nll": 0.24712603638358172, "acc": 0.925, "ece": 0.02824999999999981, "brier": 0.130945, "mean_conf": 0.9102499999999999 }, "contrastive_age_eligibility": { "n": 40, "nll": 0.025338928916240833, "acc": 1.0, "ece": 0.025000000000000022, "brier": 0.001330000000000002, "mean_conf": 0.975 }, "contrastive_quantity_limit": { "n": 40, "nll": 0.07737442505460074, "acc": 0.95, "ece": 0.049499999999999794, "brier": 0.05293, "mean_conf": 0.9719999999999999 }, "contrastive_return_window": { "n": 40, "nll": 0.7375300031177211, "acc": 0.575, "ece": 0.2710000000000001, "brier": 0.53567, "mean_conf": 0.8460000000000001 }, "contrastive_spend_threshold": { "n": 40, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0 }, "dbpedia14": { "n": 80, "nll": 0.36530005091960704, "acc": 0.95, "ece": 0.042999999999999844, "brier": 0.0808575, "mean_conf": 0.993 }, "imdb": { "n": 80, "nll": 0.22989680795572376, "acc": 0.9, "ece": 0.06987500000000015, "brier": 0.1327125, "mean_conf": 0.9036250000000001 }, "mnli": { "n": 80, "nll": 0.2919030355928468, "acc": 0.9, "ece": 0.044999999999999915, "brier": 0.15163999999999997, "mean_conf": 0.90625 }, "sst5": { "n": 80, "nll": 1.6730733736097438, "acc": 0.6375, "ece": 0.20684595959595958, "brier": 0.5775200403020099, "mean_conf": 0.7791540404040405, "score_mae": 0.4627727272727274, "ranked_probability_score": 0.08198321096061627 }, "trec": { "n": 80, "nll": 0.4660517145744967, "acc": 0.9125, "ece": 0.08024999999999996, "brier": 0.15719249999999999, "mean_conf": 0.9517499999999999 }, "yelp": { "n": 80, "nll": 1.2926501066559826, "acc": 0.6625, "ece": 0.18924242424242432, "brier": 0.5082790735639221, "mean_conf": 0.8337424242424243, "score_mae": 0.3603800505050505, "ranked_probability_score": 0.06419172747041117 }, "yelp_yn": { "n": 80, "nll": 0.26496456230337745, "acc": 0.8625, "ece": 0.08800000000000009, "brier": 0.15785, "mean_conf": 0.8460000000000001 } }, "variants": { "clean": { "n": 1200, "nll": 0.7995445079897864, "acc": 0.8325, "ece": 0.07350328282828288, "brier": 0.2567267699384416, "mean_conf": 0.9039366161616161, "score_mae": 0.44738425925925923, "ranked_probability_score": 0.0803466461436758 }, "none_absent": { "n": 72, "nll": 3.478320925911749, "acc": 0.5833333333333334, "ece": 0.30679573512906844, "brier": 0.6323930009976306, "mean_conf": 0.8643799102132436 }, "none_present": { "n": 72, "nll": 0.8478299733779013, "acc": 0.8611111111111112, "ece": 0.09243406285072964, "brier": 0.22526617011869535, "mean_conf": 0.9275659371492705 }, "permuted": { "n": 96, "nll": 0.7063334791794388, "acc": 0.8854166666666666, "ece": 0.08420244107744115, "brier": 0.19897070919634052, "mean_conf": 0.9403430134680134 } }, "heldout_tasks": {}, "permutation": { "n": 72, "mean_max_delta": 0.020611672278338947, "flip_rate": 0.0 }, "temperature": 1.0, "calibrated_clean": { "n": 1200, "nll": 0.7995445079897864, "acc": 0.8325, "ece": 0.07350328282828288, "brier": 0.2567267699384416, "mean_conf": 0.9039366161616161, "score_mae": 0.44738425925925923, "ranked_probability_score": 0.0803466461436758 }, "metric_policy": { "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 14, "returned_zeros": 11925 }, "coverage": { "requested_records": 1176, "requested_questions": 1440, "evaluated_records": 1176, "evaluated_questions": 1440, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 227.57854149999184, "p95": 390.3251252499758 }, "suite_sha256": "e0388e1d13284f5e0a2e6cb33a0fcd34f0099bfc31a6a852bd5cb619f505094d", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-18T22:03:09.733279+00:00", "calls": 1176, "input_tokens": 614328, "output_tokens": 149353, "retries_after_5xx": 0, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.025801776000000002, "budget_usd": 0.15, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-decision-v4/report.json", "source_sha256": "5c0dca8b6f6688221dc0c032029d035d1b2da219ad6d8d43ce121e863332d902", "matching_panels": [ "kev/round3/decision-r3/development", "kev/v3/decision-v3/development", "kev/v4/decision-v4/development", "kev/v5/decision-v5/development", "kev/v6/decision-v6/development", "kev/v7/decision-v7/development", "kev/v8/decision-v8/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.5273286484199085, "paired_flip": { "pairs": 80, "flip_rate": 0.875, "both_correct_rate": 0.8625, "invariant_pairs": 32, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.96875 }, "clean": { "n": 1264, "nll": 0.776262175940276, "acc": 0.8449367088607594, "ece": 0.06702299897711289, "brier": 0.23666320044195954, "mean_conf": 0.906390087584708, "confident_error_rate": 0.05221518987341772, "coverage_at_0_9": 0.7199367088607594, "accuracy_at_0_9": 0.9274725274725275, "selective": { "0.5": { "coverage": 0.502373417721519, "accuracy": 0.9559055118110236, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8022151898734177, "accuracy": 0.9072978303747534, "confidence_cutoff": 0.82 } }, "score_mae": 0.44806313131313136, "ranked_probability_score": 0.08048006914685575 }, "tasks": { "agnews": { "n": 80, "nll": 1.9503932261768973, "acc": 0.8125, "ece": 0.17762500000000003, "brier": 0.34458, "mean_conf": 0.962875, "confident_error_rate": 0.1375, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 0.8472222222222222, "selective": { "0.5": { "coverage": 0.7, "accuracy": 0.9285714285714286, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.825, "accuracy": 0.8636363636363636, "confidence_cutoff": 0.97 } } }, "agnews_yn": { "n": 160, "nll": 0.3785896700151023, "acc": 0.875, "ece": 0.0721874999999999, "brier": 0.20114875, "mean_conf": 0.9381875000000001, "confident_error_rate": 0.05625, "coverage_at_0_9": 0.83125, "accuracy_at_0_9": 0.9323308270676691, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.926829268292683, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.80625, "accuracy": 0.9302325581395349, "confidence_cutoff": 0.92 } } }, "amazon": { "n": 80, "nll": 2.5691642333981486, "acc": 0.5875, "ece": 0.1998194444444444, "brier": 0.5950064378634832, "mean_conf": 0.7873194444444445, "confident_error_rate": 0.0625, "coverage_at_0_9": 0.325, "accuracy_at_0_9": 0.8076923076923077, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.725, "confidence_cutoff": 0.8099999999999999 }, "0.8": { "coverage": 0.8, "accuracy": 0.65625, "confidence_cutoff": 0.64 } }, "score_mae": 0.5199393939393939, "ranked_probability_score": 0.0955658324405673 }, "banking77": { "n": 80, "nll": 1.6784429500178981, "acc": 0.8125, "ece": 0.14988005050505063, "brier": 0.3220046291194776, "mean_conf": 0.9160189393939394, "confident_error_rate": 0.0875, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.8833333333333333, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.926829268292683, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8461538461538461, "confidence_cutoff": 0.86 } } }, "boolq": { "n": 80, "nll": 0.25166038685395276, "acc": 0.925, "ece": 0.029999999999999895, "brier": 0.13174, "mean_conf": 0.9097500000000001, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.775, "accuracy_at_0_9": 0.9516129032258065, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.9512195121951219, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.88 } } }, "composition_atom": { "n": 16, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "composition_conditional": { "n": 16, "nll": 0.16030577836417179, "acc": 0.9375, "ece": 0.029999999999999943, "brier": 0.10744999999999999, "mean_conf": 0.9075, "confident_error_rate": 0.0, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.75, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9230769230769231, "confidence_cutoff": 0.68 } } }, "composition_conjunction": { "n": 16, "nll": 0.03803291621185537, "acc": 1.0, "ece": 0.034375, "brier": 0.0127625, "mean_conf": 0.965625, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "composition_disjunction": { "n": 16, "nll": 0.04853304934368727, "acc": 0.9375, "ece": 0.03375, "brier": 0.03645, "mean_conf": 0.97125, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9375, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.9375, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9375, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "composition_exception": { "n": 16, "nll": 0.0019037004677942859, "acc": 1.0, "ece": 0.001875000000000071, "brier": 0.0001125000000000001, "mean_conf": 0.9981249999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.9375, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9375, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "composition_negation": { "n": 16, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "composition_nested_and": { "n": 16, "nll": 0.16756474955118095, "acc": 0.8125, "ece": 0.12000000000000002, "brier": 0.12012500000000002, "mean_conf": 0.89375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.75, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.69 } } }, "composition_nested_or": { "n": 16, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "contrastive_age_eligibility": { "n": 24, "nll": 0.02619452298740957, "acc": 1.0, "ece": 0.025833333333333375, "brier": 0.0014166666666666687, "mean_conf": 0.9741666666666666, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.9166666666666666, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "contrastive_quantity_limit": { "n": 24, "nll": 0.07287683119390397, "acc": 0.9583333333333334, "ece": 0.05958333333333326, "brier": 0.036125000000000004, "mean_conf": 0.9429166666666666, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.5416666666666666, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.875, "accuracy": 1.0, "confidence_cutoff": 0.91 } } }, "contrastive_return_window": { "n": 24, "nll": 0.5244467845123877, "acc": 0.7083333333333334, "ece": 0.14208333333333337, "brier": 0.37067500000000003, "mean_conf": 0.7795833333333334, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3333333333333333, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.79 }, "0.8": { "coverage": 0.8333333333333334, "accuracy": 0.75, "confidence_cutoff": 0.61 } } }, "contrastive_spend_threshold": { "n": 24, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "dbpedia14": { "n": 80, "nll": 0.565332335467563, "acc": 0.95, "ece": 0.04350000000000006, "brier": 0.08205499999999999, "mean_conf": 0.9935, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.9875, "accuracy_at_0_9": 0.9620253164556962, "selective": { "0.5": { "coverage": 0.925, "accuracy": 0.972972972972973, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.925, "accuracy": 0.972972972972973, "confidence_cutoff": 1.0 } } }, "imdb": { "n": 80, "nll": 0.23944802918618352, "acc": 0.9, "ece": 0.04487500000000016, "brier": 0.13362749999999998, "mean_conf": 0.904875, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 0.9818181818181818, "selective": { "0.5": { "coverage": 0.525, "accuracy": 0.9761904761904762, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9846153846153847, "confidence_cutoff": 0.85 } } }, "mnli": { "n": 80, "nll": 0.29402967763766225, "acc": 0.9, "ece": 0.03775000000000005, "brier": 0.154885, "mean_conf": 0.90675, "confident_error_rate": 0.0125, "coverage_at_0_9": 0.7125, "accuracy_at_0_9": 0.9824561403508771, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.975, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.9375, "confidence_cutoff": 0.8200000000000001 } } }, "sst5": { "n": 80, "nll": 1.6697207964324985, "acc": 0.6375, "ece": 0.18999999999999997, "brier": 0.574675, "mean_conf": 0.779, "confident_error_rate": 0.125, "coverage_at_0_9": 0.375, "accuracy_at_0_9": 0.6666666666666666, "selective": { "0.5": { "coverage": 0.5125, "accuracy": 0.6585365853658537, "confidence_cutoff": 0.79 }, "0.8": { "coverage": 0.8, "accuracy": 0.65625, "confidence_cutoff": 0.62 } }, "score_mae": 0.462, "ranked_probability_score": 0.081750625 }, "trec": { "n": 80, "nll": 0.4644183961477254, "acc": 0.9125, "ece": 0.07974999999999999, "brier": 0.15528999999999998, "mean_conf": 0.9512500000000002, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.85, "accuracy_at_0_9": 0.9558823529411765, "selective": { "0.5": { "coverage": 0.65, "accuracy": 0.9807692307692307, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.96 } } }, "yelp": { "n": 80, "nll": 1.2903825646596685, "acc": 0.6625, "ece": 0.20087499999999994, "brier": 0.507875, "mean_conf": 0.8311249999999999, "confident_error_rate": 0.1625, "coverage_at_0_9": 0.475, "accuracy_at_0_9": 0.6578947368421053, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.88 }, "0.8": { "coverage": 0.8125, "accuracy": 0.7076923076923077, "confidence_cutoff": 0.7 } }, "score_mae": 0.36225, "ranked_probability_score": 0.06412375 }, "yelp_yn": { "n": 80, "nll": 0.2644469634521126, "acc": 0.8625, "ece": 0.11387500000000006, "brier": 0.1573975, "mean_conf": 0.845875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4625, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.525, "accuracy": 1.0, "confidence_cutoff": 0.88 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.76 } } } }, "variants": { "clean": { "n": 1264, "nll": 0.776262175940276, "acc": 0.8449367088607594, "ece": 0.06702299897711289, "brier": 0.23666320044195954, "mean_conf": 0.906390087584708, "confident_error_rate": 0.05221518987341772, "coverage_at_0_9": 0.7199367088607594, "accuracy_at_0_9": 0.9274725274725275, "selective": { "0.5": { "coverage": 0.502373417721519, "accuracy": 0.9559055118110236, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8022151898734177, "accuracy": 0.9072978303747534, "confidence_cutoff": 0.82 } }, "score_mae": 0.44806313131313136, "ranked_probability_score": 0.08048006914685575 }, "none_absent": { "n": 60, "nll": 4.181770827408908, "acc": 0.5, "ece": 0.3450875420875421, "brier": 0.7544118991259394, "mean_conf": 0.8358720538720539, "confident_error_rate": 0.18333333333333332, "coverage_at_0_9": 0.5666666666666667, "accuracy_at_0_9": 0.6764705882352942, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.7, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8, "accuracy": 0.5416666666666666, "confidence_cutoff": 0.68 } } }, "none_present": { "n": 60, "nll": 1.0412005035072407, "acc": 0.8333333333333334, "ece": 0.11585858585858586, "brier": 0.2807036394245485, "mean_conf": 0.9173670033670034, "confident_error_rate": 0.06666666666666667, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.9111111111111111, "selective": { "0.5": { "coverage": 0.5166666666666667, "accuracy": 0.967741935483871, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.8958333333333334, "confidence_cutoff": 0.85 } } }, "permuted": { "n": 84, "nll": 0.8146112348164126, "acc": 0.8690476190476191, "ece": 0.09744468494468486, "brier": 0.23026722054600843, "mean_conf": 0.9342219817219818, "confident_error_rate": 0.05952380952380952, "coverage_at_0_9": 0.8095238095238095, "accuracy_at_0_9": 0.9264705882352942, "selective": { "0.5": { "coverage": 0.5833333333333334, "accuracy": 0.9591836734693877, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8095238095238095, "accuracy": 0.9264705882352942, "confidence_cutoff": 0.92 } } } }, "heldout_tasks": {}, "permutation": { "n": 60, "mean_max_delta": 0.0243939393939394, "flip_rate": 0.0 }, "temperature": 1.0, "calibrated_clean": { "n": 1264, "nll": 0.776262175940276, "acc": 0.8449367088607594, "ece": 0.06702299897711289, "brier": 0.23666320044195954, "mean_conf": 0.906390087584708, "confident_error_rate": 0.05221518987341772, "coverage_at_0_9": 0.7199367088607594, "accuracy_at_0_9": 0.9274725274725275, "selective": { "0.5": { "coverage": 0.502373417721519, "accuracy": 0.9559055118110236, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8022151898734177, "accuracy": 0.9072978303747534, "confidence_cutoff": 0.82 } }, "score_mae": 0.44806313131313136, "ranked_probability_score": 0.08048006914685575 }, "metric_policy": { "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 12, "returned_zeros": 11897 }, "coverage": { "requested_records": 1204, "requested_questions": 1468, "evaluated_records": 1204, "evaluated_questions": 1468, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 213.23335400002543, "p95": 331.53796620000793 }, "suite_sha256": "1b33e566d114f9eafeff55b36c221fadb2a4ae358a1b9cc68006e82c7cfad8f1", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-18T23:45:42.182658+00:00", "calls": 1208, "input_tokens": 627273, "output_tokens": 149446, "retries_after_5xx": 4, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.026345466, "budget_usd": 0.15, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-devtools-v1/report.json", "source_sha256": "f71d032a2fda780aa5da0ae533eb41486f4872931623cb5abc6ce98f96e7d98a", "matching_panels": [ "kev/devtools-v1/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -1.4418255401899853, "paired_flip": null, "unknowable": null, "clean": { "n": 1074, "nll": 1.4702158103280576, "acc": 0.7132216014897579, "ece": 0.11682457724357165, "brier": 0.40123536573340196, "mean_conf": 0.8298227150461788, "confident_error_rate": 0.04655493482309125, "coverage_at_0_9": 0.4776536312849162, "accuracy_at_0_9": 0.9025341130604289, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.14701861963912996, "error_rate_at_0_9": 0.09746588693957114, "confidence_bias": 0.11660111355642089, "top_bins": { "0.9": { "n": 513, "errors": 50, "error_rate": 0.09746588693957114 }, "0.95": { "n": 366, "errors": 32, "error_rate": 0.08743169398907104 }, "0.99": { "n": 175, "errors": 15, "error_rate": 0.08571428571428572 } }, "selective": { "0.5": { "coverage": 0.5204841713221602, "accuracy": 0.8783542039355993, "confidence_cutoff": 0.88 }, "0.8": { "coverage": 0.8035381750465549, "accuracy": 0.7717265353418308, "confidence_cutoff": 0.6599999999999999 } } }, "tasks": { "aegis_category": { "n": 25, "nll": 1.2083425050275216, "acc": 0.84, "ece": 0.1464, "brier": 0.261568, "mean_conf": 0.9, "confident_error_rate": 0.04, "coverage_at_0_9": 0.76, "accuracy_at_0_9": 0.9473684210526315, "coverage_at_5pct_error": 0.84, "coverage_at_1pct_error": 0.0, "aurc": 0.07822198433773343, "error_rate_at_0_9": 0.05263157894736842, "confidence_bias": 0.06000000000000005, "top_bins": { "0.9": { "n": 19, "errors": 1, "error_rate": 0.05263157894736842 }, "0.95": { "n": 19, "errors": 1, "error_rate": 0.05263157894736842 }, "0.99": { "n": 16, "errors": 1, "error_rate": 0.0625 } }, "selective": { "0.5": { "coverage": 0.52, "accuracy": 0.9230769230769231, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 0.95, "confidence_cutoff": 0.8 } } }, "aegis_unsafe": { "n": 150, "nll": 0.45525512792510503, "acc": 0.7933333333333333, "ece": 0.039666666666666676, "brier": 0.29660933333333334, "mean_conf": 0.8330000000000001, "confident_error_rate": 0.04, "coverage_at_0_9": 0.49333333333333335, "accuracy_at_0_9": 0.918918918918919, "coverage_at_5pct_error": 0.30666666666666664, "coverage_at_1pct_error": 0.12, "aurc": 0.09275786713657497, "error_rate_at_0_9": 0.08108108108108109, "confidence_bias": 0.03966666666666674, "top_bins": { "0.9": { "n": 74, "errors": 6, "error_rate": 0.08108108108108109 }, "0.95": { "n": 46, "errors": 2, "error_rate": 0.043478260869565216 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.52, "accuracy": 0.9102564102564102, "confidence_cutoff": 0.89 }, "0.8": { "coverage": 0.8066666666666666, "accuracy": 0.8429752066115702, "confidence_cutoff": 0.67 } } }, "codereviewer_needs_comment": { "n": 150, "nll": 0.6928185349282461, "acc": 0.5533333333333333, "ece": 0.07940000000000004, "brier": 0.4985373333333335, "mean_conf": 0.6327333333333333, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.37937209682553286, "error_rate_at_0_9": null, "confidence_bias": 0.07939999999999992, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.52, "accuracy": 0.5769230769230769, "confidence_cutoff": 0.62 }, "0.8": { "coverage": 0.8533333333333334, "accuracy": 0.546875, "confidence_cutoff": 0.54 } } }, "commitpackft_message": { "n": 149, "nll": 0.13605480720812885, "acc": 0.959731543624161, "ece": 0.052214765100671134, "brier": 0.07045906040268457, "mean_conf": 0.9120805369127515, "confident_error_rate": 0.006711409395973154, "coverage_at_0_9": 0.738255033557047, "accuracy_at_0_9": 0.990909090909091, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9328859060402684, "aurc": 0.005156281441008902, "error_rate_at_0_9": 0.00909090909090909, "confidence_bias": -0.04765100671140954, "top_bins": { "0.9": { "n": 110, "errors": 1, "error_rate": 0.00909090909090909 }, "0.95": { "n": 87, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 64, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5167785234899329, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8053691275167785, "accuracy": 0.9916666666666667, "confidence_cutoff": 0.83 } } }, "commitpackft_type": { "n": 150, "nll": 6.171885759784734, "acc": 0.49333333333333335, "ece": 0.3631838383838384, "brier": 0.803722295956195, "mean_conf": 0.8565171717171717, "confident_error_rate": 0.14666666666666667, "coverage_at_0_9": 0.5333333333333333, "accuracy_at_0_9": 0.725, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.2894049966332535, "error_rate_at_0_9": 0.275, "confidence_bias": 0.36318383838383833, "top_bins": { "0.9": { "n": 80, "errors": 22, "error_rate": 0.275 }, "0.95": { "n": 68, "errors": 17, "error_rate": 0.25 }, "0.99": { "n": 49, "errors": 10, "error_rate": 0.20408163265306123 } }, "selective": { "0.5": { "coverage": 0.5066666666666667, "accuracy": 0.75, "confidence_cutoff": 0.92 }, "0.8": { "coverage": 0.8, "accuracy": 0.5833333333333334, "confidence_cutoff": 0.7 } } }, "flakeflagger_flaky": { "n": 150, "nll": 0.8441304625847224, "acc": 0.52, "ece": 0.2733333333333333, "brier": 0.6147173333333333, "mean_conf": 0.7933333333333333, "confident_error_rate": 0.02, "coverage_at_0_9": 0.2, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.14, "coverage_at_1pct_error": 0.11333333333333333, "aurc": 0.3025499979665429, "error_rate_at_0_9": 0.1, "confidence_bias": 0.2733333333333333, "top_bins": { "0.9": { "n": 30, "errors": 3, "error_rate": 0.1 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6933333333333334, "confidence_cutoff": 0.84 }, "0.8": { "coverage": 0.8133333333333334, "accuracy": 0.5245901639344263, "confidence_cutoff": 0.6799999999999999 } } }, "prompt_injection": { "n": 150, "nll": 0.22913395939466394, "acc": 0.8933333333333333, "ece": 0.05619999999999977, "brier": 0.13640666666666668, "mean_conf": 0.9000666666666666, "confident_error_rate": 0.013333333333333334, "coverage_at_0_9": 0.7333333333333333, "accuracy_at_0_9": 0.9818181818181818, "coverage_at_5pct_error": 0.8733333333333333, "coverage_at_1pct_error": 0.4266666666666667, "aurc": 0.019982150639180827, "error_rate_at_0_9": 0.01818181818181818, "confidence_bias": 0.006733333333333258, "top_bins": { "0.9": { "n": 110, "errors": 2, "error_rate": 0.01818181818181818 }, "0.95": { "n": 76, "errors": 2, "error_rate": 0.02631578947368421 }, "0.99": { "n": 5, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5066666666666667, "accuracy": 0.9736842105263158, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.82, "accuracy": 0.975609756097561, "confidence_cutoff": 0.84 } } }, "when2call_action": { "n": 150, "nll": 1.7969831646667616, "acc": 0.76, "ece": 0.14767138047138045, "brier": 0.40926825602829636, "mean_conf": 0.8698801346801346, "confident_error_rate": 0.1, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.8333333333333334, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.15745818237409737, "error_rate_at_0_9": 0.16666666666666666, "confidence_bias": 0.10988013468013458, "top_bins": { "0.9": { "n": 90, "errors": 15, "error_rate": 0.16666666666666666 }, "0.95": { "n": 67, "errors": 10, "error_rate": 0.14925373134328357 }, "0.99": { "n": 39, "errors": 4, "error_rate": 0.10256410256410256 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.84, "confidence_cutoff": 0.93 }, "0.8": { "coverage": 0.8, "accuracy": 0.8, "confidence_cutoff": 0.74 } } } }, "variants": { "clean": { "n": 1074, "nll": 1.4702158103280576, "acc": 0.7132216014897579, "ece": 0.11682457724357165, "brier": 0.40123536573340196, "mean_conf": 0.8298227150461788, "confident_error_rate": 0.04655493482309125, "coverage_at_0_9": 0.4776536312849162, "accuracy_at_0_9": 0.9025341130604289, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.14701861963912996, "error_rate_at_0_9": 0.09746588693957114, "confidence_bias": 0.11660111355642089, "top_bins": { "0.9": { "n": 513, "errors": 50, "error_rate": 0.09746588693957114 }, "0.95": { "n": 366, "errors": 32, "error_rate": 0.08743169398907104 }, "0.99": { "n": 175, "errors": 15, "error_rate": 0.08571428571428572 } }, "selective": { "0.5": { "coverage": 0.5204841713221602, "accuracy": 0.8783542039355993, "confidence_cutoff": 0.88 }, "0.8": { "coverage": 0.8035381750465549, "accuracy": 0.7717265353418308, "confidence_cutoff": 0.6599999999999999 } } } }, "heldout_tasks": {}, "permutation": { "n": 0, "mean_max_delta": null, "flip_rate": null }, "temperature": 1.0, "calibrated_clean": { "n": 1074, "nll": 1.4702158103280576, "acc": 0.7132216014897579, "ece": 0.11682457724357165, "brier": 0.40123536573340196, "mean_conf": 0.8298227150461788, "confident_error_rate": 0.04655493482309125, "coverage_at_0_9": 0.4776536312849162, "accuracy_at_0_9": 0.9025341130604289, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.14701861963912996, "error_rate_at_0_9": 0.09746588693957114, "confidence_bias": 0.11660111355642089, "top_bins": { "0.9": { "n": 513, "errors": 50, "error_rate": 0.09746588693957114 }, "0.95": { "n": 366, "errors": 32, "error_rate": 0.08743169398907104 }, "0.99": { "n": 175, "errors": 15, "error_rate": 0.08571428571428572 } }, "selective": { "0.5": { "coverage": 0.5204841713221602, "accuracy": 0.8783542039355993, "confidence_cutoff": 0.88 }, "0.8": { "coverage": 0.8035381750465549, "accuracy": 0.7717265353418308, "confidence_cutoff": 0.6599999999999999 } } }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 5, "returned_zeros": 1200 }, "coverage": { "requested_records": 900, "requested_questions": 1074, "evaluated_records": 900, "evaluated_questions": 1074, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 208.90775049998774, "p95": 314.60627855000115 }, "calibration": { "inference_temperature": null, "additional_temperature": 1.0, "logits_recorded": false }, "suite_sha256": "d0f7cf40cec3284684f8b2fe8f440a5f0cc40fe4a78db108eca47a5fbd141b0f", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-24T03:00:11.054851+00:00", "calls": 905, "input_tokens": 530239, "output_tokens": 35883, "retries_after_5xx": 5, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.022270038, "budget_usd": 1.0, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-ekzhang-mmlupro-v1/report.json", "source_sha256": "94603aa3df0222057ed3c5f1d72125abbd90d80a0b6d3b3b0cbb7c19c1b3a5b2", "matching_panels": [], "match_basis": "unmatched original manifest", "report": { "objective": -0.9413852086025182, "paired_flip": null, "unknowable": null, "clean": { "n": 1000, "nll": 0.8731740991527358, "acc": 0.835, "ece": 0.06941818181818175, "brier": 0.2676688794408734, "mean_conf": 0.8086234343434344, "confident_error_rate": 0.033, "coverage_at_0_9": 0.551, "accuracy_at_0_9": 0.9401088929219601, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.0, "aurc": 0.061566848551724764, "error_rate_at_0_9": 0.05989110707803993, "confidence_bias": -0.026376565656565543, "top_bins": { "0.9": { "n": 551, "errors": 33, "error_rate": 0.05989110707803993 }, "0.95": { "n": 460, "errors": 20, "error_rate": 0.043478260869565216 }, "0.99": { "n": 317, "errors": 10, "error_rate": 0.031545741324921134 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9299999999999999 }, "0.8": { "coverage": 0.801, "accuracy": 0.9013732833957553, "confidence_cutoff": 0.57 } } }, "tasks": { "mmlu_pro_biology": { "n": 48, "nll": 0.08764541781071533, "acc": 0.9791666666666666, "ece": 0.05208333333333349, "brier": 0.038504166666666666, "mean_conf": 0.9379166666666667, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9166666666666666, "aurc": 0.0017931517028128962, "error_rate_at_0_9": 0.0, "confidence_bias": -0.0412499999999999, "top_bins": { "0.9": { "n": 39, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 37, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 32, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.625, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.9 } } }, "mmlu_pro_business": { "n": 58, "nll": 1.5736689553131389, "acc": 0.7068965517241379, "ece": 0.15144374782305817, "brier": 0.4572683167445969, "mean_conf": 0.6468146987112504, "confident_error_rate": 0.017241379310344827, "coverage_at_0_9": 0.25862068965517243, "accuracy_at_0_9": 0.9333333333333333, "coverage_at_5pct_error": 0.20689655172413793, "coverage_at_1pct_error": 0.20689655172413793, "aurc": 0.14248358300144723, "error_rate_at_0_9": 0.06666666666666667, "confidence_bias": -0.06008185301288749, "top_bins": { "0.9": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.95": { "n": 12, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8275862068965517, "confidence_cutoff": 0.6900000000000001 }, "0.8": { "coverage": 0.8275862068965517, "accuracy": 0.7708333333333334, "confidence_cutoff": 0.34 } } }, "mmlu_pro_chemistry": { "n": 122, "nll": 0.5756809745867606, "acc": 0.8524590163934426, "ece": 0.14184053651266773, "brier": 0.26062108567041503, "mean_conf": 0.7106184798807749, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3442622950819672, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.680327868852459, "coverage_at_1pct_error": 0.5983606557377049, "aurc": 0.03244904805445884, "error_rate_at_0_9": 0.0, "confidence_bias": -0.1418405365126677, "top_bins": { "0.9": { "n": 42, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 36, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 28, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5081967213114754, "accuracy": 1.0, "confidence_cutoff": 0.76 }, "0.8": { "coverage": 0.8032786885245902, "accuracy": 0.9081632653061225, "confidence_cutoff": 0.41 } } }, "mmlu_pro_computer_science": { "n": 32, "nll": 0.22196345797389505, "acc": 0.96875, "ece": 0.12689393939393945, "brier": 0.11052192250790735, "mean_conf": 0.8768560606060606, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.75, "aurc": 0.008829281802594163, "error_rate_at_0_9": 0.0, "confidence_bias": -0.09189393939393942, "top_bins": { "0.9": { "n": 22, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 11, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.78 } } }, "mmlu_pro_economics": { "n": 71, "nll": 0.6446383183966767, "acc": 0.8873239436619719, "ece": 0.08514866979655714, "brier": 0.19976299888916194, "mean_conf": 0.9071205007824725, "confident_error_rate": 0.04225352112676056, "coverage_at_0_9": 0.7746478873239436, "accuracy_at_0_9": 0.9454545454545454, "coverage_at_5pct_error": 0.704225352112676, "coverage_at_1pct_error": 0.0, "aurc": 0.05193418753702269, "error_rate_at_0_9": 0.05454545454545454, "confidence_bias": 0.019796557120500657, "top_bins": { "0.9": { "n": 55, "errors": 3, "error_rate": 0.05454545454545454 }, "0.95": { "n": 46, "errors": 2, "error_rate": 0.043478260869565216 }, "0.99": { "n": 36, "errors": 2, "error_rate": 0.05555555555555555 } }, "selective": { "0.5": { "coverage": 0.5070422535211268, "accuracy": 0.9444444444444444, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8169014084507042, "accuracy": 0.9482758620689655, "confidence_cutoff": 0.87 } } }, "mmlu_pro_engineering": { "n": 81, "nll": 0.6570521375630167, "acc": 0.8148148148148148, "ece": 0.1438882653697468, "brier": 0.2932966583152891, "mean_conf": 0.6751240803092655, "confident_error_rate": 0.0, "coverage_at_0_9": 0.3333333333333333, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.7777777777777778, "coverage_at_1pct_error": 0.41975308641975306, "aurc": 0.03791200652209681, "error_rate_at_0_9": 0.0, "confidence_bias": -0.13969073450554925, "top_bins": { "0.9": { "n": 27, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 17, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 12, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5185185185185185, "accuracy": 0.9761904761904762, "confidence_cutoff": 0.77 }, "0.8": { "coverage": 0.8024691358024691, "accuracy": 0.9384615384615385, "confidence_cutoff": 0.39 } } }, "mmlu_pro_health": { "n": 55, "nll": 0.7261748079839057, "acc": 0.8545454545454545, "ece": 0.07532782369146002, "brier": 0.2171882164157646, "mean_conf": 0.9298732782369145, "confident_error_rate": 0.03636363636363636, "coverage_at_0_9": 0.8, "accuracy_at_0_9": 0.9545454545454546, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.3090909090909091, "aurc": 0.049349504764199185, "error_rate_at_0_9": 0.045454545454545456, "confidence_bias": 0.07532782369145996, "top_bins": { "0.9": { "n": 44, "errors": 2, "error_rate": 0.045454545454545456 }, "0.95": { "n": 35, "errors": 2, "error_rate": 0.05714285714285714 }, "0.99": { "n": 27, "errors": 2, "error_rate": 0.07407407407407407 } }, "selective": { "0.5": { "coverage": 0.5636363636363636, "accuracy": 0.9354838709677419, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.9545454545454546, "confidence_cutoff": 0.9 } } }, "mmlu_pro_history": { "n": 35, "nll": 3.4054716171483275, "acc": 0.7428571428571429, "ece": 0.21971428571428558, "brier": 0.4875714285714285, "mean_conf": 0.918, "confident_error_rate": 0.2, "coverage_at_0_9": 0.7714285714285715, "accuracy_at_0_9": 0.7407407407407407, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.13852301660874738, "error_rate_at_0_9": 0.25925925925925924, "confidence_bias": 0.17514285714285716, "top_bins": { "0.9": { "n": 27, "errors": 7, "error_rate": 0.25925925925925924 }, "0.95": { "n": 24, "errors": 4, "error_rate": 0.16666666666666666 }, "0.99": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.9047619047619048, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.75, "confidence_cutoff": 0.86 } } }, "mmlu_pro_law": { "n": 102, "nll": 1.4947036078270326, "acc": 0.7549019607843137, "ece": 0.10289364230540704, "brier": 0.4031500765228038, "mean_conf": 0.807207367795603, "confident_error_rate": 0.0784313725490196, "coverage_at_0_9": 0.45098039215686275, "accuracy_at_0_9": 0.8260869565217391, "coverage_at_5pct_error": 0.20588235294117646, "coverage_at_1pct_error": 0.06862745098039216, "aurc": 0.1435163624788319, "error_rate_at_0_9": 0.17391304347826086, "confidence_bias": 0.05230540701128927, "top_bins": { "0.9": { "n": 46, "errors": 8, "error_rate": 0.17391304347826086 }, "0.95": { "n": 33, "errors": 4, "error_rate": 0.12121212121212122 }, "0.99": { "n": 16, "errors": 1, "error_rate": 0.0625 } }, "selective": { "0.5": { "coverage": 0.5294117647058824, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.88 }, "0.8": { "coverage": 0.803921568627451, "accuracy": 0.7926829268292683, "confidence_cutoff": 0.6 } } }, "mmlu_pro_math": { "n": 103, "nll": 0.6397050095029277, "acc": 0.8737864077669902, "ece": 0.09190840443267626, "brier": 0.21497415797674693, "mean_conf": 0.8168196528390704, "confident_error_rate": 0.02912621359223301, "coverage_at_0_9": 0.5631067961165048, "accuracy_at_0_9": 0.9482758620689655, "coverage_at_5pct_error": 0.5922330097087378, "coverage_at_1pct_error": 0.22330097087378642, "aurc": 0.048288376688715585, "error_rate_at_0_9": 0.05172413793103448, "confidence_bias": -0.056966754927919805, "top_bins": { "0.9": { "n": 58, "errors": 3, "error_rate": 0.05172413793103448 }, "0.95": { "n": 51, "errors": 3, "error_rate": 0.058823529411764705 }, "0.99": { "n": 30, "errors": 1, "error_rate": 0.03333333333333333 } }, "selective": { "0.5": { "coverage": 0.5048543689320388, "accuracy": 0.9423076923076923, "confidence_cutoff": 0.9494949494949495 }, "0.8": { "coverage": 0.8058252427184466, "accuracy": 0.927710843373494, "confidence_cutoff": 0.65 } } }, "mmlu_pro_other": { "n": 72, "nll": 0.768506759480066, "acc": 0.8194444444444444, "ece": 0.0948891694725027, "brier": 0.2605306819598907, "mean_conf": 0.8420552749719417, "confident_error_rate": 0.041666666666666664, "coverage_at_0_9": 0.5972222222222222, "accuracy_at_0_9": 0.9302325581395349, "coverage_at_5pct_error": 0.3888888888888889, "coverage_at_1pct_error": 0.3194444444444444, "aurc": 0.056246145630593505, "error_rate_at_0_9": 0.06976744186046512, "confidence_bias": 0.02261083052749724, "top_bins": { "0.9": { "n": 43, "errors": 3, "error_rate": 0.06976744186046512 }, "0.95": { "n": 33, "errors": 2, "error_rate": 0.06060606060606061 }, "0.99": { "n": 23, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5138888888888888, "accuracy": 0.9459459459459459, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.9310344827586207, "confidence_cutoff": 0.63 } } }, "mmlu_pro_philosophy": { "n": 47, "nll": 1.2468041591257746, "acc": 0.851063829787234, "ece": 0.09828497743391351, "brier": 0.2629850523285726, "mean_conf": 0.8719019987105094, "confident_error_rate": 0.06382978723404255, "coverage_at_0_9": 0.6808510638297872, "accuracy_at_0_9": 0.90625, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.10120779658670538, "error_rate_at_0_9": 0.09375, "confidence_bias": 0.02083816892327539, "top_bins": { "0.9": { "n": 32, "errors": 3, "error_rate": 0.09375 }, "0.95": { "n": 28, "errors": 2, "error_rate": 0.07142857142857142 }, "0.99": { "n": 16, "errors": 2, "error_rate": 0.125 } }, "selective": { "0.5": { "coverage": 0.5319148936170213, "accuracy": 0.92, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8085106382978723, "accuracy": 0.8947368421052632, "confidence_cutoff": 0.67 } } }, "mmlu_pro_physics": { "n": 113, "nll": 0.7802892239826213, "acc": 0.8053097345132744, "ece": 0.09205327612407262, "brier": 0.29168083498794145, "mean_conf": 0.7548493787431841, "confident_error_rate": 0.02654867256637168, "coverage_at_0_9": 0.4778761061946903, "accuracy_at_0_9": 0.9444444444444444, "coverage_at_5pct_error": 0.5486725663716814, "coverage_at_1pct_error": 0.0, "aurc": 0.0715493045214525, "error_rate_at_0_9": 0.05555555555555555, "confidence_bias": -0.050460355770090226, "top_bins": { "0.9": { "n": 54, "errors": 3, "error_rate": 0.05555555555555555 }, "0.95": { "n": 44, "errors": 1, "error_rate": 0.022727272727272728 }, "0.99": { "n": 28, "errors": 1, "error_rate": 0.03571428571428571 } }, "selective": { "0.5": { "coverage": 0.504424778761062, "accuracy": 0.9473684210526315, "confidence_cutoff": 0.86 }, "0.8": { "coverage": 0.8053097345132744, "accuracy": 0.9010989010989011, "confidence_cutoff": 0.49 } } }, "mmlu_pro_psychology": { "n": 61, "nll": 0.3570884737403957, "acc": 0.8688524590163934, "ece": 0.06464315283987433, "brier": 0.1585901960489144, "mean_conf": 0.9114224209306175, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7704918032786885, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8360655737704918, "coverage_at_1pct_error": 0.7868852459016393, "aurc": 0.014746275222641884, "error_rate_at_0_9": 0.0, "confidence_bias": 0.04256996191422413, "top_bins": { "0.9": { "n": 47, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 44, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 31, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5081967213114754, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8032786885245902, "accuracy": 0.9795918367346939, "confidence_cutoff": 0.88 } } } }, "variants": { "clean": { "n": 1000, "nll": 0.8731740991527358, "acc": 0.835, "ece": 0.06941818181818175, "brier": 0.2676688794408734, "mean_conf": 0.8086234343434344, "confident_error_rate": 0.033, "coverage_at_0_9": 0.551, "accuracy_at_0_9": 0.9401088929219601, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.0, "aurc": 0.061566848551724764, "error_rate_at_0_9": 0.05989110707803993, "confidence_bias": -0.026376565656565543, "top_bins": { "0.9": { "n": 551, "errors": 33, "error_rate": 0.05989110707803993 }, "0.95": { "n": 460, "errors": 20, "error_rate": 0.043478260869565216 }, "0.99": { "n": 317, "errors": 10, "error_rate": 0.031545741324921134 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9299999999999999 }, "0.8": { "coverage": 0.801, "accuracy": 0.9013732833957553, "confidence_cutoff": 0.57 } } } }, "heldout_tasks": {}, "permutation": { "n": 0, "mean_max_delta": null, "flip_rate": null }, "temperature": 1.0, "calibrated_clean": { "n": 1000, "nll": 0.8731740991527358, "acc": 0.835, "ece": 0.06941818181818175, "brier": 0.2676688794408734, "mean_conf": 0.8086234343434344, "confident_error_rate": 0.033, "coverage_at_0_9": 0.551, "accuracy_at_0_9": 0.9401088929219601, "coverage_at_5pct_error": 0.5, "coverage_at_1pct_error": 0.0, "aurc": 0.061566848551724764, "error_rate_at_0_9": 0.05989110707803993, "confidence_bias": -0.026376565656565543, "top_bins": { "0.9": { "n": 551, "errors": 33, "error_rate": 0.05989110707803993 }, "0.95": { "n": 460, "errors": 20, "error_rate": 0.043478260869565216 }, "0.99": { "n": 317, "errors": 10, "error_rate": 0.031545741324921134 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.9299999999999999 }, "0.8": { "coverage": 0.801, "accuracy": 0.9013732833957553, "confidence_cutoff": 0.57 } } }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 22, "returned_zeros": 5022 }, "coverage": { "requested_records": 1000, "requested_questions": 1000, "evaluated_records": 1000, "evaluated_questions": 1000, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 254.7142110000059, "p95": 351.0079969499989 }, "calibration": { "inference_temperature": null, "additional_temperature": 1.0, "logits_recorded": false }, "suite_sha256": "6c1bc89134d371dd48d4721cb95bb0c4cf7473609ee60995412fc2018babe37b", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-22T16:16:18.307785+00:00", "calls": 1007, "input_tokens": 560475, "output_tokens": 82919, "retries_after_5xx": 7, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.02353995, "budget_usd": 1.0, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-hard-v1-r2/report.json", "source_sha256": "6527fa74f61c7a04c1166f69e4f904d7279d27698bdfdaeb55f8b4ee30f61679", "matching_panels": [ "kev/hard-v1/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.6595768726660133, "paired_flip": null, "unknowable": null, "clean": { "n": 1083, "nll": 0.6576411594522477, "acc": 0.7765466297322253, "ece": 0.03492337968792268, "brier": 0.31538264915964354, "mean_conf": 0.7873066771127714, "confident_error_rate": 0.024007386888273315, "coverage_at_0_9": 0.44967682363804246, "accuracy_at_0_9": 0.946611909650924, "coverage_at_5pct_error": 0.4164358264081256, "coverage_at_1pct_error": 0.09325946445060018, "aurc": 0.0849340976775038, "error_rate_at_0_9": 0.053388090349075976, "confidence_bias": 0.010760047380546034, "top_bins": { "0.9": { "n": 487, "errors": 26, "error_rate": 0.053388090349075976 }, "0.95": { "n": 379, "errors": 14, "error_rate": 0.036939313984168866 }, "0.99": { "n": 182, "errors": 5, "error_rate": 0.027472527472527472 } }, "selective": { "0.5": { "coverage": 0.5032317636195752, "accuracy": 0.9339449541284404, "confidence_cutoff": 0.86 }, "0.8": { "coverage": 0.8051708217913204, "accuracy": 0.8451834862385321, "confidence_cutoff": 0.57 } }, "score_mae": 0.31630014430014425, "ranked_probability_score": 0.03192551978661069 }, "tasks": { "hard_ambiguous": { "n": 100, "nll": 0.5394977619807617, "acc": 0.79, "ece": 0.10693838383838382, "brier": 0.30717337883889395, "mean_conf": 0.7392494949494952, "confident_error_rate": 0.02, "coverage_at_0_9": 0.3, "accuracy_at_0_9": 0.9333333333333333, "coverage_at_5pct_error": 0.47, "coverage_at_1pct_error": 0.26, "aurc": 0.07108975940240438, "error_rate_at_0_9": 0.06666666666666667, "confidence_bias": -0.050750505050504824, "top_bins": { "0.9": { "n": 30, "errors": 2, "error_rate": 0.06666666666666667 }, "0.95": { "n": 21, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 7, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.52, "accuracy": 0.9423076923076923, "confidence_cutoff": 0.75 }, "0.8": { "coverage": 0.8, "accuracy": 0.875, "confidence_cutoff": 0.5555555555555556 } } }, "hard_judge": { "n": 188, "nll": 0.4017902556066641, "acc": 0.7978723404255319, "ece": 0.06202450032237275, "brier": 0.2370368398144349, "mean_conf": 0.7969116698903933, "confident_error_rate": 0.005319148936170213, "coverage_at_0_9": 0.5, "accuracy_at_0_9": 0.9893617021276596, "coverage_at_5pct_error": 0.6968085106382979, "coverage_at_1pct_error": 0.46808510638297873, "aurc": 0.04449982375237592, "error_rate_at_0_9": 0.010638297872340425, "confidence_bias": -0.0009606705351385969, "top_bins": { "0.9": { "n": 94, "errors": 1, "error_rate": 0.010638297872340425 }, "0.95": { "n": 81, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9893617021276596, "confidence_cutoff": 0.9 }, "0.8": { "coverage": 0.8297872340425532, "accuracy": 0.8910256410256411, "confidence_cutoff": 0.55 } } }, "hard_long_policy": { "n": 200, "nll": 1.3876190638263513, "acc": 0.655, "ece": 0.18797474747474746, "brier": 0.5207995238240996, "mean_conf": 0.8398747474747474, "confident_error_rate": 0.1, "coverage_at_0_9": 0.505, "accuracy_at_0_9": 0.801980198019802, "coverage_at_5pct_error": 0.115, "coverage_at_1pct_error": 0.0, "aurc": 0.20041620209364905, "error_rate_at_0_9": 0.19801980198019803, "confidence_bias": 0.18487474747474741, "top_bins": { "0.9": { "n": 101, "errors": 20, "error_rate": 0.19801980198019803 }, "0.95": { "n": 73, "errors": 12, "error_rate": 0.1643835616438356 }, "0.99": { "n": 36, "errors": 5, "error_rate": 0.1388888888888889 } }, "selective": { "0.5": { "coverage": 0.505, "accuracy": 0.801980198019802, "confidence_cutoff": 0.9 }, "0.8": { "coverage": 0.8, "accuracy": 0.73125, "confidence_cutoff": 0.7 } } }, "hard_multi_hop": { "n": 124, "nll": 0.6486265091027997, "acc": 0.7741935483870968, "ece": 0.08839198435972627, "brier": 0.3441881119767239, "mean_conf": 0.7446725317693059, "confident_error_rate": 0.008064516129032258, "coverage_at_0_9": 0.3467741935483871, "accuracy_at_0_9": 0.9767441860465116, "coverage_at_5pct_error": 0.4032258064516129, "coverage_at_1pct_error": 0.18548387096774194, "aurc": 0.10023370177371982, "error_rate_at_0_9": 0.023255813953488372, "confidence_bias": -0.02952101661779083, "top_bins": { "0.9": { "n": 43, "errors": 1, "error_rate": 0.023255813953488372 }, "0.95": { "n": 34, "errors": 1, "error_rate": 0.029411764705882353 }, "0.99": { "n": 23, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9032258064516129, "confidence_cutoff": 0.79 }, "0.8": { "coverage": 0.8145161290322581, "accuracy": 0.8217821782178217, "confidence_cutoff": 0.52 } } }, "hard_probability": { "n": 155, "nll": 0.5324734810699263, "acc": 0.8064516129032258, "ece": 0.05405865102639298, "brier": 0.2883542878771422, "mean_conf": 0.7755770609318997, "confident_error_rate": 0.0064516129032258064, "coverage_at_0_9": 0.3741935483870968, "accuracy_at_0_9": 0.9827586206896551, "coverage_at_5pct_error": 0.5419354838709678, "coverage_at_1pct_error": 0.3096774193548387, "aurc": 0.06753144521497295, "error_rate_at_0_9": 0.017241379310344827, "confidence_bias": -0.030874551971326047, "top_bins": { "0.9": { "n": 58, "errors": 1, "error_rate": 0.017241379310344827 }, "0.95": { "n": 42, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 25, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5096774193548387, "accuracy": 0.9746835443037974, "confidence_cutoff": 0.78 }, "0.8": { "coverage": 0.8, "accuracy": 0.8467741935483871, "confidence_cutoff": 0.6 } }, "score_mae": 0.31630014430014425, "ranked_probability_score": 0.03192551978661069 }, "hard_temporal_numeric": { "n": 116, "nll": 0.9423103028609364, "acc": 0.5948275862068966, "ece": 0.07887931034482759, "brier": 0.5005172413793102, "mean_conf": 0.5590517241379311, "confident_error_rate": 0.0, "coverage_at_0_9": 0.09482758620689655, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.28448275862068967, "coverage_at_1pct_error": 0.1896551724137931, "aurc": 0.1829527082208703, "error_rate_at_0_9": 0.0, "confidence_bias": -0.035775862068965436, "top_bins": { "0.9": { "n": 11, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 4, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5172413793103449, "accuracy": 0.8, "confidence_cutoff": 0.53 }, "0.8": { "coverage": 0.8017241379310345, "accuracy": 0.6666666666666666, "confidence_cutoff": 0.35000000000000003 } } }, "hard_tradeoff": { "n": 200, "nll": 0.16472073421465339, "acc": 0.955, "ece": 0.03735000000000002, "brier": 0.083425, "mean_conf": 0.91765, "confident_error_rate": 0.005, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 0.9933333333333333, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.775, "aurc": 0.01022588783312255, "error_rate_at_0_9": 0.006666666666666667, "confidence_bias": -0.037349999999999994, "top_bins": { "0.9": { "n": 150, "errors": 1, "error_rate": 0.006666666666666667 }, "0.95": { "n": 121, "errors": 1, "error_rate": 0.008264462809917356 }, "0.99": { "n": 53, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.990909090909091, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.805, "accuracy": 0.9875776397515528, "confidence_cutoff": 0.87 } } } }, "variants": { "clean": { "n": 1083, "nll": 0.6576411594522477, "acc": 0.7765466297322253, "ece": 0.03492337968792268, "brier": 0.31538264915964354, "mean_conf": 0.7873066771127714, "confident_error_rate": 0.024007386888273315, "coverage_at_0_9": 0.44967682363804246, "accuracy_at_0_9": 0.946611909650924, "coverage_at_5pct_error": 0.4164358264081256, "coverage_at_1pct_error": 0.09325946445060018, "aurc": 0.0849340976775038, "error_rate_at_0_9": 0.053388090349075976, "confidence_bias": 0.010760047380546034, "top_bins": { "0.9": { "n": 487, "errors": 26, "error_rate": 0.053388090349075976 }, "0.95": { "n": 379, "errors": 14, "error_rate": 0.036939313984168866 }, "0.99": { "n": 182, "errors": 5, "error_rate": 0.027472527472527472 } }, "selective": { "0.5": { "coverage": 0.5032317636195752, "accuracy": 0.9339449541284404, "confidence_cutoff": 0.86 }, "0.8": { "coverage": 0.8051708217913204, "accuracy": 0.8451834862385321, "confidence_cutoff": 0.57 } }, "score_mae": 0.31630014430014425, "ranked_probability_score": 0.03192551978661069 } }, "heldout_tasks": {}, "permutation": { "n": 0, "mean_max_delta": null, "flip_rate": null }, "temperature": 1.0, "calibrated_clean": { "n": 1083, "nll": 0.6576411594522477, "acc": 0.7765466297322253, "ece": 0.03492337968792268, "brier": 0.31538264915964354, "mean_conf": 0.7873066771127714, "confident_error_rate": 0.024007386888273315, "coverage_at_0_9": 0.44967682363804246, "accuracy_at_0_9": 0.946611909650924, "coverage_at_5pct_error": 0.4164358264081256, "coverage_at_1pct_error": 0.09325946445060018, "aurc": 0.0849340976775038, "error_rate_at_0_9": 0.053388090349075976, "confidence_bias": 0.010760047380546034, "top_bins": { "0.9": { "n": 487, "errors": 26, "error_rate": 0.053388090349075976 }, "0.95": { "n": 379, "errors": 14, "error_rate": 0.036939313984168866 }, "0.99": { "n": 182, "errors": 5, "error_rate": 0.027472527472527472 } }, "selective": { "0.5": { "coverage": 0.5032317636195752, "accuracy": 0.9339449541284404, "confidence_cutoff": 0.86 }, "0.8": { "coverage": 0.8051708217913204, "accuracy": 0.8451834862385321, "confidence_cutoff": 0.57 } }, "score_mae": 0.31630014430014425, "ranked_probability_score": 0.03192551978661069 }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 11, "returned_zeros": 1180 }, "coverage": { "requested_records": 700, "requested_questions": 1083, "evaluated_records": 700, "evaluated_questions": 1083, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 212.42327099999966, "p95": 320.1151723000206 }, "calibration": { "inference_temperature": null, "additional_temperature": 1.0, "logits_recorded": false }, "suite_sha256": "9d8dd335f5a73840b144c987ee91bb0513301128c474a7a4cd3f754a130cfa5c", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-24T03:24:07.395180+00:00", "calls": 751, "input_tokens": 658893, "output_tokens": 49531, "retries_after_5xx": 51, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.027673506, "budget_usd": 1.0, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-scienthoon-v1/report.json", "source_sha256": "d91dcc0ba1b4804a65184b44115d923bb23a53af514f459d8e77e7c1d2f41198", "matching_panels": [ "kev/external/scienthoon-v1/development" ], "match_basis": "source panel path and complete published row IDs, ordered options, labels; no request-text hash", "published_rows_sha256": "9b301bf7a577c17ce72fff33d5f7842441ad585dba347bdbaef5f677f90f2b3a", "report": { "objective": -0.9252626912936911, "paired_flip": null, "unknowable": null, "clean": { "n": 873, "nll": 0.925262691293691, "acc": 0.7525773195876289, "ece": 0.10488493179214829, "brier": 0.34766628498179053, "mean_conf": 0.851207955847131, "confident_error_rate": 0.018327605956471937, "coverage_at_0_9": 0.4547537227949599, "accuracy_at_0_9": 0.9596977329974811, "coverage_at_5pct_error": 0.4856815578465063, "coverage_at_1pct_error": 0.041237113402061855, "confidence_bias": 0.09863063625950219, "top_bins": { "0.9": { "n": 397, "errors": 16, "error_rate": 0.04030226700251889 }, "0.95": { "n": 350, "errors": 14, "error_rate": 0.04 }, "0.99": { "n": 241, "errors": 12, "error_rate": 0.04979253112033195 } }, "selective": { "0.5": { "coverage": 0.5005727376861397, "accuracy": 0.9382151029748284, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.8029782359679267, "accuracy": 0.81169757489301, "confidence_cutoff": 0.73 } }, "score_mae": 0.5575566663195529, "ranked_probability_score": 0.13278797517330268 }, "tasks": { "scienthoon_angry": { "n": 291, "nll": 0.27974358366885455, "acc": 0.9140893470790378, "ece": 0.07048109965635742, "brier": 0.15631546391752577, "mean_conf": 0.8436082474226804, "confident_error_rate": 0.006872852233676976, "coverage_at_0_9": 0.422680412371134, "accuracy_at_0_9": 0.983739837398374, "coverage_at_5pct_error": 0.7972508591065293, "coverage_at_1pct_error": 0.08247422680412371, "confidence_bias": -0.0704810996563574, "top_bins": { "0.9": { "n": 123, "errors": 2, "error_rate": 0.016260162601626018 }, "0.95": { "n": 92, "errors": 2, "error_rate": 0.021739130434782608 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5120274914089347, "accuracy": 0.9731543624161074, "confidence_cutoff": 0.86 }, "0.8": { "coverage": 0.8006872852233677, "accuracy": 0.9484978540772532, "confidence_cutoff": 0.75 } } }, "scienthoon_priority": { "n": 291, "nll": 1.4810805852946698, "acc": 0.44673539518900346, "ece": 0.3038644173695721, "brier": 0.7353782363886706, "mean_conf": 0.740909090909091, "confident_error_rate": 0.003436426116838488, "coverage_at_0_9": 0.058419243986254296, "accuracy_at_0_9": 0.9411764705882353, "coverage_at_5pct_error": 0.03436426116838488, "coverage_at_1pct_error": 0.03436426116838488, "confidence_bias": 0.2941736957200875, "top_bins": { "0.9": { "n": 17, "errors": 1, "error_rate": 0.058823529411764705 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5292096219931272, "accuracy": 0.461038961038961, "confidence_cutoff": 0.76 }, "0.8": { "coverage": 0.8075601374570447, "accuracy": 0.4340425531914894, "confidence_cutoff": 0.64 } }, "score_mae": 0.5575566663195529, "ranked_probability_score": 0.13278797517330268 }, "scienthoon_queue": { "n": 291, "nll": 1.0149639049175487, "acc": 0.8969072164948454, "ece": 0.07982817869415812, "brier": 0.15130515463917524, "mean_conf": 0.9691065292096219, "confident_error_rate": 0.044673539518900345, "coverage_at_0_9": 0.8831615120274914, "accuracy_at_0_9": 0.9494163424124513, "coverage_at_5pct_error": 0.9106529209621993, "coverage_at_1pct_error": 0.12371134020618557, "confidence_bias": 0.07219931271477653, "top_bins": { "0.9": { "n": 257, "errors": 13, "error_rate": 0.05058365758754864 }, "0.95": { "n": 248, "errors": 12, "error_rate": 0.04838709677419355 }, "0.99": { "n": 239, "errors": 12, "error_rate": 0.0502092050209205 } }, "selective": { "0.5": { "coverage": 0.8075601374570447, "accuracy": 0.948936170212766, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8075601374570447, "accuracy": 0.948936170212766, "confidence_cutoff": 1.0 } } } }, "variants": { "clean": { "n": 873, "nll": 0.925262691293691, "acc": 0.7525773195876289, "ece": 0.10488493179214829, "brier": 0.34766628498179053, "mean_conf": 0.851207955847131, "confident_error_rate": 0.018327605956471937, "coverage_at_0_9": 0.4547537227949599, "accuracy_at_0_9": 0.9596977329974811, "coverage_at_5pct_error": 0.4856815578465063, "coverage_at_1pct_error": 0.041237113402061855, "confidence_bias": 0.09863063625950219, "top_bins": { "0.9": { "n": 397, "errors": 16, "error_rate": 0.04030226700251889 }, "0.95": { "n": 350, "errors": 14, "error_rate": 0.04 }, "0.99": { "n": 241, "errors": 12, "error_rate": 0.04979253112033195 } }, "selective": { "0.5": { "coverage": 0.5005727376861397, "accuracy": 0.9382151029748284, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.8029782359679267, "accuracy": 0.81169757489301, "confidence_cutoff": 0.73 } }, "score_mae": 0.5575566663195529, "ranked_probability_score": 0.13278797517330268 } }, "heldout_tasks": {}, "permutation": { "n": 0, "mean_max_delta": null, "flip_rate": null }, "temperature": 1.0, "calibrated_clean": { "n": 873, "nll": 0.925262691293691, "acc": 0.7525773195876289, "ece": 0.10488493179214829, "brier": 0.34766628498179053, "mean_conf": 0.851207955847131, "confident_error_rate": 0.018327605956471937, "coverage_at_0_9": 0.4547537227949599, "accuracy_at_0_9": 0.9596977329974811, "coverage_at_5pct_error": 0.4856815578465063, "coverage_at_1pct_error": 0.041237113402061855, "confidence_bias": 0.09863063625950219, "top_bins": { "0.9": { "n": 397, "errors": 16, "error_rate": 0.04030226700251889 }, "0.95": { "n": 350, "errors": 14, "error_rate": 0.04 }, "0.99": { "n": 241, "errors": 12, "error_rate": 0.04979253112033195 } }, "selective": { "0.5": { "coverage": 0.5005727376861397, "accuracy": 0.9382151029748284, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.8029782359679267, "accuracy": 0.81169757489301, "confidence_cutoff": 0.73 } }, "score_mae": 0.5575566663195529, "ranked_probability_score": 0.13278797517330268 }, "metric_policy": { "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 3, "returned_zeros": 1169 }, "source": "converted from scienthoon results/jev_synth.jsonl", "suite": "evals/external/scienthoon-v1" } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-semif-v1/report.json", "source_sha256": "3c4bf4412f8f4f9731da89f6b669abfcc54d71bed3dc04d6d73e2fc65c35f8d2", "matching_panels": [ "kev/external/semif-v1/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.0802982949412223, "paired_flip": null, "unknowable": null, "clean": { "n": 144, "nll": 0.08029829494122229, "acc": 0.9652777777777778, "ece": 0.010555555555555658, "brier": 0.046836111111111106, "mean_conf": 0.9649999999999999, "confident_error_rate": 0.006944444444444444, "coverage_at_0_9": 0.8888888888888888, "accuracy_at_0_9": 0.9921875, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.8888888888888888, "confidence_bias": -0.0002777777777779322, "top_bins": { "0.9": { "n": 128, "errors": 1, "error_rate": 0.0078125 }, "0.95": { "n": 125, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 109, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6805555555555556, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "tasks": { "semif_candidate_selection": { "n": 48, "nll": 0.09278460892467864, "acc": 0.9583333333333334, "ece": 0.019791666666666534, "brier": 0.05895416666666666, "mean_conf": 0.9722916666666667, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.875, "confidence_bias": 0.013958333333333295, "top_bins": { "0.9": { "n": 42, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 42, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 39, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.7916666666666666, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.99 } } }, "semif_evidence_interpretation": { "n": 48, "nll": 0.031238086476079483, "acc": 0.9791666666666666, "ece": 0.025208333333333555, "brier": 0.015812500000000004, "mean_conf": 0.9764583333333334, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9166666666666666, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.9791666666666666, "confidence_bias": -0.0027083333333332016, "top_bins": { "0.9": { "n": 44, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 43, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 40, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.7708333333333334, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8333333333333334, "accuracy": 1.0, "confidence_cutoff": 0.99 } } }, "semif_rule_application": { "n": 48, "nll": 0.11687218942290878, "acc": 0.9583333333333334, "ece": 0.03291666666666665, "brier": 0.06574166666666667, "mean_conf": 0.94625, "confident_error_rate": 0.020833333333333332, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 0.9761904761904762, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.8333333333333334, "confidence_bias": -0.012083333333333335, "top_bins": { "0.9": { "n": 42, "errors": 1, "error_rate": 0.023809523809523808 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 30, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.625, "accuracy": 1.0, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.96 } } } }, "variants": { "clean": { "n": 144, "nll": 0.08029829494122229, "acc": 0.9652777777777778, "ece": 0.010555555555555658, "brier": 0.046836111111111106, "mean_conf": 0.9649999999999999, "confident_error_rate": 0.006944444444444444, "coverage_at_0_9": 0.8888888888888888, "accuracy_at_0_9": 0.9921875, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.8888888888888888, "confidence_bias": -0.0002777777777779322, "top_bins": { "0.9": { "n": 128, "errors": 1, "error_rate": 0.0078125 }, "0.95": { "n": 125, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 109, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6805555555555556, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "criterion_wrapper": { "n": 36, "nll": 0.004955811665945529, "acc": 1.0, "ece": 0.004722222222222285, "brier": 0.0007055555555555557, "mean_conf": 0.9952777777777777, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9722222222222222, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.004722222222222294, "top_bins": { "0.9": { "n": 35, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 34, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9444444444444444, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9444444444444444, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "irrelevant_context": { "n": 36, "nll": 0.0019713541895255166, "acc": 1.0, "ece": 0.0019444444444444153, "brier": 8.333333333333343e-05, "mean_conf": 0.9980555555555556, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.0019444444444444153, "top_bins": { "0.9": { "n": 36, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 36, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9166666666666666, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9166666666666666, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "option_reversal": { "n": 36, "nll": 0.0022534825984660426, "acc": 1.0, "ece": 0.0022222222222221255, "brier": 0.0001000000000000001, "mean_conf": 0.9977777777777779, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.0022222222222221255, "top_bins": { "0.9": { "n": 36, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 36, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 34, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.8888888888888888, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8888888888888888, "accuracy": 1.0, "confidence_cutoff": 1.0 } } } }, "heldout_tasks": {}, "permutation": { "n": 0, "mean_max_delta": null, "flip_rate": null }, "temperature": 1.0, "calibrated_clean": { "n": 144, "nll": 0.08029829494122229, "acc": 0.9652777777777778, "ece": 0.010555555555555658, "brier": 0.046836111111111106, "mean_conf": 0.9649999999999999, "confident_error_rate": 0.006944444444444444, "coverage_at_0_9": 0.8888888888888888, "accuracy_at_0_9": 0.9921875, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.8888888888888888, "confidence_bias": -0.0002777777777779322, "top_bins": { "0.9": { "n": 128, "errors": 1, "error_rate": 0.0078125 }, "0.95": { "n": 125, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 109, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.6805555555555556, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "metric_policy": { "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 424 }, "coverage": { "requested_records": 252, "requested_questions": 252, "evaluated_records": 252, "evaluated_questions": 252, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 230.0922080000164, "p95": 401.4706125000004 }, "suite_sha256": "0de05eac16b0ddeeb2719c50a94a9148d6ae195f66303aec74aa103a3845ad11", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-20T20:20:39.044178+00:00", "calls": 261, "input_tokens": 95296, "output_tokens": 11178, "retries_after_5xx": 9, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.004002432, "budget_usd": 0.1, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-transfer-v2/report.json", "source_sha256": "fe07ca94c7073b8f03606f0914793ef3e434b2fe7915dcd69a5a4632beb52548", "matching_panels": [ "kev/transfer-v2/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.6940544412186378, "paired_flip": { "pairs": 40, "flip_rate": 0.95, "both_correct_rate": 0.95 }, "clean": { "n": 560, "nll": 0.7789994420792489, "acc": 0.8553571428571428, "ece": 0.05545364357864361, "brier": 0.2211401828166222, "mean_conf": 0.9000799062049062, "score_mae": 0.163010101010101, "ranked_probability_score": 0.03471065988164472 }, "tasks": { "contrastive_authorization": { "n": 40, "nll": 0.02276683235931674, "acc": 1.0, "ece": 0.022500000000000187, "brier": 0.0010500000000000015, "mean_conf": 0.9774999999999998 }, "contrastive_deadline": { "n": 40, "nll": 0.17611203802940129, "acc": 0.95, "ece": 0.09751515151515164, "brier": 0.09708314712784408, "mean_conf": 0.8829848484848484, "score_mae": 0.163010101010101, "ranked_probability_score": 0.03471065988164472 }, "emotion": { "n": 80, "nll": 3.552001863756293, "acc": 0.6, "ece": 0.27930808080808084, "brier": 0.6471072061524333, "mean_conf": 0.8744419191919193 }, "mmlu": { "n": 80, "nll": 0.715307888396308, "acc": 0.9, "ece": 0.0605000000000001, "brier": 0.15396749999999998, "mean_conf": 0.92125 }, "paws": { "n": 80, "nll": 0.44741928550458193, "acc": 0.7875, "ece": 0.09037499999999989, "brier": 0.2940775, "mean_conf": 0.8778749999999998 }, "qnli": { "n": 80, "nll": 0.16804925430998352, "acc": 0.925, "ece": 0.06737500000000002, "brier": 0.09560750000000001, "mean_conf": 0.8983749999999999 }, "sciq": { "n": 80, "nll": 0.03920834761263834, "acc": 0.9875, "ece": 0.023374999999999917, "brier": 0.02591, "mean_conf": 0.985125 }, "tweet_offensive": { "n": 80, "nll": 0.43157001978057996, "acc": 0.8125, "ece": 0.06125000000000004, "brier": 0.28224499999999997, "mean_conf": 0.81325 } }, "variants": { "clean": { "n": 560, "nll": 0.7789994420792489, "acc": 0.8553571428571428, "ece": 0.05545364357864361, "brier": 0.2211401828166222, "mean_conf": 0.9000799062049062, "score_mae": 0.163010101010101, "ranked_probability_score": 0.03471065988164472 }, "none_absent": { "n": 36, "nll": 2.060048254744748, "acc": 0.8055555555555556, "ece": 0.13777777777777764, "brier": 0.3665, "mean_conf": 0.8755555555555554 }, "none_present": { "n": 36, "nll": 2.207306096728024, "acc": 0.75, "ece": 0.1609876543209876, "brier": 0.39537926742169166, "mean_conf": 0.8976543209876543 }, "permuted": { "n": 36, "nll": 1.9593128108804327, "acc": 0.8055555555555556, "ece": 0.15333333333333343, "brier": 0.3099444444444444, "mean_conf": 0.9588888888888888 } }, "heldout_tasks": { "emotion": { "n": 80, "nll": 3.552001863756293, "acc": 0.6, "ece": 0.27930808080808084, "brier": 0.6471072061524333, "mean_conf": 0.8744419191919193 }, "mmlu": { "n": 80, "nll": 0.715307888396308, "acc": 0.9, "ece": 0.0605000000000001, "brier": 0.15396749999999998, "mean_conf": 0.92125 }, "paws": { "n": 80, "nll": 0.44741928550458193, "acc": 0.7875, "ece": 0.09037499999999989, "brier": 0.2940775, "mean_conf": 0.8778749999999998 }, "qnli": { "n": 80, "nll": 0.16804925430998352, "acc": 0.925, "ece": 0.06737500000000002, "brier": 0.09560750000000001, "mean_conf": 0.8983749999999999 }, "sciq": { "n": 80, "nll": 0.03920834761263834, "acc": 0.9875, "ece": 0.023374999999999917, "brier": 0.02591, "mean_conf": 0.985125 }, "tweet_offensive": { "n": 80, "nll": 0.43157001978057996, "acc": 0.8125, "ece": 0.06125000000000004, "brier": 0.28224499999999997, "mean_conf": 0.81325 } }, "permutation": { "n": 36, "mean_max_delta": 0.0175, "flip_rate": 0.0 }, "temperature": 1.0, "calibrated_clean": { "n": 560, "nll": 0.7789994420792489, "acc": 0.8553571428571428, "ece": 0.05545364357864361, "brier": 0.2211401828166222, "mean_conf": 0.9000799062049062, "score_mae": 0.163010101010101, "ranked_probability_score": 0.03471065988164472 }, "metric_policy": { "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 3, "returned_zeros": 1059 }, "coverage": { "requested_records": 668, "requested_questions": 668, "evaluated_records": 668, "evaluated_questions": 668, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 235.62814599999592, "p95": 456.5557442000041 }, "suite_sha256": "a9dd4485e3247014209ecbee28d9fe02d0556c69e95dc30cca0584d1e5c27e35", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-18T22:08:10.212000+00:00", "calls": 668, "input_tokens": 254152, "output_tokens": 24804, "retries_after_5xx": 0, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.010674384, "budget_usd": 0.15, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-transfer-v4/report.json", "source_sha256": "e93f1764154f1244c376e2cd3066e6e0bba1ddf12d8d13432e6bf43e47709886", "matching_panels": [ "kev/round3/transfer-r3/development", "kev/round5/transfer-r5/development", "kev/v3/transfer-v3/development", "kev/v4/transfer-v4/development", "kev/v5/transfer-v5/development", "kev/v6/transfer-v6/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.5650854893257732, "paired_flip": { "pairs": 64, "flip_rate": 0.90625, "both_correct_rate": 0.859375, "invariant_pairs": 24, "invariance_rate": 0.9583333333333334, "invariant_both_correct_rate": 0.875 }, "clean": { "n": 656, "nll": 0.7005616535731429, "acc": 0.8567073170731707, "ece": 0.04923780487804877, "brier": 0.2109920731707317, "mean_conf": 0.9028048780487805, "confident_error_rate": 0.036585365853658534, "coverage_at_0_9": 0.7073170731707317, "accuracy_at_0_9": 0.9482758620689655, "selective": { "0.5": { "coverage": 0.5685975609756098, "accuracy": 0.9490616621983914, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8064024390243902, "accuracy": 0.9281663516068053, "confidence_cutoff": 0.8 } }, "score_mae": 0.15949999999999998, "ranked_probability_score": 0.030907499999999997 }, "tasks": { "composition_held_and_or": { "n": 32, "nll": 0.2143738461941428, "acc": 0.90625, "ece": 0.05468749999999996, "brier": 0.12809375, "mean_conf": 0.9253125, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9615384615384616, "selective": { "0.5": { "coverage": 0.75, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.97 } } }, "composition_held_conditional": { "n": 32, "nll": 0.2997209026538331, "acc": 0.78125, "ece": 0.1440625, "brier": 0.21483125, "mean_conf": 0.8890625, "confident_error_rate": 0.0, "coverage_at_0_9": 0.65625, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.79 } } }, "composition_held_or_not": { "n": 32, "nll": 0.08213546328317517, "acc": 0.96875, "ece": 0.03187500000000012, "brier": 0.0528125, "mean_conf": 0.943125, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.98 } } }, "contrastive_authorization": { "n": 40, "nll": 0.023023244863496474, "acc": 1.0, "ece": 0.022750000000000048, "brier": 0.0010750000000000013, "mean_conf": 0.97725, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.725, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "contrastive_deadline": { "n": 40, "nll": 0.20417014714658763, "acc": 0.925, "ece": 0.0655, "brier": 0.11087499999999999, "mean_conf": 0.891, "confident_error_rate": 0.0, "coverage_at_0_9": 0.525, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8 } }, "score_mae": 0.15949999999999998, "ranked_probability_score": 0.030907499999999997 }, "emotion": { "n": 80, "nll": 3.573931146332037, "acc": 0.5875, "ece": 0.2843750000000001, "brier": 0.6490875, "mean_conf": 0.871875, "confident_error_rate": 0.1875, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.7, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.671875, "confidence_cutoff": 0.72 } } }, "mmlu": { "n": 80, "nll": 0.7246273220738594, "acc": 0.9, "ece": 0.07812499999999996, "brier": 0.1613075, "mean_conf": 0.920875, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.775, "accuracy_at_0_9": 0.9516129032258065, "selective": { "0.5": { "coverage": 0.5875, "accuracy": 0.9361702127659575, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.89 } } }, "paws": { "n": 80, "nll": 0.4499565515081502, "acc": 0.7875, "ece": 0.09124999999999994, "brier": 0.29711499999999996, "mean_conf": 0.8787500000000001, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.94, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.79 } } }, "qnli": { "n": 80, "nll": 0.16692640246587415, "acc": 0.925, "ece": 0.06762499999999996, "brier": 0.0951775, "mean_conf": 0.896875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.525, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.79 } } }, "sciq": { "n": 80, "nll": 0.04100721930598446, "acc": 0.9875, "ece": 0.019624999999999972, "brier": 0.027064999999999995, "mean_conf": 0.9846250000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.9125, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9125, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "tweet_offensive": { "n": 80, "nll": 0.436068136756365, "acc": 0.8125, "ece": 0.06437499999999997, "brier": 0.2861125, "mean_conf": 0.812875, "confident_error_rate": 0.025, "coverage_at_0_9": 0.4375, "accuracy_at_0_9": 0.9428571428571428, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.8, "accuracy": 0.828125, "confidence_cutoff": 0.67 } } } }, "variants": { "clean": { "n": 656, "nll": 0.7005616535731429, "acc": 0.8567073170731707, "ece": 0.04923780487804877, "brier": 0.2109920731707317, "mean_conf": 0.9028048780487805, "confident_error_rate": 0.036585365853658534, "coverage_at_0_9": 0.7073170731707317, "accuracy_at_0_9": 0.9482758620689655, "selective": { "0.5": { "coverage": 0.5685975609756098, "accuracy": 0.9490616621983914, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8064024390243902, "accuracy": 0.9281663516068053, "confidence_cutoff": 0.8 } }, "score_mae": 0.15949999999999998, "ranked_probability_score": 0.030907499999999997 }, "none_absent": { "n": 36, "nll": 2.05784959917515, "acc": 0.8055555555555556, "ece": 0.13666666666666644, "brier": 0.3605333333333334, "mean_conf": 0.8749999999999998, "confident_error_rate": 0.1111111111111111, "coverage_at_0_9": 0.6111111111111112, "accuracy_at_0_9": 0.8181818181818182, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8275862068965517, "confidence_cutoff": 0.71 } } }, "none_present": { "n": 36, "nll": 2.2133678477176884, "acc": 0.75, "ece": 0.14527777777777773, "brier": 0.39188333333333336, "mean_conf": 0.8952777777777778, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.6944444444444444, "accuracy_at_0_9": 0.88, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9444444444444444, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8620689655172413, "confidence_cutoff": 0.76 } } }, "permuted": { "n": 36, "nll": 1.9571278715305676, "acc": 0.8055555555555556, "ece": 0.15361111111111125, "brier": 0.3097777777777778, "mean_conf": 0.9591666666666667, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.8611111111111112, "accuracy_at_0_9": 0.9032258064516129, "selective": { "0.5": { "coverage": 0.6666666666666666, "accuracy": 0.875, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.896551724137931, "confidence_cutoff": 0.95 } } } }, "heldout_tasks": { "composition_held_and_or": { "n": 32, "nll": 0.2143738461941428, "acc": 0.90625, "ece": 0.05468749999999996, "brier": 0.12809375, "mean_conf": 0.9253125, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9615384615384616, "selective": { "0.5": { "coverage": 0.75, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.97 } } }, "composition_held_conditional": { "n": 32, "nll": 0.2997209026538331, "acc": 0.78125, "ece": 0.1440625, "brier": 0.21483125, "mean_conf": 0.8890625, "confident_error_rate": 0.0, "coverage_at_0_9": 0.65625, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.79 } } }, "composition_held_or_not": { "n": 32, "nll": 0.08213546328317517, "acc": 0.96875, "ece": 0.03187500000000012, "brier": 0.0528125, "mean_conf": 0.943125, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.53125, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.98 } } }, "contrastive_authorization": { "n": 40, "nll": 0.023023244863496474, "acc": 1.0, "ece": 0.022750000000000048, "brier": 0.0010750000000000013, "mean_conf": 0.97725, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.725, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "contrastive_deadline": { "n": 40, "nll": 0.20417014714658763, "acc": 0.925, "ece": 0.0655, "brier": 0.11087499999999999, "mean_conf": 0.891, "confident_error_rate": 0.0, "coverage_at_0_9": 0.525, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.8, "accuracy": 0.96875, "confidence_cutoff": 0.8 } }, "score_mae": 0.15949999999999998, "ranked_probability_score": 0.030907499999999997 }, "emotion": { "n": 80, "nll": 3.573931146332037, "acc": 0.5875, "ece": 0.2843750000000001, "brier": 0.6490875, "mean_conf": 0.871875, "confident_error_rate": 0.1875, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.7, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.675, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8, "accuracy": 0.671875, "confidence_cutoff": 0.72 } } }, "mmlu": { "n": 80, "nll": 0.7246273220738594, "acc": 0.9, "ece": 0.07812499999999996, "brier": 0.1613075, "mean_conf": 0.920875, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.775, "accuracy_at_0_9": 0.9516129032258065, "selective": { "0.5": { "coverage": 0.5875, "accuracy": 0.9361702127659575, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.89 } } }, "paws": { "n": 80, "nll": 0.4499565515081502, "acc": 0.7875, "ece": 0.09124999999999994, "brier": 0.29711499999999996, "mean_conf": 0.8787500000000001, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.94, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.79 } } }, "qnli": { "n": 80, "nll": 0.16692640246587415, "acc": 0.925, "ece": 0.06762499999999996, "brier": 0.0951775, "mean_conf": 0.896875, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6875, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.525, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8125, "accuracy": 1.0, "confidence_cutoff": 0.79 } } }, "sciq": { "n": 80, "nll": 0.04100721930598446, "acc": 0.9875, "ece": 0.019624999999999972, "brier": 0.027064999999999995, "mean_conf": 0.9846250000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "selective": { "0.5": { "coverage": 0.9125, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9125, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "tweet_offensive": { "n": 80, "nll": 0.436068136756365, "acc": 0.8125, "ece": 0.06437499999999997, "brier": 0.2861125, "mean_conf": 0.812875, "confident_error_rate": 0.025, "coverage_at_0_9": 0.4375, "accuracy_at_0_9": 0.9428571428571428, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.8, "accuracy": 0.828125, "confidence_cutoff": 0.67 } } } }, "permutation": { "n": 36, "mean_max_delta": 0.019722222222222228, "flip_rate": 0.0 }, "temperature": 1.0, "calibrated_clean": { "n": 656, "nll": 0.7005616535731429, "acc": 0.8567073170731707, "ece": 0.04923780487804877, "brier": 0.2109920731707317, "mean_conf": 0.9028048780487805, "confident_error_rate": 0.036585365853658534, "coverage_at_0_9": 0.7073170731707317, "accuracy_at_0_9": 0.9482758620689655, "selective": { "0.5": { "coverage": 0.5685975609756098, "accuracy": 0.9490616621983914, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8064024390243902, "accuracy": 0.9281663516068053, "confidence_cutoff": 0.8 } }, "score_mae": 0.15949999999999998, "ranked_probability_score": 0.030907499999999997 }, "metric_policy": { "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 1100 }, "coverage": { "requested_records": 764, "requested_questions": 764, "evaluated_records": 764, "evaluated_questions": 764, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 221.77577099998598, "p95": 378.30218744997694 }, "suite_sha256": "31677c2256b406222e7d94ffdc0a02a70ce05746b9efe307876024c4e77291d1", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-18T23:54:23.759938+00:00", "calls": 769, "input_tokens": 296769, "output_tokens": 27780, "retries_after_5xx": 5, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.012464298, "budget_usd": 0.15, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-transfer-v9/report.json", "source_sha256": "9629b821bca4287c483f50574160a993539e3fb55188ae0a40cf702c1c2e5f0f", "matching_panels": [ "kev/v9/transfer-v9/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.4426099425132306, "paired_flip": { "pairs": 64, "flip_rate": 0.921875, "both_correct_rate": 0.890625, "invariant_pairs": 24, "invariance_rate": 1.0, "invariant_both_correct_rate": 0.9166666666666666 }, "unknowable": { "n": 110, "mean_max_p": 0.6100000000000001, "share_at_0_9": 0.09090909090909091, "control_mean_max_p": 0.9226363636363636, "control_share_at_0_9": 0.7545454545454545, "control_acc": 0.9454545454545454, "paired_confidence_drop": 0.3126363636363636, "share_less_confident_than_control": 0.9 }, "clean": { "n": 1046, "nll": 0.643853776029141, "acc": 0.8537284894837476, "ece": 0.03348542789269361, "brier": 0.21211354829169302, "mean_conf": 0.8800055043745293, "confident_error_rate": 0.02676864244741874, "coverage_at_0_9": 0.6567877629063098, "accuracy_at_0_9": 0.9592430858806404, "coverage_at_5pct_error": 0.6950286806883366, "coverage_at_1pct_error": 0.0028680688336520078, "confidence_bias": 0.026277014890781714, "top_bins": { "0.9": { "n": 687, "errors": 28, "error_rate": 0.040756914119359534 }, "0.95": { "n": 600, "errors": 24, "error_rate": 0.04 }, "0.99": { "n": 366, "errors": 16, "error_rate": 0.04371584699453552 } }, "selective": { "0.5": { "coverage": 0.5066921606118547, "accuracy": 0.9584905660377359, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8040152963671128, "accuracy": 0.925089179548157, "confidence_cutoff": 0.76 } }, "score_mae": 0.11869999999999999, "ranked_probability_score": 0.0208005 }, "tasks": { "buried_emotion": { "n": 20, "nll": 2.147290789939311, "acc": 0.5, "ece": 0.2109141414141415, "brier": 0.6530402132435464, "mean_conf": 0.7039141414141414, "confident_error_rate": 0.0, "coverage_at_0_9": 0.15, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "confidence_bias": 0.20391414141414144, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.7272727272727273, "confidence_cutoff": 0.66 }, "0.8": { "coverage": 0.8, "accuracy": 0.5625, "confidence_cutoff": 0.62 } } }, "buried_paws": { "n": 20, "nll": 0.6224035971846659, "acc": 0.65, "ece": 0.20600000000000002, "brier": 0.44767999999999997, "mean_conf": 0.827, "confident_error_rate": 0.0, "coverage_at_0_9": 0.35, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "confidence_bias": 0.17699999999999994, "top_bins": { "0.9": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.86 }, "0.8": { "coverage": 0.8, "accuracy": 0.6875, "confidence_cutoff": 0.71 } } }, "buried_qnli": { "n": 20, "nll": 0.23718842491904066, "acc": 0.85, "ece": 0.1395, "brier": 0.14750999999999997, "mean_conf": 0.8805, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8, "coverage_at_1pct_error": 0.8, "confidence_bias": 0.03049999999999997, "top_bins": { "0.9": { "n": 14, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 8, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.81 } } }, "buried_tweet_offensive": { "n": 20, "nll": 0.49534018220868753, "acc": 0.8, "ece": 0.17649999999999996, "brier": 0.32776999999999995, "mean_conf": 0.7445, "confident_error_rate": 0.0, "coverage_at_0_9": 0.15, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "confidence_bias": -0.055499999999999994, "top_bins": { "0.9": { "n": 3, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9, "confidence_cutoff": 0.79 }, "0.8": { "coverage": 0.85, "accuracy": 0.7647058823529411, "confidence_cutoff": 0.63 } } }, "composition_held_and_or": { "n": 32, "nll": 0.17388167421601788, "acc": 0.96875, "ece": 0.08874999999999994, "brier": 0.09766250000000001, "mean_conf": 0.93875, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9615384615384616, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.78125, "confidence_bias": -0.030000000000000027, "top_bins": { "0.9": { "n": 26, "errors": 1, "error_rate": 0.038461538461538464 }, "0.95": { "n": 26, "errors": 1, "error_rate": 0.038461538461538464 }, "0.99": { "n": 24, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.75, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.96 } } }, "composition_held_conditional": { "n": 32, "nll": 0.3091480950893317, "acc": 0.78125, "ece": 0.13625000000000004, "brier": 0.22093749999999998, "mean_conf": 0.89375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.59375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8125, "coverage_at_1pct_error": 0.71875, "confidence_bias": 0.11250000000000004, "top_bins": { "0.9": { "n": 19, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.82 } } }, "composition_held_or_not": { "n": 32, "nll": 0.0746809441087405, "acc": 0.96875, "ece": 0.05750000000000002, "brier": 0.04567500000000001, "mean_conf": 0.9475, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "confidence_bias": -0.02124999999999999, "top_bins": { "0.9": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 24, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.59375, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "contrastive_authorization": { "n": 40, "nll": 0.022766832359316744, "acc": 1.0, "ece": 0.022500000000000187, "brier": 0.0010500000000000012, "mean_conf": 0.9774999999999998, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.022500000000000187, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.75, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "contrastive_deadline": { "n": 40, "nll": 0.1755083447562497, "acc": 0.95, "ece": 0.05825000000000009, "brier": 0.08714999999999999, "mean_conf": 0.89175, "confident_error_rate": 0.0, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.725, "confidence_bias": -0.05824999999999991, "top_bins": { "0.9": { "n": 23, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.875, "accuracy": 0.9714285714285714, "confidence_cutoff": 0.77 } }, "score_mae": 0.148, "ranked_probability_score": 0.025065000000000004 }, "emotion": { "n": 80, "nll": 3.575426682184326, "acc": 0.5875, "ece": 0.2855000000000001, "brier": 0.6533125, "mean_conf": 0.873, "confident_error_rate": 0.1875, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.7, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "confidence_bias": 0.2855, "top_bins": { "0.9": { "n": 50, "errors": 15, "error_rate": 0.3 }, "0.95": { "n": 46, "errors": 14, "error_rate": 0.30434782608695654 }, "0.99": { "n": 35, "errors": 10, "error_rate": 0.2857142857142857 } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.6818181818181818, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 0.65625, "confidence_cutoff": 0.73 } } }, "mmlu": { "n": 80, "nll": 0.7180639950202841, "acc": 0.9, "ece": 0.07912499999999995, "brier": 0.157055, "mean_conf": 0.922125, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.775, "accuracy_at_0_9": 0.9516129032258065, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.0375, "confidence_bias": 0.02212499999999995, "top_bins": { "0.9": { "n": 62, "errors": 3, "error_rate": 0.04838709677419355 }, "0.95": { "n": 52, "errors": 3, "error_rate": 0.057692307692307696 }, "0.99": { "n": 49, "errors": 3, "error_rate": 0.061224489795918366 } }, "selective": { "0.5": { "coverage": 0.6125, "accuracy": 0.9387755102040817, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.89 } } }, "mmlu_pro": { "n": 200, "nll": 0.655016135186899, "acc": 0.84, "ece": 0.0631464646464645, "brier": 0.22361383624119988, "mean_conf": 0.8134373737373738, "confident_error_rate": 0.025, "coverage_at_0_9": 0.585, "accuracy_at_0_9": 0.9572649572649573, "coverage_at_5pct_error": 0.78, "coverage_at_1pct_error": 0.165, "confidence_bias": -0.026562626262626177, "top_bins": { "0.9": { "n": 117, "errors": 5, "error_rate": 0.042735042735042736 }, "0.95": { "n": 95, "errors": 4, "error_rate": 0.042105263157894736 }, "0.99": { "n": 67, "errors": 3, "error_rate": 0.04477611940298507 } }, "selective": { "0.5": { "coverage": 0.505, "accuracy": 0.9504950495049505, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.81, "accuracy": 0.9382716049382716, "confidence_cutoff": 0.63 } } }, "paws": { "n": 80, "nll": 0.44334782865903916, "acc": 0.8, "ece": 0.100375, "brier": 0.29281750000000006, "mean_conf": 0.8788750000000001, "confident_error_rate": 0.025, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9583333333333334, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.3125, "confidence_bias": 0.07887500000000003, "top_bins": { "0.9": { "n": 48, "errors": 2, "error_rate": 0.041666666666666664 }, "0.95": { "n": 40, "errors": 2, "error_rate": 0.05 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.8 } } }, "qnli": { "n": 80, "nll": 0.1650349761695868, "acc": 0.9125, "ece": 0.07750000000000007, "brier": 0.09283499999999997, "mean_conf": 0.89725, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.95, "coverage_at_1pct_error": 0.775, "confidence_bias": -0.015249999999999986, "top_bins": { "0.9": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 47, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.825, "accuracy": 0.9848484848484849, "confidence_cutoff": 0.8 } } }, "sciq": { "n": 80, "nll": 0.03740746395239355, "acc": 0.9875, "ece": 0.02324999999999992, "brier": 0.0249425, "mean_conf": 0.9845, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "confidence_bias": -0.0030000000000000027, "top_bins": { "0.9": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 75, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9125, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9125, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "tweet_offensive": { "n": 80, "nll": 0.43605017703300286, "acc": 0.8125, "ece": 0.08650000000000001, "brier": 0.28639000000000003, "mean_conf": 0.8125, "confident_error_rate": 0.025, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.2625, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 32, "errors": 2, "error_rate": 0.0625 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8, "confidence_cutoff": 0.66 } } }, "unknowable_age_eligibility": { "n": 10, "nll": 0.6916449965778975, "acc": 0.5, "ece": 0.037000000000000144, "brier": 0.49850000000000005, "mean_conf": 0.5370000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "confidence_bias": 0.037000000000000144, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.7, "accuracy": 0.5714285714285714, "confidence_cutoff": 0.52 }, "0.8": { "coverage": 0.9, "accuracy": 0.5555555555555556, "confidence_cutoff": 0.51 } } }, "unknowable_authorization": { "n": 10, "nll": 0.6883402995300678, "acc": 0.5, "ece": 0.020999999999999908, "brier": 0.49522000000000005, "mean_conf": 0.5209999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "confidence_bias": 0.020999999999999908, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.5, "confidence_cutoff": 0.52 }, "0.8": { "coverage": 0.9, "accuracy": 0.4444444444444444, "confidence_cutoff": 0.51 } } }, "unknowable_control_age_eligibility": { "n": 10, "nll": 0.02534158608794977, "acc": 1.0, "ece": 0.025000000000000022, "brier": 0.0013400000000000022, "mean_conf": 0.975, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.025000000000000022, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "unknowable_control_authorization": { "n": 10, "nll": 0.023279657367676197, "acc": 1.0, "ece": 0.02299999999999991, "brier": 0.0011000000000000016, "mean_conf": 0.9770000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.02299999999999991, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.7, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "unknowable_control_deadline": { "n": 10, "nll": 0.1397380083521098, "acc": 1.0, "ece": 0.12399999999999996, "brier": 0.042179999999999995, "mean_conf": 0.876, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.124, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.89 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.84 } }, "score_mae": 0.12600000000000006, "ranked_probability_score": 0.013019999999999993 }, "unknowable_control_late_fee": { "n": 10, "nll": 0.001005033585350145, "acc": 1.0, "ece": 0.0010000000000000009, "brier": 2.000000000000002e-05, "mean_conf": 0.999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.0010000000000000009, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.002, "ranked_probability_score": 1.0000000000000018e-05 }, "unknowable_control_quantity_limit": { "n": 10, "nll": 0.041777944487873794, "acc": 1.0, "ece": 0.038999999999999924, "brier": 0.00932, "mean_conf": 0.961, "confident_error_rate": 0.0, "coverage_at_0_9": 0.9, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.039000000000000035, "top_bins": { "0.9": { "n": 9, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 7, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 6, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.94 } } }, "unknowable_control_return_window": { "n": 10, "nll": 0.7610387979355436, "acc": 0.6, "ece": 0.376, "brier": 0.5644, "mean_conf": 0.8400000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.4, "coverage_at_1pct_error": 0.4, "confidence_bias": 0.2400000000000001, "top_bins": { "0.9": { "n": 4, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.79 } } }, "unknowable_control_shipping_delay": { "n": 10, "nll": 0.15090529019237492, "acc": 1.0, "ece": 0.12400000000000003, "brier": 0.06432000000000002, "mean_conf": 0.876, "confident_error_rate": 0.0, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.124, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 5, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 1.0, "confidence_cutoff": 0.85 } }, "score_mae": 0.08999999999999998, "ranked_probability_score": 0.013439999999999999 }, "unknowable_control_sla_response": { "n": 10, "nll": 0.001005033585350145, "acc": 1.0, "ece": 0.0010000000000000009, "brier": 2.000000000000002e-05, "mean_conf": 0.999, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.0010000000000000009, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.0010000000000000009, "ranked_probability_score": 5.000000000000009e-06 }, "unknowable_control_spend_threshold": { "n": 10, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "unknowable_control_volume_discount": { "n": 10, "nll": 0.0, "acc": 1.0, "ece": 0.0, "brier": 0.0, "mean_conf": 1.0, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 10, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 1.0 } }, "score_mae": 0.0, "ranked_probability_score": 0.0 }, "unknowable_control_warranty_claim": { "n": 10, "nll": 0.5178209532761043, "acc": 0.8, "ece": 0.15399999999999997, "brier": 0.3028, "mean_conf": 0.646, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.6, "coverage_at_1pct_error": 0.6, "confidence_bias": -0.15400000000000003, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.66 }, "0.8": { "coverage": 0.9, "accuracy": 0.7777777777777778, "confidence_cutoff": 0.54 } }, "score_mae": 0.376, "ranked_probability_score": 0.08127 }, "unknowable_deadline": { "n": 10, "nll": 0.957506628041549, "acc": 0.5, "ece": 0.21000000000000002, "brier": 0.57382, "mean_conf": 0.442, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.2, "coverage_at_1pct_error": 0.2, "confidence_bias": -0.057999999999999996, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8, "confidence_cutoff": 0.45 }, "0.8": { "coverage": 0.8, "accuracy": 0.625, "confidence_cutoff": 0.4 } }, "score_mae": 0.562, "ranked_probability_score": 0.16569 }, "unknowable_late_fee": { "n": 10, "nll": 1.1371190755945695, "acc": 0.5, "ece": 0.313, "brier": 0.69322, "mean_conf": 0.6769999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.1, "coverage_at_1pct_error": 0.1, "confidence_bias": 0.17699999999999994, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.73 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.55 } }, "score_mae": 0.7680000000000001, "ranked_probability_score": 0.27716999999999997 }, "unknowable_quantity_limit": { "n": 10, "nll": 8.68352373607743, "acc": 0.5, "ece": 0.494, "brier": 0.9901599999999998, "mean_conf": 0.994, "confident_error_rate": 0.5, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 0.5, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "confidence_bias": 0.494, "top_bins": { "0.9": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.95": { "n": 10, "errors": 5, "error_rate": 0.5 }, "0.99": { "n": 9, "errors": 4, "error_rate": 0.4444444444444444 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9, "accuracy": 0.5555555555555556, "confidence_cutoff": 0.99 } } }, "unknowable_return_window": { "n": 10, "nll": 0.7055609922252111, "acc": 0.5, "ece": 0.05499999999999994, "brier": 0.5123, "mean_conf": 0.5549999999999999, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "confidence_bias": 0.05499999999999994, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.55 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.55 } } }, "unknowable_shipping_delay": { "n": 10, "nll": 1.6364781810856062, "acc": 0.2, "ece": 0.5050000000000001, "brier": 1.03208, "mean_conf": 0.7050000000000001, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "confidence_bias": 0.5050000000000001, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.2, "confidence_cutoff": 0.74 }, "0.8": { "coverage": 0.9, "accuracy": 0.2222222222222222, "confidence_cutoff": 0.67 } }, "score_mae": 0.944, "ranked_probability_score": 0.36147999999999997 }, "unknowable_sla_response": { "n": 10, "nll": 1.0740409504205626, "acc": 0.4, "ece": 0.07900000000000001, "brier": 0.6479199999999999, "mean_conf": 0.42299999999999993, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "confidence_bias": 0.02299999999999991, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.42 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.4 } }, "score_mae": 0.8100000000000002, "ranked_probability_score": 0.23806000000000002 }, "unknowable_spend_threshold": { "n": 10, "nll": 0.8364615964543166, "acc": 0.5, "ece": 0.22800000000000006, "brier": 0.5364, "mean_conf": 0.49800000000000005, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "confidence_bias": -0.0019999999999999463, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.6, "accuracy": 0.3333333333333333, "confidence_cutoff": 0.51 }, "0.8": { "coverage": 0.8, "accuracy": 0.5, "confidence_cutoff": 0.48 } } }, "unknowable_volume_discount": { "n": 10, "nll": 2.2242623476273478, "acc": 0.4, "ece": 0.444, "brier": 0.9941999999999999, "mean_conf": 0.844, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "confidence_bias": 0.44399999999999995, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.4, "confidence_cutoff": 0.85 }, "0.8": { "coverage": 0.8, "accuracy": 0.375, "confidence_cutoff": 0.83 } }, "score_mae": 0.9590000000000002, "ranked_probability_score": 0.41072499999999995 }, "unknowable_warranty_claim": { "n": 10, "nll": 1.1442566842443558, "acc": 0.4, "ece": 0.11400000000000006, "brier": 0.68136, "mean_conf": 0.514, "confident_error_rate": 0.0, "coverage_at_0_9": 0.0, "accuracy_at_0_9": null, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "confidence_bias": 0.11399999999999999, "top_bins": { "0.9": { "n": 0, "errors": 0, "error_rate": null }, "0.95": { "n": 0, "errors": 0, "error_rate": null }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.6, "confidence_cutoff": 0.53 }, "0.8": { "coverage": 0.9, "accuracy": 0.4444444444444444, "confidence_cutoff": 0.48 } }, "score_mae": 0.764, "ranked_probability_score": 0.23723 } }, "variants": { "clean": { "n": 1156, "nll": 0.7536877202467739, "acc": 0.8148788927335641, "ece": 0.04302567194435712, "brier": 0.2581510134196461, "mean_conf": 0.8543129390793751, "confident_error_rate": 0.028546712802768166, "coverage_at_0_9": 0.6029411764705882, "accuracy_at_0_9": 0.9526542324246772, "coverage_at_5pct_error": 0.6176470588235294, "coverage_at_1pct_error": 0.0025951557093425604, "confidence_bias": 0.03943404634581105, "top_bins": { "0.9": { "n": 697, "errors": 33, "error_rate": 0.047345767575322814 }, "0.95": { "n": 610, "errors": 29, "error_rate": 0.047540983606557376 }, "0.99": { "n": 375, "errors": 20, "error_rate": 0.05333333333333334 } }, "selective": { "0.5": { "coverage": 0.5025951557093425, "accuracy": 0.9518072289156626, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.892972972972973, "confidence_cutoff": 0.6799999999999999 } }, "score_mae": 0.374625, "ranked_probability_score": 0.11864749999999999 }, "none_absent": { "n": 36, "nll": 1.6047898685375264, "acc": 0.8055555555555556, "ece": 0.17999999999999983, "brier": 0.36383333333333323, "mean_conf": 0.8722222222222222, "confident_error_rate": 0.1111111111111111, "coverage_at_0_9": 0.6111111111111112, "accuracy_at_0_9": 0.8181818181818182, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "confidence_bias": 0.06666666666666665, "top_bins": { "0.9": { "n": 22, "errors": 4, "error_rate": 0.18181818181818182 }, "0.95": { "n": 20, "errors": 4, "error_rate": 0.2 }, "0.99": { "n": 14, "errors": 3, "error_rate": 0.21428571428571427 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.8333333333333334, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.7931034482758621, "confidence_cutoff": 0.74 } } }, "none_present": { "n": 36, "nll": 2.2024538694405464, "acc": 0.75, "ece": 0.1564197530864197, "brier": 0.38744481628858735, "mean_conf": 0.893641975308642, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.6944444444444444, "accuracy_at_0_9": 0.88, "coverage_at_5pct_error": 0.08333333333333333, "coverage_at_1pct_error": 0.08333333333333333, "confidence_bias": 0.14364197530864198, "top_bins": { "0.9": { "n": 25, "errors": 3, "error_rate": 0.12 }, "0.95": { "n": 24, "errors": 3, "error_rate": 0.125 }, "0.99": { "n": 22, "errors": 3, "error_rate": 0.13636363636363635 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9444444444444444, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8055555555555556, "accuracy": 0.8620689655172413, "confidence_cutoff": 0.77 } } }, "permuted": { "n": 36, "nll": 1.9484597009167162, "acc": 0.8055555555555556, "ece": 0.15861111111111115, "brier": 0.30642222222222215, "mean_conf": 0.9580555555555557, "confident_error_rate": 0.08333333333333333, "coverage_at_0_9": 0.8333333333333334, "accuracy_at_0_9": 0.9, "coverage_at_5pct_error": 0.08333333333333333, "coverage_at_1pct_error": 0.08333333333333333, "confidence_bias": 0.15250000000000008, "top_bins": { "0.9": { "n": 30, "errors": 3, "error_rate": 0.1 }, "0.95": { "n": 30, "errors": 3, "error_rate": 0.1 }, "0.99": { "n": 26, "errors": 3, "error_rate": 0.11538461538461539 } }, "selective": { "0.5": { "coverage": 0.6666666666666666, "accuracy": 0.875, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8333333333333334, "accuracy": 0.9, "confidence_cutoff": 0.95 } } } }, "heldout_tasks": { "composition_held_and_or": { "n": 32, "nll": 0.17388167421601788, "acc": 0.96875, "ece": 0.08874999999999994, "brier": 0.09766250000000001, "mean_conf": 0.93875, "confident_error_rate": 0.03125, "coverage_at_0_9": 0.8125, "accuracy_at_0_9": 0.9615384615384616, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.78125, "confidence_bias": -0.030000000000000027, "top_bins": { "0.9": { "n": 26, "errors": 1, "error_rate": 0.038461538461538464 }, "0.95": { "n": 26, "errors": 1, "error_rate": 0.038461538461538464 }, "0.99": { "n": 24, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.75, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.96 } } }, "composition_held_conditional": { "n": 32, "nll": 0.3091480950893317, "acc": 0.78125, "ece": 0.13625000000000004, "brier": 0.22093749999999998, "mean_conf": 0.89375, "confident_error_rate": 0.0, "coverage_at_0_9": 0.59375, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.8125, "coverage_at_1pct_error": 0.71875, "confidence_bias": 0.11250000000000004, "top_bins": { "0.9": { "n": 19, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8125, "accuracy": 0.9615384615384616, "confidence_cutoff": 0.82 } } }, "composition_held_or_not": { "n": 32, "nll": 0.0746809441087405, "acc": 0.96875, "ece": 0.05750000000000002, "brier": 0.04567500000000001, "mean_conf": 0.9475, "confident_error_rate": 0.0, "coverage_at_0_9": 0.875, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.96875, "confidence_bias": -0.02124999999999999, "top_bins": { "0.9": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 28, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 24, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.59375, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.84375, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "contrastive_authorization": { "n": 40, "nll": 0.022766832359316744, "acc": 1.0, "ece": 0.022500000000000187, "brier": 0.0010500000000000012, "mean_conf": 0.9774999999999998, "confident_error_rate": 0.0, "coverage_at_0_9": 1.0, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "confidence_bias": -0.022500000000000187, "top_bins": { "0.9": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 40, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 0, "errors": 0, "error_rate": null } }, "selective": { "0.5": { "coverage": 0.75, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 1.0, "accuracy": 1.0, "confidence_cutoff": 0.97 } } }, "contrastive_deadline": { "n": 40, "nll": 0.1755083447562497, "acc": 0.95, "ece": 0.05825000000000009, "brier": 0.08714999999999999, "mean_conf": 0.89175, "confident_error_rate": 0.0, "coverage_at_0_9": 0.575, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.725, "confidence_bias": -0.05824999999999991, "top_bins": { "0.9": { "n": 23, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 20, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 15, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.875, "accuracy": 0.9714285714285714, "confidence_cutoff": 0.77 } }, "score_mae": 0.148, "ranked_probability_score": 0.025065000000000004 }, "emotion": { "n": 80, "nll": 3.575426682184326, "acc": 0.5875, "ece": 0.2855000000000001, "brier": 0.6533125, "mean_conf": 0.873, "confident_error_rate": 0.1875, "coverage_at_0_9": 0.625, "accuracy_at_0_9": 0.7, "coverage_at_5pct_error": 0.05, "coverage_at_1pct_error": 0.05, "confidence_bias": 0.2855, "top_bins": { "0.9": { "n": 50, "errors": 15, "error_rate": 0.3 }, "0.95": { "n": 46, "errors": 14, "error_rate": 0.30434782608695654 }, "0.99": { "n": 35, "errors": 10, "error_rate": 0.2857142857142857 } }, "selective": { "0.5": { "coverage": 0.55, "accuracy": 0.6818181818181818, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8, "accuracy": 0.65625, "confidence_cutoff": 0.73 } } }, "mmlu": { "n": 80, "nll": 0.7180639950202841, "acc": 0.9, "ece": 0.07912499999999995, "brier": 0.157055, "mean_conf": 0.922125, "confident_error_rate": 0.0375, "coverage_at_0_9": 0.775, "accuracy_at_0_9": 0.9516129032258065, "coverage_at_5pct_error": 0.875, "coverage_at_1pct_error": 0.0375, "confidence_bias": 0.02212499999999995, "top_bins": { "0.9": { "n": 62, "errors": 3, "error_rate": 0.04838709677419355 }, "0.95": { "n": 52, "errors": 3, "error_rate": 0.057692307692307696 }, "0.99": { "n": 49, "errors": 3, "error_rate": 0.061224489795918366 } }, "selective": { "0.5": { "coverage": 0.6125, "accuracy": 0.9387755102040817, "confidence_cutoff": 0.99 }, "0.8": { "coverage": 0.8, "accuracy": 0.953125, "confidence_cutoff": 0.89 } } }, "paws": { "n": 80, "nll": 0.44334782865903916, "acc": 0.8, "ece": 0.100375, "brier": 0.29281750000000006, "mean_conf": 0.8788750000000001, "confident_error_rate": 0.025, "coverage_at_0_9": 0.6, "accuracy_at_0_9": 0.9583333333333334, "coverage_at_5pct_error": 0.625, "coverage_at_1pct_error": 0.3125, "confidence_bias": 0.07887500000000003, "top_bins": { "0.9": { "n": 48, "errors": 2, "error_rate": 0.041666666666666664 }, "0.95": { "n": 40, "errors": 2, "error_rate": 0.05 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.95, "confidence_cutoff": 0.95 }, "0.8": { "coverage": 0.8, "accuracy": 0.859375, "confidence_cutoff": 0.8 } } }, "qnli": { "n": 80, "nll": 0.1650349761695868, "acc": 0.9125, "ece": 0.07750000000000007, "brier": 0.09283499999999997, "mean_conf": 0.89725, "confident_error_rate": 0.0, "coverage_at_0_9": 0.7, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.95, "coverage_at_1pct_error": 0.775, "confidence_bias": -0.015249999999999986, "top_bins": { "0.9": { "n": 56, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 47, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 10, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.525, "accuracy": 1.0, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.825, "accuracy": 0.9848484848484849, "confidence_cutoff": 0.8 } } }, "sciq": { "n": 80, "nll": 0.03740746395239355, "acc": 0.9875, "ece": 0.02324999999999992, "brier": 0.0249425, "mean_conf": 0.9845, "confident_error_rate": 0.0, "coverage_at_0_9": 0.95, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 0.95, "confidence_bias": -0.0030000000000000027, "top_bins": { "0.9": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 76, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 75, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.9125, "accuracy": 1.0, "confidence_cutoff": 1.0 }, "0.8": { "coverage": 0.9125, "accuracy": 1.0, "confidence_cutoff": 1.0 } } }, "tweet_offensive": { "n": 80, "nll": 0.43605017703300286, "acc": 0.8125, "ece": 0.08650000000000001, "brier": 0.28639000000000003, "mean_conf": 0.8125, "confident_error_rate": 0.025, "coverage_at_0_9": 0.4, "accuracy_at_0_9": 0.9375, "coverage_at_5pct_error": 0.3, "coverage_at_1pct_error": 0.2625, "confidence_bias": 0.0, "top_bins": { "0.9": { "n": 32, "errors": 2, "error_rate": 0.0625 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.925, "confidence_cutoff": 0.87 }, "0.8": { "coverage": 0.8125, "accuracy": 0.8, "confidence_cutoff": 0.66 } } } }, "permutation": { "n": 36, "mean_max_delta": 0.01666666666666667, "flip_rate": 0.0 }, "temperature": 1.0, "calibrated_clean": { "n": 1156, "nll": 0.7536877202467739, "acc": 0.8148788927335641, "ece": 0.04302567194435712, "brier": 0.2581510134196461, "mean_conf": 0.8543129390793751, "confident_error_rate": 0.028546712802768166, "coverage_at_0_9": 0.6029411764705882, "accuracy_at_0_9": 0.9526542324246772, "coverage_at_5pct_error": 0.6176470588235294, "coverage_at_1pct_error": 0.0025951557093425604, "confidence_bias": 0.03943404634581105, "top_bins": { "0.9": { "n": 697, "errors": 33, "error_rate": 0.047345767575322814 }, "0.95": { "n": 610, "errors": 29, "error_rate": 0.047540983606557376 }, "0.99": { "n": 375, "errors": 20, "error_rate": 0.05333333333333334 } }, "selective": { "0.5": { "coverage": 0.5025951557093425, "accuracy": 0.9518072289156626, "confidence_cutoff": 0.96 }, "0.8": { "coverage": 0.8001730103806228, "accuracy": 0.892972972972973, "confidence_cutoff": 0.6799999999999999 } }, "score_mae": 0.374625, "ranked_probability_score": 0.11864749999999999 }, "metric_policy": { "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 9, "returned_zeros": 2336 }, "coverage": { "requested_records": 1264, "requested_questions": 1264, "evaluated_records": 1264, "evaluated_questions": 1264, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 231.56520850004745, "p95": 476.2518036500551 }, "suite_sha256": "3c4f0be94509a3612678bfd3a30fd99a8d0ca3c47ddfe7318075d95b2fa365e4", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-20T20:18:25.718264+00:00", "calls": 1282, "input_tokens": 530442, "output_tokens": 52174, "retries_after_5xx": 18, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.022278564, "budget_usd": 0.2, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-typesafe-v1/report.json", "source_sha256": "70a618e79b9739060b106e926d845880fbe5416ddb0016d701632978078706bc", "matching_panels": [ "kev/external/typesafe-v1/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.29238473166630585, "paired_flip": null, "unknowable": null, "clean": { "n": 102, "nll": 0.2883613963796792, "acc": 0.8921568627450981, "ece": 0.05490196078431374, "brier": 0.16740588235294115, "mean_conf": 0.861764705882353, "confident_error_rate": 0.00980392156862745, "coverage_at_0_9": 0.6274509803921569, "accuracy_at_0_9": 0.984375, "coverage_at_5pct_error": 0.8431372549019608, "coverage_at_1pct_error": 0.3627450980392157, "aurc": 0.02572431357860679, "error_rate_at_0_9": 0.015625, "confidence_bias": -0.0303921568627451, "top_bins": { "0.9": { "n": 64, "errors": 1, "error_rate": 0.015625 }, "0.95": { "n": 46, "errors": 1, "error_rate": 0.021739130434782608 }, "0.99": { "n": 25, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9803921568627451, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.803921568627451, "accuracy": 0.9512195121951219, "confidence_cutoff": 0.7 } } }, "tasks": { "typesafe_agent_trace_observability": { "n": 28, "nll": 0.5012926776942296, "acc": 0.7857142857142857, "ece": 0.12357142857142853, "brier": 0.31150714285714287, "mean_conf": 0.8128571428571431, "confident_error_rate": 0.03571428571428571, "coverage_at_0_9": 0.5357142857142857, "accuracy_at_0_9": 0.9333333333333333, "coverage_at_5pct_error": 0.14285714285714285, "coverage_at_1pct_error": 0.14285714285714285, "aurc": 0.11257058613479157, "error_rate_at_0_9": 0.06666666666666667, "confidence_bias": 0.027142857142857357, "top_bins": { "0.9": { "n": 15, "errors": 1, "error_rate": 0.06666666666666667 }, "0.95": { "n": 9, "errors": 1, "error_rate": 0.1111111111111111 }, "0.99": { "n": 2, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9285714285714286, "confidence_cutoff": 0.91 }, "0.8": { "coverage": 0.8214285714285714, "accuracy": 0.8695652173913043, "confidence_cutoff": 0.63 } } }, "typesafe_customer_service": { "n": 24, "nll": 0.2544543342534633, "acc": 0.9166666666666666, "ece": 0.0675, "brier": 0.13226666666666667, "mean_conf": 0.8883333333333333, "confident_error_rate": 0.0, "coverage_at_0_9": 0.75, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.9583333333333334, "coverage_at_1pct_error": 0.8333333333333334, "aurc": 0.00916188280318715, "error_rate_at_0_9": 0.0, "confidence_bias": -0.02833333333333332, "top_bins": { "0.9": { "n": 18, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 16, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 9, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.625, "accuracy": 1.0, "confidence_cutoff": 0.98 }, "0.8": { "coverage": 0.8333333333333334, "accuracy": 1.0, "confidence_cutoff": 0.6799999999999999 } } }, "typesafe_invoice_processing": { "n": 29, "nll": 0.07251665301719934, "acc": 1.0, "ece": 0.0627586206896551, "brier": 0.029710344827586208, "mean_conf": 0.9372413793103447, "confident_error_rate": 0.0, "coverage_at_0_9": 0.8620689655172413, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 1.0, "coverage_at_1pct_error": 1.0, "aurc": 0.0, "error_rate_at_0_9": 0.0, "confidence_bias": -0.06275862068965532, "top_bins": { "0.9": { "n": 25, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 19, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 13, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5517241379310345, "accuracy": 1.0, "confidence_cutoff": 0.97 }, "0.8": { "coverage": 0.8275862068965517, "accuracy": 1.0, "confidence_cutoff": 0.92 } } }, "typesafe_security_incidents": { "n": 21, "nll": 0.34127526170033123, "acc": 0.8571428571428571, "ece": 0.14380952380952383, "brier": 0.20558095238095236, "mean_conf": 0.7923809523809522, "confident_error_rate": 0.0, "coverage_at_0_9": 0.2857142857142857, "accuracy_at_0_9": 1.0, "coverage_at_5pct_error": 0.5714285714285714, "coverage_at_1pct_error": 0.5714285714285714, "aurc": 0.0394885732070606, "error_rate_at_0_9": 0.0, "confidence_bias": -0.06476190476190491, "top_bins": { "0.9": { "n": 6, "errors": 0, "error_rate": 0.0 }, "0.95": { "n": 2, "errors": 0, "error_rate": 0.0 }, "0.99": { "n": 1, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5238095238095238, "accuracy": 1.0, "confidence_cutoff": 0.81 }, "0.8": { "coverage": 0.8095238095238095, "accuracy": 0.9411764705882353, "confidence_cutoff": 0.6699999999999999 } } } }, "variants": { "clean": { "n": 102, "nll": 0.2883613963796792, "acc": 0.8921568627450981, "ece": 0.05490196078431374, "brier": 0.16740588235294115, "mean_conf": 0.861764705882353, "confident_error_rate": 0.00980392156862745, "coverage_at_0_9": 0.6274509803921569, "accuracy_at_0_9": 0.984375, "coverage_at_5pct_error": 0.8431372549019608, "coverage_at_1pct_error": 0.3627450980392157, "aurc": 0.02572431357860679, "error_rate_at_0_9": 0.015625, "confidence_bias": -0.0303921568627451, "top_bins": { "0.9": { "n": 64, "errors": 1, "error_rate": 0.015625 }, "0.95": { "n": 46, "errors": 1, "error_rate": 0.021739130434782608 }, "0.99": { "n": 25, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9803921568627451, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.803921568627451, "accuracy": 0.9512195121951219, "confidence_cutoff": 0.7 } } } }, "heldout_tasks": {}, "permutation": { "n": 0, "mean_max_delta": null, "flip_rate": null }, "temperature": 1.0, "calibrated_clean": { "n": 102, "nll": 0.2883613963796792, "acc": 0.8921568627450981, "ece": 0.05490196078431374, "brier": 0.16740588235294115, "mean_conf": 0.861764705882353, "confident_error_rate": 0.00980392156862745, "coverage_at_0_9": 0.6274509803921569, "accuracy_at_0_9": 0.984375, "coverage_at_5pct_error": 0.8431372549019608, "coverage_at_1pct_error": 0.3627450980392157, "aurc": 0.02572431357860679, "error_rate_at_0_9": 0.015625, "confidence_bias": -0.0303921568627451, "top_bins": { "0.9": { "n": 64, "errors": 1, "error_rate": 0.015625 }, "0.95": { "n": 46, "errors": 1, "error_rate": 0.021739130434782608 }, "0.99": { "n": 25, "errors": 0, "error_rate": 0.0 } }, "selective": { "0.5": { "coverage": 0.5, "accuracy": 0.9803921568627451, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.803921568627451, "accuracy": 0.9512195121951219, "confidence_cutoff": 0.7 } } }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 115 }, "coverage": { "requested_records": 102, "requested_questions": 102, "evaluated_records": 102, "evaluated_questions": 102, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 284.76103299999886, "p95": 443.2370654999959 }, "calibration": { "inference_temperature": null, "additional_temperature": 1.0, "logits_recorded": false }, "suite_sha256": "d3f8f5ae927f105b418190397409a14bbe7db629f79b4c88ea97909ff0220982", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-22T16:35:18.155763+00:00", "calls": 118, "input_tokens": 400169, "output_tokens": 3635, "retries_after_5xx": 16, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.016807098000000003, "budget_usd": 0.1, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } }, "typesafe": { "evaluated": { "rows": 102, "cases": 20, "equal_case_modal_agreement": 0.891474358974359, "equal_case_total_variation": 0.12456509004884006 }, "all_rows": { "rows": 102, "cases": 20, "equal_case_modal_agreement": 0.891474358974359, "equal_case_total_variation": 0.12456509004884006 } }, "comparison_sha256": "5c00f4a4a6f7f308e9679ae4b60cfcddc917185c5ca19c30f013f1d87f94be15" }, { "provenance": "published_by_Kev_not_rerun", "source_path": "runs/jev-wanli-v1/report.json", "source_sha256": "12a4353ed72fa72691ea3f905ec34edc6e7c9a1be51e74abd18d50675d34f52e", "matching_panels": [ "kev/external/wanli-v1/development" ], "match_basis": "original manifest SHA256; aliases require identical partition SHA256 and context", "report": { "objective": -0.9982054309282469, "paired_flip": null, "unknowable": null, "clean": { "n": 256, "nll": 0.9982054309282469, "acc": 0.7578125, "ece": 0.10804687499999995, "brier": 0.35753046875, "mean_conf": 0.86109375, "confident_error_rate": 0.08984375, "coverage_at_0_9": 0.59375, "accuracy_at_0_9": 0.8486842105263158, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.1406106523763559, "error_rate_at_0_9": 0.1513157894736842, "confidence_bias": 0.10328124999999999, "top_bins": { "0.9": { "n": 152, "errors": 23, "error_rate": 0.1513157894736842 }, "0.95": { "n": 124, "errors": 17, "error_rate": 0.13709677419354838 }, "0.99": { "n": 76, "errors": 7, "error_rate": 0.09210526315789473 } }, "selective": { "0.5": { "coverage": 0.515625, "accuracy": 0.8712121212121212, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8046875, "accuracy": 0.8155339805825242, "confidence_cutoff": 0.7 } } }, "tasks": { "wanli_nli": { "n": 256, "nll": 0.9982054309282469, "acc": 0.7578125, "ece": 0.10804687499999995, "brier": 0.35753046875, "mean_conf": 0.86109375, "confident_error_rate": 0.08984375, "coverage_at_0_9": 0.59375, "accuracy_at_0_9": 0.8486842105263158, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.1406106523763559, "error_rate_at_0_9": 0.1513157894736842, "confidence_bias": 0.10328124999999999, "top_bins": { "0.9": { "n": 152, "errors": 23, "error_rate": 0.1513157894736842 }, "0.95": { "n": 124, "errors": 17, "error_rate": 0.13709677419354838 }, "0.99": { "n": 76, "errors": 7, "error_rate": 0.09210526315789473 } }, "selective": { "0.5": { "coverage": 0.515625, "accuracy": 0.8712121212121212, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8046875, "accuracy": 0.8155339805825242, "confidence_cutoff": 0.7 } } } }, "variants": { "clean": { "n": 256, "nll": 0.9982054309282469, "acc": 0.7578125, "ece": 0.10804687499999995, "brier": 0.35753046875, "mean_conf": 0.86109375, "confident_error_rate": 0.08984375, "coverage_at_0_9": 0.59375, "accuracy_at_0_9": 0.8486842105263158, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.1406106523763559, "error_rate_at_0_9": 0.1513157894736842, "confidence_bias": 0.10328124999999999, "top_bins": { "0.9": { "n": 152, "errors": 23, "error_rate": 0.1513157894736842 }, "0.95": { "n": 124, "errors": 17, "error_rate": 0.13709677419354838 }, "0.99": { "n": 76, "errors": 7, "error_rate": 0.09210526315789473 } }, "selective": { "0.5": { "coverage": 0.515625, "accuracy": 0.8712121212121212, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8046875, "accuracy": 0.8155339805825242, "confidence_cutoff": 0.7 } } } }, "heldout_tasks": {}, "permutation": { "n": 0, "mean_max_delta": null, "flip_rate": null }, "temperature": 1.0, "calibrated_clean": { "n": 256, "nll": 0.9982054309282469, "acc": 0.7578125, "ece": 0.10804687499999995, "brier": 0.35753046875, "mean_conf": 0.86109375, "confident_error_rate": 0.08984375, "coverage_at_0_9": 0.59375, "accuracy_at_0_9": 0.8486842105263158, "coverage_at_5pct_error": 0.0, "coverage_at_1pct_error": 0.0, "aurc": 0.1406106523763559, "error_rate_at_0_9": 0.1513157894736842, "confidence_bias": 0.10328124999999999, "top_bins": { "0.9": { "n": 152, "errors": 23, "error_rate": 0.1513157894736842 }, "0.95": { "n": 124, "errors": 17, "error_rate": 0.13709677419354838 }, "0.99": { "n": 76, "errors": 7, "error_rate": 0.09210526315789473 } }, "selective": { "0.5": { "coverage": 0.515625, "accuracy": 0.8712121212121212, "confidence_cutoff": 0.94 }, "0.8": { "coverage": 0.8046875, "accuracy": 0.8155339805825242, "confidence_cutoff": 0.7 } } }, "metric_policy": { "version": 2, "selective_ties": "whole_confidence_groups", "coverage_at_error": "in-sample maximum over confidence thresholds; not a deployed error guarantee", "aurc": "right-step integral over whole confidence groups", "confident_error_rate": "high-confidence errors divided by all questions", "error_rate_at_0_9": "errors divided by questions accepted at p_max >= 0.9", "nll": "exact from logits when recorded; otherwise from floored probabilities", "nll_floor": 1e-09, "renormalize_returned_probabilities": true, "raw_sums_outside_1e_5": 0, "returned_zeros": 246 }, "coverage": { "requested_records": 256, "requested_questions": 256, "evaluated_records": 256, "evaluated_questions": 256, "rejected_records": 0, "truncated_records": 0 }, "latency_ms": { "median": 251.01577799999905, "p95": 332.90923024999904 }, "calibration": { "inference_temperature": null, "additional_temperature": 1.0, "logits_recorded": false }, "suite_sha256": "017f1261d52a4b2ee1e8bb6a15b7793c9b4bd8fef74792ca2ce62439b32a7dfe", "split": "development", "provider": { "model": "typesafe-ai/jev", "model_revision": "Gateway alias; provider revision not exposed by SDK result", "started_at": "2026-09-22T16:35:18.207658+00:00", "calls": 256, "input_tokens": 95462, "output_tokens": 11774, "retries_after_5xx": 0, "listed_input_usd_per_million": 0.042, "estimated_usd": 0.004009404, "budget_usd": 0.1, "sdk": "ai@7.0.105", "zero_data_retention_requested": true } } } ], "jevbench": { "provenance": "published_by_JevBench_not_rerun", "model": "Jev 1.13.0 (TypeSafe AI)", "source_path": "results/v1.2/jevbench-v1.2-per-task.json", "source_sha256": "c772b3b85809f483449ec2a6ae0272347371ed5f40f7faff8394ecf4ee22ae7c", "match_basis": "public task IDs at the pinned source revision", "note": "accuracy only; no probabilities published, so calibration is unavailable", "panels": { "jevbench/easy": { "questions": 48, "correct": 48, "accuracy": 1.0 }, "jevbench/original": { "questions": 72, "correct": 71, "accuracy": 0.9861111111111112 }, "jevbench/hard": { "questions": 111, "correct": 81, "accuracy": 0.7297297297297297 } } } }