{ "benchmark_name": "NullspanBench (Measured Decision & Extraction Leaderboard)", "benchmark_version": "3.3.0-measured", "headline_pool": "sealed", "open_pool_disclosure": "Open Pool (n=336) items come from data/val.jsonl produced by training/generate_dataset.py (disjoint seed range, 0% text overlap with train/cal_dev), which shares structural templates with the deterministic nullspan-ZeroDep rule engine. Therefore, the Sealed Pool (n=164: Stanford SQuAD v2.0 prose QA, 12 unseen multi-numeric legal/commercial agreements, and adversarial prompt-injection traps) is the headline NullspanBench table.", "draw_id": "Sealed P=164 Headline + Open S=336 Separate (v3.3-measured)", "timestamp": "2026-11-12T01:14:06Z", "n_total_observations": 500, "n_open_observations": 336, "n_sealed_observations": 164, "environment": { "os": "Darwin 25.5.0 (arm64)", "python": "3.14.8", "pytorch": "2.12.1", "onnxruntime": "1.31.0", "accelerator": "NVIDIA Tesla V100-SXM2-16GB (GCP Train) + Apple Silicon MPS/CPU (Local Eval)" }, "systems": [ { "id": "nullspan-hybrid-router", "name": "nullspan-Hybrid (Fast + ONNX Int8)", "author": "gnumanth", "system_type": "Hybrid System One Router", "open_weights": true, "params_m": 345.3, "footprint_mb": 331.1, "base_model": "Zero-Dep Router + gnumanth/nullspan Int8", "runtime": "Tiered JS/Python + ONNX Runtime", "input_rate_per_1m_usd": 0.002, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "986301d101206b7ce96831c7e9121f4b12d4fd996230e5f78a20c1ae19422875", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0003, "axes": { "intelligence": 94.5, "intelligence_ci95": [ 90.9, 97.6 ], "calibration": 95.5, "speed": 83.0, "cost": 93.8 }, "capability_score": 95.0, "composite_score": 91.4, "open_set_score": 100.0, "sealed_set_score": 94.5, "calibration_details": { "null_auroc": 0.9803, "null_ece": 0.04549, "null_brier": 0.0443 }, "latency": { "mean_ms": 63.0486, "p50_ms": 0.2482, "p95_ms": 247.0378, "p99_ms": 368.8361 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 94.5, "intelligence_ci95": [ 90.9, 97.6 ], "calibration": 95.5, "speed": 83.0, "cost": 93.8 }, "capability_score": 95.0, "composite_score": 91.4, "calibration_details": { "null_auroc": 0.9803, "null_ece": 0.04549, "null_brier": 0.0443 }, "latency": { "mean_ms": 63.0486, "p50_ms": 0.2482, "p95_ms": 247.0378, "p99_ms": 368.8361 }, "usd_per_1k_decisions": 0.0003 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 99.8, "cost": 93.8 }, "capability_score": 100.0, "composite_score": 98.3, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00201, "null_brier": 2e-05 }, "latency": { "mean_ms": 0.0278, "p50_ms": 0.0074, "p95_ms": 0.0408, "p99_ms": 0.4413 }, "usd_per_1k_decisions": 0.0003 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 98.2, "intelligence_ci95": [ 97.0, 99.2 ], "calibration": 98.6, "speed": 98.3, "cost": 93.8 }, "capability_score": 98.4, "composite_score": 97.2, "calibration_details": { "null_auroc": 0.99238, "null_ece": 0.01357, "null_brier": 0.01454 }, "latency": { "mean_ms": 20.6986, "p50_ms": 0.0138, "p95_ms": 164.8607, "p99_ms": 258.931 }, "usd_per_1k_decisions": 0.0003 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 97.2, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 97.3, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 85.0, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 1 }, { "id": "nullspan-zerodep-fast", "name": "nullspan-ZeroDep (Fast Engine)", "author": "gnumanth", "system_type": "Zero-Dep Algorithmic Engine", "open_weights": true, "params_m": 0.0, "footprint_mb": 0.0, "base_model": "Deterministic Clause + Lexical Pointer", "runtime": "Pure Python / Node / Browser JS", "input_rate_per_1m_usd": 0.0008, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "be3de37219d217d33904ec382248ff1502140dacd5b5acb7de4ee8eb39d92ebe", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0001, "axes": { "intelligence": 72.0, "intelligence_ci95": [ 65.2, 78.7 ], "calibration": 84.6, "speed": 86.1, "cost": 99.8 }, "capability_score": 78.3, "composite_score": 84.5, "open_set_score": 100.0, "sealed_set_score": 72.0, "calibration_details": { "null_auroc": 0.89839, "null_ece": 0.15249, "null_brier": 0.15672 }, "latency": { "mean_ms": 0.1456, "p50_ms": 0.1374, "p95_ms": 0.3377, "p99_ms": 0.8821 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 72.0, "intelligence_ci95": [ 65.2, 78.7 ], "calibration": 84.6, "speed": 86.1, "cost": 99.8 }, "capability_score": 78.3, "composite_score": 84.5, "calibration_details": { "null_auroc": 0.89839, "null_ece": 0.15249, "null_brier": 0.15672 }, "latency": { "mean_ms": 0.1456, "p50_ms": 0.1374, "p95_ms": 0.3377, "p99_ms": 0.8821 }, "usd_per_1k_decisions": 0.0001 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 99.8, "cost": 99.8 }, "capability_score": 100.0, "composite_score": 99.9, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00201, "null_brier": 2e-05 }, "latency": { "mean_ms": 0.0278, "p50_ms": 0.0074, "p95_ms": 0.0408, "p99_ms": 0.4413 }, "usd_per_1k_decisions": 0.0001 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 90.8, "intelligence_ci95": [ 88.2, 93.4 ], "calibration": 95.0, "speed": 98.4, "cost": 99.8 }, "capability_score": 92.9, "composite_score": 95.9, "calibration_details": { "null_auroc": 0.98239, "null_ece": 0.04866, "null_brier": 0.05142 }, "latency": { "mean_ms": 0.0664, "p50_ms": 0.0135, "p95_ms": 0.2901, "p99_ms": 0.4926 }, "usd_per_1k_decisions": 0.0001 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 78.7, "n": 108 }, "noul_sealed": { "score": 47.7, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 79.6, "n": 113 }, "Hard": { "score": 54.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 23.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 2 }, { "id": "nullspan-onnx-int8", "name": "nullspan-345M-Int8 (ONNX WASM/CPU)", "author": "gnumanth", "system_type": "Quantized ONNX WASM/CPU (512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 331.1, "base_model": "gnumanth/nullspan (Int8 Dynamic)", "runtime": "ONNX Runtime Int8 (Browser WASM / Edge)", "input_rate_per_1m_usd": 0.006, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "22d5544220f57ebdbc1c8047f042111348af54d7f31b6642d266f4cc2123a64f", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0008, "axes": { "intelligence": 94.5, "intelligence_ci95": [ 90.9, 97.6 ], "calibration": 95.8, "speed": 47.5, "cost": 84.2 }, "capability_score": 95.2, "composite_score": 74.1, "open_set_score": 100.0, "sealed_set_score": 94.5, "calibration_details": { "null_auroc": 0.97614, "null_ece": 0.04136, "null_brier": 0.04455 }, "latency": { "mean_ms": 203.0401, "p50_ms": 201.6991, "p95_ms": 292.1634, "p99_ms": 368.8361 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 94.5, "intelligence_ci95": [ 90.9, 97.6 ], "calibration": 95.8, "speed": 47.5, "cost": 84.2 }, "capability_score": 95.2, "composite_score": 74.1, "calibration_details": { "null_auroc": 0.97614, "null_ece": 0.04136, "null_brier": 0.04455 }, "latency": { "mean_ms": 203.0401, "p50_ms": 201.6991, "p95_ms": 292.1634, "p99_ms": 368.8361 }, "usd_per_1k_decisions": 0.0008 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 51.1, "cost": 84.2 }, "capability_score": 100.0, "composite_score": 77.7, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00222, "null_brier": 2e-05 }, "latency": { "mean_ms": 110.4293, "p50_ms": 101.0107, "p95_ms": 169.4236, "p99_ms": 241.4742 }, "usd_per_1k_decisions": 0.0008 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 98.2, "intelligence_ci95": [ 97.0, 99.2 ], "calibration": 98.7, "speed": 50.8, "cost": 84.2 }, "capability_score": 98.5, "composite_score": 77.1, "calibration_details": { "null_auroc": 0.99166, "null_ece": 0.01207, "null_brier": 0.01462 }, "latency": { "mean_ms": 140.8057, "p50_ms": 107.0085, "p95_ms": 264.4929, "p99_ms": 292.3606 }, "usd_per_1k_decisions": 0.0008 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 97.2, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 97.3, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 85.0, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 3 }, { "id": "nullspan-fp32", "name": "nullspan-345M (PyTorch FP32)", "author": "gnumanth", "system_type": "System One Encoder (5-Head, 512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 1317.2, "base_model": "ahotrod/electra_large_discriminator_squad2_512 + SchemaAdapter", "runtime": "PyTorch MPS/CUDA/CPU FP32", "input_rate_per_1m_usd": 0.012, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "91f8dd7ef01eb004f9c3e07eca2c0ca7705f6316f3f2126f6a59533f8a87fb87", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0017, "axes": { "intelligence": 93.9, "intelligence_ci95": [ 90.2, 97.0 ], "calibration": 95.3, "speed": 49.9, "cost": 76.8 }, "capability_score": 94.6, "composite_score": 73.8, "open_set_score": 100.0, "sealed_set_score": 93.9, "calibration_details": { "null_auroc": 0.98542, "null_ece": 0.04785, "null_brier": 0.04668 }, "latency": { "mean_ms": 128.5135, "p50_ms": 127.6058, "p95_ms": 175.881, "p99_ms": 254.4876 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 93.9, "intelligence_ci95": [ 90.2, 97.0 ], "calibration": 95.3, "speed": 49.9, "cost": 76.8 }, "capability_score": 94.6, "composite_score": 73.8, "calibration_details": { "null_auroc": 0.98542, "null_ece": 0.04785, "null_brier": 0.04668 }, "latency": { "mean_ms": 128.5135, "p50_ms": 127.6058, "p95_ms": 175.881, "p99_ms": 254.4876 }, "usd_per_1k_decisions": 0.0017 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 54.9, "cost": 76.8 }, "capability_score": 100.0, "composite_score": 78.1, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00225, "null_brier": 2e-05 }, "latency": { "mean_ms": 66.2503, "p50_ms": 49.5557, "p95_ms": 107.1403, "p99_ms": 275.8887 }, "usd_per_1k_decisions": 0.0017 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 98.0, "intelligence_ci95": [ 96.8, 99.2 ], "calibration": 98.5, "speed": 54.3, "cost": 76.8 }, "capability_score": 98.2, "composite_score": 77.2, "calibration_details": { "null_auroc": 0.99327, "null_ece": 0.01419, "null_brier": 0.01532 }, "latency": { "mean_ms": 86.6726, "p50_ms": 55.206, "p95_ms": 160.6231, "p99_ms": 270.4781 }, "usd_per_1k_decisions": 0.0017 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 83.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 4 }, { "id": "nullspan-onnx-fp32", "name": "nullspan-345M (ONNX FP32)", "author": "gnumanth", "system_type": "System One Encoder (ONNX, 512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 1313.6, "base_model": "gnumanth/nullspan (ONNX Opset 17)", "runtime": "ONNX Runtime FP32", "input_rate_per_1m_usd": 0.011, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "7a1b74a627789f05d069229df78fb8153257a79c82c2c2e32dfe5de7411a1f2f", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0015, "axes": { "intelligence": 93.9, "intelligence_ci95": [ 90.2, 97.0 ], "calibration": 96.5, "speed": 47.6, "cost": 78.0 }, "capability_score": 95.2, "composite_score": 72.9, "open_set_score": 100.0, "sealed_set_score": 93.9, "calibration_details": { "null_auroc": 0.99015, "null_ece": 0.03405, "null_brier": 0.03793 }, "latency": { "mean_ms": 188.3131, "p50_ms": 196.4426, "p95_ms": 266.5202, "p99_ms": 337.8913 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 93.9, "intelligence_ci95": [ 90.2, 97.0 ], "calibration": 96.5, "speed": 47.6, "cost": 78.0 }, "capability_score": 95.2, "composite_score": 72.9, "calibration_details": { "null_auroc": 0.99015, "null_ece": 0.03405, "null_brier": 0.03793 }, "latency": { "mean_ms": 188.3131, "p50_ms": 196.4426, "p95_ms": 266.5202, "p99_ms": 337.8913 }, "usd_per_1k_decisions": 0.0015 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 51.7, "cost": 78.0 }, "capability_score": 100.0, "composite_score": 76.7, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00225, "null_brier": 2e-05 }, "latency": { "mean_ms": 98.6641, "p50_ms": 90.9101, "p95_ms": 164.1157, "p99_ms": 219.525 }, "usd_per_1k_decisions": 0.0015 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 98.0, "intelligence_ci95": [ 96.8, 99.2 ], "calibration": 98.8, "speed": 51.3, "cost": 78.0 }, "capability_score": 98.4, "composite_score": 76.0, "calibration_details": { "null_auroc": 0.99829, "null_ece": 0.01215, "null_brier": 0.01245 }, "latency": { "mean_ms": 128.069, "p50_ms": 97.6215, "p95_ms": 235.067, "p99_ms": 266.5202 }, "usd_per_1k_decisions": 0.0015 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 83.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 5 }, { "id": "cloudflare-clairic-clef-live", "name": "@cf/cloudflare/clairic + clef (Live Workers AI)", "author": "Cloudflare", "system_type": "System One Split API (Clairic + Clef 27B)", "open_weights": false, "params_m": 27000.0, "footprint_mb": null, "base_model": "@cf/cloudflare/clairic + @cf/cloudflare/clef (180 RPCs / 246 Passes)", "runtime": "Cloudflare Workers AI Edge RPC (p50 315.3 ms/doc)", "input_rate_per_1m_usd": 0.025, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "378692149bafcdea8d8e7e9d3bae2ed81aa6e71e8d14767a4838b9426910f5c0", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 81206, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0041, "axes": { "intelligence": 95.4, "intelligence_ci95": [ 92.1, 98.2 ], "calibration": 97.8, "speed": 50.5, "cost": 68.2 }, "capability_score": 96.6, "composite_score": 72.5, "open_set_score": 89.7, "sealed_set_score": 95.4, "calibration_details": { "null_auroc": 0.99242, "null_ece": 0.02146, "null_brier": 0.02327 }, "latency": { "mean_ms": 167.0995, "p50_ms": 113.8012, "p95_ms": 373.1545, "p99_ms": 460.0218 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 95.4, "intelligence_ci95": [ 92.1, 98.2 ], "calibration": 97.8, "speed": 50.5, "cost": 68.2 }, "capability_score": 96.6, "composite_score": 72.5, "calibration_details": { "null_auroc": 0.99242, "null_ece": 0.02146, "null_brier": 0.02327 }, "latency": { "mean_ms": 167.0995, "p50_ms": 113.8012, "p95_ms": 373.1545, "p99_ms": 460.0218 }, "usd_per_1k_decisions": 0.0041 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 89.7, "intelligence_ci95": [ 86.5, 92.8 ], "calibration": 92.8, "speed": 52.0, "cost": 68.2 }, "capability_score": 91.2, "composite_score": 71.7, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.07908, "null_brier": 0.05899 }, "latency": { "mean_ms": 110.0899, "p50_ms": 86.1611, "p95_ms": 272.4725, "p99_ms": 360.5915 }, "usd_per_1k_decisions": 0.0041 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 91.6, "intelligence_ci95": [ 89.2, 93.9 ], "calibration": 94.9, "speed": 51.7, "cost": 68.2 }, "capability_score": 93.2, "composite_score": 72.1, "calibration_details": { "null_auroc": 0.98959, "null_ece": 0.05235, "null_brier": 0.04728 }, "latency": { "mean_ms": 128.789, "p50_ms": 91.323, "p95_ms": 314.023, "p99_ms": 403.1387 }, "usd_per_1k_decisions": 0.0041 }, "by_request_type": { "span_open": { "score": 83.8, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 89.0, "n": 48 }, "span_sealed": { "score": 95.8, "n": 108 }, "noul_sealed": { "score": 93.2, "n": 44 } }, "by_tier_open": { "Easy": { "score": 85.8, "n": 176 }, "Judge": { "score": 94.5, "n": 96 }, "Standard": { "score": 93.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.9, "n": 113 }, "Hard": { "score": 92.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 94.4, "n": 160 }, "compute_ledger": { "score": 82.6, "n": 144 }, "cluster_throughput_comps": { "score": 96.2, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 92.5, "n": 60 }, "contract_extraction": { "score": 97.9, "n": 96 }, "adversarial_stress": { "score": 87.5, "n": 8 } }, "official_rank": 6 }, { "id": "deberta-v3-base-squad2", "name": "deberta-v3-base-squad2 (184M SQuAD2 QA)", "author": "deepset", "system_type": "Open Weights \u00b7 Extractive QA Cross-Encoder", "open_weights": true, "params_m": 183.8, "footprint_mb": 701.3, "base_model": "deepset/deberta-v3-base-squad2", "runtime": "PyTorch MPS/CPU (512-ctx)", "input_rate_per_1m_usd": 0.032, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "c46f41c01931e088e7e4900a96be1b9a22a4b2276f5031dc7f08f1c2907ae0bd", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 57964, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0037, "axes": { "intelligence": 47.3, "intelligence_ci95": [ 39.3, 54.9 ], "calibration": 63.8, "speed": 52.8, "cost": 69.2 }, "capability_score": 55.5, "composite_score": 57.0, "open_set_score": 48.6, "sealed_set_score": 47.3, "calibration_details": { "null_auroc": 0.82121, "null_ece": 0.37927, "null_brier": 0.33034 }, "latency": { "mean_ms": 112.8393, "p50_ms": 74.1332, "p95_ms": 376.7113, "p99_ms": 376.7113 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 47.3, "intelligence_ci95": [ 39.3, 54.9 ], "calibration": 63.8, "speed": 52.8, "cost": 69.2 }, "capability_score": 55.5, "composite_score": 57.0, "calibration_details": { "null_auroc": 0.82121, "null_ece": 0.37927, "null_brier": 0.33034 }, "latency": { "mean_ms": 112.8393, "p50_ms": 74.1332, "p95_ms": 376.7113, "p99_ms": 376.7113 }, "usd_per_1k_decisions": 0.0037 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 48.6, "intelligence_ci95": [ 43.2, 54.0 ], "calibration": 62.2, "speed": 56.2, "cost": 69.2 }, "capability_score": 55.4, "composite_score": 58.1, "calibration_details": { "null_auroc": 0.82668, "null_ece": 0.4174, "null_brier": 0.30522 }, "latency": { "mean_ms": 54.7889, "p50_ms": 38.8237, "p95_ms": 116.8072, "p99_ms": 371.4963 }, "usd_per_1k_decisions": 0.0037 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 48.1, "intelligence_ci95": [ 43.9, 52.4 ], "calibration": 63.4, "speed": 54.7, "cost": 69.2 }, "capability_score": 55.8, "composite_score": 57.7, "calibration_details": { "null_auroc": 0.82087, "null_ece": 0.39493, "null_brier": 0.31346 }, "latency": { "mean_ms": 73.8294, "p50_ms": 51.2342, "p95_ms": 371.4963, "p99_ms": 376.7113 }, "usd_per_1k_decisions": 0.0037 }, "by_request_type": { "span_open": { "score": 56.4, "n": 181 }, "math": { "score": 4.9, "n": 51 }, "choice": { "score": 65.9, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 15.1, "n": 48 }, "span_sealed": { "score": 35.2, "n": 108 }, "noul_sealed": { "score": 84.1, "n": 44 } }, "by_tier_open": { "Easy": { "score": 64.2, "n": 176 }, "Judge": { "score": 9.6, "n": 96 }, "Standard": { "score": 64.1, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 33.6, "n": 113 }, "Hard": { "score": 78.0, "n": 50 }, "Judge": { "score": 50.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 50.0, "n": 160 }, "compute_ledger": { "score": 57.6, "n": 144 }, "cluster_throughput_comps": { "score": 1.3, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 80.0, "n": 60 }, "contract_extraction": { "score": 27.6, "n": 96 }, "adversarial_stress": { "score": 37.5, "n": 8 } }, "official_rank": 7 }, { "id": "all-MiniLM-L12-v2", "name": "all-MiniLM-L12-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 33.4, "footprint_mb": 127.3, "base_model": "sentence-transformers/all-MiniLM-L12-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.01, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "cc6be4a4a7811f54a8cd82a63d66b0dfc8659215e8b05aff0d4a77b45c9b1f54", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0007, "axes": { "intelligence": 8.8, "intelligence_ci95": [ 4.9, 13.4 ], "calibration": 64.5, "speed": 64.6, "cost": 85.5 }, "capability_score": 36.6, "composite_score": 25.6, "open_set_score": 17.2, "sealed_set_score": 8.8, "calibration_details": { "null_auroc": 0.42254, "null_ece": 0.35654, "null_brier": 0.35108 }, "latency": { "mean_ms": 8.14, "p50_ms": 7.9056, "p95_ms": 10.353, "p99_ms": 18.128 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 8.8, "intelligence_ci95": [ 4.9, 13.4 ], "calibration": 64.5, "speed": 64.6, "cost": 85.5 }, "capability_score": 36.6, "composite_score": 25.6, "calibration_details": { "null_auroc": 0.42254, "null_ece": 0.35654, "null_brier": 0.35108 }, "latency": { "mean_ms": 8.14, "p50_ms": 7.9056, "p95_ms": 10.353, "p99_ms": 18.128 }, "usd_per_1k_decisions": 0.0007 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 17.2, "intelligence_ci95": [ 13.5, 20.9 ], "calibration": 76.5, "speed": 65.1, "cost": 85.5 }, "capability_score": 46.9, "composite_score": 40.7, "calibration_details": { "null_auroc": 0.82201, "null_ece": 0.26802, "null_brier": 0.17378 }, "latency": { "mean_ms": 7.4393, "p50_ms": 7.28, "p95_ms": 9.3107, "p99_ms": 16.1564 }, "usd_per_1k_decisions": 0.0007 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 14.4, "intelligence_ci95": [ 11.7, 17.5 ], "calibration": 73.5, "speed": 65.0, "cost": 85.5 }, "capability_score": 44.0, "composite_score": 36.3, "calibration_details": { "null_auroc": 0.58938, "null_ece": 0.28318, "null_brier": 0.23193 }, "latency": { "mean_ms": 7.6692, "p50_ms": 7.459, "p95_ms": 9.7918, "p99_ms": 17.8974 }, "usd_per_1k_decisions": 0.0007 }, "by_request_type": { "span_open": { "score": 9.4, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 28.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 6.7, "n": 48 }, "span_sealed": { "score": 0.5, "n": 108 }, "noul_sealed": { "score": 29.5, "n": 44 } }, "by_tier_open": { "Easy": { "score": 25.0, "n": 176 }, "Judge": { "score": 3.3, "n": 96 }, "Standard": { "score": 16.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 0.9, "n": 113 }, "Hard": { "score": 27.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 19.1, "n": 160 }, "compute_ledger": { "score": 16.7, "n": 144 }, "cluster_throughput_comps": { "score": 20.0, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 1.7, "n": 60 }, "contract_extraction": { "score": 11.5, "n": 96 }, "adversarial_stress": { "score": 31.2, "n": 8 } }, "official_rank": 8 }, { "id": "all-mpnet-base-v2", "name": "all-mpnet-base-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 109.5, "footprint_mb": 417.7, "base_model": "sentence-transformers/all-mpnet-base-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.022, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "7f8ac57f8823a1bee4d47ce89f9d612fbe9f4980b1fc7584193d281a61b03282", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0016, "axes": { "intelligence": 8.2, "intelligence_ci95": [ 4.3, 12.8 ], "calibration": 64.2, "speed": 62.2, "cost": 77.4 }, "capability_score": 36.2, "composite_score": 24.0, "open_set_score": 16.2, "sealed_set_score": 8.2, "calibration_details": { "null_auroc": 0.43182, "null_ece": 0.36572, "null_brier": 0.3441 }, "latency": { "mean_ms": 15.1577, "p50_ms": 12.5622, "p95_ms": 26.3503, "p99_ms": 59.4068 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 8.2, "intelligence_ci95": [ 4.3, 12.8 ], "calibration": 64.2, "speed": 62.2, "cost": 77.4 }, "capability_score": 36.2, "composite_score": 24.0, "calibration_details": { "null_auroc": 0.43182, "null_ece": 0.36572, "null_brier": 0.3441 }, "latency": { "mean_ms": 15.1577, "p50_ms": 12.5622, "p95_ms": 26.3503, "p99_ms": 59.4068 }, "usd_per_1k_decisions": 0.0016 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 16.2, "intelligence_ci95": [ 12.5, 19.9 ], "calibration": 72.4, "speed": 62.9, "cost": 77.4 }, "capability_score": 44.3, "composite_score": 38.3, "calibration_details": { "null_auroc": 0.78916, "null_ece": 0.30333, "null_brier": 0.22501 }, "latency": { "mean_ms": 18.4708, "p50_ms": 11.0142, "p95_ms": 18.1759, "p99_ms": 175.4313 }, "usd_per_1k_decisions": 0.0016 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 13.6, "intelligence_ci95": [ 10.9, 16.7 ], "calibration": 71.0, "speed": 62.7, "cost": 77.4 }, "capability_score": 42.3, "composite_score": 34.3, "calibration_details": { "null_auroc": 0.58731, "null_ece": 0.30424, "null_brier": 0.26407 }, "latency": { "mean_ms": 17.3841, "p50_ms": 11.4679, "p95_ms": 21.0789, "p99_ms": 144.4912 }, "usd_per_1k_decisions": 0.0016 }, "by_request_type": { "span_open": { "score": 9.4, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 28.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 0.0, "n": 48 }, "span_sealed": { "score": 0.5, "n": 108 }, "noul_sealed": { "score": 27.3, "n": 44 } }, "by_tier_open": { "Easy": { "score": 25.0, "n": 176 }, "Judge": { "score": 0.0, "n": 96 }, "Standard": { "score": 16.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 0.9, "n": 113 }, "Hard": { "score": 25.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 19.1, "n": 160 }, "compute_ledger": { "score": 16.7, "n": 144 }, "cluster_throughput_comps": { "score": 0.0, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 0.0, "n": 60 }, "contract_extraction": { "score": 11.5, "n": 96 }, "adversarial_stress": { "score": 31.2, "n": 8 } }, "official_rank": 9 }, { "id": "paraphrase-MiniLM-L3-v2", "name": "paraphrase-MiniLM-L3-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 17.4, "footprint_mb": 66.3, "base_model": "sentence-transformers/paraphrase-MiniLM-L3-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.005, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "cafe97bf5b136b8d0e9dde1c3a846a75914aa1b0a544b277393ca995592a7901", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0004, "axes": { "intelligence": 7.6, "intelligence_ci95": [ 4.0, 11.9 ], "calibration": 63.7, "speed": 70.4, "cost": 91.0 }, "capability_score": 35.6, "composite_score": 23.2, "open_set_score": 12.9, "sealed_set_score": 7.6, "calibration_details": { "null_auroc": 0.45587, "null_ece": 0.36995, "null_brier": 0.35112 }, "latency": { "mean_ms": 3.0192, "p50_ms": 2.6538, "p95_ms": 4.8318, "p99_ms": 9.5178 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 7.6, "intelligence_ci95": [ 4.0, 11.9 ], "calibration": 63.7, "speed": 70.4, "cost": 91.0 }, "capability_score": 35.6, "composite_score": 23.2, "calibration_details": { "null_auroc": 0.45587, "null_ece": 0.36995, "null_brier": 0.35112 }, "latency": { "mean_ms": 3.0192, "p50_ms": 2.6538, "p95_ms": 4.8318, "p99_ms": 9.5178 }, "usd_per_1k_decisions": 0.0004 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 12.9, "intelligence_ci95": [ 9.7, 16.1 ], "calibration": 74.0, "speed": 70.4, "cost": 91.0 }, "capability_score": 43.5, "composite_score": 34.4, "calibration_details": { "null_auroc": 0.78713, "null_ece": 0.29056, "null_brier": 0.20327 }, "latency": { "mean_ms": 2.9532, "p50_ms": 2.6466, "p95_ms": 4.7012, "p99_ms": 8.5112 }, "usd_per_1k_decisions": 0.0004 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 11.1, "intelligence_ci95": [ 8.7, 13.8 ], "calibration": 72.3, "speed": 70.4, "cost": 91.0 }, "capability_score": 41.7, "composite_score": 31.0, "calibration_details": { "null_auroc": 0.58932, "null_ece": 0.29025, "null_brier": 0.25176 }, "latency": { "mean_ms": 2.9749, "p50_ms": 2.6491, "p95_ms": 4.7267, "p99_ms": 9.8108 }, "usd_per_1k_decisions": 0.0004 }, "by_request_type": { "span_open": { "score": 9.7, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 28.0, "n": 41 }, "noul_open": { "score": 44.4, "n": 27 }, "score": { "score": 6.7, "n": 48 }, "span_sealed": { "score": 0.5, "n": 108 }, "noul_sealed": { "score": 25.0, "n": 44 } }, "by_tier_open": { "Easy": { "score": 16.8, "n": 176 }, "Judge": { "score": 3.3, "n": 96 }, "Standard": { "score": 16.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 0.9, "n": 113 }, "Hard": { "score": 23.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 17.5, "n": 160 }, "compute_ledger": { "score": 8.3, "n": 144 }, "cluster_throughput_comps": { "score": 20.0, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 1.7, "n": 60 }, "contract_extraction": { "score": 10.4, "n": 96 }, "adversarial_stress": { "score": 18.8, "n": 8 } }, "official_rank": 10 }, { "id": "all-MiniLM-L6-v2", "name": "all-MiniLM-L6-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 22.7, "footprint_mb": 86.6, "base_model": "sentence-transformers/all-MiniLM-L6-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.007, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "cb306daa6a5967d18053fdb1544ae61f9d00011c3c8d899149e863155ddd15da", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0005, "axes": { "intelligence": 7.6, "intelligence_ci95": [ 4.0, 11.9 ], "calibration": 61.7, "speed": 68.2, "cost": 88.8 }, "capability_score": 34.6, "composite_score": 23.0, "open_set_score": 11.2, "sealed_set_score": 7.6, "calibration_details": { "null_auroc": 0.42756, "null_ece": 0.38988, "null_brier": 0.36948 }, "latency": { "mean_ms": 4.4962, "p50_ms": 4.0641, "p95_ms": 7.0286, "p99_ms": 8.7059 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 7.6, "intelligence_ci95": [ 4.0, 11.9 ], "calibration": 61.7, "speed": 68.2, "cost": 88.8 }, "capability_score": 34.6, "composite_score": 23.0, "calibration_details": { "null_auroc": 0.42756, "null_ece": 0.38988, "null_brier": 0.36948 }, "latency": { "mean_ms": 4.4962, "p50_ms": 4.0641, "p95_ms": 7.0286, "p99_ms": 8.7059 }, "usd_per_1k_decisions": 0.0005 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 11.2, "intelligence_ci95": [ 8.0, 14.1 ], "calibration": 73.7, "speed": 67.3, "cost": 88.8 }, "capability_score": 42.5, "composite_score": 31.0, "calibration_details": { "null_auroc": 0.7785, "null_ece": 0.29091, "null_brier": 0.21136 }, "latency": { "mean_ms": 5.2658, "p50_ms": 4.746, "p95_ms": 7.4978, "p99_ms": 12.6273 }, "usd_per_1k_decisions": 0.0005 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 10.0, "intelligence_ci95": [ 7.7, 12.3 ], "calibration": 70.9, "speed": 67.7, "cost": 88.8 }, "capability_score": 40.5, "composite_score": 28.5, "calibration_details": { "null_auroc": 0.56307, "null_ece": 0.30616, "null_brier": 0.26323 }, "latency": { "mean_ms": 5.0134, "p50_ms": 4.416, "p95_ms": 7.4636, "p99_ms": 10.5634 }, "usd_per_1k_decisions": 0.0005 }, "by_request_type": { "span_open": { "score": 8.3, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 28.0, "n": 41 }, "noul_open": { "score": 44.4, "n": 27 }, "score": { "score": 0.0, "n": 48 }, "span_sealed": { "score": 0.5, "n": 108 }, "noul_sealed": { "score": 25.0, "n": 44 } }, "by_tier_open": { "Easy": { "score": 15.3, "n": 176 }, "Judge": { "score": 0.0, "n": 96 }, "Standard": { "score": 16.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 0.9, "n": 113 }, "Hard": { "score": 23.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 19.1, "n": 160 }, "compute_ledger": { "score": 4.9, "n": 144 }, "cluster_throughput_comps": { "score": 0.0, "n": 16 }, "edge_traffic_forecast": { "score": 0.0, "n": 16 }, "squad_v2_open_qa": { "score": 1.7, "n": 60 }, "contract_extraction": { "score": 10.4, "n": 96 }, "adversarial_stress": { "score": 18.8, "n": 8 } }, "official_rank": 11 }, { "id": "claude-haiku-4.5-json", "name": "claude-haiku-4.5 (JSON Decoder \u00b7 API)", "author": "Anthropic", "system_type": "Closed API \u00b7 LLM Decoder", "open_weights": false, "params_m": null, "footprint_mb": null, "base_model": "claude-haiku-4-5 (Strict JSON Tool Schema)", "runtime": "Live Cloud JSON-Schema API (211.8 output tokens/field)", "input_rate_per_1m_usd": 1.0, "output_rate_per_1m_usd": 5.0, "measured": true, "predictions_sha256": "d1fb4e903ee98334a5cda0511c89f6c04ceb67661d7a35c8f400a1b28f25af08", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 752582, "output_tokens_total": 105913, "usd_per_1k_decisions": 2.5643, "axes": { "intelligence": 88.4, "intelligence_ci95": [ 83.5, 93.3 ], "calibration": 91.3, "speed": 35.9, "cost": 8.0 }, "capability_score": 89.8, "composite_score": 22.8, "open_set_score": 93.7, "sealed_set_score": 88.4, "calibration_details": { "null_auroc": 0.95284, "null_ece": 0.08506, "null_brier": 0.09115 }, "latency": { "mean_ms": 2030.4761, "p50_ms": 1811.16, "p95_ms": 3583.842, "p99_ms": 5955.9195 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 88.4, "intelligence_ci95": [ 83.5, 93.3 ], "calibration": 91.3, "speed": 35.9, "cost": 8.0 }, "capability_score": 89.8, "composite_score": 22.8, "calibration_details": { "null_auroc": 0.95284, "null_ece": 0.08506, "null_brier": 0.09115 }, "latency": { "mean_ms": 2030.4761, "p50_ms": 1811.16, "p95_ms": 3583.842, "p99_ms": 5955.9195 }, "usd_per_1k_decisions": 2.5643 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 93.7, "intelligence_ci95": [ 91.3, 96.0 ], "calibration": 93.1, "speed": 36.0, "cost": 8.0 }, "capability_score": 93.4, "composite_score": 23.0, "calibration_details": { "null_auroc": 0.99712, "null_ece": 0.08408, "null_brier": 0.04084 }, "latency": { "mean_ms": 1808.6166, "p50_ms": 1769.33, "p95_ms": 2433.9525, "p99_ms": 4903.8135 }, "usd_per_1k_decisions": 2.5643 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 92.0, "intelligence_ci95": [ 89.6, 94.2 ], "calibration": 94.8, "speed": 35.9, "cost": 8.0 }, "capability_score": 93.4, "composite_score": 23.0, "calibration_details": { "null_auroc": 0.94688, "null_ece": 0.0486, "null_brier": 0.05734 }, "latency": { "mean_ms": 1881.3865, "p50_ms": 1780.625, "p95_ms": 2789.522, "p99_ms": 5929.9393 }, "usd_per_1k_decisions": 2.5643 }, "by_request_type": { "span_open": { "score": 91.2, "n": 181 }, "math": { "score": 98.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 91.4, "n": 48 }, "span_sealed": { "score": 97.2, "n": 108 }, "noul_sealed": { "score": 63.6, "n": 44 } }, "by_tier_open": { "Easy": { "score": 91.5, "n": 176 }, "Judge": { "score": 94.6, "n": 96 }, "Standard": { "score": 98.4, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 98.2, "n": 113 }, "Hard": { "score": 66.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 97.2, "n": 160 }, "compute_ledger": { "score": 89.6, "n": 144 }, "cluster_throughput_comps": { "score": 89.8, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 70.0, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 87.5, "n": 8 } }, "official_rank": 12 }, { "id": "paraphrase-albert-small-v2", "name": "paraphrase-albert-small-v2 (Dense Span+Null)", "author": "sentence-transformers", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 11.7, "footprint_mb": 44.6, "base_model": "sentence-transformers/paraphrase-albert-small-v2", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.005, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "7f1c86b657b934f8edc6b2a52ae3e20df99d831c9ddb51ac3dacc06822176507", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0004, "axes": { "intelligence": 6.7, "intelligence_ci95": [ 3.4, 10.7 ], "calibration": 67.6, "speed": 63.9, "cost": 91.0 }, "capability_score": 37.1, "composite_score": 21.0, "open_set_score": 15.3, "sealed_set_score": 6.7, "calibration_details": { "null_auroc": 0.43845, "null_ece": 0.32478, "null_brier": 0.32203 }, "latency": { "mean_ms": 11.8891, "p50_ms": 9.144, "p95_ms": 45.3988, "p99_ms": 60.6356 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 6.7, "intelligence_ci95": [ 3.4, 10.7 ], "calibration": 67.6, "speed": 63.9, "cost": 91.0 }, "capability_score": 37.1, "composite_score": 21.0, "calibration_details": { "null_auroc": 0.43845, "null_ece": 0.32478, "null_brier": 0.32203 }, "latency": { "mean_ms": 11.8891, "p50_ms": 9.144, "p95_ms": 45.3988, "p99_ms": 60.6356 }, "usd_per_1k_decisions": 0.0004 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 15.3, "intelligence_ci95": [ 12.0, 18.7 ], "calibration": 79.4, "speed": 66.5, "cost": 91.0 }, "capability_score": 47.4, "composite_score": 38.5, "calibration_details": { "null_auroc": 0.78857, "null_ece": 0.23047, "null_brier": 0.16114 }, "latency": { "mean_ms": 7.0892, "p50_ms": 5.6208, "p95_ms": 8.8297, "p99_ms": 54.0583 }, "usd_per_1k_decisions": 0.0004 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 12.5, "intelligence_ci95": [ 10.0, 15.2 ], "calibration": 76.6, "speed": 65.7, "cost": 91.0 }, "capability_score": 44.5, "composite_score": 33.5, "calibration_details": { "null_auroc": 0.58482, "null_ece": 0.24547, "null_brier": 0.21391 }, "latency": { "mean_ms": 8.6636, "p50_ms": 6.5199, "p95_ms": 13.5208, "p99_ms": 62.1678 }, "usd_per_1k_decisions": 0.0004 }, "by_request_type": { "span_open": { "score": 11.0, "n": 181 }, "math": { "score": 1.0, "n": 51 }, "choice": { "score": 35.4, "n": 41 }, "noul_open": { "score": 51.9, "n": 27 }, "score": { "score": 8.4, "n": 48 }, "span_sealed": { "score": 2.3, "n": 108 }, "noul_sealed": { "score": 15.9, "n": 44 } }, "by_tier_open": { "Easy": { "score": 19.3, "n": 176 }, "Judge": { "score": 4.2, "n": 96 }, "Standard": { "score": 21.1, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 2.7, "n": 113 }, "Hard": { "score": 15.0, "n": 50 }, "Judge": { "score": 50.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 21.1, "n": 160 }, "compute_ledger": { "score": 9.7, "n": 144 }, "cluster_throughput_comps": { "score": 20.0, "n": 16 }, "edge_traffic_forecast": { "score": 3.4, "n": 16 }, "squad_v2_open_qa": { "score": 0.0, "n": 60 }, "contract_extraction": { "score": 9.9, "n": 96 }, "adversarial_stress": { "score": 18.8, "n": 8 } }, "official_rank": 13 }, { "id": "bge-small-en-v1.5", "name": "bge-small-en-v1.5 (Dense Span+Null)", "author": "BAAI", "system_type": "Open Weights \u00b7 Bi-Encoder", "open_weights": true, "params_m": 33.4, "footprint_mb": 127.3, "base_model": "BAAI/bge-small-en-v1.5", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.01, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "ab7f52e9e9dde3ac226bd44564e81217a8d8118c9226185ee29664f71a006f4a", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0007, "axes": { "intelligence": 3.4, "intelligence_ci95": [ 0.9, 6.1 ], "calibration": 75.2, "speed": 65.3, "cost": 85.5 }, "capability_score": 39.3, "composite_score": 12.0, "open_set_score": 20.3, "sealed_set_score": 3.4, "calibration_details": { "null_auroc": 0.51752, "null_ece": 0.24512, "null_brier": 0.25353 }, "latency": { "mean_ms": 8.906, "p50_ms": 7.0485, "p95_ms": 24.5518, "p99_ms": 37.8159 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 3.4, "intelligence_ci95": [ 0.9, 6.1 ], "calibration": 75.2, "speed": 65.3, "cost": 85.5 }, "capability_score": 39.3, "composite_score": 12.0, "calibration_details": { "null_auroc": 0.51752, "null_ece": 0.24512, "null_brier": 0.25353 }, "latency": { "mean_ms": 8.906, "p50_ms": 7.0485, "p95_ms": 24.5518, "p99_ms": 37.8159 }, "usd_per_1k_decisions": 0.0007 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 20.3, "intelligence_ci95": [ 16.8, 23.9 ], "calibration": 96.0, "speed": 64.6, "cost": 85.5 }, "capability_score": 58.1, "composite_score": 46.1, "calibration_details": { "null_auroc": 0.85617, "null_ece": 0.03732, "null_brier": 0.04491 }, "latency": { "mean_ms": 15.6824, "p50_ms": 8.0241, "p95_ms": 17.5403, "p99_ms": 268.6955 }, "usd_per_1k_decisions": 0.0007 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 14.7, "intelligence_ci95": [ 12.0, 17.5 ], "calibration": 89.2, "speed": 64.7, "cost": 85.5 }, "capability_score": 52.0, "composite_score": 37.6, "calibration_details": { "null_auroc": 0.68694, "null_ece": 0.10548, "null_brier": 0.11334 }, "latency": { "mean_ms": 13.4597, "p50_ms": 7.8974, "p95_ms": 18.7337, "p99_ms": 237.6608 }, "usd_per_1k_decisions": 0.0007 }, "by_request_type": { "span_open": { "score": 12.2, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 59.8, "n": 41 }, "noul_open": { "score": 44.4, "n": 27 }, "score": { "score": 24.4, "n": 48 }, "span_sealed": { "score": 2.3, "n": 108 }, "noul_sealed": { "score": 2.3, "n": 44 } }, "by_tier_open": { "Easy": { "score": 19.3, "n": 176 }, "Judge": { "score": 12.2, "n": 96 }, "Standard": { "score": 35.2, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 2.7, "n": 113 }, "Hard": { "score": 5.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 29.9, "n": 160 }, "compute_ledger": { "score": 9.7, "n": 144 }, "cluster_throughput_comps": { "score": 20.0, "n": 16 }, "edge_traffic_forecast": { "score": 20.1, "n": 16 }, "squad_v2_open_qa": { "score": 0.0, "n": 60 }, "contract_extraction": { "score": 3.1, "n": 96 }, "adversarial_stress": { "score": 31.2, "n": 8 } }, "official_rank": 14 }, { "id": "distilbert-base-control", "name": "distilbert-base-control (Raw Control)", "author": "distilbert (control)", "system_type": "Base Model Control (Untuned)", "open_weights": true, "params_m": 66.4, "footprint_mb": 253.2, "base_model": "distilbert/distilbert-base-uncased", "runtime": "PyTorch MPS/CPU", "input_rate_per_1m_usd": 0.015, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "a277779356e2ce378caa59c804942afe7a791e1ae8f0b08194c2b933df28601b", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 35946, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0011, "axes": { "intelligence": 1.8, "intelligence_ci95": [ 0.0, 4.3 ], "calibration": 80.3, "speed": 65.1, "cost": 81.1 }, "capability_score": 41.0, "composite_score": 6.7, "open_set_score": 20.2, "sealed_set_score": 1.8, "calibration_details": { "null_auroc": 0.4625, "null_ece": 0.1804, "null_brier": 0.22729 }, "latency": { "mean_ms": 9.6104, "p50_ms": 7.2909, "p95_ms": 14.4287, "p99_ms": 57.2864 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 1.8, "intelligence_ci95": [ 0.0, 4.3 ], "calibration": 80.3, "speed": 65.1, "cost": 81.1 }, "capability_score": 41.0, "composite_score": 6.7, "calibration_details": { "null_auroc": 0.4625, "null_ece": 0.1804, "null_brier": 0.22729 }, "latency": { "mean_ms": 9.6104, "p50_ms": 7.2909, "p95_ms": 14.4287, "p99_ms": 57.2864 }, "usd_per_1k_decisions": 0.0011 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 20.2, "intelligence_ci95": [ 16.6, 23.9 ], "calibration": 94.1, "speed": 65.7, "cost": 81.1 }, "capability_score": 57.1, "composite_score": 45.6, "calibration_details": { "null_auroc": 0.76268, "null_ece": 0.06075, "null_brier": 0.05552 }, "latency": { "mean_ms": 7.8717, "p50_ms": 6.4818, "p95_ms": 14.1463, "p99_ms": 24.2356 }, "usd_per_1k_decisions": 0.0011 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 14.2, "intelligence_ci95": [ 11.6, 17.0 ], "calibration": 91.5, "speed": 65.4, "cost": 81.1 }, "capability_score": 52.9, "composite_score": 36.7, "calibration_details": { "null_auroc": 0.60728, "null_ece": 0.07034, "null_brier": 0.11186 }, "latency": { "mean_ms": 8.442, "p50_ms": 6.806, "p95_ms": 14.5339, "p99_ms": 36.6088 }, "usd_per_1k_decisions": 0.0011 }, "by_request_type": { "span_open": { "score": 16.0, "n": 181 }, "math": { "score": 0.0, "n": 51 }, "choice": { "score": 59.8, "n": 41 }, "noul_open": { "score": 0.0, "n": 27 }, "score": { "score": 34.2, "n": 48 }, "span_sealed": { "score": 0.9, "n": 108 }, "noul_sealed": { "score": 0.0, "n": 44 } }, "by_tier_open": { "Easy": { "score": 16.5, "n": 176 }, "Judge": { "score": 17.1, "n": 96 }, "Standard": { "score": 35.2, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 1.8, "n": 113 }, "Hard": { "score": 2.0, "n": 50 }, "Judge": { "score": 0.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 24.2, "n": 160 }, "compute_ledger": { "score": 11.1, "n": 144 }, "cluster_throughput_comps": { "score": 27.4, "n": 16 }, "edge_traffic_forecast": { "score": 55.1, "n": 16 }, "squad_v2_open_qa": { "score": 1.7, "n": 60 }, "contract_extraction": { "score": 1.0, "n": 96 }, "adversarial_stress": { "score": 12.5, "n": 8 } }, "official_rank": 15 } ], "sealed_leaderboard": [ { "official_rank": 1, "id": "nullspan-hybrid-router", "name": "nullspan-Hybrid (Fast + ONNX Int8)", "open_weights": true, "measured": true, "predictions_sha256": "986301d101206b7ce96831c7e9121f4b12d4fd996230e5f78a20c1ae19422875", "n_observations": 164, "axes": { "intelligence": 94.5, "intelligence_ci95": [ 90.9, 97.6 ], "calibration": 95.5, "speed": 83.0, "cost": 93.8 }, "capability_score": 95.0, "composite_score": 91.4, "calibration_details": { "null_auroc": 0.9803, "null_ece": 0.04549, "null_brier": 0.0443 }, "latency": { "mean_ms": 63.0486, "p50_ms": 0.2482, "p95_ms": 247.0378, "p99_ms": 368.8361 }, "usd_per_1k_decisions": 0.0003 }, { "official_rank": 2, "id": "nullspan-zerodep-fast", "name": "nullspan-ZeroDep (Fast Engine)", "open_weights": true, "measured": true, "predictions_sha256": "be3de37219d217d33904ec382248ff1502140dacd5b5acb7de4ee8eb39d92ebe", "n_observations": 164, "axes": { "intelligence": 72.0, "intelligence_ci95": [ 65.2, 78.7 ], "calibration": 84.6, "speed": 86.1, "cost": 99.8 }, "capability_score": 78.3, "composite_score": 84.5, "calibration_details": { "null_auroc": 0.89839, "null_ece": 0.15249, "null_brier": 0.15672 }, "latency": { "mean_ms": 0.1456, "p50_ms": 0.1374, "p95_ms": 0.3377, "p99_ms": 0.8821 }, "usd_per_1k_decisions": 0.0001 }, { "official_rank": 3, "id": "nullspan-onnx-int8", "name": "nullspan-345M-Int8 (ONNX WASM/CPU)", "open_weights": true, "measured": true, "predictions_sha256": "22d5544220f57ebdbc1c8047f042111348af54d7f31b6642d266f4cc2123a64f", "n_observations": 164, "axes": { "intelligence": 94.5, "intelligence_ci95": [ 90.9, 97.6 ], "calibration": 95.8, "speed": 47.5, "cost": 84.2 }, "capability_score": 95.2, "composite_score": 74.1, "calibration_details": { "null_auroc": 0.97614, "null_ece": 0.04136, "null_brier": 0.04455 }, "latency": { "mean_ms": 203.0401, "p50_ms": 201.6991, "p95_ms": 292.1634, "p99_ms": 368.8361 }, "usd_per_1k_decisions": 0.0008 }, { "official_rank": 4, "id": "nullspan-fp32", "name": "nullspan-345M (PyTorch FP32)", "open_weights": true, "measured": true, "predictions_sha256": "91f8dd7ef01eb004f9c3e07eca2c0ca7705f6316f3f2126f6a59533f8a87fb87", "n_observations": 164, "axes": { "intelligence": 93.9, "intelligence_ci95": [ 90.2, 97.0 ], "calibration": 95.3, "speed": 49.9, "cost": 76.8 }, "capability_score": 94.6, "composite_score": 73.8, "calibration_details": { "null_auroc": 0.98542, "null_ece": 0.04785, "null_brier": 0.04668 }, "latency": { "mean_ms": 128.5135, "p50_ms": 127.6058, "p95_ms": 175.881, "p99_ms": 254.4876 }, "usd_per_1k_decisions": 0.0017 }, { "official_rank": 5, "id": "nullspan-onnx-fp32", "name": "nullspan-345M (ONNX FP32)", "open_weights": true, "measured": true, "predictions_sha256": "7a1b74a627789f05d069229df78fb8153257a79c82c2c2e32dfe5de7411a1f2f", "n_observations": 164, "axes": { "intelligence": 93.9, "intelligence_ci95": [ 90.2, 97.0 ], "calibration": 96.5, "speed": 47.6, "cost": 78.0 }, "capability_score": 95.2, "composite_score": 72.9, "calibration_details": { "null_auroc": 0.99015, "null_ece": 0.03405, "null_brier": 0.03793 }, "latency": { "mean_ms": 188.3131, "p50_ms": 196.4426, "p95_ms": 266.5202, "p99_ms": 337.8913 }, "usd_per_1k_decisions": 0.0015 }, { "official_rank": 6, "id": "cloudflare-clairic-clef-live", "name": "@cf/cloudflare/clairic + clef (Live Workers AI)", "open_weights": false, "measured": true, "predictions_sha256": "378692149bafcdea8d8e7e9d3bae2ed81aa6e71e8d14767a4838b9426910f5c0", "n_observations": 164, "axes": { "intelligence": 95.4, "intelligence_ci95": [ 92.1, 98.2 ], "calibration": 97.8, "speed": 50.5, "cost": 68.2 }, "capability_score": 96.6, "composite_score": 72.5, "calibration_details": { "null_auroc": 0.99242, "null_ece": 0.02146, "null_brier": 0.02327 }, "latency": { "mean_ms": 167.0995, "p50_ms": 113.8012, "p95_ms": 373.1545, "p99_ms": 460.0218 }, "usd_per_1k_decisions": 0.0041 }, { "official_rank": 7, "id": "deberta-v3-base-squad2", "name": "deberta-v3-base-squad2 (184M SQuAD2 QA)", "open_weights": true, "measured": true, "predictions_sha256": "c46f41c01931e088e7e4900a96be1b9a22a4b2276f5031dc7f08f1c2907ae0bd", "n_observations": 164, "axes": { "intelligence": 47.3, "intelligence_ci95": [ 39.3, 54.9 ], "calibration": 63.8, "speed": 52.8, "cost": 69.2 }, "capability_score": 55.5, "composite_score": 57.0, "calibration_details": { "null_auroc": 0.82121, "null_ece": 0.37927, "null_brier": 0.33034 }, "latency": { "mean_ms": 112.8393, "p50_ms": 74.1332, "p95_ms": 376.7113, "p99_ms": 376.7113 }, "usd_per_1k_decisions": 0.0037 }, { "official_rank": 8, "id": "all-MiniLM-L12-v2", "name": "all-MiniLM-L12-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "cc6be4a4a7811f54a8cd82a63d66b0dfc8659215e8b05aff0d4a77b45c9b1f54", "n_observations": 164, "axes": { "intelligence": 8.8, "intelligence_ci95": [ 4.9, 13.4 ], "calibration": 64.5, "speed": 64.6, "cost": 85.5 }, "capability_score": 36.6, "composite_score": 25.6, "calibration_details": { "null_auroc": 0.42254, "null_ece": 0.35654, "null_brier": 0.35108 }, "latency": { "mean_ms": 8.14, "p50_ms": 7.9056, "p95_ms": 10.353, "p99_ms": 18.128 }, "usd_per_1k_decisions": 0.0007 }, { "official_rank": 9, "id": "all-mpnet-base-v2", "name": "all-mpnet-base-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "7f8ac57f8823a1bee4d47ce89f9d612fbe9f4980b1fc7584193d281a61b03282", "n_observations": 164, "axes": { "intelligence": 8.2, "intelligence_ci95": [ 4.3, 12.8 ], "calibration": 64.2, "speed": 62.2, "cost": 77.4 }, "capability_score": 36.2, "composite_score": 24.0, "calibration_details": { "null_auroc": 0.43182, "null_ece": 0.36572, "null_brier": 0.3441 }, "latency": { "mean_ms": 15.1577, "p50_ms": 12.5622, "p95_ms": 26.3503, "p99_ms": 59.4068 }, "usd_per_1k_decisions": 0.0016 }, { "official_rank": 10, "id": "paraphrase-MiniLM-L3-v2", "name": "paraphrase-MiniLM-L3-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "cafe97bf5b136b8d0e9dde1c3a846a75914aa1b0a544b277393ca995592a7901", "n_observations": 164, "axes": { "intelligence": 7.6, "intelligence_ci95": [ 4.0, 11.9 ], "calibration": 63.7, "speed": 70.4, "cost": 91.0 }, "capability_score": 35.6, "composite_score": 23.2, "calibration_details": { "null_auroc": 0.45587, "null_ece": 0.36995, "null_brier": 0.35112 }, "latency": { "mean_ms": 3.0192, "p50_ms": 2.6538, "p95_ms": 4.8318, "p99_ms": 9.5178 }, "usd_per_1k_decisions": 0.0004 }, { "official_rank": 11, "id": "all-MiniLM-L6-v2", "name": "all-MiniLM-L6-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "cb306daa6a5967d18053fdb1544ae61f9d00011c3c8d899149e863155ddd15da", "n_observations": 164, "axes": { "intelligence": 7.6, "intelligence_ci95": [ 4.0, 11.9 ], "calibration": 61.7, "speed": 68.2, "cost": 88.8 }, "capability_score": 34.6, "composite_score": 23.0, "calibration_details": { "null_auroc": 0.42756, "null_ece": 0.38988, "null_brier": 0.36948 }, "latency": { "mean_ms": 4.4962, "p50_ms": 4.0641, "p95_ms": 7.0286, "p99_ms": 8.7059 }, "usd_per_1k_decisions": 0.0005 }, { "official_rank": 12, "id": "claude-haiku-4.5-json", "name": "claude-haiku-4.5 (JSON Decoder \u00b7 API)", "open_weights": false, "measured": true, "predictions_sha256": "d1fb4e903ee98334a5cda0511c89f6c04ceb67661d7a35c8f400a1b28f25af08", "n_observations": 164, "axes": { "intelligence": 88.4, "intelligence_ci95": [ 83.5, 93.3 ], "calibration": 91.3, "speed": 35.9, "cost": 8.0 }, "capability_score": 89.8, "composite_score": 22.8, "calibration_details": { "null_auroc": 0.95284, "null_ece": 0.08506, "null_brier": 0.09115 }, "latency": { "mean_ms": 2030.4761, "p50_ms": 1811.16, "p95_ms": 3583.842, "p99_ms": 5955.9195 }, "usd_per_1k_decisions": 2.5643 }, { "official_rank": 13, "id": "paraphrase-albert-small-v2", "name": "paraphrase-albert-small-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "7f1c86b657b934f8edc6b2a52ae3e20df99d831c9ddb51ac3dacc06822176507", "n_observations": 164, "axes": { "intelligence": 6.7, "intelligence_ci95": [ 3.4, 10.7 ], "calibration": 67.6, "speed": 63.9, "cost": 91.0 }, "capability_score": 37.1, "composite_score": 21.0, "calibration_details": { "null_auroc": 0.43845, "null_ece": 0.32478, "null_brier": 0.32203 }, "latency": { "mean_ms": 11.8891, "p50_ms": 9.144, "p95_ms": 45.3988, "p99_ms": 60.6356 }, "usd_per_1k_decisions": 0.0004 }, { "official_rank": 14, "id": "bge-small-en-v1.5", "name": "bge-small-en-v1.5 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "ab7f52e9e9dde3ac226bd44564e81217a8d8118c9226185ee29664f71a006f4a", "n_observations": 164, "axes": { "intelligence": 3.4, "intelligence_ci95": [ 0.9, 6.1 ], "calibration": 75.2, "speed": 65.3, "cost": 85.5 }, "capability_score": 39.3, "composite_score": 12.0, "calibration_details": { "null_auroc": 0.51752, "null_ece": 0.24512, "null_brier": 0.25353 }, "latency": { "mean_ms": 8.906, "p50_ms": 7.0485, "p95_ms": 24.5518, "p99_ms": 37.8159 }, "usd_per_1k_decisions": 0.0007 }, { "official_rank": 15, "id": "distilbert-base-control", "name": "distilbert-base-control (Raw Control)", "open_weights": true, "measured": true, "predictions_sha256": "a277779356e2ce378caa59c804942afe7a791e1ae8f0b08194c2b933df28601b", "n_observations": 164, "axes": { "intelligence": 1.8, "intelligence_ci95": [ 0.0, 4.3 ], "calibration": 80.3, "speed": 65.1, "cost": 81.1 }, "capability_score": 41.0, "composite_score": 6.7, "calibration_details": { "null_auroc": 0.4625, "null_ece": 0.1804, "null_brier": 0.22729 }, "latency": { "mean_ms": 9.6104, "p50_ms": 7.2909, "p95_ms": 14.4287, "p99_ms": 57.2864 }, "usd_per_1k_decisions": 0.0011 } ], "open_leaderboard": [ { "official_rank": 1, "id": "nullspan-zerodep-fast", "name": "nullspan-ZeroDep (Fast Engine)", "open_weights": true, "measured": true, "predictions_sha256": "be3de37219d217d33904ec382248ff1502140dacd5b5acb7de4ee8eb39d92ebe", "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 99.8, "cost": 99.8 }, "capability_score": 100.0, "composite_score": 99.9, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00201, "null_brier": 2e-05 }, "latency": { "mean_ms": 0.0278, "p50_ms": 0.0074, "p95_ms": 0.0408, "p99_ms": 0.4413 }, "usd_per_1k_decisions": 0.0001 }, { "official_rank": 2, "id": "nullspan-hybrid-router", "name": "nullspan-Hybrid (Fast + ONNX Int8)", "open_weights": true, "measured": true, "predictions_sha256": "986301d101206b7ce96831c7e9121f4b12d4fd996230e5f78a20c1ae19422875", "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 99.8, "cost": 93.8 }, "capability_score": 100.0, "composite_score": 98.3, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00201, "null_brier": 2e-05 }, "latency": { "mean_ms": 0.0278, "p50_ms": 0.0074, "p95_ms": 0.0408, "p99_ms": 0.4413 }, "usd_per_1k_decisions": 0.0003 }, { "official_rank": 3, "id": "nullspan-fp32", "name": "nullspan-345M (PyTorch FP32)", "open_weights": true, "measured": true, "predictions_sha256": "91f8dd7ef01eb004f9c3e07eca2c0ca7705f6316f3f2126f6a59533f8a87fb87", "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 54.9, "cost": 76.8 }, "capability_score": 100.0, "composite_score": 78.1, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00225, "null_brier": 2e-05 }, "latency": { "mean_ms": 66.2503, "p50_ms": 49.5557, "p95_ms": 107.1403, "p99_ms": 275.8887 }, "usd_per_1k_decisions": 0.0017 }, { "official_rank": 4, "id": "nullspan-onnx-int8", "name": "nullspan-345M-Int8 (ONNX WASM/CPU)", "open_weights": true, "measured": true, "predictions_sha256": "22d5544220f57ebdbc1c8047f042111348af54d7f31b6642d266f4cc2123a64f", "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 51.1, "cost": 84.2 }, "capability_score": 100.0, "composite_score": 77.7, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00222, "null_brier": 2e-05 }, "latency": { "mean_ms": 110.4293, "p50_ms": 101.0107, "p95_ms": 169.4236, "p99_ms": 241.4742 }, "usd_per_1k_decisions": 0.0008 }, { "official_rank": 5, "id": "nullspan-onnx-fp32", "name": "nullspan-345M (ONNX FP32)", "open_weights": true, "measured": true, "predictions_sha256": "7a1b74a627789f05d069229df78fb8153257a79c82c2c2e32dfe5de7411a1f2f", "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 51.7, "cost": 78.0 }, "capability_score": 100.0, "composite_score": 76.7, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00225, "null_brier": 2e-05 }, "latency": { "mean_ms": 98.6641, "p50_ms": 90.9101, "p95_ms": 164.1157, "p99_ms": 219.525 }, "usd_per_1k_decisions": 0.0015 }, { "official_rank": 6, "id": "cloudflare-clairic-clef-live", "name": "@cf/cloudflare/clairic + clef (Live Workers AI)", "open_weights": false, "measured": true, "predictions_sha256": "378692149bafcdea8d8e7e9d3bae2ed81aa6e71e8d14767a4838b9426910f5c0", "n_observations": 336, "axes": { "intelligence": 89.7, "intelligence_ci95": [ 86.5, 92.8 ], "calibration": 92.8, "speed": 52.0, "cost": 68.2 }, "capability_score": 91.2, "composite_score": 71.7, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.07908, "null_brier": 0.05899 }, "latency": { "mean_ms": 110.0899, "p50_ms": 86.1611, "p95_ms": 272.4725, "p99_ms": 360.5915 }, "usd_per_1k_decisions": 0.0041 }, { "official_rank": 7, "id": "deberta-v3-base-squad2", "name": "deberta-v3-base-squad2 (184M SQuAD2 QA)", "open_weights": true, "measured": true, "predictions_sha256": "c46f41c01931e088e7e4900a96be1b9a22a4b2276f5031dc7f08f1c2907ae0bd", "n_observations": 336, "axes": { "intelligence": 48.6, "intelligence_ci95": [ 43.2, 54.0 ], "calibration": 62.2, "speed": 56.2, "cost": 69.2 }, "capability_score": 55.4, "composite_score": 58.1, "calibration_details": { "null_auroc": 0.82668, "null_ece": 0.4174, "null_brier": 0.30522 }, "latency": { "mean_ms": 54.7889, "p50_ms": 38.8237, "p95_ms": 116.8072, "p99_ms": 371.4963 }, "usd_per_1k_decisions": 0.0037 }, { "official_rank": 8, "id": "bge-small-en-v1.5", "name": "bge-small-en-v1.5 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "ab7f52e9e9dde3ac226bd44564e81217a8d8118c9226185ee29664f71a006f4a", "n_observations": 336, "axes": { "intelligence": 20.3, "intelligence_ci95": [ 16.8, 23.9 ], "calibration": 96.0, "speed": 64.6, "cost": 85.5 }, "capability_score": 58.1, "composite_score": 46.1, "calibration_details": { "null_auroc": 0.85617, "null_ece": 0.03732, "null_brier": 0.04491 }, "latency": { "mean_ms": 15.6824, "p50_ms": 8.0241, "p95_ms": 17.5403, "p99_ms": 268.6955 }, "usd_per_1k_decisions": 0.0007 }, { "official_rank": 9, "id": "distilbert-base-control", "name": "distilbert-base-control (Raw Control)", "open_weights": true, "measured": true, "predictions_sha256": "a277779356e2ce378caa59c804942afe7a791e1ae8f0b08194c2b933df28601b", "n_observations": 336, "axes": { "intelligence": 20.2, "intelligence_ci95": [ 16.6, 23.9 ], "calibration": 94.1, "speed": 65.7, "cost": 81.1 }, "capability_score": 57.1, "composite_score": 45.6, "calibration_details": { "null_auroc": 0.76268, "null_ece": 0.06075, "null_brier": 0.05552 }, "latency": { "mean_ms": 7.8717, "p50_ms": 6.4818, "p95_ms": 14.1463, "p99_ms": 24.2356 }, "usd_per_1k_decisions": 0.0011 }, { "official_rank": 10, "id": "all-MiniLM-L12-v2", "name": "all-MiniLM-L12-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "cc6be4a4a7811f54a8cd82a63d66b0dfc8659215e8b05aff0d4a77b45c9b1f54", "n_observations": 336, "axes": { "intelligence": 17.2, "intelligence_ci95": [ 13.5, 20.9 ], "calibration": 76.5, "speed": 65.1, "cost": 85.5 }, "capability_score": 46.9, "composite_score": 40.7, "calibration_details": { "null_auroc": 0.82201, "null_ece": 0.26802, "null_brier": 0.17378 }, "latency": { "mean_ms": 7.4393, "p50_ms": 7.28, "p95_ms": 9.3107, "p99_ms": 16.1564 }, "usd_per_1k_decisions": 0.0007 }, { "official_rank": 11, "id": "paraphrase-albert-small-v2", "name": "paraphrase-albert-small-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "7f1c86b657b934f8edc6b2a52ae3e20df99d831c9ddb51ac3dacc06822176507", "n_observations": 336, "axes": { "intelligence": 15.3, "intelligence_ci95": [ 12.0, 18.7 ], "calibration": 79.4, "speed": 66.5, "cost": 91.0 }, "capability_score": 47.4, "composite_score": 38.5, "calibration_details": { "null_auroc": 0.78857, "null_ece": 0.23047, "null_brier": 0.16114 }, "latency": { "mean_ms": 7.0892, "p50_ms": 5.6208, "p95_ms": 8.8297, "p99_ms": 54.0583 }, "usd_per_1k_decisions": 0.0004 }, { "official_rank": 12, "id": "all-mpnet-base-v2", "name": "all-mpnet-base-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "7f8ac57f8823a1bee4d47ce89f9d612fbe9f4980b1fc7584193d281a61b03282", "n_observations": 336, "axes": { "intelligence": 16.2, "intelligence_ci95": [ 12.5, 19.9 ], "calibration": 72.4, "speed": 62.9, "cost": 77.4 }, "capability_score": 44.3, "composite_score": 38.3, "calibration_details": { "null_auroc": 0.78916, "null_ece": 0.30333, "null_brier": 0.22501 }, "latency": { "mean_ms": 18.4708, "p50_ms": 11.0142, "p95_ms": 18.1759, "p99_ms": 175.4313 }, "usd_per_1k_decisions": 0.0016 }, { "official_rank": 13, "id": "paraphrase-MiniLM-L3-v2", "name": "paraphrase-MiniLM-L3-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "cafe97bf5b136b8d0e9dde1c3a846a75914aa1b0a544b277393ca995592a7901", "n_observations": 336, "axes": { "intelligence": 12.9, "intelligence_ci95": [ 9.7, 16.1 ], "calibration": 74.0, "speed": 70.4, "cost": 91.0 }, "capability_score": 43.5, "composite_score": 34.4, "calibration_details": { "null_auroc": 0.78713, "null_ece": 0.29056, "null_brier": 0.20327 }, "latency": { "mean_ms": 2.9532, "p50_ms": 2.6466, "p95_ms": 4.7012, "p99_ms": 8.5112 }, "usd_per_1k_decisions": 0.0004 }, { "official_rank": 14, "id": "all-MiniLM-L6-v2", "name": "all-MiniLM-L6-v2 (Dense Span+Null)", "open_weights": true, "measured": true, "predictions_sha256": "cb306daa6a5967d18053fdb1544ae61f9d00011c3c8d899149e863155ddd15da", "n_observations": 336, "axes": { "intelligence": 11.2, "intelligence_ci95": [ 8.0, 14.1 ], "calibration": 73.7, "speed": 67.3, "cost": 88.8 }, "capability_score": 42.5, "composite_score": 31.0, "calibration_details": { "null_auroc": 0.7785, "null_ece": 0.29091, "null_brier": 0.21136 }, "latency": { "mean_ms": 5.2658, "p50_ms": 4.746, "p95_ms": 7.4978, "p99_ms": 12.6273 }, "usd_per_1k_decisions": 0.0005 }, { "official_rank": 15, "id": "claude-haiku-4.5-json", "name": "claude-haiku-4.5 (JSON Decoder \u00b7 API)", "open_weights": false, "measured": true, "predictions_sha256": "d1fb4e903ee98334a5cda0511c89f6c04ceb67661d7a35c8f400a1b28f25af08", "n_observations": 336, "axes": { "intelligence": 93.7, "intelligence_ci95": [ 91.3, 96.0 ], "calibration": 93.1, "speed": 36.0, "cost": 8.0 }, "capability_score": 93.4, "composite_score": 23.0, "calibration_details": { "null_auroc": 0.99712, "null_ece": 0.08408, "null_brier": 0.04084 }, "latency": { "mean_ms": 1808.6166, "p50_ms": 1769.33, "p95_ms": 2433.9525, "p99_ms": 4903.8135 }, "usd_per_1k_decisions": 2.5643 } ], "reference_external_anchors": [ { "id": "jev-1.13.0-ref", "name": "Jev 1.13.0 (TypeSafe AI)", "author": "TypeSafe AI", "source_benchmark": "JevBench v1.6.1 (snapshot 2026-11-11)", "jevbench_capability": 75.5, "jevbench_intelligence": 63.06, "jevbench_calibration": 87.93, "note": "Reference from JevBench v1.6.1 (different item set, not comparable to NullspanBench; unranked)." }, { "id": "h2o-lightning-4b", "name": "H2O-Lightning-4B v1.1", "author": "H2O.ai", "source_benchmark": "JevBench v1.6.1 (snapshot 2026-11-11)", "jevbench_capability": 70.64, "jevbench_intelligence": 53.75, "jevbench_calibration": 87.52, "note": "Reference from JevBench v1.6.1 (different item set, not comparable to NullspanBench; unranked)." }, { "id": "basal-omni-1.0-ultra", "name": "basal-omni-1.0-ultra", "author": "Remek Kinas", "source_benchmark": "JevBench v1.6.1 (snapshot 2026-11-11)", "jevbench_capability": 80.78, "jevbench_intelligence": 72.14, "jevbench_calibration": 89.42, "note": "Reference from JevBench v1.6.1 (different item set, not comparable to NullspanBench; unranked)." } ], "tiers": { "tier_1a_pytorch_fp32": { "id": "nullspan-fp32", "name": "nullspan-345M (PyTorch FP32)", "author": "gnumanth", "system_type": "System One Encoder (5-Head, 512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 1317.2, "base_model": "ahotrod/electra_large_discriminator_squad2_512 + SchemaAdapter", "runtime": "PyTorch MPS/CUDA/CPU FP32", "input_rate_per_1m_usd": 0.012, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "91f8dd7ef01eb004f9c3e07eca2c0ca7705f6316f3f2126f6a59533f8a87fb87", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0017, "axes": { "intelligence": 93.9, "intelligence_ci95": [ 90.2, 97.0 ], "calibration": 95.3, "speed": 49.9, "cost": 76.8 }, "capability_score": 94.6, "composite_score": 73.8, "open_set_score": 100.0, "sealed_set_score": 93.9, "calibration_details": { "null_auroc": 0.98542, "null_ece": 0.04785, "null_brier": 0.04668 }, "latency": { "mean_ms": 128.5135, "p50_ms": 127.6058, "p95_ms": 175.881, "p99_ms": 254.4876 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 93.9, "intelligence_ci95": [ 90.2, 97.0 ], "calibration": 95.3, "speed": 49.9, "cost": 76.8 }, "capability_score": 94.6, "composite_score": 73.8, "calibration_details": { "null_auroc": 0.98542, "null_ece": 0.04785, "null_brier": 0.04668 }, "latency": { "mean_ms": 128.5135, "p50_ms": 127.6058, "p95_ms": 175.881, "p99_ms": 254.4876 }, "usd_per_1k_decisions": 0.0017 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 54.9, "cost": 76.8 }, "capability_score": 100.0, "composite_score": 78.1, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00225, "null_brier": 2e-05 }, "latency": { "mean_ms": 66.2503, "p50_ms": 49.5557, "p95_ms": 107.1403, "p99_ms": 275.8887 }, "usd_per_1k_decisions": 0.0017 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 98.0, "intelligence_ci95": [ 96.8, 99.2 ], "calibration": 98.5, "speed": 54.3, "cost": 76.8 }, "capability_score": 98.2, "composite_score": 77.2, "calibration_details": { "null_auroc": 0.99327, "null_ece": 0.01419, "null_brier": 0.01532 }, "latency": { "mean_ms": 86.6726, "p50_ms": 55.206, "p95_ms": 160.6231, "p99_ms": 270.4781 }, "usd_per_1k_decisions": 0.0017 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 96.3, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 96.5, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 83.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 4 }, "tier_1b_onnx_int8": { "id": "nullspan-onnx-int8", "name": "nullspan-345M-Int8 (ONNX WASM/CPU)", "author": "gnumanth", "system_type": "Quantized ONNX WASM/CPU (512-ctx)", "open_weights": true, "params_m": 345.3, "footprint_mb": 331.1, "base_model": "gnumanth/nullspan (Int8 Dynamic)", "runtime": "ONNX Runtime Int8 (Browser WASM / Edge)", "input_rate_per_1m_usd": 0.006, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "22d5544220f57ebdbc1c8047f042111348af54d7f31b6642d266f4cc2123a64f", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0008, "axes": { "intelligence": 94.5, "intelligence_ci95": [ 90.9, 97.6 ], "calibration": 95.8, "speed": 47.5, "cost": 84.2 }, "capability_score": 95.2, "composite_score": 74.1, "open_set_score": 100.0, "sealed_set_score": 94.5, "calibration_details": { "null_auroc": 0.97614, "null_ece": 0.04136, "null_brier": 0.04455 }, "latency": { "mean_ms": 203.0401, "p50_ms": 201.6991, "p95_ms": 292.1634, "p99_ms": 368.8361 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 94.5, "intelligence_ci95": [ 90.9, 97.6 ], "calibration": 95.8, "speed": 47.5, "cost": 84.2 }, "capability_score": 95.2, "composite_score": 74.1, "calibration_details": { "null_auroc": 0.97614, "null_ece": 0.04136, "null_brier": 0.04455 }, "latency": { "mean_ms": 203.0401, "p50_ms": 201.6991, "p95_ms": 292.1634, "p99_ms": 368.8361 }, "usd_per_1k_decisions": 0.0008 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 51.1, "cost": 84.2 }, "capability_score": 100.0, "composite_score": 77.7, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00222, "null_brier": 2e-05 }, "latency": { "mean_ms": 110.4293, "p50_ms": 101.0107, "p95_ms": 169.4236, "p99_ms": 241.4742 }, "usd_per_1k_decisions": 0.0008 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 98.2, "intelligence_ci95": [ 97.0, 99.2 ], "calibration": 98.7, "speed": 50.8, "cost": 84.2 }, "capability_score": 98.5, "composite_score": 77.1, "calibration_details": { "null_auroc": 0.99166, "null_ece": 0.01207, "null_brier": 0.01462 }, "latency": { "mean_ms": 140.8057, "p50_ms": 107.0085, "p95_ms": 264.4929, "p99_ms": 292.3606 }, "usd_per_1k_decisions": 0.0008 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 97.2, "n": 108 }, "noul_sealed": { "score": 86.4, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 97.3, "n": 113 }, "Hard": { "score": 88.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 85.0, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 3 }, "tier_2_zero_dep_engine": { "id": "nullspan-zerodep-fast", "name": "nullspan-ZeroDep (Fast Engine)", "author": "gnumanth", "system_type": "Zero-Dep Algorithmic Engine", "open_weights": true, "params_m": 0.0, "footprint_mb": 0.0, "base_model": "Deterministic Clause + Lexical Pointer", "runtime": "Pure Python / Node / Browser JS", "input_rate_per_1m_usd": 0.0008, "output_rate_per_1m_usd": 0.0, "measured": true, "predictions_sha256": "be3de37219d217d33904ec382248ff1502140dacd5b5acb7de4ee8eb39d92ebe", "n_observations": 164, "n_total_observations": 500, "input_tokens_total": 69943, "output_tokens_total": 0, "usd_per_1k_decisions": 0.0001, "axes": { "intelligence": 72.0, "intelligence_ci95": [ 65.2, 78.7 ], "calibration": 84.6, "speed": 86.1, "cost": 99.8 }, "capability_score": 78.3, "composite_score": 84.5, "open_set_score": 100.0, "sealed_set_score": 72.0, "calibration_details": { "null_auroc": 0.89839, "null_ece": 0.15249, "null_brier": 0.15672 }, "latency": { "mean_ms": 0.1456, "p50_ms": 0.1374, "p95_ms": 0.3377, "p99_ms": 0.8821 }, "sealed_pool": { "n_observations": 164, "axes": { "intelligence": 72.0, "intelligence_ci95": [ 65.2, 78.7 ], "calibration": 84.6, "speed": 86.1, "cost": 99.8 }, "capability_score": 78.3, "composite_score": 84.5, "calibration_details": { "null_auroc": 0.89839, "null_ece": 0.15249, "null_brier": 0.15672 }, "latency": { "mean_ms": 0.1456, "p50_ms": 0.1374, "p95_ms": 0.3377, "p99_ms": 0.8821 }, "usd_per_1k_decisions": 0.0001 }, "open_pool": { "n_observations": 336, "axes": { "intelligence": 100.0, "intelligence_ci95": [ 100.0, 100.0 ], "calibration": 99.9, "speed": 99.8, "cost": 99.8 }, "capability_score": 100.0, "composite_score": 99.9, "calibration_details": { "null_auroc": 1.0, "null_ece": 0.00201, "null_brier": 2e-05 }, "latency": { "mean_ms": 0.0278, "p50_ms": 0.0074, "p95_ms": 0.0408, "p99_ms": 0.4413 }, "usd_per_1k_decisions": 0.0001 }, "combined_pool": { "n_observations": 500, "axes": { "intelligence": 90.8, "intelligence_ci95": [ 88.2, 93.4 ], "calibration": 95.0, "speed": 98.4, "cost": 99.8 }, "capability_score": 92.9, "composite_score": 95.9, "calibration_details": { "null_auroc": 0.98239, "null_ece": 0.04866, "null_brier": 0.05142 }, "latency": { "mean_ms": 0.0664, "p50_ms": 0.0135, "p95_ms": 0.2901, "p99_ms": 0.4926 }, "usd_per_1k_decisions": 0.0001 }, "by_request_type": { "span_open": { "score": 100.0, "n": 181 }, "math": { "score": 100.0, "n": 51 }, "choice": { "score": 100.0, "n": 41 }, "noul_open": { "score": 100.0, "n": 27 }, "score": { "score": 100.0, "n": 48 }, "span_sealed": { "score": 78.7, "n": 108 }, "noul_sealed": { "score": 47.7, "n": 44 } }, "by_tier_open": { "Easy": { "score": 100.0, "n": 176 }, "Judge": { "score": 100.0, "n": 96 }, "Standard": { "score": 100.0, "n": 64 } }, "by_tier_sealed": { "Standard": { "score": 79.6, "n": 113 }, "Hard": { "score": 54.0, "n": 50 }, "Judge": { "score": 100.0, "n": 1 } }, "by_domain": { "incident_report": { "score": 100.0, "n": 160 }, "compute_ledger": { "score": 100.0, "n": 144 }, "cluster_throughput_comps": { "score": 100.0, "n": 16 }, "edge_traffic_forecast": { "score": 100.0, "n": 16 }, "squad_v2_open_qa": { "score": 23.3, "n": 60 }, "contract_extraction": { "score": 100.0, "n": 96 }, "adversarial_stress": { "score": 100.0, "n": 8 } }, "official_rank": 2 } } }