{
  "benchmark": "mmlu",
  "exported_at": "2026-10-27 22:15:26",
  "results": [
    {
      "model": "mistralai/ministral-8b-2512",
      "benchmark": "mmlu",
      "task_quality": {
        "overall_accuracy": 0.793,
        "overall_accuracy_ci95": [
          0.7777,
          0.8075
        ],
        "macro_accuracy_subject": 0.793,
        "macro_accuracy_category": 0.7927,
        "normalised_accuracy": 0.724,
        "random_baseline": 0.25,
        "error_rate": 0.207,
        "parse_failure_rate": 0.0056,
        "subject_accuracy": {
          "abstract_algebra": {
            "correct": 32,
            "total": 50,
            "accuracy": 0.64,
            "ci95": [
              0.5014,
              0.7586
            ],
            "ci_width": 0.2572
          },
          "astronomy": {
            "correct": 42,
            "total": 50,
            "accuracy": 0.84,
            "ci95": [
              0.7149,
              0.9166
            ],
            "ci_width": 0.2018
          },
          "college_biology": {
            "correct": 45,
            "total": 50,
            "accuracy": 0.9,
            "ci95": [
              0.7864,
              0.9565
            ],
            "ci_width": 0.1701
          },
          "college_chemistry": {
            "correct": 33,
            "total": 50,
            "accuracy": 0.66,
            "ci95": [
              0.5215,
              0.7756
            ],
            "ci_width": 0.2541
          },
          "college_computer_science": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "college_mathematics": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "college_physics": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "computer_security": {
            "correct": 48,
            "total": 50,
            "accuracy": 0.96,
            "ci95": [
              0.8654,
              0.989
            ],
            "ci_width": 0.1236
          },
          "conceptual_physics": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "electrical_engineering": {
            "correct": 38,
            "total": 50,
            "accuracy": 0.76,
            "ci95": [
              0.6259,
              0.857
            ],
            "ci_width": 0.2312
          },
          "elementary_mathematics": {
            "correct": 47,
            "total": 50,
            "accuracy": 0.94,
            "ci95": [
              0.8378,
              0.9794
            ],
            "ci_width": 0.1416
          },
          "high_school_biology": {
            "correct": 42,
            "total": 50,
            "accuracy": 0.84,
            "ci95": [
              0.7149,
              0.9166
            ],
            "ci_width": 0.2018
          },
          "high_school_chemistry": {
            "correct": 41,
            "total": 50,
            "accuracy": 0.82,
            "ci95": [
              0.692,
              0.9023
            ],
            "ci_width": 0.2103
          },
          "high_school_computer_science": {
            "correct": 46,
            "total": 50,
            "accuracy": 0.92,
            "ci95": [
              0.8116,
              0.9685
            ],
            "ci_width": 0.1568
          },
          "high_school_mathematics": {
            "correct": 41,
            "total": 50,
            "accuracy": 0.82,
            "ci95": [
              0.692,
              0.9023
            ],
            "ci_width": 0.2103
          },
          "high_school_physics": {
            "correct": 39,
            "total": 50,
            "accuracy": 0.78,
            "ci95": [
              0.6476,
              0.8725
            ],
            "ci_width": 0.2249
          },
          "high_school_statistics": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "machine_learning": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "formal_logic": {
            "correct": 37,
            "total": 50,
            "accuracy": 0.74,
            "ci95": [
              0.6045,
              0.8413
            ],
            "ci_width": 0.2368
          },
          "high_school_european_history": {
            "correct": 37,
            "total": 50,
            "accuracy": 0.74,
            "ci95": [
              0.6045,
              0.8413
            ],
            "ci_width": 0.2368
          },
          "high_school_us_history": {
            "correct": 40,
            "total": 50,
            "accuracy": 0.8,
            "ci95": [
              0.6696,
              0.8876
            ],
            "ci_width": 0.2179
          },
          "high_school_world_history": {
            "correct": 42,
            "total": 50,
            "accuracy": 0.84,
            "ci95": [
              0.7149,
              0.9166
            ],
            "ci_width": 0.2018
          },
          "international_law": {
            "correct": 44,
            "total": 50,
            "accuracy": 0.88,
            "ci95": [
              0.7619,
              0.9438
            ],
            "ci_width": 0.1819
          },
          "jurisprudence": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "logical_fallacies": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "moral_disputes": {
            "correct": 33,
            "total": 50,
            "accuracy": 0.66,
            "ci95": [
              0.5215,
              0.7756
            ],
            "ci_width": 0.2541
          },
          "moral_scenarios": {
            "correct": 18,
            "total": 50,
            "accuracy": 0.36,
            "ci95": [
              0.2414,
              0.4986
            ],
            "ci_width": 0.2572
          },
          "philosophy": {
            "correct": 40,
            "total": 50,
            "accuracy": 0.8,
            "ci95": [
              0.6696,
              0.8876
            ],
            "ci_width": 0.2179
          },
          "prehistory": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "professional_law": {
            "correct": 24,
            "total": 50,
            "accuracy": 0.48,
            "ci95": [
              0.348,
              0.6149
            ],
            "ci_width": 0.2669
          },
          "world_religions": {
            "correct": 42,
            "total": 50,
            "accuracy": 0.84,
            "ci95": [
              0.7149,
              0.9166
            ],
            "ci_width": 0.2018
          },
          "econometrics": {
            "correct": 32,
            "total": 50,
            "accuracy": 0.64,
            "ci95": [
              0.5014,
              0.7586
            ],
            "ci_width": 0.2572
          },
          "high_school_geography": {
            "correct": 42,
            "total": 50,
            "accuracy": 0.84,
            "ci95": [
              0.7149,
              0.9166
            ],
            "ci_width": 0.2018
          },
          "high_school_government_and_politics": {
            "correct": 48,
            "total": 50,
            "accuracy": 0.96,
            "ci95": [
              0.8654,
              0.989
            ],
            "ci_width": 0.1236
          },
          "high_school_macroeconomics": {
            "correct": 42,
            "total": 50,
            "accuracy": 0.84,
            "ci95": [
              0.7149,
              0.9166
            ],
            "ci_width": 0.2018
          },
          "high_school_microeconomics": {
            "correct": 44,
            "total": 50,
            "accuracy": 0.88,
            "ci95": [
              0.7619,
              0.9438
            ],
            "ci_width": 0.1819
          },
          "high_school_psychology": {
            "correct": 49,
            "total": 50,
            "accuracy": 0.98,
            "ci95": [
              0.895,
              0.9965
            ],
            "ci_width": 0.1014
          },
          "human_sexuality": {
            "correct": 46,
            "total": 50,
            "accuracy": 0.92,
            "ci95": [
              0.8116,
              0.9685
            ],
            "ci_width": 0.1568
          },
          "professional_psychology": {
            "correct": 40,
            "total": 50,
            "accuracy": 0.8,
            "ci95": [
              0.6696,
              0.8876
            ],
            "ci_width": 0.2179
          },
          "public_relations": {
            "correct": 31,
            "total": 50,
            "accuracy": 0.62,
            "ci95": [
              0.4815,
              0.7414
            ],
            "ci_width": 0.2599
          },
          "security_studies": {
            "correct": 32,
            "total": 50,
            "accuracy": 0.64,
            "ci95": [
              0.5014,
              0.7586
            ],
            "ci_width": 0.2572
          },
          "sociology": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "us_foreign_policy": {
            "correct": 47,
            "total": 50,
            "accuracy": 0.94,
            "ci95": [
              0.8378,
              0.9794
            ],
            "ci_width": 0.1416
          },
          "anatomy": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "business_ethics": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "clinical_knowledge": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "college_medicine": {
            "correct": 48,
            "total": 50,
            "accuracy": 0.96,
            "ci95": [
              0.8654,
              0.989
            ],
            "ci_width": 0.1236
          },
          "global_facts": {
            "correct": 24,
            "total": 50,
            "accuracy": 0.48,
            "ci95": [
              0.348,
              0.6149
            ],
            "ci_width": 0.2669
          },
          "human_aging": {
            "correct": 39,
            "total": 50,
            "accuracy": 0.78,
            "ci95": [
              0.6476,
              0.8725
            ],
            "ci_width": 0.2249
          },
          "management": {
            "correct": 45,
            "total": 50,
            "accuracy": 0.9,
            "ci95": [
              0.7864,
              0.9565
            ],
            "ci_width": 0.1701
          },
          "marketing": {
            "correct": 46,
            "total": 50,
            "accuracy": 0.92,
            "ci95": [
              0.8116,
              0.9685
            ],
            "ci_width": 0.1568
          },
          "medical_genetics": {
            "correct": 47,
            "total": 50,
            "accuracy": 0.94,
            "ci95": [
              0.8378,
              0.9794
            ],
            "ci_width": 0.1416
          },
          "miscellaneous": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "nutrition": {
            "correct": 41,
            "total": 50,
            "accuracy": 0.82,
            "ci95": [
              0.692,
              0.9023
            ],
            "ci_width": 0.2103
          },
          "professional_accounting": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "professional_medicine": {
            "correct": 40,
            "total": 50,
            "accuracy": 0.8,
            "ci95": [
              0.6696,
              0.8876
            ],
            "ci_width": 0.2179
          },
          "virology": {
            "correct": 27,
            "total": 50,
            "accuracy": 0.54,
            "ci95": [
              0.404,
              0.6703
            ],
            "ci_width": 0.2663
          }
        },
        "category_accuracy": {
          "stem": {
            "correct": 722,
            "total": 900,
            "accuracy": 0.8022,
            "ci95": [
              0.7749,
              0.8269
            ],
            "ci_width": 0.052
          },
          "humanities": {
            "correct": 486,
            "total": 650,
            "accuracy": 0.7477,
            "ci95": [
              0.7129,
              0.7796
            ],
            "ci_width": 0.0666
          },
          "social_sciences": {
            "correct": 496,
            "total": 600,
            "accuracy": 0.8267,
            "ci95": [
              0.7943,
              0.8549
            ],
            "ci_width": 0.0605
          },
          "other": {
            "correct": 556,
            "total": 700,
            "accuracy": 0.7943,
            "ci95": [
              0.7628,
              0.8226
            ],
            "ci_width": 0.0598
          }
        },
        "option_bias": {
          "predicted_share": {
            "A": 0.2196,
            "B": 0.2723,
            "C": 0.2516,
            "D": 0.2509
          },
          "gold_share": {
            "A": 0.2379,
            "B": 0.2516,
            "C": 0.2477,
            "D": 0.2628
          },
          "total_variation_distance": 0.0274,
          "most_favoured_letter": "B"
        },
        "n_questions": 2850,
        "n_subjects": 57
      },
      "probability": {
        "available": false,
        "reason": "provider did not return token logprobs for this model",
        "coverage": 0.0
      },
      "consistency": {
        "available": false,
        "reason": "repeat/seed runs not enabled for this evaluation"
      },
      "robustness": {
        "available": false,
        "reason": "robustness sweep not enabled for this evaluation"
      },
      "api_performance": {
        "available": true,
        "latency": {
          "ttft": {
            "metric": "TTFT (s)",
            "mean": 0.4135,
            "std": 0.5063,
            "min": 0.2393,
            "p50": 0.3134,
            "p95": 0.9887,
            "p99": 1.6599,
            "max": 13.468,
            "n": 2850,
            "source": "measured"
          },
          "e2e": {
            "metric": "End-to-end (s)",
            "mean": 1.3714,
            "std": 1.0654,
            "min": 0.487,
            "p50": 1.1224,
            "p95": 2.7382,
            "p99": 4.7685,
            "max": 22.6215,
            "n": 2850,
            "source": "measured"
          },
          "tpot": {
            "metric": "TPOT (s/token)",
            "mean": 0.0105,
            "std": 0.0071,
            "min": 0.0003,
            "p50": 0.0092,
            "p95": 0.0187,
            "p99": 0.0295,
            "max": 0.2076,
            "n": 2850,
            "source": "measured"
          }
        },
        "throughput": {
          "prefill_tokens_per_s": 718.13,
          "decode_tokens_per_s": 114.44,
          "total_tokens_per_s": 185.02,
          "requests_per_s": 0.5568,
          "items_per_s": 0.5568
        },
        "wall_seconds": 5118.19,
        "source": "measured",
        "caveat": "TTFT includes network round-trip and provider queueing, so TTFT-derived prefill throughput is a lower bound on true prefill speed"
      },
      "token_efficiency": {
        "available": true,
        "prompt_tokens_total": 688211,
        "completion_tokens_total": 258736,
        "total_tokens": 946947,
        "mean_prompt_tokens": 241.5,
        "mean_completion_tokens": 90.8,
        "tokens_per_item": 332.3,
        "tokens_per_correct_answer": 419.0,
        "source": "provider_reported",
        "reasoning_tokens_total": 0,
        "mean_reasoning_tokens": 0.0,
        "reasoning_token_share": 0.0,
        "cached_prompt_tokens_total": 466048,
        "prompt_cache_hit_rate": 0.6772
      },
      "economics": {
        "available": true,
        "total_usd": 0.116086,
        "mean_per_request_usd": 4.073e-05,
        "per_1k_requests_usd": 0.0407,
        "cost_per_correct_answer_usd": 5.1e-05,
        "n_requests": 2850,
        "source": "provider_reported",
        "cost_per_1k_tokens_usd": 0.000123,
        "cost_per_1m_tokens_usd": 0.1226,
        "correct_answers_per_usd": 19468.3
      },
      "reliability": {
        "available": true,
        "n_requests": 2850,
        "successful_requests": 2850,
        "success_rate": 1.0,
        "failure_rate": 0,
        "invalid_output_rate": 0.0056,
        "timeout_rate": 0,
        "rate_limit_rate": 0,
        "server_error_rate": 0,
        "error_classes": {},
        "retry_total": 0,
        "retried_request_rate": 0,
        "retries_per_request": 0.0,
        "providers_used": {
          "Mistral": 2850
        },
        "provider_failover": false,
        "source": "measured"
      },
      "config": {
        "run_id": "mistralai_ministral-8b-2512_mmlu_TP1_PP1_DP1_SP1_CP1_EP1_20260908-043122",
        "timestamp": "2026-09-08T04:31:22",
        "schema_version": "2.0",
        "model": "mistralai/ministral-8b-2512",
        "parallelism": {
          "tensor_parallel": 1,
          "pipeline_parallel": 1,
          "data_parallel": 1,
          "sequence_parallel": 1,
          "context_parallel": 1,
          "expert_parallel": 1,
          "controlled": false,
          "note": "hosted endpoint: layout chosen by the provider, not observable or settable from the client"
        },
        "serving": {
          "backend": "openrouter",
          "dtype": "unknown",
          "quantization": null,
          "kv_cache_dtype": "unknown",
          "max_batch_size": null,
          "max_model_len": null,
          "gpu_memory_utilization": null,
          "enable_prefix_caching": null,
          "speculative_decoding": false,
          "device": "remote",
          "device_tdp_watts": null,
          "controlled": false
        },
        "decoding": {
          "temperature": 0.0,
          "top_p": 1.0,
          "top_k": null,
          "max_tokens": 384,
          "frequency_penalty": 0.0,
          "presence_penalty": 0.0,
          "seed": 42,
          "few_shot": 0
        },
        "dataset": {
          "benchmark": "mmlu",
          "split": "test",
          "subjects": [],
          "n_items": 2850,
          "questions_per_subject": null,
          "sample_seed": 42,
          "prompt_template": "baseline"
        },
        "evaluation": {
          "request_logprobs": true,
          "top_logprobs": 5,
          "stream_for_latency": true,
          "repeats_for_consistency": 1,
          "seeds_for_stability": [],
          "robustness_variants": [],
          "context_ablations": [],
          "calibration_bins": 10
        },
        "environment": {
          "python": "3.10.20",
          "platform": "macOS-26.6.2-x86_64-i386-64bit",
          "processor": "i386",
          "git_commit": "2949ebc"
        },
        "notes": ""
      },
      "taxonomy": {
        "deployment": "openrouter_hosted_api",
        "stages": [
          {
            "stage": 1,
            "layer": "Task Quality",
            "key": "task_quality",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 2,
            "layer": "Probabilistic Quality",
            "key": "probability",
            "priority": "P1",
            "openrouter": "conditional",
            "requires": "provider that returns token log-probabilities"
          },
          {
            "stage": 3,
            "layer": "Reasoning & Consistency",
            "key": "consistency",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "repeats > 1 and/or multiple seeds"
          },
          {
            "stage": 4,
            "layer": "Context Behavior",
            "key": "context",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "context ablation pass"
          },
          {
            "stage": 5,
            "layer": "Robustness",
            "key": "robustness",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "robustness pass"
          },
          {
            "stage": 6,
            "layer": "API Performance",
            "key": "api_performance",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 7,
            "layer": "Token Efficiency",
            "key": "token_efficiency",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 8,
            "layer": "Economics",
            "key": "economics",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 9,
            "layer": "Reliability",
            "key": "reliability",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          }
        ],
        "excluded": {
          "stage": 10,
          "layer": "Hardware & Distributed",
          "reason": "The benchmark is a client of a shared, auto-scaled third-party endpoint. Parallelism layout, GPU telemetry and power draw are chosen and held by the provider; none of them is exposed to the caller. Any figure reported here would be fabricated.",
          "path_to_measure": "Run the same suite against a self-hosted vLLM backend (benchmark_config.local_vllm_config), where TP/PP/DP/SP/CP/EP become settable variables and VRAM, KV cache, communication overhead and energy become directly measurable."
        }
      },
      "display_name": "Ministral-8B",
      "developer": "Mistral AI",
      "param_count": "8B",
      "colour": "#FF9800"
    },
    {
      "model": "meta-llama/llama-3.2-3b-instruct",
      "benchmark": "mmlu",
      "task_quality": {
        "overall_accuracy": 0.5467,
        "overall_accuracy_ci95": [
          0.5283,
          0.5649
        ],
        "macro_accuracy_subject": 0.5467,
        "macro_accuracy_category": 0.5574,
        "normalised_accuracy": 0.3956,
        "random_baseline": 0.25,
        "error_rate": 0.4533,
        "parse_failure_rate": 0.1172,
        "subject_accuracy": {
          "abstract_algebra": {
            "correct": 13,
            "total": 50,
            "accuracy": 0.26,
            "ci95": [
              0.1587,
              0.3955
            ],
            "ci_width": 0.2368
          },
          "astronomy": {
            "correct": 31,
            "total": 50,
            "accuracy": 0.62,
            "ci95": [
              0.4815,
              0.7414
            ],
            "ci_width": 0.2599
          },
          "college_biology": {
            "correct": 39,
            "total": 50,
            "accuracy": 0.78,
            "ci95": [
              0.6476,
              0.8725
            ],
            "ci_width": 0.2249
          },
          "college_chemistry": {
            "correct": 14,
            "total": 50,
            "accuracy": 0.28,
            "ci95": [
              0.1747,
              0.4167
            ],
            "ci_width": 0.2419
          },
          "college_computer_science": {
            "correct": 25,
            "total": 50,
            "accuracy": 0.5,
            "ci95": [
              0.3664,
              0.6336
            ],
            "ci_width": 0.2671
          },
          "college_mathematics": {
            "correct": 9,
            "total": 50,
            "accuracy": 0.18,
            "ci95": [
              0.0977,
              0.308
            ],
            "ci_width": 0.2103
          },
          "college_physics": {
            "correct": 19,
            "total": 50,
            "accuracy": 0.38,
            "ci95": [
              0.2586,
              0.5185
            ],
            "ci_width": 0.2599
          },
          "computer_security": {
            "correct": 38,
            "total": 50,
            "accuracy": 0.76,
            "ci95": [
              0.6259,
              0.857
            ],
            "ci_width": 0.2312
          },
          "conceptual_physics": {
            "correct": 27,
            "total": 50,
            "accuracy": 0.54,
            "ci95": [
              0.404,
              0.6703
            ],
            "ci_width": 0.2663
          },
          "electrical_engineering": {
            "correct": 20,
            "total": 50,
            "accuracy": 0.4,
            "ci95": [
              0.2761,
              0.5382
            ],
            "ci_width": 0.2621
          },
          "elementary_mathematics": {
            "correct": 8,
            "total": 50,
            "accuracy": 0.16,
            "ci95": [
              0.0834,
              0.2851
            ],
            "ci_width": 0.2018
          },
          "high_school_biology": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "high_school_chemistry": {
            "correct": 24,
            "total": 50,
            "accuracy": 0.48,
            "ci95": [
              0.348,
              0.6149
            ],
            "ci_width": 0.2669
          },
          "high_school_computer_science": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "high_school_mathematics": {
            "correct": 5,
            "total": 50,
            "accuracy": 0.1,
            "ci95": [
              0.0435,
              0.2136
            ],
            "ci_width": 0.1701
          },
          "high_school_physics": {
            "correct": 14,
            "total": 50,
            "accuracy": 0.28,
            "ci95": [
              0.1747,
              0.4167
            ],
            "ci_width": 0.2419
          },
          "high_school_statistics": {
            "correct": 19,
            "total": 50,
            "accuracy": 0.38,
            "ci95": [
              0.2586,
              0.5185
            ],
            "ci_width": 0.2599
          },
          "machine_learning": {
            "correct": 20,
            "total": 50,
            "accuracy": 0.4,
            "ci95": [
              0.2761,
              0.5382
            ],
            "ci_width": 0.2621
          },
          "formal_logic": {
            "correct": 23,
            "total": 50,
            "accuracy": 0.46,
            "ci95": [
              0.3297,
              0.596
            ],
            "ci_width": 0.2663
          },
          "high_school_european_history": {
            "correct": 33,
            "total": 50,
            "accuracy": 0.66,
            "ci95": [
              0.5215,
              0.7756
            ],
            "ci_width": 0.2541
          },
          "high_school_us_history": {
            "correct": 30,
            "total": 50,
            "accuracy": 0.6,
            "ci95": [
              0.4618,
              0.7239
            ],
            "ci_width": 0.2621
          },
          "high_school_world_history": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "international_law": {
            "correct": 30,
            "total": 50,
            "accuracy": 0.6,
            "ci95": [
              0.4618,
              0.7239
            ],
            "ci_width": 0.2621
          },
          "jurisprudence": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "logical_fallacies": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "moral_disputes": {
            "correct": 22,
            "total": 50,
            "accuracy": 0.44,
            "ci95": [
              0.3116,
              0.5769
            ],
            "ci_width": 0.2653
          },
          "moral_scenarios": {
            "correct": 16,
            "total": 50,
            "accuracy": 0.32,
            "ci95": [
              0.2076,
              0.4581
            ],
            "ci_width": 0.2505
          },
          "philosophy": {
            "correct": 31,
            "total": 50,
            "accuracy": 0.62,
            "ci95": [
              0.4815,
              0.7414
            ],
            "ci_width": 0.2599
          },
          "prehistory": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "professional_law": {
            "correct": 17,
            "total": 50,
            "accuracy": 0.34,
            "ci95": [
              0.2244,
              0.4785
            ],
            "ci_width": 0.2541
          },
          "world_religions": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "econometrics": {
            "correct": 19,
            "total": 50,
            "accuracy": 0.38,
            "ci95": [
              0.2586,
              0.5185
            ],
            "ci_width": 0.2599
          },
          "high_school_geography": {
            "correct": 30,
            "total": 50,
            "accuracy": 0.6,
            "ci95": [
              0.4618,
              0.7239
            ],
            "ci_width": 0.2621
          },
          "high_school_government_and_politics": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "high_school_macroeconomics": {
            "correct": 31,
            "total": 50,
            "accuracy": 0.62,
            "ci95": [
              0.4815,
              0.7414
            ],
            "ci_width": 0.2599
          },
          "high_school_microeconomics": {
            "correct": 31,
            "total": 50,
            "accuracy": 0.62,
            "ci95": [
              0.4815,
              0.7414
            ],
            "ci_width": 0.2599
          },
          "high_school_psychology": {
            "correct": 39,
            "total": 50,
            "accuracy": 0.78,
            "ci95": [
              0.6476,
              0.8725
            ],
            "ci_width": 0.2249
          },
          "human_sexuality": {
            "correct": 28,
            "total": 50,
            "accuracy": 0.56,
            "ci95": [
              0.4231,
              0.6884
            ],
            "ci_width": 0.2653
          },
          "professional_psychology": {
            "correct": 28,
            "total": 50,
            "accuracy": 0.56,
            "ci95": [
              0.4231,
              0.6884
            ],
            "ci_width": 0.2653
          },
          "public_relations": {
            "correct": 28,
            "total": 50,
            "accuracy": 0.56,
            "ci95": [
              0.4231,
              0.6884
            ],
            "ci_width": 0.2653
          },
          "security_studies": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "sociology": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "us_foreign_policy": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "anatomy": {
            "correct": 26,
            "total": 50,
            "accuracy": 0.52,
            "ci95": [
              0.3851,
              0.652
            ],
            "ci_width": 0.2669
          },
          "business_ethics": {
            "correct": 33,
            "total": 50,
            "accuracy": 0.66,
            "ci95": [
              0.5215,
              0.7756
            ],
            "ci_width": 0.2541
          },
          "clinical_knowledge": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "college_medicine": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "global_facts": {
            "correct": 14,
            "total": 50,
            "accuracy": 0.28,
            "ci95": [
              0.1747,
              0.4167
            ],
            "ci_width": 0.2419
          },
          "human_aging": {
            "correct": 33,
            "total": 50,
            "accuracy": 0.66,
            "ci95": [
              0.5215,
              0.7756
            ],
            "ci_width": 0.2541
          },
          "management": {
            "correct": 37,
            "total": 50,
            "accuracy": 0.74,
            "ci95": [
              0.6045,
              0.8413
            ],
            "ci_width": 0.2368
          },
          "marketing": {
            "correct": 39,
            "total": 50,
            "accuracy": 0.78,
            "ci95": [
              0.6476,
              0.8725
            ],
            "ci_width": 0.2249
          },
          "medical_genetics": {
            "correct": 33,
            "total": 50,
            "accuracy": 0.66,
            "ci95": [
              0.5215,
              0.7756
            ],
            "ci_width": 0.2541
          },
          "miscellaneous": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "nutrition": {
            "correct": 37,
            "total": 50,
            "accuracy": 0.74,
            "ci95": [
              0.6045,
              0.8413
            ],
            "ci_width": 0.2368
          },
          "professional_accounting": {
            "correct": 20,
            "total": 50,
            "accuracy": 0.4,
            "ci95": [
              0.2761,
              0.5382
            ],
            "ci_width": 0.2621
          },
          "professional_medicine": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "virology": {
            "correct": 25,
            "total": 50,
            "accuracy": 0.5,
            "ci95": [
              0.3664,
              0.6336
            ],
            "ci_width": 0.2671
          }
        },
        "category_accuracy": {
          "stem": {
            "correct": 389,
            "total": 900,
            "accuracy": 0.4322,
            "ci95": [
              0.4002,
              0.4648
            ],
            "ci_width": 0.0646
          },
          "humanities": {
            "correct": 376,
            "total": 650,
            "accuracy": 0.5785,
            "ci95": [
              0.5401,
              0.6159
            ],
            "ci_width": 0.0757
          },
          "social_sciences": {
            "correct": 362,
            "total": 600,
            "accuracy": 0.6033,
            "ci95": [
              0.5637,
              0.6417
            ],
            "ci_width": 0.0781
          },
          "other": {
            "correct": 431,
            "total": 700,
            "accuracy": 0.6157,
            "ci95": [
              0.5791,
              0.651
            ],
            "ci_width": 0.0719
          }
        },
        "option_bias": {
          "predicted_share": {
            "A": 0.2302,
            "B": 0.2165,
            "C": 0.2418,
            "D": 0.1944
          },
          "gold_share": {
            "A": 0.2379,
            "B": 0.2516,
            "C": 0.2477,
            "D": 0.2628
          },
          "total_variation_distance": 0.0586,
          "most_favoured_letter": "C"
        },
        "n_questions": 2850,
        "n_subjects": 57
      },
      "probability": {
        "available": false,
        "reason": "provider did not return token logprobs for this model",
        "coverage": 0.0
      },
      "consistency": {
        "available": false,
        "reason": "repeat/seed runs not enabled for this evaluation"
      },
      "robustness": {
        "available": false,
        "reason": "robustness sweep not enabled for this evaluation"
      },
      "api_performance": {
        "available": true,
        "latency": {
          "ttft": {
            "metric": "TTFT (s)",
            "mean": 0.3296,
            "std": 0.2517,
            "min": 0.1585,
            "p50": 0.2759,
            "p95": 0.734,
            "p99": 1.4116,
            "max": 2.825,
            "n": 2850,
            "source": "measured"
          },
          "e2e": {
            "metric": "End-to-end (s)",
            "mean": 0.6522,
            "std": 0.3091,
            "min": 0.2029,
            "p50": 0.5661,
            "p95": 1.1713,
            "p99": 1.8639,
            "max": 3.4858,
            "n": 2850,
            "source": "measured"
          },
          "tpot": {
            "metric": "TPOT (s/token)",
            "mean": 0.004,
            "std": 0.0016,
            "min": 0.0002,
            "p50": 0.0035,
            "p95": 0.0074,
            "p99": 0.0102,
            "max": 0.018,
            "n": 2490,
            "source": "measured"
          }
        },
        "throughput": {
          "prefill_tokens_per_s": 1022.58,
          "decode_tokens_per_s": 257.26,
          "total_tokens_per_s": 316.81,
          "requests_per_s": 1.0138,
          "items_per_s": 1.0138
        },
        "wall_seconds": 2811.13,
        "source": "measured",
        "caveat": "TTFT includes network round-trip and provider queueing, so TTFT-derived prefill throughput is a lower bound on true prefill speed"
      },
      "token_efficiency": {
        "available": true,
        "prompt_tokens_total": 676638,
        "completion_tokens_total": 213957,
        "total_tokens": 890595,
        "mean_prompt_tokens": 237.4,
        "mean_completion_tokens": 75.1,
        "tokens_per_item": 312.5,
        "tokens_per_correct_answer": 571.6,
        "source": "provider_reported",
        "reasoning_tokens_total": 0,
        "mean_reasoning_tokens": 0.0,
        "reasoning_token_share": 0.0,
        "cached_prompt_tokens_total": 0,
        "prompt_cache_hit_rate": 0.0
      },
      "economics": {
        "available": true,
        "total_usd": 0.129379,
        "mean_per_request_usd": 4.54e-05,
        "per_1k_requests_usd": 0.0454,
        "cost_per_correct_answer_usd": 8.3e-05,
        "n_requests": 2850,
        "source": "provider_reported",
        "cost_per_1k_tokens_usd": 0.000145,
        "cost_per_1m_tokens_usd": 0.1453,
        "correct_answers_per_usd": 12042.1
      },
      "reliability": {
        "available": true,
        "n_requests": 2850,
        "successful_requests": 2850,
        "success_rate": 1.0,
        "failure_rate": 0,
        "invalid_output_rate": 0.1172,
        "timeout_rate": 0,
        "rate_limit_rate": 0,
        "server_error_rate": 0,
        "error_classes": {},
        "retry_total": 0,
        "retried_request_rate": 0,
        "retries_per_request": 0.0,
        "providers_used": {
          "Cloudflare": 2850
        },
        "provider_failover": false,
        "source": "measured"
      },
      "config": {
        "run_id": "meta-llama_llama-3.2-3b-instruct_mmlu_TP1_PP1_DP1_SP1_CP1_EP1_20260908-023726",
        "timestamp": "2026-09-08T02:37:26",
        "schema_version": "2.0",
        "model": "meta-llama/llama-3.2-3b-instruct",
        "parallelism": {
          "tensor_parallel": 1,
          "pipeline_parallel": 1,
          "data_parallel": 1,
          "sequence_parallel": 1,
          "context_parallel": 1,
          "expert_parallel": 1,
          "controlled": false,
          "note": "hosted endpoint: layout chosen by the provider, not observable or settable from the client"
        },
        "serving": {
          "backend": "openrouter",
          "dtype": "unknown",
          "quantization": null,
          "kv_cache_dtype": "unknown",
          "max_batch_size": null,
          "max_model_len": null,
          "gpu_memory_utilization": null,
          "enable_prefix_caching": null,
          "speculative_decoding": false,
          "device": "remote",
          "device_tdp_watts": null,
          "controlled": false
        },
        "decoding": {
          "temperature": 0.0,
          "top_p": 1.0,
          "top_k": null,
          "max_tokens": 384,
          "frequency_penalty": 0.0,
          "presence_penalty": 0.0,
          "seed": 42,
          "few_shot": 0
        },
        "dataset": {
          "benchmark": "mmlu",
          "split": "test",
          "subjects": [],
          "n_items": 2850,
          "questions_per_subject": null,
          "sample_seed": 42,
          "prompt_template": "baseline"
        },
        "evaluation": {
          "request_logprobs": true,
          "top_logprobs": 5,
          "stream_for_latency": true,
          "repeats_for_consistency": 1,
          "seeds_for_stability": [],
          "robustness_variants": [],
          "context_ablations": [],
          "calibration_bins": 10
        },
        "environment": {
          "python": "3.10.20",
          "platform": "macOS-26.6.2-x86_64-i386-64bit",
          "processor": "i386",
          "git_commit": "2949ebc"
        },
        "notes": ""
      },
      "taxonomy": {
        "deployment": "openrouter_hosted_api",
        "stages": [
          {
            "stage": 1,
            "layer": "Task Quality",
            "key": "task_quality",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 2,
            "layer": "Probabilistic Quality",
            "key": "probability",
            "priority": "P1",
            "openrouter": "conditional",
            "requires": "provider that returns token log-probabilities"
          },
          {
            "stage": 3,
            "layer": "Reasoning & Consistency",
            "key": "consistency",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "repeats > 1 and/or multiple seeds"
          },
          {
            "stage": 4,
            "layer": "Context Behavior",
            "key": "context",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "context ablation pass"
          },
          {
            "stage": 5,
            "layer": "Robustness",
            "key": "robustness",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "robustness pass"
          },
          {
            "stage": 6,
            "layer": "API Performance",
            "key": "api_performance",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 7,
            "layer": "Token Efficiency",
            "key": "token_efficiency",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 8,
            "layer": "Economics",
            "key": "economics",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 9,
            "layer": "Reliability",
            "key": "reliability",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          }
        ],
        "excluded": {
          "stage": 10,
          "layer": "Hardware & Distributed",
          "reason": "The benchmark is a client of a shared, auto-scaled third-party endpoint. Parallelism layout, GPU telemetry and power draw are chosen and held by the provider; none of them is exposed to the caller. Any figure reported here would be fabricated.",
          "path_to_measure": "Run the same suite against a self-hosted vLLM backend (benchmark_config.local_vllm_config), where TP/PP/DP/SP/CP/EP become settable variables and VRAM, KV cache, communication overhead and energy become directly measurable."
        }
      },
      "display_name": "Llama-3.2-3B",
      "developer": "Meta",
      "param_count": "3B",
      "colour": "#4CAF50"
    },
    {
      "model": "google/gemma-3-4b-it",
      "benchmark": "mmlu",
      "task_quality": {
        "overall_accuracy": 0.6035,
        "overall_accuracy_ci95": [
          0.5854,
          0.6213
        ],
        "macro_accuracy_subject": 0.6035,
        "macro_accuracy_category": 0.6087,
        "normalised_accuracy": 0.4713,
        "random_baseline": 0.25,
        "error_rate": 0.3965,
        "parse_failure_rate": 0.0039,
        "subject_accuracy": {
          "abstract_algebra": {
            "correct": 16,
            "total": 50,
            "accuracy": 0.32,
            "ci95": [
              0.2076,
              0.4581
            ],
            "ci_width": 0.2505
          },
          "astronomy": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "college_biology": {
            "correct": 37,
            "total": 50,
            "accuracy": 0.74,
            "ci95": [
              0.6045,
              0.8413
            ],
            "ci_width": 0.2368
          },
          "college_chemistry": {
            "correct": 17,
            "total": 50,
            "accuracy": 0.34,
            "ci95": [
              0.2244,
              0.4785
            ],
            "ci_width": 0.2541
          },
          "college_computer_science": {
            "correct": 27,
            "total": 50,
            "accuracy": 0.54,
            "ci95": [
              0.404,
              0.6703
            ],
            "ci_width": 0.2663
          },
          "college_mathematics": {
            "correct": 25,
            "total": 50,
            "accuracy": 0.5,
            "ci95": [
              0.3664,
              0.6336
            ],
            "ci_width": 0.2671
          },
          "college_physics": {
            "correct": 19,
            "total": 50,
            "accuracy": 0.38,
            "ci95": [
              0.2586,
              0.5185
            ],
            "ci_width": 0.2599
          },
          "computer_security": {
            "correct": 39,
            "total": 50,
            "accuracy": 0.78,
            "ci95": [
              0.6476,
              0.8725
            ],
            "ci_width": 0.2249
          },
          "conceptual_physics": {
            "correct": 28,
            "total": 50,
            "accuracy": 0.56,
            "ci95": [
              0.4231,
              0.6884
            ],
            "ci_width": 0.2653
          },
          "electrical_engineering": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "elementary_mathematics": {
            "correct": 31,
            "total": 50,
            "accuracy": 0.62,
            "ci95": [
              0.4815,
              0.7414
            ],
            "ci_width": 0.2599
          },
          "high_school_biology": {
            "correct": 33,
            "total": 50,
            "accuracy": 0.66,
            "ci95": [
              0.5215,
              0.7756
            ],
            "ci_width": 0.2541
          },
          "high_school_chemistry": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "high_school_computer_science": {
            "correct": 38,
            "total": 50,
            "accuracy": 0.76,
            "ci95": [
              0.6259,
              0.857
            ],
            "ci_width": 0.2312
          },
          "high_school_mathematics": {
            "correct": 25,
            "total": 50,
            "accuracy": 0.5,
            "ci95": [
              0.3664,
              0.6336
            ],
            "ci_width": 0.2671
          },
          "high_school_physics": {
            "correct": 13,
            "total": 50,
            "accuracy": 0.26,
            "ci95": [
              0.1587,
              0.3955
            ],
            "ci_width": 0.2368
          },
          "high_school_statistics": {
            "correct": 24,
            "total": 50,
            "accuracy": 0.48,
            "ci95": [
              0.348,
              0.6149
            ],
            "ci_width": 0.2669
          },
          "machine_learning": {
            "correct": 32,
            "total": 50,
            "accuracy": 0.64,
            "ci95": [
              0.5014,
              0.7586
            ],
            "ci_width": 0.2572
          },
          "formal_logic": {
            "correct": 23,
            "total": 50,
            "accuracy": 0.46,
            "ci95": [
              0.3297,
              0.596
            ],
            "ci_width": 0.2663
          },
          "high_school_european_history": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "high_school_us_history": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "high_school_world_history": {
            "correct": 40,
            "total": 50,
            "accuracy": 0.8,
            "ci95": [
              0.6696,
              0.8876
            ],
            "ci_width": 0.2179
          },
          "international_law": {
            "correct": 40,
            "total": 50,
            "accuracy": 0.8,
            "ci95": [
              0.6696,
              0.8876
            ],
            "ci_width": 0.2179
          },
          "jurisprudence": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "logical_fallacies": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "moral_disputes": {
            "correct": 27,
            "total": 50,
            "accuracy": 0.54,
            "ci95": [
              0.404,
              0.6703
            ],
            "ci_width": 0.2663
          },
          "moral_scenarios": {
            "correct": 14,
            "total": 50,
            "accuracy": 0.28,
            "ci95": [
              0.1747,
              0.4167
            ],
            "ci_width": 0.2419
          },
          "philosophy": {
            "correct": 28,
            "total": 50,
            "accuracy": 0.56,
            "ci95": [
              0.4231,
              0.6884
            ],
            "ci_width": 0.2653
          },
          "prehistory": {
            "correct": 32,
            "total": 50,
            "accuracy": 0.64,
            "ci95": [
              0.5014,
              0.7586
            ],
            "ci_width": 0.2572
          },
          "professional_law": {
            "correct": 15,
            "total": 50,
            "accuracy": 0.3,
            "ci95": [
              0.191,
              0.4375
            ],
            "ci_width": 0.2465
          },
          "world_religions": {
            "correct": 38,
            "total": 50,
            "accuracy": 0.76,
            "ci95": [
              0.6259,
              0.857
            ],
            "ci_width": 0.2312
          },
          "econometrics": {
            "correct": 22,
            "total": 50,
            "accuracy": 0.44,
            "ci95": [
              0.3116,
              0.5769
            ],
            "ci_width": 0.2653
          },
          "high_school_geography": {
            "correct": 30,
            "total": 50,
            "accuracy": 0.6,
            "ci95": [
              0.4618,
              0.7239
            ],
            "ci_width": 0.2621
          },
          "high_school_government_and_politics": {
            "correct": 41,
            "total": 50,
            "accuracy": 0.82,
            "ci95": [
              0.692,
              0.9023
            ],
            "ci_width": 0.2103
          },
          "high_school_macroeconomics": {
            "correct": 32,
            "total": 50,
            "accuracy": 0.64,
            "ci95": [
              0.5014,
              0.7586
            ],
            "ci_width": 0.2572
          },
          "high_school_microeconomics": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "high_school_psychology": {
            "correct": 37,
            "total": 50,
            "accuracy": 0.74,
            "ci95": [
              0.6045,
              0.8413
            ],
            "ci_width": 0.2368
          },
          "human_sexuality": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "professional_psychology": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "public_relations": {
            "correct": 27,
            "total": 50,
            "accuracy": 0.54,
            "ci95": [
              0.404,
              0.6703
            ],
            "ci_width": 0.2663
          },
          "security_studies": {
            "correct": 30,
            "total": 50,
            "accuracy": 0.6,
            "ci95": [
              0.4618,
              0.7239
            ],
            "ci_width": 0.2621
          },
          "sociology": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "us_foreign_policy": {
            "correct": 38,
            "total": 50,
            "accuracy": 0.76,
            "ci95": [
              0.6259,
              0.857
            ],
            "ci_width": 0.2312
          },
          "anatomy": {
            "correct": 28,
            "total": 50,
            "accuracy": 0.56,
            "ci95": [
              0.4231,
              0.6884
            ],
            "ci_width": 0.2653
          },
          "business_ethics": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "clinical_knowledge": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "college_medicine": {
            "correct": 29,
            "total": 50,
            "accuracy": 0.58,
            "ci95": [
              0.4423,
              0.7063
            ],
            "ci_width": 0.2639
          },
          "global_facts": {
            "correct": 20,
            "total": 50,
            "accuracy": 0.4,
            "ci95": [
              0.2761,
              0.5382
            ],
            "ci_width": 0.2621
          },
          "human_aging": {
            "correct": 34,
            "total": 50,
            "accuracy": 0.68,
            "ci95": [
              0.5419,
              0.7924
            ],
            "ci_width": 0.2505
          },
          "management": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "marketing": {
            "correct": 43,
            "total": 50,
            "accuracy": 0.86,
            "ci95": [
              0.7381,
              0.9305
            ],
            "ci_width": 0.1924
          },
          "medical_genetics": {
            "correct": 37,
            "total": 50,
            "accuracy": 0.74,
            "ci95": [
              0.6045,
              0.8413
            ],
            "ci_width": 0.2368
          },
          "miscellaneous": {
            "correct": 36,
            "total": 50,
            "accuracy": 0.72,
            "ci95": [
              0.5833,
              0.8253
            ],
            "ci_width": 0.2419
          },
          "nutrition": {
            "correct": 35,
            "total": 50,
            "accuracy": 0.7,
            "ci95": [
              0.5625,
              0.809
            ],
            "ci_width": 0.2465
          },
          "professional_accounting": {
            "correct": 21,
            "total": 50,
            "accuracy": 0.42,
            "ci95": [
              0.2937,
              0.5577
            ],
            "ci_width": 0.2639
          },
          "professional_medicine": {
            "correct": 27,
            "total": 50,
            "accuracy": 0.54,
            "ci95": [
              0.404,
              0.6703
            ],
            "ci_width": 0.2663
          },
          "virology": {
            "correct": 27,
            "total": 50,
            "accuracy": 0.54,
            "ci95": [
              0.404,
              0.6703
            ],
            "ci_width": 0.2663
          }
        },
        "category_accuracy": {
          "stem": {
            "correct": 497,
            "total": 900,
            "accuracy": 0.5522,
            "ci95": [
              0.5196,
              0.5844
            ],
            "ci_width": 0.0648
          },
          "humanities": {
            "correct": 396,
            "total": 650,
            "accuracy": 0.6092,
            "ci95": [
              0.5712,
              0.646
            ],
            "ci_width": 0.0748
          },
          "social_sciences": {
            "correct": 386,
            "total": 600,
            "accuracy": 0.6433,
            "ci95": [
              0.6042,
              0.6806
            ],
            "ci_width": 0.0764
          },
          "other": {
            "correct": 441,
            "total": 700,
            "accuracy": 0.63,
            "ci95": [
              0.5936,
              0.665
            ],
            "ci_width": 0.0714
          }
        },
        "option_bias": {
          "predicted_share": {
            "A": 0.2937,
            "B": 0.2421,
            "C": 0.2702,
            "D": 0.1902
          },
          "gold_share": {
            "A": 0.2379,
            "B": 0.2516,
            "C": 0.2477,
            "D": 0.2628
          },
          "total_variation_distance": 0.0802,
          "most_favoured_letter": "A"
        },
        "n_questions": 2850,
        "n_subjects": 57
      },
      "probability": {
        "available": false,
        "reason": "provider did not return token logprobs for this model",
        "coverage": 0.0
      },
      "consistency": {
        "available": false,
        "reason": "repeat/seed runs not enabled for this evaluation"
      },
      "robustness": {
        "available": false,
        "reason": "robustness sweep not enabled for this evaluation"
      },
      "api_performance": {
        "available": true,
        "latency": {
          "ttft": {
            "metric": "TTFT (s)",
            "mean": 0.5818,
            "std": 0.3264,
            "min": 0.3367,
            "p50": 0.4551,
            "p95": 1.3867,
            "p99": 1.8411,
            "max": 4.0398,
            "n": 2846,
            "source": "measured"
          },
          "e2e": {
            "metric": "End-to-end (s)",
            "mean": 1.7502,
            "std": 0.781,
            "min": 0.2801,
            "p50": 1.5572,
            "p95": 3.1171,
            "p99": 5.0372,
            "max": 10.4445,
            "n": 2850,
            "source": "measured"
          },
          "tpot": {
            "metric": "TPOT (s/token)",
            "mean": 0.0141,
            "std": 0.0062,
            "min": 0.0027,
            "p50": 0.0132,
            "p95": 0.0225,
            "p99": 0.0335,
            "max": 0.1132,
            "n": 2846,
            "source": "measured"
          }
        },
        "throughput": {
          "prefill_tokens_per_s": 510.92,
          "decode_tokens_per_s": 79.95,
          "total_tokens_per_s": 136.59,
          "requests_per_s": 0.4011,
          "items_per_s": 0.4011
        },
        "wall_seconds": 7104.96,
        "source": "measured",
        "caveat": "TTFT includes network round-trip and provider queueing, so TTFT-derived prefill throughput is a lower bound on true prefill speed"
      },
      "token_efficiency": {
        "available": true,
        "prompt_tokens_total": 731647,
        "completion_tokens_total": 238841,
        "total_tokens": 970488,
        "mean_prompt_tokens": 256.7,
        "mean_completion_tokens": 83.8,
        "tokens_per_item": 340.5,
        "tokens_per_correct_answer": 564.2,
        "source": "provider_reported",
        "reasoning_tokens_total": 0,
        "mean_reasoning_tokens": 0.0,
        "reasoning_token_share": 0.0,
        "cached_prompt_tokens_total": 0,
        "prompt_cache_hit_rate": 0.0
      },
      "economics": {
        "available": true,
        "total_usd": 0.079664,
        "mean_per_request_usd": 2.799e-05,
        "per_1k_requests_usd": 0.028,
        "cost_per_correct_answer_usd": 4.6e-05,
        "n_requests": 2846,
        "source": "provider_reported",
        "cost_per_1k_tokens_usd": 8.2e-05,
        "cost_per_1m_tokens_usd": 0.0821,
        "correct_answers_per_usd": 21590.7
      },
      "reliability": {
        "available": true,
        "n_requests": 2850,
        "successful_requests": 2846,
        "success_rate": 0.9986,
        "failure_rate": 0.0014,
        "invalid_output_rate": 0.0025,
        "timeout_rate": 0,
        "rate_limit_rate": 0.0014,
        "server_error_rate": 0,
        "error_classes": {
          "rate_limited": 4
        },
        "retry_total": 40,
        "retried_request_rate": 0.0032,
        "retries_per_request": 0.014,
        "providers_used": {
          "DeepInfra": 2846
        },
        "provider_failover": false,
        "source": "measured"
      },
      "config": {
        "run_id": "google_gemma-3-4b-it_mmlu_TP1_PP1_DP1_SP1_CP1_EP1_20260908-000546",
        "timestamp": "2026-09-08T00:05:46",
        "schema_version": "2.0",
        "model": "google/gemma-3-4b-it",
        "parallelism": {
          "tensor_parallel": 1,
          "pipeline_parallel": 1,
          "data_parallel": 1,
          "sequence_parallel": 1,
          "context_parallel": 1,
          "expert_parallel": 1,
          "controlled": false,
          "note": "hosted endpoint: layout chosen by the provider, not observable or settable from the client"
        },
        "serving": {
          "backend": "openrouter",
          "dtype": "unknown",
          "quantization": null,
          "kv_cache_dtype": "unknown",
          "max_batch_size": null,
          "max_model_len": null,
          "gpu_memory_utilization": null,
          "enable_prefix_caching": null,
          "speculative_decoding": false,
          "device": "remote",
          "device_tdp_watts": null,
          "controlled": false
        },
        "decoding": {
          "temperature": 0.0,
          "top_p": 1.0,
          "top_k": null,
          "max_tokens": 384,
          "frequency_penalty": 0.0,
          "presence_penalty": 0.0,
          "seed": 42,
          "few_shot": 0
        },
        "dataset": {
          "benchmark": "mmlu",
          "split": "test",
          "subjects": [],
          "n_items": 2850,
          "questions_per_subject": null,
          "sample_seed": 42,
          "prompt_template": "baseline"
        },
        "evaluation": {
          "request_logprobs": true,
          "top_logprobs": 5,
          "stream_for_latency": true,
          "repeats_for_consistency": 1,
          "seeds_for_stability": [],
          "robustness_variants": [],
          "context_ablations": [],
          "calibration_bins": 10
        },
        "environment": {
          "python": "3.10.20",
          "platform": "macOS-26.6.2-x86_64-i386-64bit",
          "processor": "i386",
          "git_commit": "2949ebc"
        },
        "notes": ""
      },
      "taxonomy": {
        "deployment": "openrouter_hosted_api",
        "stages": [
          {
            "stage": 1,
            "layer": "Task Quality",
            "key": "task_quality",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 2,
            "layer": "Probabilistic Quality",
            "key": "probability",
            "priority": "P1",
            "openrouter": "conditional",
            "requires": "provider that returns token log-probabilities"
          },
          {
            "stage": 3,
            "layer": "Reasoning & Consistency",
            "key": "consistency",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "repeats > 1 and/or multiple seeds"
          },
          {
            "stage": 4,
            "layer": "Context Behavior",
            "key": "context",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "context ablation pass"
          },
          {
            "stage": 5,
            "layer": "Robustness",
            "key": "robustness",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "robustness pass"
          },
          {
            "stage": 6,
            "layer": "API Performance",
            "key": "api_performance",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 7,
            "layer": "Token Efficiency",
            "key": "token_efficiency",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 8,
            "layer": "Economics",
            "key": "economics",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 9,
            "layer": "Reliability",
            "key": "reliability",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          }
        ],
        "excluded": {
          "stage": 10,
          "layer": "Hardware & Distributed",
          "reason": "The benchmark is a client of a shared, auto-scaled third-party endpoint. Parallelism layout, GPU telemetry and power draw are chosen and held by the provider; none of them is exposed to the caller. Any figure reported here would be fabricated.",
          "path_to_measure": "Run the same suite against a self-hosted vLLM backend (benchmark_config.local_vllm_config), where TP/PP/DP/SP/CP/EP become settable variables and VRAM, KV cache, communication overhead and energy become directly measurable."
        }
      },
      "display_name": "Gemma-3-4B",
      "developer": "Google",
      "param_count": "4B",
      "colour": "#2196F3"
    }
  ],
  "comparison": {
    "benchmark": "mmlu",
    "ranking": [
      {
        "model": "mistralai/ministral-8b-2512",
        "quality": 0.793,
        "calibration_score": null,
        "robustness_score": null,
        "efficiency_score": 0.4756,
        "reliability_score": 0.9944,
        "components_used": [
          "quality",
          "efficiency",
          "reliability"
        ],
        "effective_weights": {
          "quality": 0.7143,
          "efficiency": 0.1429,
          "reliability": 0.1429
        },
        "composite_score": 0.7764,
        "mean_latency_s": 1.3714,
        "ttft_mean_s": 0.4135,
        "cost_total_usd": 0.116086,
        "cost_per_correct_usd": 5.1e-05,
        "cost_per_1m_tokens_usd": 0.1226,
        "total_tokens": 946947,
        "success_rate": 1.0,
        "rank": 1
      },
      {
        "model": "meta-llama/llama-3.2-3b-instruct",
        "quality": 0.5467,
        "calibration_score": null,
        "robustness_score": null,
        "efficiency_score": 1.0,
        "reliability_score": 0.8828,
        "components_used": [
          "quality",
          "efficiency",
          "reliability"
        ],
        "effective_weights": {
          "quality": 0.7143,
          "efficiency": 0.1429,
          "reliability": 0.1429
        },
        "composite_score": 0.6595,
        "mean_latency_s": 0.6522,
        "ttft_mean_s": 0.3296,
        "cost_total_usd": 0.129379,
        "cost_per_correct_usd": 8.3e-05,
        "cost_per_1m_tokens_usd": 0.1453,
        "total_tokens": 890595,
        "success_rate": 1.0,
        "rank": 2
      },
      {
        "model": "google/gemma-3-4b-it",
        "quality": 0.6035,
        "calibration_score": null,
        "robustness_score": null,
        "efficiency_score": 0.3726,
        "reliability_score": 0.9961,
        "components_used": [
          "quality",
          "efficiency",
          "reliability"
        ],
        "effective_weights": {
          "quality": 0.7143,
          "efficiency": 0.1429,
          "reliability": 0.1429
        },
        "composite_score": 0.6266,
        "mean_latency_s": 1.7502,
        "ttft_mean_s": 0.5818,
        "cost_total_usd": 0.079664,
        "cost_per_correct_usd": 4.6e-05,
        "cost_per_1m_tokens_usd": 0.0821,
        "total_tokens": 970488,
        "success_rate": 0.9986,
        "rank": 3
      }
    ],
    "comparable": true,
    "comparability_note": "all models share one run configuration - differences are attributable to the models",
    "composite_weights": {
      "quality": 0.5,
      "calibration": 0.15,
      "robustness": 0.15,
      "efficiency": 0.1,
      "reliability": 0.1
    },
    "components_used": [
      "quality",
      "efficiency",
      "reliability"
    ],
    "components_missing": [
      "calibration",
      "robustness"
    ],
    "components_uniform": true,
    "effective_weights": {
      "quality": 0.7143,
      "efficiency": 0.1429,
      "reliability": 0.1429
    },
    "composite_note": "composite renormalised over quality, efficiency, reliability; calibration, robustness not measured in this run and excluded rather than scored zero"
  }
}