{
  "benchmark": "lambada",
  "exported_at": "2026-10-27 22:14:16",
  "results": [
    {
      "model": "mistralai/ministral-8b-2512",
      "benchmark": "lambada",
      "task_quality": {
        "last_word_accuracy": 0.399,
        "last_word_accuracy_ci95": [
          0.3691,
          0.4297
        ],
        "exact_target_match": 0.399,
        "stem_match_accuracy": 0.402,
        "error_rate": 0.601,
        "empty_prediction_rate": 0.0,
        "n_passages": 1000
      },
      "probability": {
        "available": false,
        "reason": "provider did not return token logprobs for this model",
        "coverage": 0.0
      },
      "consistency": {
        "available": false,
        "reason": "repeat/seed runs not enabled for this evaluation"
      },
      "context": {
        "available": true,
        "position_sensitivity": {
          "available": true,
          "length_source": "context_words",
          "buckets": {
            "q1": {
              "word_range": [
                58,
                65
              ],
              "n": 250,
              "accuracy": 0.376
            },
            "q2": {
              "word_range": [
                65,
                70
              ],
              "n": 250,
              "accuracy": 0.388
            },
            "q3": {
              "word_range": [
                70,
                78
              ],
              "n": 250,
              "accuracy": 0.364
            },
            "q4": {
              "word_range": [
                78,
                126
              ],
              "n": 250,
              "accuracy": 0.452
            }
          },
          "shortest_vs_longest": -0.076,
          "span": 0.088,
          "superseded_note": "recomputed from true passage length; the earlier value bucketed by the truncated context_preview and is not comparable"
        }
      },
      "robustness": {
        "available": false,
        "reason": "robustness sweep not enabled for this evaluation"
      },
      "api_performance": {
        "available": true,
        "latency": {
          "ttft": {
            "metric": "TTFT (s)",
            "mean": 0.537,
            "std": 1.1991,
            "min": 0.2568,
            "p50": 0.3667,
            "p95": 1.2922,
            "p99": 1.7257,
            "max": 33.0575,
            "n": 1000,
            "source": "measured"
          },
          "e2e": {
            "metric": "End-to-end (s)",
            "mean": 0.5687,
            "std": 1.2151,
            "min": 0.2681,
            "p50": 0.3997,
            "p95": 1.3159,
            "p99": 1.7261,
            "max": 33.4741,
            "n": 1000,
            "source": "measured"
          },
          "tpot": {
            "metric": "TPOT (s/token)",
            "mean": 0.0195,
            "std": 0.0438,
            "min": 0.0002,
            "p50": 0.0095,
            "p95": 0.0905,
            "p99": 0.2365,
            "max": 0.4779,
            "n": 1000,
            "source": "measured"
          }
        },
        "throughput": {
          "prefill_tokens_per_s": 452.76,
          "decode_tokens_per_s": 816.33,
          "total_tokens_per_s": 169.19,
          "requests_per_s": 0.9175,
          "items_per_s": 0.9175
        },
        "wall_seconds": 1089.87,
        "source": "measured",
        "caveat": "TTFT includes network round-trip and provider queueing, so TTFT-derived prefill throughput is a lower bound on true prefill speed"
      },
      "token_efficiency": {
        "available": true,
        "prompt_tokens_total": 181752,
        "completion_tokens_total": 2638,
        "total_tokens": 184390,
        "mean_prompt_tokens": 181.8,
        "mean_completion_tokens": 2.6,
        "tokens_per_item": 184.4,
        "tokens_per_correct_answer": 462.1,
        "source": "provider_reported",
        "reasoning_tokens_total": 0,
        "mean_reasoning_tokens": 0.0,
        "reasoning_token_share": 0.0,
        "cached_prompt_tokens_total": 0,
        "prompt_cache_hit_rate": 0.0
      },
      "economics": {
        "available": true,
        "total_usd": 0.038396,
        "mean_per_request_usd": 3.84e-05,
        "per_1k_requests_usd": 0.0384,
        "cost_per_correct_answer_usd": 9.6e-05,
        "n_requests": 1000,
        "source": "provider_reported",
        "cost_per_1k_tokens_usd": 0.000208,
        "cost_per_1m_tokens_usd": 0.2082,
        "correct_answers_per_usd": 10391.7
      },
      "reliability": {
        "available": true,
        "n_requests": 1000,
        "successful_requests": 1000,
        "success_rate": 1.0,
        "failure_rate": 0,
        "invalid_output_rate": 0,
        "timeout_rate": 0,
        "rate_limit_rate": 0,
        "server_error_rate": 0,
        "error_classes": {},
        "retry_total": 0,
        "retried_request_rate": 0,
        "retries_per_request": 0.0,
        "providers_used": {
          "Mistral": 1000
        },
        "provider_failover": false,
        "source": "measured"
      },
      "tokenization": {
        "available": true,
        "tokenizer": "mistralai/Ministral-8B-Instruct-2410",
        "tokenizer_exact": true,
        "tokens_per_target_word": 1.52,
        "fragmentation_rate": 0.458,
        "subword_exact_match": 0.399,
        "accuracy_by_fragmentation": {
          "1_token": {
            "correct": 240,
            "total": 542,
            "accuracy": 0.4428
          },
          "2_tokens": {
            "correct": 139,
            "total": 400,
            "accuracy": 0.3475
          },
          "3plus_tokens": {
            "correct": 20,
            "total": 58,
            "accuracy": 0.3448
          }
        }
      },
      "config": {
        "run_id": "mistralai_ministral-8b-2512_lambada_TP1_PP1_DP1_SP1_CP1_EP1_20260907-233406",
        "timestamp": "2026-09-07T23:34:06",
        "schema_version": "2.0",
        "model": "mistralai/ministral-8b-2512",
        "parallelism": {
          "tensor_parallel": 1,
          "pipeline_parallel": 1,
          "data_parallel": 1,
          "sequence_parallel": 1,
          "context_parallel": 1,
          "expert_parallel": 1,
          "controlled": false,
          "note": "hosted endpoint: layout chosen by the provider, not observable or settable from the client"
        },
        "serving": {
          "backend": "openrouter",
          "dtype": "unknown",
          "quantization": null,
          "kv_cache_dtype": "unknown",
          "max_batch_size": null,
          "max_model_len": null,
          "gpu_memory_utilization": null,
          "enable_prefix_caching": null,
          "speculative_decoding": false,
          "device": "remote",
          "device_tdp_watts": null,
          "controlled": false
        },
        "decoding": {
          "temperature": 0.0,
          "top_p": 1.0,
          "top_k": null,
          "max_tokens": 8,
          "frequency_penalty": 0.0,
          "presence_penalty": 0.0,
          "seed": null,
          "few_shot": 2
        },
        "dataset": {
          "benchmark": "lambada",
          "split": "test",
          "subjects": [],
          "n_items": 1000,
          "questions_per_subject": null,
          "sample_seed": 42,
          "prompt_template": "baseline"
        },
        "evaluation": {
          "request_logprobs": true,
          "top_logprobs": 5,
          "stream_for_latency": true,
          "repeats_for_consistency": 1,
          "seeds_for_stability": [],
          "robustness_variants": [],
          "context_ablations": [],
          "calibration_bins": 10
        },
        "environment": {
          "python": "3.10.20",
          "platform": "macOS-26.6.2-x86_64-i386-64bit",
          "processor": "i386",
          "git_commit": "2949ebc"
        },
        "notes": ""
      },
      "taxonomy": {
        "deployment": "openrouter_hosted_api",
        "stages": [
          {
            "stage": 1,
            "layer": "Task Quality",
            "key": "task_quality",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 2,
            "layer": "Probabilistic Quality",
            "key": "probability",
            "priority": "P1",
            "openrouter": "conditional",
            "requires": "provider that returns token log-probabilities"
          },
          {
            "stage": 3,
            "layer": "Reasoning & Consistency",
            "key": "consistency",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "repeats > 1 and/or multiple seeds"
          },
          {
            "stage": 4,
            "layer": "Context Behavior",
            "key": "context",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "context ablation pass"
          },
          {
            "stage": 5,
            "layer": "Robustness",
            "key": "robustness",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "robustness pass"
          },
          {
            "stage": 6,
            "layer": "API Performance",
            "key": "api_performance",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 7,
            "layer": "Token Efficiency",
            "key": "token_efficiency",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 8,
            "layer": "Economics",
            "key": "economics",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 9,
            "layer": "Reliability",
            "key": "reliability",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          }
        ],
        "excluded": {
          "stage": 10,
          "layer": "Hardware & Distributed",
          "reason": "The benchmark is a client of a shared, auto-scaled third-party endpoint. Parallelism layout, GPU telemetry and power draw are chosen and held by the provider; none of them is exposed to the caller. Any figure reported here would be fabricated.",
          "path_to_measure": "Run the same suite against a self-hosted vLLM backend (benchmark_config.local_vllm_config), where TP/PP/DP/SP/CP/EP become settable variables and VRAM, KV cache, communication overhead and energy become directly measurable."
        }
      },
      "display_name": "Ministral-8B",
      "developer": "Mistral AI",
      "param_count": "8B",
      "colour": "#FF9800"
    },
    {
      "model": "meta-llama/llama-3.2-3b-instruct",
      "benchmark": "lambada",
      "task_quality": {
        "last_word_accuracy": 0.225,
        "last_word_accuracy_ci95": [
          0.2002,
          0.2519
        ],
        "exact_target_match": 0.225,
        "stem_match_accuracy": 0.232,
        "error_rate": 0.775,
        "empty_prediction_rate": 0.0,
        "n_passages": 1000
      },
      "probability": {
        "available": false,
        "reason": "provider did not return token logprobs for this model",
        "coverage": 0.0
      },
      "consistency": {
        "available": false,
        "reason": "repeat/seed runs not enabled for this evaluation"
      },
      "context": {
        "available": true,
        "position_sensitivity": {
          "available": true,
          "length_source": "context_words",
          "buckets": {
            "q1": {
              "word_range": [
                58,
                65
              ],
              "n": 250,
              "accuracy": 0.192
            },
            "q2": {
              "word_range": [
                65,
                70
              ],
              "n": 250,
              "accuracy": 0.212
            },
            "q3": {
              "word_range": [
                70,
                78
              ],
              "n": 250,
              "accuracy": 0.212
            },
            "q4": {
              "word_range": [
                78,
                126
              ],
              "n": 250,
              "accuracy": 0.28
            }
          },
          "shortest_vs_longest": -0.088,
          "span": 0.088,
          "superseded_note": "recomputed from true passage length; the earlier value bucketed by the truncated context_preview and is not comparable"
        }
      },
      "robustness": {
        "available": false,
        "reason": "robustness sweep not enabled for this evaluation"
      },
      "api_performance": {
        "available": true,
        "latency": {
          "ttft": {
            "metric": "TTFT (s)",
            "mean": 0.3447,
            "std": 0.2617,
            "min": 0.1797,
            "p50": 0.2841,
            "p95": 0.6635,
            "p99": 1.3564,
            "max": 3.7796,
            "n": 1000,
            "source": "measured"
          },
          "e2e": {
            "metric": "End-to-end (s)",
            "mean": 0.354,
            "std": 0.2621,
            "min": 0.1845,
            "p50": 0.2911,
            "p95": 0.6651,
            "p99": 1.3629,
            "max": 3.7886,
            "n": 1000,
            "source": "measured"
          },
          "tpot": {
            "metric": "TPOT (s/token)",
            "mean": 0.007,
            "std": 0.0173,
            "min": 0.0001,
            "p50": 0.0041,
            "p95": 0.0135,
            "p99": 0.0991,
            "max": 0.199,
            "n": 1000,
            "source": "measured"
          }
        },
        "throughput": {
          "prefill_tokens_per_s": 744.55,
          "decode_tokens_per_s": 673.88,
          "total_tokens_per_s": 304.76,
          "requests_per_s": 1.4202,
          "items_per_s": 1.4202
        },
        "wall_seconds": 704.11,
        "source": "measured",
        "caveat": "TTFT includes network round-trip and provider queueing, so TTFT-derived prefill throughput is a lower bound on true prefill speed"
      },
      "token_efficiency": {
        "available": true,
        "prompt_tokens_total": 211997,
        "completion_tokens_total": 2586,
        "total_tokens": 214583,
        "mean_prompt_tokens": 212.0,
        "mean_completion_tokens": 2.6,
        "tokens_per_item": 214.6,
        "tokens_per_correct_answer": 953.7,
        "source": "provider_reported",
        "reasoning_tokens_total": 0,
        "mean_reasoning_tokens": 0.0,
        "reasoning_token_share": 0.0,
        "cached_prompt_tokens_total": 0,
        "prompt_cache_hit_rate": 0.0
      },
      "economics": {
        "available": true,
        "total_usd": 0.022783,
        "mean_per_request_usd": 2.278e-05,
        "per_1k_requests_usd": 0.0228,
        "cost_per_correct_answer_usd": 0.000101,
        "n_requests": 1000,
        "source": "provider_reported",
        "cost_per_1k_tokens_usd": 0.000106,
        "cost_per_1m_tokens_usd": 0.1062,
        "correct_answers_per_usd": 9876.0
      },
      "reliability": {
        "available": true,
        "n_requests": 1000,
        "successful_requests": 1000,
        "success_rate": 1.0,
        "failure_rate": 0,
        "invalid_output_rate": 0,
        "timeout_rate": 0,
        "rate_limit_rate": 0,
        "server_error_rate": 0,
        "error_classes": {},
        "retry_total": 0,
        "retried_request_rate": 0,
        "retries_per_request": 0.0,
        "providers_used": {
          "Cloudflare": 1000
        },
        "provider_failover": false,
        "source": "measured"
      },
      "tokenization": {
        "available": true,
        "tokenizer": "cl100k_base",
        "tokenizer_exact": false,
        "tokens_per_target_word": 1.446,
        "fragmentation_rate": 0.406,
        "subword_exact_match": 0.225,
        "accuracy_by_fragmentation": {
          "1_token": {
            "correct": 175,
            "total": 594,
            "accuracy": 0.2946
          },
          "2_tokens": {
            "correct": 49,
            "total": 367,
            "accuracy": 0.1335
          },
          "3plus_tokens": {
            "correct": 1,
            "total": 39,
            "accuracy": 0.0256
          }
        }
      },
      "config": {
        "run_id": "meta-llama_llama-3.2-3b-instruct_lambada_TP1_PP1_DP1_SP1_CP1_EP1_20260907-231336",
        "timestamp": "2026-09-07T23:13:36",
        "schema_version": "2.0",
        "model": "meta-llama/llama-3.2-3b-instruct",
        "parallelism": {
          "tensor_parallel": 1,
          "pipeline_parallel": 1,
          "data_parallel": 1,
          "sequence_parallel": 1,
          "context_parallel": 1,
          "expert_parallel": 1,
          "controlled": false,
          "note": "hosted endpoint: layout chosen by the provider, not observable or settable from the client"
        },
        "serving": {
          "backend": "openrouter",
          "dtype": "unknown",
          "quantization": null,
          "kv_cache_dtype": "unknown",
          "max_batch_size": null,
          "max_model_len": null,
          "gpu_memory_utilization": null,
          "enable_prefix_caching": null,
          "speculative_decoding": false,
          "device": "remote",
          "device_tdp_watts": null,
          "controlled": false
        },
        "decoding": {
          "temperature": 0.0,
          "top_p": 1.0,
          "top_k": null,
          "max_tokens": 8,
          "frequency_penalty": 0.0,
          "presence_penalty": 0.0,
          "seed": null,
          "few_shot": 2
        },
        "dataset": {
          "benchmark": "lambada",
          "split": "test",
          "subjects": [],
          "n_items": 1000,
          "questions_per_subject": null,
          "sample_seed": 42,
          "prompt_template": "baseline"
        },
        "evaluation": {
          "request_logprobs": true,
          "top_logprobs": 5,
          "stream_for_latency": true,
          "repeats_for_consistency": 1,
          "seeds_for_stability": [],
          "robustness_variants": [],
          "context_ablations": [],
          "calibration_bins": 10
        },
        "environment": {
          "python": "3.10.20",
          "platform": "macOS-26.6.2-x86_64-i386-64bit",
          "processor": "i386",
          "git_commit": "2949ebc"
        },
        "notes": ""
      },
      "taxonomy": {
        "deployment": "openrouter_hosted_api",
        "stages": [
          {
            "stage": 1,
            "layer": "Task Quality",
            "key": "task_quality",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 2,
            "layer": "Probabilistic Quality",
            "key": "probability",
            "priority": "P1",
            "openrouter": "conditional",
            "requires": "provider that returns token log-probabilities"
          },
          {
            "stage": 3,
            "layer": "Reasoning & Consistency",
            "key": "consistency",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "repeats > 1 and/or multiple seeds"
          },
          {
            "stage": 4,
            "layer": "Context Behavior",
            "key": "context",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "context ablation pass"
          },
          {
            "stage": 5,
            "layer": "Robustness",
            "key": "robustness",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "robustness pass"
          },
          {
            "stage": 6,
            "layer": "API Performance",
            "key": "api_performance",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 7,
            "layer": "Token Efficiency",
            "key": "token_efficiency",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 8,
            "layer": "Economics",
            "key": "economics",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 9,
            "layer": "Reliability",
            "key": "reliability",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          }
        ],
        "excluded": {
          "stage": 10,
          "layer": "Hardware & Distributed",
          "reason": "The benchmark is a client of a shared, auto-scaled third-party endpoint. Parallelism layout, GPU telemetry and power draw are chosen and held by the provider; none of them is exposed to the caller. Any figure reported here would be fabricated.",
          "path_to_measure": "Run the same suite against a self-hosted vLLM backend (benchmark_config.local_vllm_config), where TP/PP/DP/SP/CP/EP become settable variables and VRAM, KV cache, communication overhead and energy become directly measurable."
        }
      },
      "display_name": "Llama-3.2-3B",
      "developer": "Meta",
      "param_count": "3B",
      "colour": "#4CAF50"
    },
    {
      "model": "google/gemma-3-4b-it",
      "benchmark": "lambada",
      "task_quality": {
        "last_word_accuracy": 0.18,
        "last_word_accuracy_ci95": [
          0.1574,
          0.205
        ],
        "exact_target_match": 0.18,
        "stem_match_accuracy": 0.185,
        "error_rate": 0.82,
        "empty_prediction_rate": 0.001,
        "n_passages": 1000
      },
      "probability": {
        "available": false,
        "reason": "provider did not return token logprobs for this model",
        "coverage": 0.0
      },
      "consistency": {
        "available": false,
        "reason": "repeat/seed runs not enabled for this evaluation"
      },
      "context": {
        "available": true,
        "position_sensitivity": {
          "available": true,
          "length_source": "context_words",
          "buckets": {
            "q1": {
              "word_range": [
                58,
                65
              ],
              "n": 250,
              "accuracy": 0.168
            },
            "q2": {
              "word_range": [
                65,
                70
              ],
              "n": 250,
              "accuracy": 0.188
            },
            "q3": {
              "word_range": [
                70,
                78
              ],
              "n": 250,
              "accuracy": 0.176
            },
            "q4": {
              "word_range": [
                78,
                126
              ],
              "n": 250,
              "accuracy": 0.188
            }
          },
          "shortest_vs_longest": -0.02,
          "span": 0.02,
          "superseded_note": "recomputed from true passage length; the earlier value bucketed by the truncated context_preview and is not comparable"
        }
      },
      "robustness": {
        "available": false,
        "reason": "robustness sweep not enabled for this evaluation"
      },
      "api_performance": {
        "available": true,
        "latency": {
          "ttft": {
            "metric": "TTFT (s)",
            "mean": 0.5311,
            "std": 0.3152,
            "min": 0.3242,
            "p50": 0.413,
            "p95": 1.3347,
            "p99": 1.6205,
            "max": 3.396,
            "n": 1000,
            "source": "measured"
          },
          "e2e": {
            "metric": "End-to-end (s)",
            "mean": 0.5388,
            "std": 0.3159,
            "min": 0.3266,
            "p50": 0.4193,
            "p95": 1.3359,
            "p99": 1.6255,
            "max": 3.405,
            "n": 1000,
            "source": "measured"
          },
          "tpot": {
            "metric": "TPOT (s/token)",
            "mean": 0.0071,
            "std": 0.0155,
            "min": 0.0003,
            "p50": 0.0049,
            "p95": 0.016,
            "p99": 0.0411,
            "max": 0.2605,
            "n": 1000,
            "source": "measured"
          }
        },
        "throughput": {
          "prefill_tokens_per_s": 413.68,
          "decode_tokens_per_s": 587.51,
          "total_tokens_per_s": 178.1,
          "requests_per_s": 0.9335,
          "items_per_s": 0.9335
        },
        "wall_seconds": 1071.22,
        "source": "measured",
        "caveat": "TTFT includes network round-trip and provider queueing, so TTFT-derived prefill throughput is a lower bound on true prefill speed"
      },
      "token_efficiency": {
        "available": true,
        "prompt_tokens_total": 188663,
        "completion_tokens_total": 2122,
        "total_tokens": 190785,
        "mean_prompt_tokens": 188.7,
        "mean_completion_tokens": 2.1,
        "tokens_per_item": 190.8,
        "tokens_per_correct_answer": 1059.9,
        "source": "provider_reported",
        "reasoning_tokens_total": 0,
        "mean_reasoning_tokens": 0.0,
        "reasoning_token_share": 0.0,
        "cached_prompt_tokens_total": 0,
        "prompt_cache_hit_rate": 0.0
      },
      "economics": {
        "available": true,
        "total_usd": 0.019178,
        "mean_per_request_usd": 1.918e-05,
        "per_1k_requests_usd": 0.0192,
        "cost_per_correct_answer_usd": 0.000107,
        "n_requests": 1000,
        "source": "provider_reported",
        "cost_per_1k_tokens_usd": 0.000101,
        "cost_per_1m_tokens_usd": 0.1005,
        "correct_answers_per_usd": 9385.5
      },
      "reliability": {
        "available": true,
        "n_requests": 1000,
        "successful_requests": 1000,
        "success_rate": 1.0,
        "failure_rate": 0,
        "invalid_output_rate": 0.001,
        "timeout_rate": 0,
        "rate_limit_rate": 0,
        "server_error_rate": 0,
        "error_classes": {},
        "retry_total": 0,
        "retried_request_rate": 0,
        "retries_per_request": 0.0,
        "providers_used": {
          "DeepInfra": 1000
        },
        "provider_failover": false,
        "source": "measured"
      },
      "tokenization": {
        "available": true,
        "tokenizer": "cl100k_base",
        "tokenizer_exact": false,
        "tokens_per_target_word": 1.446,
        "fragmentation_rate": 0.406,
        "subword_exact_match": 0.18,
        "accuracy_by_fragmentation": {
          "1_token": {
            "correct": 140,
            "total": 594,
            "accuracy": 0.2357
          },
          "2_tokens": {
            "correct": 39,
            "total": 367,
            "accuracy": 0.1063
          },
          "3plus_tokens": {
            "correct": 1,
            "total": 39,
            "accuracy": 0.0256
          }
        }
      },
      "config": {
        "run_id": "google_gemma-3-4b-it_lambada_TP1_PP1_DP1_SP1_CP1_EP1_20260907-224937",
        "timestamp": "2026-09-07T22:49:37",
        "schema_version": "2.0",
        "model": "google/gemma-3-4b-it",
        "parallelism": {
          "tensor_parallel": 1,
          "pipeline_parallel": 1,
          "data_parallel": 1,
          "sequence_parallel": 1,
          "context_parallel": 1,
          "expert_parallel": 1,
          "controlled": false,
          "note": "hosted endpoint: layout chosen by the provider, not observable or settable from the client"
        },
        "serving": {
          "backend": "openrouter",
          "dtype": "unknown",
          "quantization": null,
          "kv_cache_dtype": "unknown",
          "max_batch_size": null,
          "max_model_len": null,
          "gpu_memory_utilization": null,
          "enable_prefix_caching": null,
          "speculative_decoding": false,
          "device": "remote",
          "device_tdp_watts": null,
          "controlled": false
        },
        "decoding": {
          "temperature": 0.0,
          "top_p": 1.0,
          "top_k": null,
          "max_tokens": 8,
          "frequency_penalty": 0.0,
          "presence_penalty": 0.0,
          "seed": null,
          "few_shot": 2
        },
        "dataset": {
          "benchmark": "lambada",
          "split": "test",
          "subjects": [],
          "n_items": 1000,
          "questions_per_subject": null,
          "sample_seed": 42,
          "prompt_template": "baseline"
        },
        "evaluation": {
          "request_logprobs": true,
          "top_logprobs": 5,
          "stream_for_latency": true,
          "repeats_for_consistency": 1,
          "seeds_for_stability": [],
          "robustness_variants": [],
          "context_ablations": [],
          "calibration_bins": 10
        },
        "environment": {
          "python": "3.10.20",
          "platform": "macOS-26.6.2-x86_64-i386-64bit",
          "processor": "i386",
          "git_commit": "2949ebc"
        },
        "notes": ""
      },
      "taxonomy": {
        "deployment": "openrouter_hosted_api",
        "stages": [
          {
            "stage": 1,
            "layer": "Task Quality",
            "key": "task_quality",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 2,
            "layer": "Probabilistic Quality",
            "key": "probability",
            "priority": "P1",
            "openrouter": "conditional",
            "requires": "provider that returns token log-probabilities"
          },
          {
            "stage": 3,
            "layer": "Reasoning & Consistency",
            "key": "consistency",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "repeats > 1 and/or multiple seeds"
          },
          {
            "stage": 4,
            "layer": "Context Behavior",
            "key": "context",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "context ablation pass"
          },
          {
            "stage": 5,
            "layer": "Robustness",
            "key": "robustness",
            "priority": "P1",
            "openrouter": "yes",
            "requires": "robustness pass"
          },
          {
            "stage": 6,
            "layer": "API Performance",
            "key": "api_performance",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 7,
            "layer": "Token Efficiency",
            "key": "token_efficiency",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 8,
            "layer": "Economics",
            "key": "economics",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          },
          {
            "stage": 9,
            "layer": "Reliability",
            "key": "reliability",
            "priority": "P0",
            "openrouter": "yes",
            "requires": null
          }
        ],
        "excluded": {
          "stage": 10,
          "layer": "Hardware & Distributed",
          "reason": "The benchmark is a client of a shared, auto-scaled third-party endpoint. Parallelism layout, GPU telemetry and power draw are chosen and held by the provider; none of them is exposed to the caller. Any figure reported here would be fabricated.",
          "path_to_measure": "Run the same suite against a self-hosted vLLM backend (benchmark_config.local_vllm_config), where TP/PP/DP/SP/CP/EP become settable variables and VRAM, KV cache, communication overhead and energy become directly measurable."
        }
      },
      "display_name": "Gemma-3-4B",
      "developer": "Google",
      "param_count": "4B",
      "colour": "#2196F3"
    }
  ],
  "comparison": {
    "benchmark": "lambada",
    "ranking": [
      {
        "model": "mistralai/ministral-8b-2512",
        "quality": 0.399,
        "calibration_score": null,
        "robustness_score": null,
        "efficiency_score": 0.6225,
        "reliability_score": 1.0,
        "components_used": [
          "quality",
          "efficiency",
          "reliability"
        ],
        "effective_weights": {
          "quality": 0.7143,
          "efficiency": 0.1429,
          "reliability": 0.1429
        },
        "composite_score": 0.5168,
        "mean_latency_s": 0.5687,
        "ttft_mean_s": 0.537,
        "cost_total_usd": 0.038396,
        "cost_per_correct_usd": 9.6e-05,
        "cost_per_1m_tokens_usd": 0.2082,
        "total_tokens": 184390,
        "success_rate": 1.0,
        "rank": 1
      },
      {
        "model": "meta-llama/llama-3.2-3b-instruct",
        "quality": 0.225,
        "calibration_score": null,
        "robustness_score": null,
        "efficiency_score": 1.0,
        "reliability_score": 1.0,
        "components_used": [
          "quality",
          "efficiency",
          "reliability"
        ],
        "effective_weights": {
          "quality": 0.7143,
          "efficiency": 0.1429,
          "reliability": 0.1429
        },
        "composite_score": 0.4464,
        "mean_latency_s": 0.354,
        "ttft_mean_s": 0.3447,
        "cost_total_usd": 0.022783,
        "cost_per_correct_usd": 0.000101,
        "cost_per_1m_tokens_usd": 0.1062,
        "total_tokens": 214583,
        "success_rate": 1.0,
        "rank": 2
      },
      {
        "model": "google/gemma-3-4b-it",
        "quality": 0.18,
        "calibration_score": null,
        "robustness_score": null,
        "efficiency_score": 0.657,
        "reliability_score": 0.999,
        "components_used": [
          "quality",
          "efficiency",
          "reliability"
        ],
        "effective_weights": {
          "quality": 0.7143,
          "efficiency": 0.1429,
          "reliability": 0.1429
        },
        "composite_score": 0.3651,
        "mean_latency_s": 0.5388,
        "ttft_mean_s": 0.5311,
        "cost_total_usd": 0.019178,
        "cost_per_correct_usd": 0.000107,
        "cost_per_1m_tokens_usd": 0.1005,
        "total_tokens": 190785,
        "success_rate": 1.0,
        "rank": 3
      }
    ],
    "comparable": true,
    "comparability_note": "all models share one run configuration - differences are attributable to the models",
    "composite_weights": {
      "quality": 0.5,
      "calibration": 0.15,
      "robustness": 0.15,
      "efficiency": 0.1,
      "reliability": 0.1
    },
    "components_used": [
      "quality",
      "efficiency",
      "reliability"
    ],
    "components_missing": [
      "calibration",
      "robustness"
    ],
    "components_uniform": true,
    "effective_weights": {
      "quality": 0.7143,
      "efficiency": 0.1429,
      "reliability": 0.1429
    },
    "composite_note": "composite renormalised over quality, efficiency, reliability; calibration, robustness not measured in this run and excluded rather than scored zero"
  }
}