{
 "name": "Claude Fable 5.2 at Medium effort on 30 tasks of the coding-agent benchmark (1 run)",
 "publisher": "CodeRouter (https://www.coderouter.io)",
 "disclosure": "CodeRouter is our product. Claude Fable 5.2 was called directly through Anthropic's API, not through CodeRouter.",
 "run_date": "2026-11-10",
 "other_setups_run_dates": {
  "claude_fable_5_2_high": "2026-11-10 (morning, US Pacific; Medium ran in the evening)",
  "all_opus, all_sonnet, all_haiku, coderouter_default": "2026-11-09 (run 1 of the 99-task benchmark)"
 },
 "prices_as_of": "2026-11-10",
 "post": "https://www.coderouter.io/blog/claude-fable-5-2-medium-effort-coding-agent-benchmark",
 "high_effort_run": {
  "post": "https://www.coderouter.io/blog/claude-fable-5-2-coding-agent-benchmark",
  "data": "https://www.coderouter.io/benchmarks/coderouter-agentic-fable52-2026-11.json"
 },
 "benchmark_99_tasks": "https://www.coderouter.io/benchmark/agentic",
 "sessions_csv": "/benchmarks/coderouter-agentic-fable52-medium-2026-11-sessions.csv",
 "method": {
  "change_from_the_high_effort_run": "output_config {\"effort\": \"medium\"} on every request. Nothing else changed: same 30 tasks, agent, prompts, limits, max_tokens and caching. The API default for Fable 5.2 is high effort.",
  "model": "claude-fable-5-2 through Anthropic's Messages API with a separate API key, not through CodeRouter",
  "agent": "mini-swe-agent 2.4.6 (bash tool only, benchmarks/swebench.yaml prompts unchanged) via LiteLLM, as in the 99-task benchmark",
  "limits_per_session": {
   "steps": 150,
   "cost_usd": 10.0,
   "time_s": 3600,
   "per_command_timeout_s": 60,
   "max_tokens": 8192
  },
  "limits_other_setups": "the same, except cost_usd 2.5 per session (2026-11-09 benchmark)",
  "prompt_caching": "mini-swe-agent's default for Anthropic models: a 5-minute cache marker on the last message of every turn",
  "other_setups": "All-Opus, all-Sonnet and all-Haiku named their model through CodeRouter's pass-through path (no routing, no fallback), which adds its own prompt-cache markers; CodeRouter default routing went through CodeRouter's production API; both Fable runs called Anthropic's API directly.",
  "grading": "swebench 5.0.2 harness (Docker) with the SWE-rebench evaluation spec: apply the patch, run the task's tests; solved = the required failing tests pass and the passing tests still pass",
  "cost": "List price: every request's input, cache-read, cache-write and output tokens x the model's list price, summed per session.",
  "intervals": "Completion: Wilson 95%. Paired differences and cost ratios: task bootstrap (10,000 resamples, seed 20261104). p-values: exact McNemar on the tasks where exactly one of the two setups solved the task.",
  "comparison_sets": "Pre-registered: Medium vs High on the tasks where neither Fable session was stopped by a limit or the budget (24: the High run had six budget-stopped sessions); Medium vs the other setups on Medium's own analysis set (all 30: no Medium session was stopped). Sensitivity: Medium vs High on all 30 with High's stopped sessions counted as failures.",
  "machine": "Agent containers and grading ran on a rented cloud VM, used only to run containers. Nothing here measures the machine."
 },
 "tasks": {
  "dataset": "nebius/SWE-rebench-leaderboard",
  "revision": "34d5a58864acf91613740a09ec5d205228dcfa39",
  "file": "data/2026_03-00000-of-00001.parquet",
  "license": "CC BY 4.0",
  "selection": "the 30 tasks of the high-effort run: random.Random(20261110).sample of the 2026-11-09 benchmark's 99 tasks",
  "task_ids": [
   "itsdnns__docsight-437",
   "koxudaxi__datamodel-code-generator-3070",
   "copier-org__copier-2646",
   "pypa__twine-1309",
   "databricks__dbt-databricks-1428",
   "tobymao__sqlglot-7479",
   "koxudaxi__datamodel-code-generator-3071",
   "nesquena__hermes-webui-330_interface",
   "koxudaxi__datamodel-code-generator-3046",
   "oemof__tespy-921",
   "hiero-ledger__hiero-sdk-python-1914_interface",
   "line__line-bot-sdk-python-981_interface",
   "getmoto__moto-9903",
   "nesquena__hermes-webui-1818",
   "spack__spack-52244",
   "vprusso__toqito-1484_interface",
   "pennylaneai__pennylane-9298",
   "keras-team__keras-22330",
   "tox-dev__tox-3846",
   "huggingface__huggingface_hub-4056",
   "scrapy__scrapy-7351_interface",
   "agronholm__anyio-1134",
   "marshmallow-code__marshmallow-2925_interface",
   "pallets-eco__wtforms-892_interface",
   "raullenchai__rapid-mlx-228",
   "meltano__meltano-9929",
   "graphistry__pygraphistry-1107_interface",
   "macbre__sql-metadata-625_interface",
   "keras-team__keras-22642",
   "raullenchai__rapid-mlx-289"
  ],
  "medium_vs_high_set": [
   "itsdnns__docsight-437",
   "pypa__twine-1309",
   "databricks__dbt-databricks-1428",
   "tobymao__sqlglot-7479",
   "koxudaxi__datamodel-code-generator-3071",
   "nesquena__hermes-webui-330_interface",
   "koxudaxi__datamodel-code-generator-3046",
   "oemof__tespy-921",
   "hiero-ledger__hiero-sdk-python-1914_interface",
   "line__line-bot-sdk-python-981_interface",
   "getmoto__moto-9903",
   "vprusso__toqito-1484_interface",
   "pennylaneai__pennylane-9298",
   "keras-team__keras-22330",
   "scrapy__scrapy-7351_interface",
   "agronholm__anyio-1134",
   "marshmallow-code__marshmallow-2925_interface",
   "pallets-eco__wtforms-892_interface",
   "raullenchai__rapid-mlx-228",
   "meltano__meltano-9929",
   "graphistry__pygraphistry-1107_interface",
   "macbre__sql-metadata-625_interface",
   "keras-team__keras-22642",
   "raullenchai__rapid-mlx-289"
  ],
  "high_run_budget_stopped": [
   {
    "task": "koxudaxi__datamodel-code-generator-3070",
    "high_cost_list_usd": 3.205235,
    "medium_solved": false,
    "medium_cost_list_usd": 0.51386
   },
   {
    "task": "copier-org__copier-2646",
    "high_cost_list_usd": 1.613257,
    "medium_solved": true,
    "medium_cost_list_usd": 0.494479
   },
   {
    "task": "nesquena__hermes-webui-1818",
    "high_cost_list_usd": 2.852821,
    "medium_solved": false,
    "medium_cost_list_usd": 1.100096
   },
   {
    "task": "spack__spack-52244",
    "high_cost_list_usd": 2.843738,
    "medium_solved": true,
    "medium_cost_list_usd": 0.813009
   },
   {
    "task": "tox-dev__tox-3846",
    "high_cost_list_usd": 1.11827,
    "medium_solved": false,
    "medium_cost_list_usd": 0.321379
   },
   {
    "task": "huggingface__huggingface_hub-4056",
    "high_cost_list_usd": 3.348409,
    "medium_solved": false,
    "medium_cost_list_usd": 0.248978
   }
  ]
 },
 "medium_vs_high_24_tasks": {
  "tasks": [
   "itsdnns__docsight-437",
   "pypa__twine-1309",
   "databricks__dbt-databricks-1428",
   "tobymao__sqlglot-7479",
   "koxudaxi__datamodel-code-generator-3071",
   "nesquena__hermes-webui-330_interface",
   "koxudaxi__datamodel-code-generator-3046",
   "oemof__tespy-921",
   "hiero-ledger__hiero-sdk-python-1914_interface",
   "line__line-bot-sdk-python-981_interface",
   "getmoto__moto-9903",
   "vprusso__toqito-1484_interface",
   "pennylaneai__pennylane-9298",
   "keras-team__keras-22330",
   "scrapy__scrapy-7351_interface",
   "agronholm__anyio-1134",
   "marshmallow-code__marshmallow-2925_interface",
   "pallets-eco__wtforms-892_interface",
   "raullenchai__rapid-mlx-228",
   "meltano__meltano-9929",
   "graphistry__pygraphistry-1107_interface",
   "macbre__sql-metadata-625_interface",
   "keras-team__keras-22642",
   "raullenchai__rapid-mlx-289"
  ],
  "setups": {
   "claude_fable_5_2_medium": {
    "label": "Claude Fable 5.2, Medium",
    "how_called": "claude-fable-5-2 through Anthropic's Messages API with a separate API key, not through CodeRouter, with output_config {\"effort\": \"medium\"} on every request; otherwise the same request as the high-effort run (max_tokens 8192, no temperature or thinking parameter). Run on 2026-11-10 (evening, US Pacific).",
    "sessions": 24,
    "solved": 15,
    "solve_rate": 0.625,
    "solve_rate_ci95_wilson": [
     0.4271,
     0.7884
    ],
    "cost_list_usd_total": 10.474929,
    "cost_per_task_list_usd": 0.436455,
    "cost_per_solved_list_usd": 0.698329,
    "cost_per_task_median_list_usd": 0.339653,
    "steps_per_task": 8.625,
    "model_requests": 207,
    "exit_statuses": {
     "Submitted": 24
    },
    "empty_patches": 0
   },
   "claude_fable_5_2_high": {
    "label": "Claude Fable 5.2, High (API default)",
    "how_called": "The high-effort quick run: the same as Medium without output_config, so the API default (high effort) applied. Run on 2026-11-10 (morning, US Pacific); published at https://www.coderouter.io/blog/claude-fable-5-2-coding-agent-benchmark",
    "sessions": 24,
    "solved": 15,
    "solve_rate": 0.625,
    "solve_rate_ci95_wilson": [
     0.4271,
     0.7884
    ],
    "cost_list_usd_total": 37.695213,
    "cost_per_task_list_usd": 1.570634,
    "cost_per_solved_list_usd": 2.513014,
    "cost_per_task_median_list_usd": 1.403254,
    "steps_per_task": 17.9583,
    "model_requests": 431,
    "exit_statuses": {
     "Submitted": 24
    },
    "empty_patches": 0
   },
   "all_opus": {
    "label": "All Claude Opus 5.5",
    "how_called": "claude-opus-5-5 named on every request; CodeRouter's pass-through path (named model, no routing, no fallback). Run 1 of the 2026-11-09 benchmark.",
    "sessions": 24,
    "solved": 16,
    "solve_rate": 0.666667,
    "solve_rate_ci95_wilson": [
     0.4671,
     0.8203
    ],
    "cost_list_usd_total": 4.695636,
    "cost_per_task_list_usd": 0.195652,
    "cost_per_solved_list_usd": 0.293477,
    "cost_per_task_median_list_usd": 0.16972,
    "steps_per_task": 11.1667,
    "model_requests": 268,
    "exit_statuses": {
     "Submitted": 24
    },
    "empty_patches": 0
   },
   "all_sonnet": {
    "label": "All Claude Sonnet 5.5",
    "how_called": "claude-sonnet-5-5 named on every request; same pass-through path as all-Opus. Run 1 of the 2026-11-09 benchmark.",
    "sessions": 24,
    "solved": 15,
    "solve_rate": 0.625,
    "solve_rate_ci95_wilson": [
     0.4271,
     0.7884
    ],
    "cost_list_usd_total": 1.845853,
    "cost_per_task_list_usd": 0.076911,
    "cost_per_solved_list_usd": 0.123057,
    "cost_per_task_median_list_usd": 0.069118,
    "steps_per_task": 8.5,
    "model_requests": 204,
    "exit_statuses": {
     "Submitted": 24
    },
    "empty_patches": 0
   }
  },
  "paired_medium_vs_high": {
   "a": "claude_fable_5_2_medium",
   "b": "claude_fable_5_2_high",
   "tasks": 24,
   "solved_a": 15,
   "solved_b": 15,
   "solve_rate_diff_a_minus_b": 0.0,
   "solve_rate_diff_ci95_task_bootstrap": [
    0.0,
    0.0
   ],
   "tasks_only_a_solved": [],
   "tasks_only_b_solved": [],
   "mcnemar_exact_p": 1.0,
   "cost_ratio_list_a_over_b": 0.2779,
   "cost_ratio_list_ci95_task_bootstrap": [
    0.2166,
    0.3486
   ],
   "cost_ratio_list_b_over_a": 3.5986,
   "cost_per_solved_ratio_list_a_over_b": 0.2779,
   "cost_per_solved_ratio_list_ci95_task_bootstrap": [
    0.2166,
    0.3486
   ],
   "tasks_where_a_cost_less": 24
  }
 },
 "medium_vs_other_setups_30_tasks": {
  "tasks": [
   "itsdnns__docsight-437",
   "koxudaxi__datamodel-code-generator-3070",
   "copier-org__copier-2646",
   "pypa__twine-1309",
   "databricks__dbt-databricks-1428",
   "tobymao__sqlglot-7479",
   "koxudaxi__datamodel-code-generator-3071",
   "nesquena__hermes-webui-330_interface",
   "koxudaxi__datamodel-code-generator-3046",
   "oemof__tespy-921",
   "hiero-ledger__hiero-sdk-python-1914_interface",
   "line__line-bot-sdk-python-981_interface",
   "getmoto__moto-9903",
   "nesquena__hermes-webui-1818",
   "spack__spack-52244",
   "vprusso__toqito-1484_interface",
   "pennylaneai__pennylane-9298",
   "keras-team__keras-22330",
   "tox-dev__tox-3846",
   "huggingface__huggingface_hub-4056",
   "scrapy__scrapy-7351_interface",
   "agronholm__anyio-1134",
   "marshmallow-code__marshmallow-2925_interface",
   "pallets-eco__wtforms-892_interface",
   "raullenchai__rapid-mlx-228",
   "meltano__meltano-9929",
   "graphistry__pygraphistry-1107_interface",
   "macbre__sql-metadata-625_interface",
   "keras-team__keras-22642",
   "raullenchai__rapid-mlx-289"
  ],
  "setups": {
   "claude_fable_5_2_medium": {
    "label": "Claude Fable 5.2, Medium",
    "how_called": "claude-fable-5-2 through Anthropic's Messages API with a separate API key, not through CodeRouter, with output_config {\"effort\": \"medium\"} on every request; otherwise the same request as the high-effort run (max_tokens 8192, no temperature or thinking parameter). Run on 2026-11-10 (evening, US Pacific).",
    "sessions": 30,
    "solved": 17,
    "solve_rate": 0.566667,
    "solve_rate_ci95_wilson": [
     0.392,
     0.7262
    ],
    "cost_list_usd_total": 13.96673,
    "cost_per_task_list_usd": 0.465558,
    "cost_per_solved_list_usd": 0.821572,
    "cost_per_task_median_list_usd": 0.348688,
    "steps_per_task": 9.4,
    "model_requests": 282,
    "exit_statuses": {
     "Submitted": 30
    },
    "empty_patches": 0
   },
   "all_opus": {
    "label": "All Claude Opus 5.5",
    "how_called": "claude-opus-5-5 named on every request; CodeRouter's pass-through path (named model, no routing, no fallback). Run 1 of the 2026-11-09 benchmark.",
    "sessions": 30,
    "solved": 18,
    "solve_rate": 0.6,
    "solve_rate_ci95_wilson": [
     0.4232,
     0.7541
    ],
    "cost_list_usd_total": 6.485635,
    "cost_per_task_list_usd": 0.216188,
    "cost_per_solved_list_usd": 0.360313,
    "cost_per_task_median_list_usd": 0.179342,
    "steps_per_task": 12.5667,
    "model_requests": 377,
    "exit_statuses": {
     "Submitted": 30
    },
    "empty_patches": 0
   },
   "all_sonnet": {
    "label": "All Claude Sonnet 5.5",
    "how_called": "claude-sonnet-5-5 named on every request; same pass-through path as all-Opus. Run 1 of the 2026-11-09 benchmark.",
    "sessions": 30,
    "solved": 17,
    "solve_rate": 0.566667,
    "solve_rate_ci95_wilson": [
     0.392,
     0.7262
    ],
    "cost_list_usd_total": 2.45174,
    "cost_per_task_list_usd": 0.081725,
    "cost_per_solved_list_usd": 0.14422,
    "cost_per_task_median_list_usd": 0.070059,
    "steps_per_task": 8.8667,
    "model_requests": 266,
    "exit_statuses": {
     "Submitted": 30
    },
    "empty_patches": 0
   },
   "coderouter_default": {
    "label": "CodeRouter default routing",
    "how_called": "model: auto through CodeRouter's production API, as any user got it on 2026-11-09. Cost = list price. Run 1 of the 2026-11-09 benchmark.",
    "sessions": 30,
    "solved": 15,
    "solve_rate": 0.5,
    "solve_rate_ci95_wilson": [
     0.3315,
     0.6685
    ],
    "cost_list_usd_total": 2.43806,
    "cost_per_task_list_usd": 0.081269,
    "cost_per_solved_list_usd": 0.162537,
    "cost_per_task_median_list_usd": 0.077201,
    "steps_per_task": 9.1,
    "model_requests": 273,
    "exit_statuses": {
     "Submitted": 30
    },
    "empty_patches": 0
   },
   "all_haiku": {
    "label": "All Claude Haiku 5.5",
    "how_called": "claude-haiku-5-5 named on every request; same pass-through path as all-Opus. Run 1 of the 2026-11-09 benchmark.",
    "sessions": 30,
    "solved": 13,
    "solve_rate": 0.433333,
    "solve_rate_ci95_wilson": [
     0.2738,
     0.608
    ],
    "cost_list_usd_total": 0.29373,
    "cost_per_task_list_usd": 0.009791,
    "cost_per_solved_list_usd": 0.022595,
    "cost_per_task_median_list_usd": 0.006099,
    "steps_per_task": 15.8667,
    "model_requests": 476,
    "exit_statuses": {
     "Submitted": 30
    },
    "empty_patches": 1
   }
  },
  "paired_vs_medium": [
   {
    "a": "claude_fable_5_2_medium",
    "b": "all_opus",
    "tasks": 30,
    "solved_a": 17,
    "solved_b": 18,
    "solve_rate_diff_a_minus_b": -0.033333,
    "solve_rate_diff_ci95_task_bootstrap": [
     -0.133333,
     0.066667
    ],
    "tasks_only_a_solved": [
     "raullenchai__rapid-mlx-289"
    ],
    "tasks_only_b_solved": [
     "pypa__twine-1309",
     "databricks__dbt-databricks-1428"
    ],
    "mcnemar_exact_p": 1.0,
    "cost_ratio_list_a_over_b": 2.1535,
    "cost_ratio_list_ci95_task_bootstrap": [
     1.8776,
     2.5275
    ],
    "cost_ratio_list_b_over_a": 0.4644,
    "cost_per_solved_ratio_list_a_over_b": 2.2802,
    "cost_per_solved_ratio_list_ci95_task_bootstrap": [
     1.9447,
     2.8059
    ],
    "tasks_where_a_cost_less": 0
   },
   {
    "a": "claude_fable_5_2_medium",
    "b": "all_sonnet",
    "tasks": 30,
    "solved_a": 17,
    "solved_b": 17,
    "solve_rate_diff_a_minus_b": 0.0,
    "solve_rate_diff_ci95_task_bootstrap": [
     -0.1,
     0.1
    ],
    "tasks_only_a_solved": [
     "spack__spack-52244"
    ],
    "tasks_only_b_solved": [
     "tox-dev__tox-3846"
    ],
    "mcnemar_exact_p": 1.0,
    "cost_ratio_list_a_over_b": 5.6967,
    "cost_ratio_list_ci95_task_bootstrap": [
     5.1393,
     6.2849
    ],
    "cost_ratio_list_b_over_a": 0.1755,
    "cost_per_solved_ratio_list_a_over_b": 5.6967,
    "cost_per_solved_ratio_list_ci95_task_bootstrap": [
     4.5779,
     6.9729
    ],
    "tasks_where_a_cost_less": 0
   },
   {
    "a": "claude_fable_5_2_medium",
    "b": "coderouter_default",
    "tasks": 30,
    "solved_a": 17,
    "solved_b": 15,
    "solve_rate_diff_a_minus_b": 0.066667,
    "solve_rate_diff_ci95_task_bootstrap": [
     0.0,
     0.166667
    ],
    "tasks_only_a_solved": [
     "nesquena__hermes-webui-330_interface",
     "spack__spack-52244"
    ],
    "tasks_only_b_solved": [],
    "mcnemar_exact_p": 0.5,
    "cost_ratio_list_a_over_b": 5.7286,
    "cost_ratio_list_ci95_task_bootstrap": [
     4.7653,
     6.8895
    ],
    "cost_ratio_list_b_over_a": 0.1746,
    "cost_per_solved_ratio_list_a_over_b": 5.0547,
    "cost_per_solved_ratio_list_ci95_task_bootstrap": [
     3.7279,
     6.5102
    ],
    "tasks_where_a_cost_less": 0
   },
   {
    "a": "claude_fable_5_2_medium",
    "b": "all_haiku",
    "tasks": 30,
    "solved_a": 17,
    "solved_b": 13,
    "solve_rate_diff_a_minus_b": 0.133333,
    "solve_rate_diff_ci95_task_bootstrap": [
     0.033333,
     0.266667
    ],
    "tasks_only_a_solved": [
     "tobymao__sqlglot-7479",
     "getmoto__moto-9903",
     "raullenchai__rapid-mlx-228",
     "graphistry__pygraphistry-1107_interface"
    ],
    "tasks_only_b_solved": [],
    "mcnemar_exact_p": 0.125,
    "cost_ratio_list_a_over_b": 47.5496,
    "cost_ratio_list_ci95_task_bootstrap": [
     35.0159,
     66.1362
    ],
    "cost_ratio_list_b_over_a": 0.021,
    "cost_per_solved_ratio_list_a_over_b": 36.3614,
    "cost_per_solved_ratio_list_ci95_task_bootstrap": [
     24.0012,
     53.7582
    ],
    "tasks_where_a_cost_less": 0
   }
  ]
 },
 "sensitivity_high_budget_stopped_as_failures": {
  "what": "Medium vs High on all 30 tasks, with the six budget-stopped High sessions counted as not solved (their cost counted).",
  "claude_fable_5_2_high": {
   "sessions": 30,
   "solved": 15,
   "solve_rate": 0.5,
   "solve_rate_ci95_wilson": [
    0.3315,
    0.6685
   ],
   "cost_list_usd_total": 52.676943,
   "cost_per_task_list_usd": 1.755898,
   "cost_per_solved_list_usd": 3.511796
  },
  "paired_medium_vs_high": {
   "a": "claude_fable_5_2_medium",
   "b": "claude_fable_5_2_high",
   "tasks": 30,
   "solved_a": 17,
   "solved_b": 15,
   "solve_rate_diff_a_minus_b": 0.066667,
   "solve_rate_diff_ci95_task_bootstrap": [
    0.0,
    0.166667
   ],
   "tasks_only_a_solved": [
    "copier-org__copier-2646",
    "spack__spack-52244"
   ],
   "tasks_only_b_solved": [],
   "mcnemar_exact_p": 0.5,
   "cost_ratio_list_a_over_b": 0.2651,
   "cost_ratio_list_ci95_task_bootstrap": [
    0.2124,
    0.3258
   ],
   "cost_ratio_list_b_over_a": 3.7716,
   "cost_per_solved_ratio_list_a_over_b": 0.2339,
   "cost_per_solved_ratio_list_ci95_task_bootstrap": [
    0.1737,
    0.3033
   ],
   "tasks_where_a_cost_less": 30
  }
 },
 "cost_structure_24_tasks": {
  "claude_fable_5_2_medium": {
   "sessions": 24,
   "model_requests": 207,
   "steps_per_task": 8.625,
   "cost_per_request_usd": 0.050604,
   "output_tokens_per_request_thinking_included": 600.64,
   "thinking_tokens_per_request_estimated": 171.6,
   "thinking_share_of_output_estimated": 0.2858,
   "output_tokens_per_task": 5180.54,
   "cache_read_tokens_per_request": 14603.24,
   "cache_write_tokens_per_request": 2363.93,
   "requests_with_a_thinking_block": 112,
   "thinking_block_share_of_requests": 0.5411,
   "requests_that_hit_max_tokens_8192": 0,
   "list_cost_share": {
    "output": 0.4748,
    "cache_write": 0.4671,
    "cache_read": 0.0577,
    "input": 0.0004
   }
  },
  "claude_fable_5_2_high": {
   "sessions": 24,
   "model_requests": 431,
   "steps_per_task": 17.9583,
   "cost_per_request_usd": 0.08746,
   "output_tokens_per_request_thinking_included": 1262.97,
   "thinking_tokens_per_request_estimated": 710.9,
   "thinking_share_of_output_estimated": 0.5629,
   "output_tokens_per_task": 22680.88,
   "cache_read_tokens_per_request": 37849.92,
   "cache_write_tokens_per_request": 2935.38,
   "requests_with_a_thinking_block": 338,
   "thinking_block_share_of_requests": 0.7842,
   "requests_that_hit_max_tokens_8192": 10,
   "list_cost_share": {
    "output": 0.5776,
    "cache_write": 0.3356,
    "cache_read": 0.0866,
    "input": 0.0002
   }
  },
  "all_opus": {
   "sessions": 24,
   "model_requests": 268,
   "steps_per_task": 11.1667,
   "cost_per_request_usd": 0.017521,
   "output_tokens_per_request_thinking_included": 374.47,
   "thinking_tokens_per_request_estimated": 81.6,
   "thinking_share_of_output_estimated": 0.2179,
   "output_tokens_per_task": 4181.58,
   "cache_read_tokens_per_request": 12076.59,
   "cache_write_tokens_per_request": 1521.52,
   "requests_with_a_thinking_block": 120,
   "thinking_block_share_of_requests": 0.4478,
   "requests_that_hit_max_tokens_8192": 0,
   "list_cost_share": {
    "output": 0.4275,
    "cache_write": 0.4342,
    "cache_read": 0.1379,
    "input": 0.0005
   }
  },
  "ratio_medium_over_high": {
   "steps_per_task": 0.4803,
   "cost_per_request_usd": 0.5786,
   "output_tokens_per_request_thinking_included": 0.4756,
   "thinking_tokens_per_request_estimated": 0.2414,
   "output_tokens_per_task": 0.2284,
   "cache_read_tokens_per_request": 0.3858,
   "cache_write_tokens_per_request": 0.8053
  },
  "ratio_medium_over_opus": {
   "steps_per_task": 0.7724,
   "cost_per_request_usd": 2.8882,
   "output_tokens_per_request_thinking_included": 1.604,
   "thinking_tokens_per_request_estimated": 2.1029,
   "output_tokens_per_task": 1.2389,
   "cache_read_tokens_per_request": 1.2092,
   "cache_write_tokens_per_request": 1.5537
  },
  "thinking_estimate_method": "Estimated, not reported by the API: thinking ~= output tokens minus the visible reply's tokens, predicted from its text and tool-call lengths with a least-squares fit on the Fable requests without a thinking block (both efforts), clipped to [0, output tokens].",
  "note": "Per request = per agent step (one model request per step). Medium vs High and vs all-Opus run 1 on the same 24 tasks."
 },
 "list_prices": {
  "unit": "USD per million tokens",
  "as_of": "2026-11-10",
  "cache_write_1h": "2 x input",
  "models": {
   "claude-fable-5-2": {
    "input": 8.0,
    "cache_read": 0.2,
    "cache_write_5m": 10.0,
    "output": 40.0,
    "cache_write_1h": 16.0
   },
   "claude-opus-5-5": {
    "input": 4.0,
    "cache_read": 0.2,
    "cache_write_5m": 5.0,
    "output": 20.0,
    "cache_write_1h": 8.0
   },
   "claude-sonnet-5-5": {
    "input": 2.0,
    "cache_read": 0.1,
    "cache_write_5m": 2.5,
    "output": 10.0,
    "cache_write_1h": 4.0
   },
   "claude-haiku-5-5": {
    "input": 0.1,
    "cache_read": 0.01,
    "cache_write_5m": 0.125,
    "output": 0.5,
    "cache_write_1h": 0.2
   }
  }
 },
 "checks": {
  "patches_identical_to_upstream_fix": {
   "sessions": 30,
   "identical": 0,
   "median_line_jaccard_vs_upstream_fix": 0.1096
  },
  "commands_fetching_the_tasks_own_repository_or_package": 0,
  "reference_fix_on_the_grading_machine": {
   "checked": 30,
   "passed": 30
  },
  "empty_change_on_the_grading_machine": {
   "tasks_checked": 30,
   "resolved_without_a_fix": 0
  },
  "network_isolation": {
   "method": "Every packet from a container to outside the host was rejected and logged, for agent and grading containers alike. Before the start a probe container checked that DNS, a GitHub IP on 443 and 1.1.1.1:53 were unreachable.",
   "probes_before_start": 1,
   "probes_all_blocked": true,
   "packets_blocked_logged": 11148,
   "packets_blocked_logged_by_kind": {
    "probe": 18,
    "agent": 1122,
    "eval": 10008
   }
  },
  "grading": {
   "patches_graded": 30,
   "apply_failed": 0,
   "grading_errors": 0
  },
  "responses": {
   "total": 282,
   "model_field": {
    "claude-fable-5-2": 282
   },
   "inference_geo": {
    "global": 282
   },
   "note": "Every response's model field was checked during the run; a different model would have stopped the run. The effort level is not reported in responses; before the run, an invalid effort value was rejected by the API, showing the field is read."
  },
  "sessions_stopped_by_a_limit_or_the_budget": 0
 },
 "limitations": [
  "30 tasks, one run per setup; a one- or two-task difference is noise at this size.",
  "The High run was on the morning and the Medium run on the evening of 2026-11-10 (US Pacific); the Opus, Sonnet, Haiku and CodeRouter runs on 2026-11-09. Not interleaved.",
  "The High run had six sessions stopped by its budget, so Medium vs High uses 24 tasks.",
  "All-Opus, all-Sonnet and all-Haiku ran through CodeRouter's pass-through path, which adds its own prompt-cache markers; Fable was called directly.",
  "One agent harness (mini-swe-agent, bash tool only). Claude Code uses other tools and prompts.",
  "Thinking tokens are an estimate (see cost_structure_24_tasks.thinking_estimate_method).",
  "List prices on 2026-11-10. Models and prices change."
 ]
}
