{
 "name": "Claude Fable 5.2 on 30 tasks of the coding-agent benchmark (quick run, 1 run)",
 "publisher": "CodeRouter (https://www.coderouter.io)",
 "disclosure": "CodeRouter is our product. Claude Fable 5.2 was called directly through Anthropic's API, not through CodeRouter.",
 "run_date": "2026-11-10",
 "other_setups_run_date": "2026-11-09 (run 1 of the 99-task benchmark)",
 "prices_as_of": "2026-11-10",
 "post": "https://www.coderouter.io/blog/claude-fable-5-2-coding-agent-benchmark",
 "benchmark_99_tasks": "https://www.coderouter.io/benchmark/agentic",
 "sessions_csv": "/benchmarks/coderouter-agentic-fable52-2026-11-sessions.csv",
 "method": {
  "model": "claude-fable-5-2 through Anthropic's Messages API with a separate API key, not through CodeRouter",
  "settings": "API defaults: max_tokens 8192, no temperature, thinking or effort parameter (the API default for Fable 5.2 is high effort)",
  "agent": "mini-swe-agent 2.4.6 (bash tool only, benchmarks/swebench.yaml prompts unchanged) via LiteLLM, as in the 99-task benchmark",
  "limits_per_session": {
   "steps": 150,
   "cost_usd": 10.0,
   "time_s": 3600,
   "per_command_timeout_s": 60,
   "max_tokens": 8192
  },
  "limits_other_setups": "the same, except cost_usd 2.5 per session (2026-11-09 benchmark)",
  "prompt_caching": "mini-swe-agent's default for Anthropic models: a 5-minute cache marker on the last message of every turn",
  "other_setups": "The five published setups' run 1 of the 2026-11-09 benchmark on the same tasks, paired by task. All-Opus, all-Sonnet and all-Haiku named their model through CodeRouter's pass-through path (no routing, no fallback); Fable 5.2 was called directly.",
  "grading": "swebench 5.0.2 harness (Docker) with the SWE-rebench evaluation spec: apply the patch, run the task's tests; solved = the required failing tests pass and the passing tests still pass",
  "cost": "List price: every request's input, cache-read, cache-write and output tokens x the model's list price (prices below), summed per session. OpenRouter Auto Code: OpenRouter's usage.cost per request.",
  "intervals": "Completion: Wilson 95%. Paired differences and cost ratios: task bootstrap (10,000 resamples, seed 20261104). p-values: exact McNemar on the tasks where exactly one of the two setups solved the task.",
  "exclusion_rule": "Pre-registered: a Fable 5.2 session stopped by the $10 per-session limit or by the run's overall budget is left out of the main comparison and listed, not counted as a failure; a session that reaches 150 steps or 60 minutes counts as not solved. Sensitivity: the same comparison with the left-out sessions counted as failures.",
  "interruption": "The run stopped once on an API billing error on our side. The 12 sessions that had already finished were kept; the other 18 tasks were rerun from scratch about 45 minutes later with the same settings, and their interrupted first attempts are not in the results. The change was recorded before the rerun.",
  "machine": "Agent containers and grading ran on rented cloud VMs (a new one for the rerun), used only to run containers. Nothing here measures the machine."
 },
 "tasks": {
  "dataset": "nebius/SWE-rebench-leaderboard",
  "revision": "34d5a58864acf91613740a09ec5d205228dcfa39",
  "file": "data/2026_03-00000-of-00001.parquet",
  "license": "CC BY 4.0",
  "drawn_from": "the 99 task ids of the 2026-11-09 benchmark (see benchmark_99_tasks)",
  "selection": "random.Random(20261110).sample(the 99 ids in the benchmark's run order, 30), listed in the benchmark's run order",
  "seed": 20261110,
  "task_ids": [
   "itsdnns__docsight-437",
   "koxudaxi__datamodel-code-generator-3070",
   "copier-org__copier-2646",
   "pypa__twine-1309",
   "databricks__dbt-databricks-1428",
   "tobymao__sqlglot-7479",
   "koxudaxi__datamodel-code-generator-3071",
   "nesquena__hermes-webui-330_interface",
   "koxudaxi__datamodel-code-generator-3046",
   "oemof__tespy-921",
   "hiero-ledger__hiero-sdk-python-1914_interface",
   "line__line-bot-sdk-python-981_interface",
   "getmoto__moto-9903",
   "nesquena__hermes-webui-1818",
   "spack__spack-52244",
   "vprusso__toqito-1484_interface",
   "pennylaneai__pennylane-9298",
   "keras-team__keras-22330",
   "tox-dev__tox-3846",
   "huggingface__huggingface_hub-4056",
   "scrapy__scrapy-7351_interface",
   "agronholm__anyio-1134",
   "marshmallow-code__marshmallow-2925_interface",
   "pallets-eco__wtforms-892_interface",
   "raullenchai__rapid-mlx-228",
   "meltano__meltano-9929",
   "graphistry__pygraphistry-1107_interface",
   "macbre__sql-metadata-625_interface",
   "keras-team__keras-22642",
   "raullenchai__rapid-mlx-289"
  ],
  "analysis_set": [
   "itsdnns__docsight-437",
   "pypa__twine-1309",
   "databricks__dbt-databricks-1428",
   "tobymao__sqlglot-7479",
   "koxudaxi__datamodel-code-generator-3071",
   "nesquena__hermes-webui-330_interface",
   "koxudaxi__datamodel-code-generator-3046",
   "oemof__tespy-921",
   "hiero-ledger__hiero-sdk-python-1914_interface",
   "line__line-bot-sdk-python-981_interface",
   "getmoto__moto-9903",
   "vprusso__toqito-1484_interface",
   "pennylaneai__pennylane-9298",
   "keras-team__keras-22330",
   "scrapy__scrapy-7351_interface",
   "agronholm__anyio-1134",
   "marshmallow-code__marshmallow-2925_interface",
   "pallets-eco__wtforms-892_interface",
   "raullenchai__rapid-mlx-228",
   "meltano__meltano-9929",
   "graphistry__pygraphistry-1107_interface",
   "macbre__sql-metadata-625_interface",
   "keras-team__keras-22642",
   "raullenchai__rapid-mlx-289"
  ],
  "budget_stopped": [
   {
    "task": "koxudaxi__datamodel-code-generator-3070",
    "cost_list_usd": 3.205235,
    "steps": 37,
    "model_requests": 36,
    "solved_by_in_2026_11_09_run1": []
   },
   {
    "task": "copier-org__copier-2646",
    "cost_list_usd": 1.613257,
    "steps": 22,
    "model_requests": 21,
    "solved_by_in_2026_11_09_run1": [
     "all_opus",
     "openrouter_auto_code",
     "all_sonnet",
     "coderouter_default",
     "all_haiku"
    ]
   },
   {
    "task": "nesquena__hermes-webui-1818",
    "cost_list_usd": 2.852821,
    "steps": 31,
    "model_requests": 30,
    "solved_by_in_2026_11_09_run1": []
   },
   {
    "task": "spack__spack-52244",
    "cost_list_usd": 2.843738,
    "steps": 39,
    "model_requests": 38,
    "solved_by_in_2026_11_09_run1": [
     "all_opus",
     "openrouter_auto_code",
     "all_haiku"
    ]
   },
   {
    "task": "tox-dev__tox-3846",
    "cost_list_usd": 1.11827,
    "steps": 26,
    "model_requests": 25,
    "solved_by_in_2026_11_09_run1": [
     "all_sonnet"
    ]
   },
   {
    "task": "huggingface__huggingface_hub-4056",
    "cost_list_usd": 3.348409,
    "steps": 31,
    "model_requests": 30,
    "solved_by_in_2026_11_09_run1": []
   }
  ],
  "budget_stopped_note": "Stopped before they finished when the run's overall budget was reached (none had reached the $10 per-session limit); left out of the main comparison as pre-registered. Their solved_by lists are the five published setups' run 1 on the same task.",
  "interruption_batches": {
   "kept_first_run": {
    "tasks": [
     "agronholm__anyio-1134",
     "databricks__dbt-databricks-1428",
     "getmoto__moto-9903",
     "hiero-ledger__hiero-sdk-python-1914_interface",
     "itsdnns__docsight-437",
     "keras-team__keras-22330",
     "koxudaxi__datamodel-code-generator-3071",
     "nesquena__hermes-webui-330_interface",
     "pypa__twine-1309",
     "scrapy__scrapy-7351_interface",
     "tobymao__sqlglot-7479",
     "vprusso__toqito-1484_interface"
    ],
    "all_opus_run1_solved_same_tasks": 9,
    "claude_fable_5_2_solved": 7,
    "sessions_in_analysis_set": 12
   },
   "rerun": {
    "tasks": [
     "copier-org__copier-2646",
     "graphistry__pygraphistry-1107_interface",
     "huggingface__huggingface_hub-4056",
     "keras-team__keras-22642",
     "koxudaxi__datamodel-code-generator-3046",
     "koxudaxi__datamodel-code-generator-3070",
     "line__line-bot-sdk-python-981_interface",
     "macbre__sql-metadata-625_interface",
     "marshmallow-code__marshmallow-2925_interface",
     "meltano__meltano-9929",
     "nesquena__hermes-webui-1818",
     "oemof__tespy-921",
     "pallets-eco__wtforms-892_interface",
     "pennylaneai__pennylane-9298",
     "raullenchai__rapid-mlx-228",
     "raullenchai__rapid-mlx-289",
     "spack__spack-52244",
     "tox-dev__tox-3846"
    ],
    "budget_stopped": [
     "copier-org__copier-2646",
     "huggingface__huggingface_hub-4056",
     "koxudaxi__datamodel-code-generator-3070",
     "nesquena__hermes-webui-1818",
     "spack__spack-52244",
     "tox-dev__tox-3846"
    ],
    "all_opus_run1_solved_same_tasks": 7,
    "claude_fable_5_2_solved": 8,
    "sessions_in_analysis_set": 12
   }
  }
 },
 "setups": {
  "claude_fable_5_2": {
   "label": "Claude Fable 5.2",
   "how_called": "claude-fable-5-2 through Anthropic's Messages API (api.anthropic.com) with a separate API key, not through CodeRouter. Same request as the other setups: max_tokens 8192, no temperature, thinking or effort parameter, so the model ran at its API defaults (for Fable 5.2: high effort). Cost = tokens x Anthropic list price. Run on 2026-11-10.",
   "sessions": 24,
   "solved": 15,
   "solve_rate": 0.625,
   "solve_rate_ci95_wilson": [
    0.4271,
    0.7884
   ],
   "cost_list_usd_total": 37.695213,
   "cost_per_task_list_usd": 1.570634,
   "cost_per_solved_list_usd": 2.513014,
   "cost_per_task_median_list_usd": 1.403254,
   "model_requests": 431,
   "steps_mean": 17.96,
   "exit_statuses": {
    "Submitted": 24
   },
   "empty_patches": 0
  },
  "all_opus": {
   "label": "All Claude Opus 5.5",
   "how_called": "claude-opus-5-5 named on every request; CodeRouter's pass-through path (named model, no routing, no fallback). Cost = tokens x Anthropic list price. Run 1 of the 2026-11-09 benchmark.",
   "sessions": 24,
   "solved": 16,
   "solve_rate": 0.666667,
   "solve_rate_ci95_wilson": [
    0.4671,
    0.8203
   ],
   "cost_list_usd_total": 4.695636,
   "cost_per_task_list_usd": 0.195652,
   "cost_per_solved_list_usd": 0.293477,
   "cost_per_task_median_list_usd": 0.16972,
   "model_requests": 268,
   "steps_mean": 11.17,
   "exit_statuses": {
    "Submitted": 24
   },
   "empty_patches": 0
  },
  "openrouter_auto_code": {
   "label": "OpenRouter Auto Code",
   "how_called": "model: openrouter/auto-code through OpenRouter's API. Cost = usage.cost returned by OpenRouter on each request. Run 1 of the 2026-11-09 benchmark.",
   "sessions": 24,
   "solved": 16,
   "solve_rate": 0.666667,
   "solve_rate_ci95_wilson": [
    0.4671,
    0.8203
   ],
   "cost_list_usd_total": 5.469285,
   "cost_per_task_list_usd": 0.227887,
   "cost_per_solved_list_usd": 0.34183,
   "cost_per_task_median_list_usd": 0.152825,
   "model_requests": 346,
   "steps_mean": 14.42,
   "exit_statuses": {
    "Submitted": 24
   },
   "empty_patches": 0
  },
  "all_sonnet": {
   "label": "All Claude Sonnet 5.5",
   "how_called": "claude-sonnet-5-5 named on every request; same pass-through path as all-Opus. Run 1 of the 2026-11-09 benchmark.",
   "sessions": 24,
   "solved": 15,
   "solve_rate": 0.625,
   "solve_rate_ci95_wilson": [
    0.4271,
    0.7884
   ],
   "cost_list_usd_total": 1.845853,
   "cost_per_task_list_usd": 0.076911,
   "cost_per_solved_list_usd": 0.123057,
   "cost_per_task_median_list_usd": 0.069118,
   "model_requests": 204,
   "steps_mean": 8.5,
   "exit_statuses": {
    "Submitted": 24
   },
   "empty_patches": 0
  },
  "coderouter_default": {
   "label": "CodeRouter default routing",
   "how_called": "model: auto through CodeRouter's production API, as any user got it on 2026-11-09. Cost = list price. Run 1 of the 2026-11-09 benchmark.",
   "sessions": 24,
   "solved": 14,
   "solve_rate": 0.583333,
   "solve_rate_ci95_wilson": [
    0.3883,
    0.7553
   ],
   "cost_list_usd_total": 1.674484,
   "cost_per_task_list_usd": 0.06977,
   "cost_per_solved_list_usd": 0.119606,
   "cost_per_task_median_list_usd": 0.069759,
   "model_requests": 190,
   "steps_mean": 7.92,
   "exit_statuses": {
    "Submitted": 24
   },
   "empty_patches": 0
  },
  "all_haiku": {
   "label": "All Claude Haiku 5.5",
   "how_called": "claude-haiku-5-5 named on every request; same pass-through path as all-Opus. Run 1 of the 2026-11-09 benchmark.",
   "sessions": 24,
   "solved": 11,
   "solve_rate": 0.458333,
   "solve_rate_ci95_wilson": [
    0.2789,
    0.6493
   ],
   "cost_list_usd_total": 0.188668,
   "cost_per_task_list_usd": 0.007861,
   "cost_per_solved_list_usd": 0.017152,
   "cost_per_task_median_list_usd": 0.005971,
   "model_requests": 311,
   "steps_mean": 12.96,
   "exit_statuses": {
    "Submitted": 24
   },
   "empty_patches": 1
  }
 },
 "paired_vs_claude_fable_5_2": [
  {
   "a": "claude_fable_5_2",
   "b": "all_opus",
   "tasks": 24,
   "solved_a": 15,
   "solved_b": 16,
   "solve_rate_diff_a_minus_b": -0.041667,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.208333,
    0.083333
   ],
   "tasks_only_a_solved": [
    "raullenchai__rapid-mlx-289"
   ],
   "tasks_only_b_solved": [
    "pypa__twine-1309",
    "databricks__dbt-databricks-1428"
   ],
   "mcnemar_exact_p": 1.0,
   "cost_ratio_list_a_over_b": 8.0277,
   "cost_ratio_list_ci95_task_bootstrap": [
    6.305,
    10.1852
   ],
   "cost_per_solved_ratio_list_a_over_b": 8.5629,
   "cost_per_solved_ratio_list_ci95_task_bootstrap": [
    6.3243,
    11.8776
   ],
   "tasks_where_a_cost_less": 0
  },
  {
   "a": "claude_fable_5_2",
   "b": "openrouter_auto_code",
   "tasks": 24,
   "solved_a": 15,
   "solved_b": 16,
   "solve_rate_diff_a_minus_b": -0.041667,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.166667,
    0.083333
   ],
   "tasks_only_a_solved": [
    "nesquena__hermes-webui-330_interface"
   ],
   "tasks_only_b_solved": [
    "pypa__twine-1309",
    "agronholm__anyio-1134"
   ],
   "mcnemar_exact_p": 1.0,
   "cost_ratio_list_a_over_b": 6.8922,
   "cost_ratio_list_ci95_task_bootstrap": [
    5.0231,
    9.8846
   ],
   "cost_per_solved_ratio_list_a_over_b": 7.3516,
   "cost_per_solved_ratio_list_ci95_task_bootstrap": [
    5.1108,
    11.4212
   ],
   "tasks_where_a_cost_less": 0
  },
  {
   "a": "claude_fable_5_2",
   "b": "all_sonnet",
   "tasks": 24,
   "solved_a": 15,
   "solved_b": 15,
   "solve_rate_diff_a_minus_b": 0.0,
   "solve_rate_diff_ci95_task_bootstrap": [
    0.0,
    0.0
   ],
   "tasks_only_a_solved": [],
   "tasks_only_b_solved": [],
   "mcnemar_exact_p": 1.0,
   "cost_ratio_list_a_over_b": 20.4216,
   "cost_ratio_list_ci95_task_bootstrap": [
    16.8468,
    25.1369
   ],
   "cost_per_solved_ratio_list_a_over_b": 20.4216,
   "cost_per_solved_ratio_list_ci95_task_bootstrap": [
    16.8468,
    25.1369
   ],
   "tasks_where_a_cost_less": 0
  },
  {
   "a": "claude_fable_5_2",
   "b": "coderouter_default",
   "tasks": 24,
   "solved_a": 15,
   "solved_b": 14,
   "solve_rate_diff_a_minus_b": 0.041667,
   "solve_rate_diff_ci95_task_bootstrap": [
    0.0,
    0.125
   ],
   "tasks_only_a_solved": [
    "nesquena__hermes-webui-330_interface"
   ],
   "tasks_only_b_solved": [],
   "mcnemar_exact_p": 1.0,
   "cost_ratio_list_a_over_b": 22.5115,
   "cost_ratio_list_ci95_task_bootstrap": [
    19.7788,
    25.7922
   ],
   "cost_per_solved_ratio_list_a_over_b": 21.0108,
   "cost_per_solved_ratio_list_ci95_task_bootstrap": [
    17.268,
    24.7907
   ],
   "tasks_where_a_cost_less": 0
  },
  {
   "a": "claude_fable_5_2",
   "b": "all_haiku",
   "tasks": 24,
   "solved_a": 15,
   "solved_b": 11,
   "solve_rate_diff_a_minus_b": 0.166667,
   "solve_rate_diff_ci95_task_bootstrap": [
    0.041667,
    0.333333
   ],
   "tasks_only_a_solved": [
    "tobymao__sqlglot-7479",
    "getmoto__moto-9903",
    "raullenchai__rapid-mlx-228",
    "graphistry__pygraphistry-1107_interface"
   ],
   "tasks_only_b_solved": [],
   "mcnemar_exact_p": 0.125,
   "cost_ratio_list_a_over_b": 199.7965,
   "cost_ratio_list_ci95_task_bootstrap": [
    152.8153,
    265.8976
   ],
   "cost_per_solved_ratio_list_a_over_b": 146.5175,
   "cost_per_solved_ratio_list_ci95_task_bootstrap": [
    92.2764,
    213.0963
   ],
   "tasks_where_a_cost_less": 0
  }
 ],
 "sensitivity_budget_stopped_as_failures": {
  "what": "The six budget-stopped Fable 5.2 sessions counted as not solved, with their cost counted: all 30 tasks.",
  "claude_fable_5_2": {
   "sessions": 30,
   "solved": 15,
   "solve_rate": 0.5,
   "solve_rate_ci95_wilson": [
    0.3315,
    0.6685
   ],
   "cost_list_usd_total": 52.676943,
   "cost_per_task_list_usd": 1.755898,
   "cost_per_solved_list_usd": 3.511796
  },
  "all_opus": {
   "sessions": 30,
   "solved": 18,
   "solve_rate": 0.6,
   "solve_rate_ci95_wilson": [
    0.4232,
    0.7541
   ],
   "cost_list_usd_total": 6.485635,
   "cost_per_task_list_usd": 0.216188,
   "cost_per_solved_list_usd": 0.360313
  },
  "paired_vs_all_opus": {
   "a": "claude_fable_5_2",
   "b": "all_opus",
   "tasks": 30,
   "solved_a": 15,
   "solved_b": 18,
   "solve_rate_diff_a_minus_b": -0.1,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.233333,
    0.033333
   ],
   "tasks_only_a_solved": [
    "raullenchai__rapid-mlx-289"
   ],
   "tasks_only_b_solved": [
    "copier-org__copier-2646",
    "pypa__twine-1309",
    "databricks__dbt-databricks-1428",
    "spack__spack-52244"
   ],
   "mcnemar_exact_p": 0.375,
   "cost_ratio_list_a_over_b": 8.1221,
   "cost_ratio_list_ci95_task_bootstrap": [
    6.5823,
    10.0974
   ],
   "cost_per_solved_ratio_list_a_over_b": 9.7465,
   "cost_per_solved_ratio_list_ci95_task_bootstrap": [
    7.1915,
    13.6647
   ],
   "tasks_where_a_cost_less": 0
  }
 },
 "per_session_limit_of_the_other_setups": {
  "limit_usd": 2.5,
  "sessions_at_or_over": 7,
  "in_analysis_set": 3,
  "in_analysis_set_solved": 3,
  "budget_stopped_sessions_already_over": 4,
  "all_opus_sessions_at_or_over_in_the_full_99_task_benchmark": 0
 },
 "cost_structure_fable_vs_opus": {
  "claude_fable_5_2": {
   "sessions": 24,
   "model_requests": 431,
   "cost_per_request_usd": 0.08746,
   "output_tokens_per_request_thinking_included": 1262.97,
   "model_requests_per_session": 17.9583,
   "output_tokens_per_session": 22680.88,
   "cache_read_tokens_per_request": 37849.92,
   "cache_write_tokens_per_request": 2935.38,
   "requests_with_a_thinking_block": 338,
   "thinking_block_share_of_requests": 0.7842,
   "requests_that_hit_max_tokens_8192": 10,
   "list_cost_share": {
    "output": 0.5776,
    "cache_write": 0.3356,
    "cache_read": 0.0866,
    "input": 0.0002
   }
  },
  "all_opus": {
   "sessions": 24,
   "model_requests": 268,
   "cost_per_request_usd": 0.017521,
   "output_tokens_per_request_thinking_included": 374.47,
   "model_requests_per_session": 11.1667,
   "output_tokens_per_session": 4181.58,
   "cache_read_tokens_per_request": 12076.59,
   "cache_write_tokens_per_request": 1521.52,
   "requests_with_a_thinking_block": 120,
   "thinking_block_share_of_requests": 0.4478,
   "requests_that_hit_max_tokens_8192": 0,
   "list_cost_share": {
    "output": 0.4275,
    "cache_write": 0.4342,
    "cache_read": 0.1379,
    "input": 0.0005
   }
  },
  "ratio_fable_over_opus": {
   "cost_per_request_usd": 4.9917,
   "output_tokens_per_request_thinking_included": 3.3727,
   "model_requests_per_session": 1.6082,
   "output_tokens_per_session": 5.424,
   "cache_read_tokens_per_request": 3.1342,
   "cache_write_tokens_per_request": 1.9292
  }
 },
 "price_vs_behaviour": {
  "what": "The 24 tasks' sessions re-priced at the other model's list prices (same tokens).",
  "opus_tokens_at_opus_prices_usd": 4.695641,
  "opus_tokens_at_fable_prices_usd": 8.743977,
  "fable_tokens_at_opus_prices_usd": 20.478938,
  "fable_tokens_at_fable_prices_usd": 37.695213,
  "price_factor_on_opus_tokens": 1.8621,
  "price_factor_on_fable_tokens": 1.8407,
  "behaviour_factor_at_opus_prices": 4.3613,
  "behaviour_factor_at_fable_prices": 4.311,
  "total_cost_ratio": 8.0277,
  "note": "price_factor_on_fable_tokens x behaviour_factor_at_opus_prices = total_cost_ratio exactly (as does price_factor_on_opus_tokens x behaviour_factor_at_fable_prices)."
 },
 "list_prices": {
  "unit": "USD per million tokens",
  "as_of": "2026-11-10",
  "cache_write_1h": "2 x input",
  "models": {
   "claude-fable-5-2": {
    "input": 8.0,
    "cache_read": 0.2,
    "cache_write_5m": 10.0,
    "output": 40.0,
    "cache_write_1h": 16.0
   },
   "claude-opus-5-5": {
    "input": 4.0,
    "cache_read": 0.2,
    "cache_write_5m": 5.0,
    "output": 20.0,
    "cache_write_1h": 8.0
   },
   "claude-sonnet-5-5": {
    "input": 2.0,
    "cache_read": 0.1,
    "cache_write_5m": 2.5,
    "output": 10.0,
    "cache_write_1h": 4.0
   },
   "claude-haiku-5-5": {
    "input": 0.1,
    "cache_read": 0.01,
    "cache_write_5m": 0.125,
    "output": 0.5,
    "cache_write_1h": 0.2
   }
  },
  "openrouter_auto_code": "usage.cost returned by OpenRouter on each request"
 },
 "checks": {
  "patches_identical_to_upstream_fix": {
   "sessions": 30,
   "identical": 0,
   "median_line_jaccard_vs_upstream_fix": 0.057
  },
  "commands_fetching_the_tasks_own_repository_or_package": 0,
  "reference_fix_on_the_grading_machine": {
   "checked": 30,
   "passed": 30
  },
  "empty_change_on_the_grading_machine": {
   "tasks_checked": 30,
   "resolved_without_a_fix": 0
  },
  "network_isolation": {
   "method": "Every packet from a container to outside the host was rejected and logged, for agent and grading containers alike. Before each start (the first run and the rerun) a probe container checked that DNS, a GitHub IP on 443 and 1.1.1.1:53 were unreachable.",
   "probes_before_start": 2,
   "probes_all_blocked": true,
   "packets_blocked_logged": 12414,
   "packets_blocked_logged_by_kind": {
    "probe": 18,
    "agent": 2980,
    "eval": 9416
   },
   "logged_scope": "The rerun machine (the 18 rerun sessions and the grading of all 30). The first machine's per-packet log was not kept."
  },
  "grading": {
   "patches_graded": 24,
   "apply_failed": 0,
   "grading_errors": 0
  },
  "responses": {
   "in_the_30_published_sessions": 611,
   "in_the_30_published_sessions_model_field": {
    "claude-fable-5-2": 611
   },
   "in_the_30_published_sessions_inference_geo": {
    "global": 611
   },
   "all_attempts_including_the_discarded_interrupted_sessions": 829,
   "all_attempts_model_field": {
    "claude-fable-5-2": 829
   },
   "note": "Every response's model field was checked during the run; a different model would have stopped the run."
  }
 },
 "limitations": [
  "30 tasks, one run, 24 in the main comparison. A one- or two-task difference is noise at this size.",
  "The other setups ran on 2026-11-09 and Fable 5.2 on 2026-11-10, not interleaved, so day-to-day changes in models or serving aren't controlled.",
  "Fable 5.2 had a $10 per-session limit and the other setups $2.50. No all-Opus session in the full benchmark reached $2.50.",
  "The six budget-stopped sessions were among the longest and most expensive, so leaving them out may flatter Fable 5.2's cost; counting them, see sensitivity_budget_stopped_as_failures.",
  "One agent harness (mini-swe-agent, bash tool only) at the API's default effort. Claude Code, Codex or Cursor sessions use other tools, prompts and effort settings.",
  "List prices on 2026-11-10. Models and prices change."
 ]
}
