{
 "name": "Coding-agent benchmark: 99 SWE-rebench tasks, 5 setups, 2 runs each",
 "publisher": "CodeRouter (https://www.coderouter.io)",
 "disclosure": "CodeRouter is our product. OpenRouter Auto Code is a competing router; we ran it through OpenRouter's public API like any customer.",
 "run_date": "2026-11-09",
 "prices_as_of": "2026-11-09",
 "page": "https://www.coderouter.io/benchmark/agentic",
 "sessions_csv": "/benchmarks/coderouter-agentic-99-2026-11-sessions.csv",
 "method": {
  "agent": "mini-swe-agent 2.4.6 (bash tool only, benchmarks/swebench.yaml prompts unchanged) via LiteLLM 1.108.0",
  "limits_per_session": {
   "steps": 150,
   "cost_usd": 2.5,
   "time_s": 3600,
   "per_command_timeout_s": 60,
   "max_tokens": 8192,
   "consecutive_format_errors": 10
  },
  "prompt_caching": "mini-swe-agent's default for Anthropic models: a 5-minute cache marker on the last message of every turn, the same for every setup",
  "reasoning_effort": "Auto Code chooses its own reasoning effort; every other setup used the model's default settings",
  "grading": "swebench 5.0.2 harness (Docker) with the SWE-rebench evaluation spec: apply the patch, run the task's tests; resolved = the required failing tests pass and the passing tests still pass",
  "runs": "Each setup ran every task twice on 2026-11-09. Run 2 started after run 1 ended, with the task order rotated, so the same setup reached the same task about an hour apart (no shared prompt cache). The five setups ran interleaved, 20 sessions at a time.",
  "session": "one setup on one task in one run; 99 tasks x 2 runs = 198 sessions per setup",
  "intervals": "Completion: task bootstrap (both runs of a task resampled together; 10,000 resamples, seed 20261104) and Wilson over sessions. Costs and paired differences: task bootstrap. Paired p-values: exact McNemar on sessions where exactly one of the two setups solved the task (same task, same run).",
  "cost": "List price: every request's input, cache-read, cache-write (5 min / 1 h) and output tokens x the model's list price, summed per session. Auto Code: OpenRouter's usage.cost per request. Customer-paid: see customer_paid_rule per setup.",
  "machine": "Agent containers and grading ran on one rented cloud VM, used only to run containers. Nothing here measures the machine."
 },
 "tasks": {
  "dataset": "nebius/SWE-rebench-leaderboard",
  "revision": "34d5a58864acf91613740a09ec5d205228dcfa39",
  "file": "data/2026_03-00000-of-00001.parquet",
  "sha256": "18e198ac18b3c25b307c0aa5d9b6e20d338886e186bf7c12addb759ad4165a61",
  "license": "CC BY 4.0",
  "pool": 110,
  "pool_created_from": "2026-03-01",
  "selection": "SWE-rebench 2026_03 split, tasks created on or after 2026-03-01 with a pytest log parser; sorted ids shuffled with random.Random(20261104)",
  "ran": 108,
  "reference_fix_check": {
   "rule": "with the network off, the upstream fix is graded 3 times; a task is kept only if all 3 pass, and an empty change must fail it",
   "checked": 110,
   "passed_all_3": 100,
   "offline_runs_resolved": [
    101,
    100,
    101
   ],
   "empty_change_resolved": 0,
   "empty_change_checked": 110
  },
  "dropped_from_the_108": [
   {
    "task": "azure-samples__azure-search-openai-demo-3025",
    "reason": "its tests need network access; offline the test run does not start"
   },
   {
    "task": "destine-climate-dt__aqua-2768",
    "reason": "some of its tests need network access (e.g. git clone from GitHub, data downloads, DNS); they fail offline even with the upstream fix"
   },
   {
    "task": "geopython__pygeoapi-2338",
    "reason": "some of its tests need network access (e.g. git clone from GitHub, data downloads, DNS); they fail offline even with the upstream fix"
   },
   {
    "task": "mempalace__mempalace-1004",
    "reason": "some of its tests need network access (e.g. git clone from GitHub, data downloads, DNS); they fail offline even with the upstream fix"
   },
   {
    "task": "olofk__fusesoc-776_interface",
    "reason": "some of its tests need network access (e.g. git clone from GitHub, data downloads, DNS); they fail offline even with the upstream fix"
   },
   {
    "task": "pdm-project__pdm-3759",
    "reason": "some of its tests need network access (e.g. git clone from GitHub, data downloads, DNS); they fail offline even with the upstream fix"
   },
   {
    "task": "pgmpy__pgmpy-3137",
    "reason": "some of its tests need network access (e.g. git clone from GitHub, data downloads, DNS); they fail offline even with the upstream fix"
   },
   {
    "task": "reframe-hpc__reframe-3660_interface",
    "reason": "some of its tests need network access (e.g. git clone from GitHub, data downloads, DNS); they fail offline even with the upstream fix"
   },
   {
    "task": "zauberzeug__nicegui-5914",
    "reason": "a test that starts a local server failed in 1 of 3 offline reference-fix runs"
   }
  ],
  "main_analysis": 99,
  "repositories": 70,
  "created_at_range": [
   "2026-03-01",
   "2026-05-12"
  ],
  "task_ids": [
   "aallan__vera-662_interface",
   "agentscope-ai__qwenpaw-3278",
   "agronholm__anyio-1121",
   "agronholm__anyio-1134",
   "alibaba__opensandbox-426",
   "alibaba__opensandbox-816",
   "apache__iceberg-python-3295",
   "astronomy-commons__hats-648",
   "astronomy-commons__lsdb-1349",
   "astropy__astropy-19438",
   "beever-ai__beever-atlas-102",
   "celestoai__smolvm-172",
   "con__nwb2bids-347_interface",
   "conan-io__conan-19735_interface",
   "conan-io__conan-19750",
   "containers__ramalama-2487",
   "copier-org__copier-2646",
   "cpp-linter__cpp-linter-hooks-206_interface",
   "cs-si__eodag-2176",
   "dagster-io__dagster-33605",
   "databricks__dbt-databricks-1428",
   "delgan__loguru-1451",
   "deltares__hydrolib-core-1049_interface",
   "genericmappingtools__pygmt-4463",
   "getmoto__moto-9841",
   "getmoto__moto-9903",
   "graphistry__pygraphistry-1107_interface",
   "graphistry__pygraphistry-1277",
   "harumiweb__exstruct-113",
   "hiero-ledger__hiero-sdk-python-1914_interface",
   "hiero-ledger__hiero-sdk-python-2007_interface",
   "hkuds__openharness-217",
   "holoviz__param-1117",
   "huggingface__huggingface_hub-4056",
   "idaholab__montepy-933_interface",
   "itsdnns__docsight-437",
   "keras-team__keras-22316",
   "keras-team__keras-22330",
   "keras-team__keras-22642",
   "koxudaxi__datamodel-code-generator-3046",
   "koxudaxi__datamodel-code-generator-3070",
   "koxudaxi__datamodel-code-generator-3071",
   "kubeflow__pipelines-13112",
   "line__line-bot-sdk-python-981_interface",
   "macbre__sql-metadata-625_interface",
   "marshmallow-code__marshmallow-2925_interface",
   "meltano__meltano-9929",
   "meltano__meltano-9950",
   "microsoft__qcodes-8039",
   "nesquena__hermes-webui-1818",
   "nesquena__hermes-webui-2056",
   "nesquena__hermes-webui-330_interface",
   "oemof__tespy-921",
   "pallets-eco__wtforms-892_interface",
   "pallets__click-3239",
   "pandas-dev__pandas-64796",
   "pandas-dev__pandas-64797",
   "pandas-dev__pandas-64816",
   "pennylaneai__pennylane-9298",
   "pipecat-ai__pipecat-4283",
   "psd-tools__psd-tools-593",
   "pycqa__isort-2491",
   "pyinfra-dev__pyinfra-1630",
   "pyinfra-dev__pyinfra-1665",
   "pyinfra-dev__pyinfra-1679_interface",
   "pypa__build-1027",
   "pypa__twine-1309",
   "pypsa__pypsa-1653",
   "python-scim__scim2-models-139_interface",
   "python-wheel-build__fromager-1106",
   "python-wheel-build__fromager-1124",
   "raullenchai__rapid-mlx-227",
   "raullenchai__rapid-mlx-228",
   "raullenchai__rapid-mlx-289",
   "raullenchai__rapid-mlx-341_interface",
   "schemathesis__schemathesis-3778",
   "schemathesis__schemathesis-3933",
   "schemathesis__schemathesis-4087",
   "scikit-learn__scikit-learn-33565",
   "scrapy__scrapy-7351_interface",
   "spack__spack-52244",
   "sqlfluff__sqlfluff-7615",
   "sqlfluff__sqlfluff-7617",
   "sqlfluff__sqlfluff-7622",
   "stanfordnlp__dspy-9478_interface",
   "sunpy__sunpy-8548_interface",
   "tobymao__sqlglot-7187",
   "tobymao__sqlglot-7457",
   "tobymao__sqlglot-7479",
   "tox-dev__tox-3846",
   "tox-dev__tox-3931",
   "ucfopen__canvasapi-716",
   "ultraplot__ultraplot-696",
   "unit8co__darts-3065",
   "vprusso__toqito-1484_interface",
   "vprusso__toqito-1538",
   "youssofal__mtplx-21",
   "zarr-developers__virtualizarr-979",
   "zauberzeug__nicegui-5858"
  ]
 },
 "setups": {
  "all_opus": {
   "label": "All Claude Opus 5.5",
   "how_called": "claude-opus-5-5 named on every request; CodeRouter's pass-through path (named model, no routing, no fallback). Cost = tokens x Anthropic list price.",
   "sessions": 198,
   "resolved": 130,
   "solve_rate": 0.656566,
   "solve_rate_ci95_task_bootstrap": [
    0.565657,
    0.747475
   ],
   "solve_rate_ci95_wilson": [
    0.588,
    0.7192
   ],
   "resolved_by_run": {
    "run1": 64,
    "run2": 66
   },
   "cost_list_usd_total": 48.9136,
   "cost_per_task_list_usd": 0.247,
   "cost_per_task_list_ci95": [
    0.2083,
    0.2918
   ],
   "cost_per_task_median_list_usd": 0.1723,
   "customer_paid_rule": "list price (Anthropic API)",
   "customer_multiplier": 1.0,
   "cost_customer_usd_total": 48.9136,
   "cost_per_task_customer_usd": 0.247,
   "cost_per_solved_list_usd": 0.3763,
   "cost_per_solved_list_ci95": [
    0.2992,
    0.4775
   ],
   "cost_per_solved_customer_usd": 0.3763,
   "cache_read_share_of_prompt_tokens": 0.9184,
   "model_switches": 0,
   "sessions_with_model_switch": 0,
   "model_switches_per_session": 0.0,
   "steps_median": 11.0,
   "steps_mean": 13.7,
   "exit_statuses": {
    "Submitted": 198
   },
   "empty_patches": 0,
   "malformed_submissions": [],
   "served_model_requests": {
    "claude-opus-5.5": 2711
   },
   "served_model_cost_list_usd": {
    "claude-opus-5.5": 48.9136
   },
   "first_request_model": {
    "claude-opus-5.5": 198
   }
  },
  "openrouter_auto_code": {
   "label": "OpenRouter Auto Code (openrouter/auto-code)",
   "how_called": "model: openrouter/auto-code through OpenRouter's API. Cost = usage.cost returned by OpenRouter on each request.",
   "sessions": 198,
   "resolved": 125,
   "solve_rate": 0.631313,
   "solve_rate_ci95_task_bootstrap": [
    0.540404,
    0.722222
   ],
   "solve_rate_ci95_wilson": [
    0.5622,
    0.6954
   ],
   "resolved_by_run": {
    "run1": 64,
    "run2": 61
   },
   "cost_list_usd_total": 48.7941,
   "cost_per_task_list_usd": 0.2464,
   "cost_per_task_list_ci95": [
    0.2109,
    0.2864
   ],
   "cost_per_task_median_list_usd": 0.1819,
   "customer_paid_rule": "list price x 1.055 (OpenRouter's 5.5% fee on credit purchases, Standard)",
   "customer_multiplier": 1.055,
   "cost_customer_usd_total": 51.4778,
   "cost_per_task_customer_usd": 0.26,
   "cost_per_solved_list_usd": 0.3904,
   "cost_per_solved_list_ci95": [
    0.3135,
    0.4959
   ],
   "cost_per_solved_customer_usd": 0.4118,
   "cache_read_share_of_prompt_tokens": 0.8995,
   "model_switches": 44,
   "sessions_with_model_switch": 19,
   "model_switches_per_session": 0.222,
   "steps_median": 12.0,
   "steps_mean": 14.7,
   "exit_statuses": {
    "Submitted": 198
   },
   "empty_patches": 0,
   "malformed_submissions": [],
   "served_model_requests": {
    "openai/gpt-6.1-sol": 1235,
    "anthropic/claude-opus-5.5": 1001,
    "moonshotai/kimi-k3.1-pro": 386,
    "z-ai/glm-5.3": 268,
    "google/gemini-4-argon": 28
   },
   "served_model_cost_list_usd": {
    "openai/gpt-6.1-sol": 21.9648,
    "anthropic/claude-opus-5.5": 19.7689,
    "moonshotai/kimi-k3.1-pro": 5.4348,
    "z-ai/glm-5.3": 1.0699,
    "google/gemini-4-argon": 0.5557
   },
   "first_request_model": {
    "openai/gpt-6.1-sol": 92,
    "anthropic/claude-opus-5.5": 84,
    "moonshotai/kimi-k3.1-pro": 14,
    "z-ai/glm-5.3": 8
   }
  },
  "all_sonnet": {
   "label": "All Claude Sonnet 5.5",
   "how_called": "claude-sonnet-5-5 named on every request; same pass-through path as all-Opus.",
   "sessions": 198,
   "resolved": 118,
   "solve_rate": 0.59596,
   "solve_rate_ci95_task_bootstrap": [
    0.505051,
    0.686869
   ],
   "solve_rate_ci95_wilson": [
    0.5264,
    0.6619
   ],
   "resolved_by_run": {
    "run1": 60,
    "run2": 58
   },
   "cost_list_usd_total": 18.6454,
   "cost_per_task_list_usd": 0.0942,
   "cost_per_task_list_ci95": [
    0.0791,
    0.1129
   ],
   "cost_per_task_median_list_usd": 0.0731,
   "customer_paid_rule": "list price (Anthropic API)",
   "customer_multiplier": 1.0,
   "cost_customer_usd_total": 18.6454,
   "cost_per_task_customer_usd": 0.0942,
   "cost_per_solved_list_usd": 0.158,
   "cost_per_solved_list_ci95": [
    0.1232,
    0.2068
   ],
   "cost_per_solved_customer_usd": 0.158,
   "cache_read_share_of_prompt_tokens": 0.8954,
   "model_switches": 0,
   "sessions_with_model_switch": 0,
   "model_switches_per_session": 0.0,
   "steps_median": 9.0,
   "steps_mean": 10.2,
   "exit_statuses": {
    "Submitted": 198
   },
   "empty_patches": 0,
   "malformed_submissions": [
    "run1/schemathesis__schemathesis-3778",
    "run1/spack__spack-52244",
    "run2/astropy__astropy-19438",
    "run2/copier-org__copier-2646",
    "run2/pyinfra-dev__pyinfra-1679_interface"
   ],
   "served_model_requests": {
    "claude-sonnet-5.5": 2010
   },
   "served_model_cost_list_usd": {
    "claude-sonnet-5.5": 18.6454
   },
   "first_request_model": {
    "claude-sonnet-5.5": 198
   }
  },
  "coderouter_default": {
   "label": "CodeRouter default routing (model: auto)",
   "how_called": "model: auto through CodeRouter's production API, as any user gets it (the version live on 2026-11-09).",
   "sessions": 198,
   "resolved": 117,
   "solve_rate": 0.590909,
   "solve_rate_ci95_task_bootstrap": [
    0.5,
    0.681818
   ],
   "solve_rate_ci95_wilson": [
    0.5213,
    0.657
   ],
   "resolved_by_run": {
    "run1": 55,
    "run2": 62
   },
   "cost_list_usd_total": 22.6729,
   "cost_per_task_list_usd": 0.1145,
   "cost_per_task_list_ci95": [
    0.0888,
    0.152
   ],
   "cost_per_task_median_list_usd": 0.0781,
   "customer_paid_rule": "list price x 1.08 (CodeRouter pay as you go: model list price +8%)",
   "customer_multiplier": 1.08,
   "cost_customer_usd_total": 24.4867,
   "cost_per_task_customer_usd": 0.1237,
   "cost_per_solved_list_usd": 0.1938,
   "cost_per_solved_list_ci95": [
    0.1403,
    0.2745
   ],
   "cost_per_solved_customer_usd": 0.2093,
   "cache_read_share_of_prompt_tokens": 0.8899,
   "model_switches": 25,
   "sessions_with_model_switch": 25,
   "model_switches_per_session": 0.126,
   "steps_median": 8.0,
   "steps_mean": 10.5,
   "exit_statuses": {
    "Submitted": 198
   },
   "empty_patches": 0,
   "malformed_submissions": [
    "run1/astropy__astropy-19438",
    "run1/spack__spack-52244",
    "run1/vprusso__toqito-1538",
    "run2/copier-org__copier-2646",
    "run2/line__line-bot-sdk-python-981_interface"
   ],
   "served_model_requests": {
    "claude-sonnet-5.5": 1834,
    "claude-opus-5.5": 229,
    "claude-haiku-5.5": 8
   },
   "served_model_cost_list_usd": {
    "claude-sonnet-5.5": 16.7288,
    "claude-opus-5.5": 5.9413,
    "claude-haiku-5.5": 0.0029
   },
   "first_request_model": {
    "claude-sonnet-5.5": 196,
    "claude-haiku-5.5": 2
   }
  },
  "all_haiku": {
   "label": "All Claude Haiku 5.5",
   "how_called": "claude-haiku-5-5 named on every request; same pass-through path as all-Opus.",
   "sessions": 198,
   "resolved": 111,
   "solve_rate": 0.560606,
   "solve_rate_ci95_task_bootstrap": [
    0.469697,
    0.651515
   ],
   "solve_rate_ci95_wilson": [
    0.491,
    0.6279
   ],
   "resolved_by_run": {
    "run1": 56,
    "run2": 55
   },
   "cost_list_usd_total": 1.9261,
   "cost_per_task_list_usd": 0.0097,
   "cost_per_task_list_ci95": [
    0.0081,
    0.0116
   ],
   "cost_per_task_median_list_usd": 0.0067,
   "customer_paid_rule": "list price (Anthropic API)",
   "customer_multiplier": 1.0,
   "cost_customer_usd_total": 1.9261,
   "cost_per_task_customer_usd": 0.0097,
   "cost_per_solved_list_usd": 0.0174,
   "cost_per_solved_list_ci95": [
    0.0137,
    0.0222
   ],
   "cost_per_solved_customer_usd": 0.0174,
   "cache_read_share_of_prompt_tokens": 0.9242,
   "model_switches": 0,
   "sessions_with_model_switch": 0,
   "model_switches_per_session": 0.0,
   "steps_median": 12.0,
   "steps_mean": 15.5,
   "exit_statuses": {
    "Submitted": 196,
    "RepeatedFormatError": 2
   },
   "empty_patches": 8,
   "malformed_submissions": [],
   "served_model_requests": {
    "claude-haiku-5.5": 3060
   },
   "served_model_cost_list_usd": {
    "claude-haiku-5.5": 1.9261
   },
   "first_request_model": {
    "claude-haiku-5.5": 198
   }
  }
 },
 "paired": [
  {
   "a": "openrouter_auto_code",
   "b": "all_opus",
   "sessions": 198,
   "solved_a": 125,
   "solved_b": 130,
   "solve_rate_diff_a_minus_b": -0.025253,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.085859,
    0.035354
   ],
   "sessions_only_a_solved": 12,
   "sessions_only_b_solved": 17,
   "mcnemar_exact_p": 0.4583,
   "cost_ratio_list_a_over_b": 0.9976,
   "cost_ratio_list_ci95": [
    0.9058,
    1.1039
   ],
   "cost_ratio_customer_a_over_b": 1.0524,
   "cost_ratio_customer_ci95": [
    0.9556,
    1.1646
   ],
   "cost_per_solved_ratio_list_a_over_b": 1.0375,
   "cost_per_solved_ratio_list_ci95": [
    0.9148,
    1.1881
   ]
  },
  {
   "a": "all_sonnet",
   "b": "all_opus",
   "sessions": 198,
   "solved_a": 118,
   "solved_b": 130,
   "solve_rate_diff_a_minus_b": -0.060606,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.116162,
    -0.010101
   ],
   "sessions_only_a_solved": 4,
   "sessions_only_b_solved": 16,
   "mcnemar_exact_p": 0.0118,
   "cost_ratio_list_a_over_b": 0.3812,
   "cost_ratio_list_ci95": [
    0.3516,
    0.4113
   ],
   "cost_ratio_customer_a_over_b": 0.3812,
   "cost_ratio_customer_ci95": [
    0.3516,
    0.4113
   ],
   "cost_per_solved_ratio_list_a_over_b": 0.42,
   "cost_per_solved_ratio_list_ci95": [
    0.3737,
    0.4741
   ]
  },
  {
   "a": "coderouter_default",
   "b": "all_opus",
   "sessions": 198,
   "solved_a": 117,
   "solved_b": 130,
   "solve_rate_diff_a_minus_b": -0.065657,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.116162,
    -0.020202
   ],
   "sessions_only_a_solved": 5,
   "sessions_only_b_solved": 18,
   "mcnemar_exact_p": 0.0106,
   "cost_ratio_list_a_over_b": 0.4635,
   "cost_ratio_list_ci95": [
    0.3906,
    0.548
   ],
   "cost_ratio_customer_a_over_b": 0.5006,
   "cost_ratio_customer_ci95": [
    0.4219,
    0.5918
   ],
   "cost_per_solved_ratio_list_a_over_b": 0.515,
   "cost_per_solved_ratio_list_ci95": [
    0.4272,
    0.6193
   ]
  },
  {
   "a": "all_haiku",
   "b": "all_opus",
   "sessions": 198,
   "solved_a": 111,
   "solved_b": 130,
   "solve_rate_diff_a_minus_b": -0.09596,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.161616,
    -0.035354
   ],
   "sessions_only_a_solved": 7,
   "sessions_only_b_solved": 26,
   "mcnemar_exact_p": 0.0013,
   "cost_ratio_list_a_over_b": 0.0394,
   "cost_ratio_list_ci95": [
    0.0349,
    0.0452
   ],
   "cost_ratio_customer_a_over_b": 0.0394,
   "cost_ratio_customer_ci95": [
    0.0349,
    0.0452
   ],
   "cost_per_solved_ratio_list_a_over_b": 0.0461,
   "cost_per_solved_ratio_list_ci95": [
    0.0396,
    0.0546
   ]
  },
  {
   "a": "coderouter_default",
   "b": "openrouter_auto_code",
   "sessions": 198,
   "solved_a": 117,
   "solved_b": 125,
   "solve_rate_diff_a_minus_b": -0.040404,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.10101,
    0.020202
   ],
   "sessions_only_a_solved": 10,
   "sessions_only_b_solved": 18,
   "mcnemar_exact_p": 0.1849,
   "cost_ratio_list_a_over_b": 0.4647,
   "cost_ratio_list_ci95": [
    0.3853,
    0.5601
   ],
   "cost_ratio_customer_a_over_b": 0.4757,
   "cost_ratio_customer_ci95": [
    0.3944,
    0.5733
   ],
   "cost_per_solved_ratio_list_a_over_b": 0.4964,
   "cost_per_solved_ratio_list_ci95": [
    0.4009,
    0.6107
   ]
  },
  {
   "a": "coderouter_default",
   "b": "all_sonnet",
   "sessions": 198,
   "solved_a": 117,
   "solved_b": 118,
   "solve_rate_diff_a_minus_b": -0.005051,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.050505,
    0.040404
   ],
   "sessions_only_a_solved": 9,
   "sessions_only_b_solved": 10,
   "mcnemar_exact_p": 1.0,
   "cost_ratio_list_a_over_b": 1.216,
   "cost_ratio_list_ci95": [
    1.0569,
    1.3969
   ],
   "cost_ratio_customer_a_over_b": 1.3133,
   "cost_ratio_customer_ci95": [
    1.1414,
    1.5087
   ],
   "cost_per_solved_ratio_list_a_over_b": 1.2264,
   "cost_per_solved_ratio_list_ci95": [
    1.0454,
    1.4335
   ]
  },
  {
   "a": "openrouter_auto_code",
   "b": "all_sonnet",
   "sessions": 198,
   "solved_a": 125,
   "solved_b": 118,
   "solve_rate_diff_a_minus_b": 0.035354,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.020202,
    0.090909
   ],
   "sessions_only_a_solved": 16,
   "sessions_only_b_solved": 9,
   "mcnemar_exact_p": 0.2295,
   "cost_ratio_list_a_over_b": 2.6169,
   "cost_ratio_list_ci95": [
    2.3747,
    2.9058
   ],
   "cost_ratio_customer_a_over_b": 2.7609,
   "cost_ratio_customer_ci95": [
    2.5053,
    3.0656
   ],
   "cost_per_solved_ratio_list_a_over_b": 2.4704,
   "cost_per_solved_ratio_list_ci95": [
    2.1601,
    2.8419
   ]
  },
  {
   "a": "all_sonnet",
   "b": "all_haiku",
   "sessions": 198,
   "solved_a": 118,
   "solved_b": 111,
   "solve_rate_diff_a_minus_b": 0.035354,
   "solve_rate_diff_ci95_task_bootstrap": [
    -0.025253,
    0.10101
   ],
   "sessions_only_a_solved": 20,
   "sessions_only_b_solved": 13,
   "mcnemar_exact_p": 0.2962,
   "cost_ratio_list_a_over_b": 9.6804,
   "cost_ratio_list_ci95": [
    8.6,
    10.7748
   ],
   "cost_ratio_customer_a_over_b": 9.6804,
   "cost_ratio_customer_ci95": [
    8.6,
    10.7748
   ],
   "cost_per_solved_ratio_list_a_over_b": 9.1062,
   "cost_per_solved_ratio_list_ci95": [
    7.8113,
    10.5233
   ]
  }
 ],
 "coderouter_default_sessions_that_changed_model": {
  "sessions_that_changed_model": 25,
  "sessions_moved_up_to_opus": 24,
  "sessions_moved_haiku_to_sonnet": 1,
  "in_sessions_moved_up_to_opus": {
   "sessions": 24,
   "solved_by_coderouter": 9,
   "solved_by_all_sonnet_same_task_and_run": 12,
   "solved_by_all_opus_same_task_and_run": 13
  },
  "opus_cost_list_usd": 5.9413,
  "opus_share_of_setup_cost": 0.262
 },
 "run_to_run": {
  "all_opus": {
   "solved_run1": 64,
   "solved_run2": 66,
   "solved_both": 61,
   "solved_one_only": 8,
   "solved_neither": 30,
   "mcnemar_run1_vs_run2_p": 0.7266,
   "per_task_cost_ratio_run2_over_run1_median": 0.973
  },
  "openrouter_auto_code": {
   "solved_run1": 64,
   "solved_run2": 61,
   "solved_both": 59,
   "solved_one_only": 7,
   "solved_neither": 33,
   "mcnemar_run1_vs_run2_p": 0.4531,
   "per_task_cost_ratio_run2_over_run1_median": 0.958
  },
  "all_sonnet": {
   "solved_run1": 60,
   "solved_run2": 58,
   "solved_both": 55,
   "solved_one_only": 8,
   "solved_neither": 36,
   "mcnemar_run1_vs_run2_p": 0.7266,
   "per_task_cost_ratio_run2_over_run1_median": 1.021
  },
  "coderouter_default": {
   "solved_run1": 55,
   "solved_run2": 62,
   "solved_both": 53,
   "solved_one_only": 11,
   "solved_neither": 35,
   "mcnemar_run1_vs_run2_p": 0.0654,
   "per_task_cost_ratio_run2_over_run1_median": 0.964
  },
  "all_haiku": {
   "solved_run1": 56,
   "solved_run2": 55,
   "solved_both": 48,
   "solved_one_only": 15,
   "solved_neither": 36,
   "mcnemar_run1_vs_run2_p": 1.0,
   "per_task_cost_ratio_run2_over_run1_median": 0.932
  }
 },
 "checks": {
  "patches_identical_to_upstream_fix": {
   "all_opus": {
    "run1": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.1053
    },
    "run2": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.1053
    }
   },
   "openrouter_auto_code": {
    "run1": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.1194
    },
    "run2": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.1094
    }
   },
   "all_sonnet": {
    "run1": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.1
    },
    "run2": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.0909
    }
   },
   "coderouter_default": {
    "run1": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.08
    },
    "run2": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.095
    }
   },
   "all_haiku": {
    "run1": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.0952
    },
    "run2": {
     "n": 99,
     "identical_to_gold": 0,
     "median_jaccard_vs_gold": 0.1042
    }
   }
  },
  "upstream_fetch_attempts_manual_review": [
   {
    "setup": "all_opus",
    "run": "run1",
    "task": "psd-tools__psd-tools-593",
    "command": "pip download psd-tools==1.14.2",
    "blocked": true,
    "resolved": false
   },
   {
    "setup": "all_opus",
    "run": "run2",
    "task": "schemathesis__schemathesis-4087",
    "command": "pip download schemathesis==99",
    "blocked": true,
    "resolved": false
   },
   {
    "setup": "coderouter_default",
    "run": "run1",
    "task": "psd-tools__psd-tools-593",
    "command": "pip download psd-tools",
    "blocked": true,
    "resolved": false
   },
   {
    "setup": "coderouter_default",
    "run": "run2",
    "task": "psd-tools__psd-tools-593",
    "command": "pip download psd-tools",
    "blocked": true,
    "resolved": false
   },
   {
    "setup": "openrouter_auto_code",
    "run": "run1",
    "task": "agronholm__anyio-1134",
    "command": "pip download anyio==4.12.0",
    "blocked": true,
    "resolved": true
   },
   {
    "setup": "openrouter_auto_code",
    "run": "run2",
    "task": "oemof__tespy-921",
    "command": "urllib request for a source file on the project's GitHub dev branch",
    "blocked": true,
    "resolved": true
   },
   {
    "setup": "openrouter_auto_code",
    "run": "run2",
    "task": "pypa__twine-1309",
    "command": "pip download twine==6.3.0",
    "blocked": true,
    "resolved": false
   }
  ],
  "network_isolation": {
   "method": "Every packet from a container to outside the host was rejected and logged (Docker DOCKER-USER chain), for agent containers and grading containers alike. Verified before the run with a probe container: DNS, a GitHub IP on 443 and 1.1.1.1:53 were all unreachable. Images were pulled by the host's Docker daemon.",
   "packets_blocked": 306628,
   "packets_blocked_by_kind": {
    "probe": 18,
    "eval": 211299,
    "agent": 95311
   },
   "agent_packets_blocked_by_setup": {
    "all_haiku": 23118,
    "coderouter_default": 15761,
    "all_opus": 33287,
    "openrouter_auto_code": 14599,
    "all_sonnet": 8546
   },
   "tasks_with_a_blocked_agent_packet_by_setup_of_108": {
    "all_opus": 44,
    "all_sonnet": 32,
    "coderouter_default": 37,
    "openrouter_auto_code": 43,
    "all_haiku": 33
   },
   "destination_ports": {
    "53": 306552,
    "80": 60,
    "443": 16
   }
  },
  "sensitivity_dropped_tasks_regraded": {
   "what": "The 8 gradable dropped tasks re-graded with the tests that fail offline even with the upstream fix excluded, added to the 99: 107 tasks x 2 runs = 214 sessions per setup.",
   "excluded_tests_per_task": {
    "azure-samples__azure-search-openai-demo-3025": 0,
    "destine-climate-dt__aqua-2768": 1,
    "geopython__pygeoapi-2338": 3,
    "mempalace__mempalace-1004": 9,
    "olofk__fusesoc-776_interface": 1,
    "pdm-project__pdm-3759": 1,
    "pgmpy__pgmpy-3137": 6,
    "reframe-hpc__reframe-3660_interface": 6,
    "zauberzeug__nicegui-5914": 1
   },
   "setups": {
    "all_opus": {
     "resolved": 142,
     "sessions": 214,
     "solve_rate": 0.6636,
     "wilson95": [
      0.5979,
      0.7235
     ],
     "resolved_on_regraded_tasks": 12
    },
    "openrouter_auto_code": {
     "resolved": 137,
     "sessions": 214,
     "solve_rate": 0.6402,
     "wilson95": [
      0.5739,
      0.7015
     ],
     "resolved_on_regraded_tasks": 12
    },
    "all_sonnet": {
     "resolved": 128,
     "sessions": 214,
     "solve_rate": 0.5981,
     "wilson95": [
      0.5313,
      0.6615
     ],
     "resolved_on_regraded_tasks": 10
    },
    "coderouter_default": {
     "resolved": 128,
     "sessions": 214,
     "solve_rate": 0.5981,
     "wilson95": [
      0.5313,
      0.6615
     ],
     "resolved_on_regraded_tasks": 11
    },
    "all_haiku": {
     "resolved": 119,
     "sessions": 214,
     "solve_rate": 0.5561,
     "wilson95": [
      0.4891,
      0.6211
     ],
     "resolved_on_regraded_tasks": 8
    }
   }
  },
  "grading_errors": 0
 },
 "limitations": [
  "One agent harness (mini-swe-agent, bash tool only). Claude Code, Codex or Cursor sessions differ: other tools, prompts and context handling.",
  "99 Python repository tasks (SWE-rebench, created March to May 2026). Claude Opus 5.5's published knowledge cutoff (June 2026) is later than the task dates, so we can't prove no model saw the fixes; no submitted patch was identical to the upstream fix.",
  "2 runs per task. One setup's completion rate has a 95% interval of about ±7 (Wilson) to ±9 points (task bootstrap); paired differences between two setups have intervals of about ±5 to ±6 points, so smaller differences can't be told apart from noise.",
  "9 of the 108 tasks we ran were dropped because their upstream fix doesn't pass offline (8 need network for some tests, 1 flaky). Re-grading 8 of them without the network-dependent tests doesn't change the order.",
  "List prices on 2026-11-09. Prices, models and routers change; results may change with model updates.",
  "OpenRouter Auto Code picks models per request and its routes change over time; this is one day's behavior.",
  "Limits: 150 steps, $2.50 and 60 minutes per session; max_tokens 8,192."
 ]
}
