{
  "name": "CodeRouter 176-task coding benchmark — aggregates, November 2026",
  "measured_by": "TokenWatt, a measurement project from the same team as CodeRouter (not an independent benchmark)",
  "page": "https://www.coderouter.io/benchmark",
  "updated": "2026-11-05",
  "contents": "Aggregate results only. No per-task data.",
  "summary": "96–100% of tasks solved for 56–70% less than sending every request to Opus, across runs.",
  "method": {
    "tasks": 176,
    "task_sources": ["Exercism (Python)", "Aider polyglot benchmark, hard set (Python)"],
    "task_shape": "Single-file Python tasks. The model sees the instructions and the stub file and returns the whole file.",
    "scoring": "The task's tests are run. On failure, one retry with the failing test output. 'First try' = tests pass without the retry; 'with one retry' = tests pass after at most one retry.",
    "pricing": "Provider list prices as checked 2026-11-05 (DeepSeek at its peak-hour price; GLM-5.3 at Zhipu's open.bigmodel.cn price, converted at 6.5 CNY per USD), applied to each request's logged tokens. Costs include the retry.",
    "baseline": "Every request sent to Claude Opus 5.5."
  },
  "baseline_all_opus": {
    "model": "Claude Opus 5.5",
    "tasks": 176,
    "solved_with_retry": 176,
    "cost_usd": 2.56
  },
  "router_runs": [
    {
      "router": "Earlier router (before routing v3)",
      "period": "2026-10",
      "runs": 3,
      "solved_pct_min": 96.0,
      "solved_pct_max": 97.7,
      "cost_pct_of_all_opus_min": 41,
      "cost_pct_of_all_opus_max": 44,
      "savings_vs_all_opus_pct_min": 56,
      "savings_vs_all_opus_pct_max": 59
    },
    {
      "router": "Routing v3",
      "date": "2026-11-04",
      "runs": 1,
      "tasks": 176,
      "solved_with_retry": 176,
      "solved_pct": 100,
      "first_try_pct": 88.6,
      "cost_usd": 0.77,
      "cost_pct_of_all_opus": 30,
      "savings_vs_all_opus_pct": 70
    }
  ],
  "model_matrix": {
    "date": "2026-11-03",
    "how": "Each model called directly through CodeRouter's production path on the same tasks, one run per model.",
    "tasks_compared": 175,
    "note": "175 tasks are used for the matrix comparison because one Kimi K3.1 Pro request errored.",
    "models": [
      { "model": "Claude Opus 5.5", "first_try_pct": 98.3, "with_one_retry_pct": 100, "total_cost_usd": 2.61 },
      { "model": "Claude Sonnet 5.5", "first_try_pct": 93.7, "with_one_retry_pct": 100, "total_cost_usd": 1.48 },
      { "model": "Kimi K2.6", "first_try_pct": 86.9, "with_one_retry_pct": 94.9, "total_cost_usd": 0.57 },
      { "model": "Claude Haiku 5.5", "first_try_pct": 72.0, "with_one_retry_pct": 93.7, "total_cost_usd": 0.15 },
      { "model": "DeepSeek Chat", "first_try_pct": 70.3, "with_one_retry_pct": 80.6, "total_cost_usd": 0.11 },
      { "model": "DeepSeek V4 Pro", "first_try_pct": 68.6, "with_one_retry_pct": 78.9, "total_cost_usd": 0.36 },
      { "model": "Kimi K3.1 Pro", "first_try_pct": 67.4, "with_one_retry_pct": 90.9, "total_cost_usd": 1.67 },
      { "model": "GLM-5.3", "first_try_pct": 62.3, "with_one_retry_pct": 89.1, "total_cost_usd": 0.99 }
    ]
  },
  "strategy_comparison": {
    "tasks_compared": 175,
    "strategies": [
      {
        "strategy": "Cheapest-first cascade",
        "description": "DeepSeek Chat, then Kimi K2.6, then Claude Sonnet 5.5; escalate on test failure.",
        "solved_pct": 97.1,
        "cost_usd": 0.49
      },
      {
        "strategy": "Oracle",
        "description": "The cheapest model that passes each task, chosen after the fact. A lower bound, not something a live router can do.",
        "solved_pct": 99.4,
        "cost_usd": 0.29
      }
    ]
  },
  "limitations": [
    "Single-file Python tasks only.",
    "Public tasks: models may have seen them during training.",
    "One run per model in the matrix, and one run of routing v3.",
    "Single-turn, short contexts with no prompt-cache cost — unlike long coding-agent sessions.",
    "Measured by TokenWatt, which is run by the same team as CodeRouter."
  ],
  "team_usage_2026_05_09": {
    "what": "Real per-developer usage on CodeRouter's own team (not customers): one API key per engineer, May–September 2026.",
    "developer_days": 385,
    "inclusion": "Active developer-days with at least 20 requests.",
    "basis": "Spend, not results: each request at its model's list price (official provider pages, checked 2026-11-05) — uncached input, cache writes, cache reads and output. Claude cache writes were not logged before 2026-11-02; for May–September they are estimated at 6.6% of the Claude prompt (input + cache reads + cache writes), the share measured on our team's November 2026 traffic. The *_without_write_estimate figures leave them out.",
    "per_developer_per_active_day": {
      "requests": { "median": 163, "p75": 378 },
      "routed_cost_usd": { "median": 4.30, "p75": 13.03 },
      "routed_cost_usd_without_write_estimate": { "median": 4.17, "p75": 11.52 },
      "note": "Medians and 75th percentiles are computed separately for each row."
    },
    "period_totals_usd": {
      "routed": 5581,
      "routed_without_write_estimate": 4574
    },
    "per_month_20_working_days": {
      "typical_developer_median": { "requests": 3300, "routed_cost_usd": 86 },
      "heavy_developer_p75": { "requests": 7600, "routed_cost_usd": 261 }
    },
    "cache_read_share_of_input_tokens_pct": 79,
    "estimated_claude_cache_write_share_pct": 6.6,
    "corrected": {
      "date": "2026-11-07",
      "previously_published": {
        "per_active_day_routed_cost_usd": { "median": 4.34, "p75": 17.01 },
        "period_total_routed_usd": 8043,
        "per_month_routed_cost_usd": { "typical": 87, "heavy": 340 },
        "cache_read_share_of_input_tokens_pct": 53
      },
      "why": [
        "The earlier figures were CodeRouter's recorded costs, not list prices. Until 2026-10-29 CodeRouter's model table priced Claude Opus 4.7 at $15 / $75 per million input / output tokens (list: $5 / $25), so its requests were recorded at 3× list; other models were recorded at the table's prices of the day.",
        "The earlier cache-read share counted cached tokens twice on providers whose input count already includes them.",
        "Removed: the same requests priced at Claude Opus 5.5 and Sonnet 5.5 list prices. A different model takes a different number of steps to finish the same task, so repricing these tokens on one model doesn't estimate what that setup would cost. For model-against-model cost see router_runs and model_matrix: the same tasks run on each setup."
      ]
    },
    "caveats": [
      "This is our own team's usage, not customers'.",
      "It shows what was spent, not what got solved: the answers weren't graded.",
      "May–September Claude cache writes are an estimate (see basis); without it the figures are lower."
    ]
  }
}