{
 "name": "Claude Code on open models: 30 SWE-rebench repository tasks, 4 models, 1 run each",
 "measured_by": "CodeRouter (we build it; TokenWatt, the same team, ran the harness)",
 "date": "2026-11-12",
 "agent": "Claude Code 2.1.282, headless (claude -p), run as shipped: default system prompt and tools, thinking at its default; WebSearch and WebFetch disabled; --max-turns 150; 60-minute limit per task; $3 limit per task",
 "models": "Open models through CodeRouter with the model named explicitly (no routing, no fallback); Claude Sonnet 5.5 through the Anthropic API directly. Every model variable (main, Opus, Sonnet, Haiku, subagent) set to the model under test.",
 "tasks": "30 tasks drawn at random (seed 20261110) from the 99 SWE-rebench tasks of our coding-agent benchmark: Python pull requests created 2026-03-01..05-12 (nebius/SWE-rebench-leaderboard, CC BY 4.0). Gold patches 30/30 and empty patches 0/30 on the grading machine.",
 "isolation": "Every container had no network except a relay to the model proxy; blocked attempts were logged (DNS lookups only, none reached the internet).",
 "grading": "Official swebench 5.0.2 harness; resolved = the task's failing tests pass and passing tests still pass.",
 "cost": "List price x the tokens each request reported, cache reads and writes included (DeepSeek at its peak price). Open models: CodeRouter's model list on 2026-11-12, the provider's list price, no markup. Claude Sonnet 5.5: Anthropic list price.",
 "rerun_note": "Zhipu's API returned an out-of-balance error from 17:43 PT on our account, cutting 15 GLM-5.3 sessions short; those 15 tasks were rerun from scratch after the top-up, and only the reruns are counted.",
 "limitations": [
  "One run per model; 30 tasks, so intervals are wide.",
  "Claude Code's defaults favour Claude models: its prompts and tools are tuned for them.",
  "Kimi K2.6 hit the 60-minute limit on many tasks; a longer limit would change its numbers.",
  "Python repositories only: bug fixes and small features from real pull requests."
 ],
 "summary": {
  "Claude Sonnet 5.5": {
   "model_id": "claude-sonnet-5-5",
   "tasks": 30,
   "resolved": 16,
   "resolved_pct": 53.3,
   "resolved_ci95": [
    36.1,
    69.8
   ],
   "cost_usd_list": 4.36,
   "cost_per_task_usd": 0.145,
   "cost_per_resolved_usd": 0.272,
   "median_model_requests": 8.0,
   "median_turns_finished_sessions": 8.5,
   "median_wall_min": 1.0,
   "output_tokens_per_task_median": 3579,
   "cache_read_share_of_input_pct": 93.3,
   "ended": {
    "finished": 30,
    "turn_limit": 0,
    "time_limit": 0,
    "error": 0
   },
   "empty_patches": 0
  },
  "DeepSeek V4.1 Flash": {
   "model_id": "deepseek-chat",
   "tasks": 30,
   "resolved": 9,
   "resolved_pct": 30.0,
   "resolved_ci95": [
    16.7,
    47.9
   ],
   "cost_usd_list": 0.98,
   "cost_per_task_usd": 0.033,
   "cost_per_resolved_usd": 0.109,
   "median_model_requests": 25.0,
   "median_turns_finished_sessions": 26.0,
   "median_wall_min": 4.0,
   "output_tokens_per_task_median": 4993,
   "cache_read_share_of_input_pct": 98.4,
   "ended": {
    "finished": 27,
    "turn_limit": 3,
    "time_limit": 0,
    "error": 0
   },
   "empty_patches": 0,
   "vs_sonnet": {
    "only_this_solved": 1,
    "only_sonnet_solved": 8,
    "mcnemar_exact_p": 0.0391
   }
  },
  "Kimi K2.6": {
   "model_id": "kimi-k2.6",
   "tasks": 30,
   "resolved": 9,
   "resolved_pct": 30.0,
   "resolved_ci95": [
    16.7,
    47.9
   ],
   "cost_usd_list": 21.09,
   "cost_per_task_usd": 0.703,
   "cost_per_resolved_usd": 2.343,
   "median_model_requests": 51.5,
   "median_turns_finished_sessions": 22,
   "median_wall_min": 54.4,
   "output_tokens_per_task_median": 59847,
   "cache_read_share_of_input_pct": 94.6,
   "ended": {
    "finished": 15,
    "turn_limit": 0,
    "time_limit": 15,
    "error": 0
   },
   "empty_patches": 7,
   "vs_sonnet": {
    "only_this_solved": 2,
    "only_sonnet_solved": 9,
    "mcnemar_exact_p": 0.0654
   }
  },
  "GLM-5.3": {
   "model_id": "glm-5.3",
   "tasks": 30,
   "resolved": 14,
   "resolved_pct": 46.7,
   "resolved_ci95": [
    30.2,
    63.9
   ],
   "cost_usd_list": 17.5,
   "cost_per_task_usd": 0.583,
   "cost_per_resolved_usd": 1.25,
   "median_model_requests": 31.5,
   "median_turns_finished_sessions": 32.5,
   "median_wall_min": 4.5,
   "output_tokens_per_task_median": 8368,
   "cache_read_share_of_input_pct": 97.1,
   "ended": {
    "finished": 29,
    "turn_limit": 1,
    "time_limit": 0,
    "error": 0
   },
   "empty_patches": 0,
   "vs_sonnet": {
    "only_this_solved": 1,
    "only_sonnet_solved": 3,
    "mcnemar_exact_p": 0.625
   }
  }
 },
 "requests_refused_by_cost_limit": 1
}