{
  "updated": "2026-10-21",
  "benchmarks": [
    {
      "id": "swe-bench-verified",
      "name": "SWE-bench Verified",
      "metric": "Resolved %",
      "url": "https://www.swebench.com/verified.html",
      "rows": [
        {
          "agent": "Claude Code (Claude Sonnet 5.5)",
          "score": 72.8,
          "tokensPerTask": null,
          "costPerTaskUsd": 0.85,
          "sourceUrl": "https://www.trustthebench.com/agents"
        },
        {
          "agent": "OpenHands + critic model",
          "score": 72,
          "tokensPerTask": null,
          "costPerTaskUsd": null,
          "sourceUrl": "https://raw.githubusercontent.com/MohamedAbdallah-14/Wazir/refs/heads/main/docs/research/agent-architecture/swe-bench.md"
        },
        {
          "agent": "Devin (cloud agent)",
          "score": 70.4,
          "tokensPerTask": null,
          "costPerTaskUsd": 1.4,
          "sourceUrl": "https://www.trustthebench.com/agents"
        },
        {
          "agent": "Cursor Agent (Composer 2.5)",
          "score": 69.2,
          "tokensPerTask": null,
          "costPerTaskUsd": 0.68,
          "sourceUrl": "https://www.trustthebench.com/agents"
        },
        {
          "agent": "Aider (open source CLI)",
          "score": 67.5,
          "tokensPerTask": null,
          "costPerTaskUsd": 0.48,
          "sourceUrl": "https://www.trustthebench.com/agents"
        },
        {
          "agent": "Cline (VS Code, BYOK)",
          "score": 66.8,
          "tokensPerTask": null,
          "costPerTaskUsd": 0.42,
          "sourceUrl": "https://www.trustthebench.com/agents"
        },
        {
          "agent": "GitHub Copilot coding agent",
          "score": 65,
          "tokensPerTask": null,
          "costPerTaskUsd": 0.75,
          "sourceUrl": "https://www.trustthebench.com/agents"
        },
        {
          "agent": "AutoCodeRover (AST agent)",
          "score": 46.2,
          "tokensPerTask": null,
          "costPerTaskUsd": 0.7,
          "sourceUrl": "https://raw.githubusercontent.com/MohamedAbdallah-14/Wazir/refs/heads/main/docs/research/agent-architecture/swe-bench.md"
        }
      ]
    },
    {
      "id": "terminal-bench-3",
      "name": "Terminal-Bench 3.0",
      "metric": "Reward %",
      "url": "https://bench.turing.com/terminal-bench-3/leaderboard/",
      "rows": [
        {
          "agent": "GPT-5.6 Sol via codex (max)",
          "score": 56.6,
          "tokensPerTask": 577000,
          "costPerTaskUsd": 0.99,
          "sourceUrl": "https://bench.turing.com/terminal-bench-3/leaderboard/"
        },
        {
          "agent": "Claude Fable 5 via claude-code (high)",
          "score": 47.3,
          "tokensPerTask": 455000,
          "costPerTaskUsd": 1.59,
          "sourceUrl": "https://bench.turing.com/terminal-bench-3/leaderboard/"
        },
        {
          "agent": "Grok-4.5 via terminus-2 (high)",
          "score": 44,
          "tokensPerTask": 555000,
          "costPerTaskUsd": 0.48,
          "sourceUrl": "https://bench.turing.com/terminal-bench-3/leaderboard/"
        },
        {
          "agent": "GLM-5.2 via terminus-2 (high)",
          "score": 41.5,
          "tokensPerTask": 251000,
          "costPerTaskUsd": 0.21,
          "sourceUrl": "https://bench.turing.com/terminal-bench-3/leaderboard/"
        },
        {
          "agent": "Kimi K3 via terminus-2 (thinking)",
          "score": 39.8,
          "tokensPerTask": 126000,
          "costPerTaskUsd": 0.35,
          "sourceUrl": "https://bench.turing.com/terminal-bench-3/leaderboard/"
        },
        {
          "agent": "Claude Opus 4.8 via terminus-2 (xhigh)",
          "score": 36.6,
          "tokensPerTask": 870000,
          "costPerTaskUsd": 5.54,
          "sourceUrl": "https://bench.turing.com/terminal-bench-3/leaderboard/"
        },
        {
          "agent": "Kimi K2.7 via terminus-2 (max)",
          "score": 27.6,
          "tokensPerTask": 575000,
          "costPerTaskUsd": 0.23,
          "sourceUrl": "https://bench.turing.com/terminal-bench-3/leaderboard/"
        }
      ]
    },
    {
      "id": "coding-agent-index",
      "name": "Artificial Analysis Coding Agent Index v1.5",
      "metric": "Index score (0-100)",
      "url": "https://artificialanalysis.ai/agents/coding-agents",
      "rows": [
        {
          "agent": "Claude Code — Sonnet 5.5 (max)",
          "score": 68,
          "tokensPerTask": 27700000,
          "costPerTaskUsd": 14.19,
          "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex"
        },
        {
          "agent": "Claude Code — Opus 5.5 (max)",
          "score": 66,
          "tokensPerTask": 15600000,
          "costPerTaskUsd": 13.04,
          "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex"
        },
        {
          "agent": "Codex — GPT-6.1 Sol (xhigh)",
          "score": 63,
          "tokensPerTask": 3200000,
          "costPerTaskUsd": 1.04,
          "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex"
        },
        {
          "agent": "Claude Code — Sonnet 5.5 (xhigh)",
          "score": 63,
          "tokensPerTask": 7100000,
          "costPerTaskUsd": 3.33,
          "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex"
        },
        {
          "agent": "Claude Code — Fable 5.1 (max)",
          "score": 62,
          "tokensPerTask": 5700000,
          "costPerTaskUsd": 12.39,
          "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex"
        },
        {
          "agent": "Cursor — Composer 2.5 (fast)",
          "score": 62,
          "tokensPerTask": null,
          "costPerTaskUsd": 0.44,
          "sourceUrl": "https://artificialanalysis.ai/de/articles/cursor-composer-2-5-coding-agent-index"
        },
        {
          "agent": "Claude Code — Sonnet 5.5 (high)",
          "score": 55,
          "tokensPerTask": 2700000,
          "costPerTaskUsd": 1.24,
          "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/comparisons/claude-code-vs-codex"
        },
        {
          "agent": "Cursor — Composer 2.5 (standard)",
          "score": 62,
          "tokensPerTask": null,
          "costPerTaskUsd": 0.07,
          "sourceUrl": "https://artificialanalysis.ai/de/articles/cursor-composer-2-5-coding-agent-index"
        }
      ]
    }
  ],
  "frameworkOverhead": [
    {
      "framework": "AutoGen",
      "overheadPct": 0,
      "note": "Leanest baseline in a 450-run study: 379 tokens per tool-use run on GPT-4o",
      "sourceUrl": "https://github.com/waleedabujaish/agentic-framework-bench/blob/main/RESULTS.md"
    },
    {
      "framework": "LangGraph",
      "overheadPct": 11,
      "note": "422 tokens per run vs AutoGen's 379 (+11%); same tasks, temperature 0",
      "sourceUrl": "https://github.com/waleedabujaish/agentic-framework-bench/blob/main/RESULTS.md"
    },
    {
      "framework": "OpenAI Agents SDK",
      "overheadPct": 25,
      "note": "472 tokens per run vs AutoGen's 379 (+25%); role/tool scaffolding adds context",
      "sourceUrl": "https://github.com/waleedabujaish/agentic-framework-bench/blob/main/RESULTS.md"
    },
    {
      "framework": "OpenAI Agents SDK + MCP tools",
      "overheadPct": 25,
      "note": "MCP tool routing adds no measurable tokens once the server connection is reused",
      "sourceUrl": "https://github.com/waleedabujaish/agentic-framework-bench/blob/main/RESULTS.md"
    },
    {
      "framework": "CrewAI",
      "overheadPct": 1056,
      "note": "4,382 tokens per run vs AutoGen's 379 (about 11.6x); role/goal/backstory prompts dominate",
      "sourceUrl": "https://github.com/waleedabujaish/agentic-framework-bench/blob/main/RESULTS.md"
    }
  ]
}