{ "summary": "Published exact-task results: GPT-6 Astra (xhigh) and Claude Opus 5 (max). Opus 5.5 task-level results were not found.", "note": "Official mini-swe-agent runs; our runs use OpenCode. Counts are completed passes across published attempts, not matched-harness rankings.", "records": [ { "task": "ofetch-per-origin-circuit-breaker", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/ofetch-per-origin-circuit-breaker", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "ofetch-per-origin-circuit-breake__RY6v4PG", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 47, "f2p_total": 47, "p2p_passed": 13, "p2p_total": 13, "agent_duration_seconds": 1161.582489, "n_output_tokens": 87879, "n_agent_steps": 48, "cost_usd": 5.522882250000001 }, { "trial_name": "ofetch-per-origin-circuit-breake__YpCCesS", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 47, "f2p_total": 47, "p2p_passed": 13, "p2p_total": 13, "agent_duration_seconds": 884.662459, "n_output_tokens": 72684, "n_agent_steps": 42, "cost_usd": 4.3263039999999995 }, { "trial_name": "ofetch-per-origin-circuit-breake__foKa4G8", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 47, "f2p_total": 47, "p2p_passed": 13, "p2p_total": 13, "agent_duration_seconds": 952.312546, "n_output_tokens": 78246, "n_agent_steps": 42, "cost_usd": 4.614413000000001 }, { "trial_name": "ofetch-per-origin-circuit-breake__2YPBe7g", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 47, "f2p_total": 47, "p2p_passed": 13, "p2p_total": 13, "agent_duration_seconds": 1005.424478, "n_output_tokens": 81732, "n_agent_steps": 43, "cost_usd": 4.7747382499999995 } ] }, { "task": "ofetch-per-origin-circuit-breaker", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/ofetch-per-origin-circuit-breaker", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "ofetch-per-origin-circuit-breake__unMetbB", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 568, "n_output_tokens": 19755, "n_agent_steps": 16, "cost_usd": 2.603392 }, { "trial_name": "ofetch-per-origin-circuit-breake__5bJdxGA", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 834, "n_output_tokens": 25904, "n_agent_steps": 17, "cost_usd": 2.475287 }, { "trial_name": "ofetch-per-origin-circuit-breake__eNRm4qj", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 692, "n_output_tokens": 25412, "n_agent_steps": 19, "cost_usd": 2.710117 }, { "trial_name": "ofetch-per-origin-circuit-breake__MzKXbUr", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 647, "n_output_tokens": 23550, "n_agent_steps": 15, "cost_usd": 2.3549325 } ] }, { "task": "happy-dom-abort-pending-body-reads", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/happy-dom-abort-pending-body-reads", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "happy-dom-abort-pending-body-rea__2qtdxuG", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 14, "f2p_total": 14, "p2p_passed": 165, "p2p_total": 165, "agent_duration_seconds": 2708.480873, "n_output_tokens": 116032, "n_agent_steps": 173, "cost_usd": 23.138919250000008 }, { "trial_name": "happy-dom-abort-pending-body-rea__Ktpc6uE", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 14, "f2p_total": 14, "p2p_passed": 165, "p2p_total": 165, "agent_duration_seconds": 3231.783231, "n_output_tokens": 138708, "n_agent_steps": 204, "cost_usd": 28.34598625 }, { "trial_name": "happy-dom-abort-pending-body-rea__SZAVQG2", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 14, "f2p_total": 14, "p2p_passed": 165, "p2p_total": 165, "agent_duration_seconds": 2603.846341, "n_output_tokens": 99156, "n_agent_steps": 169, "cost_usd": 20.175400749999987 }, { "trial_name": "happy-dom-abort-pending-body-rea__FThBYq8", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 14, "f2p_total": 14, "p2p_passed": 165, "p2p_total": 165, "agent_duration_seconds": 2468.890389, "n_output_tokens": 95524, "n_agent_steps": 128, "cost_usd": 15.088696500000001 } ] }, { "task": "happy-dom-abort-pending-body-reads", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/happy-dom-abort-pending-body-reads", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "happy-dom-abort-pending-body-rea__okk4Zc2", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1255, "n_output_tokens": 35572, "n_agent_steps": 35, "cost_usd": 5.979226 }, { "trial_name": "happy-dom-abort-pending-body-rea__9vSDRyr", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1180, "n_output_tokens": 30069, "n_agent_steps": 39, "cost_usd": 6.519934 }, { "trial_name": "happy-dom-abort-pending-body-rea__FNuAzQ5", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1031, "n_output_tokens": 28163, "n_agent_steps": 37, "cost_usd": 6.3795495 }, { "trial_name": "happy-dom-abort-pending-body-rea__Hvinrxe", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1220, "n_output_tokens": 33468, "n_agent_steps": 34, "cost_usd": 7.5196905 } ] }, { "task": "httpx-streaming-json-iteration", "model": "claude-opus-5", "effort": "max", "passed": 0, "attempts": 4, "result": "0/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/httpx-streaming-json-iteration", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "httpx-streaming-json-iteration__yMurRqN", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 107, "f2p_total": 108, "p2p_passed": 1404, "p2p_total": 1404, "agent_duration_seconds": 1335.339138, "n_output_tokens": 93296, "n_agent_steps": 67, "cost_usd": 6.698524500000001 }, { "trial_name": "httpx-streaming-json-iteration__AuQTSHb", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 107, "f2p_total": 108, "p2p_passed": 1404, "p2p_total": 1404, "agent_duration_seconds": 1491.422436, "n_output_tokens": 98738, "n_agent_steps": 67, "cost_usd": 7.1356927500000005 }, { "trial_name": "httpx-streaming-json-iteration__DGAzSUL", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 107, "f2p_total": 108, "p2p_passed": 1404, "p2p_total": 1404, "agent_duration_seconds": 1510.887046, "n_output_tokens": 92517, "n_agent_steps": 76, "cost_usd": 7.231788 }, { "trial_name": "httpx-streaming-json-iteration__dXYxvQ9", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 107, "f2p_total": 108, "p2p_passed": 1404, "p2p_total": 1404, "agent_duration_seconds": 1335.706, "n_output_tokens": 90164, "n_agent_steps": 66, "cost_usd": 6.450494 } ] }, { "task": "httpx-streaming-json-iteration", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/httpx-streaming-json-iteration", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "httpx-streaming-json-iteration__p6dMe8k", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 2973, "n_output_tokens": 34782, "n_agent_steps": 26, "cost_usd": 3.8359495 }, { "trial_name": "httpx-streaming-json-iteration__6Dum86i", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1110, "n_output_tokens": 27860, "n_agent_steps": 24, "cost_usd": 3.6246775 }, { "trial_name": "httpx-streaming-json-iteration__YwhVXta", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1025, "n_output_tokens": 33595, "n_agent_steps": 29, "cost_usd": 4.6301035 }, { "trial_name": "httpx-streaming-json-iteration__YMh4tYz", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1100, "n_output_tokens": 27376, "n_agent_steps": 22, "cost_usd": 2.8672445 } ] }, { "task": "fd-deterministic-multi-key-sorting", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/fd-deterministic-multi-key-sorting", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "fd-deterministic-multi-key-sorti__iERyM5Z", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 43, "f2p_total": 43, "p2p_passed": 109, "p2p_total": 109, "agent_duration_seconds": 1373.322362, "n_output_tokens": 84197, "n_agent_steps": 89, "cost_usd": 8.111945500000001 }, { "trial_name": "fd-deterministic-multi-key-sorti__rytfZYh", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 43, "f2p_total": 43, "p2p_passed": 109, "p2p_total": 109, "agent_duration_seconds": 1073.985383, "n_output_tokens": 69573, "n_agent_steps": 82, "cost_usd": 7.357061749999998 }, { "trial_name": "fd-deterministic-multi-key-sorti__WGCxVZs", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 43, "f2p_total": 43, "p2p_passed": 109, "p2p_total": 109, "agent_duration_seconds": 1577.514978, "n_output_tokens": 88197, "n_agent_steps": 90, "cost_usd": 8.688030500000002 }, { "trial_name": "fd-deterministic-multi-key-sorti__R6WQfun", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 43, "f2p_total": 43, "p2p_passed": 109, "p2p_total": 109, "agent_duration_seconds": 1453.924083, "n_output_tokens": 86455, "n_agent_steps": 101, "cost_usd": 9.458184999999999 } ] }, { "task": "fd-deterministic-multi-key-sorting", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/fd-deterministic-multi-key-sorting", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "fd-deterministic-multi-key-sorti__HSHnAYm", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 804, "n_output_tokens": 23035, "n_agent_steps": 27, "cost_usd": 3.412312 }, { "trial_name": "fd-deterministic-multi-key-sorti__2qVDwzw", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 628, "n_output_tokens": 18582, "n_agent_steps": 24, "cost_usd": 3.232165 }, { "trial_name": "fd-deterministic-multi-key-sorti__wst8Dzk", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1077, "n_output_tokens": 21401, "n_agent_steps": 19, "cost_usd": 2.459779 }, { "trial_name": "fd-deterministic-multi-key-sorti__ycakNQZ", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 788, "n_output_tokens": 27171, "n_agent_steps": 24, "cost_usd": 4.2423195 } ] }, { "task": "aiomonitor-task-snapshots-diff", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/aiomonitor-task-snapshots-diff", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "aiomonitor-task-snapshots-diff__N2bniJA", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 53, "f2p_total": 53, "p2p_passed": 8, "p2p_total": 8, "agent_duration_seconds": 1285.875558, "n_output_tokens": 110775, "n_agent_steps": 79, "cost_usd": 9.35192775 }, { "trial_name": "aiomonitor-task-snapshots-diff__sPzNJE5", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 53, "f2p_total": 53, "p2p_passed": 8, "p2p_total": 8, "agent_duration_seconds": 1426.199757, "n_output_tokens": 107091, "n_agent_steps": 81, "cost_usd": 9.675126 }, { "trial_name": "aiomonitor-task-snapshots-diff__dQypxvV", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 53, "f2p_total": 53, "p2p_passed": 8, "p2p_total": 8, "agent_duration_seconds": 1218.579518, "n_output_tokens": 94018, "n_agent_steps": 70, "cost_usd": 7.797832999999997 }, { "trial_name": "aiomonitor-task-snapshots-diff__BwG65bg", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 53, "f2p_total": 53, "p2p_passed": 8, "p2p_total": 8, "agent_duration_seconds": 1250.966493, "n_output_tokens": 103588, "n_agent_steps": 84, "cost_usd": 9.51255125 } ] }, { "task": "aiomonitor-task-snapshots-diff", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/aiomonitor-task-snapshots-diff", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "aiomonitor-task-snapshots-diff__nhZXPnv", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 798, "n_output_tokens": 26130, "n_agent_steps": 23, "cost_usd": 3.693057 }, { "trial_name": "aiomonitor-task-snapshots-diff__bQo92za", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 861, "n_output_tokens": 27051, "n_agent_steps": 24, "cost_usd": 2.986787 }, { "trial_name": "aiomonitor-task-snapshots-diff__fqmdpyR", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 649, "n_output_tokens": 24819, "n_agent_steps": 20, "cost_usd": 2.654592 }, { "trial_name": "aiomonitor-task-snapshots-diff__8RMYkQj", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 806, "n_output_tokens": 25303, "n_agent_steps": 19, "cost_usd": 3.387016 } ] }, { "task": "sqlite-utils-safe-import-checkpoints", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/sqlite-utils-safe-import-checkpoints", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "sqlite-utils-safe-import-checkpo__pRsBvGa", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 60, "f2p_total": 60, "p2p_passed": 1038, "p2p_total": 1038, "agent_duration_seconds": 1889.934143, "n_output_tokens": 151328, "n_agent_steps": 88, "cost_usd": 11.5269825 }, { "trial_name": "sqlite-utils-safe-import-checkpo__y4zR25A", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 60, "f2p_total": 60, "p2p_passed": 1038, "p2p_total": 1038, "agent_duration_seconds": 1937.243343, "n_output_tokens": 153287, "n_agent_steps": 106, "cost_usd": 13.00683774999999 }, { "trial_name": "sqlite-utils-safe-import-checkpo__v2ErSsT", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 60, "f2p_total": 60, "p2p_passed": 1038, "p2p_total": 1038, "agent_duration_seconds": 1808.28272, "n_output_tokens": 141480, "n_agent_steps": 97, "cost_usd": 11.816896249999997 }, { "trial_name": "sqlite-utils-safe-import-checkpo__bCbPTAe", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 60, "f2p_total": 60, "p2p_passed": 1038, "p2p_total": 1038, "agent_duration_seconds": 1822.538524, "n_output_tokens": 155611, "n_agent_steps": 74, "cost_usd": 10.324739250000002 } ] }, { "task": "sqlite-utils-safe-import-checkpoints", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/sqlite-utils-safe-import-checkpoints", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "sqlite-utils-safe-import-checkpo__jncZZcq", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 954, "n_output_tokens": 35663, "n_agent_steps": 27, "cost_usd": 3.7017295 }, { "trial_name": "sqlite-utils-safe-import-checkpo__2dvVd3P", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1023, "n_output_tokens": 34900, "n_agent_steps": 26, "cost_usd": 3.831769 }, { "trial_name": "sqlite-utils-safe-import-checkpo__HMT86Zk", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1012, "n_output_tokens": 32735, "n_agent_steps": 25, "cost_usd": 3.6349375 }, { "trial_name": "sqlite-utils-safe-import-checkpo__GwNfXJV", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 882, "n_output_tokens": 33662, "n_agent_steps": 22, "cost_usd": 3.7615055 } ] }, { "task": "cattrs-partial-structuring-recovery", "model": "claude-opus-5", "effort": "max", "passed": 1, "attempts": 4, "result": "1/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/cattrs-partial-structuring-recovery", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "cattrs-partial-structuring-recov__LT9pV4B", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 67, "f2p_total": 69, "p2p_passed": 7, "p2p_total": 7, "agent_duration_seconds": 1257.11763, "n_output_tokens": 90894, "n_agent_steps": 74, "cost_usd": 7.663733749999999 }, { "trial_name": "cattrs-partial-structuring-recov__mQ8CKBe", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 69, "f2p_total": 69, "p2p_passed": 7, "p2p_total": 7, "agent_duration_seconds": 1105.87346, "n_output_tokens": 88062, "n_agent_steps": 63, "cost_usd": 6.804415250000001 }, { "trial_name": "cattrs-partial-structuring-recov__sQZBifp", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 67, "f2p_total": 69, "p2p_passed": 7, "p2p_total": 7, "agent_duration_seconds": 1630.637625, "n_output_tokens": 114588, "n_agent_steps": 101, "cost_usd": 11.318763749999999 }, { "trial_name": "cattrs-partial-structuring-recov__c7Yrwex", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 67, "f2p_total": 69, "p2p_passed": 7, "p2p_total": 7, "agent_duration_seconds": 1727.828606, "n_output_tokens": 127690, "n_agent_steps": 84, "cost_usd": 10.498603499999998 } ] }, { "task": "cattrs-partial-structuring-recovery", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/cattrs-partial-structuring-recovery", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "cattrs-partial-structuring-recov__7DosmYF", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 859, "n_output_tokens": 25951, "n_agent_steps": 21, "cost_usd": 2.8373925 }, { "trial_name": "cattrs-partial-structuring-recov__GxfA44o", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 838, "n_output_tokens": 32090, "n_agent_steps": 22, "cost_usd": 3.999448 }, { "trial_name": "cattrs-partial-structuring-recov__8QHKABb", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 736, "n_output_tokens": 25894, "n_agent_steps": 27, "cost_usd": 3.0529315 }, { "trial_name": "cattrs-partial-structuring-recov__zqBtrsJ", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 618, "n_output_tokens": 25388, "n_agent_steps": 16, "cost_usd": 2.580318 } ] }, { "task": "tomlkit-toml-table-converters", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/tomlkit-toml-table-converters", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "tomlkit-toml-table-converters__NmySf4G", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 60, "f2p_total": 60, "p2p_passed": 964, "p2p_total": 964, "agent_duration_seconds": 2072.030482, "n_output_tokens": 170386, "n_agent_steps": 83, "cost_usd": 13.358928750000004 }, { "trial_name": "tomlkit-toml-table-converters__tugZsJV", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 60, "f2p_total": 60, "p2p_passed": 964, "p2p_total": 964, "agent_duration_seconds": 2241.812246, "n_output_tokens": 186255, "n_agent_steps": 98, "cost_usd": 14.7187675 }, { "trial_name": "tomlkit-toml-table-converters__zdx9URS", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 60, "f2p_total": 60, "p2p_passed": 964, "p2p_total": 964, "agent_duration_seconds": 2909.86045, "n_output_tokens": 237985, "n_agent_steps": 119, "cost_usd": 22.03270675 }, { "trial_name": "tomlkit-toml-table-converters__96pcKB9", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 60, "f2p_total": 60, "p2p_passed": 964, "p2p_total": 964, "agent_duration_seconds": 2069.530378, "n_output_tokens": 166525, "n_agent_steps": 82, "cost_usd": 12.472251250000001 } ] }, { "task": "tomlkit-toml-table-converters", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/tomlkit-toml-table-converters", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "tomlkit-toml-table-converters__HRrjKhh", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 961, "n_output_tokens": 30214, "n_agent_steps": 24, "cost_usd": 3.93782 }, { "trial_name": "tomlkit-toml-table-converters__yTPMVfM", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 867, "n_output_tokens": 32889, "n_agent_steps": 26, "cost_usd": 3.8454085 }, { "trial_name": "tomlkit-toml-table-converters__EjYaHVU", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 912, "n_output_tokens": 27756, "n_agent_steps": 26, "cost_usd": 4.7788515 }, { "trial_name": "tomlkit-toml-table-converters__edReCYH", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 928, "n_output_tokens": 28620, "n_agent_steps": 22, "cost_usd": 3.8549195 } ] }, { "task": "dateutil-rfc5545-timezone-interop", "model": "claude-opus-5", "effort": "max", "passed": 3, "attempts": 4, "result": "3/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/dateutil-rfc5545-timezone-interop", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "dateutil-rfc5545-timezone-intero__8JaA8Fr", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 67, "f2p_total": 67, "p2p_passed": 2035, "p2p_total": 2035, "agent_duration_seconds": 1291.80661, "n_output_tokens": 115311, "n_agent_steps": 75, "cost_usd": 8.340422250000001 }, { "trial_name": "dateutil-rfc5545-timezone-intero__CDDAcJR", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 66, "f2p_total": 67, "p2p_passed": 2035, "p2p_total": 2035, "agent_duration_seconds": 1619.792914, "n_output_tokens": 147040, "n_agent_steps": 78, "cost_usd": 10.100814000000003 }, { "trial_name": "dateutil-rfc5545-timezone-intero__P5QADTs", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 67, "f2p_total": 67, "p2p_passed": 2035, "p2p_total": 2035, "agent_duration_seconds": 1435.964478, "n_output_tokens": 120870, "n_agent_steps": 62, "cost_usd": 7.4979275 }, { "trial_name": "dateutil-rfc5545-timezone-intero__tvzZYDh", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 67, "f2p_total": 67, "p2p_passed": 2035, "p2p_total": 2035, "agent_duration_seconds": 1712.240967, "n_output_tokens": 143227, "n_agent_steps": 86, "cost_usd": 10.960925999999999 } ] }, { "task": "dateutil-rfc5545-timezone-interop", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/dateutil-rfc5545-timezone-interop", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "dateutil-rfc5545-timezone-intero__5ZPHzx2", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 747, "n_output_tokens": 27090, "n_agent_steps": 27, "cost_usd": 3.117666 }, { "trial_name": "dateutil-rfc5545-timezone-intero__GJhUMFQ", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 721, "n_output_tokens": 26550, "n_agent_steps": 21, "cost_usd": 3.465418 }, { "trial_name": "dateutil-rfc5545-timezone-intero__znB3HUv", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 2650, "n_output_tokens": 32758, "n_agent_steps": 28, "cost_usd": 4.37748 }, { "trial_name": "dateutil-rfc5545-timezone-intero__TT2iwAb", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 684, "n_output_tokens": 27746, "n_agent_steps": 22, "cost_usd": 4.685071 } ] }, { "task": "etree-xml-diff-patch", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/etree-xml-diff-patch", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "etree-xml-diff-patch__D7i7k7C", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 52, "f2p_total": 52, "p2p_passed": 15, "p2p_total": 15, "agent_duration_seconds": 2043.409216, "n_output_tokens": 188059, "n_agent_steps": 50, "cost_usd": 10.128508250000001 }, { "trial_name": "etree-xml-diff-patch__tHgb9Vr", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 52, "f2p_total": 52, "p2p_passed": 15, "p2p_total": 15, "agent_duration_seconds": 1629.923978, "n_output_tokens": 144958, "n_agent_steps": 50, "cost_usd": 8.444230499999998 }, { "trial_name": "etree-xml-diff-patch__iq2c9jw", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 52, "f2p_total": 52, "p2p_passed": 15, "p2p_total": 15, "agent_duration_seconds": 1706.856726, "n_output_tokens": 150969, "n_agent_steps": 55, "cost_usd": 9.123385 }, { "trial_name": "etree-xml-diff-patch__wKQENqu", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 52, "f2p_total": 52, "p2p_passed": 15, "p2p_total": 15, "agent_duration_seconds": 2081.910878, "n_output_tokens": 188139, "n_agent_steps": 45, "cost_usd": 9.616640750000002 } ] }, { "task": "etree-xml-diff-patch", "model": "gpt-6-astra", "effort": "xhigh", "passed": 2, "attempts": 4, "result": "2/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/etree-xml-diff-patch", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "etree-xml-diff-patch__ookNdnZ", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 915, "n_output_tokens": 39259, "n_agent_steps": 19, "cost_usd": 3.790143 }, { "trial_name": "etree-xml-diff-patch__a9cW8hk", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1104, "n_output_tokens": 46685, "n_agent_steps": 23, "cost_usd": 4.436001 }, { "trial_name": "etree-xml-diff-patch__Qe4Ggnv", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1163, "n_output_tokens": 42593, "n_agent_steps": 19, "cost_usd": 3.791299 }, { "trial_name": "etree-xml-diff-patch__aroRUz6", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1106, "n_output_tokens": 44357, "n_agent_steps": 24, "cost_usd": 4.010965 } ] }, { "task": "bandit-structured-nosec-directives", "model": "claude-opus-5", "effort": "max", "passed": 0, "attempts": 4, "result": "0/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/bandit-structured-nosec-directives", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "bandit-structured-nosec-directiv__jQmhbFN", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 69, "f2p_total": 69, "p2p_passed": 281, "p2p_total": 282, "agent_duration_seconds": 1486.782495, "n_output_tokens": 108072, "n_agent_steps": 75, "cost_usd": 8.424927499999999 }, { "trial_name": "bandit-structured-nosec-directiv__TYM7wB6", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 67, "f2p_total": 69, "p2p_passed": 281, "p2p_total": 282, "agent_duration_seconds": 1329.035701, "n_output_tokens": 105159, "n_agent_steps": 85, "cost_usd": 8.37353 }, { "trial_name": "bandit-structured-nosec-directiv__ae5KeaD", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 67, "f2p_total": 69, "p2p_passed": 281, "p2p_total": 282, "agent_duration_seconds": 1355.421626, "n_output_tokens": 106951, "n_agent_steps": 73, "cost_usd": 8.260703750000001 }, { "trial_name": "bandit-structured-nosec-directiv__XHnMS5t", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": 67, "f2p_total": 69, "p2p_passed": 281, "p2p_total": 282, "agent_duration_seconds": 1305.185228, "n_output_tokens": 97537, "n_agent_steps": 61, "cost_usd": 6.922166500000001 } ] }, { "task": "bandit-structured-nosec-directives", "model": "gpt-6-astra", "effort": "xhigh", "passed": 0, "attempts": 4, "result": "0/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/bandit-structured-nosec-directives", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "bandit-structured-nosec-directiv__xKTjkSo", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 727, "n_output_tokens": 26651, "n_agent_steps": 17, "cost_usd": 3.218079 }, { "trial_name": "bandit-structured-nosec-directiv__L2G3V6F", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 886, "n_output_tokens": 29188, "n_agent_steps": 21, "cost_usd": 2.989483 }, { "trial_name": "bandit-structured-nosec-directiv__irsZNY8", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1020, "n_output_tokens": 30967, "n_agent_steps": 24, "cost_usd": 3.899955 }, { "trial_name": "bandit-structured-nosec-directiv__nvgrfRW", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 683, "n_output_tokens": 25993, "n_agent_steps": 21, "cost_usd": 3.2258675 } ] }, { "task": "participle-grammar-conflict-analysis", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/participle-grammar-conflict-analysis", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "participle-grammar-conflict-anal__47iL2mD", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 91, "f2p_total": 91, "p2p_passed": 153, "p2p_total": 153, "agent_duration_seconds": 1809.95734, "n_output_tokens": 124958, "n_agent_steps": 81, "cost_usd": 10.124839 }, { "trial_name": "participle-grammar-conflict-anal__ZW9aCRW", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 91, "f2p_total": 91, "p2p_passed": 153, "p2p_total": 153, "agent_duration_seconds": 1479.877502, "n_output_tokens": 114268, "n_agent_steps": 61, "cost_usd": 7.894513249999999 }, { "trial_name": "participle-grammar-conflict-anal__6ZF8BFo", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 91, "f2p_total": 91, "p2p_passed": 153, "p2p_total": 153, "agent_duration_seconds": 1251.129051, "n_output_tokens": 99362, "n_agent_steps": 57, "cost_usd": 7.064859999999999 }, { "trial_name": "participle-grammar-conflict-anal__i5nCFk6", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 91, "f2p_total": 91, "p2p_passed": 153, "p2p_total": 153, "agent_duration_seconds": 1542.73084, "n_output_tokens": 116760, "n_agent_steps": 65, "cost_usd": 8.723291249999999 } ] }, { "task": "participle-grammar-conflict-analysis", "model": "gpt-6-astra", "effort": "xhigh", "passed": 0, "attempts": 4, "result": "0/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/participle-grammar-conflict-analysis", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "participle-grammar-conflict-anal__KWDXjU3", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 853, "n_output_tokens": 30648, "n_agent_steps": 22, "cost_usd": 3.2125195 }, { "trial_name": "participle-grammar-conflict-anal__jqfoz78", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 839, "n_output_tokens": 27687, "n_agent_steps": 20, "cost_usd": 2.880025 }, { "trial_name": "participle-grammar-conflict-anal__pZrXCR6", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 817, "n_output_tokens": 25474, "n_agent_steps": 19, "cost_usd": 2.689252 }, { "trial_name": "participle-grammar-conflict-anal__93c9Dkc", "outcome": "fail", "passed": false, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1205, "n_output_tokens": 32020, "n_agent_steps": 20, "cost_usd": 3.1523505 } ] }, { "task": "valibot-recursive-schema-composition", "model": "claude-opus-5", "effort": "max", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/valibot-recursive-schema-composition", "note": "max reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "valibot-recursive-schema-composi__qQXhidL", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 10, "f2p_total": 10, "p2p_passed": 209, "p2p_total": 209, "agent_duration_seconds": 4156.16086, "n_output_tokens": 239242, "n_agent_steps": 196, "cost_usd": 30.439942499999983 }, { "trial_name": "valibot-recursive-schema-composi__n3SLLP5", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 10, "f2p_total": 10, "p2p_passed": 209, "p2p_total": 209, "agent_duration_seconds": 3389.293582, "n_output_tokens": 239920, "n_agent_steps": 166, "cost_usd": 27.059483500000006 }, { "trial_name": "valibot-recursive-schema-composi__8Dk6JVc", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 10, "f2p_total": 10, "p2p_passed": 209, "p2p_total": 209, "agent_duration_seconds": 3932.567493, "n_output_tokens": 208528, "n_agent_steps": 165, "cost_usd": 24.85514999999999 }, { "trial_name": "valibot-recursive-schema-composi__tdggFci", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": 10, "f2p_total": 10, "p2p_passed": 209, "p2p_total": 209, "agent_duration_seconds": 3611.779631, "n_output_tokens": 219210, "n_agent_steps": 167, "cost_usd": 25.615049750000004 } ] }, { "task": "valibot-recursive-schema-composition", "model": "gpt-6-astra", "effort": "xhigh", "passed": 4, "attempts": 4, "result": "4/4 passed", "source": "https://deepswe.datacurve.ai/data/v1.1/tasks/valibot-recursive-schema-composition", "note": "xhigh reasoning \u00b7 official mini-swe-agent runs", "trials": [ { "trial_name": "valibot-recursive-schema-composi__bDwC3t8", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 3865, "n_output_tokens": 30070, "n_agent_steps": 36, "cost_usd": 4.3731405 }, { "trial_name": "valibot-recursive-schema-composi__BeHcxk6", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 3592, "n_output_tokens": 35052, "n_agent_steps": 31, "cost_usd": 5.330469 }, { "trial_name": "valibot-recursive-schema-composi__7hFdtgw", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1067, "n_output_tokens": 29581, "n_agent_steps": 43, "cost_usd": 5.842106 }, { "trial_name": "valibot-recursive-schema-composi__psb6JCu", "outcome": "pass", "passed": true, "included_in_score": true, "f2p_passed": null, "f2p_total": null, "p2p_passed": null, "p2p_total": null, "agent_duration_seconds": 1188, "n_output_tokens": 28507, "n_agent_steps": 35, "cost_usd": 4.971248 } ] } ] }