{ "exercises": [ { "group": "Async state & resource ownership", "task": "ofetch-per-origin-circuit-breaker", "project": "ofetch", "stack": "TypeScript \u00b7 HTTP client", "headline": "Coordinate a circuit breaker across retries, hooks, and shared clients", "requirements": [ "Track closed/open/half-open state by the effective URL origin after request hooks.", "Share state across derived clients; hold a recovery-probe slot through retries.", "Count one failure per logical request, including parsing and hook failures\u2014not one per retry." ], "critique": "A strong integration result: the completed patch had to coordinate state across multiple entry points, not merely implement a counter. It passed in the October 7 batch and again overnight. This is better repeat evidence than a single successful generation." }, { "group": "Async state & resource ownership", "task": "happy-dom-abort-pending-body-reads", "project": "Happy DOM", "stack": "TypeScript \u00b7 browser/DOM runtime", "headline": "Abort in-flight body reads without breaking buffered responses", "requirements": [ "Abort Request/Response reads and multipart parsing when a browser, page, or active page state closes.", "Preserve already-buffered response bodies.", "Clear timers and animation callbacks owned by discarded state." ], "critique": "Pass \u2192 timeout \u2192 pass is a delivery-reliability problem, not a simple inability to handle async code. The overnight session consumed its full 45 minutes and submitted nothing usable; the follow-up finished in about 30. A prior successful implementation did not make this a dependable assignment." }, { "group": "Async state & resource ownership", "task": "httpx-streaming-json-iteration", "project": "HTTPX", "stack": "Python \u00b7 sync/async streaming", "headline": "Stream JSON, NDJSON, and JSON sequences\u2014and close on every exit path", "requirements": [ "Implement synchronous and asynchronous JSON iterators with media-type validation and encoding detection.", "Handle array elements, line-delimited records, and record-separator sequences incrementally.", "Close consumed streams, reject a second iteration, and preserve repeatable in-memory responses." ], "critique": "The follow-up got the parser almost entirely right and still leaked the streaming response after an invalid NDJSON line. That cleanup obligation is part of the API, not an optional refinement. The overnight clean pass followed by the same kind of miss seen earlier makes exception-path consistency the weakness to target." }, { "group": "Async state & resource ownership", "task": "aiomonitor-task-snapshots-diff", "project": "aiomonitor", "stack": "Python \u00b7 asyncio / CLI / web API", "headline": "Freeze async task state and expose snapshot diffs across terminal and web", "requirements": [ "Capture running and terminated tasks with stable snapshot IDs, names, stacks, and timing fields.", "Diff snapshots by task identity; implement retention that favors named snapshots.", "Add terminal commands, completion signaling, JSON endpoints, and a snapshots page." ], "critique": "Both recorded attempts passed across a broad feature surface. The earlier patch review still found a hole: once every stored snapshot was named, the implementation evicted a named snapshot despite the preservation requirement. The grader accepted it. This is a concrete limitation of the test coverage, not a reason to relabel the official pass." }, { "group": "Transactions & partial failure", "task": "sqlite-utils-safe-import-checkpoints", "project": "sqlite-utils", "stack": "Python + SQL \u00b7 database API / CLI", "headline": "Roll back failed imports\u2014including schema changes\u2014and enforce write invariants", "requirements": [ "Support nested checkpoints with explicit commit, rollback, and cleanup lifecycles.", "Persist and evaluate table invariants after writes; restore tables, columns, indexes, and triggers after failure.", "Expose safe insert/upsert/import behavior and CLI controls, including bulk UPDATE." ], "critique": "The earlier run missed invariant validation on bulk UPDATE\u2014a central safety guarantee. The overnight failures were different: the enable-safe-import and disable-safe-import CLI checks exited with code 2 instead of succeeding. The newer patch passed most behavior checks but failed the commands users need to operate the feature. Do not carry the older UPDATE diagnosis onto this run." }, { "group": "Transactions & partial failure", "task": "cattrs-partial-structuring-recovery", "project": "cattrs", "stack": "Python \u00b7 attrs / dataclasses / TypedDict", "headline": "Recover valid fields from failed conversions without inventing valid objects", "requirements": [ "Return a PartialResult separating successful fields, failed fields, errors, and any constructible value.", "Recurse into nested objects, treat collection conversion atomically, and honor defaults and required fields.", "Refine failed fields without overwriting successful ones; respect detailed-validation settings." ], "critique": "The first recorded patch returned a partial TypedDict when a required field failed, and its own tests endorsed that wrong contract. A separate later implementation passed after manual submission recovery, but the agent had omitted the required commit. The follow-up committed correctly and missed successful-field accounting with detailed validation disabled. Three distinct problems: understanding the contract, shipping the patch, and covering a configuration branch." }, { "group": "Representation, identity & ordering", "task": "fd-deterministic-multi-key-sorting", "project": "fd", "stack": "Rust \u00b7 filesystem search / clap CLI", "headline": "Make multi-key filesystem sorting deterministic\u2014even for seeded randomness", "requirements": [ "Apply repeatable sort keys left-to-right with deterministic path tie-breaks independent of traversal order.", "Combine natural ordering, case handling, missing metadata, file-kind grouping, and seeded random ordering.", "Sort and reverse before applying result limits; preserve filtering and unsorted output behavior." ], "critique": "Two clean results: about 23 minutes, then 17. The earlier source review confirmed the important operation order: buffer, sort/reverse, then truncate. This is a stronger result than adding CLI flags alone; combinations and determinism are the feature." }, { "group": "Representation, identity & ordering", "task": "tomlkit-toml-table-converters", "project": "TOMLKit", "stack": "Python \u00b7 format-preserving TOML", "headline": "Convert table representations without losing comments or round-trip integrity", "requirements": [ "Convert among header tables, inline tables, dotted keys, and grouped super-tables.", "Move comments to the correct new location while preserving values and parse/serialize round trips.", "Mutate the existing document, handle nested conversions and depth limits, and reject unsupported paths." ], "critique": "The October 7 session explored the code, lost precise API requirements during context summarization, and asked for clarification instead of delivering. The overnight attempt passed. The original failure was not a nearly finished converter: it produced no implementation. Retaining the literal API contract across a long session mattered as much as navigating the parser internals." }, { "group": "Representation, identity & ordering", "task": "dateutil-rfc5545-timezone-interop", "project": "python-dateutil", "stack": "Python \u00b7 recurrence / iCalendar", "headline": "Round-trip recurrence rules through RFC 5545 timezones and calendars", "requirements": [ "Parse TZID/VALUE parameters and embedded VTIMEZONE definitions with explicit resolution precedence.", "Serialize timezone-aware rules and recurrence sets, including UTC Z suffixes and non-UTC TZIDs.", "Add equality, hashing, representations, calendar export, and recurrence-set operations." ], "critique": "The first attempt flooded its context with a recurrence list extending to the year 7980. The later runs delivered code: 64/67, then 66/67 new checks. The remaining TZID serialization failure is an interoperability defect in the feature's core purpose. The follow-up also took about 22 minutes versus 13; closer to completion did not mean faster." }, { "group": "Representation, identity & ordering", "task": "etree-xml-diff-patch", "project": "etree", "stack": "Go \u00b7 XML diff / patch / merge", "headline": "Diff XML by identity, reverse patches, and reconcile three-way conflicts", "requirements": [ "Match elements by position, key attribute, or content hash; distinguish replacement from movement.", "Generate and apply XML patches for elements, attributes, and text, then reverse them.", "Merge base/ours/theirs with conflict classification and configurable resolution." ], "critique": "The early patch panicked after removing an element and then using its cleared parent pointer. Overnight, 49/52 new checks passed; failures covered ignored attributes and delete-parent/add-child conflict classification. The follow-up reached 51/52, failing the move check. This is substantial progress across different implementations, but identity and structural edits are still the unfinished part\u2014not peripheral formatting." }, { "group": "Language analysis & recursive types", "task": "bandit-structured-nosec-directives", "project": "Bandit", "stack": "Python \u00b7 security-analysis directives", "headline": "Parse suppression expressions and apply them to whole statements", "requirements": [ "Add nested region and next-statement directives with indentation-sensitive lifetimes.", "Resolve selector unions, intersections, differences, negation, and fallback parsing.", "Combine suppressions across multiline statements and preserve blanket-versus-specific metrics." ], "critique": "Both newer runs passed every new-feature check and broke the same existing multiline suppression-union check. That is a repeat regression, not a random near miss. The patch extended the directive language without fully preserving how suppression combines across statement lines. New tests alone would have given the wrong verdict." }, { "group": "Language analysis & recursive types", "task": "participle-grammar-conflict-analysis", "project": "Participle", "stack": "Go \u00b7 parser construction / static analysis", "headline": "Analyze FIRST/FOLLOW conflicts in recursive grammars", "requirements": [ "Detect overlapping alternatives, repetition/follow conflicts, and unreachable branches.", "Propagate epsilon through embedded grammar nodes and respect lookahead and negation rules.", "Produce useful conflict reports, build-tagged APIs, filters, and a strict parser-build mode." ], "critique": "The earlier submission failed to build against the new tests. The overnight run got farther, then TestAnalyzeRecursiveStructure triggered a fatal stack overflow. Fourteen new checks were recorded as passing before the aborted suite. This is not 77 separately diagnosed wrong answers: the recursive-analysis path crashed the process. Termination on recursive grammars is foundational here." }, { "group": "Language analysis & recursive types", "task": "valibot-recursive-schema-composition", "project": "Valibot", "stack": "TypeScript \u00b7 runtime validation / type inference", "headline": "Resolve recursive schema placeholders through containers, pipes, and intersections", "requirements": [ "Add Recur plus recursive/recursiveAsync wrappers across sync and async parsing.", "Resolve recursion through arrays, records, maps, sets, pipes, and intersections.", "Keep input/output types genuinely self-referential and reject unresolved placeholders at typed parse calls." ], "critique": "Both recorded attempts exhausted 45 minutes without a usable submission. The earlier trace identified a real obstacle: a piped schema's execution closure retained original items after cloning. Finding that obstacle was useful debugging, but it did not become a finished repair. This remains a delivery failure on a task combining runtime graph rewriting with compile-time type behavior." } ], "runs": { "ofetch-per-origin-circuit-breaker": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Pass", "new": "47/47", "preservation": "13/13", "minutes": 18.94, "setup_seconds": 27.17, "grading_seconds": 17.53, "patch_bytes": 35979, "requests": 59, "output_tokens": 75532, "peak_context": 117059, "agent_steps": 58, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "ofetch-per-origin-circuit-breake__k2FbL8H" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Pass", "new": "47/47", "preservation": "13/13", "minutes": 16.37, "setup_seconds": 3.47, "grading_seconds": 16.17, "patch_bytes": 38325, "requests": 45, "output_tokens": 58883, "peak_context": 86219, "agent_steps": 44, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "ofetch-per-origin-circuit-breake__WZFS28q" } ], "happy-dom-abort-pending-body-reads": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Pass", "new": "14/14", "preservation": "165/165", "minutes": 40.08, "setup_seconds": 32.57, "grading_seconds": 23.73, "patch_bytes": 32026, "requests": 195, "output_tokens": 121104, "peak_context": 122055, "agent_steps": 194, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "happy-dom-abort-pending-body-rea__ZCFqcVj" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Timed out", "new": "\u2014", "preservation": "\u2014", "minutes": 45.0, "setup_seconds": 3.51, "grading_seconds": 28.74, "patch_bytes": 0, "requests": 97, "output_tokens": 40583, "peak_context": 85793, "agent_steps": 44, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "happy-dom-abort-pending-body-rea__J8MWLHE" }, { "checkpoint": "Oct 11 \u00b7 IQ3 / 196K", "outcome": "Pass", "new": "14/14", "preservation": "165/165", "minutes": 30.39, "setup_seconds": 3.71, "grading_seconds": 22.61, "patch_bytes": 31508, "requests": 144, "output_tokens": 67279, "peak_context": 152338, "agent_steps": 143, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 196608, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "196,608 tokens", "Reasoning": "medium", "GPU expert-cache reserve": "2,432 MiB", "Agent time limit": "45 minutes" }, "run_id": "happy-dom-abort-pending-body-rea__bsNBNwE" } ], "httpx-streaming-json-iteration": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Failed checks", "new": "107/108", "preservation": "1404/1404", "minutes": 17.31, "setup_seconds": 31.61, "grading_seconds": 23.25, "patch_bytes": 23647, "requests": 63, "output_tokens": 68354, "peak_context": 106589, "agent_steps": 62, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "httpx-streaming-json-iteration__XUh7CYz" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Pass", "new": "108/108", "preservation": "1404/1404", "minutes": 20.02, "setup_seconds": 3.72, "grading_seconds": 21.99, "patch_bytes": 22568, "requests": 95, "output_tokens": 57944, "peak_context": 90963, "agent_steps": 94, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "httpx-streaming-json-iteration__6uPh3qf" }, { "checkpoint": "Oct 11 \u00b7 IQ3 / 196K", "outcome": "Failed checks", "new": "107/108", "preservation": "1404/1404", "minutes": 21.04, "setup_seconds": 3.68, "grading_seconds": 22.27, "patch_bytes": 30697, "requests": 64, "output_tokens": 65326, "peak_context": 100600, "agent_steps": 63, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 196608, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "196,608 tokens", "Reasoning": "medium", "GPU expert-cache reserve": "2,432 MiB", "Agent time limit": "45 minutes" }, "run_id": "httpx-streaming-json-iteration__EVcbvPd" } ], "aiomonitor-task-snapshots-diff": [ { "checkpoint": "Oct 10 \u00b7 IQ2", "outcome": "Pass", "new": "53/53", "preservation": "8/8", "minutes": 18.7, "setup_seconds": 27.41, "grading_seconds": 21.38, "patch_bytes": 43057, "requests": 80, "output_tokens": 70461, "peak_context": 90272, "agent_steps": 79, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB \u00b7 before the upgrade checkpoint", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "aiomonitor-task-snapshots-diff__h6mGTqL" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Pass", "new": "53/53", "preservation": "8/8", "minutes": 15.11, "setup_seconds": 3.55, "grading_seconds": 20.13, "patch_bytes": 50651, "requests": 69, "output_tokens": 48486, "peak_context": 91040, "agent_steps": 68, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "aiomonitor-task-snapshots-diff__YqXqtu2" } ], "sqlite-utils-safe-import-checkpoints": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Failed checks", "new": "59/60", "preservation": "1038/1038", "minutes": 25.09, "setup_seconds": 25.72, "grading_seconds": 45.12, "patch_bytes": 60297, "requests": 163, "output_tokens": 90350, "peak_context": 88370, "agent_steps": 162, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "sqlite-utils-safe-import-checkpo__wjZdVut" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Failed checks", "new": "58/60", "preservation": "1038/1038", "minutes": 19.14, "setup_seconds": 3.5, "grading_seconds": 43.68, "patch_bytes": 78457, "requests": 95, "output_tokens": 57376, "peak_context": 91126, "agent_steps": 94, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "sqlite-utils-safe-import-checkpo__HZRqagG" } ], "cattrs-partial-structuring-recovery": [ { "checkpoint": "Oct 10 \u00b7 IQ2", "outcome": "Failed checks", "new": "68/69", "preservation": "7/7", "minutes": 18.09, "setup_seconds": 25.96, "grading_seconds": 17.41, "patch_bytes": 28682, "requests": 60, "output_tokens": 68918, "peak_context": 88710, "agent_steps": 59, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB \u00b7 before the upgrade checkpoint", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "cattrs-partial-structuring-recov__BrzkHWA" }, { "checkpoint": "Oct 11 \u00b7 IQ3 / 196K", "outcome": "Failed checks", "new": "68/69", "preservation": "7/7", "minutes": 17.37, "setup_seconds": 3.82, "grading_seconds": 16.27, "patch_bytes": 26973, "requests": 72, "output_tokens": 48263, "peak_context": 97561, "agent_steps": 71, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 196608, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "196,608 tokens", "Reasoning": "medium", "GPU expert-cache reserve": "2,432 MiB", "Agent time limit": "45 minutes" }, "run_id": "cattrs-partial-structuring-recov__kEmo95n" } ], "fd-deterministic-multi-key-sorting": [ { "checkpoint": "Oct 10 \u00b7 IQ2", "outcome": "Pass", "new": "43/43", "preservation": "109/109", "minutes": 22.6, "setup_seconds": 30.06, "grading_seconds": 20.53, "patch_bytes": 39185, "requests": 96, "output_tokens": 86735, "peak_context": 90214, "agent_steps": 95, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB \u00b7 before the upgrade checkpoint", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "fd-deterministic-multi-key-sorti__7kQFYy7" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Pass", "new": "43/43", "preservation": "109/109", "minutes": 17.32, "setup_seconds": 3.55, "grading_seconds": 19.08, "patch_bytes": 38310, "requests": 67, "output_tokens": 55334, "peak_context": 89904, "agent_steps": 66, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "fd-deterministic-multi-key-sorti__Li5LoqF" } ], "tomlkit-toml-table-converters": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Empty submission", "new": "\u2014", "preservation": "\u2014", "minutes": 22.58, "setup_seconds": 28.01, "grading_seconds": 17.86, "patch_bytes": 0, "requests": 34, "output_tokens": 87037, "peak_context": 107453, "agent_steps": 33, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "tomlkit-toml-table-converters__KaECs6E" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Pass", "new": "60/60", "preservation": "964/964", "minutes": 17.98, "setup_seconds": 3.52, "grading_seconds": 16.73, "patch_bytes": 26840, "requests": 62, "output_tokens": 57473, "peak_context": 89940, "agent_steps": 61, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "tomlkit-toml-table-converters__LseFe3C" } ], "dateutil-rfc5545-timezone-interop": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Interrupted", "new": "\u2014", "preservation": "\u2014", "minutes": 13.93, "setup_seconds": 24.9, "grading_seconds": 25.18, "patch_bytes": 0, "requests": 61, "output_tokens": 55178, "peak_context": 112132, "agent_steps": 59, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "dateutil-rfc5545-timezone-intero__deZCCVw" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Failed checks", "new": "64/67", "preservation": "2035/2035", "minutes": 13.29, "setup_seconds": 3.74, "grading_seconds": 19.76, "patch_bytes": 37090, "requests": 55, "output_tokens": 44061, "peak_context": 90396, "agent_steps": 54, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "dateutil-rfc5545-timezone-intero__V7xMczF" }, { "checkpoint": "Oct 11 \u00b7 IQ3 / 196K", "outcome": "Failed checks", "new": "66/67", "preservation": "2035/2035", "minutes": 22.03, "setup_seconds": 3.53, "grading_seconds": 20.31, "patch_bytes": 52793, "requests": 58, "output_tokens": 71559, "peak_context": 121031, "agent_steps": 57, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 196608, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "196,608 tokens", "Reasoning": "medium", "GPU expert-cache reserve": "2,432 MiB", "Agent time limit": "45 minutes" }, "run_id": "dateutil-rfc5545-timezone-intero__xf6J3AP" } ], "etree-xml-diff-patch": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Failed checks", "new": "2/52", "preservation": "15/15", "minutes": 10.1, "setup_seconds": 24.88, "grading_seconds": 23.18, "patch_bytes": 47591, "requests": 33, "output_tokens": 44635, "peak_context": 64006, "agent_steps": 32, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "etree-xml-diff-patch__u5AdYmy" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Failed checks", "new": "49/52", "preservation": "15/15", "minutes": 9.0, "setup_seconds": 3.53, "grading_seconds": 21.76, "patch_bytes": 37653, "requests": 30, "output_tokens": 33356, "peak_context": 50829, "agent_steps": 29, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "etree-xml-diff-patch__Ww5FXQR" }, { "checkpoint": "Oct 11 \u00b7 IQ3 / 196K", "outcome": "Failed checks", "new": "51/52", "preservation": "15/15", "minutes": 20.36, "setup_seconds": 3.54, "grading_seconds": 22.25, "patch_bytes": 44191, "requests": 49, "output_tokens": 70812, "peak_context": 108015, "agent_steps": 48, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 196608, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "196,608 tokens", "Reasoning": "medium", "GPU expert-cache reserve": "2,432 MiB", "Agent time limit": "45 minutes" }, "run_id": "etree-xml-diff-patch__eQUeBHU" } ], "bandit-structured-nosec-directives": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Failed checks", "new": "67/69", "preservation": "280/282", "minutes": 22.85, "setup_seconds": 25.72, "grading_seconds": 29.27, "patch_bytes": 35507, "requests": 72, "output_tokens": 79997, "peak_context": 119746, "agent_steps": 71, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "bandit-structured-nosec-directiv__2oaTY4J" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Failed checks", "new": "69/69", "preservation": "281/282", "minutes": 18.69, "setup_seconds": 3.55, "grading_seconds": 28.3, "patch_bytes": 27475, "requests": 70, "output_tokens": 58651, "peak_context": 89437, "agent_steps": 69, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "bandit-structured-nosec-directiv__cSnTYyN" }, { "checkpoint": "Oct 11 \u00b7 IQ3 / 196K", "outcome": "Failed checks", "new": "69/69", "preservation": "281/282", "minutes": 19.2, "setup_seconds": 3.55, "grading_seconds": 27.85, "patch_bytes": 28617, "requests": 70, "output_tokens": 59699, "peak_context": 96960, "agent_steps": 69, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 196608, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "196,608 tokens", "Reasoning": "medium", "GPU expert-cache reserve": "2,432 MiB", "Agent time limit": "45 minutes" }, "run_id": "bandit-structured-nosec-directiv__K5PDoYv" } ], "participle-grammar-conflict-analysis": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Failed checks", "new": "2/91", "preservation": "153/153", "minutes": 25.82, "setup_seconds": 25.63, "grading_seconds": 28.6, "patch_bytes": 28145, "requests": 70, "output_tokens": 99101, "peak_context": 86747, "agent_steps": 69, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "participle-grammar-conflict-anal__hQgpo3r" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Failed checks", "new": "14/91", "preservation": "153/153", "minutes": 12.49, "setup_seconds": 3.52, "grading_seconds": 30.49, "patch_bytes": 32538, "requests": 44, "output_tokens": 37260, "peak_context": 91199, "agent_steps": 43, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "participle-grammar-conflict-anal__e9DiKUD" } ], "valibot-recursive-schema-composition": [ { "checkpoint": "Oct 7 \u00b7 IQ2", "outcome": "Timed out", "new": "\u2014", "preservation": "\u2014", "minutes": 45.0, "setup_seconds": 35.66, "grading_seconds": 25.87, "patch_bytes": 0, "requests": 125, "output_tokens": 160024, "peak_context": 90914, "agent_steps": 123, "settings": [ { "model": "qwen3.8-flash-next-iq2_xs", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "32 GB", "Model": "Flash-Next \u00b7 IQ2_XS", "Runtime": "Strata \u00b7 exact version not recorded in campaign manifest", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "valibot-recursive-schema-composi__kCv6AA4" }, { "checkpoint": "Oct 10\u201311 \u00b7 IQ3", "outcome": "Timed out", "new": "\u2014", "preservation": "\u2014", "minutes": 45.0, "setup_seconds": 3.63, "grading_seconds": 24.76, "patch_bytes": 0, "requests": 140, "output_tokens": 119011, "peak_context": 90915, "agent_steps": 138, "settings": [ { "model": "qwen3.8-flash-next-iq3_s", "reasoning_effort": "medium", "max_tokens": 131072, "temperature": 1.0, "top_p": 0.95, "top_k": 20, "min_p": 0.0, "presence_penalty": 0.0, "repetition_penalty": 1.0 } ], "configuration": { "GPU": "RX 7900 XTX \u00b7 24 GB VRAM", "System RAM": "64 GB", "Model": "Flash-Next \u00b7 IQ3_S", "Runtime": "Strata 0.1.42 \u00b7 rebuilt locally", "Agent": "OpenCode / Pier", "Total context": "131,072 tokens", "Reasoning": "medium", "Agent time limit": "45 minutes" }, "run_id": "valibot-recursive-schema-composi__5fqpbth" } ] }, "attempt_count": 31 }