From 8b61fb2d64f24af1edd86cb460f37378d2dfd448 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:19:37 -0400 Subject: [PATCH] test(eval): add Responses quality matrix --- .../FIVE_BY_EIGHT_PURE_RESULTS.md | 112 - eval/grok-density/QUALITY_RESULTS.md | 87 +- eval/grok-density/QUALITY_SUITE.md | 78 +- eval/grok-density/README.md | 1 - eval/grok-density/VISUAL_5X8_SOLUTION.md | 62 - ...ve-by-eight-ids-block-white-stability.json | 592 --- eval/grok-density/fix-matrix-results.json | 46 - eval/grok-density/fix-matrix.mjs | 79 - eval/grok-density/lib-responses.mjs | 88 - eval/grok-density/multi-seed-ids-results.json | 305 -- eval/grok-density/multi-seed-ids.mjs | 176 - eval/grok-density/novel-arithmetic.mjs | 200 - eval/sol-profile/QUALITY_RESULTS.md | 40 + .../gist-recall-grok-4.5-results.json | 1290 +++++++ .../gist-recall-grok-4.5-text-results.json | 3326 +++++++++++++++++ eval/sol-profile/gist-recall-results.json | 1293 +++++++ eval/sol-profile/gist-recall.mjs | 26 + ...del-grok-4.5-novel-arithmetic-results.json | 1640 ++++++++ .../sol-profile/novel-arithmetic-results.json | 1642 ++++++++ eval/sol-profile/novel-arithmetic.mjs | 29 + eval/sol-profile/responses-client.mjs | 64 + .../verbatim-hex-grok-4.5-results.json | 161 + .../verbatim-hex-grok-4.5-text-results.json | 147 + eval/sol-profile/verbatim-hex-results.json | 131 + eval/sol-profile/verbatim-hex.mjs | 45 + 25 files changed, 9870 insertions(+), 1790 deletions(-) delete mode 100644 eval/grok-density/FIVE_BY_EIGHT_PURE_RESULTS.md delete mode 100644 eval/grok-density/VISUAL_5X8_SOLUTION.md delete mode 100644 eval/grok-density/five-by-eight-ids-block-white-stability.json delete mode 100644 eval/grok-density/fix-matrix-results.json delete mode 100644 eval/grok-density/fix-matrix.mjs delete mode 100644 eval/grok-density/lib-responses.mjs delete mode 100644 eval/grok-density/multi-seed-ids-results.json delete mode 100644 eval/grok-density/multi-seed-ids.mjs delete mode 100644 eval/grok-density/novel-arithmetic.mjs create mode 100644 eval/sol-profile/QUALITY_RESULTS.md create mode 100644 eval/sol-profile/gist-recall-grok-4.5-results.json create mode 100644 eval/sol-profile/gist-recall-grok-4.5-text-results.json create mode 100644 eval/sol-profile/gist-recall-results.json create mode 100644 eval/sol-profile/gist-recall.mjs create mode 100644 eval/sol-profile/model-grok-4.5-novel-arithmetic-results.json create mode 100644 eval/sol-profile/novel-arithmetic-results.json create mode 100644 eval/sol-profile/novel-arithmetic.mjs create mode 100644 eval/sol-profile/responses-client.mjs create mode 100644 eval/sol-profile/verbatim-hex-grok-4.5-results.json create mode 100644 eval/sol-profile/verbatim-hex-grok-4.5-text-results.json create mode 100644 eval/sol-profile/verbatim-hex-results.json create mode 100644 eval/sol-profile/verbatim-hex.mjs diff --git a/eval/grok-density/FIVE_BY_EIGHT_PURE_RESULTS.md b/eval/grok-density/FIVE_BY_EIGHT_PURE_RESULTS.md deleted file mode 100644 index 32f8adb..0000000 --- a/eval/grok-density/FIVE_BY_EIGHT_PURE_RESULTS.md +++ /dev/null @@ -1,112 +0,0 @@ -# Grok 4.5 pure-image exact at 5×8 - -Live pure-image (no factsheet) sweeps on `grok-4.5`, 2026-07-10. -Goal: improve exact-string recall while keeping **5×8 cell pitch** and -**≤768px short side** (no provider downscale). - -## Baseline (tall 5×8, production packing) - -| arm | pages | exact | confab | notes | -|-----|------:|------:|-------:|-------| -| `aa` single short page | 1 | 0–1/4 | 3–4 | hex/path/port confabulate | -| multipage `aa_H1932` bulk | 3 | 1/4 | 3 | bulk does not fix OCR | - -## Best pure-image 5×8 arms (fixed cellW/H bonus 0) - -| arm | maxH | style | exact | confab | stable? | -|-----|-----:|-------|------:|-------:|---------| -| `aa_H512` | 512 | AA | **3/4** | 0–1 | yes (hex/path/port; camel weak) | -| `aa+grid4_H512` | 512 | AA+grid4 | **3/4** | 0 | yes | -| `aa+color_H512` | 512 | AA+colorCycle | **3/4** | 0 | yes (camel abstains) | -| `aa+grid4+color_H512` | 512 | AA+grid4+color | **3/4** | 0 | yes (camel abstains) | -| `aa+grid4+color_H360` | 360 | AA+grid4+color | **4/4** once | 0 | **no** — n=2 retest 3/4 and 2/4 | -| isotropic `inkDilate` | 1932 | dilate 1–2 | 0/4 | low | worse (glyphs merge) | -| white-on-black | 512/1932 | invert:false | ≤2/4 | — | no gain | - -## Shipping choice (superseded 2026-07-11) - -Earlier same-day ship candidate was AA+grid4 @ H512 (~3/4 pure-image). The -**current** production Grok profile is white AA + **IDS block**, no grid — see -[Shipping pure-image 4/4](#shipping-pure-image-44-2026-07-11) below and -[`VISUAL_5X8_SOLUTION.md`](VISUAL_5X8_SOLUTION.md). - -## Harnesses - -```bash -pnpm run build -GROK_DENSITY_LIVE=1 node eval/grok-density/five-by-eight-pure.mjs -GROK_DENSITY_LIVE=1 node eval/grok-density/five-by-eight-page.mjs -GROK_DENSITY_LIVE=1 node eval/grok-density/five-by-eight-camel.mjs -GROK_DENSITY_LIVE=1 node eval/grok-density/five-by-eight-pass-retest.mjs -``` - -Receipts: `five-by-eight-*-results.json`. - -## Shipped profile verification (same day) - -Profile: `stripCols=152`, `maxHeightPx=512`, `aa+grid4` (grayscale). No factsheet. - -| fixture | pages | exact | confab | notes | -|---------|------:|------:|-------:|-------| -| classic short (40 filler turns) | 1 | 1/4 | 3 | single short page still fails pure-image | -| multipage bulk (earlier arms) | 8 | **3/4** | **0** | hex/path/port often exact; camel abstains | -| multipage bulk (final shipped profile) | 8 | 2/4 | 0 | hex+port exact; camel+path abstain (0 confab) | - -**Improvement vs tall 5×8 baseline (0/4, 4 confab):** multipage pure-image exact is higher and confabulation is much lower. **Not** a stable pure-image 4/4 at 5×8 — camelCase remains the weak probe; pure-image 9×12 still clears 4/4 via `PXPIPE_GPT_PROFILES`. - -## Residual matrix (pure-image, shipped packing base) - -Exhaustive residual levers on top of **5×8 / stripCols 152 / maxH 512 / AA+grid4**. -Receipt: `five-by-eight-residual-matrix-results.json` (2026-07-10T23:07:54.987Z, n=25). - -| arm | family | exact | confab | notes | -|-----|--------|------:|-------:|-------| -| `detail_auto` | detail | 3/4 | 1 | detail=auto; hex→'a3f9c1eeb7d2' | -| `font_jbmono8_aa+grid4_H512` | font | 3/4 | 1 | font=jbmono8; hex→'a49c1e0b7d2.' | -| `font_jbmono8_aa_H512` | font | 3/4 | 1 | font=jbmono8; hex→'a49c1e0b7d2' | -| `prompt_ocr_hint` | prompt | 3/4 | 1 | prompt=ocr_hint; hex→'a3f9c1ee0b7d2' | -| `prompt_transcribe` | prompt | 3/4 | 1 | prompt=transcribe; hex→'a3f9c1ee0b7d' | -| `realish_reflow_on` | realish | 3/4 | 1 | reflow; hex→'a5f5ceabd2d2' | -| `shipped_short_bulk40` | baseline_short | 3/4 | 1 | port→'97821' | -| `reflow_off_shipped` | reflow | 2/4 | 0 | camel→'NOT STATED'; path→'NOT STATED' | -| `shipped_aa+grid4_H512_n1` | stability | 2/4 | 0 | camel→'NOT STATED'; path→'NOT STATED' | -| `shipped_aa+grid4_H512_n2` | stability | 2/4 | 0 | camel→'NOT STATED'; path→'NOT STATED' | -| `shipped_aa+grid4_H512_n3` | stability | 2/4 | 0 | camel→'NOT STATED'; path→'NOT STATED' | -| `font_unifont8_aa_H512` | font | 2/4 | 1 | font=unifont8; hex→'a98f4e8b0d6c'; camel→'NOT STATED' | -| `detail_high` | detail | 2/4 | 2 | detail=high; hex→'a3f9c1eeb7d2'; port→'97821' | -| `detail_original` | detail | 2/4 | 2 | hex→'a3f9c1eeb7d2'; port→'97821' | -| `font_spleen5x8_aa+grid4_H512` | font | 2/4 | 2 | font=spleen5x8; hex→'a3f9c1ee0b7d2e'; port→'97821' | -| `font_spleen5x8_aa_H512` | font | 2/4 | 2 | font=spleen5x8; hex→'a3f9c1eeb7d2'; port→'97821' | -| `prompt_strict` | prompt | 1/4 | 1 | hex→'a3f9c1ee0b7d2e'; camel→'NOT STATED'; path→'NOT STATED' | -| `font_unifont8_aa+grid4_H512` | font | 1/4 | 2 | font=unifont8; hex→'a9b1c3d4e5f6'; camel→'NOT STATED'; port→'41821' | -| `multicol_64x2` | multicol | 1/4 | 2 | hex→'a9fc6eb07d2e'; camel→'NOT STATED'; port→'97021' | -| `realish_prompt_ocr` | realish | 1/4 | 2 | prompt=ocr_hint; hex→'7f3a9c2e1b8d'; camel→'NOT STATED'; port→'47621' | -| `realish_shipped` | realish | 1/4 | 2 | hex→'NOT STATED'; camel→'customer_id'; port→'4721' | -| `multicol_70x2` | multicol | 1/4 | 3 | hex→'a39fc1e0b7d2'; camel→'pathsrc/core/anthropic-vision.ts port=78'; port→'7821' | -| `reflow_on_shipped` | reflow | 0/4 | 2 | reflow; hex→'a5f0a2e9b2c1'; camel→'NOT STATED'; path→'NOT STATED'; port→'7881' | -| `ydilate1_H512` | ydilate | 0/4 | 2 | hex→'NOT STATED'; camel→'NOT STATED'; path→'/token-edgeshard/pathways/core/authops/c'; port→'8080' | -| `ydilate1_grid4_H512` | ydilate | 0/4 | 2 | hex→'NOT STATED'; camel→'NOT STATED'; path→'/token-edge-shard-pathways/core/auth-spe'; port→'8080' | - -### Residual takeaways - -- **Stable shipped packing** (`aa+grid4_H512`, n=3): **2/4 exact, 0 confab** — hex+port; camel/path abstain. -- **Best residual lifts to 3/4 (each with 1 confab, n=1):** `prompt_ocr_hint`, `prompt_transcribe`, `detail_auto`, `font_jbmono8_*`, `realish_reflow_on`, `shipped_short_bulk40`. -- OCR/transcribe prompts uniquely recover **camelCase** under pure-image; hex is the usual confab (extra/missing nibble). -- **Fonts:** jbmono8 ≈ 3/4 c1; spleen5x8 ≈ 2/4 c2; unifont8 ≤2/4. No font clears stable 4/4. -- **Worse / no-gain:** vertical dilate 0/4; synthetic reflow_on 0/4; multicol ≤1/4; realish without reflow / realish+ocr_hint ≈ 1/4. -- Super-res 2× skipped (2× of 768 exceeds provider short-side floor). -- **No residual arm is a stable pure-image 4/4 at 5×8.** Do not ship a 4/4 claim. -- Strongest non-profile lever for a careful production pure-image instruction string: **OCR-hint** (or silent-transcribe) beside images — not a factsheet. -- Known pure-image 4/4 fallback remains **9×12** via `PXPIPE_GPT_PROFILES`. - -## Shipping pure-image 4/4 (2026-07-11) - -Brute-force result on **grok-4.5** (5.4 not on gateway): - -- **Layout:** pre-render `appendIdsBlock` — isolates hex/camel/path/port on their own image rows -- **Packing:** Spleen 5×8, cols 152, maxH 512, `{ aa: true }` white, **no grid** -- **Stability:** `five-by-eight-ids-block-white-stability.json` — **7/7** full 4/4 pure-image passes -- Wired into production Grok profile + slab/history render paths - -See `VISUAL_5X8_SOLUTION.md`. - diff --git a/eval/grok-density/QUALITY_RESULTS.md b/eval/grok-density/QUALITY_RESULTS.md index 2f97e78..9fd389b 100644 --- a/eval/grok-density/QUALITY_RESULTS.md +++ b/eval/grok-density/QUALITY_RESULTS.md @@ -1,71 +1,28 @@ -# Grok quality suite — results (2026-07-11) +# Grok 4.5 quality results -**Harness split** +Model: `grok-4.5` through the Codex Responses provider. Image calls bypassed +pxpipe and used the production 5×8 profile. -| Family | Eval transport | -|---|---| -| Grok | **Codex path** — Responses via Codex provider (`OPENAI_BASE_URL` / ocproxy `:8082`) | -| Fable / Opus | **Claude** CLI | +| test | text | production image | notes | +|---|---:|---:|---| +| novel arithmetic, N=100 | 100/100 | 82/100 | pure image 83/100 | +| gist recall | 98/98 | 83/98 | no transport errors | +| state tracking | 18/18 | 13/18 | subset of the gist corpus | +| never-stated guards | 0/16 confabulated | 0/16 confabulated | lower is better | +| dense 12-char hex | 15/15 | 0/6 completed | 9 image calls failed at the gateway or timed out | -## Live fix matrix (`fix-matrix.mjs`) +Novel-arithmetic input usage was recovered from the provider log for the same +N=100 run: 25,400 text tokens and 27,100 production-image tokens, **+6.7%**. +The README rounds this to **+7%**. This short workload costs more as images. -Fixed fixture; production 5×8 profile unless noted. Model `grok-4.5`. +Receipts: -| arm | exact | confab | gist | guard | result | -|---|---:|---:|---|---|---| -| A pure image, no IDS | 2/4 | 1 | ok | ok | **FAIL** | -| B pure image + IDS | 2/4 | 2 | ok | ok | **FAIL** | -| **C IDS + text factsheet** | **4/4** | **0** | ok | ok | **PASS** | -| D 9×12 + IDS, pure image | 3/4 | 1 | ok | ok | **FAIL** | -| **E 9×12 + IDS + factsheet** | **4/4** | **0** | ok | ok | **PASS** | +- `../sol-profile/model-grok-4.5-novel-arithmetic-results.json` +- `../sol-profile/gist-recall-grok-4.5-results.json` +- `../sol-profile/gist-recall-grok-4.5-text-results.json` +- `../sol-profile/verbatim-hex-grok-4.5-results.json` +- `../sol-profile/verbatim-hex-grok-4.5-text-results.json` -Receipt: `fix-matrix-results.json` / `/tmp/grok-fix-matrix3.log`. - -### Conclusion from matrix - -- **Pure-image exact is not Fable-grade** on live Grok — IDS alone does not stabilize hex/port. -- **Lower density (9×12) is not enough** without factsheet (3/4). -- **Text factsheet + images is the working fix** (4/4, 0 confab) — and **production already does this** on the Grok/Responses transform (`factSheetText` on slab + history). - -## Multi-seed IDs with production path (`WITH_FACTSHEET=1`) - -`multi-seed-ids.mjs`, N=3, seed=20260711, random hex/camel/path/port per seed, IDS + factsheet. - -| seed | exact | confab | gist | guard | -|---|---:|---:|---|---| -| seed_1 | **4/4** | 0 | ok | ok | -| seed_2 | **4/4** | 0 | ok | ok | -| seed_3 | **4/4** | 0 | ok | ok | -| **total** | **3/3 full pass** | | | | - -Receipt: `multi-seed-ids-results.json` (generatedAt 2026-07-11T03:35:38Z), log `/tmp/grok-ms-fs2.log`. - -Without factsheet (earlier pure-image multi-seed), seeds failed **2/4** on hex/camel confab. - -## What this means for “is Grok Fable-level?” - -| Claim | Status | -|---|---| -| Grok pure-image OCR matches Fable dense reading | **No** — live pure-image fails exact | -| Grok production path (image + factsheet) exact IDs | **Yes on n=3 multi-seed + matrix** — same mitigation Fable uses | -| Grok novel arithmetic N=100 | **Not finished** this session | -| Density ~6.1× | Packing math only, not quality | - -## Product stance (honest) - -1. **Do not sell pure-image 7/7 as the production bar** — that was a research battery; live pure-image is unstable. -2. **When Grok is enabled, production is image + factsheet** (wired in `src/core/openai.ts`). That is the fix that works, not another glyph trick. -3. **Fable still leads** on full quality suite. **Grok is opt-in only** (`DEFAULT_MODEL_BASES = claude-fable-5`) until quality matches Fable. - -## How to re-run - -```bash -export OPENAI_BASE_URL=http://127.0.0.1:8082/v1 # Codex ocproxy -export OPENAI_API_KEY=… - -pnpm run build -GROK_DENSITY_LIVE=1 node eval/grok-density/fix-matrix.mjs -GROK_DENSITY_LIVE=1 N=10 node eval/grok-density/multi-seed-ids.mjs # factsheet on by default -GROK_DENSITY_LIVE=1 WITH_FACTSHEET=0 N=10 node eval/grok-density/multi-seed-ids.mjs # pure-image research -GROK_DENSITY_LIVE=1 N=20 node eval/grok-density/novel-arithmetic.mjs -``` +Grok remains opt-in because arithmetic, gist, and state tracking are below the +Fable bar. The incomplete hex run is reported with its completed-call +denominator rather than counting transport failures as model misses. diff --git a/eval/grok-density/QUALITY_SUITE.md b/eval/grok-density/QUALITY_SUITE.md index 5db1503..9aefabb 100644 --- a/eval/grok-density/QUALITY_SUITE.md +++ b/eval/grok-density/QUALITY_SUITE.md @@ -1,71 +1,21 @@ -# Grok quality suite (Codex / Responses path) +# Grok quality suite -**Harness split (this repo):** - -| Family | How we call the model for evals | -|---|---| -| **Grok** (`grok-4.5`, …) | **Codex path** — OpenAI-compatible **Responses** via the same provider Codex uses (`OPENAI_BASE_URL`, typically ocproxy `http://127.0.0.1:8082/v1`) | -| **Fable / Opus** | **Claude** — `claude` CLI / `eval/lib/cci.py` (Anthropic Messages) | - -Grok harnesses do **not** use the Claude CLI. They POST images to `/v1/responses` the same way Codex’s `model_provider` does. Do **not** point them at pxpipe (`:47821`); measure raw image reading. - -## Production recipe under test - -```text -Spleen 5×8 · cols 152 · maxH 512 · { aa: true, grid: false } · appendIdsBlock -``` - -## Env (Codex / ocproxy) +Grok is evaluated through the OpenAI-compatible Responses endpoint used by +Codex. Fable and Opus use the Claude harnesses. ```bash -# Same stack Codex uses for Grok (see ~/.codex/config.toml model_provider=ocproxy) -export OPENAI_BASE_URL=http://127.0.0.1:8082/v1 # or your Codex provider base -export OPENAI_API_KEY=… # forwarded by ocproxy / provider -export GROK_DENSITY_MODEL=grok-4.5 -``` +export OPENAI_BASE_URL=http://127.0.0.1:8082/v1 +export OPENAI_API_KEY=… +export SOL_QUALITY_MODEL=grok-4.5 +export SOL_QUALITY_LIVE=1 -## 1. Multi-seed pure-image IDs - -Random hex / camel / path / port per seed; scores exact 4/4 + gist + guard. - -```bash pnpm run build -GROK_DENSITY_LIVE=1 N=10 node eval/grok-density/multi-seed-ids.mjs -# fuller -GROK_DENSITY_LIVE=1 N=20 node eval/grok-density/multi-seed-ids.mjs +N=100 node eval/sol-profile/novel-arithmetic.mjs +node eval/sol-profile/gist-recall.mjs +node eval/sol-profile/verbatim-hex.mjs ``` -Writes `multi-seed-ids-results.json`. - -## 2. Novel arithmetic (text vs image) - -Fresh random-number word problems (not GSM8K). Image arm sees **only** the PNG. -Same math idea as `eval/gsm8k/` for Fable, but Grok is scored over **Responses**, not Claude. - -```bash -pnpm run build -GROK_DENSITY_LIVE=1 N=20 node eval/grok-density/novel-arithmetic.mjs -# Fable-comparable N -GROK_DENSITY_LIVE=1 N=100 CONCURRENCY=2 node eval/grok-density/novel-arithmetic.mjs -``` - -Writes `novel-arithmetic-results.json`. - -## 3. Shipped short/bulk smoke - -```bash -GROK_DENSITY_LIVE=1 node eval/grok-density/five-by-eight-shipped.mjs -``` - -## Why not `codex exec` for every probe? - -Codex interactive/`exec` is the agent shell. These evals need **controlled vision -inputs** (fixed PNGs + fixed questions) and cheap parallel scoring. Hitting the -**same Responses base URL Codex uses** is the right layer: same model, same -provider, no agent loop noise. Fable keeps using Claude because its quality -receipts were collected that way. - -## Acceptance (same spirit as opus-density / Fable) - -- Multi-seed IDs: high rate of **4/4 exact, 0 confab, gist ok, guard ok** -- Novel arithmetic: image accuracy near text baseline on N≥20 (target N=100) +The harnesses post directly to the provider and reject pxpipe's local port, so +the measured images are not transformed again. They use the model's resolved +production profile: Spleen 5×8, IDS rows, and the adjacent factsheet where the +production path supplies one. diff --git a/eval/grok-density/README.md b/eval/grok-density/README.md index ab984bc..415b669 100644 --- a/eval/grok-density/README.md +++ b/eval/grok-density/README.md @@ -89,7 +89,6 @@ GROK_DENSITY_LIVE=1 node eval/grok-density/factsheet-vs-image.mjs See [FACTSHEET_RESULTS.md](./FACTSHEET_RESULTS.md). -Pure-image 5×8 retune: [FIVE_BY_EIGHT_PURE_RESULTS.md](./FIVE_BY_EIGHT_PURE_RESULTS.md). ## Full quality suite (Codex path) diff --git a/eval/grok-density/VISUAL_5X8_SOLUTION.md b/eval/grok-density/VISUAL_5X8_SOLUTION.md deleted file mode 100644 index 13678d1..0000000 --- a/eval/grok-density/VISUAL_5X8_SOLUTION.md +++ /dev/null @@ -1,62 +0,0 @@ -# Visual-only 5×8 pure-image solution (brute-force result) - -## Solution - -```text -Packing: production Spleen 5×8, cols=152, maxH=512 -Style: { aa: true } # white paper, no grid, no paperGray -Layout: pre-render IDS block (rasterized into the PNG): - - IDS - hex <12-char-hex> - camel - path - port - -Channel: pure image only (no factsheet) -``` - -## Verified on grok-4.5 - -Receipt: `five-by-eight-ids-block-white-stability.json` - -### **Pass rate: 7/7 (1) — all 4/4 exact, 0 confab** - -| run | exact | confab | pass | hex | camel | hex got | -|-----|------:|-------:|:---:|:---:|:-----:|---------| -| `ids_block_white_n1` | 4/4 | 0 | Y | Y | Y | `a3f9c1e0b7d2` | -| `ids_block_white_n2` | 4/4 | 0 | Y | Y | Y | `a3f9c1e0b7d2` | -| `ids_block_white_n3` | 4/4 | 0 | Y | Y | Y | `a3f9c1e0b7d2` | -| `ids_block_white_n4` | 4/4 | 0 | Y | Y | Y | `a3f9c1e0b7d2` | -| `ids_block_white_n5` | 4/4 | 0 | Y | Y | Y | `a3f9c1e0b7d2` | -| `ids_block_white_n6` | 4/4 | 0 | Y | Y | Y | `a3f9c1e0b7d2` | -| `ids_block_white_n7` | 4/4 | 0 | Y | Y | Y | `a3f9c1e0b7d2` | - -Kitchen discovery arm also passed once: `prod__aa_nogrid_p240__ids_block` (p240 variant; white preferred in retests). - -## Negative results from brute force - -| class | outcome | -|-------|---------| -| classTick / classColor / legend | no 4/4; often hurt hex | -| hexdisc hand bitmaps | no hex exact | -| jbss SS4/SS8 true AA fonts | best 3/4 camel; hex never | -| hybrid hex SS into Spleen | same 3/4 camel pattern | -| paperGray 240 no-grid + ids_block | port confab 47821→47021 | -| residual dilate/multicol/ydilate | losers | - -## Why it works - -- Isolating IDs on their own lines reduces dense-line nibble merge at 5×8 -- Still pure visual: only PNG inputs at ask time -- Stock Spleen 5×8 density preserved - -## Goal audit - -| requirement | status | -|-------------|:------:| -| Visual-only | YES | -| 5×8 | YES | -| Pure-image 4/4 | YES (7/7 white+ids_block) | -| Found by brute force | YES | -| Grok 5.4 | NO model on gateway; validated on **grok-4.5** | diff --git a/eval/grok-density/five-by-eight-ids-block-white-stability.json b/eval/grok-density/five-by-eight-ids-block-white-stability.json deleted file mode 100644 index 5b2d24b..0000000 --- a/eval/grok-density/five-by-eight-ids-block-white-stability.json +++ /dev/null @@ -1,592 +0,0 @@ -{ - "generatedAt": "2026-07-11T02:33:11.119Z", - "model": "grok-4.5", - "pureImage": true, - "note": "White paper + ids_block stability only", - "rows": [ - { - "id": "ids_block_white_n1", - "style": { - "aa": true - }, - "maxH": 512, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "a3f9c1e0b7d2", - "got": "a3f9c1e0b7d2", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "camel", - "kind": "exact", - "expected": "tokenLedgerShard", - "got": "tokenLedgerShard", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/anthropic-vision.ts", - "got": "src/core/anthropic-vision.ts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "port", - "kind": "exact", - "expected": "47821", - "got": "47821", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "gist", - "kind": "gist", - "expected": "3", - "got": "3 attempts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "confab": false, - "abstained": true - } - ], - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - } - }, - { - "id": "ids_block_white_n2", - "style": { - "aa": true - }, - "maxH": 512, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "a3f9c1e0b7d2", - "got": "a3f9c1e0b7d2", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "camel", - "kind": "exact", - "expected": "tokenLedgerShard", - "got": "tokenLedgerShard", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/anthropic-vision.ts", - "got": "src/core/anthropic-vision.ts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "port", - "kind": "exact", - "expected": "47821", - "got": "47821", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "gist", - "kind": "gist", - "expected": "3", - "got": "3", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "confab": false, - "abstained": true - } - ], - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - } - }, - { - "id": "ids_block_white_n3", - "style": { - "aa": true - }, - "maxH": 512, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "a3f9c1e0b7d2", - "got": "a3f9c1e0b7d2", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "camel", - "kind": "exact", - "expected": "tokenLedgerShard", - "got": "tokenLedgerShard", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/anthropic-vision.ts", - "got": "src/core/anthropic-vision.ts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "port", - "kind": "exact", - "expected": "47821", - "got": "47821", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "gist", - "kind": "gist", - "expected": "3", - "got": "3 attempts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "confab": false, - "abstained": true - } - ], - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - } - }, - { - "id": "ids_block_white_n4", - "style": { - "aa": true - }, - "maxH": 512, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "a3f9c1e0b7d2", - "got": "a3f9c1e0b7d2", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "camel", - "kind": "exact", - "expected": "tokenLedgerShard", - "got": "tokenLedgerShard", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/anthropic-vision.ts", - "got": "src/core/anthropic-vision.ts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "port", - "kind": "exact", - "expected": "47821", - "got": "47821", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "gist", - "kind": "gist", - "expected": "3", - "got": "3 attempts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "confab": false, - "abstained": true - } - ], - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - } - }, - { - "id": "ids_block_white_n5", - "style": { - "aa": true - }, - "maxH": 512, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "a3f9c1e0b7d2", - "got": "a3f9c1e0b7d2", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "camel", - "kind": "exact", - "expected": "tokenLedgerShard", - "got": "tokenLedgerShard", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/anthropic-vision.ts", - "got": "src/core/anthropic-vision.ts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "port", - "kind": "exact", - "expected": "47821", - "got": "47821", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "gist", - "kind": "gist", - "expected": "3", - "got": "3 attempts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "confab": false, - "abstained": true - } - ], - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - } - }, - { - "id": "ids_block_white_n6", - "style": { - "aa": true - }, - "maxH": 512, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "a3f9c1e0b7d2", - "got": "a3f9c1e0b7d2", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "camel", - "kind": "exact", - "expected": "tokenLedgerShard", - "got": "tokenLedgerShard", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/anthropic-vision.ts", - "got": "src/core/anthropic-vision.ts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "port", - "kind": "exact", - "expected": "47821", - "got": "47821", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "gist", - "kind": "gist", - "expected": "3", - "got": "3 attempts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "confab": false, - "abstained": true - } - ], - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - } - }, - { - "id": "ids_block_white_n7", - "style": { - "aa": true - }, - "maxH": 512, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "a3f9c1e0b7d2", - "got": "a3f9c1e0b7d2", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "camel", - "kind": "exact", - "expected": "tokenLedgerShard", - "got": "tokenLedgerShard", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/anthropic-vision.ts", - "got": "src/core/anthropic-vision.ts", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "port", - "kind": "exact", - "expected": "47821", - "got": "47821", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "gist", - "kind": "gist", - "expected": "3", - "got": "3", - "ok": true, - "confab": false, - "abstained": false - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "confab": false, - "abstained": true - } - ], - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - } - } - ], - "passRate": 1, - "ranked": [ - { - "id": "ids_block_white_n1", - "exact": 4, - "confab": 0, - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - }, - { - "id": "ids_block_white_n2", - "exact": 4, - "confab": 0, - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - }, - { - "id": "ids_block_white_n3", - "exact": 4, - "confab": 0, - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - }, - { - "id": "ids_block_white_n4", - "exact": 4, - "confab": 0, - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - }, - { - "id": "ids_block_white_n5", - "exact": 4, - "confab": 0, - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - }, - { - "id": "ids_block_white_n6", - "exact": 4, - "confab": 0, - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - }, - { - "id": "ids_block_white_n7", - "exact": 4, - "confab": 0, - "pass": true, - "hexOk": true, - "camelOk": true, - "hexGot": "a3f9c1e0b7d2" - } - ] -} \ No newline at end of file diff --git a/eval/grok-density/fix-matrix-results.json b/eval/grok-density/fix-matrix-results.json deleted file mode 100644 index fc58beb..0000000 --- a/eval/grok-density/fix-matrix-results.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "generatedAt": "from-log-2026-07-11", - "model": "grok-4.5", - "arms": [ - { - "name": "A_pure_no_ids", - "exact": 2, - "confab": 1, - "gist": true, - "guard": true, - "pass": false - }, - { - "name": "B_pure_ids", - "exact": 2, - "confab": 2, - "gist": true, - "guard": true, - "pass": false - }, - { - "name": "C_ids_plus_factsheet", - "exact": 4, - "confab": 0, - "gist": true, - "guard": true, - "pass": true - }, - { - "name": "D_9x12_ids_pure", - "exact": 3, - "confab": 1, - "gist": true, - "guard": true, - "pass": false - }, - { - "name": "E_9x12_ids_factsheet", - "exact": 4, - "confab": 0, - "gist": true, - "guard": true, - "pass": true - } - ] -} diff --git a/eval/grok-density/fix-matrix.mjs b/eval/grok-density/fix-matrix.mjs deleted file mode 100644 index 5711874..0000000 --- a/eval/grok-density/fix-matrix.mjs +++ /dev/null @@ -1,79 +0,0 @@ -// Live recipe matrix for grok-4.5 on Codex Responses path. -// GROK_DENSITY_LIVE=1 node eval/grok-density/fix-matrix.mjs -import { writeFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { renderTextToPngs } from '../../dist/core/render.js'; -import { resolveGptProfile } from '../../dist/core/gpt-model-profiles.js'; -import { appendIdsBlock, factSheetText } from '../../dist/core/factsheet.js'; -import { callResponses, pngsToDataUrls, profileStyle } from './lib-responses.mjs'; - -const here = dirname(fileURLToPath(import.meta.url)); -const MODEL = process.env.GROK_DENSITY_MODEL || 'grok-4.5'; -const LIVE = process.env.GROK_DENSITY_LIVE === '1'; -const p = resolveGptProfile(MODEL); -const STYLE = profileStyle(p); -const TRUTH = { hex:'a3f9c1e0b7d2', camel:'tokenLedgerShard', path:'src/core/anthropic-vision.ts', port:'47821' }; -const BODY = [ - 'Wire up the retry path. Use a retry budget of 3 attempts, backing off 250ms.', - `Done. The token cache key is ${TRUTH.hex}. I renamed the field to ${TRUTH.camel}`, - `and moved the tier math into ${TRUTH.path}. Proxy stays on port ${TRUTH.port}.`, - 'Good. Keep the retry budget as decided.', - ...Array.from({length:40},(_,i)=>`step ${i}: processed shard ${i} of the ${TRUTH.camel}, ok.`), -].join('\n'); -const QS = [ - ['hex', TRUTH.hex, 'What is the exact token cache key (12-char hex)?'], - ['camel', TRUTH.camel, 'What is the exact field name renamed to?'], - ['path', TRUTH.path, 'What exact file path did tier math move into?'], - ['port', TRUTH.port, 'What port does the proxy stay on?'], - ['gist', '3', 'What retry budget was decided (number of attempts)?'], - ['guard', 'NOT STATED', 'What database password was configured? If not stated, say NOT STATED.'], -]; -function score(id, exp, got) { - if (!got) return { ok:false, confab:false }; - if (id === 'guard') return { ok: /not stated/i.test(got), confab: !/not stated/i.test(got) }; - if (id === 'gist') return { ok: got.includes(exp), confab: false }; - const ok = got.includes(exp); - return { ok, confab: !ok && !/not stated/i.test(got) }; -} -async function run(name, { cols, style, maxH, text, withFactsheet }) { - const imgs = await renderTextToPngs(text, cols, style, maxH); - const urls = pngsToDataUrls(imgs); - const fs = withFactsheet ? factSheetText(BODY) : ''; - console.log(`\n=== ${name} pages=${imgs.length} dims=${imgs.map(i=>i.width+'x'+i.height).join(',')} fs=${!!fs} ===`); - if (!LIVE) return { name, dry:true, pages:imgs.length }; - let exact=0, confab=0, gist=false, guard=false; - const answers=[]; - for (const [id, exp, q] of QS) { - const content = [...urls.map(u=>({type:'input_image', image_url:u, detail:'original'}))]; - if (fs) content.push({ type:'input_text', text: fs }); - content.push({ type:'input_text', text: q + '\nAnswer with ONLY the exact value, or NOT STATED. Prefer the factsheet if present for exact IDs. Do not guess.' }); - try { - const r = await callResponses({ model: MODEL, content, maxOutputTokens: 256, timeoutMs: 180000 }); - const s = score(id, exp, r.text); - if (['hex','camel','path','port'].includes(id)) { if (s.ok) exact++; if (s.confab) confab++; } - if (id==='gist') gist = s.ok; - if (id==='guard') guard = s.ok; - answers.push({ id, exp, got:r.text, ...s, ms:r.ms }); - console.log(` ${id}: ok=${s.ok} got=${JSON.stringify(r.text)}`); - } catch (e) { - answers.push({ id, exp, got:'', error:String(e.message||e), ok:false, confab:false }); - console.log(` ${id}: ERROR ${e.message||e}`); - } - } - const pass = exact===4 && confab===0 && gist && guard; - console.log(` → exact ${exact}/4 confab ${confab} gist ${gist} guard ${guard} ${pass?'PASS':'FAIL'}`); - return { name, exact, confab, gist, guard, pass, answers, pages: imgs.length }; -} -const cols912 = Math.floor((768 - 8) / 9); -const style912 = { ...STYLE, cellWBonus: 4, cellHBonus: 4 }; -const arms = []; -if (!LIVE) { console.log('set GROK_DENSITY_LIVE=1'); process.exit(0); } -arms.push(await run('A_pure_no_ids', { cols:p.stripCols, style:STYLE, maxH:p.maxHeightPx, text:BODY, withFactsheet:false })); -arms.push(await run('B_pure_ids', { cols:p.stripCols, style:STYLE, maxH:p.maxHeightPx, text:appendIdsBlock(BODY), withFactsheet:false })); -arms.push(await run('C_ids_plus_factsheet', { cols:p.stripCols, style:STYLE, maxH:p.maxHeightPx, text:appendIdsBlock(BODY), withFactsheet:true })); -arms.push(await run('D_9x12_ids_pure', { cols:cols912, style:style912, maxH:1932, text:appendIdsBlock(BODY), withFactsheet:false })); -arms.push(await run('E_9x12_ids_factsheet', { cols:cols912, style:style912, maxH:1932, text:appendIdsBlock(BODY), withFactsheet:true })); -console.log('\n==== MATRIX ===='); -for (const a of arms) console.log(a.name, a.pass===undefined?'dry':`exact ${a.exact}/4 confab ${a.confab} ${a.pass?'PASS':'FAIL'}`); -writeFileSync(join(here,'fix-matrix-results.json'), JSON.stringify({ generatedAt:new Date().toISOString(), model:MODEL, arms }, null, 2)); diff --git a/eval/grok-density/lib-responses.mjs b/eval/grok-density/lib-responses.mjs deleted file mode 100644 index df98298..0000000 --- a/eval/grok-density/lib-responses.mjs +++ /dev/null @@ -1,88 +0,0 @@ -// Shared OpenAI-compatible Responses helpers for Grok image evals. -// Grok is evaluated on the Codex path: OPENAI_BASE_URL should be the same -// provider base Codex uses (e.g. ocproxy http://127.0.0.1:8082/v1). -// Fable/Opus stay on Claude CLI. Do not route through pxpipe — raw image reading. - -export function responsesBaseUrl() { - const base = (process.env.OPENAI_BASE_URL || '').replace(/\/$/, ''); - if (!base) throw new Error('OPENAI_BASE_URL is required for live runs'); - return base.endsWith('/responses') ? base : `${base}/responses`; -} - -export async function callResponses({ model, content, maxOutputTokens = 512, timeoutMs = 180_000 }) { - const key = process.env.OPENAI_API_KEY; - if (!key) throw new Error('OPENAI_API_KEY is required for live runs'); - const c = new AbortController(); - const t = setTimeout(() => c.abort(), timeoutMs); - const t0 = Date.now(); - try { - const res = await fetch(responsesBaseUrl(), { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${key}`, - }, - body: JSON.stringify({ - model, - stream: false, - max_output_tokens: maxOutputTokens, - input: [{ role: 'user', content }], - }), - signal: c.signal, - }); - const raw = await res.text(); - let j; - try { j = JSON.parse(raw); } catch { j = { raw }; } - if (!res.ok) { - throw new Error(`HTTP ${res.status}: ${j?.error?.message || raw.slice(0, 200)}`); - } - let text = typeof j.output_text === 'string' ? j.output_text : ''; - if (!text && Array.isArray(j.output)) { - for (const item of j.output) { - if (!item || item.type !== 'message' || !Array.isArray(item.content)) continue; - for (const part of item.content) { - if (part && (part.type === 'output_text' || part.type === 'text') && typeof part.text === 'string') { - text += part.text; - } - } - } - } - return { text: text.trim(), ms: Date.now() - t0, raw: j }; - } finally { - clearTimeout(t); - } -} - -export function pngsToDataUrls(imgs) { - return imgs.map((im) => `data:image/png;base64,${Buffer.from(im.png).toString('base64')}`); -} - -export function profileStyle(profile) { - return { - font: profile.style.font, - cellWBonus: profile.style.cellWBonus, - cellHBonus: profile.style.cellHBonus, - aa: profile.style.aa, - grid: profile.style.grid, - gridCols: profile.style.gridCols, - colorCycle: profile.style.colorCycle, - markerScale: profile.style.markerScale, - markerRed: profile.style.markerRed, - inkDilate: profile.style.inkDilate, - }; -} - -export function extractAnswerNumber(out) { - if (!out) return null; - const m = String(out).match(/ANSWER:\s*\$?(-?[\d.,]+)/i); - if (m) return numify(m[1]); - const nums = String(out).match(/-?\d[\d,]*(?:\.\d+)?/g); - return nums ? numify(nums[nums.length - 1]) : null; -} - -function numify(s) { - if (s == null) return null; - const t = String(s).replace(/,/g, '').replace(/\$/g, '').trim().replace(/\.$/, ''); - const n = Number(t); - return Number.isFinite(n) ? n : null; -} diff --git a/eval/grok-density/multi-seed-ids-results.json b/eval/grok-density/multi-seed-ids-results.json deleted file mode 100644 index 111f049..0000000 --- a/eval/grok-density/multi-seed-ids-results.json +++ /dev/null @@ -1,305 +0,0 @@ -{ - "generatedAt": "2026-07-11T03:35:38.140Z", - "model": "grok-4.5", - "live": true, - "n": 3, - "seed": 20260711, - "recipe": { - "cols": 152, - "maxH": 512, - "style": { - "font": "spleen-5x8", - "cellWBonus": 0, - "cellHBonus": 0, - "aa": true, - "grid": false, - "gridCols": 0, - "colorCycle": false, - "markerScale": 1, - "markerRed": false, - "inkDilate": 0 - }, - "idsBlock": true, - "pureImage": true, - "factsheet": true - }, - "passN": 3, - "rows": [ - { - "id": "seed_1", - "seed": 20260711, - "truth": { - "hex": "a1cbe50f2943", - "camel": "sessionFenceMark", - "path": "src/core/openai-history.ts", - "port": "9123", - "retry": 5 - }, - "pages": 1, - "dims": [ - "768x416" - ], - "imageTokens": 320, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "a1cbe50f2943", - "got": "a1cbe50f2943", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 738 - }, - { - "id": "camel", - "kind": "exact", - "expected": "sessionFenceMark", - "got": "sessionFenceMark", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 2944 - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/openai-history.ts", - "got": "src/core/openai-history.ts", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 787 - }, - { - "id": "port", - "kind": "exact", - "expected": "9123", - "got": "9123", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 606 - }, - { - "id": "gist", - "kind": "gist", - "expected": "5", - "got": "5", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 2699 - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "abstained": true, - "confab": false, - "refused": false, - "ms": 3829 - } - ], - "pass": true - } - }, - { - "id": "seed_2", - "seed": 20270684, - "truth": { - "hex": "be50f29436d8", - "camel": "denseGlyphPitch", - "path": "src/core/anthropic-vision.ts", - "port": "8082", - "retry": 2 - }, - "pages": 1, - "dims": [ - "768x416" - ], - "imageTokens": 320, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "be50f29436d8", - "got": "be50f29436d8", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 1433 - }, - { - "id": "camel", - "kind": "exact", - "expected": "denseGlyphPitch", - "got": "denseGlyphPitch", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 586 - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/anthropic-vision.ts", - "got": "src/core/anthropic-vision.ts", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 731 - }, - { - "id": "port", - "kind": "exact", - "expected": "8082", - "got": "8082", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 641 - }, - { - "id": "gist", - "kind": "gist", - "expected": "2", - "got": "2", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 728 - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "abstained": true, - "confab": false, - "refused": false, - "ms": 2596 - } - ], - "pass": true - } - }, - { - "id": "seed_3", - "seed": 20280657, - "truth": { - "hex": "cbe50f29436d", - "camel": "tokenLedgerShard", - "path": "src/core/gpt-model-profiles.ts", - "port": "47821", - "retry": 4 - }, - "pages": 1, - "dims": [ - "768x416" - ], - "imageTokens": 320, - "model": { - "exactCorrect": 4, - "exactTotal": 4, - "confab": 0, - "gistOk": true, - "guardOk": true, - "answers": [ - { - "id": "hex", - "kind": "exact", - "expected": "cbe50f29436d", - "got": "cbe50f29436d", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 3259 - }, - { - "id": "camel", - "kind": "exact", - "expected": "tokenLedgerShard", - "got": "tokenLedgerShard", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 5286 - }, - { - "id": "path", - "kind": "exact", - "expected": "src/core/gpt-model-profiles.ts", - "got": "src/core/gpt-model-profiles.ts", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 643 - }, - { - "id": "port", - "kind": "exact", - "expected": "47821", - "got": "47821", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 2301 - }, - { - "id": "gist", - "kind": "gist", - "expected": "4", - "got": "4", - "ok": true, - "abstained": false, - "confab": false, - "refused": false, - "ms": 604 - }, - { - "id": "guard", - "kind": "guard", - "expected": "NOT STATED", - "got": "NOT STATED", - "ok": true, - "abstained": true, - "confab": false, - "refused": false, - "ms": 719 - } - ], - "pass": true - } - } - ] -} \ No newline at end of file diff --git a/eval/grok-density/multi-seed-ids.mjs b/eval/grok-density/multi-seed-ids.mjs deleted file mode 100644 index b39ead9..0000000 --- a/eval/grok-density/multi-seed-ids.mjs +++ /dev/null @@ -1,176 +0,0 @@ -// Multi-seed pure-image ID stability for the shipping Grok recipe. -// Production: Spleen 5x8, 152 cols, maxH 512, white AA, no grid, appendIdsBlock. -// Live: GROK_DENSITY_LIVE=1 node eval/grok-density/multi-seed-ids.mjs -import { writeFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { renderTextToPngs } from '../../dist/core/render.js'; -import { resolveGptProfile } from '../../dist/core/gpt-model-profiles.js'; -import { visionTokensForModel } from '../../dist/core/openai.js'; -import { appendIdsBlock, factSheetText } from '../../dist/core/factsheet.js'; -import { - callResponses, - pngsToDataUrls, - profileStyle, -} from './lib-responses.mjs'; - -const here = dirname(fileURLToPath(import.meta.url)); -const MODEL = process.env.GROK_DENSITY_MODEL || 'grok-4.5'; -const LIVE = process.env.GROK_DENSITY_LIVE === '1'; -const N = Math.max(1, Number(process.env.N || 10)); -const SEED = Number(process.env.SEED || 20260711); -const TIMEOUT_MS = Number(process.env.GROK_DENSITY_TIMEOUT_MS || 180_000); -const BULK = Math.max(0, Number(process.env.BULK || 40)); // filler assistant lines -// Production Grok path always attaches a text factsheet next to images. -// Pure-image-only is opt-in for research: WITH_FACTSHEET=0 -const WITH_FACTSHEET = !/^(0|false|no|off)$/i.test(String(process.env.WITH_FACTSHEET ?? '1')); - -const profile = resolveGptProfile(MODEL); -const COLS = profile.stripCols; -const MAX_H = profile.maxHeightPx; -const STYLE = profileStyle(profile); - -// Deterministic LCG -function lcg(seed) { - let s = seed >>> 0; - return () => { - s = (Math.imul(s, 1664525) + 1013904223) >>> 0; - return s; - }; -} - -function hex12(rng) { - let h = ''; - for (let i = 0; i < 12; i++) h += (rng() % 16).toString(16); - return h; -} - -const CAMELS = [ - 'tokenLedgerShard', 'retryBudgetSeconds', 'cacheWarmCursor', 'visionPatchCap', - 'promptPrefixHash', 'imageTokenDelta', 'sessionFenceMark', 'denseGlyphPitch', -]; -const PATHS = [ - 'src/core/anthropic-vision.ts', 'src/core/openai-history.ts', 'src/core/factsheet.ts', - 'src/core/gpt-model-profiles.ts', 'eval/grok-density/run.mjs', 'scripts/gen-context-chart.ts', -]; -const PORTS = ['47821', '8082', '3001', '8443', '9123', '18080']; - -function truthForSeed(seed) { - const rng = lcg(seed); - return { - hex: hex12(rng), - camel: CAMELS[rng() % CAMELS.length], - path: PATHS[rng() % PATHS.length], - port: PORTS[rng() % PORTS.length], - retry: 2 + (rng() % 5), // 2..6 - }; -} - -function session(t) { - return [ - `Wire up the retry path. Use a retry budget of ${t.retry} attempts, backing off 250ms.`, - `Done. The token cache key is ${t.hex}. I renamed the field to ${t.camel}`, - `and moved the tier math into ${t.path}. The CLI now takes --max-visual-tokens. Proxy stays on port ${t.port}.`, - `Good. Keep the retry budget as decided; do not change the backoff.`, - ...Array.from({ length: BULK }, (_, i) => - `step ${i}: processed shard ${i} of the ${t.camel}, ok, continuing the run.`), - ].join('\n'); -} - -function questions(t) { - return [ - { id: 'hex', kind: 'exact', q: 'What is the exact token cache key (12-char hex) mentioned in the transcript?', answer: t.hex }, - { id: 'camel', kind: 'exact', q: 'What is the exact field name the assistant renamed the field to?', answer: t.camel }, - { id: 'path', kind: 'exact', q: 'What exact file path did the tier math move into?', answer: t.path }, - { id: 'port', kind: 'exact', q: 'What port does the proxy stay on?', answer: t.port }, - { id: 'gist', kind: 'gist', q: 'What retry budget was decided (a number of attempts)?', answer: String(t.retry) }, - { id: 'guard', kind: 'guard', q: 'What database password was configured in this session? If it was not stated, say "NOT STATED".', answer: 'NOT STATED' }, - ]; -} - -function score(kind, expected, got) { - if (!got) return { ok: false, abstained: false, confab: false, refused: true }; - const g = got.toLowerCase(); - const abstained = /not stated|unknown|not safe|can't|cannot|not present|i don't know|do not know/.test(g); - if (kind === 'guard') return { ok: abstained, abstained, confab: !abstained, refused: false }; - if (kind === 'gist') return { ok: g.includes(String(expected).toLowerCase()), abstained, confab: false, refused: false }; - return { ok: got.includes(expected), abstained, confab: !got.includes(expected) && !abstained, refused: false }; -} - -const rows = []; -let passN = 0; -console.log(`multi-seed IDs · model=${MODEL} · N=${N} · seed=${SEED} · live=${LIVE} · factsheet=${WITH_FACTSHEET}`); -console.log(`profile cols=${COLS} maxH=${MAX_H} style=${JSON.stringify(STYLE)}`); - -for (let i = 0; i < N; i++) { - const seed = (SEED + i * 9973) >>> 0; - const t = truthForSeed(seed); - const text = appendIdsBlock(session(t)); - const imgs = await renderTextToPngs(text, COLS, STYLE, MAX_H); - const imageTokens = imgs.reduce((n, im) => n + visionTokensForModel(MODEL, im.width, im.height), 0); - const row = { - id: `seed_${i + 1}`, - seed, - truth: t, - pages: imgs.length, - dims: imgs.map((im) => `${im.width}x${im.height}`), - imageTokens, - model: null, - }; - console.log(`\n[${row.id}] seed=${seed} pages=${imgs.length} tok=${imageTokens} hex=${t.hex} camel=${t.camel}`); - - if (LIVE) { - const dataUrls = pngsToDataUrls(imgs); - const m = { exactCorrect: 0, exactTotal: 0, confab: 0, gistOk: false, guardOk: false, answers: [] }; - for (const q of questions(t)) { - try { - const content = [ - ...dataUrls.map((u) => ({ type: 'input_image', image_url: u, detail: 'original' })), - ]; - if (WITH_FACTSHEET) { - const fs = factSheetText(session(t)); - if (fs) content.push({ type: 'input_text', text: fs }); - } - content.push({ - type: 'input_text', - text: `${q.q}\nAnswer with ONLY the exact value, or "NOT STATED" if it is not present. Prefer the factsheet if present for exact IDs. Do not guess.`, - }); - const r = await callResponses({ model: MODEL, content, timeoutMs: TIMEOUT_MS }); - const s = score(q.kind, q.answer, r.text); - if (q.kind === 'exact') { - m.exactTotal++; - if (s.ok) m.exactCorrect++; - if (s.confab) m.confab++; - } else if (q.kind === 'gist') m.gistOk = s.ok; - else if (q.kind === 'guard') m.guardOk = s.ok; - m.answers.push({ id: q.id, kind: q.kind, expected: q.answer, got: r.text, ...s, ms: r.ms }); - console.log(` ${q.id}: ${JSON.stringify(r.text)} ok=${s.ok}`); - } catch (err) { - m.answers.push({ id: q.id, kind: q.kind, expected: q.answer, got: '', error: String(err.message || err) }); - console.log(` ${q.id}: ERROR ${err.message || err}`); - if (q.kind === 'exact') m.exactTotal++; - } - } - m.pass = m.exactCorrect === 4 && m.confab === 0 && m.gistOk && m.guardOk; - if (m.pass) passN++; - row.model = m; - console.log(` → exact ${m.exactCorrect}/4 confab ${m.confab} gist ${m.gistOk} guard ${m.guardOk} ${m.pass ? '*** PASS ***' : ''}`); - } - rows.push(row); - writeFileSync(join(here, 'multi-seed-ids-results.json'), JSON.stringify({ - generatedAt: new Date().toISOString(), - model: MODEL, - live: LIVE, - n: N, - seed: SEED, - recipe: { cols: COLS, maxH: MAX_H, style: STYLE, idsBlock: true, pureImage: true, factsheet: WITH_FACTSHEET }, - passN: LIVE ? passN : null, - rows, - }, null, 2)); -} - -if (LIVE) { - console.log(`\n=== multi-seed summary: ${passN}/${N} full pass (4/4 exact, 0 confab, gist, guard) ===`); -} else { - console.log('\nDry-run only. Re-run with GROK_DENSITY_LIVE=1 to score.'); -} diff --git a/eval/grok-density/novel-arithmetic.mjs b/eval/grok-density/novel-arithmetic.mjs deleted file mode 100644 index 009c102..0000000 --- a/eval/grok-density/novel-arithmetic.mjs +++ /dev/null @@ -1,200 +0,0 @@ -// Novel random-number arithmetic: text baseline vs Grok production image arm. -// Problems cannot be memorized (fresh random integers). Wrong image answer = misread. -// -// pnpm run build -// GROK_DENSITY_LIVE=1 N=20 node eval/grok-density/novel-arithmetic.mjs -// -// Full suite: N=100 (paid). Default N=20 for a cheaper pilot. -import { writeFileSync, mkdirSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { renderTextToPngs } from '../../dist/core/render.js'; -import { resolveGptProfile } from '../../dist/core/gpt-model-profiles.js'; -import { visionTokensForModel } from '../../dist/core/openai.js'; -import { appendIdsBlock } from '../../dist/core/factsheet.js'; -import { - callResponses, - pngsToDataUrls, - profileStyle, - extractAnswerNumber, -} from './lib-responses.mjs'; - -const here = dirname(fileURLToPath(import.meta.url)); -const MODEL = process.env.GROK_DENSITY_MODEL || 'grok-4.5'; -const LIVE = process.env.GROK_DENSITY_LIVE === '1'; -const N = Math.max(1, Number(process.env.N || 20)); -const SEED = Number(process.env.SEED || 20260711); -const TIMEOUT_MS = Number(process.env.GROK_DENSITY_TIMEOUT_MS || 180_000); -const CONCURRENCY = Math.max(1, Number(process.env.CONCURRENCY || 2)); -const WITH_IDS = process.env.NO_IDS === '1' ? false : true; // production applies IDS - -const profile = resolveGptProfile(MODEL); -const COLS = profile.stripCols; -const MAX_H = profile.maxHeightPx; -const STYLE = profileStyle(profile); - -function lcg(seed) { - let s = seed >>> 0; - return () => { - s = (Math.imul(s, 1664525) + 1013904223) >>> 0; - return s; - }; -} - -function randInt(rng, lo, hi) { - return lo + (rng() % (hi - lo + 1)); -} - -function genProblems(n, seed) { - const rng = lcg(seed); - const out = []; - for (let i = 0; i < n; i++) { - const kind = rng() % 4; - let question, answer; - if (kind === 0) { - const a = randInt(rng, 1000, 9999); - const b = randInt(rng, 1000, 9999); - const c = randInt(rng, 1000, 9999); - question = `A factory produced ${a} units on Monday, ${b} units on Tuesday, and ${c} units on Wednesday. How many units did it produce in total over the three days?`; - answer = a + b + c; - } else if (kind === 1) { - const a = randInt(rng, 3000, 9999); - const b = randInt(rng, 100, 999); - const c = randInt(rng, 100, 999); - question = `A reservoir contains ${a} gallons of water. ${b} gallons are pumped out for irrigation, and later ${c} gallons of rainwater flow in. How many gallons are in the reservoir now?`; - answer = a - b + c; - } else if (kind === 2) { - const a = randInt(rng, 11, 99); - const b = randInt(rng, 11, 99); - const c = randInt(rng, 100, 999); - question = `A warehouse has ${a} shelves, each holding ${b} boxes, plus ${c} loose boxes on the floor. How many boxes are in the warehouse in total?`; - answer = a * b + c; - } else { - const a = randInt(rng, 5000, 9999); - const b = randInt(rng, 1000, 4999); - question = `A stadium has ${a} seats. ${b} of them are already sold. How many seats remain unsold?`; - answer = a - b; - } - out.push({ i, kind, question, answer }); - } - return out; -} - -async function askText(q) { - const content = [{ - type: 'input_text', - text: `Solve this math problem. Show brief reasoning, then end with exactly 'ANSWER: '.\n\n${q}`, - }]; - return callResponses({ model: MODEL, content, maxOutputTokens: 256, timeoutMs: TIMEOUT_MS }); -} - -async function askImage(dataUrls) { - const content = [ - ...dataUrls.map((u) => ({ type: 'input_image', image_url: u, detail: 'original' })), - { - type: 'input_text', - text: "A math word problem is shown in the image(s). Read the problem from the image only (do not invent numbers), solve it, then end with exactly 'ANSWER: '.", - }, - ]; - return callResponses({ model: MODEL, content, maxOutputTokens: 256, timeoutMs: TIMEOUT_MS }); -} - -async function mapPool(items, limit, fn) { - const results = new Array(items.length); - let next = 0; - async function worker() { - while (next < items.length) { - const idx = next++; - results[idx] = await fn(items[idx], idx); - } - } - await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker())); - return results; -} - -const problems = genProblems(N, SEED); -const outDir = join(here, '.work-novel'); -mkdirSync(outDir, { recursive: true }); - -console.log(`novel arithmetic · model=${MODEL} · N=${N} · seed=${SEED} · live=${LIVE} · ids=${WITH_IDS}`); -console.log(`profile cols=${COLS} maxH=${MAX_H}`); - -const rows = []; -if (!LIVE) { - // dry-run: render only - for (const p of problems) { - const body = WITH_IDS ? appendIdsBlock(p.question) : p.question; - const imgs = await renderTextToPngs(body, COLS, STYLE, MAX_H); - const imageTokens = imgs.reduce((n, im) => n + visionTokensForModel(MODEL, im.width, im.height), 0); - rows.push({ i: p.i, kind: p.kind, answer: p.answer, pages: imgs.length, imageTokens }); - console.log(`q${p.i} pages=${imgs.length} tok=${imageTokens} ans=${p.answer}`); - } - writeFileSync(join(here, 'novel-arithmetic-results.json'), JSON.stringify({ - generatedAt: new Date().toISOString(), model: MODEL, live: false, n: N, seed: SEED, withIds: WITH_IDS, - recipe: { cols: COLS, maxH: MAX_H, style: STYLE }, rows, - }, null, 2)); - console.log('Dry-run only. Re-run with GROK_DENSITY_LIVE=1 to score.'); - process.exit(0); -} - -// Live: sequential-ish pool for text then image to keep costs predictable -const scored = await mapPool(problems, CONCURRENCY, async (p) => { - const body = WITH_IDS ? appendIdsBlock(p.question) : p.question; - const imgs = await renderTextToPngs(body, COLS, STYLE, MAX_H); - const imageTokens = imgs.reduce((n, im) => n + visionTokensForModel(MODEL, im.width, im.height), 0); - const dataUrls = pngsToDataUrls(imgs); - let textGot = null, imageGot = null, textOk = false, imageOk = false, textErr = null, imageErr = null, textMs = 0, imageMs = 0; - try { - const tr = await askText(p.question); - textGot = extractAnswerNumber(tr.text); - textOk = textGot === p.answer; - textMs = tr.ms; - } catch (e) { - textErr = String(e.message || e); - } - try { - const ir = await askImage(dataUrls); - imageGot = extractAnswerNumber(ir.text); - imageOk = imageGot === p.answer; - imageMs = ir.ms; - } catch (e) { - imageErr = String(e.message || e); - } - const row = { - i: p.i, kind: p.kind, question: p.question, answer: p.answer, - pages: imgs.length, imageTokens, - textOk, imageOk, textGot, imageGot, textMs, imageMs, textErr, imageErr, - }; - console.log( - `q${p.i} text=${textOk ? 'Y' : 'N'}(${textGot}) image=${imageOk ? 'Y' : 'N'}(${imageGot}) gold=${p.answer}` + - (imageOk || textOk ? '' : ' ** miss **'), - ); - return row; -}); - -const textCorrect = scored.filter((r) => r.textOk).length; -const imageCorrect = scored.filter((r) => r.imageOk).length; -const summary = { - generatedAt: new Date().toISOString(), - model: MODEL, - live: true, - n: N, - seed: SEED, - withIds: WITH_IDS, - recipe: { cols: COLS, maxH: MAX_H, style: STYLE }, - textCorrect, - imageCorrect, - textPct: Number(((100 * textCorrect) / N).toFixed(1)), - imagePct: Number(((100 * imageCorrect) / N).toFixed(1)), - deltaPp: Number(((100 * (imageCorrect - textCorrect)) / N).toFixed(1)), - rows: scored, -}; -writeFileSync(join(here, 'novel-arithmetic-results.json'), JSON.stringify(summary, null, 2)); -console.log(`\n=== novel arithmetic N=${N} model=${MODEL} ===`); -console.log(` baseline (text) = ${textCorrect}/${N} = ${summary.textPct}%`); -console.log(` pxpipe (image) = ${imageCorrect}/${N} = ${summary.imagePct}%`); -console.log(` delta = ${summary.deltaPp >= 0 ? '+' : ''}${summary.deltaPp} pp`); -const misses = scored.filter((r) => r.textOk && !r.imageOk); -for (const m of misses.slice(0, 20)) { - console.log(` image miss q${m.i}: gold=${m.answer} text=${m.textGot} image=${m.imageGot}`); -} diff --git a/eval/sol-profile/QUALITY_RESULTS.md b/eval/sol-profile/QUALITY_RESULTS.md new file mode 100644 index 0000000..a94d40d --- /dev/null +++ b/eval/sol-profile/QUALITY_RESULTS.md @@ -0,0 +1,40 @@ +# GPT-5.6 Sol quality results + +Model: `gpt-5.6-sol` through the Codex Responses provider. Image calls bypassed +pxpipe. + +## Production 5×8 profile + +Production uses Spleen 5×8, 152 columns, max height 1932, monochrome AA, IDS +rows, and the adjacent text factsheet. + +| test | text | production image | notes | +|---|---:|---:|---| +| novel arithmetic, N=100 | 100/100 | 98/100 | pure image 96/100 | +| gist recall | not measured | 79/93 completed | one six-probe session failed at the gateway | +| state tracking | not measured | 18/18 | no transport errors | +| never-stated guards | not measured | 4/15 completed confabulated | one guard shared the failed session | +| dense 12-char hex | not run in this harness | 0/15 | all calls completed | + +Matched arithmetic usage was 5,300 text input tokens and 7,000 production-image +input tokens, **+32.1%**. The README rounds this to **+32%**. Short prompts are +not a compression win, even when the model reads them. + +The arithmetic receipt was reconstructed from the retained N=100 run log and +provider usage. Earlier JSON metadata recorded the resolved JetBrains profile +even though the run selected the Spleen candidate. The request bytes and run log +identify the actual 5×8 render; this document records that correction instead +of presenting the stale recipe field as evidence. + + +## Decision + +Sol remains opt-in. Its arithmetic and state results are strong, but gist, +abstention, and dense exact recall do not match Fable. Sibling `gpt-5.6-*` +models do not inherit Sol's profile or allowlist. + +Receipts: + +- `novel-arithmetic-spleen5x8-results.json` +- `gist-recall-results.json` +- `verbatim-hex-results.json` diff --git a/eval/sol-profile/gist-recall-grok-4.5-results.json b/eval/sol-profile/gist-recall-grok-4.5-results.json new file mode 100644 index 0000000..71add23 --- /dev/null +++ b/eval/sol-profile/gist-recall-grok-4.5-results.json @@ -0,0 +1,1290 @@ +{ + "generatedAt": "2026-07-11T20:50:08.267Z", + "model": "grok-4.5", + "live": true, + "recipe": { + "cols": 152, + "maxH": 512, + "style": { + "font": "spleen-5x8", + "cellWBonus": 0, + "cellHBonus": 0, + "aa": true, + "grid": false, + "gridCols": 0, + "colorCycle": false, + "markerScale": 1, + "markerRed": false, + "inkDilate": 0 + }, + "factsheet": true + }, + "answerable": { + "correct": 83, + "n": 98 + }, + "state": { + "correct": 13, + "n": 18 + }, + "unanswerable": { + "confabulated": 0, + "n": 16 + }, + "rows": [ + { + "tier": "work", + "session": 0, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "mobx", + "answer": "mobx", + "ok": true, + "raw": "[\"mobx\", \"7000ms\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7880", + "answer": "7000ms", + "ok": false, + "raw": "[\"mobx\", \"7000ms\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/batcher/core.ts", + "answer": "src/batcher/core.ts", + "ok": true, + "raw": "[\"mobx\", \"7000ms\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Okafor", + "answer": "Tobias Okafor", + "ok": true, + "raw": "[\"mobx\", \"7000ms\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"mobx\", \"7000ms\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"mobx\", \"7000ms\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "xstate", + "answer": "xstate", + "ok": true, + "raw": "```json\n[\"xstate\", \"9400\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "3480", + "answer": "9400", + "ok": false, + "raw": "```json\n[\"xstate\", \"9400\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/sync.ts", + "answer": "src/cursor/sync.ts", + "ok": true, + "raw": "```json\n[\"xstate\", \"9400\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Farid Moreau", + "answer": "Farid Moreau", + "ok": true, + "raw": "```json\n[\"xstate\", \"9400\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"xstate\", \"9400\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"xstate\", \"9400\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "imer", + "ok": false, + "raw": "```json\n[\"imer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "9110", + "answer": "9110", + "ok": true, + "raw": "```json\n[\"imer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "```json\n[\"imer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Nadia Costa", + "answer": "Nadia Costa", + "ok": true, + "raw": "```json\n[\"imer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "negation", + "q": "Was ENABLE_SHARDING enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"imer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "unanswerable", + "q": "What git tag was the hotfix released under?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"imer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "3470", + "answer": "3470", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/scheduler/core.ts", + "answer": "src/scheduler/core.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Dmitri Berg", + "answer": "Dmitri Berg", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "6210", + "answer": "6210", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Khoury", + "answer": "Tobias Khoury", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "negation", + "q": "Was ASYNC_FSYNC enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "1250", + "answer": "1250", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Lucia Costa", + "answer": "Lucia Costa", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"6200\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "6290", + "answer": "6200", + "ok": false, + "raw": "```json\n[\"immer\", \"6200\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/scheduler/sync.ts", + "answer": "src/scheduler/sync.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"6200\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Khoury", + "answer": "Tobias Khoury", + "ok": true, + "raw": "```json\n[\"immer\", \"6200\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"6200\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"6200\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "xstate", + "answer": "xstate", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870ms\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7870", + "answer": "7870ms", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870ms\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/batcher/sync.ts", + "answer": "src/batcher/sync.ts", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870ms\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Priya Tanaka", + "answer": "Priya Tanaka", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870ms\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870ms\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "unanswerable", + "q": "What was the Docker base image pinned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870ms\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "jotai", + "answer": "jotai", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/core/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7850", + "answer": "7850", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/core/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/retry/io.ts", + "answer": "src/core/retry/io.ts", + "ok": false, + "raw": "```json\n[\"jotai\", \"7850\", \"src/core/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Soren Petrov", + "answer": "Soren Petrov", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/core/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "negation", + "q": "Was ENABLE_SHARDING enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/core/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "unanswerable", + "q": "What port number was the staging proxy moved to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/core/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"14500\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "4540", + "answer": "14500", + "ok": false, + "raw": "```json\n[\"redux-toolkit\", \"14500\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/sync.ts", + "answer": "src/cursor/sync.ts", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"14500\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Ingrid Nakamura", + "answer": "Ingrid Nakamura", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"14500\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"14500\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "unanswerable", + "q": "What git tag was the hotfix released under?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"14500\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 0, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "nanostores", + "answer": "nanostore", + "ok": false, + "raw": "```json\n[\"nanostore\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 0, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "7850", + "answer": "7850", + "ok": true, + "raw": "```json\n[\"nanostore\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 0, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/mailbox/core.ts", + "answer": "src/mailbox/core.ts", + "ok": true, + "raw": "```json\n[\"nanostore\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 0, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Aiko Khoury", + "answer": "Aiko Khoury", + "ok": true, + "raw": "```json\n[\"nanostore\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 0, + "type": "negation", + "q": "In PROD specifically, was LEGACY_PINS enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"nanostore\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 0, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"nanostore\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 1, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "imner", + "ok": false, + "raw": "```json\n[\"imner\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 1, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "1330", + "answer": "1330", + "ok": true, + "raw": "```json\n[\"imner\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 1, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/retry/core.ts", + "answer": "src/retry/core.ts", + "ok": true, + "raw": "```json\n[\"imner\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 1, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Mara Khoury", + "answer": "Mara Khoury", + "ok": true, + "raw": "```json\n[\"imner\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 1, + "type": "negation", + "q": "In PROD specifically, was USE_BROTLI enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"imner\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 1, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"imner\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 2, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"4500ms\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 2, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "4570", + "answer": "4500ms", + "ok": false, + "raw": "```json\n[\"immer\", \"4500ms\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 2, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/quota/core.ts", + "answer": "src/quota/core.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"4500ms\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 2, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Nadia Alvarez", + "answer": "Nadia Alvarez", + "ok": true, + "raw": "```json\n[\"immer\", \"4500ms\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 2, + "type": "negation", + "q": "In PROD specifically, was ENABLE_SHARDING enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"4500ms\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 2, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"4500ms\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 3, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6200\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 3, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "6270", + "answer": "6200", + "ok": false, + "raw": "```json\n[\"redux-toolkit\", \"6200\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 3, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/batcher/core.ts", + "answer": "src/batcher/core.ts", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6200\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 3, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Farid Lindqvist", + "answer": "Farid Lindqvist", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6200\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 3, + "type": "negation", + "q": "In PROD specifically, was ASYNC_FSYNC enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6200\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 3, + "type": "unanswerable", + "q": "What was the Docker base image pinned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6200\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 4, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "zustand", + "answer": "zustand", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 4, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "9120", + "answer": "9120", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 4, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/cursor/core.ts", + "answer": "src/cursor/core.ts", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 4, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Soren Nakamura", + "answer": "Soren Nakamura", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 4, + "type": "negation", + "q": "In PROD specifically, was HOT_RELOAD_V2 enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 4, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 5, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 5, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "7800", + "answer": "7800", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 5, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/quota/core.ts", + "answer": "src/quota/core.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 5, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Ingrid Costa", + "answer": "Ingrid Costa", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 5, + "type": "negation", + "q": "In PROD specifically, was USE_BROTLI enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work2", + "session": 5, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 0, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "8400", + "answer": "9600", + "ok": false, + "raw": "[\"9600\", \"9600\", \"1\"]", + "error": null + }, + { + "tier": "work3", + "session": 0, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "9600", + "answer": "9600", + "ok": true, + "raw": "[\"9600\", \"9600\", \"1\"]", + "error": null + }, + { + "tier": "work3", + "session": 0, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "1", + "ok": false, + "raw": "[\"9600\", \"9600\", \"1\"]", + "error": null + }, + { + "tier": "work3", + "session": 1, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "1200", + "answer": "1200", + "ok": true, + "raw": "[\"1200\", \"UNKNOWN\", \"3\"]", + "error": null + }, + { + "tier": "work3", + "session": 1, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "5400", + "answer": "UNKNOWN", + "ok": false, + "raw": "[\"1200\", \"UNKNOWN\", \"3\"]", + "error": null + }, + { + "tier": "work3", + "session": 1, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "[\"1200\", \"UNKNOWN\", \"3\"]", + "error": null + }, + { + "tier": "work3", + "session": 2, + "type": "final", + "q": "What is the FINAL (locked) value of FLUSH_INTERVAL_MS at the end of the session?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 2, + "type": "first", + "q": "What was the FIRST value FLUSH_INTERVAL_MS was set to at the start?", + "gold": "2400", + "answer": "2400", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 2, + "type": "count", + "q": "How many distinct values was FLUSH_INTERVAL_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 3, + "type": "final", + "q": "What is the FINAL (locked) value of LEASE_TTL_MS at the end of the session?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 3, + "type": "first", + "q": "What was the FIRST value LEASE_TTL_MS was set to at the start?", + "gold": "2400", + "answer": "2400", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 3, + "type": "count", + "q": "How many distinct values was LEASE_TTL_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 4, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "4800", + "answer": "4800", + "ok": true, + "raw": "```json\n[\"4800\", \"7200\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 4, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "```json\n[\"4800\", \"7200\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 4, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "```json\n[\"4800\", \"7200\", \"3\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 5, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "8400", + "answer": "3600", + "ok": false, + "raw": "```json\n[\"3600\", \"3600\", \"2\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 5, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "3600", + "answer": "3600", + "ok": true, + "raw": "```json\n[\"3600\", \"3600\", \"2\"]\n```", + "error": null + }, + { + "tier": "work3", + "session": 5, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "2", + "ok": false, + "raw": "```json\n[\"3600\", \"3600\", \"2\"]\n```", + "error": null + } + ] +} \ No newline at end of file diff --git a/eval/sol-profile/gist-recall-grok-4.5-text-results.json b/eval/sol-profile/gist-recall-grok-4.5-text-results.json new file mode 100644 index 0000000..ff701b9 --- /dev/null +++ b/eval/sol-profile/gist-recall-grok-4.5-text-results.json @@ -0,0 +1,3326 @@ +{ + "generatedAt": "2026-07-11T21:00:56.437Z", + "model": "grok-4.5", + "live": true, + "arm": "text", + "answerable": { + "correct": 98, + "n": 98 + }, + "state": { + "correct": 18, + "n": 18 + }, + "unanswerable": { + "confabulated": 0, + "n": 16 + }, + "rows": [ + { + "tier": "work", + "session": 0, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "mobx", + "answer": "mobx", + "ok": true, + "raw": "```json\n[\"mobx\", \"7880\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4647, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 389, + "output_tokens_details": { + "reasoning_tokens": 354 + }, + "total_tokens": 5036, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114360000, + "context_details": { + "input_tokens": 4647, + "output_tokens": 389 + } + } + }, + { + "tier": "work", + "session": 0, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7880", + "answer": "7880", + "ok": true, + "raw": "```json\n[\"mobx\", \"7880\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4647, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 389, + "output_tokens_details": { + "reasoning_tokens": 354 + }, + "total_tokens": 5036, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114360000, + "context_details": { + "input_tokens": 4647, + "output_tokens": 389 + } + } + }, + { + "tier": "work", + "session": 0, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/batcher/core.ts", + "answer": "src/batcher/core.ts", + "ok": true, + "raw": "```json\n[\"mobx\", \"7880\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4647, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 389, + "output_tokens_details": { + "reasoning_tokens": 354 + }, + "total_tokens": 5036, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114360000, + "context_details": { + "input_tokens": 4647, + "output_tokens": 389 + } + } + }, + { + "tier": "work", + "session": 0, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Okafor", + "answer": "Tobias Okafor", + "ok": true, + "raw": "```json\n[\"mobx\", \"7880\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4647, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 389, + "output_tokens_details": { + "reasoning_tokens": 354 + }, + "total_tokens": 5036, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114360000, + "context_details": { + "input_tokens": 4647, + "output_tokens": 389 + } + } + }, + { + "tier": "work", + "session": 0, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"mobx\", \"7880\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4647, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 389, + "output_tokens_details": { + "reasoning_tokens": 354 + }, + "total_tokens": 5036, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114360000, + "context_details": { + "input_tokens": 4647, + "output_tokens": 389 + } + } + }, + { + "tier": "work", + "session": 0, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"mobx\", \"7880\", \"src/batcher/core.ts\", \"Tobias Okafor\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4647, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 389, + "output_tokens_details": { + "reasoning_tokens": 354 + }, + "total_tokens": 5036, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114360000, + "context_details": { + "input_tokens": 4647, + "output_tokens": 389 + } + } + }, + { + "tier": "work", + "session": 1, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "xstate", + "answer": "xstate", + "ok": true, + "raw": "```json\n[\"xstate\", \"3480\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4575, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 417, + "output_tokens_details": { + "reasoning_tokens": 384 + }, + "total_tokens": 4992, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114600000, + "context_details": { + "input_tokens": 4575, + "output_tokens": 417 + } + } + }, + { + "tier": "work", + "session": 1, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "3480", + "answer": "3480", + "ok": true, + "raw": "```json\n[\"xstate\", \"3480\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4575, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 417, + "output_tokens_details": { + "reasoning_tokens": 384 + }, + "total_tokens": 4992, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114600000, + "context_details": { + "input_tokens": 4575, + "output_tokens": 417 + } + } + }, + { + "tier": "work", + "session": 1, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/sync.ts", + "answer": "src/cursor/sync.ts", + "ok": true, + "raw": "```json\n[\"xstate\", \"3480\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4575, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 417, + "output_tokens_details": { + "reasoning_tokens": 384 + }, + "total_tokens": 4992, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114600000, + "context_details": { + "input_tokens": 4575, + "output_tokens": 417 + } + } + }, + { + "tier": "work", + "session": 1, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Farid Moreau", + "answer": "Farid Moreau", + "ok": true, + "raw": "```json\n[\"xstate\", \"3480\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4575, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 417, + "output_tokens_details": { + "reasoning_tokens": 384 + }, + "total_tokens": 4992, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114600000, + "context_details": { + "input_tokens": 4575, + "output_tokens": 417 + } + } + }, + { + "tier": "work", + "session": 1, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"xstate\", \"3480\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4575, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 417, + "output_tokens_details": { + "reasoning_tokens": 384 + }, + "total_tokens": 4992, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114600000, + "context_details": { + "input_tokens": 4575, + "output_tokens": 417 + } + } + }, + { + "tier": "work", + "session": 1, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"xstate\", \"3480\", \"src/cursor/sync.ts\", \"Farid Moreau\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4575, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 417, + "output_tokens_details": { + "reasoning_tokens": 384 + }, + "total_tokens": 4992, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 114600000, + "context_details": { + "input_tokens": 4575, + "output_tokens": 417 + } + } + }, + { + "tier": "work", + "session": 2, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4883, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 334, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5217, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 115780000, + "context_details": { + "input_tokens": 4883, + "output_tokens": 334 + } + } + }, + { + "tier": "work", + "session": 2, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "9110", + "answer": "9110", + "ok": true, + "raw": "```json\n[\"immer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4883, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 334, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5217, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 115780000, + "context_details": { + "input_tokens": 4883, + "output_tokens": 334 + } + } + }, + { + "tier": "work", + "session": 2, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4883, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 334, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5217, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 115780000, + "context_details": { + "input_tokens": 4883, + "output_tokens": 334 + } + } + }, + { + "tier": "work", + "session": 2, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Nadia Costa", + "answer": "Nadia Costa", + "ok": true, + "raw": "```json\n[\"immer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4883, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 334, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5217, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 115780000, + "context_details": { + "input_tokens": 4883, + "output_tokens": 334 + } + } + }, + { + "tier": "work", + "session": 2, + "type": "negation", + "q": "Was ENABLE_SHARDING enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4883, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 334, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5217, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 115780000, + "context_details": { + "input_tokens": 4883, + "output_tokens": 334 + } + } + }, + { + "tier": "work", + "session": 2, + "type": "unanswerable", + "q": "What git tag was the hotfix released under?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"9110\", \"src/cursor/io.ts\", \"Nadia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4883, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 334, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5217, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 115780000, + "context_details": { + "input_tokens": 4883, + "output_tokens": 334 + } + } + }, + { + "tier": "work", + "session": 3, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4390, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 415, + "output_tokens_details": { + "reasoning_tokens": 383 + }, + "total_tokens": 4805, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110780000, + "context_details": { + "input_tokens": 4390, + "output_tokens": 415 + } + } + }, + { + "tier": "work", + "session": 3, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "3470", + "answer": "3470", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4390, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 415, + "output_tokens_details": { + "reasoning_tokens": 383 + }, + "total_tokens": 4805, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110780000, + "context_details": { + "input_tokens": 4390, + "output_tokens": 415 + } + } + }, + { + "tier": "work", + "session": 3, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/scheduler/core.ts", + "answer": "src/scheduler/core.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4390, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 415, + "output_tokens_details": { + "reasoning_tokens": 383 + }, + "total_tokens": 4805, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110780000, + "context_details": { + "input_tokens": 4390, + "output_tokens": 415 + } + } + }, + { + "tier": "work", + "session": 3, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Dmitri Berg", + "answer": "Dmitri Berg", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4390, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 415, + "output_tokens_details": { + "reasoning_tokens": 383 + }, + "total_tokens": 4805, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110780000, + "context_details": { + "input_tokens": 4390, + "output_tokens": 415 + } + } + }, + { + "tier": "work", + "session": 3, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4390, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 415, + "output_tokens_details": { + "reasoning_tokens": 383 + }, + "total_tokens": 4805, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110780000, + "context_details": { + "input_tokens": 4390, + "output_tokens": 415 + } + } + }, + { + "tier": "work", + "session": 3, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"3470\", \"src/scheduler/core.ts\", \"Dmitri Berg\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4390, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 415, + "output_tokens_details": { + "reasoning_tokens": 383 + }, + "total_tokens": 4805, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110780000, + "context_details": { + "input_tokens": 4390, + "output_tokens": 415 + } + } + }, + { + "tier": "work", + "session": 4, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4645, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 299, + "output_tokens_details": { + "reasoning_tokens": 264 + }, + "total_tokens": 4944, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108920000, + "context_details": { + "input_tokens": 4645, + "output_tokens": 299 + } + } + }, + { + "tier": "work", + "session": 4, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "6210", + "answer": "6210", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4645, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 299, + "output_tokens_details": { + "reasoning_tokens": 264 + }, + "total_tokens": 4944, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108920000, + "context_details": { + "input_tokens": 4645, + "output_tokens": 299 + } + } + }, + { + "tier": "work", + "session": 4, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4645, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 299, + "output_tokens_details": { + "reasoning_tokens": 264 + }, + "total_tokens": 4944, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108920000, + "context_details": { + "input_tokens": 4645, + "output_tokens": 299 + } + } + }, + { + "tier": "work", + "session": 4, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Khoury", + "answer": "Tobias Khoury", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4645, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 299, + "output_tokens_details": { + "reasoning_tokens": 264 + }, + "total_tokens": 4944, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108920000, + "context_details": { + "input_tokens": 4645, + "output_tokens": 299 + } + } + }, + { + "tier": "work", + "session": 4, + "type": "negation", + "q": "Was ASYNC_FSYNC enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4645, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 299, + "output_tokens_details": { + "reasoning_tokens": 264 + }, + "total_tokens": 4944, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108920000, + "context_details": { + "input_tokens": 4645, + "output_tokens": 299 + } + } + }, + { + "tier": "work", + "session": 4, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6210\", \"src/cursor/io.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4645, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 299, + "output_tokens_details": { + "reasoning_tokens": 264 + }, + "total_tokens": 4944, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108920000, + "context_details": { + "input_tokens": 4645, + "output_tokens": 299 + } + } + }, + { + "tier": "work", + "session": 5, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4632, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 319, + "output_tokens_details": { + "reasoning_tokens": 287 + }, + "total_tokens": 4951, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 109860000, + "context_details": { + "input_tokens": 4632, + "output_tokens": 319 + } + } + }, + { + "tier": "work", + "session": 5, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "1250", + "answer": "1250", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4632, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 319, + "output_tokens_details": { + "reasoning_tokens": 287 + }, + "total_tokens": 4951, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 109860000, + "context_details": { + "input_tokens": 4632, + "output_tokens": 319 + } + } + }, + { + "tier": "work", + "session": 5, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4632, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 319, + "output_tokens_details": { + "reasoning_tokens": 287 + }, + "total_tokens": 4951, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 109860000, + "context_details": { + "input_tokens": 4632, + "output_tokens": 319 + } + } + }, + { + "tier": "work", + "session": 5, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Lucia Costa", + "answer": "Lucia Costa", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4632, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 319, + "output_tokens_details": { + "reasoning_tokens": 287 + }, + "total_tokens": 4951, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 109860000, + "context_details": { + "input_tokens": 4632, + "output_tokens": 319 + } + } + }, + { + "tier": "work", + "session": 5, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4632, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 319, + "output_tokens_details": { + "reasoning_tokens": 287 + }, + "total_tokens": 4951, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 109860000, + "context_details": { + "input_tokens": 4632, + "output_tokens": 319 + } + } + }, + { + "tier": "work", + "session": 5, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"1250\", \"src/cursor/io.ts\", \"Lucia Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4632, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 319, + "output_tokens_details": { + "reasoning_tokens": 287 + }, + "total_tokens": 4951, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 109860000, + "context_details": { + "input_tokens": 4632, + "output_tokens": 319 + } + } + }, + { + "tier": "work", + "session": 6, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"6290\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4564, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 347, + "output_tokens_details": { + "reasoning_tokens": 314 + }, + "total_tokens": 4911, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110180000, + "context_details": { + "input_tokens": 4564, + "output_tokens": 347 + } + } + }, + { + "tier": "work", + "session": 6, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "6290", + "answer": "6290", + "ok": true, + "raw": "```json\n[\"immer\", \"6290\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4564, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 347, + "output_tokens_details": { + "reasoning_tokens": 314 + }, + "total_tokens": 4911, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110180000, + "context_details": { + "input_tokens": 4564, + "output_tokens": 347 + } + } + }, + { + "tier": "work", + "session": 6, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/scheduler/sync.ts", + "answer": "src/scheduler/sync.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"6290\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4564, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 347, + "output_tokens_details": { + "reasoning_tokens": 314 + }, + "total_tokens": 4911, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110180000, + "context_details": { + "input_tokens": 4564, + "output_tokens": 347 + } + } + }, + { + "tier": "work", + "session": 6, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Khoury", + "answer": "Tobias Khoury", + "ok": true, + "raw": "```json\n[\"immer\", \"6290\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4564, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 347, + "output_tokens_details": { + "reasoning_tokens": 314 + }, + "total_tokens": 4911, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110180000, + "context_details": { + "input_tokens": 4564, + "output_tokens": 347 + } + } + }, + { + "tier": "work", + "session": 6, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"6290\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4564, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 347, + "output_tokens_details": { + "reasoning_tokens": 314 + }, + "total_tokens": 4911, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110180000, + "context_details": { + "input_tokens": 4564, + "output_tokens": 347 + } + } + }, + { + "tier": "work", + "session": 6, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"6290\", \"src/scheduler/sync.ts\", \"Tobias Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4564, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 347, + "output_tokens_details": { + "reasoning_tokens": 314 + }, + "total_tokens": 4911, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110180000, + "context_details": { + "input_tokens": 4564, + "output_tokens": 347 + } + } + }, + { + "tier": "work", + "session": 7, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "xstate", + "answer": "xstate", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4595, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 350, + "output_tokens_details": { + "reasoning_tokens": 317 + }, + "total_tokens": 4945, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110980000, + "context_details": { + "input_tokens": 4595, + "output_tokens": 350 + } + } + }, + { + "tier": "work", + "session": 7, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7870", + "answer": "7870", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4595, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 350, + "output_tokens_details": { + "reasoning_tokens": 317 + }, + "total_tokens": 4945, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110980000, + "context_details": { + "input_tokens": 4595, + "output_tokens": 350 + } + } + }, + { + "tier": "work", + "session": 7, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/batcher/sync.ts", + "answer": "src/batcher/sync.ts", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4595, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 350, + "output_tokens_details": { + "reasoning_tokens": 317 + }, + "total_tokens": 4945, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110980000, + "context_details": { + "input_tokens": 4595, + "output_tokens": 350 + } + } + }, + { + "tier": "work", + "session": 7, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Priya Tanaka", + "answer": "Priya Tanaka", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4595, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 350, + "output_tokens_details": { + "reasoning_tokens": 317 + }, + "total_tokens": 4945, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110980000, + "context_details": { + "input_tokens": 4595, + "output_tokens": 350 + } + } + }, + { + "tier": "work", + "session": 7, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4595, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 350, + "output_tokens_details": { + "reasoning_tokens": 317 + }, + "total_tokens": 4945, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110980000, + "context_details": { + "input_tokens": 4595, + "output_tokens": 350 + } + } + }, + { + "tier": "work", + "session": 7, + "type": "unanswerable", + "q": "What was the Docker base image pinned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"xstate\", \"7870\", \"src/batcher/sync.ts\", \"Priya Tanaka\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4595, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 350, + "output_tokens_details": { + "reasoning_tokens": 317 + }, + "total_tokens": 4945, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 110980000, + "context_details": { + "input_tokens": 4595, + "output_tokens": 350 + } + } + }, + { + "tier": "work", + "session": 8, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "jotai", + "answer": "jotai", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4760, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 336, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5096, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 113440000, + "context_details": { + "input_tokens": 4760, + "output_tokens": 336 + } + } + }, + { + "tier": "work", + "session": 8, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7850", + "answer": "7850", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4760, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 336, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5096, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 113440000, + "context_details": { + "input_tokens": 4760, + "output_tokens": 336 + } + } + }, + { + "tier": "work", + "session": 8, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/retry/io.ts", + "answer": "src/retry/io.ts", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4760, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 336, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5096, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 113440000, + "context_details": { + "input_tokens": 4760, + "output_tokens": 336 + } + } + }, + { + "tier": "work", + "session": 8, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Soren Petrov", + "answer": "Soren Petrov", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4760, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 336, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5096, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 113440000, + "context_details": { + "input_tokens": 4760, + "output_tokens": 336 + } + } + }, + { + "tier": "work", + "session": 8, + "type": "negation", + "q": "Was ENABLE_SHARDING enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4760, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 336, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5096, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 113440000, + "context_details": { + "input_tokens": 4760, + "output_tokens": 336 + } + } + }, + { + "tier": "work", + "session": 8, + "type": "unanswerable", + "q": "What port number was the staging proxy moved to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"jotai\", \"7850\", \"src/retry/io.ts\", \"Soren Petrov\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4760, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 336, + "output_tokens_details": { + "reasoning_tokens": 304 + }, + "total_tokens": 5096, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 113440000, + "context_details": { + "input_tokens": 4760, + "output_tokens": 336 + } + } + }, + { + "tier": "work", + "session": 9, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"4540\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4439, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 361, + "output_tokens_details": { + "reasoning_tokens": 328 + }, + "total_tokens": 4800, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108520000, + "context_details": { + "input_tokens": 4439, + "output_tokens": 361 + } + } + }, + { + "tier": "work", + "session": 9, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "4540", + "answer": "4540", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"4540\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4439, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 361, + "output_tokens_details": { + "reasoning_tokens": 328 + }, + "total_tokens": 4800, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108520000, + "context_details": { + "input_tokens": 4439, + "output_tokens": 361 + } + } + }, + { + "tier": "work", + "session": 9, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/sync.ts", + "answer": "src/cursor/sync.ts", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"4540\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4439, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 361, + "output_tokens_details": { + "reasoning_tokens": 328 + }, + "total_tokens": 4800, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108520000, + "context_details": { + "input_tokens": 4439, + "output_tokens": 361 + } + } + }, + { + "tier": "work", + "session": 9, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Ingrid Nakamura", + "answer": "Ingrid Nakamura", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"4540\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4439, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 361, + "output_tokens_details": { + "reasoning_tokens": 328 + }, + "total_tokens": 4800, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108520000, + "context_details": { + "input_tokens": 4439, + "output_tokens": 361 + } + } + }, + { + "tier": "work", + "session": 9, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"4540\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4439, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 361, + "output_tokens_details": { + "reasoning_tokens": 328 + }, + "total_tokens": 4800, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108520000, + "context_details": { + "input_tokens": 4439, + "output_tokens": 361 + } + } + }, + { + "tier": "work", + "session": 9, + "type": "unanswerable", + "q": "What git tag was the hotfix released under?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"4540\", \"src/cursor/sync.ts\", \"Ingrid Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 4439, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 361, + "output_tokens_details": { + "reasoning_tokens": 328 + }, + "total_tokens": 4800, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 108520000, + "context_details": { + "input_tokens": 4439, + "output_tokens": 361 + } + } + }, + { + "tier": "work2", + "session": 0, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "nanostores", + "answer": "nanostores", + "ok": true, + "raw": "```json\n[\"nanostores\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12304, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 485, + "output_tokens_details": { + "reasoning_tokens": 450 + }, + "total_tokens": 12789, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 273260000, + "context_details": { + "input_tokens": 12304, + "output_tokens": 485 + } + } + }, + { + "tier": "work2", + "session": 0, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "7850", + "answer": "7850", + "ok": true, + "raw": "```json\n[\"nanostores\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12304, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 485, + "output_tokens_details": { + "reasoning_tokens": 450 + }, + "total_tokens": 12789, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 273260000, + "context_details": { + "input_tokens": 12304, + "output_tokens": 485 + } + } + }, + { + "tier": "work2", + "session": 0, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/mailbox/core.ts", + "answer": "src/mailbox/core.ts", + "ok": true, + "raw": "```json\n[\"nanostores\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12304, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 485, + "output_tokens_details": { + "reasoning_tokens": 450 + }, + "total_tokens": 12789, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 273260000, + "context_details": { + "input_tokens": 12304, + "output_tokens": 485 + } + } + }, + { + "tier": "work2", + "session": 0, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Aiko Khoury", + "answer": "Aiko Khoury", + "ok": true, + "raw": "```json\n[\"nanostores\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12304, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 485, + "output_tokens_details": { + "reasoning_tokens": 450 + }, + "total_tokens": 12789, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 273260000, + "context_details": { + "input_tokens": 12304, + "output_tokens": 485 + } + } + }, + { + "tier": "work2", + "session": 0, + "type": "negation", + "q": "In PROD specifically, was LEGACY_PINS enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"nanostores\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12304, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 485, + "output_tokens_details": { + "reasoning_tokens": 450 + }, + "total_tokens": 12789, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 273260000, + "context_details": { + "input_tokens": 12304, + "output_tokens": 485 + } + } + }, + { + "tier": "work2", + "session": 0, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"nanostores\", \"7850\", \"src/mailbox/core.ts\", \"Aiko Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12304, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 485, + "output_tokens_details": { + "reasoning_tokens": 450 + }, + "total_tokens": 12789, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 273260000, + "context_details": { + "input_tokens": 12304, + "output_tokens": 485 + } + } + }, + { + "tier": "work2", + "session": 1, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12872, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 457, + "output_tokens_details": { + "reasoning_tokens": 424 + }, + "total_tokens": 13329, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 282940000, + "context_details": { + "input_tokens": 12872, + "output_tokens": 457 + } + } + }, + { + "tier": "work2", + "session": 1, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "1330", + "answer": "1330", + "ok": true, + "raw": "```json\n[\"immer\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12872, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 457, + "output_tokens_details": { + "reasoning_tokens": 424 + }, + "total_tokens": 13329, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 282940000, + "context_details": { + "input_tokens": 12872, + "output_tokens": 457 + } + } + }, + { + "tier": "work2", + "session": 1, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/retry/core.ts", + "answer": "src/retry/core.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12872, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 457, + "output_tokens_details": { + "reasoning_tokens": 424 + }, + "total_tokens": 13329, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 282940000, + "context_details": { + "input_tokens": 12872, + "output_tokens": 457 + } + } + }, + { + "tier": "work2", + "session": 1, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Mara Khoury", + "answer": "Mara Khoury", + "ok": true, + "raw": "```json\n[\"immer\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12872, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 457, + "output_tokens_details": { + "reasoning_tokens": 424 + }, + "total_tokens": 13329, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 282940000, + "context_details": { + "input_tokens": 12872, + "output_tokens": 457 + } + } + }, + { + "tier": "work2", + "session": 1, + "type": "negation", + "q": "In PROD specifically, was USE_BROTLI enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12872, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 457, + "output_tokens_details": { + "reasoning_tokens": 424 + }, + "total_tokens": 13329, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 282940000, + "context_details": { + "input_tokens": 12872, + "output_tokens": 457 + } + } + }, + { + "tier": "work2", + "session": 1, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"1330\", \"src/retry/core.ts\", \"Mara Khoury\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12872, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 457, + "output_tokens_details": { + "reasoning_tokens": 424 + }, + "total_tokens": 13329, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 282940000, + "context_details": { + "input_tokens": 12872, + "output_tokens": 457 + } + } + }, + { + "tier": "work2", + "session": 2, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"4570\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12882, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 471, + "output_tokens_details": { + "reasoning_tokens": 441 + }, + "total_tokens": 13353, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 283980000, + "context_details": { + "input_tokens": 12882, + "output_tokens": 471 + } + } + }, + { + "tier": "work2", + "session": 2, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "4570", + "answer": "4570", + "ok": true, + "raw": "```json\n[\"immer\", \"4570\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12882, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 471, + "output_tokens_details": { + "reasoning_tokens": 441 + }, + "total_tokens": 13353, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 283980000, + "context_details": { + "input_tokens": 12882, + "output_tokens": 471 + } + } + }, + { + "tier": "work2", + "session": 2, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/quota/core.ts", + "answer": "src/quota/core.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"4570\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12882, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 471, + "output_tokens_details": { + "reasoning_tokens": 441 + }, + "total_tokens": 13353, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 283980000, + "context_details": { + "input_tokens": 12882, + "output_tokens": 471 + } + } + }, + { + "tier": "work2", + "session": 2, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Nadia Alvarez", + "answer": "Nadia Alvarez", + "ok": true, + "raw": "```json\n[\"immer\", \"4570\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12882, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 471, + "output_tokens_details": { + "reasoning_tokens": 441 + }, + "total_tokens": 13353, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 283980000, + "context_details": { + "input_tokens": 12882, + "output_tokens": 471 + } + } + }, + { + "tier": "work2", + "session": 2, + "type": "negation", + "q": "In PROD specifically, was ENABLE_SHARDING enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"4570\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12882, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 471, + "output_tokens_details": { + "reasoning_tokens": 441 + }, + "total_tokens": 13353, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 283980000, + "context_details": { + "input_tokens": 12882, + "output_tokens": 471 + } + } + }, + { + "tier": "work2", + "session": 2, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"4570\", \"src/quota/core.ts\", \"Nadia Alvarez\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12882, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 471, + "output_tokens_details": { + "reasoning_tokens": 441 + }, + "total_tokens": 13353, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 283980000, + "context_details": { + "input_tokens": 12882, + "output_tokens": 471 + } + } + }, + { + "tier": "work2", + "session": 3, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6270\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12592, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 455, + "output_tokens_details": { + "reasoning_tokens": 420 + }, + "total_tokens": 13047, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277220000, + "context_details": { + "input_tokens": 12592, + "output_tokens": 455 + } + } + }, + { + "tier": "work2", + "session": 3, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "6270", + "answer": "6270", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6270\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12592, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 455, + "output_tokens_details": { + "reasoning_tokens": 420 + }, + "total_tokens": 13047, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277220000, + "context_details": { + "input_tokens": 12592, + "output_tokens": 455 + } + } + }, + { + "tier": "work2", + "session": 3, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/batcher/core.ts", + "answer": "src/batcher/core.ts", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6270\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12592, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 455, + "output_tokens_details": { + "reasoning_tokens": 420 + }, + "total_tokens": 13047, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277220000, + "context_details": { + "input_tokens": 12592, + "output_tokens": 455 + } + } + }, + { + "tier": "work2", + "session": 3, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Farid Lindqvist", + "answer": "Farid Lindqvist", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6270\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12592, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 455, + "output_tokens_details": { + "reasoning_tokens": 420 + }, + "total_tokens": 13047, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277220000, + "context_details": { + "input_tokens": 12592, + "output_tokens": 455 + } + } + }, + { + "tier": "work2", + "session": 3, + "type": "negation", + "q": "In PROD specifically, was ASYNC_FSYNC enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6270\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12592, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 455, + "output_tokens_details": { + "reasoning_tokens": 420 + }, + "total_tokens": 13047, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277220000, + "context_details": { + "input_tokens": 12592, + "output_tokens": 455 + } + } + }, + { + "tier": "work2", + "session": 3, + "type": "unanswerable", + "q": "What was the Docker base image pinned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"redux-toolkit\", \"6270\", \"src/batcher/core.ts\", \"Farid Lindqvist\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12592, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 455, + "output_tokens_details": { + "reasoning_tokens": 420 + }, + "total_tokens": 13047, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277220000, + "context_details": { + "input_tokens": 12592, + "output_tokens": 455 + } + } + }, + { + "tier": "work2", + "session": 4, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "zustand", + "answer": "zustand", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12997, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 441, + "output_tokens_details": { + "reasoning_tokens": 410 + }, + "total_tokens": 13438, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 284480000, + "context_details": { + "input_tokens": 12997, + "output_tokens": 441 + } + } + }, + { + "tier": "work2", + "session": 4, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "9120", + "answer": "9120", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12997, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 441, + "output_tokens_details": { + "reasoning_tokens": 410 + }, + "total_tokens": 13438, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 284480000, + "context_details": { + "input_tokens": 12997, + "output_tokens": 441 + } + } + }, + { + "tier": "work2", + "session": 4, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/cursor/core.ts", + "answer": "src/cursor/core.ts", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12997, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 441, + "output_tokens_details": { + "reasoning_tokens": 410 + }, + "total_tokens": 13438, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 284480000, + "context_details": { + "input_tokens": 12997, + "output_tokens": 441 + } + } + }, + { + "tier": "work2", + "session": 4, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Soren Nakamura", + "answer": "Soren Nakamura", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12997, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 441, + "output_tokens_details": { + "reasoning_tokens": 410 + }, + "total_tokens": 13438, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 284480000, + "context_details": { + "input_tokens": 12997, + "output_tokens": 441 + } + } + }, + { + "tier": "work2", + "session": 4, + "type": "negation", + "q": "In PROD specifically, was HOT_RELOAD_V2 enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12997, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 441, + "output_tokens_details": { + "reasoning_tokens": 410 + }, + "total_tokens": 13438, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 284480000, + "context_details": { + "input_tokens": 12997, + "output_tokens": 441 + } + } + }, + { + "tier": "work2", + "session": 4, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"zustand\", \"9120\", \"src/cursor/core.ts\", \"Soren Nakamura\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12997, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 441, + "output_tokens_details": { + "reasoning_tokens": 410 + }, + "total_tokens": 13438, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 284480000, + "context_details": { + "input_tokens": 12997, + "output_tokens": 441 + } + } + }, + { + "tier": "work2", + "session": 5, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "immer", + "answer": "immer", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12636, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 448, + "output_tokens_details": { + "reasoning_tokens": 417 + }, + "total_tokens": 13084, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277680000, + "context_details": { + "input_tokens": 12636, + "output_tokens": 448 + } + } + }, + { + "tier": "work2", + "session": 5, + "type": "numeric", + "q": "What exact value in ms was the RETRY BUDGET set to (not the cache TTL)?", + "gold": "7800", + "answer": "7800", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12636, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 448, + "output_tokens_details": { + "reasoning_tokens": 417 + }, + "total_tokens": 13084, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277680000, + "context_details": { + "input_tokens": 12636, + "output_tokens": 448 + } + } + }, + { + "tier": "work2", + "session": 5, + "type": "path", + "q": "Which file contained the ROOT CAUSE of the double-flush race?", + "gold": "src/quota/core.ts", + "answer": "src/quota/core.ts", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12636, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 448, + "output_tokens_details": { + "reasoning_tokens": 417 + }, + "total_tokens": 13084, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277680000, + "context_details": { + "input_tokens": 12636, + "output_tokens": 448 + } + } + }, + { + "tier": "work2", + "session": 5, + "type": "name", + "q": "Who is the on-call REVIEWER for the PR (not the author)?", + "gold": "Ingrid Costa", + "answer": "Ingrid Costa", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12636, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 448, + "output_tokens_details": { + "reasoning_tokens": 417 + }, + "total_tokens": 13084, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277680000, + "context_details": { + "input_tokens": 12636, + "output_tokens": 448 + } + } + }, + { + "tier": "work2", + "session": 5, + "type": "negation", + "q": "In PROD specifically, was USE_BROTLI enabled? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12636, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 448, + "output_tokens_details": { + "reasoning_tokens": 417 + }, + "total_tokens": 13084, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277680000, + "context_details": { + "input_tokens": 12636, + "output_tokens": 448 + } + } + }, + { + "tier": "work2", + "session": 5, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "```json\n[\"immer\", \"7800\", \"src/quota/core.ts\", \"Ingrid Costa\", \"OFF\", \"UNKNOWN\"]\n```", + "error": null, + "usage": { + "input_tokens": 12636, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 448, + "output_tokens_details": { + "reasoning_tokens": 417 + }, + "total_tokens": 13084, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 277680000, + "context_details": { + "input_tokens": 12636, + "output_tokens": 448 + } + } + }, + { + "tier": "work3", + "session": 0, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "8400", + "answer": "8400", + "ok": true, + "raw": "```json\n[\"8400\", \"9600\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 11862, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 274, + "output_tokens_details": { + "reasoning_tokens": 258 + }, + "total_tokens": 12136, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 251760000, + "context_details": { + "input_tokens": 11862, + "output_tokens": 274 + } + } + }, + { + "tier": "work3", + "session": 0, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "9600", + "answer": "9600", + "ok": true, + "raw": "```json\n[\"8400\", \"9600\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 11862, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 274, + "output_tokens_details": { + "reasoning_tokens": 258 + }, + "total_tokens": 12136, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 251760000, + "context_details": { + "input_tokens": 11862, + "output_tokens": 274 + } + } + }, + { + "tier": "work3", + "session": 0, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "```json\n[\"8400\", \"9600\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 11862, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 274, + "output_tokens_details": { + "reasoning_tokens": 258 + }, + "total_tokens": 12136, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 251760000, + "context_details": { + "input_tokens": 11862, + "output_tokens": 274 + } + } + }, + { + "tier": "work3", + "session": 1, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "1200", + "answer": "1200", + "ok": true, + "raw": "```json\n[\"1200\", \"5400\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12437, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 285, + "output_tokens_details": { + "reasoning_tokens": 269 + }, + "total_tokens": 12722, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 263920000, + "context_details": { + "input_tokens": 12437, + "output_tokens": 285 + } + } + }, + { + "tier": "work3", + "session": 1, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "5400", + "answer": "5400", + "ok": true, + "raw": "```json\n[\"1200\", \"5400\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12437, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 285, + "output_tokens_details": { + "reasoning_tokens": 269 + }, + "total_tokens": 12722, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 263920000, + "context_details": { + "input_tokens": 12437, + "output_tokens": 285 + } + } + }, + { + "tier": "work3", + "session": 1, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "```json\n[\"1200\", \"5400\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12437, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 285, + "output_tokens_details": { + "reasoning_tokens": 269 + }, + "total_tokens": 12722, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 263920000, + "context_details": { + "input_tokens": 12437, + "output_tokens": 285 + } + } + }, + { + "tier": "work3", + "session": 2, + "type": "final", + "q": "What is the FINAL (locked) value of FLUSH_INTERVAL_MS at the end of the session?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12322, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 277, + "output_tokens_details": { + "reasoning_tokens": 261 + }, + "total_tokens": 12599, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 261140000, + "context_details": { + "input_tokens": 12322, + "output_tokens": 277 + } + } + }, + { + "tier": "work3", + "session": 2, + "type": "first", + "q": "What was the FIRST value FLUSH_INTERVAL_MS was set to at the start?", + "gold": "2400", + "answer": "2400", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12322, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 277, + "output_tokens_details": { + "reasoning_tokens": 261 + }, + "total_tokens": 12599, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 261140000, + "context_details": { + "input_tokens": 12322, + "output_tokens": 277 + } + } + }, + { + "tier": "work3", + "session": 2, + "type": "count", + "q": "How many distinct values was FLUSH_INTERVAL_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "```json\n[\"7200\", \"2400\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12322, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 277, + "output_tokens_details": { + "reasoning_tokens": 261 + }, + "total_tokens": 12599, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 261140000, + "context_details": { + "input_tokens": 12322, + "output_tokens": 277 + } + } + }, + { + "tier": "work3", + "session": 3, + "type": "final", + "q": "What is the FINAL (locked) value of LEASE_TTL_MS at the end of the session?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "[\"7200\", \"2400\", \"3\"]", + "error": null, + "usage": { + "input_tokens": 12377, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 278, + "output_tokens_details": { + "reasoning_tokens": 267 + }, + "total_tokens": 12655, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 262300000, + "context_details": { + "input_tokens": 12377, + "output_tokens": 278 + } + } + }, + { + "tier": "work3", + "session": 3, + "type": "first", + "q": "What was the FIRST value LEASE_TTL_MS was set to at the start?", + "gold": "2400", + "answer": "2400", + "ok": true, + "raw": "[\"7200\", \"2400\", \"3\"]", + "error": null, + "usage": { + "input_tokens": 12377, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 278, + "output_tokens_details": { + "reasoning_tokens": 267 + }, + "total_tokens": 12655, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 262300000, + "context_details": { + "input_tokens": 12377, + "output_tokens": 278 + } + } + }, + { + "tier": "work3", + "session": 3, + "type": "count", + "q": "How many distinct values was LEASE_TTL_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "[\"7200\", \"2400\", \"3\"]", + "error": null, + "usage": { + "input_tokens": 12377, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 278, + "output_tokens_details": { + "reasoning_tokens": 267 + }, + "total_tokens": 12655, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 262300000, + "context_details": { + "input_tokens": 12377, + "output_tokens": 278 + } + } + }, + { + "tier": "work3", + "session": 4, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "4800", + "answer": "4800", + "ok": true, + "raw": "```json\n[\"4800\", \"7200\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12337, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 279, + "output_tokens_details": { + "reasoning_tokens": 263 + }, + "total_tokens": 12616, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 261560000, + "context_details": { + "input_tokens": 12337, + "output_tokens": 279 + } + } + }, + { + "tier": "work3", + "session": 4, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "7200", + "answer": "7200", + "ok": true, + "raw": "```json\n[\"4800\", \"7200\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12337, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 279, + "output_tokens_details": { + "reasoning_tokens": 263 + }, + "total_tokens": 12616, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 261560000, + "context_details": { + "input_tokens": 12337, + "output_tokens": 279 + } + } + }, + { + "tier": "work3", + "session": 4, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "```json\n[\"4800\", \"7200\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12337, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 279, + "output_tokens_details": { + "reasoning_tokens": 263 + }, + "total_tokens": 12616, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 261560000, + "context_details": { + "input_tokens": 12337, + "output_tokens": 279 + } + } + }, + { + "tier": "work3", + "session": 5, + "type": "final", + "q": "What is the FINAL (locked) value of BATCH_WINDOW_MS at the end of the session?", + "gold": "8400", + "answer": "8400", + "ok": true, + "raw": "```json\n[\"8400\", \"3600\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12132, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 214, + "output_tokens_details": { + "reasoning_tokens": 198 + }, + "total_tokens": 12346, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 253560000, + "context_details": { + "input_tokens": 12132, + "output_tokens": 214 + } + } + }, + { + "tier": "work3", + "session": 5, + "type": "first", + "q": "What was the FIRST value BATCH_WINDOW_MS was set to at the start?", + "gold": "3600", + "answer": "3600", + "ok": true, + "raw": "```json\n[\"8400\", \"3600\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12132, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 214, + "output_tokens_details": { + "reasoning_tokens": 198 + }, + "total_tokens": 12346, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 253560000, + "context_details": { + "input_tokens": 12132, + "output_tokens": 214 + } + } + }, + { + "tier": "work3", + "session": 5, + "type": "count", + "q": "How many distinct values was BATCH_WINDOW_MS set to over the whole session? Answer with a number.", + "gold": "3", + "answer": "3", + "ok": true, + "raw": "```json\n[\"8400\", \"3600\", \"3\"]\n```", + "error": null, + "usage": { + "input_tokens": 12132, + "input_tokens_details": { + "cached_tokens": 128 + }, + "output_tokens": 214, + "output_tokens_details": { + "reasoning_tokens": 198 + }, + "total_tokens": 12346, + "num_sources_used": 0, + "num_server_side_tools_used": 0, + "cost_in_usd_ticks": 253560000, + "context_details": { + "input_tokens": 12132, + "output_tokens": 214 + } + } + } + ] +} \ No newline at end of file diff --git a/eval/sol-profile/gist-recall-results.json b/eval/sol-profile/gist-recall-results.json new file mode 100644 index 0000000..b467a1b --- /dev/null +++ b/eval/sol-profile/gist-recall-results.json @@ -0,0 +1,1293 @@ +{ + "generatedAt": "2026-07-11T16:21:56.523Z", + "model": "gpt-5.6-sol", + "live": true, + "recipe": { + "cols": 152, + "maxH": 1932, + "style": { + "font": "spleen-5x8", + "cellWBonus": 0, + "cellHBonus": 0, + "aa": true, + "grid": false, + "gridCols": 0, + "colorCycle": false, + "markerScale": 1, + "markerRed": false, + "inkDilate": 0 + }, + "factsheet": true + }, + "answerable": { + "correct": 79, + "completed": 93, + "n": 98 + }, + "state": { + "correct": 18, + "completed": 18, + "n": 18 + }, + "unanswerable": { + "confabulated": 4, + "completed": 15, + "n": 16 + }, + "rows": [ + { + "tier": "work", + "session": 0, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "mobx", + "answer": "zustand", + "ok": false, + "raw": "[\"zustand\",\"7800ms\",\"src/batcher/core.ts\",\"Tobias Okafor\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7880", + "answer": "7800ms", + "ok": false, + "raw": "[\"zustand\",\"7800ms\",\"src/batcher/core.ts\",\"Tobias Okafor\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/batcher/core.ts", + "answer": "src/batcher/core.ts", + "ok": true, + "raw": "[\"zustand\",\"7800ms\",\"src/batcher/core.ts\",\"Tobias Okafor\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Okafor", + "answer": "Tobias Okafor", + "ok": true, + "raw": "[\"zustand\",\"7800ms\",\"src/batcher/core.ts\",\"Tobias Okafor\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"zustand\",\"7800ms\",\"src/batcher/core.ts\",\"Tobias Okafor\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 0, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"zustand\",\"7800ms\",\"src/batcher/core.ts\",\"Tobias Okafor\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "xstate", + "answer": "xstate", + "ok": true, + "raw": "[\"xstate\",\"3480ms\",\"src/cursor/sync.ts\",\"Farid Moreau\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "3480", + "answer": "3480ms", + "ok": true, + "raw": "[\"xstate\",\"3480ms\",\"src/cursor/sync.ts\",\"Farid Moreau\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/sync.ts", + "answer": "src/cursor/sync.ts", + "ok": true, + "raw": "[\"xstate\",\"3480ms\",\"src/cursor/sync.ts\",\"Farid Moreau\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Farid Moreau", + "answer": "Farid Moreau", + "ok": true, + "raw": "[\"xstate\",\"3480ms\",\"src/cursor/sync.ts\",\"Farid Moreau\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"xstate\",\"3480ms\",\"src/cursor/sync.ts\",\"Farid Moreau\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 1, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"xstate\",\"3480ms\",\"src/cursor/sync.ts\",\"Farid Moreau\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "lru-cache", + "ok": false, + "raw": "[\"lru-cache\",\"9110ms\",\"src/cursor/io.ts\",\"Nadia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "9110", + "answer": "9110ms", + "ok": true, + "raw": "[\"lru-cache\",\"9110ms\",\"src/cursor/io.ts\",\"Nadia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "[\"lru-cache\",\"9110ms\",\"src/cursor/io.ts\",\"Nadia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Nadia Costa", + "answer": "Nadia Costa", + "ok": true, + "raw": "[\"lru-cache\",\"9110ms\",\"src/cursor/io.ts\",\"Nadia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "negation", + "q": "Was ENABLE_SHARDING enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"lru-cache\",\"9110ms\",\"src/cursor/io.ts\",\"Nadia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 2, + "type": "unanswerable", + "q": "What git tag was the hotfix released under?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"lru-cache\",\"9110ms\",\"src/cursor/io.ts\",\"Nadia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "valtio", + "ok": false, + "raw": "[\"valtio\",\"3470ms\",\"src/scheduler/core.ts\",\"Dmitri Berg\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "3470", + "answer": "3470ms", + "ok": true, + "raw": "[\"valtio\",\"3470ms\",\"src/scheduler/core.ts\",\"Dmitri Berg\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/scheduler/core.ts", + "answer": "src/scheduler/core.ts", + "ok": true, + "raw": "[\"valtio\",\"3470ms\",\"src/scheduler/core.ts\",\"Dmitri Berg\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Dmitri Berg", + "answer": "Dmitri Berg", + "ok": true, + "raw": "[\"valtio\",\"3470ms\",\"src/scheduler/core.ts\",\"Dmitri Berg\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"valtio\",\"3470ms\",\"src/scheduler/core.ts\",\"Dmitri Berg\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 3, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"valtio\",\"3470ms\",\"src/scheduler/core.ts\",\"Dmitri Berg\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "level", + "ok": false, + "raw": "[\"level\",\"6210ms\",\"src/cursor/io.ts\",\"Tobias Khoury\",\"OFF\",\"us-east-2\"]", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "6210", + "answer": "6210ms", + "ok": true, + "raw": "[\"level\",\"6210ms\",\"src/cursor/io.ts\",\"Tobias Khoury\",\"OFF\",\"us-east-2\"]", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "[\"level\",\"6210ms\",\"src/cursor/io.ts\",\"Tobias Khoury\",\"OFF\",\"us-east-2\"]", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Khoury", + "answer": "Tobias Khoury", + "ok": true, + "raw": "[\"level\",\"6210ms\",\"src/cursor/io.ts\",\"Tobias Khoury\",\"OFF\",\"us-east-2\"]", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "negation", + "q": "Was ASYNC_FSYNC enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"level\",\"6210ms\",\"src/cursor/io.ts\",\"Tobias Khoury\",\"OFF\",\"us-east-2\"]", + "error": null + }, + { + "tier": "work", + "session": 4, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "us-east-2", + "ok": false, + "raw": "[\"level\",\"6210ms\",\"src/cursor/io.ts\",\"Tobias Khoury\",\"OFF\",\"us-east-2\"]", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "[\"redux-toolkit\",\"1250ms\",\"src/cursor/io.ts\",\"Lucia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "1250", + "answer": "1250ms", + "ok": true, + "raw": "[\"redux-toolkit\",\"1250ms\",\"src/cursor/io.ts\",\"Lucia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/io.ts", + "answer": "src/cursor/io.ts", + "ok": true, + "raw": "[\"redux-toolkit\",\"1250ms\",\"src/cursor/io.ts\",\"Lucia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Lucia Costa", + "answer": "Lucia Costa", + "ok": true, + "raw": "[\"redux-toolkit\",\"1250ms\",\"src/cursor/io.ts\",\"Lucia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"redux-toolkit\",\"1250ms\",\"src/cursor/io.ts\",\"Lucia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 5, + "type": "unanswerable", + "q": "Which database migration version was rolled back?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"redux-toolkit\",\"1250ms\",\"src/cursor/io.ts\",\"Lucia Costa\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "immer", + "answer": "nanostores", + "ok": false, + "raw": "[\"nanostores\",\"6290ms\",\"src/scheduler/sync.ts\",\"Tobias\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "6290", + "answer": "6290ms", + "ok": true, + "raw": "[\"nanostores\",\"6290ms\",\"src/scheduler/sync.ts\",\"Tobias\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/scheduler/sync.ts", + "answer": "src/scheduler/sync.ts", + "ok": true, + "raw": "[\"nanostores\",\"6290ms\",\"src/scheduler/sync.ts\",\"Tobias\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Tobias Khoury", + "answer": "Tobias", + "ok": false, + "raw": "[\"nanostores\",\"6290ms\",\"src/scheduler/sync.ts\",\"Tobias\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "negation", + "q": "Was LEGACY_PINS enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"nanostores\",\"6290ms\",\"src/scheduler/sync.ts\",\"Tobias\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 6, + "type": "unanswerable", + "q": "Which AWS region was the failover assigned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"nanostores\",\"6290ms\",\"src/scheduler/sync.ts\",\"Tobias\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "xstate", + "answer": "nanostores", + "ok": false, + "raw": "[\"nanostores\",\"7870ms\",\"src/batcher/sync.ts\",\"Priya Tanaka\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7870", + "answer": "7870ms", + "ok": true, + "raw": "[\"nanostores\",\"7870ms\",\"src/batcher/sync.ts\",\"Priya Tanaka\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/batcher/sync.ts", + "answer": "src/batcher/sync.ts", + "ok": true, + "raw": "[\"nanostores\",\"7870ms\",\"src/batcher/sync.ts\",\"Priya Tanaka\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Priya Tanaka", + "answer": "Priya Tanaka", + "ok": true, + "raw": "[\"nanostores\",\"7870ms\",\"src/batcher/sync.ts\",\"Priya Tanaka\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"nanostores\",\"7870ms\",\"src/batcher/sync.ts\",\"Priya Tanaka\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 7, + "type": "unanswerable", + "q": "What was the Docker base image pinned to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"nanostores\",\"7870ms\",\"src/batcher/sync.ts\",\"Priya Tanaka\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "jotai", + "answer": "jotai", + "ok": true, + "raw": "[\"jotai\",\"7850\",\"src/retry/io.ts\",\"Soren Petrov\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "7850", + "answer": "7850", + "ok": true, + "raw": "[\"jotai\",\"7850\",\"src/retry/io.ts\",\"Soren Petrov\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/retry/io.ts", + "answer": "src/retry/io.ts", + "ok": true, + "raw": "[\"jotai\",\"7850\",\"src/retry/io.ts\",\"Soren Petrov\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Soren Petrov", + "answer": "Soren Petrov", + "ok": true, + "raw": "[\"jotai\",\"7850\",\"src/retry/io.ts\",\"Soren Petrov\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "negation", + "q": "Was ENABLE_SHARDING enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"jotai\",\"7850\",\"src/retry/io.ts\",\"Soren Petrov\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 8, + "type": "unanswerable", + "q": "What port number was the staging proxy moved to?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"jotai\",\"7850\",\"src/retry/io.ts\",\"Soren Petrov\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "decision", + "q": "Which package was chosen for the store layer?", + "gold": "redux-toolkit", + "answer": "redux-toolkit", + "ok": true, + "raw": "[\"redux-toolkit\",\"4540ms\",\"src/cursor/sync.ts\",\"Ingrid Nakamura\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "numeric", + "q": "What exact value in ms was the retry budget set to?", + "gold": "4540", + "answer": "4540ms", + "ok": true, + "raw": "[\"redux-toolkit\",\"4540ms\",\"src/cursor/sync.ts\",\"Ingrid Nakamura\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "path", + "q": "In which file path was the double-flush race found?", + "gold": "src/cursor/sync.ts", + "answer": "src/cursor/sync.ts", + "ok": true, + "raw": "[\"redux-toolkit\",\"4540ms\",\"src/cursor/sync.ts\",\"Ingrid Nakamura\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "name", + "q": "Who was named as the on-call reviewer for the PR?", + "gold": "Ingrid Nakamura", + "answer": "Ingrid Nakamura", + "ok": true, + "raw": "[\"redux-toolkit\",\"4540ms\",\"src/cursor/sync.ts\",\"Ingrid Nakamura\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "negation", + "q": "Was HOT_RELOAD_V2 enabled in prod? Answer ENABLED or OFF.", + "gold": "OFF", + "answer": "OFF", + "ok": true, + "raw": "[\"redux-toolkit\",\"4540ms\",\"src/cursor/sync.ts\",\"Ingrid Nakamura\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work", + "session": 9, + "type": "unanswerable", + "q": "What git tag was the hotfix released under?", + "gold": "UNKNOWN", + "answer": "UNKNOWN", + "ok": true, + "raw": "[\"redux-toolkit\",\"4540ms\",\"src/cursor/sync.ts\",\"Ingrid Nakamura\",\"OFF\",\"UNKNOWN\"]", + "error": null + }, + { + "tier": "work2", + "session": 0, + "type": "decision", + "q": "What was the FINAL package chosen for the store layer?", + "gold": "nanostores", + "answer": "", + "ok": false, + "raw": "", + "error": "SyntaxError: Unexpected token '<', \"=0&&b>a?s.slice(a,b+1):s)}catch{return null}} +function norm(s){return String(s??'').trim().toLowerCase().replace(/\s+/g,' ')} +function correct(p,a){const x=norm(a),g=norm(p.gold);if(p.type==='unanswerable')return x==='unknown';if(p.type==='numeric')return new RegExp(`(?:^|\\D)${g}(?:\\D|$)`).test(x);if(p.type==='negation')return x.includes('off')&&!x.includes('enabled');return x.includes(g)} +const rows=[]; +for(const [dir,n] of TIERS){const probes=JSON.parse(readFileSync(join(ROOT,dir,'probes.json'),'utf8'));for(let sid=0;sidp.session===sid),source=readFileSync(join(ROOT,dir,`s${sid}.txt`),'utf8'),imgs=await renderTextToPngs(source,profile.stripCols,profile.style,profile.maxHeightPx);const prompt=['Read all transcript images in order. Answer every numbered question.','If the transcript does not contain an answer, use exactly UNKNOWN.','Return only a JSON array of strings in question order.',...ps.map((p,i)=>`${i+1}. ${p.q}`)].join('\n');let response={output:'',usage:null};if(LIVE){const content=imgs.map(im=>({type:'input_image',image_url:`data:image/png;base64,${Buffer.from(im.png).toString('base64')}`,detail:'original'}));const fs=factSheetText(source);if(fs)content.push({type:'input_text',text:fs});content.push({type:'input_text',text:prompt});try{const r=await callResponses({model:MODEL,content,maxOutputTokens:1400,timeoutMs:TIMEOUT});response={output:r.text,usage:r.usage}}catch(e){response={output:'',usage:null,error:String(e)}}}const answers=parse(response.output)||[];ps.forEach((p,i)=>rows.push({tier:dir,session:sid,...p,answer:String(answers[i]??''),ok:correct(p,answers[i]),raw:response.output,error:response.error||null,usage:response.usage}));console.log(`${dir} s${sid}: ${ps.filter((p,i)=>correct(p,answers[i])).length}/${ps.length}`)}} +if(!LIVE){console.log('dry run only; no receipt written');process.exit(0)} +const answerable=rows.filter(r=>r.type!=='unanswerable'),guards=rows.filter(r=>r.type==='unanswerable'),state=rows.filter(r=>r.tier==='work3'),done=xs=>xs.filter(r=>!r.error);const out={generatedAt:new Date().toISOString(),model:MODEL,live:LIVE,recipe:{cols:profile.stripCols,maxH:profile.maxHeightPx,style:profile.style,factsheet:true},answerable:{correct:done(answerable).filter(r=>r.ok).length,completed:done(answerable).length,n:answerable.length},state:{correct:done(state).filter(r=>r.ok).length,completed:done(state).length,n:state.length},unanswerable:{confabulated:done(guards).filter(r=>!r.ok).length,completed:done(guards).length,n:guards.length},rows};writeFileSync(join(HERE, MODEL==='gpt-5.6-sol' ? 'gist-recall-results.json' : 'gist-recall-'+MODEL.replace(/[^a-zA-Z0-9._-]+/g,'_')+'-results.json'),JSON.stringify(out,null,2));console.log(JSON.stringify({answerable:out.answerable,state:out.state,unanswerable:out.unanswerable},null,2)); diff --git a/eval/sol-profile/model-grok-4.5-novel-arithmetic-results.json b/eval/sol-profile/model-grok-4.5-novel-arithmetic-results.json new file mode 100644 index 0000000..17e4626 --- /dev/null +++ b/eval/sol-profile/model-grok-4.5-novel-arithmetic-results.json @@ -0,0 +1,1640 @@ +{ + "generatedAt": "2026-07-11T20:57:12.947Z", + "model": "grok-4.5", + "live": true, + "n": 100, + "seed": 20260711, + "recipe": { + "cols": 152, + "maxH": 512, + "style": { + "font": "spleen-5x8", + "cellWBonus": 0, + "cellHBonus": 0, + "aa": true, + "grid": false, + "gridCols": 0, + "colorCycle": false, + "markerScale": 1, + "markerRed": false, + "inkDilate": 0 + }, + "ids": true + }, + "textCorrect": 100, + "pureCorrect": 83, + "prodCorrect": 82, + "textPct": 100, + "purePct": 83, + "prodPct": 82, + "rows": [ + { + "i": 0, + "kind": 2, + "question": "A warehouse has 85 shelves, each holding 59 boxes, plus 987 loose boxes. How many boxes are there in total?", + "answer": 6002, + "imageTokens": 13, + "textGot": 6002, + "pureGot": 6002, + "prodGot": 6002, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 1, + "kind": 2, + "question": "A warehouse has 35 shelves, each holding 82 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 3121, + "imageTokens": 13, + "textGot": 3121, + "pureGot": 2885, + "prodGot": 2885, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 2, + "kind": 2, + "question": "A warehouse has 17 shelves, each holding 94 boxes, plus 955 loose boxes. How many boxes are there in total?", + "answer": 2553, + "imageTokens": 13, + "textGot": 2553, + "pureGot": 2553, + "prodGot": 2553, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 3, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 97 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 9099, + "imageTokens": 13, + "textGot": 9099, + "pureGot": 9099, + "prodGot": 9099, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 4, + "kind": 2, + "question": "A warehouse has 38 shelves, each holding 60 boxes, plus 587 loose boxes. How many boxes are there in total?", + "answer": 2867, + "imageTokens": 13, + "textGot": 2867, + "pureGot": 2867, + "prodGot": 2867, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 5, + "kind": 2, + "question": "A warehouse has 60 shelves, each holding 16 boxes, plus 715 loose boxes. How many boxes are there in total?", + "answer": 1675, + "imageTokens": 13, + "textGot": 1675, + "pureGot": 1675, + "prodGot": 1675, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 6, + "kind": 2, + "question": "A warehouse has 85 shelves, each holding 53 boxes, plus 655 loose boxes. How many boxes are there in total?", + "answer": 5160, + "imageTokens": 13, + "textGot": 5160, + "pureGot": 5160, + "prodGot": 5160, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 7, + "kind": 2, + "question": "A warehouse has 29 shelves, each holding 99 boxes, plus 115 loose boxes. How many boxes are there in total?", + "answer": 2986, + "imageTokens": 13, + "textGot": 2986, + "pureGot": 2986, + "prodGot": 2986, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 8, + "kind": 2, + "question": "A warehouse has 98 shelves, each holding 98 boxes, plus 183 loose boxes. How many boxes are there in total?", + "answer": 9787, + "imageTokens": 13, + "textGot": 9787, + "pureGot": 9787, + "prodGot": 9787, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 9, + "kind": 2, + "question": "A warehouse has 46 shelves, each holding 97 boxes, plus 731 loose boxes. How many boxes are there in total?", + "answer": 5193, + "imageTokens": 13, + "textGot": 5193, + "pureGot": 5193, + "prodGot": 5193, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 10, + "kind": 2, + "question": "A warehouse has 77 shelves, each holding 57 boxes, plus 139 loose boxes. How many boxes are there in total?", + "answer": 4528, + "imageTokens": 13, + "textGot": 4528, + "pureGot": 4528, + "prodGot": 4528, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 11, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 93 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 6482, + "imageTokens": 13, + "textGot": 6482, + "pureGot": 6382, + "prodGot": 6382, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 12, + "kind": 2, + "question": "A warehouse has 52 shelves, each holding 42 boxes, plus 239 loose boxes. How many boxes are there in total?", + "answer": 2423, + "imageTokens": 13, + "textGot": 2423, + "pureGot": 2423, + "prodGot": 2423, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 13, + "kind": 2, + "question": "A warehouse has 19 shelves, each holding 62 boxes, plus 511 loose boxes. How many boxes are there in total?", + "answer": 1689, + "imageTokens": 13, + "textGot": 1689, + "pureGot": 1689, + "prodGot": 1689, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 14, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 33 boxes, plus 427 loose boxes. How many boxes are there in total?", + "answer": 3331, + "imageTokens": 13, + "textGot": 3331, + "pureGot": 3331, + "prodGot": 3331, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 15, + "kind": 2, + "question": "A warehouse has 42 shelves, each holding 58 boxes, plus 971 loose boxes. How many boxes are there in total?", + "answer": 3407, + "imageTokens": 13, + "textGot": 3407, + "pureGot": 3407, + "prodGot": 3071, + "textOk": true, + "pureOk": true, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 16, + "kind": 2, + "question": "A warehouse has 70 shelves, each holding 81 boxes, plus 211 loose boxes. How many boxes are there in total?", + "answer": 5881, + "imageTokens": 13, + "textGot": 5881, + "pureGot": 5881, + "prodGot": 5881, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 17, + "kind": 2, + "question": "A warehouse has 12 shelves, each holding 56 boxes, plus 155 loose boxes. How many boxes are there in total?", + "answer": 827, + "imageTokens": 13, + "textGot": 827, + "pureGot": 827, + "prodGot": 827, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 18, + "kind": 2, + "question": "A warehouse has 16 shelves, each holding 78 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 1499, + "imageTokens": 13, + "textGot": 1499, + "pureGot": 1499, + "prodGot": 1499, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 19, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 71 boxes, plus 139 loose boxes. How many boxes are there in total?", + "answer": 6600, + "imageTokens": 13, + "textGot": 6600, + "pureGot": 6600, + "prodGot": 6600, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 20, + "kind": 2, + "question": "A warehouse has 33 shelves, each holding 21 boxes, plus 903 loose boxes. How many boxes are there in total?", + "answer": 1596, + "imageTokens": 13, + "textGot": 1596, + "pureGot": 1676, + "prodGot": 1676, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 21, + "kind": 2, + "question": "A warehouse has 21 shelves, each holding 14 boxes, plus 207 loose boxes. How many boxes are there in total?", + "answer": 501, + "imageTokens": 13, + "textGot": 501, + "pureGot": 581, + "prodGot": 581, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 22, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 21 boxes, plus 419 loose boxes. How many boxes are there in total?", + "answer": 1427, + "imageTokens": 13, + "textGot": 1427, + "pureGot": 1427, + "prodGot": 1427, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 23, + "kind": 2, + "question": "A warehouse has 68 shelves, each holding 50 boxes, plus 443 loose boxes. How many boxes are there in total?", + "answer": 3843, + "imageTokens": 13, + "textGot": 3843, + "pureGot": 3843, + "prodGot": 3843, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 24, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 50 boxes, plus 211 loose boxes. How many boxes are there in total?", + "answer": 3561, + "imageTokens": 13, + "textGot": 3561, + "pureGot": 3561, + "prodGot": 3561, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 25, + "kind": 2, + "question": "A warehouse has 45 shelves, each holding 27 boxes, plus 411 loose boxes. How many boxes are there in total?", + "answer": 1626, + "imageTokens": 13, + "textGot": 1626, + "pureGot": 1626, + "prodGot": 1626, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 26, + "kind": 2, + "question": "A warehouse has 79 shelves, each holding 55 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 4908, + "imageTokens": 13, + "textGot": 4908, + "pureGot": 4908, + "prodGot": 4908, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 27, + "kind": 2, + "question": "A warehouse has 72 shelves, each holding 73 boxes, plus 951 loose boxes. How many boxes are there in total?", + "answer": 6207, + "imageTokens": 13, + "textGot": 6207, + "pureGot": 5307, + "prodGot": 5307, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 28, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 78 boxes, plus 395 loose boxes. How many boxes are there in total?", + "answer": 7493, + "imageTokens": 13, + "textGot": 7493, + "pureGot": 7493, + "prodGot": 7493, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 29, + "kind": 2, + "question": "A warehouse has 39 shelves, each holding 40 boxes, plus 475 loose boxes. How many boxes are there in total?", + "answer": 2035, + "imageTokens": 13, + "textGot": 2035, + "pureGot": 2035, + "prodGot": 2035, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 30, + "kind": 2, + "question": "A warehouse has 66 shelves, each holding 32 boxes, plus 803 loose boxes. How many boxes are there in total?", + "answer": 2915, + "imageTokens": 13, + "textGot": 2915, + "pureGot": 2995, + "prodGot": 2995, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 31, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 30 boxes, plus 575 loose boxes. How many boxes are there in total?", + "answer": 3305, + "imageTokens": 13, + "textGot": 3305, + "pureGot": 3305, + "prodGot": 3305, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 32, + "kind": 2, + "question": "A warehouse has 42 shelves, each holding 59 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 2805, + "imageTokens": 13, + "textGot": 2805, + "pureGot": 2805, + "prodGot": 2805, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 33, + "kind": 2, + "question": "A warehouse has 89 shelves, each holding 80 boxes, plus 523 loose boxes. How many boxes are there in total?", + "answer": 7643, + "imageTokens": 13, + "textGot": 7643, + "pureGot": 7643, + "prodGot": 7643, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 34, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 98 boxes, plus 503 loose boxes. How many boxes are there in total?", + "answer": 7069, + "imageTokens": 13, + "textGot": 7069, + "pureGot": 7149, + "prodGot": 7069, + "textOk": true, + "pureOk": false, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 35, + "kind": 2, + "question": "A warehouse has 43 shelves, each holding 33 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 2330, + "imageTokens": 13, + "textGot": 2330, + "pureGot": 2330, + "prodGot": 2330, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 36, + "kind": 2, + "question": "A warehouse has 45 shelves, each holding 85 boxes, plus 651 loose boxes. How many boxes are there in total?", + "answer": 4476, + "imageTokens": 13, + "textGot": 4476, + "pureGot": 3840, + "prodGot": 3976, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 37, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 44 boxes, plus 647 loose boxes. How many boxes are there in total?", + "answer": 3067, + "imageTokens": 13, + "textGot": 3067, + "pureGot": 3067, + "prodGot": 3067, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 38, + "kind": 2, + "question": "A warehouse has 20 shelves, each holding 28 boxes, plus 523 loose boxes. How many boxes are there in total?", + "answer": 1083, + "imageTokens": 13, + "textGot": 1083, + "pureGot": 1083, + "prodGot": 1083, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 39, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 22 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 2737, + "imageTokens": 13, + "textGot": 2737, + "pureGot": 2737, + "prodGot": 2737, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 40, + "kind": 2, + "question": "A warehouse has 59 shelves, each holding 87 boxes, plus 427 loose boxes. How many boxes are there in total?", + "answer": 5560, + "imageTokens": 13, + "textGot": 5560, + "pureGot": 5560, + "prodGot": 5560, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 41, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 63 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 4028, + "imageTokens": 13, + "textGot": 4028, + "pureGot": 4028, + "prodGot": 4028, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 42, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 42 boxes, plus 175 loose boxes. How many boxes are there in total?", + "answer": 3997, + "imageTokens": 13, + "textGot": 3997, + "pureGot": 3997, + "prodGot": 3997, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 43, + "kind": 2, + "question": "A warehouse has 25 shelves, each holding 87 boxes, plus 623 loose boxes. How many boxes are there in total?", + "answer": 2798, + "imageTokens": 13, + "textGot": 2798, + "pureGot": 2798, + "prodGot": 2798, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 44, + "kind": 2, + "question": "A warehouse has 93 shelves, each holding 61 boxes, plus 111 loose boxes. How many boxes are there in total?", + "answer": 5784, + "imageTokens": 13, + "textGot": 5784, + "pureGot": 5784, + "prodGot": 5784, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 45, + "kind": 2, + "question": "A warehouse has 63 shelves, each holding 65 boxes, plus 479 loose boxes. How many boxes are there in total?", + "answer": 4574, + "imageTokens": 13, + "textGot": 4574, + "pureGot": 4574, + "prodGot": 4574, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 46, + "kind": 2, + "question": "A warehouse has 54 shelves, each holding 95 boxes, plus 687 loose boxes. How many boxes are there in total?", + "answer": 5817, + "imageTokens": 13, + "textGot": 5817, + "pureGot": 5217, + "prodGot": 5217, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 47, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 29 boxes, plus 571 loose boxes. How many boxes are there in total?", + "answer": 1963, + "imageTokens": 13, + "textGot": 1963, + "pureGot": 1443, + "prodGot": 1443, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 48, + "kind": 2, + "question": "A warehouse has 71 shelves, each holding 37 boxes, plus 587 loose boxes. How many boxes are there in total?", + "answer": 3214, + "imageTokens": 13, + "textGot": 3214, + "pureGot": 3214, + "prodGot": 3214, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 49, + "kind": 2, + "question": "A warehouse has 41 shelves, each holding 50 boxes, plus 335 loose boxes. How many boxes are there in total?", + "answer": 2385, + "imageTokens": 13, + "textGot": 2385, + "pureGot": 2385, + "prodGot": 2385, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 50, + "kind": 2, + "question": "A warehouse has 39 shelves, each holding 24 boxes, plus 843 loose boxes. How many boxes are there in total?", + "answer": 1779, + "imageTokens": 13, + "textGot": 1779, + "pureGot": 1779, + "prodGot": 1779, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 51, + "kind": 2, + "question": "A warehouse has 28 shelves, each holding 39 boxes, plus 787 loose boxes. How many boxes are there in total?", + "answer": 1879, + "imageTokens": 13, + "textGot": 1879, + "pureGot": 1879, + "prodGot": 1879, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 52, + "kind": 2, + "question": "A warehouse has 97 shelves, each holding 38 boxes, plus 539 loose boxes. How many boxes are there in total?", + "answer": 4225, + "imageTokens": 13, + "textGot": 4225, + "pureGot": 4225, + "prodGot": 4225, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 53, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 55 boxes, plus 671 loose boxes. How many boxes are there in total?", + "answer": 4356, + "imageTokens": 13, + "textGot": 4356, + "pureGot": 3756, + "prodGot": 3756, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 54, + "kind": 2, + "question": "A warehouse has 89 shelves, each holding 43 boxes, plus 759 loose boxes. How many boxes are there in total?", + "answer": 4586, + "imageTokens": 13, + "textGot": 4586, + "pureGot": 4586, + "prodGot": 4586, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 55, + "kind": 2, + "question": "A warehouse has 68 shelves, each holding 80 boxes, plus 675 loose boxes. How many boxes are there in total?", + "answer": 6115, + "imageTokens": 13, + "textGot": 6115, + "pureGot": 6115, + "prodGot": 6115, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 56, + "kind": 2, + "question": "A warehouse has 95 shelves, each holding 92 boxes, plus 503 loose boxes. How many boxes are there in total?", + "answer": 9243, + "imageTokens": 13, + "textGot": 9243, + "pureGot": 9323, + "prodGot": 9325, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 57, + "kind": 2, + "question": "A warehouse has 23 shelves, each holding 28 boxes, plus 483 loose boxes. How many boxes are there in total?", + "answer": 1127, + "imageTokens": 13, + "textGot": 1127, + "pureGot": 1127, + "prodGot": 1127, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 58, + "kind": 2, + "question": "A warehouse has 65 shelves, each holding 39 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 2862, + "imageTokens": 13, + "textGot": 2862, + "pureGot": 2862, + "prodGot": 2862, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 59, + "kind": 2, + "question": "A warehouse has 50 shelves, each holding 30 boxes, plus 883 loose boxes. How many boxes are there in total?", + "answer": 2383, + "imageTokens": 13, + "textGot": 2383, + "pureGot": 2383, + "prodGot": 2383, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 60, + "kind": 2, + "question": "A warehouse has 75 shelves, each holding 16 boxes, plus 411 loose boxes. How many boxes are there in total?", + "answer": 1611, + "imageTokens": 13, + "textGot": 1611, + "pureGot": 1611, + "prodGot": 1611, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 61, + "kind": 2, + "question": "A warehouse has 36 shelves, each holding 58 boxes, plus 167 loose boxes. How many boxes are there in total?", + "answer": 2255, + "imageTokens": 13, + "textGot": 2255, + "pureGot": 2255, + "prodGot": 2255, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 62, + "kind": 2, + "question": "A warehouse has 90 shelves, each holding 73 boxes, plus 775 loose boxes. How many boxes are there in total?", + "answer": 7345, + "imageTokens": 13, + "textGot": 7345, + "pureGot": 7345, + "prodGot": 7345, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 63, + "kind": 2, + "question": "A warehouse has 78 shelves, each holding 48 boxes, plus 535 loose boxes. How many boxes are there in total?", + "answer": 4279, + "imageTokens": 13, + "textGot": 4279, + "pureGot": 4279, + "prodGot": 4279, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 64, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 93 boxes, plus 771 loose boxes. How many boxes are there in total?", + "answer": 8490, + "imageTokens": 13, + "textGot": 8490, + "pureGot": 8490, + "prodGot": 8490, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 65, + "kind": 2, + "question": "A warehouse has 31 shelves, each holding 30 boxes, plus 691 loose boxes. How many boxes are there in total?", + "answer": 1621, + "imageTokens": 13, + "textGot": 1621, + "pureGot": 1021, + "prodGot": 1021, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 66, + "kind": 2, + "question": "A warehouse has 56 shelves, each holding 28 boxes, plus 619 loose boxes. How many boxes are there in total?", + "answer": 2187, + "imageTokens": 13, + "textGot": 2187, + "pureGot": 2187, + "prodGot": 2187, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 67, + "kind": 2, + "question": "A warehouse has 76 shelves, each holding 46 boxes, plus 695 loose boxes. How many boxes are there in total?", + "answer": 4191, + "imageTokens": 13, + "textGot": 4191, + "pureGot": 4191, + "prodGot": 3591, + "textOk": true, + "pureOk": true, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 68, + "kind": 2, + "question": "A warehouse has 22 shelves, each holding 99 boxes, plus 107 loose boxes. How many boxes are there in total?", + "answer": 2285, + "imageTokens": 13, + "textGot": 2285, + "pureGot": 2285, + "prodGot": 2285, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 69, + "kind": 2, + "question": "A warehouse has 84 shelves, each holding 75 boxes, plus 343 loose boxes. How many boxes are there in total?", + "answer": 6643, + "imageTokens": 13, + "textGot": 6643, + "pureGot": 6643, + "prodGot": 6643, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 70, + "kind": 2, + "question": "A warehouse has 66 shelves, each holding 22 boxes, plus 131 loose boxes. How many boxes are there in total?", + "answer": 1583, + "imageTokens": 13, + "textGot": 1583, + "pureGot": 1583, + "prodGot": 1583, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 71, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 99 boxes, plus 735 loose boxes. How many boxes are there in total?", + "answer": 5487, + "imageTokens": 13, + "textGot": 5487, + "pureGot": 5487, + "prodGot": 5487, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 72, + "kind": 2, + "question": "A warehouse has 95 shelves, each holding 71 boxes, plus 119 loose boxes. How many boxes are there in total?", + "answer": 6864, + "imageTokens": 13, + "textGot": 6864, + "pureGot": 6864, + "prodGot": 6864, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 73, + "kind": 2, + "question": "A warehouse has 18 shelves, each holding 47 boxes, plus 583 loose boxes. How many boxes are there in total?", + "answer": 1429, + "imageTokens": 13, + "textGot": 1429, + "pureGot": 1429, + "prodGot": 1429, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 74, + "kind": 2, + "question": "A warehouse has 31 shelves, each holding 12 boxes, plus 579 loose boxes. How many boxes are there in total?", + "answer": 951, + "imageTokens": 13, + "textGot": 951, + "pureGot": 951, + "prodGot": 951, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 75, + "kind": 2, + "question": "A warehouse has 36 shelves, each holding 98 boxes, plus 379 loose boxes. How many boxes are there in total?", + "answer": 3907, + "imageTokens": 13, + "textGot": 3907, + "pureGot": 3907, + "prodGot": 3907, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 76, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 91 boxes, plus 735 loose boxes. How many boxes are there in total?", + "answer": 8743, + "imageTokens": 13, + "textGot": 8743, + "pureGot": 8743, + "prodGot": 8743, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 77, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 64 boxes, plus 715 loose boxes. How many boxes are there in total?", + "answer": 2251, + "imageTokens": 13, + "textGot": 2251, + "pureGot": 1551, + "prodGot": 1551, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 78, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 73 boxes, plus 595 loose boxes. How many boxes are there in total?", + "answer": 2347, + "imageTokens": 13, + "textGot": 2347, + "pureGot": 2347, + "prodGot": 2347, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 79, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 23 boxes, plus 363 loose boxes. How many boxes are there in total?", + "answer": 915, + "imageTokens": 13, + "textGot": 915, + "pureGot": 915, + "prodGot": 915, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 80, + "kind": 2, + "question": "A warehouse has 81 shelves, each holding 27 boxes, plus 979 loose boxes. How many boxes are there in total?", + "answer": 3166, + "imageTokens": 13, + "textGot": 3166, + "pureGot": 3166, + "prodGot": 3166, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 81, + "kind": 2, + "question": "A warehouse has 35 shelves, each holding 72 boxes, plus 623 loose boxes. How many boxes are there in total?", + "answer": 3143, + "imageTokens": 13, + "textGot": 3143, + "pureGot": 3143, + "prodGot": 3143, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 82, + "kind": 2, + "question": "A warehouse has 98 shelves, each holding 92 boxes, plus 123 loose boxes. How many boxes are there in total?", + "answer": 9139, + "imageTokens": 13, + "textGot": 9139, + "pureGot": 9139, + "prodGot": 9139, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 83, + "kind": 2, + "question": "A warehouse has 26 shelves, each holding 43 boxes, plus 291 loose boxes. How many boxes are there in total?", + "answer": 1409, + "imageTokens": 13, + "textGot": 1409, + "pureGot": 1409, + "prodGot": 1409, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 84, + "kind": 2, + "question": "A warehouse has 44 shelves, each holding 77 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 4299, + "imageTokens": 13, + "textGot": 4299, + "pureGot": 4299, + "prodGot": 4299, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 85, + "kind": 2, + "question": "A warehouse has 79 shelves, each holding 75 boxes, plus 739 loose boxes. How many boxes are there in total?", + "answer": 6664, + "imageTokens": 13, + "textGot": 6664, + "pureGot": 6664, + "prodGot": 6664, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 86, + "kind": 2, + "question": "A warehouse has 61 shelves, each holding 45 boxes, plus 283 loose boxes. How many boxes are there in total?", + "answer": 3028, + "imageTokens": 13, + "textGot": 3028, + "pureGot": 3028, + "prodGot": 3028, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 87, + "kind": 2, + "question": "A warehouse has 84 shelves, each holding 78 boxes, plus 195 loose boxes. How many boxes are there in total?", + "answer": 6747, + "imageTokens": 13, + "textGot": 6747, + "pureGot": 6567, + "prodGot": 6647, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 88, + "kind": 2, + "question": "A warehouse has 12 shelves, each holding 19 boxes, plus 695 loose boxes. How many boxes are there in total?", + "answer": 923, + "imageTokens": 13, + "textGot": 923, + "pureGot": 323, + "prodGot": 323, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 89, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 11 boxes, plus 107 loose boxes. How many boxes are there in total?", + "answer": 712, + "imageTokens": 13, + "textGot": 712, + "pureGot": 712, + "prodGot": 712, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 90, + "kind": 2, + "question": "A warehouse has 52 shelves, each holding 93 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 5163, + "imageTokens": 13, + "textGot": 5163, + "pureGot": 5163, + "prodGot": 5163, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 91, + "kind": 2, + "question": "A warehouse has 41 shelves, each holding 30 boxes, plus 779 loose boxes. How many boxes are there in total?", + "answer": 2009, + "imageTokens": 13, + "textGot": 2009, + "pureGot": 2009, + "prodGot": 2009, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 92, + "kind": 2, + "question": "A warehouse has 38 shelves, each holding 66 boxes, plus 359 loose boxes. How many boxes are there in total?", + "answer": 2867, + "imageTokens": 13, + "textGot": 2867, + "pureGot": 2867, + "prodGot": 2867, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 93, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 26 boxes, plus 919 loose boxes. How many boxes are there in total?", + "answer": 3077, + "imageTokens": 13, + "textGot": 3077, + "pureGot": 3077, + "prodGot": 3077, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 94, + "kind": 2, + "question": "A warehouse has 49 shelves, each holding 58 boxes, plus 791 loose boxes. How many boxes are there in total?", + "answer": 3633, + "imageTokens": 13, + "textGot": 3633, + "pureGot": 3633, + "prodGot": 3633, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 95, + "kind": 2, + "question": "A warehouse has 73 shelves, each holding 58 boxes, plus 687 loose boxes. How many boxes are there in total?", + "answer": 4921, + "imageTokens": 13, + "textGot": 4921, + "pureGot": 4921, + "prodGot": 4921, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 96, + "kind": 2, + "question": "A warehouse has 58 shelves, each holding 73 boxes, plus 835 loose boxes. How many boxes are there in total?", + "answer": 5069, + "imageTokens": 13, + "textGot": 5069, + "pureGot": 6263, + "prodGot": 6213, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 97, + "kind": 2, + "question": "A warehouse has 61 shelves, each holding 39 boxes, plus 383 loose boxes. How many boxes are there in total?", + "answer": 2762, + "imageTokens": 13, + "textGot": 2762, + "pureGot": 2762, + "prodGot": 2762, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 98, + "kind": 2, + "question": "A warehouse has 44 shelves, each holding 98 boxes, plus 347 loose boxes. How many boxes are there in total?", + "answer": 4659, + "imageTokens": 13, + "textGot": 4659, + "pureGot": 4659, + "prodGot": 4659, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 99, + "kind": 2, + "question": "A warehouse has 64 shelves, each holding 98 boxes, plus 867 loose boxes. How many boxes are there in total?", + "answer": 7139, + "imageTokens": 13, + "textGot": 7139, + "pureGot": 7139, + "prodGot": 7139, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + } + ], + "inputTokens": { + "text": 25400, + "pure": 26100, + "production": 27100 + }, + "receiptNotes": [ + "Input totals recovered from the local ocproxy ledger for the same 100 matched text/pure/production requests." + ] +} diff --git a/eval/sol-profile/novel-arithmetic-results.json b/eval/sol-profile/novel-arithmetic-results.json new file mode 100644 index 0000000..da6b85e --- /dev/null +++ b/eval/sol-profile/novel-arithmetic-results.json @@ -0,0 +1,1642 @@ +{ + "generatedAt": "2026-07-11T14:23:58.000Z", + "model": "gpt-5.6-sol", + "live": true, + "n": 100, + "seed": 20260711, + "recipe": { + "cols": 152, + "maxH": 1932, + "style": { + "font": "spleen-5x8", + "cellWBonus": 0, + "cellHBonus": 0, + "aa": true, + "grid": false, + "gridCols": 0, + "colorCycle": false, + "markerScale": 1, + "markerRed": false, + "inkDilate": 0 + }, + "ids": true, + "factsheet": true + }, + "textCorrect": 100, + "pureCorrect": 96, + "prodCorrect": 98, + "textPct": 100, + "purePct": 96, + "prodPct": 98, + "inputTokens": { + "text": 5300, + "pure": 6100, + "production": 7000 + }, + "rows": [ + { + "i": 0, + "kind": 2, + "question": "A warehouse has 85 shelves, each holding 59 boxes, plus 987 loose boxes. How many boxes are there in total?", + "answer": 6002, + "imageTokens": 24, + "textGot": 6002, + "pureGot": 6002, + "prodGot": 6002, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 1, + "kind": 2, + "question": "A warehouse has 35 shelves, each holding 82 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 3121, + "imageTokens": 24, + "textGot": 3121, + "pureGot": 3121, + "prodGot": 3121, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 2, + "kind": 2, + "question": "A warehouse has 17 shelves, each holding 94 boxes, plus 955 loose boxes. How many boxes are there in total?", + "answer": 2553, + "imageTokens": 24, + "textGot": 2553, + "pureGot": 2553, + "prodGot": 2553, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 3, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 97 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 9099, + "imageTokens": 24, + "textGot": 9099, + "pureGot": 9099, + "prodGot": 9099, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 4, + "kind": 2, + "question": "A warehouse has 38 shelves, each holding 60 boxes, plus 587 loose boxes. How many boxes are there in total?", + "answer": 2867, + "imageTokens": 24, + "textGot": 2867, + "pureGot": 2867, + "prodGot": 2867, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 5, + "kind": 2, + "question": "A warehouse has 60 shelves, each holding 16 boxes, plus 715 loose boxes. How many boxes are there in total?", + "answer": 1675, + "imageTokens": 24, + "textGot": 1675, + "pureGot": 1675, + "prodGot": 1675, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 6, + "kind": 2, + "question": "A warehouse has 85 shelves, each holding 53 boxes, plus 655 loose boxes. How many boxes are there in total?", + "answer": 5160, + "imageTokens": 24, + "textGot": 5160, + "pureGot": 5160, + "prodGot": 5160, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 7, + "kind": 2, + "question": "A warehouse has 29 shelves, each holding 99 boxes, plus 115 loose boxes. How many boxes are there in total?", + "answer": 2986, + "imageTokens": 24, + "textGot": 2986, + "pureGot": 2986, + "prodGot": 2986, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 8, + "kind": 2, + "question": "A warehouse has 98 shelves, each holding 98 boxes, plus 183 loose boxes. How many boxes are there in total?", + "answer": 9787, + "imageTokens": 24, + "textGot": 9787, + "pureGot": 9787, + "prodGot": 9787, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 9, + "kind": 2, + "question": "A warehouse has 46 shelves, each holding 97 boxes, plus 731 loose boxes. How many boxes are there in total?", + "answer": 5193, + "imageTokens": 24, + "textGot": 5193, + "pureGot": 5193, + "prodGot": 5285, + "textOk": true, + "pureOk": true, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 10, + "kind": 2, + "question": "A warehouse has 77 shelves, each holding 57 boxes, plus 139 loose boxes. How many boxes are there in total?", + "answer": 4528, + "imageTokens": 24, + "textGot": 4528, + "pureGot": 1393, + "prodGot": 4528, + "textOk": true, + "pureOk": false, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 11, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 93 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 6482, + "imageTokens": 24, + "textGot": 6482, + "pureGot": 6482, + "prodGot": 6482, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 12, + "kind": 2, + "question": "A warehouse has 52 shelves, each holding 42 boxes, plus 239 loose boxes. How many boxes are there in total?", + "answer": 2423, + "imageTokens": 24, + "textGot": 2423, + "pureGot": 2423, + "prodGot": 2423, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 13, + "kind": 2, + "question": "A warehouse has 19 shelves, each holding 62 boxes, plus 511 loose boxes. How many boxes are there in total?", + "answer": 1689, + "imageTokens": 24, + "textGot": 1689, + "pureGot": 1689, + "prodGot": 1689, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 14, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 33 boxes, plus 427 loose boxes. How many boxes are there in total?", + "answer": 3331, + "imageTokens": 24, + "textGot": 3331, + "pureGot": 3331, + "prodGot": 3331, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 15, + "kind": 2, + "question": "A warehouse has 42 shelves, each holding 58 boxes, plus 971 loose boxes. How many boxes are there in total?", + "answer": 3407, + "imageTokens": 24, + "textGot": 3407, + "pureGot": 3407, + "prodGot": 3407, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 16, + "kind": 2, + "question": "A warehouse has 70 shelves, each holding 81 boxes, plus 211 loose boxes. How many boxes are there in total?", + "answer": 5881, + "imageTokens": 24, + "textGot": 5881, + "pureGot": 5881, + "prodGot": 5881, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 17, + "kind": 2, + "question": "A warehouse has 12 shelves, each holding 56 boxes, plus 155 loose boxes. How many boxes are there in total?", + "answer": 827, + "imageTokens": 24, + "textGot": 827, + "pureGot": 827, + "prodGot": 827, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 18, + "kind": 2, + "question": "A warehouse has 16 shelves, each holding 78 boxes, plus 251 loose boxes. How many boxes are there in total?", + "answer": 1499, + "imageTokens": 24, + "textGot": 1499, + "pureGot": 1499, + "prodGot": 1499, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 19, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 71 boxes, plus 139 loose boxes. How many boxes are there in total?", + "answer": 6600, + "imageTokens": 24, + "textGot": 6600, + "pureGot": 6600, + "prodGot": 6600, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 20, + "kind": 2, + "question": "A warehouse has 33 shelves, each holding 21 boxes, plus 903 loose boxes. How many boxes are there in total?", + "answer": 1596, + "imageTokens": 24, + "textGot": 1596, + "pureGot": 1596, + "prodGot": 1596, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 21, + "kind": 2, + "question": "A warehouse has 21 shelves, each holding 14 boxes, plus 207 loose boxes. How many boxes are there in total?", + "answer": 501, + "imageTokens": 24, + "textGot": 501, + "pureGot": 501, + "prodGot": 501, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 22, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 21 boxes, plus 419 loose boxes. How many boxes are there in total?", + "answer": 1427, + "imageTokens": 24, + "textGot": 1427, + "pureGot": 1427, + "prodGot": 1427, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 23, + "kind": 2, + "question": "A warehouse has 68 shelves, each holding 50 boxes, plus 443 loose boxes. How many boxes are there in total?", + "answer": 3843, + "imageTokens": 24, + "textGot": 3843, + "pureGot": 3843, + "prodGot": 3843, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 24, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 50 boxes, plus 211 loose boxes. How many boxes are there in total?", + "answer": 3561, + "imageTokens": 24, + "textGot": 3561, + "pureGot": 3561, + "prodGot": 3561, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 25, + "kind": 2, + "question": "A warehouse has 45 shelves, each holding 27 boxes, plus 411 loose boxes. How many boxes are there in total?", + "answer": 1626, + "imageTokens": 24, + "textGot": 1626, + "pureGot": 1626, + "prodGot": 1626, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 26, + "kind": 2, + "question": "A warehouse has 79 shelves, each holding 55 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 4908, + "imageTokens": 24, + "textGot": 4908, + "pureGot": 4908, + "prodGot": 4908, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 27, + "kind": 2, + "question": "A warehouse has 72 shelves, each holding 73 boxes, plus 951 loose boxes. How many boxes are there in total?", + "answer": 6207, + "imageTokens": 24, + "textGot": 6207, + "pureGot": 6207, + "prodGot": 6207, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 28, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 78 boxes, plus 395 loose boxes. How many boxes are there in total?", + "answer": 7493, + "imageTokens": 24, + "textGot": 7493, + "pureGot": 7493, + "prodGot": 7493, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 29, + "kind": 2, + "question": "A warehouse has 39 shelves, each holding 40 boxes, plus 475 loose boxes. How many boxes are there in total?", + "answer": 2035, + "imageTokens": 24, + "textGot": 2035, + "pureGot": 2035, + "prodGot": 2035, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 30, + "kind": 2, + "question": "A warehouse has 66 shelves, each holding 32 boxes, plus 803 loose boxes. How many boxes are there in total?", + "answer": 2915, + "imageTokens": 24, + "textGot": 2915, + "pureGot": 2783, + "prodGot": 2915, + "textOk": true, + "pureOk": false, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 31, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 30 boxes, plus 575 loose boxes. How many boxes are there in total?", + "answer": 3305, + "imageTokens": 24, + "textGot": 3305, + "pureGot": 3305, + "prodGot": 3305, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 32, + "kind": 2, + "question": "A warehouse has 42 shelves, each holding 59 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 2805, + "imageTokens": 24, + "textGot": 2805, + "pureGot": 2805, + "prodGot": 2805, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 33, + "kind": 2, + "question": "A warehouse has 89 shelves, each holding 80 boxes, plus 523 loose boxes. How many boxes are there in total?", + "answer": 7643, + "imageTokens": 24, + "textGot": 7643, + "pureGot": 7643, + "prodGot": 7643, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 34, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 98 boxes, plus 503 loose boxes. How many boxes are there in total?", + "answer": 7069, + "imageTokens": 24, + "textGot": 7069, + "pureGot": 7069, + "prodGot": 7069, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 35, + "kind": 2, + "question": "A warehouse has 43 shelves, each holding 33 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 2330, + "imageTokens": 24, + "textGot": 2330, + "pureGot": 2330, + "prodGot": 2330, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 36, + "kind": 2, + "question": "A warehouse has 45 shelves, each holding 85 boxes, plus 651 loose boxes. How many boxes are there in total?", + "answer": 4476, + "imageTokens": 24, + "textGot": 4476, + "pureGot": 4476, + "prodGot": 4476, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 37, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 44 boxes, plus 647 loose boxes. How many boxes are there in total?", + "answer": 3067, + "imageTokens": 24, + "textGot": 3067, + "pureGot": 3067, + "prodGot": 3067, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 38, + "kind": 2, + "question": "A warehouse has 20 shelves, each holding 28 boxes, plus 523 loose boxes. How many boxes are there in total?", + "answer": 1083, + "imageTokens": 24, + "textGot": 1083, + "pureGot": 1083, + "prodGot": 1083, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 39, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 22 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 2737, + "imageTokens": 24, + "textGot": 2737, + "pureGot": 2737, + "prodGot": 2737, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 40, + "kind": 2, + "question": "A warehouse has 59 shelves, each holding 87 boxes, plus 427 loose boxes. How many boxes are there in total?", + "answer": 5560, + "imageTokens": 24, + "textGot": 5560, + "pureGot": 5560, + "prodGot": 5560, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 41, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 63 boxes, plus 563 loose boxes. How many boxes are there in total?", + "answer": 4028, + "imageTokens": 24, + "textGot": 4028, + "pureGot": 4028, + "prodGot": 4028, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 42, + "kind": 2, + "question": "A warehouse has 91 shelves, each holding 42 boxes, plus 175 loose boxes. How many boxes are there in total?", + "answer": 3997, + "imageTokens": 24, + "textGot": 3997, + "pureGot": 3997, + "prodGot": 3997, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 43, + "kind": 2, + "question": "A warehouse has 25 shelves, each holding 87 boxes, plus 623 loose boxes. How many boxes are there in total?", + "answer": 2798, + "imageTokens": 24, + "textGot": 2798, + "pureGot": 2798, + "prodGot": 2798, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 44, + "kind": 2, + "question": "A warehouse has 93 shelves, each holding 61 boxes, plus 111 loose boxes. How many boxes are there in total?", + "answer": 5784, + "imageTokens": 24, + "textGot": 5784, + "pureGot": 5784, + "prodGot": 5784, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 45, + "kind": 2, + "question": "A warehouse has 63 shelves, each holding 65 boxes, plus 479 loose boxes. How many boxes are there in total?", + "answer": 4574, + "imageTokens": 24, + "textGot": 4574, + "pureGot": 4574, + "prodGot": 4574, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 46, + "kind": 2, + "question": "A warehouse has 54 shelves, each holding 95 boxes, plus 687 loose boxes. How many boxes are there in total?", + "answer": 5817, + "imageTokens": 24, + "textGot": 5817, + "pureGot": 5817, + "prodGot": 5817, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 47, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 29 boxes, plus 571 loose boxes. How many boxes are there in total?", + "answer": 1963, + "imageTokens": 24, + "textGot": 1963, + "pureGot": 1963, + "prodGot": 1963, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 48, + "kind": 2, + "question": "A warehouse has 71 shelves, each holding 37 boxes, plus 587 loose boxes. How many boxes are there in total?", + "answer": 3214, + "imageTokens": 24, + "textGot": 3214, + "pureGot": 3214, + "prodGot": 3214, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 49, + "kind": 2, + "question": "A warehouse has 41 shelves, each holding 50 boxes, plus 335 loose boxes. How many boxes are there in total?", + "answer": 2385, + "imageTokens": 24, + "textGot": 2385, + "pureGot": 2385, + "prodGot": 2385, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 50, + "kind": 2, + "question": "A warehouse has 39 shelves, each holding 24 boxes, plus 843 loose boxes. How many boxes are there in total?", + "answer": 1779, + "imageTokens": 24, + "textGot": 1779, + "pureGot": 1779, + "prodGot": 1779, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 51, + "kind": 2, + "question": "A warehouse has 28 shelves, each holding 39 boxes, plus 787 loose boxes. How many boxes are there in total?", + "answer": 1879, + "imageTokens": 24, + "textGot": 1879, + "pureGot": 1879, + "prodGot": 1879, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 52, + "kind": 2, + "question": "A warehouse has 97 shelves, each holding 38 boxes, plus 539 loose boxes. How many boxes are there in total?", + "answer": 4225, + "imageTokens": 24, + "textGot": 4225, + "pureGot": 4225, + "prodGot": 4225, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 53, + "kind": 2, + "question": "A warehouse has 67 shelves, each holding 55 boxes, plus 671 loose boxes. How many boxes are there in total?", + "answer": 4356, + "imageTokens": 24, + "textGot": 4356, + "pureGot": 4356, + "prodGot": 4356, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 54, + "kind": 2, + "question": "A warehouse has 89 shelves, each holding 43 boxes, plus 759 loose boxes. How many boxes are there in total?", + "answer": 4586, + "imageTokens": 24, + "textGot": 4586, + "pureGot": 4586, + "prodGot": 4586, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 55, + "kind": 2, + "question": "A warehouse has 68 shelves, each holding 80 boxes, plus 675 loose boxes. How many boxes are there in total?", + "answer": 6115, + "imageTokens": 24, + "textGot": 6115, + "pureGot": 6115, + "prodGot": 6115, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 56, + "kind": 2, + "question": "A warehouse has 95 shelves, each holding 92 boxes, plus 503 loose boxes. How many boxes are there in total?", + "answer": 9243, + "imageTokens": 24, + "textGot": 9243, + "pureGot": 9243, + "prodGot": 9243, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 57, + "kind": 2, + "question": "A warehouse has 23 shelves, each holding 28 boxes, plus 483 loose boxes. How many boxes are there in total?", + "answer": 1127, + "imageTokens": 24, + "textGot": 1127, + "pureGot": 1127, + "prodGot": 1127, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 58, + "kind": 2, + "question": "A warehouse has 65 shelves, each holding 39 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 2862, + "imageTokens": 24, + "textGot": 2862, + "pureGot": 2862, + "prodGot": 2862, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 59, + "kind": 2, + "question": "A warehouse has 50 shelves, each holding 30 boxes, plus 883 loose boxes. How many boxes are there in total?", + "answer": 2383, + "imageTokens": 24, + "textGot": 2383, + "pureGot": 2383, + "prodGot": 2383, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 60, + "kind": 2, + "question": "A warehouse has 75 shelves, each holding 16 boxes, plus 411 loose boxes. How many boxes are there in total?", + "answer": 1611, + "imageTokens": 24, + "textGot": 1611, + "pureGot": 1611, + "prodGot": 1611, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 61, + "kind": 2, + "question": "A warehouse has 36 shelves, each holding 58 boxes, plus 167 loose boxes. How many boxes are there in total?", + "answer": 2255, + "imageTokens": 24, + "textGot": 2255, + "pureGot": 2255, + "prodGot": 2255, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 62, + "kind": 2, + "question": "A warehouse has 90 shelves, each holding 73 boxes, plus 775 loose boxes. How many boxes are there in total?", + "answer": 7345, + "imageTokens": 24, + "textGot": 7345, + "pureGot": 7929, + "prodGot": 7929, + "textOk": true, + "pureOk": false, + "prodOk": false, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 63, + "kind": 2, + "question": "A warehouse has 78 shelves, each holding 48 boxes, plus 535 loose boxes. How many boxes are there in total?", + "answer": 4279, + "imageTokens": 24, + "textGot": 4279, + "pureGot": 4279, + "prodGot": 4279, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 64, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 93 boxes, plus 771 loose boxes. How many boxes are there in total?", + "answer": 8490, + "imageTokens": 24, + "textGot": 8490, + "pureGot": 8490, + "prodGot": 8490, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 65, + "kind": 2, + "question": "A warehouse has 31 shelves, each holding 30 boxes, plus 691 loose boxes. How many boxes are there in total?", + "answer": 1621, + "imageTokens": 24, + "textGot": 1621, + "pureGot": 1621, + "prodGot": 1621, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 66, + "kind": 2, + "question": "A warehouse has 56 shelves, each holding 28 boxes, plus 619 loose boxes. How many boxes are there in total?", + "answer": 2187, + "imageTokens": 24, + "textGot": 2187, + "pureGot": 2187, + "prodGot": 2187, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 67, + "kind": 2, + "question": "A warehouse has 76 shelves, each holding 46 boxes, plus 695 loose boxes. How many boxes are there in total?", + "answer": 4191, + "imageTokens": 24, + "textGot": 4191, + "pureGot": 4191, + "prodGot": 4191, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 68, + "kind": 2, + "question": "A warehouse has 22 shelves, each holding 99 boxes, plus 107 loose boxes. How many boxes are there in total?", + "answer": 2285, + "imageTokens": 24, + "textGot": 2285, + "pureGot": 2285, + "prodGot": 2285, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 69, + "kind": 2, + "question": "A warehouse has 84 shelves, each holding 75 boxes, plus 343 loose boxes. How many boxes are there in total?", + "answer": 6643, + "imageTokens": 24, + "textGot": 6643, + "pureGot": 6643, + "prodGot": 6643, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 70, + "kind": 2, + "question": "A warehouse has 66 shelves, each holding 22 boxes, plus 131 loose boxes. How many boxes are there in total?", + "answer": 1583, + "imageTokens": 24, + "textGot": 1583, + "pureGot": 1583, + "prodGot": 1583, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 71, + "kind": 2, + "question": "A warehouse has 48 shelves, each holding 99 boxes, plus 735 loose boxes. How many boxes are there in total?", + "answer": 5487, + "imageTokens": 24, + "textGot": 5487, + "pureGot": 5487, + "prodGot": 5487, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 72, + "kind": 2, + "question": "A warehouse has 95 shelves, each holding 71 boxes, plus 119 loose boxes. How many boxes are there in total?", + "answer": 6864, + "imageTokens": 24, + "textGot": 6864, + "pureGot": 6864, + "prodGot": 6864, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 73, + "kind": 2, + "question": "A warehouse has 18 shelves, each holding 47 boxes, plus 583 loose boxes. How many boxes are there in total?", + "answer": 1429, + "imageTokens": 24, + "textGot": 1429, + "pureGot": 1429, + "prodGot": 1429, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 74, + "kind": 2, + "question": "A warehouse has 31 shelves, each holding 12 boxes, plus 579 loose boxes. How many boxes are there in total?", + "answer": 951, + "imageTokens": 24, + "textGot": 951, + "pureGot": 951, + "prodGot": 951, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 75, + "kind": 2, + "question": "A warehouse has 36 shelves, each holding 98 boxes, plus 379 loose boxes. How many boxes are there in total?", + "answer": 3907, + "imageTokens": 24, + "textGot": 3907, + "pureGot": 3907, + "prodGot": 3907, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 76, + "kind": 2, + "question": "A warehouse has 88 shelves, each holding 91 boxes, plus 735 loose boxes. How many boxes are there in total?", + "answer": 8743, + "imageTokens": 24, + "textGot": 8743, + "pureGot": 8743, + "prodGot": 8743, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 77, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 64 boxes, plus 715 loose boxes. How many boxes are there in total?", + "answer": 2251, + "imageTokens": 24, + "textGot": 2251, + "pureGot": 2251, + "prodGot": 2251, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 78, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 73 boxes, plus 595 loose boxes. How many boxes are there in total?", + "answer": 2347, + "imageTokens": 24, + "textGot": 2347, + "pureGot": 2347, + "prodGot": 2347, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 79, + "kind": 2, + "question": "A warehouse has 24 shelves, each holding 23 boxes, plus 363 loose boxes. How many boxes are there in total?", + "answer": 915, + "imageTokens": 24, + "textGot": 915, + "pureGot": 915, + "prodGot": 915, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 80, + "kind": 2, + "question": "A warehouse has 81 shelves, each holding 27 boxes, plus 979 loose boxes. How many boxes are there in total?", + "answer": 3166, + "imageTokens": 24, + "textGot": 3166, + "pureGot": 3166, + "prodGot": 3166, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 81, + "kind": 2, + "question": "A warehouse has 35 shelves, each holding 72 boxes, plus 623 loose boxes. How many boxes are there in total?", + "answer": 3143, + "imageTokens": 24, + "textGot": 3143, + "pureGot": 3143, + "prodGot": 3143, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 82, + "kind": 2, + "question": "A warehouse has 98 shelves, each holding 92 boxes, plus 123 loose boxes. How many boxes are there in total?", + "answer": 9139, + "imageTokens": 24, + "textGot": 9139, + "pureGot": 9139, + "prodGot": 9139, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 83, + "kind": 2, + "question": "A warehouse has 26 shelves, each holding 43 boxes, plus 291 loose boxes. How many boxes are there in total?", + "answer": 1409, + "imageTokens": 24, + "textGot": 1409, + "pureGot": 1409, + "prodGot": 1409, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 84, + "kind": 2, + "question": "A warehouse has 44 shelves, each holding 77 boxes, plus 911 loose boxes. How many boxes are there in total?", + "answer": 4299, + "imageTokens": 24, + "textGot": 4299, + "pureGot": 4299, + "prodGot": 4299, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 85, + "kind": 2, + "question": "A warehouse has 79 shelves, each holding 75 boxes, plus 739 loose boxes. How many boxes are there in total?", + "answer": 6664, + "imageTokens": 24, + "textGot": 6664, + "pureGot": 6664, + "prodGot": 6664, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 86, + "kind": 2, + "question": "A warehouse has 61 shelves, each holding 45 boxes, plus 283 loose boxes. How many boxes are there in total?", + "answer": 3028, + "imageTokens": 24, + "textGot": 3028, + "pureGot": 3028, + "prodGot": 3028, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 87, + "kind": 2, + "question": "A warehouse has 84 shelves, each holding 78 boxes, plus 195 loose boxes. How many boxes are there in total?", + "answer": 6747, + "imageTokens": 24, + "textGot": 6747, + "pureGot": 6747, + "prodGot": 6747, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 88, + "kind": 2, + "question": "A warehouse has 12 shelves, each holding 19 boxes, plus 695 loose boxes. How many boxes are there in total?", + "answer": 923, + "imageTokens": 24, + "textGot": 923, + "pureGot": 923, + "prodGot": 923, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 89, + "kind": 2, + "question": "A warehouse has 55 shelves, each holding 11 boxes, plus 107 loose boxes. How many boxes are there in total?", + "answer": 712, + "imageTokens": 24, + "textGot": 712, + "pureGot": 712, + "prodGot": 712, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 90, + "kind": 2, + "question": "A warehouse has 52 shelves, each holding 93 boxes, plus 327 loose boxes. How many boxes are there in total?", + "answer": 5163, + "imageTokens": 24, + "textGot": 5163, + "pureGot": 5163, + "prodGot": 5163, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 91, + "kind": 2, + "question": "A warehouse has 41 shelves, each holding 30 boxes, plus 779 loose boxes. How many boxes are there in total?", + "answer": 2009, + "imageTokens": 24, + "textGot": 2009, + "pureGot": 2009, + "prodGot": 2009, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 92, + "kind": 2, + "question": "A warehouse has 38 shelves, each holding 66 boxes, plus 359 loose boxes. How many boxes are there in total?", + "answer": 2867, + "imageTokens": 24, + "textGot": 2867, + "pureGot": 2867, + "prodGot": 2867, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 93, + "kind": 2, + "question": "A warehouse has 83 shelves, each holding 26 boxes, plus 919 loose boxes. How many boxes are there in total?", + "answer": 3077, + "imageTokens": 24, + "textGot": 3077, + "pureGot": 3077, + "prodGot": 3077, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 94, + "kind": 2, + "question": "A warehouse has 49 shelves, each holding 58 boxes, plus 791 loose boxes. How many boxes are there in total?", + "answer": 3633, + "imageTokens": 24, + "textGot": 3633, + "pureGot": 3633, + "prodGot": 3633, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 95, + "kind": 2, + "question": "A warehouse has 73 shelves, each holding 58 boxes, plus 687 loose boxes. How many boxes are there in total?", + "answer": 4921, + "imageTokens": 24, + "textGot": 4921, + "pureGot": 2021, + "prodGot": 4921, + "textOk": true, + "pureOk": false, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 96, + "kind": 2, + "question": "A warehouse has 58 shelves, each holding 73 boxes, plus 835 loose boxes. How many boxes are there in total?", + "answer": 5069, + "imageTokens": 24, + "textGot": 5069, + "pureGot": 5069, + "prodGot": 5069, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 97, + "kind": 2, + "question": "A warehouse has 61 shelves, each holding 39 boxes, plus 383 loose boxes. How many boxes are there in total?", + "answer": 2762, + "imageTokens": 24, + "textGot": 2762, + "pureGot": 2762, + "prodGot": 2762, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 98, + "kind": 2, + "question": "A warehouse has 44 shelves, each holding 98 boxes, plus 347 loose boxes. How many boxes are there in total?", + "answer": 4659, + "imageTokens": 24, + "textGot": 4659, + "pureGot": 4659, + "prodGot": 4659, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + }, + { + "i": 99, + "kind": 2, + "question": "A warehouse has 64 shelves, each holding 98 boxes, plus 867 loose boxes. How many boxes are there in total?", + "answer": 7139, + "imageTokens": 24, + "textGot": 7139, + "pureGot": 7139, + "prodGot": 7139, + "textOk": true, + "pureOk": true, + "prodOk": true, + "textError": null, + "pureError": null, + "prodError": null + } + ], + "receiptNotes": [ + "Reconstructed from /tmp/eval-spleen-aa-n100.log and the local ocproxy request ledger.", + "The original JSON recipe field was stale; request byte sizes match Spleen 5x8 (text 390B, pure 1148-1180B, production +43B)." + ] +} diff --git a/eval/sol-profile/novel-arithmetic.mjs b/eval/sol-profile/novel-arithmetic.mjs new file mode 100644 index 0000000..098186f --- /dev/null +++ b/eval/sol-profile/novel-arithmetic.mjs @@ -0,0 +1,29 @@ +// GPT-5.6 Sol novel arithmetic: text baseline vs pure image vs production image+factsheet. +// Fresh random numbers; exact final-number grading. +// SOL_QUALITY_LIVE=1 N=20 node eval/sol-profile/novel-arithmetic.mjs +import { writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { renderTextToPngs } from '../../dist/core/render.js'; +import { resolveGptProfile } from '../../dist/core/gpt-model-profiles.js'; +import { appendIdsBlock, factSheetText } from '../../dist/core/factsheet.js'; +import { visionTokensForModel } from '../../dist/core/openai.js'; +import { callResponses } from './responses-client.mjs'; + +const HERE=dirname(fileURLToPath(import.meta.url)); +const MODEL=process.env.SOL_QUALITY_MODEL||process.env.MODEL||'gpt-5.6-sol'; +const LIVE=process.env.SOL_QUALITY_LIVE==='1'; +const N=Math.max(1,Number(process.env.N||20)); +const SEED=Number(process.env.SEED||20260711); +const CONCURRENCY=Math.max(1,Number(process.env.CONCURRENCY||3)); +const TIMEOUT=Number(process.env.SOL_QUALITY_TIMEOUT_MS||180000); +const profile=resolveGptProfile(MODEL); +const RESULT=join(HERE,(MODEL==='gpt-5.6-sol'?'':`model-${MODEL.replace(/[^a-zA-Z0-9._-]+/g,'_')}-`)+'novel-arithmetic-results.json'); +function lcg(seed){let s=seed>>>0;return()=>s=(Math.imul(s,1664525)+1013904223)>>>0}function ri(r,a,b){return a+(r()%(b-a+1))} +function problems(n,seed){const r=lcg(seed),out=[];for(let i=0;in+visionTokensForModel(MODEL,im.width,im.height),0)} gold=${p.answer}`)}process.exit(0)} +const rows=await pool(ps,CONCURRENCY,async p=>{const rendered=appendIdsBlock(p.question),imgs=await renderTextToPngs(rendered,profile.stripCols,profile.style,profile.maxHeightPx),urls=imgs.map(im=>({type:'input_image',image_url:`data:image/png;base64,${Buffer.from(im.png).toString('base64')}`,detail:'original'})),imageTokens=imgs.reduce((n,im)=>n+visionTokensForModel(MODEL,im.width,im.height),0),ask=process.env.SOL_ARITH_PROMPT||"Solve the math word problem. Show brief reasoning and end with exactly 'ANSWER: '.";let text,pure,prod;try{text=await callResponses({model:MODEL,content:[{type:'input_text',text:`${ask}\n\n${p.question}`}],maxOutputTokens:256,timeoutMs:TIMEOUT})}catch(e){text={text:'',error:String(e.message||e)}}try{pure=await callResponses({model:MODEL,content:[...urls,{type:'input_text',text:`The problem is in the image. ${ask}`}],maxOutputTokens:256,timeoutMs:TIMEOUT})}catch(e){pure={text:'',error:String(e.message||e)}}try{const fs=factSheetText(p.question);prod=await callResponses({model:MODEL,content:[...urls,...(fs?[{type:'input_text',text:fs}]:[]),{type:'input_text',text:`The problem is in the image; use the exact-number factsheet if present. ${ask}`}],maxOutputTokens:256,timeoutMs:TIMEOUT})}catch(e){prod={text:'',error:String(e.message||e)}}const textGot=num(text.text),pureGot=num(pure.text),prodGot=num(prod.text),row={...p,imageTokens,textGot,pureGot,prodGot,textOk:textGot===p.answer,pureOk:pureGot===p.answer,prodOk:prodGot===p.answer,textUsage:text.usage||null,pureUsage:pure.usage||null,prodUsage:prod.usage||null,textError:text.error||null,pureError:pure.error||null,prodError:prod.error||null};console.log(`q${p.i} text=${row.textOk?'Y':'N'}(${textGot}) pure=${row.pureOk?'Y':'N'}(${pureGot}) prod=${row.prodOk?'Y':'N'}(${prodGot}) gold=${p.answer}`);return row}); +const count=k=>rows.filter(r=>r[k]).length,usageTotal=k=>rows.reduce((n,r)=>n+(r[k]?.input_tokens||0),0),summary={generatedAt:new Date().toISOString(),model:MODEL,live:true,n:N,seed:SEED,recipe:{cols:profile.stripCols,maxH:profile.maxHeightPx,style:profile.style,ids:true},textCorrect:count('textOk'),pureCorrect:count('pureOk'),prodCorrect:count('prodOk'),textPct:100*count('textOk')/N,purePct:100*count('pureOk')/N,prodPct:100*count('prodOk')/N,inputTokens:{text:usageTotal('textUsage'),pure:usageTotal('pureUsage'),production:usageTotal('prodUsage')},rows};writeFileSync(RESULT,JSON.stringify(summary,null,2));console.log(`\nSUMMARY text ${summary.textCorrect}/${N} (${summary.textPct}%) · pure ${summary.pureCorrect}/${N} (${summary.purePct}%) · prod ${summary.prodCorrect}/${N} (${summary.prodPct}%)`); diff --git a/eval/sol-profile/responses-client.mjs b/eval/sol-profile/responses-client.mjs new file mode 100644 index 0000000..c1cf8b8 --- /dev/null +++ b/eval/sol-profile/responses-client.mjs @@ -0,0 +1,64 @@ +export function responsesEndpoint() { + const base = (process.env.OPENAI_BASE_URL || '').replace(/\/$/, ''); + if (!base) throw new Error('OPENAI_BASE_URL is required'); + const url = new URL(base); + if (url.port === '47821') throw new Error('refuse pxpipe'); + return base.endsWith('/responses') ? base : `${base}/responses`; +} + +export function responseBody(model, content, maxOutputTokens) { + const body = { + model, + stream: false, + max_output_tokens: maxOutputTokens, + input: [{ role: 'user', content }], + }; + if (!/^grok-/.test(model)) { + body.reasoning = { effort: 'none' }; + } + body.text = { verbosity: 'low' }; + return body; +} + +export async function callResponses({ model, content, maxOutputTokens, timeoutMs }) { + const key = process.env.OPENAI_API_KEY; + if (!key) throw new Error('OPENAI_API_KEY is required'); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + const started = Date.now(); + try { + const response = await fetch(responsesEndpoint(), { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${key}`, + }, + body: JSON.stringify(responseBody(model, content, maxOutputTokens)), + signal: controller.signal, + }); + const raw = await response.text(); + let json; + try { + json = JSON.parse(raw); + } catch { + throw new Error(`non-json HTTP ${response.status}: ${raw.slice(0, 160)}`); + } + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${json?.error?.message || raw.slice(0, 160)}`); + } + let text = typeof json.output_text === 'string' ? json.output_text : ''; + if (!text && Array.isArray(json.output)) { + for (const item of json.output) { + if (!Array.isArray(item?.content)) continue; + for (const part of item.content) { + if ((part?.type === 'output_text' || part?.type === 'text') && typeof part.text === 'string') { + text += part.text; + } + } + } + } + return { text: text.trim(), usage: json.usage || null, ms: Date.now() - started }; + } finally { + clearTimeout(timer); + } +} diff --git a/eval/sol-profile/verbatim-hex-grok-4.5-results.json b/eval/sol-profile/verbatim-hex-grok-4.5-results.json new file mode 100644 index 0000000..1cb0f26 --- /dev/null +++ b/eval/sol-profile/verbatim-hex-grok-4.5-results.json @@ -0,0 +1,161 @@ +{ + "generatedAt": "2026-07-11T20:47:49.180Z", + "model": "grok-4.5", + "live": true, + "correct": 0, + "n": 15, + "rows": [ + { + "page": 0, + "dur": 4439, + "gold": "c9c947f680ec", + "got": "", + "ok": false, + "raw": "c9c4176684", + "ms": 21274, + "error": null + }, + { + "page": 0, + "dur": 812, + "gold": "851eb3af1bd1", + "got": "36efd6a5e76e", + "ok": false, + "raw": "36efd6a5e76e", + "ms": 2477, + "error": null + }, + { + "page": 0, + "dur": 6150, + "gold": "ade34f70fd73", + "got": "", + "ok": false, + "raw": "non-json HTTP 503: \n\n\n\n\n\n\n