From 514a4f0b741ce1dec455cd020c415af89ee702f8 Mon Sep 17 00:00:00 2001 From: teamchong <25894545+teamchong@users.noreply.github.com> Date: Tue, 16 Jun 2026 12:22:55 -0400 Subject: [PATCH] feat(gate): config-driven model scope (PXPIPE_MODELS) + Opus 4.8 re-test - applicability: isPxpipeSupportedModel reads PXPIPE_MODELS (comma-sep, default claude-fable-5 only) and strips bracket/variant tags like [1m]; Opus becomes opt-in with no code change. Default behavior unchanged. - FINDINGS: Opus 4.8 re-test - arithmetic 98/100 (-2pp), verbatim 6/15 (was 93/100, 0/15): improved but still taxed -> default stays Fable-only. - FINDINGS + eval/glyph-matrix/sweep: root cause of the read tax is per-glyph RESOLUTION, not font shape (confusions scale with px and vanish by 10x16). Opus needs ~4x glyph area to read reliably; the 1568px ceiling makes that break-even, so there is no profitable Opus operating point. --- FINDINGS.md | 72 ++++++++++++++++++++++- eval/glyph-matrix/sweep/README.md | 45 ++++++++++++++ eval/glyph-matrix/sweep/gen_sweep.mjs | 57 ++++++++++++++++++ eval/glyph-matrix/sweep/golds.json | 1 + eval/glyph-matrix/sweep/grade_sweep.py | 47 +++++++++++++++ eval/glyph-matrix/sweep/out_opus_s0_0.txt | 7 +++ eval/glyph-matrix/sweep/out_opus_s0_1.txt | 7 +++ eval/glyph-matrix/sweep/out_opus_s0_2.txt | 9 +++ eval/glyph-matrix/sweep/out_opus_s0_3.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s1_0.txt | 13 ++++ eval/glyph-matrix/sweep/out_opus_s1_1.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s1_2.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s1_3.txt | 11 ++++ eval/glyph-matrix/sweep/out_opus_s2_0.txt | 7 +++ eval/glyph-matrix/sweep/out_opus_s2_1.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s2_2.txt | 7 +++ eval/glyph-matrix/sweep/out_opus_s2_3.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s3_0.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s3_1.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s3_2.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s3_3.txt | 7 +++ eval/glyph-matrix/sweep/out_opus_s4_0.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s4_1.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s4_2.txt | 5 ++ eval/glyph-matrix/sweep/out_opus_s4_3.txt | 5 ++ eval/glyph-matrix/sweep/results-opus.txt | 7 +++ eval/glyph-matrix/sweep/run_sweep.sh | 20 +++++++ src/core/applicability.ts | 50 +++++++++++++--- tests/public-api.test.ts | 22 +++++++ 29 files changed, 437 insertions(+), 12 deletions(-) create mode 100644 eval/glyph-matrix/sweep/README.md create mode 100644 eval/glyph-matrix/sweep/gen_sweep.mjs create mode 100644 eval/glyph-matrix/sweep/golds.json create mode 100644 eval/glyph-matrix/sweep/grade_sweep.py create mode 100644 eval/glyph-matrix/sweep/out_opus_s0_0.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s0_1.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s0_2.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s0_3.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s1_0.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s1_1.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s1_2.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s1_3.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s2_0.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s2_1.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s2_2.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s2_3.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s3_0.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s3_1.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s3_2.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s3_3.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s4_0.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s4_1.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s4_2.txt create mode 100644 eval/glyph-matrix/sweep/out_opus_s4_3.txt create mode 100644 eval/glyph-matrix/sweep/results-opus.txt create mode 100644 eval/glyph-matrix/sweep/run_sweep.sh diff --git a/FINDINGS.md b/FINDINGS.md index 7521f07..d359e22 100644 --- a/FINDINGS.md +++ b/FINDINGS.md @@ -1,13 +1,79 @@ # FINDINGS — pxpipe (text→PNG token compression) **Status:** ⚠️ **VERDICT REVERSED — see correction below.** Originally ruled "dead"; live measurement shows pxpipe is a working *lossy gist-compressor* saving ~68% on real (dense) Claude Code traffic, with a known verbatim-recall gap. -**Date:** 2026-05-28 (original) · 2026-05-29 (correction) · 2026-06-09 (Fable 5 update) · 2026-06-10 (gist-recall A/B, SWE-bench pilot) · 2026-06-12 (field observation, n=1) -**Models tested:** `claude-opus-4-5` (original run), `claude-opus-4-8` (re-test after a model bump), `claude-fable-5` (2026-06-09) -**Model scope (current):** Fable 5 only, enforced in library + proxy (Opus disabled 2026-06-09 — see update below). +**Date:** 2026-05-28 (original) · 2026-05-29 (correction) · 2026-06-09 (Fable 5 update) · 2026-06-10 (gist-recall A/B, SWE-bench pilot) · 2026-06-12 (field observation, n=1) · 2026-06-16 (Opus 4.8 re-test) +**Models tested:** `claude-opus-4-5` (original run), `claude-opus-4-8` (model-bump re-test, and again 2026-06-16), `claude-fable-5` (2026-06-09) +**Model scope (current):** Fable 5 only by default, enforced in library + proxy (Opus disabled 2026-06-09; re-tested 2026-06-16 still taxed; now opt-in via `PXPIPE_MODELS` — see updates below). **Harness:** `eval/needle-haystack/` (receipts preserved from `/tmp/needle_eval`) --- +## Update (2026-06-16) — Opus 4.8 re-test: read tax shrank but did not close + +Re-ran the two measurements that disabled Opus on 2026-06-09, against the +current `claude-opus-4-8` CLI snapshot (proxy bypassed, subscription billing). + +| axis | Opus (at disablement) | Opus 4.8 (2026-06-16) | Fable 5 | +|---|---|---|---| +| novel-arithmetic, N=100 (readable content) | 93/100 (−7pp) | **98/100 (−2pp)** | 100/100 | +| verbatim dense-hex recall, n=15 | 0/15 | **6/15** | 13/15 | + +Method: `eval/gsm8k/` (fixed seed → same problems as the original Opus run) and +`eval/verbatim-15/` re-pointed to `claude-opus-4-8`. Every `None`/empty result +was a load-induced dropped call (the text arm answered the same problem +correctly) and re-ran to a HIT on a quiet machine; the genuine misses are 2 +arithmetic (`10200`→`9400`, `7873`→`7793`) and 9 verbatim, all silent +single-glyph confusions — same failure mode as before. + +Read: the newer Opus snapshot reads pxpipe renders materially better than the +one that was disabled, but still pays a measurable read tax and still +confabulates silently on dense content — below Fable on both axes at identical +image billing. So the default scope stays Fable-only; Opus is now opt-in via +`PXPIPE_MODELS` (see `src/core/applicability.ts`). Open follow-up: whether a +zero-cost render style (`eval/glyph-matrix/`) closes the remaining Opus gap — +if so, re-enabling Opus becomes free. + +## Update (2026-06-16, later) — root cause of the Opus read tax: per-glyph resolution, not font shape + +Why does Opus misread pxpipe renders while Fable doesn't? Swept the glyph cell +size on identical content, Opus reading **by eye** (`--disallowedTools Bash`, so +it cannot zoom/crop), all images kept **<1568px** so Anthropic does not +downscale them back. n=20 ids/size, dense 12-char hex transcription. + +| glyph cell | rel. area | Opus exact-read | +|---|---|---| +| 5×8 (production) | 1× | **10%** | +| 7×10 | 1.75× | 35% | +| 10×16 | 4× | **95%** | +| 14×22 | 7.7× | 95% | +| 20×32 | 16× | 100% | + +Reading: accuracy is a **monotonic function of pixels-per-glyph** with a knee at +~10×16 (≈4× production area). Per-char confusions are broad at 5×8 +(`f→5, 1→d, 7→f, 4→d`…) and vanish by 10×16 (a single `3→5`) — they scale with +resolution rather than persisting as fixed glyph-shape pairs, so the cause is +**resolution (glyphs-per-encoder-patch), not an intrinsically confusable font.** +A "bag of ids, ignore label" score equals the per-label score at every size, so +it is pure *reading*, not row/label localization. + +Mechanism: Anthropic encodes/bills images in ~750px² units; a 5×8 cell packs +~18 glyphs into one visual token, so sub-token glyph detail is recoverable only +by a strong-enough encoder. Fable recovers it (≈87% verbatim / 100% arithmetic +at 5×8); Opus 4.8 does not (10% here). + +**Economic consequence:** the 1568px ceiling locks pixels-per-glyph to +chars-per-image. Opus needs ~4× the glyph area to read reliably → ~4× the image +tokens for the same text → pxpipe's ~74% compressed-content saving inverts to +roughly **break-even/negative**. There is no glyph size at which Opus both reads +renders *and* saves tokens. This is the mechanistic reason the scope is +Fable-only: Fable reads at the 5×8 cell where imaging is profitable. + +The Fable control on the identical sweep was **blocked — Fable 5 is currently +unavailable** ("Claude Fable 5 is currently unavailable"), which is the reason +this Opus re-test was run at all; Fable's 5×8 reading is sourced from prior runs. +Caveats: n=20/size, one font, one task; the break-even is directional (published +savings × measured 4× area). Receipts: [`eval/glyph-matrix/sweep/`](eval/glyph-matrix/sweep/). + ## Update (2026-06-12) — field observation: live verbatim misreads in a real session (n=1, anecdotal) **Not a controlled eval.** Logged because it is the first *unprompted, diff --git a/eval/glyph-matrix/sweep/README.md b/eval/glyph-matrix/sweep/README.md new file mode 100644 index 0000000..6f79b85 --- /dev/null +++ b/eval/glyph-matrix/sweep/README.md @@ -0,0 +1,45 @@ +# Per-glyph resolution sweep — why Opus misreads pxpipe renders + +Isolates **one** variable — pixels-per-glyph (render cell size) — to find the +root cause of Opus 4.8's read tax on dense renders. See the dated entry in +[`/FINDINGS.md`](../../../FINDINGS.md) (2026-06-16, later) for full analysis. + +## Result (Opus 4.8, n=20 ids/size) + +| glyph cell | rel. area | Opus exact-read | +|---|---|---| +| 5×8 (production) | 1× | 10% | +| 7×10 | 1.75× | 35% | +| 10×16 | 4× | 95% | +| 14×22 | 7.7× | 95% | +| 20×32 | 16× | 100% | + +Accuracy is a monotonic function of pixels-per-glyph (knee at ~10×16 ≈ 4× area). +Per-char confusions are broad at 5×8 and vanish by 10×16 → the cause is +**resolution (glyphs-per-encoder-patch), not font shape**. Because Opus needs +~4× the glyph area to read reliably, and the 1568px ceiling locks +pixels-per-glyph to chars-per-image, pxpipe's ~74% compression inverts to +break-even on Opus. Fable reads at 5×8 — the size where imaging is profitable. + +## Method / controls + +- **No zoom:** reader runs with `--disallowedTools Bash` so it must read by eye + (an earlier pass cheated by upscaling via code). +- **No downscale:** rendered at `cols=72` so even the 20×32 cell stays <1568px + wide; Anthropic downscales anything wider, which would silently undo the size. +- **Same content** rendered at every cell size (only resolution varies). +- **Two metrics:** per-label (right id under right label) and bag-of-ids (gold + id anywhere) — they match at every size, so it's reading, not localization. + +## Reproduce + +```bash +pnpm run build # render imports ../../../dist +node eval/glyph-matrix/sweep/gen_sweep.mjs # -> /tmp/sweep/*.png + golds.json +MODEL=claude-opus-4-8 TAG=opus bash eval/glyph-matrix/sweep/run_sweep.sh +python3 eval/glyph-matrix/sweep/grade_sweep.py opus +``` + +Needs the `claude` CLI (Max plan). `out_opus_*.txt` are the raw model +transcriptions banked from the run graded in `results-opus.txt`. The Fable arm +is blocked while Fable 5 access is unavailable. diff --git a/eval/glyph-matrix/sweep/gen_sweep.mjs b/eval/glyph-matrix/sweep/gen_sweep.mjs new file mode 100644 index 0000000..66f851e --- /dev/null +++ b/eval/glyph-matrix/sweep/gen_sweep.mjs @@ -0,0 +1,57 @@ +// Root-cause sweep: SAME content rendered at increasing glyph-cell sizes. +// Isolates ONE variable — pixels-per-glyph (cell size) — from density/search. +// Short lines so every cell size fits one <=1568px page. Same content across +// sizes => only resolution changes. Reader accuracy vs cell size = the curve. +import { renderTextToPngs } from '/Users/user/Downloads/repos/pixelpipe/dist/core/render.js'; +import { writeFileSync, mkdirSync } from 'node:fs'; +const OUT = '/tmp/sweep'; mkdirSync(OUT, { recursive: true }); + +const PAGES = 4; +// (cellWBonus, cellHBonus) -> cell = (5+wb) x (8+hb). prod is (0,0)=5x8. +const SIZES = [ + ['s0', 0, 0], // 5x8 prod + ['s1', 2, 2], // 7x10 + ['s2', 5, 8], // 10x16 (2x linear) + ['s3', 9, 14], // 14x22 (~2.8x) + ['s4', 15, 24], // 20x32 (4x) +]; + +function mulberry32(a){return function(){a|=0;a=(a+0x6D2B79F5)|0;let t=Math.imul(a^(a>>>15),1|a);t=(t+Math.imul(t^(t>>>7),61|t))^t;return((t^(t>>>14))>>>0)/4294967296;};} +const rnd = mulberry32(20260616); +const hex = (n)=>Array.from({length:n},()=>'0123456789abcdef'[(rnd()*16)|0]).join(''); +const ri = (lo,hi)=>lo+Math.floor(rnd()*(hi-lo+1)); + +// short line (~45 chars) so large cells still fit one page +function line(label){ + const id = hex(12); + const rec = label ? {label, id, dur: ri(100,9999)} : {lvl:['info','warn','dbg'][ri(0,2)], id, dur: ri(100,9999)}; + return { text: JSON.stringify(rec), id }; +} + +const golds = {}; SIZES.forEach(([k])=>golds[k]=[]); +const pageTexts = []; +for (let p=0; p=0){ const {text,id}=line(labels[idx]); gold[labels[idx]]=id; rows.push(text);} + else rows.push(line(null).text); + } + pageTexts.push(rows.join('\n')); + SIZES.forEach(([k])=>golds[k].push(gold)); // identical content across sizes +} + +for (const [k, wb, hb] of SIZES){ + for (let p=0; p 72*20=1440px < 1568, so NO downscale. + // Lines are ~45 chars so they fit one row at every size. Glyph px now + // genuinely varies at the encoder (the whole point). + const pngs = await renderTextToPngs(pageTexts[p], 72, {aa:true, cellWBonus:wb, cellHBonus:hb}); + if (pngs.length!==1) console.error(`WARN ${k}_${p}: ${pngs.length} pages`); + writeFileSync(`${OUT}/${k}_${p}.png`, pngs[0].png); + if (p===0) console.log(`${k} cell=${5+wb}x${8+hb}px page0=${pngs[0].width}x${pngs[0].height}px img_tokens~${Math.round(pngs[0].width*pngs[0].height/750)}`); + } +} +writeFileSync(`${OUT}/golds.json`, JSON.stringify(golds)); +console.log(`done: ${SIZES.length} sizes x ${PAGES} pages`); diff --git a/eval/glyph-matrix/sweep/golds.json b/eval/glyph-matrix/sweep/golds.json new file mode 100644 index 0000000..12fc1d2 --- /dev/null +++ b/eval/glyph-matrix/sweep/golds.json @@ -0,0 +1 @@ +{"s0":[{"C":"fa2587c3db43","A":"0a9016292918","B":"4aefc5667127","E":"3b57511a2d37","D":"4e557a6941a3"},{"D":"d9c1a44d9d82","C":"f68401231baa","B":"2556bf62cbeb","E":"1497869832e0","A":"9422c7d44eab"},{"C":"423406430208","B":"e04e32424126","D":"112c052c3cd3","E":"c7977df6e8e7","A":"7808126d9ce2"},{"B":"c1e378234a1f","D":"fc496e0325ca","C":"c3e61c70b689","E":"fd8d4a1f08ef","A":"4c1d5dab770b"}],"s1":[{"C":"fa2587c3db43","A":"0a9016292918","B":"4aefc5667127","E":"3b57511a2d37","D":"4e557a6941a3"},{"D":"d9c1a44d9d82","C":"f68401231baa","B":"2556bf62cbeb","E":"1497869832e0","A":"9422c7d44eab"},{"C":"423406430208","B":"e04e32424126","D":"112c052c3cd3","E":"c7977df6e8e7","A":"7808126d9ce2"},{"B":"c1e378234a1f","D":"fc496e0325ca","C":"c3e61c70b689","E":"fd8d4a1f08ef","A":"4c1d5dab770b"}],"s2":[{"C":"fa2587c3db43","A":"0a9016292918","B":"4aefc5667127","E":"3b57511a2d37","D":"4e557a6941a3"},{"D":"d9c1a44d9d82","C":"f68401231baa","B":"2556bf62cbeb","E":"1497869832e0","A":"9422c7d44eab"},{"C":"423406430208","B":"e04e32424126","D":"112c052c3cd3","E":"c7977df6e8e7","A":"7808126d9ce2"},{"B":"c1e378234a1f","D":"fc496e0325ca","C":"c3e61c70b689","E":"fd8d4a1f08ef","A":"4c1d5dab770b"}],"s3":[{"C":"fa2587c3db43","A":"0a9016292918","B":"4aefc5667127","E":"3b57511a2d37","D":"4e557a6941a3"},{"D":"d9c1a44d9d82","C":"f68401231baa","B":"2556bf62cbeb","E":"1497869832e0","A":"9422c7d44eab"},{"C":"423406430208","B":"e04e32424126","D":"112c052c3cd3","E":"c7977df6e8e7","A":"7808126d9ce2"},{"B":"c1e378234a1f","D":"fc496e0325ca","C":"c3e61c70b689","E":"fd8d4a1f08ef","A":"4c1d5dab770b"}],"s4":[{"C":"fa2587c3db43","A":"0a9016292918","B":"4aefc5667127","E":"3b57511a2d37","D":"4e557a6941a3"},{"D":"d9c1a44d9d82","C":"f68401231baa","B":"2556bf62cbeb","E":"1497869832e0","A":"9422c7d44eab"},{"C":"423406430208","B":"e04e32424126","D":"112c052c3cd3","E":"c7977df6e8e7","A":"7808126d9ce2"},{"B":"c1e378234a1f","D":"fc496e0325ca","C":"c3e61c70b689","E":"fd8d4a1f08ef","A":"4c1d5dab770b"}]} \ No newline at end of file diff --git a/eval/glyph-matrix/sweep/grade_sweep.py b/eval/glyph-matrix/sweep/grade_sweep.py new file mode 100644 index 0000000..72538e6 --- /dev/null +++ b/eval/glyph-matrix/sweep/grade_sweep.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# Grade the cell-size sweep. Two metrics per size: +# per-label = right hex id under the right label (strict; needs read + row-assoc) +# bag-of-ids = gold ids found ANYWHERE in output (isolates pure glyph reading) +import json, os, re, sys +from collections import Counter + +D = "/tmp/sweep" +golds = json.load(open(f"{D}/golds.json")) +TAG = sys.argv[1] if len(sys.argv) > 1 else "opus" +CELL = {"s0":"5x8","s1":"7x10","s2":"10x16","s3":"14x22","s4":"20x32"} +LINE = re.compile(r"^\s*([A-E])\s*[:.]\s*`?([0-9a-fA-F]{12})`?\s*$") +HEX12 = re.compile(r"[0-9a-fA-F]{12}") + +print(f"tag={TAG}") +print(f"{'cell':7s} {'per-label':>10s} {'bag-of-ids':>11s} pages top confusions") +for k in ["s0","s1","s2","s3","s4"]: + lab_hit=lab_tot=bag_hit=bag_tot=pages=bad=0 + conf=Counter() + for i in range(len(golds[k])): + p=f"{D}/out_{TAG}_{k}_{i}.txt" + if not os.path.exists(p) or os.path.getsize(p)==0: bad+=1; continue + txt=open(p).read() + labeled={} + for ln in txt.splitlines(): + m=LINE.match(ln.strip()) + if m: labeled[m.group(1)]=m.group(2).lower() + allids={h.lower() for h in HEX12.findall(txt)} + gold=golds[k][i] + pages+=1 + # bag: each gold id present anywhere? + for lbl,gid in gold.items(): + bag_tot+=1 + if gid in allids: bag_hit+=1 + # per-label (only if model gave 5 clean labeled lines) + if len(labeled)==5: + for lbl,gid in gold.items(): + lab_tot+=1 + g=labeled.get(lbl,"") + if g==gid: lab_hit+=1 + elif len(g)==12: + for a,b in zip(gid,g): + if a!=b: conf[(a,b)]+=1 + lp=f"{100*lab_hit/lab_tot:.0f}%" if lab_tot else "--" + bp=f"{100*bag_hit/bag_tot:.0f}%" if bag_tot else "--" + cf=", ".join(f"{a}->{b}x{n}" for (a,b),n in sorted(conf.items(),key=lambda x:-x[1])[:6]) + print(f"{CELL[k]:7s} {lab_hit:2d}/{lab_tot:<2d} {lp:>4s} {bag_hit:2d}/{bag_tot:<2d} {bp:>4s} {pages}pg b{bad} {cf}") diff --git a/eval/glyph-matrix/sweep/out_opus_s0_0.txt b/eval/glyph-matrix/sweep/out_opus_s0_0.txt new file mode 100644 index 0000000..ef710a0 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s0_0.txt @@ -0,0 +1,7 @@ +Based on 9 independent ordered reads with per-line-position majority voting (and resolving the label↔line mapping against the structural constraint that exactly five lines carry labels A–E): + +A: a3a9d6292918 +B: 4e8573691187 +C: fa2587c3db93 +D: e5d4af2ad187 +E: aaef5567127f diff --git a/eval/glyph-matrix/sweep/out_opus_s0_1.txt b/eval/glyph-matrix/sweep/out_opus_s0_1.txt new file mode 100644 index 0000000..b585dc0 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s0_1.txt @@ -0,0 +1,7 @@ +Reading the image visually: + +A: 5a52bfeb21a6 +B: 9a22c0d4eeab +C: f669d1233baa +D: d0c1a4d9d6d2 +E: 1497869832e0 diff --git a/eval/glyph-matrix/sweep/out_opus_s0_2.txt b/eval/glyph-matrix/sweep/out_opus_s0_2.txt new file mode 100644 index 0000000..8710a24 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s0_2.txt @@ -0,0 +1,9 @@ +I ran two independent ensemble passes (19 total visual reads) and resolved the label→line assignment with max-weight matching (the agents systematically confused B/D and A/C/E on the hardest lines, so per-character majority voting plus optimal bijection was needed), then majority-voted each character. + +Structure recovered: 8 JSON lines total — rows 5–7 are `"lvl"` lines (dbg/warn/info), not labels. The 5 labeled lines are rows 1,2,3,4,8 → C, E, D, B, A respectively. + +A: 708018ed9ce2 +B: 132c052c3cd3 +C: 423406430208 +D: d09a17212dc3 +E: 404433542132 diff --git a/eval/glyph-matrix/sweep/out_opus_s0_3.txt b/eval/glyph-matrix/sweep/out_opus_s0_3.txt new file mode 100644 index 0000000..eab9858 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s0_3.txt @@ -0,0 +1,5 @@ +A: d8d9a3f80a01 +B: c1d37823a41f +C: c8e61c706649 +D: 5c1d5dab776b +E: fc96803525af diff --git a/eval/glyph-matrix/sweep/out_opus_s1_0.txt b/eval/glyph-matrix/sweep/out_opus_s1_0.txt new file mode 100644 index 0000000..248eb16 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s1_0.txt @@ -0,0 +1,13 @@ +Reconciling all 7 reads (my 2 direct reads + 6 workflow reads) with per-character majority voting: + +- **C** — unanimous across all reads: `fa2587c3db93` +- **A** — middle chars contested; consensus `9a9816293918` (3 exact-match reads + majority per position) +- **B** — body `aefc5667127` confirmed by multiple reads; leading char read as `e` +- **D** — leading char `4` (6 reads) over my `e`; position-9 `a` by 4–3 +- **E** — `3b57511a2d_7`; penultimate char `3` by 4–3 + +A: 9a9816293918 +B: eaefc5667127 +C: fa2587c3db93 +D: 4e557a69a1a3 +E: 3b57511a2d37 diff --git a/eval/glyph-matrix/sweep/out_opus_s1_1.txt b/eval/glyph-matrix/sweep/out_opus_s1_1.txt new file mode 100644 index 0000000..4111fa3 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s1_1.txt @@ -0,0 +1,5 @@ +A: 9422c7d44eab +B: 2556bf62cbeb +C: 584612311b5a +D: d9c1444d9d82 +E: 149786993220 diff --git a/eval/glyph-matrix/sweep/out_opus_s1_2.txt b/eval/glyph-matrix/sweep/out_opus_s1_2.txt new file mode 100644 index 0000000..56c0ed9 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s1_2.txt @@ -0,0 +1,5 @@ +A: 7808126d9ce2 +B: 804032424126 +C: 423406438208 +D: 112c052c3cd3 +E: c7977df6a8e7 diff --git a/eval/glyph-matrix/sweep/out_opus_s1_3.txt b/eval/glyph-matrix/sweep/out_opus_s1_3.txt new file mode 100644 index 0000000..297d862 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s1_3.txt @@ -0,0 +1,11 @@ +Based on 6 independent vision readers and per-character consensus (resolving ambiguous tails by whole-string agreement): + +``` +A: 4c1d5dab770b +B: c1e37823a41f +C: c3e61c70b669 +D: fc496e033c5e +E: fd8d4a1f08ef +``` + +A, B, and E were unanimous across all readers. C and D had noisy tails — C resolved to `c3e61c70b669` (5/6 on char 2, 4/6 on char 3); D's last three chars were the least legible, with `fc496e033c5e` being the most-agreed full string (best-guessed). diff --git a/eval/glyph-matrix/sweep/out_opus_s2_0.txt b/eval/glyph-matrix/sweep/out_opus_s2_0.txt new file mode 100644 index 0000000..e302579 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s2_0.txt @@ -0,0 +1,7 @@ +Reading the labeled lines (those with a `"label"` field) and transcribing their 12-char hex ids: + +A: 0a9016292918 +B: 4aefc5667127 +C: fa2587c5db43 +D: 4e557a6941a3 +E: 3b57511a2d37 diff --git a/eval/glyph-matrix/sweep/out_opus_s2_1.txt b/eval/glyph-matrix/sweep/out_opus_s2_1.txt new file mode 100644 index 0000000..a5a5df0 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s2_1.txt @@ -0,0 +1,5 @@ +A: 9422c7d44eab +B: 2556bf62cbeb +C: f68401231baa +D: d9c1a44d9d82 +E: 1497869832e0 diff --git a/eval/glyph-matrix/sweep/out_opus_s2_2.txt b/eval/glyph-matrix/sweep/out_opus_s2_2.txt new file mode 100644 index 0000000..773a93c --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s2_2.txt @@ -0,0 +1,7 @@ +Reading the labeled lines (those with a `"label"` field) visually: + +A: 7808126d9ce2 +B: e04e32424126 +C: 423406430208 +D: 112c052c3cd3 +E: c7977df6e8e7 diff --git a/eval/glyph-matrix/sweep/out_opus_s2_3.txt b/eval/glyph-matrix/sweep/out_opus_s2_3.txt new file mode 100644 index 0000000..7162c4e --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s2_3.txt @@ -0,0 +1,5 @@ +A: 4c1d5dab770b +B: c1e378234a1f +C: c3e61c70b689 +D: fc496e0325ca +E: fd8d4a1f08ef diff --git a/eval/glyph-matrix/sweep/out_opus_s3_0.txt b/eval/glyph-matrix/sweep/out_opus_s3_0.txt new file mode 100644 index 0000000..1baebd8 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s3_0.txt @@ -0,0 +1,5 @@ +A: 0a9016292918 +B: 4aefc5667127 +C: fa2587c3db43 +D: 4e557a6941a3 +E: 3b57511a2d37 diff --git a/eval/glyph-matrix/sweep/out_opus_s3_1.txt b/eval/glyph-matrix/sweep/out_opus_s3_1.txt new file mode 100644 index 0000000..a5a5df0 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s3_1.txt @@ -0,0 +1,5 @@ +A: 9422c7d44eab +B: 2556bf62cbeb +C: f68401231baa +D: d9c1a44d9d82 +E: 1497869832e0 diff --git a/eval/glyph-matrix/sweep/out_opus_s3_2.txt b/eval/glyph-matrix/sweep/out_opus_s3_2.txt new file mode 100644 index 0000000..515bf05 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s3_2.txt @@ -0,0 +1,5 @@ +A: 7808126d9ce2 +B: e04e32424126 +C: 423406430208 +D: 112c052c3cd3 +E: c7977df6e8e7 diff --git a/eval/glyph-matrix/sweep/out_opus_s3_3.txt b/eval/glyph-matrix/sweep/out_opus_s3_3.txt new file mode 100644 index 0000000..4cc4a47 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s3_3.txt @@ -0,0 +1,7 @@ +Re-read confirms positions 1–5 (`fc496`) and 7–12 (`0325ca`) for D; the only contested glyph is position 6. Across all reads it splits 4×`8`, 2×`0`, 1×`e` — majority is `8`. The other four IDs had unanimous 6/6 agreement. + +A: 4c1d5dab770b +B: c1e378234a1f +C: c3e61c70b689 +D: fc49680325ca +E: fd8d4a1f08ef diff --git a/eval/glyph-matrix/sweep/out_opus_s4_0.txt b/eval/glyph-matrix/sweep/out_opus_s4_0.txt new file mode 100644 index 0000000..1baebd8 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s4_0.txt @@ -0,0 +1,5 @@ +A: 0a9016292918 +B: 4aefc5667127 +C: fa2587c3db43 +D: 4e557a6941a3 +E: 3b57511a2d37 diff --git a/eval/glyph-matrix/sweep/out_opus_s4_1.txt b/eval/glyph-matrix/sweep/out_opus_s4_1.txt new file mode 100644 index 0000000..a5a5df0 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s4_1.txt @@ -0,0 +1,5 @@ +A: 9422c7d44eab +B: 2556bf62cbeb +C: f68401231baa +D: d9c1a44d9d82 +E: 1497869832e0 diff --git a/eval/glyph-matrix/sweep/out_opus_s4_2.txt b/eval/glyph-matrix/sweep/out_opus_s4_2.txt new file mode 100644 index 0000000..515bf05 --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s4_2.txt @@ -0,0 +1,5 @@ +A: 7808126d9ce2 +B: e04e32424126 +C: 423406430208 +D: 112c052c3cd3 +E: c7977df6e8e7 diff --git a/eval/glyph-matrix/sweep/out_opus_s4_3.txt b/eval/glyph-matrix/sweep/out_opus_s4_3.txt new file mode 100644 index 0000000..7162c4e --- /dev/null +++ b/eval/glyph-matrix/sweep/out_opus_s4_3.txt @@ -0,0 +1,5 @@ +A: 4c1d5dab770b +B: c1e378234a1f +C: c3e61c70b689 +D: fc496e0325ca +E: fd8d4a1f08ef diff --git a/eval/glyph-matrix/sweep/results-opus.txt b/eval/glyph-matrix/sweep/results-opus.txt new file mode 100644 index 0000000..710451c --- /dev/null +++ b/eval/glyph-matrix/sweep/results-opus.txt @@ -0,0 +1,7 @@ +tag=opus +cell per-label bag-of-ids pages top confusions +5x8 2/20 10% 2/20 10% 4pg b0 f->5x4, 1->dx3, 7->fx3, 4->dx3, 4->2x3, 4->9x2 +7x10 7/20 35% 7/20 35% 4pg b0 2->3x3, 0->8x2, 4->ax2, a->4x2, 4->9x1, 0->9x1 +10x16 19/20 95% 19/20 95% 4pg b0 3->5x1 +14x22 19/20 95% 19/20 95% 4pg b0 e->8x1 +20x32 20/20 100% 20/20 100% 4pg b0 diff --git a/eval/glyph-matrix/sweep/run_sweep.sh b/eval/glyph-matrix/sweep/run_sweep.sh new file mode 100644 index 0000000..c10486d --- /dev/null +++ b/eval/glyph-matrix/sweep/run_sweep.sh @@ -0,0 +1,20 @@ +#!/bin/bash +D=/tmp/sweep; export MODEL="${MODEL:-claude-opus-4-8}"; export TAG="${TAG:-opus}"; PMAX="${PMAX:-4}" +export SIZES="${SIZES:-s0 s1 s2 s3 s4}" +rm -f $D/HALT +run_one() { + k=$1; i=$2; f=/tmp/sweep/${k}_${i}.png; out=/tmp/sweep/out_${TAG}_${k}_${i}.txt + [ -e /tmp/sweep/HALT ] && return 0 + [ -s "$out" ] && return 0 + local P="Read the image at $f. It shows JSON lines; exactly five have a \"label\" field A-E. Transcribe the 12-char hex id from each labeled line. Read visually; do NOT run code/tools to crop or zoom. Reply with five lines 'A: '..'E: ' only; best-guess unreadable chars rather than refusing." + tmp=$(mktemp) + printf '%s' "$P" | env -u ANTHROPIC_BASE_URL ~/.claude/local/claude -p --model "$MODEL" --disallowedTools Bash > "$tmp" 2>/dev/null + if grep -qiE "hit your (session|usage) limit" "$tmp"; then rm -f "$tmp"; touch /tmp/sweep/HALT; echo HALT >&2; return 1; fi + if [ ! -s "$tmp" ]; then rm -f "$tmp"; return 1; fi + mv "$tmp" "$out" +} +export -f run_one +for k in $SIZES; do for i in $(seq 0 $((PMAX-1))); do echo "$k $i"; done; done \ + | xargs -P 2 -n 2 bash -c 'run_one "$@"' _ +[ -e $D/HALT ] && { echo HALTED-ON-LIMIT; exit 1; } +echo "DONE tag=$TAG" diff --git a/src/core/applicability.ts b/src/core/applicability.ts index c0ace92..2d9ec7e 100644 --- a/src/core/applicability.ts +++ b/src/core/applicability.ts @@ -14,16 +14,48 @@ export interface PxpipeApplicabilityInput { readonly bodyBytes?: number | null; } -/** Pxpipe's validated production scope: Fable 5 only. - * Measured 2026-06-09: Fable 5 reads pxpipe renders at 100/100 on the - * novel-arithmetic eval (Opus 4.8: 93/100) and bills the same image tokens - * (w·h/750, same tokenizer as Opus 4.7+). Opus is disabled — its ~7% read - * tax is the wrong trade now that a tax-free model exists. Mythos 5 is - * unmeasured (no access). Suffix aliases such as `claude-fable-5-high` are - * accepted because hosts may check either the client alias or the resolved - * upstream model. */ +/** Bracketed variant tags that don't change reading behavior and so must not + * affect the gate — e.g. the context-window tag in `claude-opus-4-8[1m]`. + * Stripped before matching so a base model and its `[1m]` form gate alike. */ +const VARIANT_TAG = /\[[^\]]*\]/g; + +function baseModelId(model: string): string { + return model.replace(VARIANT_TAG, ''); +} + +/** Base model ids pxpipe is allowed to transform, from `PXPIPE_MODELS` + * (comma-separated). Defaults to Fable 5 only — the validated production + * scope. Read per call so the scope can be widened/narrowed by env alone, + * no rebuild or restart. + * + * Validated 2026-06-09: Fable 5 reads pxpipe renders at 100/100 on the + * novel-arithmetic eval (Opus 4.8: 93/100) and bills the same image tokens + * (w·h/750, same tokenizer as Opus 4.7+). Opus was the original scope but + * carried a ~7% read tax, so it was dropped once a tax-free model existed. + * Re-tested 2026-06-16 on a newer Opus 4.8 snapshot: improved to 98/100 + * (-2pp) on arithmetic and 6/15 (was 0/15) on dense-hex recall, but still + * taxed and still silently confabulating vs Fable's 100/100 and 13/15 — so + * the default stays Fable-only. + * To re-enable it (e.g. while re-evaluating a newer Opus snapshot): + * PXPIPE_MODELS=claude-fable-5,claude-opus-4-8 + * Mythos 5 is unmeasured (no access). */ +function allowedModelBases(): string[] { + const raw = process.env.PXPIPE_MODELS; + return (raw && raw.trim() ? raw : 'claude-fable-5') + .split(',') + .map((s) => s.trim()) + .filter(Boolean); +} + +/** True when pxpipe is allowed to transform requests for this model. A model + * matches an allowed base when it equals the base or extends it with a + * `-suffix` alias (`claude-fable-5-high`) — hosts may send either the client + * alias or the resolved upstream id. Bracketed variant tags (`[1m]`) are + * stripped first so `claude-opus-4-8[1m]` matches its base. */ export function isPxpipeSupportedModel(model: string | null | undefined): boolean { - return typeof model === 'string' && /^claude-fable-5(?:-|$)/.test(model); + if (typeof model !== 'string') return false; + const base = baseModelId(model); + return allowedModelBases().some((b) => base === b || base.startsWith(`${b}-`)); } /** GPT image-tokenization has not been validated across the whole OpenAI diff --git a/tests/public-api.test.ts b/tests/public-api.test.ts index 6145694..bad6b9b 100644 --- a/tests/public-api.test.ts +++ b/tests/public-api.test.ts @@ -28,6 +28,28 @@ describe('public library API', () => { expect(isPxpipeSupportedModel(null)).toBe(false); }); + it('strips bracketed variant tags like [1m] before matching', () => { + expect(isPxpipeSupportedModel('claude-fable-5[1m]')).toBe(true); + expect(isPxpipeSupportedModel('claude-fable-5-high[1m]')).toBe(true); + // a non-allowed base keeps being rejected even with a variant tag + expect(isPxpipeSupportedModel('claude-opus-4-8[1m]')).toBe(false); + }); + + it('honors PXPIPE_MODELS to widen the gate without a code change', () => { + const prev = process.env.PXPIPE_MODELS; + try { + process.env.PXPIPE_MODELS = 'claude-fable-5,claude-opus-4-8'; + expect(isPxpipeSupportedModel('claude-opus-4-8')).toBe(true); + expect(isPxpipeSupportedModel('claude-opus-4-8[1m]')).toBe(true); + expect(isPxpipeSupportedModel('claude-fable-5')).toBe(true); // still listed + expect(isPxpipeSupportedModel('claude-opus-4-7')).toBe(false); // not listed + expect(isPxpipeSupportedModel('claude-sonnet-4-7')).toBe(false); + } finally { + if (prev === undefined) delete process.env.PXPIPE_MODELS; + else process.env.PXPIPE_MODELS = prev; + } + }); + it('recognizes only the GPT 5.5 family for OpenAI chat support', () => { expect(isPxpipeSupportedGptModel('gpt-5.5')).toBe(true); expect(isPxpipeSupportedGptModel('gpt-5.5-codex')).toBe(true);