#!/usr/bin/env node // Probe 2: OCR accuracy vs glyph↔patch phase, using the PRODUCTION renderer. // gcd(CELL_W=5,28)=1 → one image contains all 28 horizontal phases; // gcd(CELL_H=8,28)=4 → 7 vertical phases. Content is gibberish lowercase // 4-letter groups (no language prior, not credential-shaped → avoids the // refusal classifier that fires on mixed-case alnum strings). Every 5th cell // is a space; cols must be a multiple of lcm(5,28)=140 so each phase sees an // equal number of letter cells (4 letters + 1 space per phase per 140 cols). // // Usage: CC_OAUTH_TOKEN=... node accuracy-phase-probe.mjs [model] [cols] [rows] [seed] // cols: multiple of 140, rows: multiple of 7. // NOTE: real inference — costs output tokens. import { writeFile } from 'node:fs/promises'; import { renderTextToPngs, PAD_X, PAD_Y, CELL_W, CELL_H } from '../../dist/core/render.js'; const model = process.argv[2] ?? 'claude-fable-5'; const COLS = +(process.argv[3] ?? 140); const ROWS = +(process.argv[4] ?? 21); const seedArg = process.argv[5] ?? '1'; const PADL = +(process.argv[6] ?? 0); // blank lines prepended to the IMAGE only: shifts row phase by PADL*CELL_H px, content identical const fileMode = !/^\d+$/.test(seedArg); // non-numeric 5th arg = path to a real text file const seed = fileMode ? 1 : +seedArg; const P = 28; if (!fileMode && (COLS % 140 || ROWS % 7)) console.error(`warn: cols%140=${COLS % 140} rows%7=${ROWS % 7} — phases unbalanced`); let s = (seed >>> 0) || 1; const rnd = () => (s ^= s << 13, s ^= s >>> 17, s ^= s << 5, (s >>> 0) / 2 ** 32); // Random-order REAL dictionary words: refusal-safe, production-representative. // Language prior is phase-independent, so straddle effects survive as relative // error-rate differences. Phase balance is statistical, not exact (per-phase // totals are tracked, so unevenness is handled in the rates). const { readFileSync } = await import('node:fs'); let grid; if (fileMode) { // Production-representative content: real source text, pre-wrapped to COLS. grid = readFileSync(seedArg, 'utf8').split('\n') .map(l => l.replace(/\t/g, ' ').replace(/[^\x20-\x7e]/g, '?').trimEnd().slice(0, COLS)) .filter(l => l.trim().length >= 8) .slice(0, ROWS); if (grid.length < ROWS) console.error(`warn: file only yielded ${grid.length} lines`); } else { const WORDS = readFileSync('/usr/share/dict/words', 'utf8').split('\n') .filter(w => /^[a-z]{3,7}$/.test(w)); grid = Array.from({ length: ROWS }, () => { let line = ''; for (;;) { const w = WORDS[(rnd() * WORDS.length) | 0]; if (line.length + w.length + (line ? 1 : 0) > COLS) break; line += (line ? ' ' : '') + w; } return line; }); } const imgs = await renderTextToPngs('\n'.repeat(PADL) + grid.join('\n'), COLS); if (PADL) console.error(`padLines=${PADL} (row shift ${PADL * CELL_H}px, phase +${(PADL * CELL_H) % 28} mod 28)`); if (imgs.length !== 1) throw new Error(`expected 1 image, got ${imgs.length}`); const img = imgs[0]; const png = img.png ?? img.data ?? img.buffer ?? Object.values(img).find(v => v instanceof Uint8Array); if (!png) throw new Error('no png buffer; keys=' + Object.keys(img).join(',')); console.error(`image: ${img.width ?? '?'}x${img.height ?? '?'}px ${png.length}B; predicted image_tokens=` + (img.width && img.height ? 3 + Math.ceil(img.width / P) * Math.ceil(img.height / P) : '?')); const BASE = process.env.ANTHROPIC_BASE_URL ?? 'https://api.anthropic.com'; const res = await fetch(`${BASE}/v1/messages`, { method: 'POST', headers: { 'authorization': `Bearer ${process.env.CC_OAUTH_TOKEN}`, 'anthropic-version': '2023-06-01', 'anthropic-beta': 'oauth-2025-04-20', 'content-type': 'application/json', }, body: JSON.stringify({ model, max_tokens: ROWS * (COLS + 2) + 1500, system: "You are Claude Code, Anthropic's official CLI for Claude.", messages: [{ role: 'user', content: [ { type: 'image', source: { type: 'base64', media_type: 'image/png', data: Buffer.from(png).toString('base64') } }, { type: 'text', text: `OCR this image to plain text. Preserve line breaks. Output only the text.` }, ], }], }), }); const j = await res.json(); if (!res.ok) { console.error('API error:', JSON.stringify(j)); process.exit(1); } await writeFile(`/tmp/phase-probe-${model}-${Date.now()}.json`, JSON.stringify({ model, cols: COLS, rows: ROWS, seedArg, padLines: PADL, grid, resp: j }, null, 2)); const out = (j.content?.find(b => b.type === 'text')?.text ?? '') .replace(/```[a-z]*\n?/g, '').split('\n').map(l => l.trimEnd()).filter(l => l.length); console.error(`usage=${JSON.stringify(j.usage)} stop=${j.stop_reason} lines=${out.length}/${ROWS}`); if (out.length < ROWS) console.error('rawTextHead: ' + JSON.stringify((j.content?.find(b => b.type === 'text')?.text ?? '').slice(0, 200))); // Alignment-based scoring: Levenshtein backtrace marks which truth positions // matched exactly; everything else (sub or indel) is an error at its truth pos. function alignOk(truth, got, subs) { const T = truth.length, G = got.length; const dp = Array.from({ length: T + 1 }, () => new Array(G + 1).fill(0)); for (let i = 0; i <= T; i++) dp[i][0] = i; for (let jj = 0; jj <= G; jj++) dp[0][jj] = jj; for (let i = 1; i <= T; i++) for (let jj = 1; jj <= G; jj++) dp[i][jj] = Math.min(dp[i - 1][jj - 1] + (truth[i - 1] === got[jj - 1] ? 0 : 1), dp[i - 1][jj] + 1, dp[i][jj - 1] + 1); const ok = new Array(T).fill(false); let i = T, jj = G; while (i > 0 && jj > 0) { if (dp[i][jj] === dp[i - 1][jj - 1] + (truth[i - 1] === got[jj - 1] ? 0 : 1)) { if (truth[i - 1] === got[jj - 1]) ok[i - 1] = true; else subs.set(`${truth[i - 1]}->${got[jj - 1]}`, (subs.get(`${truth[i - 1]}->${got[jj - 1]}`) ?? 0) + 1); i--; jj--; } else if (dp[i][jj] === dp[i - 1][jj] + 1) i--; else jj--; } return ok; } // Line-level alignment BEFORE char scoring: the model may emit preamble lines, // wrap long lines (1 truth : 2 out), or drop lines. Positional r->r pairing // turns one such slip into a phase-flat error smear. Banded DP, merge-aware. function lev(a, b) { const m = b.length; let prev = Array.from({ length: m + 1 }, (_, k) => k), cur = new Array(m + 1); for (let i = 1; i <= a.length; i++) { cur[0] = i; for (let k = 1; k <= m; k++) cur[k] = Math.min(prev[k - 1] + (a[i - 1] === b[k - 1] ? 0 : 1), prev[k] + 1, cur[k - 1] + 1); [prev, cur] = [cur, prev]; } return prev[m]; } function alignLines(truth, got) { const T = truth.length, G = got.length, BAND = 25, INF = 1e9; const dp = Array.from({ length: T + 1 }, () => new Array(G + 1).fill(INF)); const bt = Array.from({ length: T + 1 }, () => new Array(G + 1).fill(null)); dp[0][0] = 0; for (let jj = 1; jj <= G; jj++) { dp[0][jj] = dp[0][jj - 1] + 2; bt[0][jj] = ['spur']; } for (let i = 1; i <= T; i++) { dp[i][0] = dp[i - 1][0] + truth[i - 1].length; bt[i][0] = ['miss']; for (let jj = Math.max(1, i - BAND); jj <= Math.min(G, i + BAND); jj++) { let c = dp[i - 1][jj - 1] + lev(truth[i - 1], got[jj - 1]), b = ['m11']; if (jj >= 2 && dp[i - 1][jj - 2] < INF) { const merged = [got[jj - 2] + ' ' + got[jj - 1], got[jj - 2] + got[jj - 1]]; for (const mtxt of merged) { const c2 = dp[i - 1][jj - 2] + lev(truth[i - 1], mtxt); if (c2 < c) { c = c2; b = ['m12', mtxt]; } } } if (dp[i][jj - 1] + 2 < c) { c = dp[i][jj - 1] + 2; b = ['spur']; } if (dp[i - 1][jj] + truth[i - 1].length < c) { c = dp[i - 1][jj] + truth[i - 1].length; b = ['miss']; } dp[i][jj] = c; bt[i][jj] = b; } } const matched = new Array(T).fill(null); let i = T, jj = G; while ((i > 0 || jj > 0) && bt[i][jj]) { const b = bt[i][jj]; if (b[0] === 'spur') jj--; else if (b[0] === 'miss') i--; else if (b[0] === 'm12') { matched[i - 1] = b[1]; i--; jj -= 2; } else { matched[i - 1] = got[jj - 1]; i--; jj--; } } return matched; } const mk = () => ({ err: new Array(P).fill(0), tot: new Array(P).fill(0) }); const col = mk(), row = mk(); let errs = 0, tot = 0, missedLines = 0; const subs = new Map(); const matched = alignLines(grid, out); for (let r = 0; r < ROWS && r < grid.length; r++) { const truth = grid[r]; if (matched[r] == null) { missedLines++; continue; } // structural, not phase-scorable const ok = alignOk(truth, matched[r], subs); for (let c = 0; c < truth.length; c++) { if (truth[c] === ' ') continue; // letters only const cp = (PAD_X + c * CELL_W) % P, rp = (PAD_Y + (r + PADL) * CELL_H) % P; col.tot[cp]++; row.tot[rp]++; tot++; if (!ok[c]) { errs++; col.err[cp]++; row.err[rp]++; } } } const agg = (m, pred) => { let e = 0, t = 0; for (let p = 0; p < P; p++) if (m.tot[p] && pred(p)) { e += m.err[p]; t += m.tot[p]; } return t ? `${(100 * e / t).toFixed(2)}% (${e}/${t})` : 'n/a'; }; console.log(`model=${model} cols=${COLS} rows=${ROWS} seed=${seed}`); console.log(`overall acc=${(100 * (1 - errs / tot)).toFixed(2)}% errs=${errs}/${tot} scoredLines=${grid.length - missedLines}/${ROWS} missedLines=${missedLines} rawOutLines=${out.length}`); console.log(`colStraddle(x%28>=24): ${agg(col, p => p >= 24)} vs aligned: ${agg(col, p => p < 24)}`); console.log(`rowStraddle(y%28>=21): ${agg(row, p => p >= 21)} vs aligned: ${agg(row, p => p < 21)}`); console.log('axis,phase,err,tot,errPct'); for (let p = 0; p < P; p++) if (col.tot[p]) console.log(`col,${p},${col.err[p]},${col.tot[p]},${(100 * col.err[p] / col.tot[p]).toFixed(1)}`); for (let p = 0; p < P; p++) if (row.tot[p]) console.log(`row,${p},${row.err[p]},${row.tot[p]},${(100 * row.err[p] / row.tot[p]).toFixed(1)}`); console.log('topConfusions: ' + [...subs.entries()].sort((a, b) => b[1] - a[1]).slice(0, 12).map(([k, n]) => `${k}×${n}`).join(' '));