diff --git a/src/core/factsheet.ts b/src/core/factsheet.ts new file mode 100644 index 0000000..f0244a8 --- /dev/null +++ b/src/core/factsheet.ts @@ -0,0 +1,79 @@ +/** + * Verbatim fact-sheet for imaged content. + * + * When pxpipe renders a block (system slab, history, tool_result, reminder) to a PNG, + * the precision-critical, hard-to-OCR strings inside it — file paths, URLs, SHAs/UUIDs, + * version numbers, CLI flags, large numbers, CONST_IDS — are exactly what a model is + * most likely to misread off the image yet most likely to need quoted verbatim. This + * module extracts those tokens so they ride next to the image as plain text: the model + * quotes them without re-reading the PNG, and they stay in the cached prefix. + * + * Deterministic by construction (fixed pattern order, length-desc/lexical sort, no + * Date/random) → the emitted text is byte-stable across turns and never busts the + * Anthropic prompt cache. Empirically ~5% of source chars on production history + * (median 4.9%, max 12.1%, N=10), which preserves the imaging token win. + */ + +/** ReDoS-safe extraction patterns (each global). Ordered most- to least-specific so the + * longest, most-identifying tokens are kept first when the substring filter runs. */ +const PATTERNS: readonly RegExp[] = [ + /\bhttps?:\/\/[^\s)"'<>]+/g, // URLs + /\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g, // UUID + /(?:[\w@~+-]+)?(?:\/[\w.@+-]+)+\.[A-Za-z]\w{0,8}\b/g, // path with a file extension (multi-dot ok: .test.ts) + /\/[\w.@+-]+(?:\/[\w.@+-]+)+\/?/g, // dir path (>=2 segments) + /\b(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b/g, // git sha / long hex (must contain a digit) + /\bv?\d+\.\d+(?:\.\d+)?(?:[-+][\w.]+)?\b/g, // version string + /(?:^|[^\w-])(--?[A-Za-z][\w-]+)/g, // CLI flag (token in capture group 1) + /\b\d[\d,_]{3,}\b/g, // large / separated number + /\b\d+\.\d+\b/g, // decimal + /\b[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+\b/g, // CONST_IDS / env var names +]; + +const MIN_LEN = 3; +const MAX_LEN = 120; +const MAX_TOKENS = 64; // budget cap per block — longest/most-specific kept first +const MAX_SCAN = 262_144; // defensive input bound; tool_results are already paged +const MAX_CHUNK = 512; // whitespace-free chunks longer than this are blobs (base64, minified) — skip + +/** + * Extract deduped, precision-critical tokens from `text`, longest-first, with any token + * that is a substring of a longer kept token dropped (so `/github.com` inside the full + * URL, `lib/x.ts` inside `src/lib/x.ts`, etc. collapse to the most specific form). + * + * Every token class is whitespace-free, so we split on whitespace first and skip + * blob-length chunks. That bounds each regex to a short chunk and keeps extraction + * strictly O(n) — no quadratic backtracking on delimiter-heavy input like base64 or + * minified bundles (which embed `/` and would otherwise make the path patterns blow up). + */ +export function extractFactSheetTokens(text: string): string[] { + const scan = text.length > MAX_SCAN ? text.slice(0, MAX_SCAN) : text; + const seen = new Set(); + for (const chunk of scan.split(/\s+/)) { + if (chunk.length < MIN_LEN || chunk.length > MAX_CHUNK) continue; + for (const re of PATTERNS) { + for (const m of chunk.matchAll(re)) { + // Strip trailing sentence punctuation pulled in from prose (`pull/93.` → `pull/93`); + // no real identifier we extract ends in these. + const tok = (m[1] ?? m[0]).trim().replace(/[.,;:!?]+$/, ''); + if (tok.length >= MIN_LEN && tok.length <= MAX_LEN) seen.add(tok); + } + } + } + // Total order: length desc, then lexical — independent of Set iteration / sort stability. + const ordered = [...seen].sort((a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0)); + const kept: string[] = []; + for (const t of ordered) { + if (kept.length >= MAX_TOKENS) break; + if (!kept.some((k) => k.includes(t))) kept.push(t); + } + return kept; +} + +const OPEN = + '[Exact identifiers from the rendered context above (paths, ids, versions, numbers) — quote these verbatim instead of transcribing them from the image: '; + +/** One-line fact-sheet string for `text`, or `''` when nothing notable was found. */ +export function factSheetText(text: string): string { + const toks = extractFactSheetTokens(text); + return toks.length > 0 ? OPEN + toks.join(' · ') + ']' : ''; +} diff --git a/src/core/history.ts b/src/core/history.ts index 76550a9..401d941 100644 --- a/src/core/history.ts +++ b/src/core/history.ts @@ -13,6 +13,7 @@ import type { CacheControl, ContentBlock, ImageBlock, Message, TextBlock, ToolUseBlock, ToolResultBlock } from './types.js'; import { DENSE_CONTENT_CHARS_PER_IMAGE, DENSE_CONTENT_COLS, DENSE_RENDER_STYLE, neutralizeSentinel, reflow, renderTextToPngsWithCharLimit, roleSlotSegment, SLOT_MARK_ASSISTANT, SLOT_MARK_USER } from './render.js'; +import { factSheetText } from './factsheet.js'; import { bytesToBase64 } from './png.js'; /** @@ -479,10 +480,12 @@ export async function collapseHistory( return { messages, info }; } const latestUserPointer = latestCollapsedUserPointer(messages, collapseLen, protectedPrefix); + const historyFactSheet = factSheetText(text); const syntheticContent: ContentBlock[] = [ { type: 'text', text: HISTORY_SYNTHETIC_INTRO }, ...imageBlocks, ...(latestUserPointer ? [latestUserPointer] : []), + ...(historyFactSheet ? [{ type: 'text' as const, text: historyFactSheet }] : []), { type: 'text', text: HISTORY_SYNTHETIC_OUTRO }, ]; const syntheticUser: Message = { diff --git a/src/core/openai.ts b/src/core/openai.ts index 9ae6bc2..2cdc384 100644 --- a/src/core/openai.ts +++ b/src/core/openai.ts @@ -36,6 +36,7 @@ import { type GptHistoryOptions, } from './openai-history.js'; import { HISTORY_SYNTHETIC_INTRO, HISTORY_SYNTHETIC_OUTRO } from './history.js'; +import { factSheetText } from './factsheet.js'; import { countTokens as o200kCountTokens } from 'gpt-tokenizer/encoding/o200k_base'; // Per-model GPT rendering + vision-cost profiles (portrait-strip width, image-token @@ -673,10 +674,15 @@ export async function transformOpenAIChatCompletions( info.imagePngs = images.map((img) => img.png); info.imageDims = images.map((img) => ({ width: img.width, height: img.height })); + // Verbatim fact-sheet: precision-critical tokens (paths, ids, versions, flags) + // pulled from the pre-image text so exact strings survive OCR loss. Deterministic + // → stays inside the cached prefix. See src/core/factsheet.ts. + const slabFactSheet = factSheetText(combinedRaw); const slabUserMsg: OpenAIChatMessage = { role: 'user', content: [ ...imageParts, + ...(slabFactSheet ? [{ type: 'text', text: slabFactSheet } as OpenAIContentPart] : []), { type: 'text', text: '[End of rendered GPT system/tool context.]' }, ], }; @@ -716,6 +722,9 @@ export async function transformOpenAIChatCompletions( content.push({ type: 'text', text: pinnedRequestBlock(plan.pinText) }); for (const img of plan.imagesAfter) content.push(openAIImagePart(img)); } + // Verbatim fact-sheet for the imaged transcript (exact ids survive OCR loss). + const histFactSheet = factSheetText(plan.text); + if (histFactSheet) content.push({ type: 'text', text: histFactSheet }); content.push({ type: 'text', text: HISTORY_TRANSCRIPT_OUTRO }); const synthetic: OpenAIChatMessage = { role: 'user', content }; const guard: OpenAIChatMessage = { @@ -870,6 +879,11 @@ export async function transformOpenAIResponses( const imagePartsResp: ResponsesInputImagePart[] = images.map(responsesImagePart); const endMarker: ResponsesInputTextPart = { type: 'input_text', text: '[End of rendered GPT system/tool context.]' }; + // Verbatim fact-sheet (see src/core/factsheet.ts): exact tokens that survive OCR loss. + const slabFactSheet = factSheetText(combinedRaw); + const slabFactSheetPart: ResponsesInputTextPart[] = slabFactSheet + ? [{ type: 'input_text', text: slabFactSheet }] + : []; if (inputWasString) { // Wrap bare string input into a user item with images prepended. @@ -877,6 +891,7 @@ export async function transformOpenAIResponses( role: 'user', content: [ ...imagePartsResp, + ...slabFactSheetPart, endMarker, { type: 'input_text', text: originalInputString! }, ], @@ -887,7 +902,7 @@ export async function transformOpenAIResponses( // and protecting it made stale first-turn requests look live. const slabUserItem: ResponsesInputItem = { role: 'user', - content: [...imagePartsResp, endMarker], + content: [...imagePartsResp, ...slabFactSheetPart, endMarker], }; inputItems = [ ...inputItems.slice(0, firstUserIdx), @@ -940,6 +955,9 @@ export async function transformOpenAIResponses( content.push({ type: 'input_text', text: pinnedRequestBlock(plan.pinText) }); for (const img of plan.imagesAfter) content.push(responsesImagePart(img)); } + // Verbatim fact-sheet for the imaged transcript (exact ids survive OCR loss). + const histFactSheet = factSheetText(plan.text); + if (histFactSheet) content.push({ type: 'input_text', text: histFactSheet }); content.push({ type: 'input_text', text: HISTORY_TRANSCRIPT_OUTRO }); const synthetic: ResponsesInputItem = { role: 'user', content }; const guard: ResponsesInputItem = { diff --git a/src/core/transform.ts b/src/core/transform.ts index 9de07de..a14fd5d 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -34,6 +34,7 @@ import { DENSE_RENDER_STYLE, renderTextToPngsWithCharLimit, } from './render.js'; +import { factSheetText } from './factsheet.js'; import { bytesToBase64 } from './png.js'; import { collapseHistory, HISTORY_SYNTHETIC_INTRO } from './history.js'; import type { GptHistoryOptions } from './openai-history.js'; @@ -1597,6 +1598,8 @@ export async function transformRequest( processedExisting.push(out as ImageBlock); info.imageBytes += approxBlockBytes(img); } + const reminderFactSheet = factSheetText(reminderRaw); + if (reminderFactSheet) processedExisting.push({ type: 'text', text: reminderFactSheet }); info.imagePixels = (info.imagePixels ?? 0) + pixels; info.reminderImgs = (info.reminderImgs ?? 0) + imgs.length; await recordRecoverable(info, o.emitRecoverable, { @@ -1616,8 +1619,10 @@ export async function transformRequest( processedExisting.push(...existing); } + const slabFactSheet = factSheetText(combinedRaw); m.content = [ ...imageBlocks, + ...(slabFactSheet ? [{ type: 'text' as const, text: slabFactSheet }] : []), { type: 'text' as const, text: '[End of rendered context.]' }, ...processedExisting, ]; @@ -1681,7 +1686,11 @@ export async function transformRequest( for (const [cp, n] of dcp) { droppedCodepoints.set(cp, (droppedCodepoints.get(cp) ?? 0) + n); } - rewritten.push({ ...tr, content: imgs }); + const trFactSheet = factSheetText(innerRaw); + rewritten.push({ + ...tr, + content: trFactSheet ? [...imgs, { type: 'text' as const, text: trFactSheet }] : imgs, + }); changed = true; bumpBucket(info, toolResultBucket(classifyContent(inner)), innerRaw.length); } @@ -1738,6 +1747,8 @@ export async function transformRequest( newInner.push(out as ImageBlock); info.imageBytes += approxBlockBytes(img); } + const partFactSheet = factSheetText(innerTextRaw); + if (partFactSheet) newInner.push({ type: 'text', text: partFactSheet }); info.imagePixels = (info.imagePixels ?? 0) + pixels; info.toolResultImgs = (info.toolResultImgs ?? 0) + imgs.length; info.imageCount += imgs.length; diff --git a/tests/design-behavior-e2e.test.ts b/tests/design-behavior-e2e.test.ts index 2f77cee..e3d157f 100644 --- a/tests/design-behavior-e2e.test.ts +++ b/tests/design-behavior-e2e.test.ts @@ -123,11 +123,13 @@ describe('design: HISTORY COLLAPSE (Anthropic)', () => { ); const hay = JSON.stringify(out); expect(imageCount(out)).toBeGreaterThan(0); - // Recent turns survive as text (the working set the model still reasons over). - expect(hay).toContain('TURNMARK_29 '); - expect(hay).toContain('TURNMARK_28 '); - // A mid-history turn was collapsed into an image → not present as text. - expect(hay).not.toContain('TURNMARK_5 '); + // Recent turns survive as legible text BODY (the working set the model still reasons over). + expect(hay).toContain('TURNMARK_29 ' + 'x'.repeat(100)); + expect(hay).toContain('TURNMARK_28 ' + 'x'.repeat(100)); + // A mid-history turn's BODY was collapsed into an image → its content is not legible text. + // Its bare identifier may appear in the verbatim fact-sheet beside the image (by design — + // precision-critical tokens are preserved as text); the 4000-char body is not. + expect(hay).not.toContain('TURNMARK_5 ' + 'x'.repeat(100)); }); }); diff --git a/tests/factsheet.test.ts b/tests/factsheet.test.ts new file mode 100644 index 0000000..92f4677 --- /dev/null +++ b/tests/factsheet.test.ts @@ -0,0 +1,46 @@ +import { describe, it, expect } from 'vitest'; +import { extractFactSheetTokens, factSheetText } from '../src/core/factsheet.js'; + +describe('factsheet extraction', () => { + it('captures precision-critical, hard-to-OCR tokens', () => { + const text = [ + 'Edited src/lib/__tests__/livekit-egress.test.ts and agents/transcription/agent.ts', + 'opened https://github.com/Keplogic/atlas/pull/93 at commit 6d80bd6', + 'set LIVEKIT_API_SECRET and ran with --max-tokens 64000, coverage 97.82', + ].join('\n'); + const toks = extractFactSheetTokens(text); + expect(toks).toContain('src/lib/__tests__/livekit-egress.test.ts'); + expect(toks).toContain('https://github.com/Keplogic/atlas/pull/93'); + expect(toks).toContain('6d80bd6'); + expect(toks).toContain('LIVEKIT_API_SECRET'); + expect(toks).toContain('--max-tokens'); + expect(toks).toContain('97.82'); + }); + + it('drops substrings of longer kept tokens', () => { + const toks = extractFactSheetTokens('see https://github.com/o/r/pull/9 in repo'); + // The bare /github.com path must collapse into the full URL. + expect(toks).toContain('https://github.com/o/r/pull/9'); + expect(toks).not.toContain('/github.com'); + }); + + it('does not flag pure-letter hex words (decade, facade)', () => { + const toks = extractFactSheetTokens('this decade the facade was added'); + expect(toks).not.toContain('decade'); + expect(toks).not.toContain('facade'); + }); + + it('is deterministic — identical input yields byte-identical output (cache stability)', () => { + const text = 'paths /a/b/c.ts /d/e/f.ts ids 1a2b3c4 9f8e7d6 nums 12345 6789.0 FLAG_X FLAG_Y'; + expect(factSheetText(text)).toBe(factSheetText(text)); + }); + + it('returns empty string when nothing notable is present', () => { + expect(factSheetText('the quick brown fox jumps over')).toBe(''); + }); + + it('caps the token budget', () => { + const many = Array.from({ length: 200 }, (_, i) => `/dir${i}/file${i}.ts`).join(' '); + expect(extractFactSheetTokens(many).length).toBeLessThanOrEqual(64); + }); +}); diff --git a/tests/openai-gpt5.test.ts b/tests/openai-gpt5.test.ts index e46cb2d..9e2320c 100644 --- a/tests/openai-gpt5.test.ts +++ b/tests/openai-gpt5.test.ts @@ -413,7 +413,10 @@ describe('transformOpenAIResponses — history collapse', () => { expect((out.input[historyIdx + 1] as { role?: string }).role).toBe('developer'); expect(JSON.stringify(out.input[historyIdx + 1])).toContain('live current request'); const serialized = JSON.stringify(out.input); - expect(serialized).not.toContain(OPENING_PROMPT_MARKER); + // The opening prompt's BODY was collapsed into an image → its legible text is gone. + // Its bare marker may surface once in the verbatim fact-sheet beside the image (by + // design — precision-critical ids are kept as text); the repeated body does not. + expect(serialized).not.toContain(`${OPENING_PROMPT_MARKER} ${OPENING_PROMPT_MARKER}`); expect(serialized).toContain(LIVE_PROMPT_MARKER); // The recent tail is still raw text items (function_call / user), not collapsed. const lastUser = [...out.input].reverse().find( @@ -501,7 +504,10 @@ describe('transformOpenAIChatCompletions — history collapse', () => { expect((out.messages[historyIdx + 1] as { role?: string }).role).toBe('developer'); expect(JSON.stringify(out.messages[historyIdx + 1])).toContain('live current request'); const serialized = JSON.stringify(out.messages); - expect(serialized).not.toContain(OPENING_PROMPT_MARKER); + // The opening prompt's BODY was collapsed into an image → its legible text is gone. + // Its bare marker may surface once in the verbatim fact-sheet beside the image (by + // design — precision-critical ids are kept as text); the repeated body does not. + expect(serialized).not.toContain(`${OPENING_PROMPT_MARKER} ${OPENING_PROMPT_MARKER}`); expect(serialized).toContain(LIVE_PROMPT_MARKER); }); });