From 59a416cb09d32b3d86cbb27c2e98b3b830e92c7b Mon Sep 17 00:00:00 2001 From: teamchong <25894545+teamchong@users.noreply.github.com> Date: Tue, 19 May 2026 23:54:11 -0400 Subject: [PATCH] feat(history): always-on history compression + fix gate-vs-renderer asymmetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rip the compressHistory opt-in and run Variant C history-image collapse on every request that has a closed-prefix run. There is now ONE codepath: no toggle, no env switch, no per-request opt-in. The reason the option was defaulted-off was a -250% live measurement on 2026-05-19 (commit c3506e3 era); that wasn't a "marginal upside" problem, it was a math bug. This commit fixes the bug and turns the feature on. Root cause of the -250% loss ---------------------------- `isCompressionProfitable()` has two paths: a row-aware path (string input, matches `renderTextToPngs` image budgeting exactly) and a looser chars-only path (number input, assumes dense lines). History.ts was calling it with `text.length` (number) → loose chars-only estimate. History text is newline-heavy (`--- role ---` headers, JSON arg blocks, [tool_use] / [tool_result] labels), so the chars-only estimate under-counted images by 5-10× and let net-losers through the gate. Secondary contributor: tool_use args were serialised with `JSON.stringify(input, null, 2)` — pretty-printed, one short row per JSON field. At cols=100 each field claimed a full row instead of wrapping at ~100 chars, inflating image count for tool-heavy histories. Changes ------- - src/core/history.ts - `ProfitableFn` signature: `(text: string, cols: number)` (was `textLen: number`). Documented the asymmetry so callers can't regress this. - `blocksToText` tool_use: compact `JSON.stringify(input)`, no indent. - Pass `text` (not `text.length`) into the profitability check. - src/core/transform.ts - Delete `compressHistory`, `historyKeepTail`, `historyMinPrefix` from TransformOptions and DEFAULTS. - History collapse is now unconditional when `messages[]` is non-empty. - Wrap the gate in a closure that pins `numCols=1` + per-request `cpt`, so the gate's row-aware decision is byte-identical to the single-col renderer's actual image count. - src/worker.ts: drop `COMPRESS_HISTORY`, `HISTORY_KEEP_TAIL`, `HISTORY_MIN_PREFIX` env vars from `Env` + transform options. - src/node.ts: stale comment refresh — no toggles, period. - tests/history.test.ts: - Rename describe "transformRequest + compressHistory" → "transformRequest history compression (always-on)". - Remove every `compressHistory: true, historyKeepTail: X, historyMinPrefix: Y` from transformRequest calls — options no longer exist. - Bump fixtures for default `keepTail=4` / `minCollapsePrefix=10` (14-turn body, 3500 chars/turn) so the now-honest row-aware gate accepts. Renamed "8-closed + 2-live" → "10-closed + 4-live" to match the new defaults. - Bump straddle-test per-turn body 2500→3500 chars: tool block labels add ~65 chars of header overhead that pushed the tighter fixture under the row-aware boundary. - Rename "JSON-pretty args" test → "COMPACT JSON args"; assert no pretty indentation. Verification ------------ - npm test → 263/263 green - npm run typecheck → clean - npm run build → dist/node.js builds clean --- src/core/history.ts | 30 ++++++++++++++----- src/core/transform.ts | 68 +++++++++++++++++-------------------------- src/node.ts | 6 ++-- src/worker.ts | 11 ------- tests/history.test.ts | 63 ++++++++++++++++++++------------------- 5 files changed, 85 insertions(+), 93 deletions(-) diff --git a/src/core/history.ts b/src/core/history.ts index afd9620..23f7423 100644 --- a/src/core/history.ts +++ b/src/core/history.ts @@ -44,8 +44,18 @@ import { bytesToBase64 } from './png.js'; * caller (transform.ts) rather than imported here to keep `src/core/history.ts` * free of a cycle with `src/core/transform.ts`. transform.ts already imports * history.ts to invoke `collapseHistory`; importing back the other way would - * create an evaluation-order trap. */ -export type ProfitableFn = (textLen: number, cols: number) => boolean; + * create an evaluation-order trap. + * + * IMPORTANT — takes the full `text`, NOT `text.length`. The downstream + * `isCompressionProfitable` has two paths: a row-aware path for strings + * (matches renderTextToPngs() image budgeting exactly) and a looser + * chars-only fallback for numbers (assumes dense lines, no newlines). + * History text is *newline-heavy* — `--- role ---` headers, JSON args, + * `[tool_use]` / `[tool_result]` labels — so the chars-only estimate + * under-predicts image count by ~5-10× and used to let net-losers + * through. The 2026-05-19 production -250% savings measurement traces + * back to that asymmetry. Always pass the string. */ +export type ProfitableFn = (text: string, cols: number) => boolean; /** Configuration for history collapse. */ export interface HistoryCollapseOptions { @@ -167,13 +177,16 @@ export function blocksToText(content: string | ContentBlock[]): string { break; case 'tool_use': { const tu = blk as ToolUseBlock; - // Render as a labelled block so the model can re-attribute. The - // arg JSON is included verbatim — for very large args this will - // bloat the history image, but the alternative (dropping args) - // breaks model self-attribution worse than the bloat does. + // Render as a labelled block so the model can re-attribute. Args + // are serialised COMPACT (no 2-space indent) — pretty-printing + // bloats the history text ~5× via per-field newlines, which + // multiplies image cost since the renderer is row-aware and + // every JSON field gets its own row. Compact JSON wraps at + // `cols` like normal text, packing ~14k chars per single-col + // image instead of one short line per field. let argsStr: string; try { - argsStr = JSON.stringify(tu.input, null, 2); + argsStr = JSON.stringify(tu.input); } catch { argsStr = String(tu.input); } @@ -287,7 +300,8 @@ export async function collapseHistory( info.reason = 'render_empty'; return { messages, info }; } - if (!isProfitable(text.length, o.cols)) { + // Row-aware: pass the string, not its length. See ProfitableFn jsdoc. + if (!isProfitable(text, o.cols)) { info.reason = 'not_profitable'; info.collapsedChars = text.length; // surface what we DIDN'T compress return { messages, info }; diff --git a/src/core/transform.ts b/src/core/transform.ts index 9b75b9d..adb1c61 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -59,22 +59,6 @@ export interface TransformOptions { * rendering so the request stays under Anthropic's 100-image-per-request * cap even when a single tool dumps a huge log. Default 10. */ maxImagesPerToolResult?: number; - /** Variant C history-image compression: walk `messages[]` from the head, - * find the largest closed-tool-sequence prefix, render its text into one - * prepended user message with image blocks, and collapse those messages - * out of the live tail. Off by default — round-3 spec marked this as - * MARGINAL (~1% per-call cost reduction) with HIGH risk on cache topology. - * Enable opt-in once telemetry confirms the cache breakpoint won't fight - * with Claude Code's own upstream breakpoint placement. */ - compressHistory?: boolean; - /** Number of tail turns to KEEP as text when `compressHistory` is on. The - * most-recent assistant turn (carrying Opus 4.7's thinking signature) is - * always in the tail by construction. Default 4. */ - historyKeepTail?: number; - /** Minimum closed-prefix turn count before we bother collapsing. Cache- - * amortization math from round-3 only pays out at scale — collapsing 2-3 - * turns costs more in image overhead than it saves. Default 10. */ - historyMinPrefix?: number; /** R2 multi-column rendering: pack N text columns side-by-side per image * so each image covers `N×LINES_PER_IMAGE` wrapped lines instead of one. * Default 1 (single column = current behavior). 2 roughly halves image @@ -115,15 +99,6 @@ const DEFAULTS: Required = { // `find` over a big tree or `grep -r` can easily exceed this; the paging // marker tells the model what was elided. Tuneable per session. maxImagesPerToolResult: 10, - // Variant C history-image: OFF. Round-3 spec called this MARGINAL - // (~1× per-call) against HIGH cache-topology risk. Live measurement on - // 2026-05-19 confirmed the warning: with 128-turn history, replacing - // ~21k chars of text with ~140k tokens of imagery LOSES money on every - // request (img cost exceeds text replaced). Re-enable per-deployment - // only after measuring a positive delta on your specific traffic shape. - compressHistory: false, - historyKeepTail: 4, - historyMinPrefix: 10, // English ~4 chars/tok default (= the CHARS_PER_TOKEN constant declared // later in this file — kept as a literal here to avoid forward-reference). // Host overrides per-request when the dashboard's live fit has converged. @@ -395,7 +370,8 @@ export interface TransformInfo { omittedChars?: number; /** Variant C history-image: how many original `messages[]` entries got * collapsed into the prepended synthetic user message. 0 / unset when - * no collapse happened (compressHistory off, no closed prefix, etc.). */ + * no collapse happened (no closed prefix, too few turns, gate rejected + * as not_profitable, etc. — see `historyReason`). */ collapsedTurns?: number; /** Variant C: total chars of text serialized into the history image(s) * before render (pre-OCR loss). */ @@ -1562,24 +1538,32 @@ export async function transformRequest( if (toolsRewritten) req.tools = toolsRewritten; - // 6. Variant C history-image compression. Runs AFTER all per-message - // rewrites so the collapsed prefix reflects final state. Off by default — - // round-3 spec marks the savings (~1% per call) as marginal vs the cache- - // topology risk. When on, walks messages[] back-to-front tracking open - // tool_use_ids; collapses the largest closed-prefix run into one prepended - // synthetic user message. Live tail (keepTail turns + anything in an open - // tool sequence) stays as text. History image carries NO cache_control on - // first ship — the static-slab breakpoint remains the sole pixelpipe - // breakpoint until telemetry shows otherwise. - if (o.compressHistory && Array.isArray(req.messages) && req.messages.length > 0) { + // 6. Variant C history-image compression. ALWAYS-ON, unconditional. + // Runs AFTER all per-message rewrites so the collapsed prefix reflects + // final state. Walks messages[] tracking open tool_use_ids; collapses + // the largest closed-prefix run into one prepended synthetic user + // message. Live tail (HISTORY_DEFAULTS.keepTail turns + anything in an + // open tool sequence) stays as text. History image carries NO + // cache_control — the static-slab breakpoint remains pixelpipe's sole + // breakpoint. + // + // The per-block break-even gate (`isCompressionProfitable`) is passed + // numCols=1 + the request's charsPerToken so its row-aware estimate + // matches the single-col `renderTextToPngs` exactly. This closes the + // 2026-05-19 -250% measurement gap: the old call passed `text.length` + // (number → loose chars-only estimate) which under-counted images by + // 5-10× on newline-heavy history text and let net-losers through. + if (Array.isArray(req.messages) && req.messages.length > 0) { + // Closure that gives the row-aware gate the same numCols/cpt context + // the renderer will use. History is single-col; pinning numCols=1 + // here makes the gate decision identical to the renderer's image + // count after wrapping. + const historyProfitable = (text: string, cols: number): boolean => + isCompressionProfitable(text, cols, undefined, 1, o.charsPerToken); const { messages: newMessages, info: histInfo } = await collapseHistory( req.messages, - isCompressionProfitable, - { - keepTail: o.historyKeepTail, - minCollapsePrefix: o.historyMinPrefix, - cols: o.cols, - }, + historyProfitable, + { cols: o.cols }, ); if (histInfo.collapsedTurns > 0) { req.messages = newMessages; diff --git a/src/node.ts b/src/node.ts index 4ae982b..0a46b34 100644 --- a/src/node.ts +++ b/src/node.ts @@ -419,8 +419,10 @@ async function main(): Promise { const argv = process.argv.slice(2); const opts = parseCli(argv); // Transform options pass through empty — the proxy uses the DEFAULTS - // baked into transform.ts (every compression on, history on, all - // tuning parameters at their measured-best values). Per-request α + // baked into transform.ts. There are no behavior toggles: system slab, + // reminders, tool_results, and history compression all run + // unconditionally; the per-block break-even gate decides per-call + // whether to actually image each piece. Per-request α // injection happens later via the function-form `transform` in // ProxyConfig so the gate gets the dashboard's live empirical rate. const tracker: Tracker = new FileTracker(opts.eventsFile); diff --git a/src/worker.ts b/src/worker.ts index 60e2a25..f68a834 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -24,11 +24,6 @@ export interface Env { COMPRESS_SCHEMAS?: string; COMPRESS_REMINDERS?: string; COMPRESS_TOOL_RESULTS?: string; - /** Variant C history-image compression. OFF by default — round-3 spec - * rated savings as MARGINAL vs HIGH cache-topology risk. */ - COMPRESS_HISTORY?: string; - HISTORY_KEEP_TAIL?: string; - HISTORY_MIN_PREFIX?: string; MIN_COMPRESS_CHARS?: string; MIN_REMINDER_CHARS?: string; MIN_TOOL_RESULT_CHARS?: string; @@ -53,12 +48,6 @@ export default { compressSchemas: truthy(env.COMPRESS_SCHEMAS, true), compressReminders: truthy(env.COMPRESS_REMINDERS, true), compressToolResults: truthy(env.COMPRESS_TOOL_RESULTS, true), - // Variant C history-image: OFF by default. Round-3 spec marks the - // savings as MARGINAL (~1% per-call) against HIGH cache-topology risk. - // Flip via COMPRESS_HISTORY=1 once telemetry confirms safe rollout. - compressHistory: truthy(env.COMPRESS_HISTORY, false), - historyKeepTail: env.HISTORY_KEEP_TAIL ? Number(env.HISTORY_KEEP_TAIL) : 4, - historyMinPrefix: env.HISTORY_MIN_PREFIX ? Number(env.HISTORY_MIN_PREFIX) : 10, minCompressChars: env.MIN_COMPRESS_CHARS ? Number(env.MIN_COMPRESS_CHARS) : 2000, // Raised to 10,000 — per-block break-even point at current renderer // config (Unifont 10px, cell 5×11, 100 cols). Real gate is diff --git a/tests/history.test.ts b/tests/history.test.ts index 933ca69..e5c2e97 100644 --- a/tests/history.test.ts +++ b/tests/history.test.ts @@ -123,7 +123,9 @@ describe('blocksToText', () => { ).toBe('first paragraph\n\nsecond paragraph'); }); - it('serialises tool_use with its name and JSON-pretty args', () => { + it('serialises tool_use with its name and COMPACT JSON args', () => { + // Compact JSON (no 2-space indent) keeps row counts low so the + // history image stays small — pretty-printing inflates rows ~5×. const out = blocksToText([ { type: 'tool_use', @@ -133,8 +135,11 @@ describe('blocksToText', () => { }, ]); expect(out).toContain('[tool_use Read]'); - expect(out).toContain('"file_path": "/etc/hosts"'); - expect(out).toContain('"limit": 50'); + expect(out).toContain('"file_path":"/etc/hosts"'); + expect(out).toContain('"limit":50'); + // Negative: no pretty-print artefacts. + expect(out).not.toContain(' "file_path"'); + expect(out).not.toContain('"limit": 50'); }); it('serialises tool_result string content', () => { @@ -300,9 +305,13 @@ describe('collapseHistory', () => { it('preserves a tool_use sequence that straddles the live-tail boundary', async () => { // 14 turns: 10 closed turns, then an open tool_use at index 10 that closes at index 12. + // Per-turn body bumped to 3500 chars so the row-aware gate (numCols=1) clears + // the per-block break-even point. The tool_use/tool_result block labels + // add ~65 chars of header overhead that pushes a tighter fixture under + // the boundary; 3500-char turns leave headroom. const msgs: Message[] = []; for (let i = 0; i < 10; i++) { - const body = `turn ${i}: ` + 'x'.repeat(2500); + const body = `turn ${i}: ` + 'x'.repeat(3500); msgs.push(i % 2 === 0 ? usr(body) : asst(body)); } msgs.push(asst([{ type: 'tool_use', id: 'X', name: 't', input: {} }])); @@ -359,7 +368,7 @@ describe('collapseHistory', () => { }); }); -describe('transformRequest + compressHistory', () => { +describe('transformRequest history compression (always-on)', () => { function bigPlain(n: number): string { return 'x'.repeat(n); } @@ -393,19 +402,19 @@ describe('transformRequest + compressHistory', () => { expect(Array.isArray(reparsed.messages)).toBe(true); }); - it('compressHistory:true collapses an 8-closed + 2-live conversation', async () => { - // 12 turns total. With keepTail=2 + minPrefix=5, we expect 10 turns to - // collapse into 1 synthetic prepended user + 2 live tail = 3 total. + it('collapses a 10-closed + 4-live conversation under the default keepTail=4', async () => { + // 14 turns total. Default keepTail=4 + minPrefix=10 means 10 turns + // collapse into 1 synthetic prepended user + 4 live tail = 5 total. + // Per-turn body 3500 chars puts the fixture comfortably above the + // row-aware profitability gate (numCols=1 at the renderer means each + // 3500-char `x` line wraps to 35 rows; 10 turns × ~36 rows = ~360 rows + // = 3 single-col images ≪ text-cost). const msgs: Message[] = []; - for (let i = 0; i < 12; i++) { - const body = `turn ${i}: ` + bigPlain(2500); + for (let i = 0; i < 14; i++) { + const body = `turn ${i}: ` + bigPlain(3500); msgs.push(i % 2 === 0 ? usr(body) : asst(body)); } - const { body, info } = await transformRequest(mkBody(msgs, bigPlain(80_000)), { - compressHistory: true, - historyKeepTail: 2, - historyMinPrefix: 5, - }); + const { body, info } = await transformRequest(mkBody(msgs, bigPlain(80_000))); expect(info.collapsedTurns).toBe(10); expect(info.collapsedChars).toBeGreaterThan(0); expect(info.collapsedImages).toBeGreaterThanOrEqual(1); @@ -413,7 +422,7 @@ describe('transformRequest + compressHistory', () => { expect(info.imageCount).toBeGreaterThanOrEqual(1 + (info.collapsedImages ?? 0)); const reparsed = JSON.parse(new TextDecoder().decode(body)); - expect(reparsed.messages.length).toBe(3); // 1 synthetic + 2 live tail + expect(reparsed.messages.length).toBe(5); // 1 synthetic + 4 live tail expect(reparsed.messages[0].role).toBe('user'); const content = reparsed.messages[0].content; expect(Array.isArray(content)).toBe(true); @@ -424,34 +433,28 @@ describe('transformRequest + compressHistory', () => { }); }); - it('compressHistory:true sets historyReason when no closed prefix exists', async () => { - // First message opens a tool_use; nothing closes it. + it('sets historyReason=no_closed_prefix when an open tool_use precedes the tail', async () => { + // First message opens a tool_use; nothing closes it. With default + // keepTail=4 and 4 messages total, cutoff=0, so the boundary search + // runs over an empty range [0..-1] and returns -1 → no_closed_prefix. const msgs: Message[] = [ asst([{ type: 'tool_use', id: 'X', name: 't', input: {} }]), usr('plain'), asst('plain'), usr('plain'), ]; - const { info } = await transformRequest(mkBody(msgs, bigPlain(80_000)), { - compressHistory: true, - historyKeepTail: 1, - historyMinPrefix: 2, - }); + const { info } = await transformRequest(mkBody(msgs, bigPlain(80_000))); expect(info.collapsedTurns).toBeUndefined(); expect(info.historyReason).toBe('no_closed_prefix'); }); it('history-image blocks carry NO cache_control (conservative first-cut)', async () => { const msgs: Message[] = []; - for (let i = 0; i < 12; i++) { - const body = `turn ${i}: ` + bigPlain(2500); + for (let i = 0; i < 14; i++) { + const body = `turn ${i}: ` + bigPlain(3500); msgs.push(i % 2 === 0 ? usr(body) : asst(body)); } - const { body, info } = await transformRequest(mkBody(msgs, bigPlain(80_000)), { - compressHistory: true, - historyKeepTail: 2, - historyMinPrefix: 5, - }); + const { body, info } = await transformRequest(mkBody(msgs, bigPlain(80_000))); expect(info.collapsedImages).toBeGreaterThanOrEqual(1); const reparsed = JSON.parse(new TextDecoder().decode(body)); const synth = reparsed.messages[0];