From e27c19c2a3dffdaef55235770a8c9903e0f0d2db Mon Sep 17 00:00:00 2001 From: teamchong <25894545+teamchong@users.noreply.github.com> Date: Tue, 19 May 2026 14:46:12 -0400 Subject: [PATCH] =?UTF-8?q?feat(transform):=20live-=CE=B1=20charsPerToken?= =?UTF-8?q?=20closes=20not=5Fprofitable=20max-savings=20leak?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The break-even gate (isCompressionProfitable) used a hard-coded CHARS_PER_TOKEN=4 (Anthropic's English avg). Live Claude Code traffic has α≈0.5-0.9 (1.1-2 chars/tok) — JSON-dense tool docs, structured CLAUDE.md slabs, etc. Off-by ~3.5× on the text side at α=0.88. Production trace (events.jsonl, 2026-05-19): 200+ requests with system slabs 113K-170K chars hit reason="not_profitable" because the gate's textTokensEq = len/4 was a fraction of the row-aware image-token cost. Example event: slab=169_632 chars, cache_create_tokens=148_891. Gate's textEq=42K rejected vs imgCost=127K. With live α (1.14 ch/tok), textEq=149K → ACCEPT. We were leaving real cache_create money on the table on every one of those calls. - TransformOptions gains charsPerToken?: number. Default 4 (back-compat). - isCompressionProfitable() takes charsPerToken as a 5th optional param; defensive clamp on ≤0 / NaN / Infinity falls back to CHARS_PER_TOKEN. - All 4 internal gate call sites (static slab + reminder + tool_result string + tool_result array) thread o.charsPerToken through. - ProxyConfig.transform now also accepts a function: () => TransformOptions. Resolved per-request inside createProxy so the host can inject dynamic values. Static-object form unchanged for Workers / tests. - node.ts wraps the static transform options in a closure that asks dashboardState.fitCosts() each request and passes the live chars_per_token. Fit-null falls through to the baked-in default — no chicken-and-egg deadlock since normal-sized slabs still feed the fit ring even before the huge-slab cases get unblocked. - 4 new gate tests: stale-default rejects the 170K newline-heavy slab; live α=1.14 flips the same slab to profitable; defensive clamp on bogus charsPerToken; end-to-end transformRequest passes through. Tests: 269 → 273 (+4). Typecheck + build clean. --- src/core/proxy.ts | 15 ++++++++-- src/core/transform.ts | 39 +++++++++++++++++++++---- src/node.ts | 16 +++++++++- tests/render.test.ts | 68 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 9 deletions(-) diff --git a/src/core/proxy.ts b/src/core/proxy.ts index 417f22c..346e10e 100644 --- a/src/core/proxy.ts +++ b/src/core/proxy.ts @@ -15,8 +15,12 @@ export interface ProxyConfig { upstream?: string; /** Override or supply an API key. If unset, we forward whatever the client sent. */ apiKey?: string; - /** Per-request transform options. */ - transform?: TransformOptions; + /** Per-request transform options. Pass a function when the host wants to + * inject DYNAMIC values per request (e.g. live empirical `charsPerToken` + * from the dashboard's converging fit) — the proxy invokes it once per + * /v1/messages POST. Static object form is used by the Workers host and + * tests that don't need dynamic state. */ + transform?: TransformOptions | (() => TransformOptions); /** Called after every request — useful for logging / metrics in the host. */ onRequest?: (event: ProxyEvent) => void | Promise; } @@ -321,7 +325,12 @@ export function createProxy(config: ProxyConfig = {}) { if (isMessages) { const bodyIn = new Uint8Array(await req.arrayBuffer()); try { - const r = await transformRequest(bodyIn, config.transform); + // Resolve transform options per-request when the host passed a + // function — lets dashboardState.fitCosts()'s live α flow into + // isCompressionProfitable on every call. + const transformOpts = + typeof config.transform === 'function' ? config.transform() : config.transform; + const r = await transformRequest(bodyIn, transformOpts); // Cast: TS narrows Uint8Array away from BodyInit, but // it's a valid body and we never use SharedArrayBuffer. bodyOut = r.body as unknown as BodyInit; diff --git a/src/core/transform.ts b/src/core/transform.ts index 53ad666..7c4b94a 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -84,6 +84,19 @@ export interface TransformOptions { * off until a smoke test against the real slab confirms ordering. * Auto-clamped if the resulting canvas would exceed 1568 px wide. */ multiCol?: number; + /** Chars-per-token assumption used by `isCompressionProfitable()`. Default + * 4 (Anthropic's published English-text average ≈ α=0.25). The host can + * inject the live empirical `chars_per_token` from `dashboardState. + * fitCosts()` per-request — content with dense tokenization (JSON-heavy, + * tool definitions, Claude Code's CLAUDE.md slabs at α≈0.5-0.9) makes + * the stale 4 chars/tok wildly under-estimate text-token equivalence, + * and the gate then rejects compressions that would actually save real + * tokens. With the live value plumbed in, the gate auto-tightens or + * loosens to match the model's actual tokenization rate. + * Production trace 2026-05-19: 200+ events with slabs 113K-170K chars + * hit `not_profitable` because the gate used 4 chars/tok while live α + * was ≈0.88 (1.14 chars/tok) — gate was off by ~3.5× on the text side. */ + charsPerToken?: number; } const DEFAULTS: Required = { @@ -118,6 +131,10 @@ const DEFAULTS: Required = { compressHistory: false, historyKeepTail: 4, historyMinPrefix: 10, + // English ~4 chars/tok default (= the CHARS_PER_TOKEN constant declared + // later in this file — kept as a literal here to avoid forward-reference). + // Host overrides per-request when the dashboard's live fit has converged. + charsPerToken: 4, // R2 multi-column ON (2 cols) — at single-col the break-even gate // correctly rejects compression on real tool-doc-shaped slabs (~38 chars/ // row → ~29 imgs vs 39k text tokens → net loss). Two columns packs ~2× @@ -204,6 +221,12 @@ export function isCompressionProfitable( cols: number = DEFAULTS.cols, imageCountCap?: number, numCols: number = 1, + /** Chars-per-token assumption for the text side of the break-even math. + * Default 4 (Anthropic's English-text average). Host can pass the live + * empirical `chars_per_token` from dashboardState.fitCosts() so the gate + * auto-tunes to the model's actual tokenization rate. Lower values = + * more profitable text compressions (each char buys more tokens back). */ + charsPerToken: number = CHARS_PER_TOKEN, ): boolean { const n = Math.max(1, numCols | 0); let estImages: number; @@ -225,8 +248,14 @@ export function isCompressionProfitable( if (imageCountCap !== undefined && imageCountCap > 0) { estImages = Math.min(estImages, imageCountCap); } + // Defensive clamp: a corrupt or pathological charsPerToken (≤0 / NaN) + // would either crash or give a misleading-true. Fall back to the + // baked-in default in that case. + const cpt = Number.isFinite(charsPerToken) && charsPerToken > 0 + ? charsPerToken + : CHARS_PER_TOKEN; const imageTokensCost = estImages * TOKENS_PER_IMAGE; - const textTokensEquivalent = textLen / CHARS_PER_TOKEN; + const textTokensEquivalent = textLen / cpt; return imageTokensCost < textTokensEquivalent; } @@ -1241,7 +1270,7 @@ export async function transformRequest( Math.max(1, (o.multiCol | 0) || 1), Math.max(1, maxFittingCols(o.cols)), ); - if (!isCompressionProfitable(combined, o.cols, undefined, numCols)) { + if (!isCompressionProfitable(combined, o.cols, undefined, numCols, o.charsPerToken)) { info.reason = `not_profitable (slab=${combined.length} chars)`; bumpPassthrough(info, 'not_profitable'); info.outgoingTextChars = countOutgoingTextChars(req); @@ -1345,7 +1374,7 @@ export async function transformRequest( continue; } const reminderText = (blk as TextBlock).text; - if (!isCompressionProfitable(reminderText, o.cols, undefined, numCols)) { + if (!isCompressionProfitable(reminderText, o.cols, undefined, numCols, o.charsPerToken)) { // Above threshold but image cost ≥ text cost. Net loss to compress. bumpPassthrough(info, 'not_profitable'); processedExisting.push(blk); @@ -1406,7 +1435,7 @@ export async function transformRequest( if (inner.length < o.minToolResultChars) { bumpPassthrough(info, 'below_threshold'); rewritten.push(blk); - } else if (!isCompressionProfitable(inner, o.cols, o.maxImagesPerToolResult, numCols)) { + } else if (!isCompressionProfitable(inner, o.cols, o.maxImagesPerToolResult, numCols, o.charsPerToken)) { bumpPassthrough(info, 'not_profitable'); rewritten.push(blk); } else { @@ -1450,7 +1479,7 @@ export async function transformRequest( newInner.push(ib as TextBlock | ImageBlock); continue; } - if (!isCompressionProfitable(innerText, o.cols, o.maxImagesPerToolResult, numCols)) { + if (!isCompressionProfitable(innerText, o.cols, o.maxImagesPerToolResult, numCols, o.charsPerToken)) { bumpPassthrough(info, 'not_profitable'); newInner.push(ib as TextBlock | ImageBlock); continue; diff --git a/src/node.ts b/src/node.ts index 0f156fa..e7b83d8 100644 --- a/src/node.ts +++ b/src/node.ts @@ -515,7 +515,21 @@ async function main(): Promise { const config: ProxyConfig = { upstream: opts.upstream, - transform, + // Resolve transform options per request so the live empirical + // chars/token from dashboardState.fitCosts() flows into the + // isCompressionProfitable gate. With α at the stale default of 0.25 + // (4 chars/tok), the gate rejects slabs as small as ~138K chars + // because the row-aware image-count estimate exceeds the textual + // token equivalence. Live α on Claude Code traffic runs ≈0.5-0.9 + // — once the fit has 3+ samples, the gate auto-tunes and accepts + // compressions that the stale assumption was leaving on the table. + // When fit is null, falls through to the baked-in DEFAULTS.charsPerToken. + transform: () => { + const fit = dashboard.fitCosts(); + return fit && fit.chars_per_token > 0 + ? { ...transform, charsPerToken: fit.chars_per_token } + : transform; + }, onRequest: async (e) => { // Feed the dashboard BEFORE tracker.emit — toTrackEvent strips // info.firstImagePng, so capturing has to happen on the raw event. diff --git a/tests/render.test.ts b/tests/render.test.ts index 71d9b7c..5cf66ef 100644 --- a/tests/render.test.ts +++ b/tests/render.test.ts @@ -1709,6 +1709,74 @@ describe('transform', () => { expect(isCompressionProfitable(40000)).toBe(true); }); + // --- Live-α override: gate auto-tunes when dashboard's chars/token diverges --- + // + // The default `CHARS_PER_TOKEN = 4` corresponds to Anthropic's English + // average (α=0.25). Real Claude Code traffic has α≈0.5-0.9 — JSON-dense + // tool definitions, structured CLAUDE.md slabs, etc. tokenize at 1.1-2 + // chars/tok. The gate now accepts a per-request override so the host + // (node.ts) can pipe dashboardState.fitCosts().chars_per_token through. + // + // Production trace 2026-05-19: a 169_632-char slab with 88 lines of + // markdown got `not_profitable` rejected because the gate used 4 ch/tok + // (textEq=42_408) while actual upstream billed 148_891 tokens (ch/tok= + // 1.14). With the live override (1.14 ch/tok), the gate flips to ACCEPT. + + it('isCompressionProfitable: stale default rejects newline-heavy 170K slab', () => { + // CLAUDE.md-like slab: many short lines, ~138K chars. Default ch/tok=4 + // gives textEq=34_515 vs imgCost~95_000 (multiCol=2). Reject. + const slab = ( + '## Section\n\n- Bullet point with reasonable length.\n- Another bullet.\n\n' + + '```ts\nconst x = 42;\nconst y = "string";\n```\n\n' + ).repeat(Math.floor(170000 / 160)); + expect(isCompressionProfitable(slab, 100, undefined, 2)).toBe(false); + }); + + it('isCompressionProfitable: live α≈0.88 (1.14 ch/tok) flips the same slab to PROFITABLE', () => { + const slab = ( + '## Section\n\n- Bullet point with reasonable length.\n- Another bullet.\n\n' + + '```ts\nconst x = 42;\nconst y = "string";\n```\n\n' + ).repeat(Math.floor(170000 / 160)); + // 1.14 ch/tok ≈ the actual rate the upstream billed on the production + // event. Now textEq = len/1.14 ≈ 121K tokens vs imgCost ≈ 95K → ACCEPT. + expect(isCompressionProfitable(slab, 100, undefined, 2, 1.14)).toBe(true); + }); + + it('isCompressionProfitable: defensive clamp on bogus chars/token (≤0 / NaN → falls back to 4)', () => { + // Corrupt values would either crash or produce wildly wrong gate + // decisions. The function falls back to CHARS_PER_TOKEN=4 silently. + // Confirm: a 5000-char input is rejected at 4 ch/tok regardless of + // whether we pass 0, -1, NaN, or Infinity. + expect(isCompressionProfitable(5000, 100, undefined, 1, 0)).toBe(false); + expect(isCompressionProfitable(5000, 100, undefined, 1, -1)).toBe(false); + expect(isCompressionProfitable(5000, 100, undefined, 1, NaN)).toBe(false); + expect(isCompressionProfitable(5000, 100, undefined, 1, Infinity)).toBe(false); + }); + + it('TransformOptions.charsPerToken: low value unlocks a previously-rejected slab end-to-end', async () => { + // Borderline-newline-heavy slab that the DEFAULT gate rejects. + const slab = ( + '## Section\n\n- Bullet point with reasonable length.\n- Another bullet.\n\n' + + '```ts\nconst x = 42;\nconst y = "string";\n```\n\n' + ).repeat(Math.floor(170000 / 160)); + const req = JSON.stringify({ + model: 'claude-3-5-sonnet', + messages: [{ role: 'user', content: 'hi' }], + system: slab, + }); + const bytes = new TextEncoder().encode(req); + + // Default ch/tok=4: rejected as not_profitable. + const stale = await transformRequest(bytes); + expect(stale.info.compressed).toBe(false); + expect(stale.info.reason).toMatch(/^not_profitable/); + + // Live α equivalent (1.14 ch/tok): same slab now compresses. + const live = await transformRequest(bytes, { charsPerToken: 1.14 }); + expect(live.info.compressed).toBe(true); + expect(live.info.imageCount ?? 0).toBeGreaterThan(0); + }); + // --- Adaptive break-even: CHARS_PER_IMAGE derived from atlas cell, not hardcoded --- // Brief: when font-rater swaps to a smaller cell (e.g. Cozette 4×7), more chars // pack into one image, so the N-image break-even thresholds shift. Tests below