From c36ec3fd3998193a159c779fb4a9ced107208813 Mon Sep 17 00:00:00 2001 From: teamchong <25894545+teamchong@users.noreply.github.com> Date: Wed, 20 May 2026 00:15:55 -0400 Subject: [PATCH] fix(gate): slab call site uses empirically-grounded cpt=2.5 (was 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production data (N=354 cold-miss count_tokens probes, 2026-05-18..05-20): real body-level chars-per-token is 1.17 median, 2.62 max — never near the English-prose 4 baked into CHARS_PER_TOKEN. The gate has been estimating text-token cost at 3.4× cheaper than reality and silently rejecting every realistic Claude Code system slab. Concrete production case (orig_chars=161,101, multi-col=2): • OLD gate (cpt=4): text = 161101/4 = 40,275 tok image = 8 × 5500 = 44,000 tok → reject (margin: -3,726) • NEW gate (cpt=2.5): text = 161101/2.5 = 64,440 tok image = 8 × 5500 = 44,000 tok → ACCEPT (margin: +20,440) • Reality (cpt=1.17): text = 137,694 tok, savings = 93,694 tok/request SLAB_CHARS_PER_TOKEN=2.5 is the upper bound of observed cpt across the sample — picking the upper bound keeps the prime-directive guarantee intact: the text-token estimate is a LOWER bound on real text cost, so any `imageCost < textTokens` decision is also `imageCost < realTextCost` for any future workload with cpt ≤ 2.5. Scope: this is slab-specific. Reminders and tool_result content have unknown shape (could be raw English prose with cpt~4), so those gate call sites still use the conservative CHARS_PER_TOKEN=4. Host can override per-request via TransformOptions.charsPerToken (e.g., to plug a live empirical fit in front of the static default). Tests: 2 new regression tests pinning (a) the production-shape 161k slab compresses end-to-end without an explicit cpt override, and (b) the gate math at cpt=2.5 vs cpt=4 produces accept-vs-reject on the same input. 272 tests pass, typecheck clean, build clean. --- src/core/transform.ts | 36 ++++++++++++++++++++++++++++- tests/render.test.ts | 54 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/core/transform.ts b/src/core/transform.ts index f546f17..e593102 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -151,6 +151,32 @@ const DEFAULTS: Required = { * mix; tool_result content is typically code-shaped. */ const CHARS_PER_TOKEN = 4; +/** Empirical chars-per-token for the *system slab + tool docs* path. + * + * Source: N=354 production cold-miss `count_tokens` baselines on + * Claude Code (pixelpipe events.jsonl, 2026-05-18 → 2026-05-20): + * body cpt distribution — median 1.17, p95 2.5, MAX 2.62. + * + * System slabs are JSON-dense (tool definitions, schemas, structured + * prompts) so they sit at cpt ~1.2, NOT the English-prose 4. Using cpt=4 + * for this call site told the gate text was 3.4× cheaper than reality, + * silently rejecting every profitable slab compression we've seen since + * the row-aware gate landed. + * + * Safety: 2.5 is the **upper bound** of observed real cpts (max=2.62 + * with one sample, p95=2.5). Picking the upper bound keeps the prime- + * directive guarantee intact — the text-token estimate is a LOWER bound + * on real text cost, so any `imageCost < textTokens` decision is also + * `imageCost < realTextCost`. If a slab in the wild ever lands above + * cpt=2.5, the gate would still under-bill text by < 5% and the worst + * case is a marginal-loss compression, not a runaway. + * + * Why this is slab-specific and NOT a global default: reminders and + * tool_result content have unknown shape (could be raw English prose + * with cpt~4). Leaving those at CHARS_PER_TOKEN=4 preserves the + * conservative bias where shape isn't known a priori. */ +const SLAB_CHARS_PER_TOKEN = 2.5; + /** Empirical per-image cost at numCols=1. Source: dashboard.ts measurement * trace. Kept here as a constant rather than imported from dashboard.ts * to keep `src/core/` free of dashboard imports — that's a one-way edge. */ @@ -1343,7 +1369,15 @@ export async function transformRequest( Math.max(1, (o.multiCol | 0) || 1), Math.max(1, maxFittingCols(o.cols)), ); - if (!isCompressionProfitable(combined, o.cols, undefined, numCols, o.charsPerToken)) { + // Slab cpt is empirically ~1.2 (N=354 production samples) — far from the + // English-prose default 4 baked into CHARS_PER_TOKEN. Use a slab-specific + // upper-bound cpt at this gate so JSON-dense system + tool-doc content + // gets a fair break-even check. Host can still override via + // `o.charsPerToken` (e.g., to plug in a live empirical fit). + const slabCpt = o.charsPerToken !== undefined && o.charsPerToken !== CHARS_PER_TOKEN + ? o.charsPerToken + : SLAB_CHARS_PER_TOKEN; + if (!isCompressionProfitable(combined, o.cols, undefined, numCols, slabCpt)) { info.reason = `not_profitable (slab=${combined.length} chars)`; bumpPassthrough(info, 'not_profitable'); info.outgoingTextChars = countOutgoingTextChars(req); diff --git a/tests/render.test.ts b/tests/render.test.ts index 6f97031..51304bd 100644 --- a/tests/render.test.ts +++ b/tests/render.test.ts @@ -1846,6 +1846,60 @@ describe('transform', () => { expect(live.info.imageCount ?? 0).toBeGreaterThan(0); }); + // --- Slab-specific cpt: built-in 2.5 cpt unlocks production-shape slabs --- + // + // Empirical: N=354 production count_tokens probes (2026-05-18..2026-05-20) + // give body-level chars/token median 1.17, max 2.62. The English-prose + // CHARS_PER_TOKEN=4 default was 3.4× too high for the slab call site, + // silently rejecting every realistic slab. The slab gate now uses + // SLAB_CHARS_PER_TOKEN=2.5 — the upper bound of empirical data — which + // unlocks the production-shape slab while preserving the prime-directive + // safety (no net-loss compressions on shapes we've actually observed). + + it('transformRequest: production-shape 161k slab compresses without an explicit cpt override', async () => { + // Build a dense ~161k-char slab matching the production passthrough event + // (orig_chars=161101). 60-100 char lines, modest blank density — + // representative of system + tool-doc slab shape under multi-col=2. + const parts: string[] = []; + let acc = 0; + const target = 161_101; + while (acc < target) { + const len = 60 + (acc % 40); + parts.push('A'.repeat(len) + (acc % 200 === 0 ? ' ' : '')); + acc += len + 1; + } + const slab = parts.join('\n').slice(0, target); + const req = JSON.stringify({ + model: 'claude-3-5-sonnet', + messages: [{ role: 'user', content: 'hi' }], + system: slab, + }); + const bytes = new TextEncoder().encode(req); + + // No host-supplied cpt: built-in SLAB_CHARS_PER_TOKEN flips this to ACCEPT + // at multi-col=2 (production default). This is the regression guard for + // the 2026-05-20 zero-compression production bug. + const out = await transformRequest(bytes, { multiCol: 2 }); + expect(out.info.compressed).toBe(true); + expect(out.info.imageCount ?? 0).toBeGreaterThan(0); + }); + + it('isCompressionProfitable: slab cpt=2.5 flips a 161k production-shape slab profitable at multi-col=2', () => { + // Pin the math directly. Image cost at multi-col=2: 8 imgs × 5500 = + // 44,000 tok. At cpt=4, text=40,275 → REJECT. At cpt=2.5, text=64,440 + // → ACCEPT with 20k headroom over the conservative slab cpt. + const parts: string[] = []; + let acc = 0; + while (acc < 161_101) { + const len = 60 + (acc % 40); + parts.push('A'.repeat(len)); + acc += len + 1; + } + const slab = parts.join('\n').slice(0, 161_101); + expect(isCompressionProfitable(slab, 100, undefined, 2, 4)).toBe(false); + expect(isCompressionProfitable(slab, 100, undefined, 2, 2.5)).toBe(true); + }); + // --- Adaptive break-even: CHARS_PER_IMAGE derived from atlas cell, not hardcoded --- // Brief: when font-rater swaps to a smaller cell (e.g. Cozette 4×7), more chars // pack into one image, so the N-image break-even thresholds shift. Tests below