From 0cfe08a868dd05193b2374a4e00a2237a101e7f4 Mon Sep 17 00:00:00 2001 From: Steven Chong <25894545+teamchong@users.noreply.github.com> Date: Fri, 22 May 2026 23:54:05 -0400 Subject: [PATCH] feat(gate): symmetric warm-cache burn + gateEval telemetry (anti-flapping) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production data (ocproxy Opus 4.7, 2026-05-22 23:15–23:44) showed the slab break-even gate ping-ponging between text and image modes within a single session, paying cache_create on whichever side it just flipped away from. Two flips in 30 minutes cost ~$0.71 + ~$0.40 of avoidable cache_create on a single conversation. Root cause: priorWarmTokens was asymmetric. It penalised compressing when the un-rewritten text path was warm, but did not penalise decompressing when the rewritten image path was warm. Per-turn cost favored flipping; the flip forced cache_create on the new side; next turn flipped back. This change adds the symmetric counterpart: text→image flip: warm text invalidated → burn applied to IMAGE side (existing behavior, priorWarmTokens) image→text flip: warm image invalidated → burn applied to TEXT side (NEW, priorWarmImageTokens) Once a session commits to a mode, staying in that mode is cheaper than flipping by the burn cost. Mode changes only happen when the per-turn delta exceeds the burn, which is exactly when the flip is actually worth the cache_create. Foundational fix — no thresholds. Observability: - New exported `evalCompressionProfitability()` decomposes the slab gate's evaluation into imageTokens, textTokens, burnImageSide, burnTextSide, and verdict. Same constants and helper as the gate itself, no drift risk. - New `TransformInfo.gateEval` field stamps that decomposition into every applied or not_profitable event so hosts can persist the verdict margin and prove flap-prevention efficacy from telemetry. Back-compat: - New params default to 0; single-arg callers behave identically. - 315 existing tests unchanged; +7 new tests pin the new semantics (back-compat, asymmetric ⇒ symmetric, both-sides-warm cancellation, eval verdict matches gate verdict, burn term math). --- src/core/transform.ts | 182 ++++++++++++++++++++++++++++++++++++---- tests/gate-flap.test.ts | 104 +++++++++++++++++++++++ 2 files changed, 272 insertions(+), 14 deletions(-) create mode 100644 tests/gate-flap.test.ts diff --git a/src/core/transform.ts b/src/core/transform.ts index 991648f..7569a06 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -120,6 +120,32 @@ export interface TransformOptions { * Cold-start safe: 0 disables the burn term entirely. Negative or * non-finite values are clamped to 0. */ priorWarmTokens?: number; + /** Symmetric counterpart of `priorWarmTokens` for the IMAGE-mode side. + * + * When the prior turn rendered the static prefix as image blocks + * (pixelpipe applied), Anthropic's prompt cache holds the IMAGE prefix, + * not the text prefix. Declining compression on this turn — sending + * plain text — invalidates the image-prefix cache key and forces a + * fresh `cache_create` on the un-rewritten text prefix. Symmetric to + * `priorWarmTokens`, the burn cost is + * + * burn = priorWarmImageTokens × (CACHE_CREATE_RATE − CACHE_READ_RATE) + * + * and is added to the TEXT side of the break-even comparison so the + * gate stays in image mode when the image cache is already warm. This + * is the foundational fix for gate flapping: without it the gate + * re-decides every turn purely on per-turn cost and ping-pongs between + * modes, paying cache_create on both sides. + * + * Set by the host from a session-keyed LRU that remembers the prior + * turn's chosen mode AND the upstream-observed cacheable prefix size + * on that turn. Hosts populating `priorWarmTokens` SHOULD also + * populate this; supplying one without the other biases the gate + * toward whichever side has the asymmetric burn term. + * + * Cold-start safe: 0 disables the burn term entirely. Negative or + * non-finite values are clamped to 0. */ + priorWarmImageTokens?: number; /** R3 reflow: re-pack each image-bound text block into a continuous * sentinel-delimited stream so rendered rows fill `cols` instead of * leaving line-end dead margin (measured glyph-fill ~29% → ~75-80%). @@ -173,6 +199,7 @@ const DEFAULTS: Required = { // integer ≥ 2 via `historyAmortizationHorizon`. See option jsdoc. historyAmortizationHorizon: 1, priorWarmTokens: 0, + priorWarmImageTokens: 0, // R2 multi-column ON (2 cols) — at single-col the break-even gate // correctly rejects compression on real tool-doc-shaped slabs (~38 chars/ // row → ~29 imgs vs 39k text tokens → net loss). Two columns packs ~2× @@ -413,6 +440,65 @@ function maybeReflow(text: string, enabled: boolean): string { * tests that pass only `textLen` keep working byte-identically at the * current atlas. New call sites should pass `o.cols` so a runtime * `--cols` override flows into the break-even math too. */ +/** Decompose the slab break-even gate's evaluation into its components. + * Returns the exact `imageTokens`, `textTokens`, and symmetric burn + * terms the gate uses internally. Pairs with `isCompressionProfitable` + * for telemetry: callers can record the numbers without re-implementing + * the formula and risking drift. + * + * Returns `null` when the inputs are not finite or `textLen ≤ 0`. + * Stays in sync with `isCompressionProfitable` because both use the + * same constants and `effectiveTokensPerImage`. */ +export function evalCompressionProfitability( + textOrLen: string | number, + cols: number, + imageCountCap: number | undefined = undefined, + numCols: number = 1, + charsPerToken: number = CHARS_PER_TOKEN, + priorWarmTokens: number = 0, + priorWarmImageTokens: number = 0, +): { + imageTokens: number; + textTokens: number; + burnImageSide: number; + burnTextSide: number; + profitable: boolean; +} | null { + const n = Math.max(1, numCols | 0); + let estImages: number; + let textLen: number; + if (typeof textOrLen === 'string') { + estImages = estimateImageCount(textOrLen, cols, n); + textLen = textOrLen.length; + } else { + const charsPerImage = maxCharsPerImage(cols) * n; + estImages = Math.max(1, Math.ceil(textOrLen / charsPerImage)); + textLen = textOrLen; + } + if (imageCountCap !== undefined && imageCountCap > 0) { + estImages = Math.min(estImages, imageCountCap); + } + if (!Number.isFinite(textLen) || textLen <= 0) return null; + const cpt = Number.isFinite(charsPerToken) && charsPerToken > 0 + ? charsPerToken + : CHARS_PER_TOKEN; + const imageTokens = estImages * effectiveTokensPerImage(n); + const textTokens = textLen / cpt; + const burnImageSide = Number.isFinite(priorWarmTokens) && priorWarmTokens > 0 + ? priorWarmTokens * (CACHE_CREATE_RATE - CACHE_READ_RATE) + : 0; + const burnTextSide = Number.isFinite(priorWarmImageTokens) && priorWarmImageTokens > 0 + ? priorWarmImageTokens * (CACHE_CREATE_RATE - CACHE_READ_RATE) + : 0; + return { + imageTokens, + textTokens, + burnImageSide, + burnTextSide, + profitable: imageTokens + burnImageSide < textTokens + burnTextSide, + }; +} + export function isCompressionProfitable( textOrLen: string | number, cols: number = DEFAULTS.cols, @@ -428,6 +514,14 @@ export function isCompressionProfitable( * rewritten prefix invalidates Anthropic's prior cache key. Default 0 * (cold-start behavior; matches pre-burn-aware callers byte-for-byte). */ priorWarmTokens: number = 0, + /** Symmetric image-side burn: tokens the rewritten (IMAGE) path would + * have hit cache on. Adds a one-time burn penalty of + * `priorWarmImageTokens × (CACHE_CREATE_RATE − CACHE_READ_RATE)` to the + * TEXT side — the cost of forcing a fresh cache_create on the + * un-rewritten text prefix when the rewritten image prefix was warm. + * Default 0 (back-compat; existing single-arg callers behave unchanged). + * See PixelpipeOptions.priorWarmImageTokens for the foundational rationale. */ + priorWarmImageTokens: number = 0, ): boolean { const n = Math.max(1, numCols | 0); let estImages: number; @@ -457,16 +551,28 @@ export function isCompressionProfitable( : CHARS_PER_TOKEN; const imageTokensCost = estImages * effectiveTokensPerImage(n); const textTokensEquivalent = textLen / cpt; - // Cache-burn penalty: rewriting the cacheable prefix invalidates - // Anthropic's prior cache key. The un-rewritten path would have paid - // CACHE_READ_RATE on `priorWarmTokens`; the rewritten path pays - // CACHE_CREATE_RATE on the new prefix. The delta is borne entirely on - // this turn (subsequent turns warm the new prefix). Per-turn gate uses - // the full delta with no amortization; that's what `horizon=1` means. - const safeBurn = Number.isFinite(priorWarmTokens) && priorWarmTokens > 0 + // Cache-burn penalty (symmetric form, ANTI-FLAPPING): + // + // text→image flip: invalidate the warm text cache. Burn applied to + // the IMAGE side — discourages compressing when + // the text prefix is already warm. + // image→text flip: invalidate the warm image cache. Burn applied to + // the TEXT side — discourages decompressing when + // the image prefix is already warm. + // + // Without the symmetric term the gate ping-pongs: once a session + // commits to a mode, single-turn cost can favor flipping, but the + // flip forces cache_create on the new side, then the next turn flips + // back. We pay cache_create twice. The symmetric burn pins the + // session in its current mode unless the per-turn delta exceeds the + // burn cost. + const burnImageSide = Number.isFinite(priorWarmTokens) && priorWarmTokens > 0 ? priorWarmTokens * (CACHE_CREATE_RATE - CACHE_READ_RATE) : 0; - return imageTokensCost + safeBurn < textTokensEquivalent; + const burnTextSide = Number.isFinite(priorWarmImageTokens) && priorWarmImageTokens > 0 + ? priorWarmImageTokens * (CACHE_CREATE_RATE - CACHE_READ_RATE) + : 0; + return imageTokensCost + burnImageSide < textTokensEquivalent + burnTextSide; } /** @@ -517,9 +623,12 @@ export function isCompressionProfitableAmortized( * Amortized across `horizon` turns at this gate (the prior cache is * burned exactly once, on the first rewritten turn). Default 0. */ priorWarmTokens: number = 0, + /** Symmetric image-side burn — see same-named param on + * `isCompressionProfitable`. Default 0 (back-compat). */ + priorWarmImageTokens: number = 0, ): boolean { if (!Number.isFinite(horizon) || horizon <= 1) { - return isCompressionProfitable(textOrLen, cols, imageCountCap, numCols, charsPerToken, priorWarmTokens); + return isCompressionProfitable(textOrLen, cols, imageCountCap, numCols, charsPerToken, priorWarmTokens, priorWarmImageTokens); } const N = Math.max(2, Math.floor(horizon)); const n = Math.max(1, numCols | 0); @@ -547,12 +656,16 @@ export function isCompressionProfitableAmortized( // assumptions. const imageLifetime = imageTokens * (CACHE_CREATE_RATE + CACHE_READ_RATE * (N - 1)); const textLifetime = textTokens * CACHE_READ_RATE * N; - // Burn is paid exactly once (on the rewritten-prefix's first turn), - // amortized across N turns. Adds to the image side. - const safeBurn = Number.isFinite(priorWarmTokens) && priorWarmTokens > 0 + // Symmetric burn — each side pays its own cache_create invalidation + // cost when the verdict flips its mode. See + // `isCompressionProfitable` for the full anti-flapping argument. + const burnImageSide = Number.isFinite(priorWarmTokens) && priorWarmTokens > 0 ? priorWarmTokens * (CACHE_CREATE_RATE - CACHE_READ_RATE) : 0; - return imageLifetime + safeBurn < textLifetime; + const burnTextSide = Number.isFinite(priorWarmImageTokens) && priorWarmImageTokens > 0 + ? priorWarmImageTokens * (CACHE_CREATE_RATE - CACHE_READ_RATE) + : 0; + return imageLifetime + burnImageSide < textLifetime + burnTextSide; } @@ -706,6 +819,34 @@ export interface TransformInfo { * returned false (image cost ≥ text cost at current cell config) * Only emitted when at least one counter is > 0. */ passthroughReasons?: { below_threshold?: number; not_profitable?: number }; + /** Per-gate-call diagnostics — exactly what the slab break-even gate + * saw and compared. Foundational observability for why a turn flipped + * or stayed: hosts that persist this can compute the verdict margin + * (`textTokens + burnTextSide − imageTokens − burnImageSide`), + * measure flap-prevention efficacy (`burnTextSide` > 0 rows that + * declined despite `imageTokens < textTokens`), and tune + * `historyAmortizationHorizon` against observed cache lifetimes. + * + * Currently emitted for the session-anchor slab gate only. Per-block + * reminder / tool_result gates remain summarised in `passthroughReasons` + * + `bucketChars` — adding per-call diagnostics there would multiply + * event size without commensurate signal (those gates don't flap). */ + gateEval?: { + /** "slab" today; reserved for future gates if they grow flapping risk. */ + readonly site: 'slab'; + /** Image-side cost estimate the gate used (token-equivalents). */ + readonly imageTokens: number; + /** Text-side cost estimate the gate used (token-equivalents). */ + readonly textTokens: number; + /** `priorWarmTokens × (1.25 − 0.10)` applied to the image side. */ + readonly burnImageSide: number; + /** `priorWarmImageTokens × (1.25 − 0.10)` applied to the text side. + * Non-zero rows are the anti-flapping anchor — text would otherwise + * have looked cheaper. */ + readonly burnTextSide: number; + /** Gate's verdict; `true` ⇒ compression applied on this turn. */ + readonly profitable: boolean; + }; /** Per-bucket sum of TEXT chars that flowed through each gate-call site * (static slab, reminder, tool_result by classifyContent shape, history). * Pre-compaction lengths — stays comparable to `origChars` and to what @@ -1801,7 +1942,20 @@ export async function transformRequest( const slabCpt = opts.charsPerToken !== undefined ? o.charsPerToken : SLAB_CHARS_PER_TOKEN; - if (!isCompressionProfitable(combined, o.cols, undefined, numCols, slabCpt, o.priorWarmTokens)) { + const slabGateEval = evalCompressionProfitability( + combined, o.cols, undefined, numCols, slabCpt, o.priorWarmTokens, o.priorWarmImageTokens, + ); + if (slabGateEval) { + info.gateEval = { + site: 'slab', + imageTokens: slabGateEval.imageTokens, + textTokens: slabGateEval.textTokens, + burnImageSide: slabGateEval.burnImageSide, + burnTextSide: slabGateEval.burnTextSide, + profitable: slabGateEval.profitable, + }; + } + if (!isCompressionProfitable(combined, o.cols, undefined, numCols, slabCpt, o.priorWarmTokens, o.priorWarmImageTokens)) { info.reason = `not_profitable (slab=${combined.length} chars)`; bumpPassthrough(info, 'not_profitable'); // Slab failed the break-even gate, but message history may still be diff --git a/tests/gate-flap.test.ts b/tests/gate-flap.test.ts new file mode 100644 index 0000000..17f8be1 --- /dev/null +++ b/tests/gate-flap.test.ts @@ -0,0 +1,104 @@ +/** + * Foundational gate-flapping tests. + * + * Background: the slab break-even gate originally penalised compressing + * when the un-rewritten text path was warm (`priorWarmTokens`). It did + * NOT symmetrically penalise decompressing when the rewritten image path + * was warm. Sessions ping-ponged between modes — each flip paid a fresh + * `cache_create` on the new side, then the next turn flipped back. + * + * Fix: symmetric burn (`priorWarmImageTokens`) on the TEXT side. Once a + * session commits to a mode, the same-mode side gets a discount equal to + * `priorWarm × (CACHE_CREATE_RATE − CACHE_READ_RATE)`. Mode changes only + * happen when the per-turn delta exceeds the burn cost. + * + * These tests pin the new behavior end-to-end at the public API: + * `isCompressionProfitable` + `evalCompressionProfitability`. + */ +import { describe, expect, it } from 'vitest'; +import { + isCompressionProfitable, + evalCompressionProfitability, +} from '../src/core/transform.js'; + +describe('symmetric warm-cache burn (anti-flapping)', () => { + it('back-compat: omitting both warm args matches previous behavior', () => { + // A clearly-profitable case (large text, few images) must still + // come out as `true` when no warm-cache burn is supplied. + const text = 'x'.repeat(50_000); + expect(isCompressionProfitable(text, 100, undefined, 2, 4)).toBe(true); + }); + + it('priorWarmTokens alone penalises compressing when text is warm', () => { + // Borderline-profitable text length — small enough that adding any + // image-side burn flips the verdict to "not profitable". + const text = 'x'.repeat(12_000); + const cold = isCompressionProfitable(text, 100, undefined, 2, 4, 0, 0); + const warm = isCompressionProfitable(text, 100, undefined, 2, 4, 50_000, 0); + // If the cold gate says compress, the warm-text gate must NOT (asymmetric + // burn now pins the session in text mode). + if (cold) expect(warm).toBe(false); + }); + + it('priorWarmImageTokens symmetrically pins a session in image mode', () => { + // Pick text length where the cold gate says NOT profitable (text is + // small relative to image fixed cost). Once image mode is warm, the + // symmetric burn must flip the verdict to "profitable" so we stay in + // image mode rather than re-creating the text cache. + const text = 'x'.repeat(2_000); + const cold = isCompressionProfitable(text, 100, undefined, 1, 4, 0, 0); + const warmImage = isCompressionProfitable(text, 100, undefined, 1, 4, 0, 60_000); + // Cold should reject this short text; warm-image must accept it to + // avoid the flap-induced cache_create on the text side. + expect(cold).toBe(false); + expect(warmImage).toBe(true); + }); + + it('warm on BOTH sides cancels out (no asymmetric bias)', () => { + const text = 'x'.repeat(12_000); + const cold = isCompressionProfitable(text, 100, undefined, 2, 4, 0, 0); + const both = isCompressionProfitable(text, 100, undefined, 2, 4, 25_000, 25_000); + expect(both).toBe(cold); + }); +}); + +describe('evalCompressionProfitability observability', () => { + it('returns the same verdict as isCompressionProfitable', () => { + const samples = [ + { text: 'x'.repeat(50_000), pw: 0, pwi: 0 }, + { text: 'x'.repeat(12_000), pw: 50_000, pwi: 0 }, + { text: 'x'.repeat(2_000), pw: 0, pwi: 60_000 }, + { text: 'x'.repeat(12_000), pw: 25_000, pwi: 25_000 }, + ]; + for (const s of samples) { + const verdict = isCompressionProfitable(s.text, 100, undefined, 2, 4, s.pw, s.pwi); + const evaled = evalCompressionProfitability(s.text, 100, undefined, 2, 4, s.pw, s.pwi); + expect(evaled).not.toBeNull(); + expect(evaled!.profitable).toBe(verdict); + } + }); + + it('exposes the symmetric burn terms used internally', () => { + const e = evalCompressionProfitability( + 'x'.repeat(12_000), + 100, + undefined, + 2, + 4, + 40_000, + 60_000, + ); + expect(e).not.toBeNull(); + // burn = warm × (1.25 − 0.10) = warm × 1.15 + expect(e!.burnImageSide).toBeCloseTo(40_000 * 1.15, 5); + expect(e!.burnTextSide).toBeCloseTo(60_000 * 1.15, 5); + // imageTokens / textTokens are positive token-equivalents. + expect(e!.imageTokens).toBeGreaterThan(0); + expect(e!.textTokens).toBeGreaterThan(0); + }); + + it('returns null when textLen ≤ 0 (defensive against degenerate inputs)', () => { + expect(evalCompressionProfitability(0, 100)).toBeNull(); + expect(evalCompressionProfitability(-5, 100)).toBeNull(); + }); +});