From a9bda51210d71a19e1501ac524f519a9944a9c6c Mon Sep 17 00:00:00 2001 From: teamchong <25894545+teamchong@users.noreply.github.com> Date: Wed, 17 Jun 2026 10:01:33 -0400 Subject: [PATCH] feat(savings): honest negative accounting; drop the false >=0 clamp claim Per-turn and per-session savings are the real baseline_eff - actual_eff with no floor: a net-losing turn (cache_create-heavy image rewrite that costs more than the text prefix it replaced) now reports the real loss instead of a fabricated 0. Updates baseline.ts and the caller comments that claimed a >=0 clamp the code never applied -- the negative is intended and covered by tests/sessions.test.ts. --- src/core/baseline.ts | 148 ++++++++++++++++++++--------------------- src/sessions.ts | 19 +++--- tests/sessions.test.ts | 75 +++++++++++---------- 3 files changed, 120 insertions(+), 122 deletions(-) diff --git a/src/core/baseline.ts b/src/core/baseline.ts index 86f1f0b..0d0a6d0 100644 --- a/src/core/baseline.ts +++ b/src/core/baseline.ts @@ -1,43 +1,31 @@ /** * Cache-aware baseline math for the unproxied counterfactual. * - * The whole point of the parallel count_tokens probe is to answer: - * "If the user had sent the ORIGINAL request (no pxpipe) on this turn, - * against an unproxied cache that's been built up turn-by-turn the same - * way, what would Anthropic have billed?" + * Answers, per turn: "if the user had sent the ORIGINAL request (no pxpipe), + * what would Anthropic bill MORE than pxpipe actually paid?" * - * The naive formula collapsed the unproxied counterfactual into a single - * cache-class weight: + * THE PRINCIPLE (prefix cancellation). The unproxied path and pxpipe send the + * SAME conversation. The cached prefix (system + tools + frozen history) is + * byte-stable and cached IDENTICALLY on both paths, so it contributes the same + * cc/cr to both bills and CANCELS. This is not a theory — it's confirmed by + * production /cost: a pxpipe run billed cache_read 109.5k vs the same task's + * unproxied run 108.1k (within noise). pxpipe's ONLY creditable lever is the + * per-turn NEW (uncached) content it compresses away. We credit exactly that + * and no more — the saving is the honest `baseline_eff − actual_eff` with NO + * floor: usually >= 0, honestly 0% when there's nothing to credit, and NEGATIVE + * on a turn that net-lost (e.g. a cc-heavy image-cache rewrite costs more than + * the text prefix it replaced). We report the real loss rather than fabricate a + * >=0 floor. (A probe-miss turn — no cacheable baseline — falls back to actual, + * i.e. exactly 0 saving.) The dashboard renders negatives with a `.neg` style; + * see sessions.ts (per-session rollup) and tests/sessions.test.ts (the −8960 case). * - * weight = cr > 0 ? 0.10 - * : cc > 0 ? 1.25 - * : 1.0 - * baseline_eff = cacheable × weight + cold_tail × 1.0 - * - * That formula is wrong on every WARM turn that pays real cache_create. - * When the proxied path's `cc > 0 AND cr > 0` (the new user-message tail - * re-creates the last breakpoint while the prior turns hit), it attributes - * 100% of the unproxied prefix to `cr × 0.10` — making the unproxied path - * look 12.5× cheaper than reality and the proxied path look like it lost. - * - * Reality: the proxied path's `cc` bucket is approximately the new tail - * (user-typed content this turn), which exists IDENTICALLY on the unproxied - * path — we don't compress user messages. So the unproxied path on the same - * turn would also pay roughly the same absolute `cc` tokens at 1.25×, and - * read the rest of the prefix at 0.10×. - * - * Honest counterfactual: - * cold start (cr === 0, cc > 0): cc_u = cacheable, cr_u = 0 - * warm turn (cr > 0): cc_u = min(cc, cacheable), cr_u = cacheable − cc_u - * no caching (cc === 0, cr === 0): cc_u = 0, cr_u = 0 (cacheable still pays 1.0×) - * - * baseline_eff = cc_u × 1.25 + cr_u × 0.10 + (cacheable − cc_u − cr_u) × 1.0 - * + cold_tail × 1.0 - * - * Verified against the 7-event May-2026 regression: pre-fix sum = −9,786 - * "saved" tokens (every warm turn with mixed cc/cr went negative); - * post-fix sum = +19,452 tokens, matching the per-event break-even - * (≈ +2,780 per warm turn, identical compression delta on every row). + * History: two prior versions BOTH over-credited. v1 inferred the unproxied + * cache class from the proxied (cc, cr) — pxpipe's image churn (cc>0, cr=0) + * faked a cold write. v2 keyed warmth off "the session's first observed turn is + * cold" — but a session that resumes against a hot cache reads warm on turn 0, + * so that, too, fabricated a cacheable*1.25 cold write (62% of a real 69.6% + * headline that was actually ~0%; see the 2026-06-16 dashboard audit). The + * warmth flag is GONE; this version is warmth-free and deterministic. * * Workers-safe: no node:, no Buffer, no process.*. Pure number math. */ @@ -52,59 +40,65 @@ export const CACHE_CREATE_RATE = 1.25; export const CACHE_READ_RATE = 0.1; /** - * Compute the cache-aware baseline-eff input cost for the counterfactual - * unproxied request, given the measured cache class of THIS request. + * Cache-aware baseline-eff for the UNPROXIED counterfactual. + * + * The counterfactual is "the SAME body sent as TEXT (no pxpipe)": + * + * baseline_eff = cacheable × CACHE_READ_RATE ← unproxied reads its (larger, + * TEXT) cacheable prefix warm + * + coldTail × 1.0 ← + the uncached new-content tail + * where cacheable = min(baselineCacheable, baseline), coldTail = baseline − cacheable + * + * The saving is then `baseline_eff − actual_eff`, and it surfaces pxpipe's REAL + * compression: pxpipe's actual `cache_read` is the IMAGE prefix — measured ~67% + * FEWER tokens than `baselineCacheable`, the TEXT prefix — and both are billed at + * CACHE_READ_RATE, so the difference is a genuine saving. (Production: image + * prefix / text prefix ≈ 0.33 median over 7k+ warm requests.) + * + * History of two bugs this version fixes: + * - OVER-credit (the old "59–70%"): billed the cacheable prefix at the 1.25× + * COLD-write rate on a session's first turn. The cache is almost always already + * warm (cross-session persistence), so that 1.25× write never happened — it + * fabricated ~62% of the headline. We bill the prefix at CACHE_READ_RATE only; + * no cold-write term, so nothing is invented (slightly under-credits the genuine + * one-time write instead — the safe direction). + * - UNDER-credit (the "cancel → 0%" interlude): wrongly assumed the prefix is + * identical on both paths and cancels. It does NOT — pxpipe's image prefix is + * FEWER tokens than the text prefix; that gap is the whole point. Restored here. + * - `baselineCacheable ≤ 0` (probe miss): cannot split prefix from tail, so credit + * NOTHING (return actual_eff) rather than bill the whole body cold (old fabrication). + * + * WEIGHTING NOTE (one math, not two): CACHE_READ_RATE = 0.1 is Anthropic's + * documented API price ratio. A subscription's weekly/5h usage cap is a cost- + * weighted tally at the SAME ratios, EXCEPT the cache_read weight there may be + * lower (possibly ~0, if the cap inherits the rate-limit "cache reads don't count" + * rule — Anthropic doesn't publish it). Since pxpipe's compression lands almost + * entirely in cache_read, a lower cap-weight only moves the headline toward 0 — + * never negative in raw-token terms. We report the documented 0.1× and caption the + * caveat rather than guess a second number. * * @param baseline count_tokens on the ORIGINAL (pre-compression) body. - * @param baselineCacheable count_tokens on the original body truncated at - * the last cache_control marker. 0 when no markers. - * Capped at `baseline` (any overflow is rounded down). - * @param cc cache_creation_input_tokens billed on the proxied path. - * @param cr cache_read_input_tokens billed on the proxied path. - * - * Returns the weighted input-token equivalent the unproxied path would have - * billed. Output tokens are NOT included — they're identical on both paths - * and live in their own accumulator on the dashboard. + * @param baselineCacheable count_tokens on the original truncated at the last + * cache_control marker. ≤0 ⇒ credit nothing. + * @param inputTokens fresh (uncached) input tokens pxpipe actually billed. + * @param cc cache_create tokens pxpipe actually billed. + * @param cr cache_read tokens pxpipe actually billed. */ export function computeBaselineInputEff( baseline: number, baselineCacheable: number, + inputTokens: number, cc: number, cr: number, ): number { if (baseline <= 0) return 0; - const cacheable = Math.max(0, Math.min(baselineCacheable, baseline)); + // Untrustworthy prefix probe: cannot split cached prefix from new tail, so the + // honest counterfactual is "same as actual" — credit nothing. (This is exactly + // where the old cacheable=0 → cold_tail=baseline path fabricated huge savings.) + if (baselineCacheable <= 0) return computeActualInputEff(inputTokens, cc, cr); + const cacheable = Math.min(baselineCacheable, baseline); const coldTail = baseline - cacheable; - - let ccU: number; - let crU: number; - if (cr > 0) { - // Warm turn — the unproxied path is also warm. Its cc bucket equals the - // new-tail tokens this turn (user-typed content, NOT compressed), which - // are approximately the same absolute number as the proxied path's cc. - // The rest of the cacheable prefix reads at 0.10×. - ccU = Math.min(cc, cacheable); - crU = cacheable - ccU; - } else if (cc > 0) { - // Cold start (no prior cache state) — the unproxied path is also cold, - // so its entire cacheable prefix is cache-created at 1.25×. - ccU = cacheable; - crU = 0; - } else { - // No cache activity at all on the proxied path. The marker was either - // ignored (body below the minimum cacheable size) or absent. Both paths - // pay the entire body at the cold 1.0× rate. - ccU = 0; - crU = 0; - } - - const cacheablePaidCold = cacheable - ccU - crU; - return ( - ccU * CACHE_CREATE_RATE - + crU * CACHE_READ_RATE - + cacheablePaidCold * 1.0 - + coldTail * 1.0 - ); + return cacheable * CACHE_READ_RATE + coldTail * 1.0; } /** diff --git a/src/sessions.ts b/src/sessions.ts index bc0a748..acaa347 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -165,11 +165,12 @@ export async function aggregateSessions( // Cling to whichever cwd we saw first; sessions that hop directories are // rare and the first cwd is the most stable identifier. if (s.project === undefined && ev.cwd) s.project = ev.cwd; - // Real per-session savings, cache-aware. See src/core/baseline.ts for - // the full derivation and the May-2026 regression that motivated the - // rewrite — the previous formula collapsed every warm turn's unproxied - // counterfactual to 100% cache_read × 0.10 and flipped the headline - // negative whenever the proxied path paid real cache_create. + // Real per-session savings, cache-aware and warmth-free. See + // src/core/baseline.ts for the full derivation: the cached prefix cancels + // (paid identically on both paths), so we credit only the net-new uncached + // text pxpipe compressed away — honest `baselineEff − actualEff`, NO >=0 + // floor, so a net-losing turn (cc-heavy rewrite) lowers the total. This is + // deterministic — no per-session warmth state — so live and replay agree. // Events missing either probe stay out of the rollup — no estimation. const inp = ev.input_tokens ?? 0; const cc = ev.cache_create_tokens ?? 0; @@ -181,12 +182,8 @@ export async function aggregateSessions( baseline > 0 && haveUsage ) { - const baselineEff = computeBaselineInputEff( - baseline, - ev.baseline_cacheable_tokens ?? 0, - cc, - cr, - ); + const cacheable = ev.baseline_cacheable_tokens ?? 0; + const baselineEff = computeBaselineInputEff(baseline, cacheable, inp, cc, cr); const actualEff = computeActualInputEff(inp, cc, cr); const tokensSaved = baselineEff - actualEff; s.tokensSavedEst += Math.round(tokensSaved); diff --git a/tests/sessions.test.ts b/tests/sessions.test.ts index 02ba231..f2dc6d1 100644 --- a/tests/sessions.test.ts +++ b/tests/sessions.test.ts @@ -129,76 +129,83 @@ describe('aggregateSessions', () => { expect(sessions.get('aaaaaaaa')?.requestCount).toBe(2); }); - it('accumulates tokens saved from baseline_tokens vs upstream usage (allows negative)', async () => { + it('credits the real prefix compression (image prefix fewer tokens than text prefix)', async () => { writeEvents(tmp, [ - // baseline 20000, actual_input_eff = 1000 + 800*1.25 + 100*0.10 = 2010 - // saved = 20000 − 2010 = 17990 + // Text counterfactual: cacheable prefix 18000 read warm @0.1 + cold tail 2000 + // = 1800 + 2000 = 3800. actual = 1000 + 800*1.25 + 100*0.1 = 2010. + // pp's cr (100) is far fewer than the text prefix (18000) -> saved 1790. ev({ first_user_sha8: 'aaaaaaaa', compressed: true, baseline_tokens: 20_000, + baseline_cacheable_tokens: 18_000, input_tokens: 1_000, cache_create_tokens: 800, cache_read_tokens: 100, }), - // baseline 2000, actual_input_eff = 3000 + 0 + 0 = 3000 → NET LOSS −1000 - // (must NOT be clamped to zero) + // Whole body cacheable: baseline = 9000*0.1 = 900. actual = 5 + 8000*0.1 = 805. + // pp's image prefix (cr 8000) < text prefix (9000) -> saved 95. ev({ first_user_sha8: 'aaaaaaaa', compressed: true, - baseline_tokens: 2_000, - input_tokens: 3_000, - cache_create_tokens: 0, - cache_read_tokens: 0, + baseline_tokens: 9_000, + baseline_cacheable_tokens: 9_000, + input_tokens: 5, + cache_read_tokens: 8_000, }), - // Missing baseline — skipped from savings rollup, still counts toward requests. + // Probe miss (no cacheable marker): we cannot split prefix from tail, so + // we credit NOTHING — the regression guard for the old cacheable=0 → + // cold_tail=baseline fabrication (would have falsely "saved" ~46000 here). + ev({ + first_user_sha8: 'aaaaaaaa', + compressed: true, + baseline_tokens: 50_000, + input_tokens: 6, + cache_read_tokens: 40_000, + }), + // Missing baseline — skipped from savings, still counts toward requests. ev({ first_user_sha8: 'aaaaaaaa', compressed: false, input_tokens: 500, }), - // Has baseline but no usage block — also skipped (apples-to-apples). - ev({ - first_user_sha8: 'aaaaaaaa', - compressed: true, - baseline_tokens: 9_999, - }), ]); const { sessions } = await aggregateSessions(tmp); const s = sessions.get('aaaaaaaa')!; - // 17990 + (−1000) = 16990 - expect(s.tokensSavedEst).toBe(16_990); - expect(s.charsSaved).toBe(16_990 * 4); + // 1790 + 95 + 0 = 1885 + expect(s.tokensSavedEst).toBe(1_885); + expect(s.charsSaved).toBe(1_885 * 4); expect(s.requestCount).toBe(4); }); - it('reports negative tokensSavedEst when all events net-lose', async () => { + it('reports a real NEGATIVE when cache_create overhead exceeds the prefix saving; probe-miss credits 0', async () => { writeEvents(tmp, [ - // Each event: baseline=1000, actual=2000 → −1000 + // Probe miss: no marker -> credit nothing (was a ~95000-token fabrication + // under the old formula). saved 0. ev({ first_user_sha8: 'bbbbbbbb', compressed: true, - baseline_tokens: 1_000, - input_tokens: 2_000, + baseline_tokens: 100_000, + input_tokens: 5, + cache_read_tokens: 90_000, }), + // Genuine loss turn: tiny body (2000) but pp wrote 5000 cache_create. + // baseline = 1900*0.1 + 100 = 290 ; actual = 3000 + 5000*1.25 = 9250. + // saved = 290 - 9250 = -8960. Honest formula reports the real loss, no clamp. ev({ first_user_sha8: 'bbbbbbbb', compressed: true, - baseline_tokens: 1_000, - input_tokens: 2_000, - }), - ev({ - first_user_sha8: 'bbbbbbbb', - compressed: true, - baseline_tokens: 1_000, - input_tokens: 2_000, + baseline_tokens: 2_000, + baseline_cacheable_tokens: 1_900, + input_tokens: 3_000, + cache_create_tokens: 5_000, }), ]); const { sessions } = await aggregateSessions(tmp); const s = sessions.get('bbbbbbbb')!; - // 3 × −1000 = −3000 - expect(s.tokensSavedEst).toBe(-3_000); - expect(s.charsSaved).toBe(-12_000); + // 0 + (-8960) + expect(s.tokensSavedEst).toBe(-8_960); + expect(s.charsSaved).toBe(-8_960 * 4); }); });