From 8fbd53ea3efcfe2ef0eed30d2febe91e855d366e Mon Sep 17 00:00:00 2001 From: teamchong <25894545+teamchong@users.noreply.github.com> Date: Mon, 18 May 2026 23:01:12 -0400 Subject: [PATCH] dashboard: fix bogus savings math + emit compressed_chars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard headline was showing 0% saved because the math was apples-to-oranges: - orig_chars = static slab + tool docs ONLY (~135 KB) - image_count = ALL images including reminders + tool_results (~33) - txtReplaced = 135k/4 = 33.7k tokens - imgTokensEst = 33 × 2500 = 82.5k tokens - extraText = max(0, 33.7k - 82.5k) = 0 ← clamped negative - baseline == paid → 'saved 0%' headline despite real savings happening Two fixes: 1. Track 'compressedChars' separately from 'origChars'. It accumulates only when a block actually image-renders (passes the break-even gate AND any per-block min thresholds). Includes the static slab + all compressed reminders + all compressed tool_results. 2. Remove the Math.max(0,...) clamp in baselineCost(). Negative savings surface honestly — they're the signal that small-block compressions are cost-bleeding, which the break-even gate is supposed to prevent. Wires: - TransformInfo.compressedChars (new field, mandatory). - All 4 image-render success paths accumulate (static slab + reminder + tool_result string branch + tool_result array branch). - TrackEvent.compressed_chars (new optional field, emitted when > 0). - dashboard.baselineCost() takes compressedChars now; clamp removed. - sessions.aggregateSessions() also uses compressed_chars + honest math. 229/229 tests passing. Bundle stays at 496 KB gzipped. Post-restart, the dashboard 'tokens saved' will show real numbers including negatives. Empirically expect ~1-5% net positive in steady-state cache-hit traffic. --- src/core/tracker.ts | 7 +++++++ src/core/transform.ts | 24 +++++++++++++++++++++++- src/dashboard.ts | 15 +++++++++++---- src/sessions.ts | 14 +++++--------- tests/sessions.test.ts | 8 +++++--- 5 files changed, 51 insertions(+), 17 deletions(-) diff --git a/src/core/tracker.ts b/src/core/tracker.ts index 2cf0cf7..a8ceb96 100644 --- a/src/core/tracker.ts +++ b/src/core/tracker.ts @@ -29,6 +29,10 @@ export interface TrackEvent { compressed?: boolean; reason?: string; orig_chars?: number; + /** Sum of text-chars actually replaced by image blocks this request (static + * slab + reminders + tool_results). Apples-to-apples with image_count for + * savings math: textTokens(compressed_chars/4) vs imageTokens(image_count×2500). */ + compressed_chars?: number; image_count?: number; image_bytes?: number; static_chars?: number; @@ -160,6 +164,9 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { if (info.compressed !== undefined) out.compressed = info.compressed; if (info.reason) out.reason = info.reason; if (info.origChars !== undefined) out.orig_chars = info.origChars; + if (info.compressedChars !== undefined && info.compressedChars > 0) { + out.compressed_chars = info.compressedChars; + } if (info.imageCount !== undefined) out.image_count = info.imageCount; if (info.imageBytes !== undefined) out.image_bytes = info.imageBytes; if (info.staticChars !== undefined) out.static_chars = info.staticChars; diff --git a/src/core/transform.ts b/src/core/transform.ts index 2198aed..8404cd0 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -197,6 +197,15 @@ export interface TransformInfo { compressed: boolean; reason?: string; origChars: number; + /** Total chars of source text that were image-encoded across ALL blocks + * this request (static slab + reminders + tool_results). Pairs with + * `imageCount` for honest savings math: + * textTokens = compressedChars / 4 + * imageTokens = imageCount × 2500 + * savings = textTokens − imageTokens + * Unlike `origChars` (which is just static slab + tool docs), + * `compressedChars` reflects what `imageCount` actually replaced. */ + compressedChars: number; imageCount: number; imageBytes: number; /** Length of the static (cacheable) slab rendered into the image. */ @@ -973,6 +982,7 @@ export async function transformRequest( const info: TransformInfo = { compressed: false, origChars: 0, + compressedChars: 0, imageCount: 0, imageBytes: 0, staticChars: 0, @@ -1089,6 +1099,10 @@ export async function transformRequest( // cache key (= image bytes) stays stable across turns. const combined = [staticText, toolDocsText].filter((s) => s.length > 0).join('\n\n'); info.origChars = combined.length; + // Track chars of the static slab+tools that DO end up imaged. The + // break-even gate below may reject — bump only when the slab actually + // renders. Reminder/tool_result compressions add to this at their sites. + info.compressedChars = 0; // Hash the EXACT text that goes into the image. Repeats of this hash across // turns = cache_control should be earning its keep. if (combined) info.systemSha8 = await sha8(combined); @@ -1123,6 +1137,8 @@ export async function transformRequest( imageBlocks.push(makeImageBlock(b64, i === images.length - 1)); } info.imageCount = imageBlocks.length; + // Static slab made it through the break-even gate and rendered. + info.compressedChars += combined.length; // Stash the first image's raw bytes + dimensions for the dashboard preview. // Stripped before persisting to JSONL by toTrackEvent. Memory cost is bounded // (we only ever keep ONE — the latest — via the dashboard's replace-on-update). @@ -1202,13 +1218,15 @@ export async function transformRequest( processedExisting.push(blk); continue; } + const reminderText = (blk as TextBlock).text; const { blocks: imgs, droppedChars, droppedCodepoints: dcp } = - await textToImageBlocks((blk as TextBlock).text, o.cols); + await textToImageBlocks(reminderText, o.cols); for (const img of imgs) { processedExisting.push(img); info.imageBytes += approxBlockBytes(img); } info.reminderImgs = (info.reminderImgs ?? 0) + imgs.length; + info.compressedChars += reminderText.length; info.imageCount += imgs.length; info.droppedChars = (info.droppedChars ?? 0) + droppedChars; for (const [cp, n] of dcp) { @@ -1270,6 +1288,9 @@ export async function transformRequest( for (const img of imgs) info.imageBytes += approxBlockBytes(img); info.toolResultImgs = (info.toolResultImgs ?? 0) + imgs.length; info.imageCount += imgs.length; + // Use original (pre-paging) length: that's what we would have + // paid for as text. + info.compressedChars += inner.length; info.droppedChars = (info.droppedChars ?? 0) + droppedChars; for (const [cp, n] of dcp) { droppedCodepoints.set(cp, (droppedCodepoints.get(cp) ?? 0) + n); @@ -1313,6 +1334,7 @@ export async function transformRequest( } info.toolResultImgs = (info.toolResultImgs ?? 0) + imgs.length; info.imageCount += imgs.length; + info.compressedChars += innerText.length; info.droppedChars = (info.droppedChars ?? 0) + droppedChars; for (const [cp, n] of dcp) { droppedCodepoints.set(cp, (droppedCodepoints.get(cp) ?? 0) + n); diff --git a/src/dashboard.ts b/src/dashboard.ts index 29a3ad5..cf0fa5b 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -129,14 +129,21 @@ function effectiveCost( */ function baselineCost( actualEff: number, - origChars: number, + compressedChars: number, imageCount: number, cacheCreate: number, cacheRead: number, ): number { - const txtReplaced = Math.floor(origChars / 4); // ~4 chars per token in English + // compressedChars is the actual text we IMAGE-encoded (static slab + + // reminders + tool_results that passed the break-even gate). NOT the + // total origChars — that would include text-only blocks that stayed + // as text and wouldn't have appeared as `imageCount` either, making + // the apples-to-apples comparison invalid. + const txtReplaced = Math.floor(compressedChars / 4); // ~4 chars per token const imgTokensEst = estImageTokens(imageCount); - const extraText = Math.max(0, txtReplaced - imgTokensEst); + // extraText CAN be negative when image cost > text cost. Don't clamp — + // surfaces honest negatives so the operator can see cost-bleed. + const extraText = txtReplaced - imgTokensEst; const cachedTotal = cacheCreate + cacheRead; const baselineRate = cachedTotal > 0 ? (cacheCreate / cachedTotal) * 1.25 + (cacheRead / cachedTotal) * 0.1 : 0.1; @@ -208,7 +215,7 @@ export class DashboardState { const eff = haveUsage ? effectiveCost(inp, cc, cr) : 0; const baselineEff = haveUsage && compressed - ? baselineCost(eff, info?.origChars ?? 0, info?.imageCount ?? 0, cc, cr) + ? baselineCost(eff, info?.compressedChars ?? 0, info?.imageCount ?? 0, cc, cr) : eff; const prevSaved = this.totals.effectiveInputBaselineEst - this.totals.effectiveInputActual; diff --git a/src/sessions.ts b/src/sessions.ts index 7773696..9bc95ab 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -159,17 +159,13 @@ export async function aggregateSessions( if (s.project === undefined && ev.cwd) s.project = ev.cwd; if ( ev.compressed && - typeof ev.orig_chars === 'number' && + typeof ev.compressed_chars === 'number' && typeof ev.image_count === 'number' ) { - // Honest savings math, mirrors src/dashboard.ts: - // textTokens = orig_chars / 4 (Anthropic ~4 chars/token rule) - // imageTokens = image_count × 2500 (empirical per-image cost) - // tokensSaved = textTokens − imageTokens (CAN be negative when - // small per-block compressions cost more than they save) - // Previous formula compared chars to PNG bytes (mismatched units) AND - // clamped via `if (saved > 0)` which hid the cost-bleed entirely. - const textTokens = ev.orig_chars / 4; + // Honest savings math (mirrors src/dashboard.ts:baselineCost). + // compressed_chars is the actual text we IMAGE-encoded; orig_chars + // would include text-only blocks that didn't contribute to image_count. + const textTokens = ev.compressed_chars / 4; const imageTokens = ev.image_count * 2500; const tokensSaved = textTokens - imageTokens; s.tokensSavedEst += Math.round(tokensSaved); diff --git a/tests/sessions.test.ts b/tests/sessions.test.ts index a50771e..5b9adbe 100644 --- a/tests/sessions.test.ts +++ b/tests/sessions.test.ts @@ -145,6 +145,7 @@ describe('aggregateSessions', () => { first_user_sha8: 'aaaaaaaa', compressed: true, orig_chars: 100_000, + compressed_chars: 100_000, image_count: 2, }), // 5,000 chars compressed to 1 image — NET LOSS. @@ -155,6 +156,7 @@ describe('aggregateSessions', () => { first_user_sha8: 'aaaaaaaa', compressed: true, orig_chars: 5_000, + compressed_chars: 5_000, image_count: 1, }), // Not compressed at all — skipped. @@ -175,9 +177,9 @@ describe('aggregateSessions', () => { it('reports negative tokensSavedEst when all compressions net-lose', async () => { writeEvents(tmp, [ // 3 small blocks each net-lose ~1250 tokens. - ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, image_count: 1 }), - ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, image_count: 1 }), - ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, image_count: 1 }), + ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, compressed_chars: 5000, image_count: 1 }), + ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, compressed_chars: 5000, image_count: 1 }), + ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, compressed_chars: 5000, image_count: 1 }), ]); const { sessions } = await aggregateSessions(tmp); const s = sessions.get('bbbbbbbb')!;