diff --git a/src/core/tracker.ts b/src/core/tracker.ts index 2cf0cf7..a8ceb96 100644 --- a/src/core/tracker.ts +++ b/src/core/tracker.ts @@ -29,6 +29,10 @@ export interface TrackEvent { compressed?: boolean; reason?: string; orig_chars?: number; + /** Sum of text-chars actually replaced by image blocks this request (static + * slab + reminders + tool_results). Apples-to-apples with image_count for + * savings math: textTokens(compressed_chars/4) vs imageTokens(image_count×2500). */ + compressed_chars?: number; image_count?: number; image_bytes?: number; static_chars?: number; @@ -160,6 +164,9 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent { if (info.compressed !== undefined) out.compressed = info.compressed; if (info.reason) out.reason = info.reason; if (info.origChars !== undefined) out.orig_chars = info.origChars; + if (info.compressedChars !== undefined && info.compressedChars > 0) { + out.compressed_chars = info.compressedChars; + } if (info.imageCount !== undefined) out.image_count = info.imageCount; if (info.imageBytes !== undefined) out.image_bytes = info.imageBytes; if (info.staticChars !== undefined) out.static_chars = info.staticChars; diff --git a/src/core/transform.ts b/src/core/transform.ts index 2198aed..8404cd0 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -197,6 +197,15 @@ export interface TransformInfo { compressed: boolean; reason?: string; origChars: number; + /** Total chars of source text that were image-encoded across ALL blocks + * this request (static slab + reminders + tool_results). Pairs with + * `imageCount` for honest savings math: + * textTokens = compressedChars / 4 + * imageTokens = imageCount × 2500 + * savings = textTokens − imageTokens + * Unlike `origChars` (which is just static slab + tool docs), + * `compressedChars` reflects what `imageCount` actually replaced. */ + compressedChars: number; imageCount: number; imageBytes: number; /** Length of the static (cacheable) slab rendered into the image. */ @@ -973,6 +982,7 @@ export async function transformRequest( const info: TransformInfo = { compressed: false, origChars: 0, + compressedChars: 0, imageCount: 0, imageBytes: 0, staticChars: 0, @@ -1089,6 +1099,10 @@ export async function transformRequest( // cache key (= image bytes) stays stable across turns. const combined = [staticText, toolDocsText].filter((s) => s.length > 0).join('\n\n'); info.origChars = combined.length; + // Track chars of the static slab+tools that DO end up imaged. The + // break-even gate below may reject — bump only when the slab actually + // renders. Reminder/tool_result compressions add to this at their sites. + info.compressedChars = 0; // Hash the EXACT text that goes into the image. Repeats of this hash across // turns = cache_control should be earning its keep. if (combined) info.systemSha8 = await sha8(combined); @@ -1123,6 +1137,8 @@ export async function transformRequest( imageBlocks.push(makeImageBlock(b64, i === images.length - 1)); } info.imageCount = imageBlocks.length; + // Static slab made it through the break-even gate and rendered. + info.compressedChars += combined.length; // Stash the first image's raw bytes + dimensions for the dashboard preview. // Stripped before persisting to JSONL by toTrackEvent. Memory cost is bounded // (we only ever keep ONE — the latest — via the dashboard's replace-on-update). @@ -1202,13 +1218,15 @@ export async function transformRequest( processedExisting.push(blk); continue; } + const reminderText = (blk as TextBlock).text; const { blocks: imgs, droppedChars, droppedCodepoints: dcp } = - await textToImageBlocks((blk as TextBlock).text, o.cols); + await textToImageBlocks(reminderText, o.cols); for (const img of imgs) { processedExisting.push(img); info.imageBytes += approxBlockBytes(img); } info.reminderImgs = (info.reminderImgs ?? 0) + imgs.length; + info.compressedChars += reminderText.length; info.imageCount += imgs.length; info.droppedChars = (info.droppedChars ?? 0) + droppedChars; for (const [cp, n] of dcp) { @@ -1270,6 +1288,9 @@ export async function transformRequest( for (const img of imgs) info.imageBytes += approxBlockBytes(img); info.toolResultImgs = (info.toolResultImgs ?? 0) + imgs.length; info.imageCount += imgs.length; + // Use original (pre-paging) length: that's what we would have + // paid for as text. + info.compressedChars += inner.length; info.droppedChars = (info.droppedChars ?? 0) + droppedChars; for (const [cp, n] of dcp) { droppedCodepoints.set(cp, (droppedCodepoints.get(cp) ?? 0) + n); @@ -1313,6 +1334,7 @@ export async function transformRequest( } info.toolResultImgs = (info.toolResultImgs ?? 0) + imgs.length; info.imageCount += imgs.length; + info.compressedChars += innerText.length; info.droppedChars = (info.droppedChars ?? 0) + droppedChars; for (const [cp, n] of dcp) { droppedCodepoints.set(cp, (droppedCodepoints.get(cp) ?? 0) + n); diff --git a/src/dashboard.ts b/src/dashboard.ts index 29a3ad5..cf0fa5b 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -129,14 +129,21 @@ function effectiveCost( */ function baselineCost( actualEff: number, - origChars: number, + compressedChars: number, imageCount: number, cacheCreate: number, cacheRead: number, ): number { - const txtReplaced = Math.floor(origChars / 4); // ~4 chars per token in English + // compressedChars is the actual text we IMAGE-encoded (static slab + + // reminders + tool_results that passed the break-even gate). NOT the + // total origChars — that would include text-only blocks that stayed + // as text and wouldn't have appeared as `imageCount` either, making + // the apples-to-apples comparison invalid. + const txtReplaced = Math.floor(compressedChars / 4); // ~4 chars per token const imgTokensEst = estImageTokens(imageCount); - const extraText = Math.max(0, txtReplaced - imgTokensEst); + // extraText CAN be negative when image cost > text cost. Don't clamp — + // surfaces honest negatives so the operator can see cost-bleed. + const extraText = txtReplaced - imgTokensEst; const cachedTotal = cacheCreate + cacheRead; const baselineRate = cachedTotal > 0 ? (cacheCreate / cachedTotal) * 1.25 + (cacheRead / cachedTotal) * 0.1 : 0.1; @@ -208,7 +215,7 @@ export class DashboardState { const eff = haveUsage ? effectiveCost(inp, cc, cr) : 0; const baselineEff = haveUsage && compressed - ? baselineCost(eff, info?.origChars ?? 0, info?.imageCount ?? 0, cc, cr) + ? baselineCost(eff, info?.compressedChars ?? 0, info?.imageCount ?? 0, cc, cr) : eff; const prevSaved = this.totals.effectiveInputBaselineEst - this.totals.effectiveInputActual; diff --git a/src/sessions.ts b/src/sessions.ts index 7773696..9bc95ab 100644 --- a/src/sessions.ts +++ b/src/sessions.ts @@ -159,17 +159,13 @@ export async function aggregateSessions( if (s.project === undefined && ev.cwd) s.project = ev.cwd; if ( ev.compressed && - typeof ev.orig_chars === 'number' && + typeof ev.compressed_chars === 'number' && typeof ev.image_count === 'number' ) { - // Honest savings math, mirrors src/dashboard.ts: - // textTokens = orig_chars / 4 (Anthropic ~4 chars/token rule) - // imageTokens = image_count × 2500 (empirical per-image cost) - // tokensSaved = textTokens − imageTokens (CAN be negative when - // small per-block compressions cost more than they save) - // Previous formula compared chars to PNG bytes (mismatched units) AND - // clamped via `if (saved > 0)` which hid the cost-bleed entirely. - const textTokens = ev.orig_chars / 4; + // Honest savings math (mirrors src/dashboard.ts:baselineCost). + // compressed_chars is the actual text we IMAGE-encoded; orig_chars + // would include text-only blocks that didn't contribute to image_count. + const textTokens = ev.compressed_chars / 4; const imageTokens = ev.image_count * 2500; const tokensSaved = textTokens - imageTokens; s.tokensSavedEst += Math.round(tokensSaved); diff --git a/tests/sessions.test.ts b/tests/sessions.test.ts index a50771e..5b9adbe 100644 --- a/tests/sessions.test.ts +++ b/tests/sessions.test.ts @@ -145,6 +145,7 @@ describe('aggregateSessions', () => { first_user_sha8: 'aaaaaaaa', compressed: true, orig_chars: 100_000, + compressed_chars: 100_000, image_count: 2, }), // 5,000 chars compressed to 1 image — NET LOSS. @@ -155,6 +156,7 @@ describe('aggregateSessions', () => { first_user_sha8: 'aaaaaaaa', compressed: true, orig_chars: 5_000, + compressed_chars: 5_000, image_count: 1, }), // Not compressed at all — skipped. @@ -175,9 +177,9 @@ describe('aggregateSessions', () => { it('reports negative tokensSavedEst when all compressions net-lose', async () => { writeEvents(tmp, [ // 3 small blocks each net-lose ~1250 tokens. - ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, image_count: 1 }), - ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, image_count: 1 }), - ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, image_count: 1 }), + ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, compressed_chars: 5000, image_count: 1 }), + ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, compressed_chars: 5000, image_count: 1 }), + ev({ first_user_sha8: 'bbbbbbbb', compressed: true, orig_chars: 5000, compressed_chars: 5000, image_count: 1 }), ]); const { sessions } = await aggregateSessions(tmp); const s = sessions.get('bbbbbbbb')!;