ai-space/pxpipe
3
0
Fork 0
mirror of https://github.com/teamchong/pxpipe.git synced 2026-07-22 02:02:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat(proxy): cache-aware baseline via second count_tokens probe

Browse Source 
The baseline_tokens probe alone reports the COLD cost of the unproxied
body — but the user's request actually carries `cache_control` markers
that would have cached on the unproxied path too. Comparing cold-
baseline against warm-actual was an apples-to-oranges inflation of
saved%.

Fix: fire a SECOND count_tokens probe on the body TRUNCATED at the last
`cache_control` marker. Difference between the two is (cacheable_prefix,
cold_tail). Dashboard weights the prefix by the SAME cache class the
actual response landed in:

    cacheable = baseline_cacheable_tokens or 0
    cold_tail = baseline_tokens − cacheable
    weight    = cr>0 ? 0.10 (warm hit — both paths cache)
              : cc>0 ? 1.25 (cold create — both paths build cache)
              :        1.0  (no caching — marker rejected or absent)
    baseline_input_eff = cacheable·weight + cold_tail·1.0
    actual_input_eff   = input + cache_create·1.25 + cache_read·0.10
    saved              = baseline_input_eff − actual_input_eff

Now the headline number is the exact unproxied counterfactual, not a
cold-every-time approximation. No estimation, no α, no assumptions —
every input is a measurement.

Wiring:
- src/core/proxy.ts: buildCountTokensCacheablePrefix() walks messages
  backward → system backward → tools backward (reverse cache order) for
  the last cache_control marker, truncates, attaches a sentinel user
  message when needed. Second probe fires in parallel; null/missing
  leaves the field absent.
- src/core/transform.ts + tracker.ts: new baselineCacheableTokens /
  baseline_cacheable_tokens fields, emitted only when > 0.
- src/dashboard.ts: per-event cache-aware math in update() and
  replay(); Totals renamed to baselineInputWeighted; /proxy-stats
  payload field is `baseline_input_weighted`.
- src/sessions.ts: per-session tokensSavedEst uses the same formula so
  session and global rollups agree.

Tests: 2 new in proxy-usage covering (a) body with cache_control →
two probes fire with different bodies and both numbers land on info,
(b) cacheable-prefix probe failure doesn't poison the full probe.
260→262 tests, all green; typecheck and build clean.
This commit is contained in:
teamchong
2026-05-19 22:29:07 -04:00
parent 424f84172a
commit dfd3bf514f
6 changed files with 468 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/proxy.ts
+170 -19
View File
@@ -291,6 +291,126 @@ function buildCountTokensBody(bytes: Uint8Array): Uint8Array | null {
}
}
/** Type guard for content blocks that may carry a cache_control marker.
* Anthropic accepts `cache_control` on: any system block, any message
* content block, any tool definition. We don't care about the marker's
* value — only its presence/position. */
function hasCacheControl(x: unknown): boolean {
return (
typeof x === 'object'
&& x !== null
&& (x as { cache_control?: unknown }).cache_control != null
);
}
/** Build a body that contains EXACTLY the tokens forming the longest
* cacheable prefix on the unproxied path — everything up to and INCLUDING
* the last `cache_control` marker in the original request, with everything
* after that marker stripped. count_tokens on this body returns
* `cacheable_prefix_tokens`; subtracting from the full count_tokens gives
* `cold_tail_tokens` (always-cold input on both proxied and unproxied paths).
*
* Anthropic's cache-traversal order is tools → system → messages. We walk
* messages first (latest), then system, then tools — the FIRST one we find
* a marker in (walking backward) is the latest in cache order. Everything
* after that marker in the same section is dropped; later sections are
* dropped wholesale.
*
* Returns null when the original body has zero `cache_control` markers
* anywhere — caller treats that as `cacheable_prefix_tokens = 0`. The
* unproxied path doesn't cache anything in that case, so the full body
* is cold input. */
function buildCountTokensCacheablePrefix(bytes: Uint8Array): Uint8Array | null {
let obj: Record<string, unknown>;
try {
obj = JSON.parse(new TextDecoder().decode(bytes)) as Record<string, unknown>;
} catch {
return null;
}
if (typeof obj.model !== 'string') return null;
const system = obj.system;
const messages = obj.messages;
const tools = obj.tools;
// Walk messages backward; for each message walk content blocks backward.
// First block we hit with cache_control is the latest marker in the
// entire request (messages come last in cache traversal order).
let truncated: Record<string, unknown> | null = null;
if (Array.isArray(messages)) {
for (let mi = messages.length - 1; mi >= 0 && truncated == null; mi--) {
const msg = messages[mi] as { role?: unknown; content?: unknown };
const content = msg?.content;
if (Array.isArray(content)) {
for (let bi = content.length - 1; bi >= 0; bi--) {
if (hasCacheControl(content[bi])) {
// Truncate this message's content at and including the marker.
const truncatedMsg = { ...msg, content: content.slice(0, bi + 1) };
const truncatedMessages = messages.slice(0, mi).concat([truncatedMsg]);
truncated = {
model: obj.model,
messages: truncatedMessages,
};
if (system !== undefined) truncated.system = system;
if (tools !== undefined) truncated.tools = tools;
break;
}
}
} else if (hasCacheControl(msg)) {
// String-content message with a marker — keep up to this message.
truncated = {
model: obj.model,
messages: messages.slice(0, mi + 1),
};
if (system !== undefined) truncated.system = system;
if (tools !== undefined) truncated.tools = tools;
}
}
}
// No marker in messages — check system (next-earliest in cache order).
// count_tokens requires non-empty messages, so we append a 1-token sentinel
// user message and subtract its weight on the consumer side. Or simpler:
// include a single-char user message and accept ~1 token of noise.
if (truncated == null && Array.isArray(system)) {
for (let si = system.length - 1; si >= 0; si--) {
if (hasCacheControl(system[si])) {
truncated = {
model: obj.model,
system: system.slice(0, si + 1),
messages: [{ role: 'user', content: 'x' }],
};
if (tools !== undefined) truncated.tools = tools;
break;
}
}
}
// No marker in system — check tools (earliest in cache order). Same
// sentinel-message trick to keep count_tokens happy.
if (truncated == null && Array.isArray(tools)) {
for (let ti = tools.length - 1; ti >= 0; ti--) {
if (hasCacheControl(tools[ti])) {
truncated = {
model: obj.model,
tools: tools.slice(0, ti + 1),
messages: [{ role: 'user', content: 'x' }],
};
break;
}
}
}
// Filter through the same whitelist as the full-body probe to avoid 400s
// from stray fields slipping into the truncated copy.
if (truncated == null) return null;
const out: Record<string, unknown> = {};
for (const k of Object.keys(truncated)) {
if (COUNT_TOKENS_FIELDS.has(k)) out[k] = truncated[k];
}
return new TextEncoder().encode(JSON.stringify(out));
}
/** POST /v1/messages/count_tokens with the given body. Returns the upstream's
* `input_tokens` number or null on any failure. count_tokens is documented
* as a free endpoint (no input-token billing) — we use it once per request
@@ -354,16 +474,27 @@ export function createProxy(config: ProxyConfig = {}) {
// gzip failure is non-fatal — drop the body sample, keep the rest.
}
}
// Wait for the baseline count_tokens probe before persisting the
// event so baseline_tokens lands on the same row as the usage block.
// null result (probe failed) leaves the field absent — dashboard
// skips that event from the savings math.
if (baselinePromise && info) {
try {
const b = await baselinePromise;
if (b !== null) info.baselineTokens = b;
} catch {
/* probe threw — drop, keep the rest of the event intact */
// Wait for the baseline count_tokens probes before persisting the
// event so both numbers land on the same row as the usage block.
// Each probe is independent: full-body baseline can land even if the
// cacheable-prefix probe fails (and vice versa). null/missing leaves
// the field absent; the dashboard's per-event math degrades cleanly.
if (info) {
if (baselinePromise) {
try {
const b = await baselinePromise;
if (b !== null) info.baselineTokens = b;
} catch {
/* probe threw — drop, keep the rest of the event intact */
}
}
if (baselineCacheablePromise) {
try {
const c = await baselineCacheablePromise;
if (c !== null) info.baselineCacheableTokens = c;
} catch {
/* probe threw — leave field absent */
}
}
}
await config.onRequest?.({
@@ -389,11 +520,20 @@ export function createProxy(config: ProxyConfig = {}) {
let bodyOut: BodyInit | null = null;
let info: TransformInfo | undefined;
// Ground-truth baseline measurement. Fires /v1/messages/count_tokens on
// the PRE-COMPRESSION body in parallel with the main forward. Resolves
// to a number (baseline tokens) or null (probe failed, e.g. count_tokens
// 4xx'd). Lands on info.baselineTokens before the host event persists.
// Ground-truth baseline measurement. Fires /v1/messages/count_tokens TWO
// ways on the PRE-COMPRESSION body in parallel with the main forward:
// baselinePromise → full-body input_tokens (cold cost)
// baselineCacheablePromise → input_tokens of the body TRUNCATED at the
// last cache_control marker (the prefix that
// would have cached on the unproxied path)
// Difference of the two is `cold_tail_tokens` (always-cold input on both
// proxied and unproxied paths). The dashboard combines them with the
// actual usage block's cache class to get an exact cache-aware baseline,
// not a cold-every-time approximation. Both resolve to a number or null
// (probe failed or no markers); land on info.baseline*Tokens before the
// host event persists.
let baselinePromise: Promise<number | null> | undefined;
let baselineCacheablePromise: Promise<number | null> | undefined;
if (isMessages) {
const bodyIn = new Uint8Array(await req.arrayBuffer());
@@ -410,17 +550,28 @@ export function createProxy(config: ProxyConfig = {}) {
reqBodySha8 = await sha8Bytes(r.body);
}
// Kick off the count_tokens probe on the ORIGINAL body BEFORE the
// main forward so the two calls overlap. Anthropic doesn't bill
// count_tokens, and the result is the only honest baseline we can
// get — the post-compression token count comes back free in the
// /v1/messages response usage block.
// Kick off the count_tokens probes on the ORIGINAL body BEFORE the
// main forward so all three calls (full probe, cacheable-prefix probe,
// main /v1/messages) overlap. Anthropic doesn't bill count_tokens, so
// the cost is wall-clock only — typically ~30-80ms, fully hidden by
// the main forward latency.
const ctBody = buildCountTokensBody(bodyIn);
if (ctBody) {
const ctHeaders = filterHeaders(req.headers, STRIP_REQ_HEADERS);
ctHeaders.set('content-type', 'application/json');
if (config.apiKey) ctHeaders.set('x-api-key', config.apiKey);
baselinePromise = countTokensUpstream(upstream, ctBody, ctHeaders);
// Second probe: body truncated at the last cache_control marker.
// Null body = no markers exist → cacheable=0 by definition, no
// probe needed.
const ctCacheableBody = buildCountTokensCacheablePrefix(bodyIn);
if (ctCacheableBody) {
baselineCacheablePromise = countTokensUpstream(
upstream,
ctCacheableBody,
new Headers(ctHeaders),
);
}
}
} catch (e) {
fire(502, undefined, `transform_error: ${(e as Error).message}`);
src/core/tracker.ts
+15 -3
View File
@@ -109,10 +109,16 @@ export interface TrackEvent {
/** Ground-truth pre-compression token count from a parallel call to
* /v1/messages/count_tokens on the ORIGINAL request body. The endpoint
* is free (no billing). Absent when the probe failed; those events are
* excluded from the dashboard's savings rollup. The post-compression
* total comes from the usage block above (input + cache_create +
* cache_read), so no second probe is needed. */
* excluded from the dashboard's savings rollup. */
baseline_tokens?: number;
/** Second baseline probe: input_tokens of the original body TRUNCATED at
* the last `cache_control` marker. With `baseline_tokens` it decomposes
* the unproxied path's cost into (cacheable_prefix, cold_tail) so the
* dashboard can apply the SAME cache class the actual request landed in
* for a true apples-to-apples counterfactual. Absent when the original
* body has no cache_control markers (cacheable=0, the whole body is the
* cold tail). */
baseline_cacheable_tokens?: number;
// Errors:
error?: string;
@@ -235,6 +241,12 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent {
if (info.baselineTokens !== undefined && info.baselineTokens > 0) {
out.baseline_tokens = info.baselineTokens;
}
if (
info.baselineCacheableTokens !== undefined
&& info.baselineCacheableTokens > 0
) {
out.baseline_cacheable_tokens = info.baselineCacheableTokens;
}
}
if (env) {
if (env.cwd) out.cwd = env.cwd;
src/core/transform.ts
+12 -5
View File
@@ -384,12 +384,19 @@ export interface TransformInfo {
| 'collapsed';
/** Ground-truth baseline token count for THIS request, from a parallel
* call to /v1/messages/count_tokens on the PRE-COMPRESSION body. The
* endpoint is free (no input-token billing). When present, the dashboard
* computes the real saved_pct directly: actual = input + cache_create +
* cache_read (from /v1/messages usage), saved = baseline - actual.
* Absent when the probe failed (network, 4xx) — that event is then
* excluded from the savings rollup. */
* endpoint is free (no input-token billing). Absent when the probe
* failed (network, 4xx) — that event is then excluded from the
* savings rollup. */
baselineTokens?: number;
/** Second baseline probe: input_tokens of the original body TRUNCATED at
* the last `cache_control` marker — the prefix that would have cached
* on the unproxied path. Used by the dashboard to weight the baseline by
* the SAME cache class the proxied request landed in (cache_create ×1.25,
* cache_read ×0.10, no-cache ×1.0), giving an exact cache-aware
* counterfactual instead of cold-every-time. Absent when the original
* body has no cache_control markers anywhere (in which case the unproxied
* path doesn't cache and cacheable_prefix_tokens = 0). */
baselineCacheableTokens?: number;
}
// --- helpers ---------------------------------------------------------------
src/dashboard.ts
+90 -57
View File
@@ -82,29 +82,38 @@ export interface RecentRow {
/** Aggregate over the whole session. Reset on process restart unless
* replay() is called to seed from the JSONL file.
*
* Both sides of the savings math are INPUT-ONLY:
* baseline_input = count_tokens(originalBody).input_tokens (free probe)
* actual_input = input + cache_create×1.25 + cache_read×0.10 (upstream usage)
* saved = baseline_input − actual_input
* Output is deliberately excluded — it's identical with or without
* compression (same prompt → same response) so it cancels in the
* numerator and would only drag saved_pct toward zero by inflating
* the denominator. Headline reads "% of input bill saved", not "% of
* total bill saved", and that's the honest number. */
* Both sides of the savings math are INPUT-ONLY, and the baseline is
* CACHE-AWARE — it weights the unproxied path's tokens by the SAME cache
* class the proxied request actually landed in, so the comparison is
* apples-to-apples (caching vs caching, not caching vs no-caching).
*
* Per event:
* cacheable = baseline_cacheable_tokens || 0 (tokens up to last cache_control)
* cold_tail = baseline_tokens − cacheable (always-cold input on both paths)
* weight = cr > 0 ? 0.10 (warm hit — both paths would have cached)
* : cc > 0 ? 1.25 (cold create — both paths would create cache)
* : 1.0 (no caching — marker rejected or absent)
* baseline_input_eff = cacheable × weight + cold_tail × 1.0
* actual_input_eff = input + cache_create×1.25 + cache_read×0.10
* saved = baseline_input_eff − actual_input_eff
*
* Roll-up:
* saved_pct = Σ saved / Σ baseline_input_eff × 100
*
* Output is deliberately excluded — identical with/without compression,
* same prompt → same response, so it cancels and would only drag the
* percentage toward zero. Headline reads "% of input bill saved." */
interface Totals {
requests: number;
compressedRequests: number;
/** Sum of weighted actual input tokens we paid for, across all events that
* also carried a baseline measurement (input + cache_create×1.25 +
* cache_read×0.10). Output is excluded — output cost is identical whether
* the prompt was imaged or not, so it cancels in any saved_pct that means
* "fraction of INPUT spend the proxy shaved off". */
* also carried a baseline_tokens measurement (input + cache_create×1.25 +
* cache_read×0.10). */
actualInputWeighted: number;
/** Sum of `baseline_tokens` (count_tokens result on the ORIGINAL body) for
* the same events that contributed to `actualInputWeighted`. Treated as
* cold-input cost — the counter-factual we'd have paid if we sent the
* uncompressed body. Pairs 1:1 with `actualInputWeighted`. */
baselineInputTokens: number;
/** Sum of the cache-aware baseline (see formula above) across the same
* events that contributed to `actualInputWeighted`. The honest counter-
* factual cost of the unproxied path. */
baselineInputWeighted: number;
startedAt: number;
}
@@ -146,7 +155,7 @@ export class DashboardState {
requests: 0,
compressedRequests: 0,
actualInputWeighted: 0,
baselineInputTokens: 0,
baselineInputWeighted: 0,
startedAt: Date.now() / 1000,
};
private latestPng: Uint8Array | null = null;
@@ -201,10 +210,9 @@ export class DashboardState {
/** Fold one event into the running totals + ring buffer.
*
* Savings math is gated on a per-request `baseline_tokens` measurement
* from the parallel count_tokens probe. When the probe is missing (e.g.
* upstream 4xx'd the side call, or the request was a non-/v1/messages
* passthrough), we still count the request but skip its savings
* contribution — no estimation, no α-regression, no inferred numbers. */
* from the parallel count_tokens probe AND an upstream usage block.
* When either is missing, we still count the request but skip its
* savings contribution — no estimation. */
update(ev: ProxyEvent): void {
// Stash the image bytes before they get GC'd by the request finishing.
if (ev.info) this.captureImage(ev.info);
@@ -221,20 +229,31 @@ export class DashboardState {
const baseline = info?.baselineTokens;
const haveBaseline = typeof baseline === 'number' && baseline > 0;
// Weighted INPUT cost we actually paid this turn. Output is identical
// with or without compression so it's deliberately not in the savings
// numerator/denominator — see Totals.actualInputWeighted comment.
// Weighted INPUT cost we actually paid this turn.
const actualInputEff = haveUsage ? inp + cc * 1.25 + cr * 0.1 : 0;
// Cache-aware baseline: decompose the unproxied counterfactual into
// (cacheable_prefix, cold_tail) using the second count_tokens probe,
// then weight the prefix by the SAME cache class the actual request
// landed in. No assumptions — every input is a measurement.
// cacheable = tokens up to last cache_control marker (or 0 if no markers)
// cold_tail = baseline − cacheable (always cold input)
// weight = cr > 0 ? 0.10 (warm hit — both paths cache)
// : cc > 0 ? 1.25 (cold create — both paths build the cache)
// : 1.0 (no caching — marker rejected or absent)
let baselineInputEff = 0;
if (haveBaseline && haveUsage) {
const cacheable = Math.min(info?.baselineCacheableTokens ?? 0, baseline);
const coldTail = baseline - cacheable;
const weight = cr > 0 ? 0.1 : cc > 0 ? 1.25 : 1.0;
baselineInputEff = cacheable * weight + coldTail * 1.0;
}
this.totals.requests += 1;
if (compressed) this.totals.compressedRequests += 1;
// Real savings rollup: ONLY events with both a count_tokens baseline
// and a billed usage block. Counter-factual: if we'd sent the original
// body cold, we'd have paid `baseline × 1.0` input rate; we actually
// paid `actualInputEff`. saved = baseline − actualInputEff.
if (haveBaseline && haveUsage) {
this.totals.baselineInputTokens += baseline;
this.totals.baselineInputWeighted += baselineInputEff;
this.totals.actualInputWeighted += actualInputEff;
}
@@ -249,9 +268,9 @@ export class DashboardState {
cache_create: haveUsage ? cc : undefined,
cache_read: haveUsage ? cr : undefined,
actual_input: haveUsage ? round1(actualInputEff) : undefined,
baseline_input: haveBaseline ? baseline : undefined,
baseline_input: haveBaseline && haveUsage ? round1(baselineInputEff) : undefined,
session_saved_so_far_delta:
haveBaseline && haveUsage ? round1(baseline - actualInputEff) : undefined,
haveBaseline && haveUsage ? round1(baselineInputEff - actualInputEff) : undefined,
};
this.recent.push(row);
if (this.recent.length > RECENT_CAP) this.recent.splice(0, this.recent.length - RECENT_CAP);
@@ -287,8 +306,17 @@ export class DashboardState {
const cr = t.cache_read_tokens ?? 0;
const haveUsage = inp > 0 || cc > 0 || cr > 0;
const baseline = (t as { baseline_tokens?: number }).baseline_tokens;
const cacheable = (t as { baseline_cacheable_tokens?: number })
.baseline_cacheable_tokens ?? 0;
const haveBaseline = typeof baseline === 'number' && baseline > 0;
const actualInputEff = haveUsage ? inp + cc * 1.25 + cr * 0.1 : 0;
let baselineInputEff = 0;
if (haveBaseline && haveUsage) {
const c = Math.min(cacheable, baseline as number);
const coldTail = (baseline as number) - c;
const weight = cr > 0 ? 0.1 : cc > 0 ? 1.25 : 1.0;
baselineInputEff = c * weight + coldTail * 1.0;
}
const row: RecentRow = {
ts: Date.parse(t.ts) / 1000,
method: t.method,
@@ -300,7 +328,8 @@ export class DashboardState {
cache_create: t.cache_create_tokens,
cache_read: t.cache_read_tokens,
actual_input: haveUsage ? round1(actualInputEff) : undefined,
baseline_input: haveBaseline ? baseline : undefined,
baseline_input:
haveBaseline && haveUsage ? round1(baselineInputEff) : undefined,
};
this.recent.push(row);
}
@@ -309,16 +338,16 @@ export class DashboardState {
// ---- HTTP handlers ------------------------------------------------------
serveStats(): Response {
// Real saved% from REAL numbers:
// baseline = sum of count_tokens(originalBody) across events with a probe
// actual = sum of (input + cache_create*1.25 + cache_read*0.10) over the same events
// saved = baseline - actual
// Output is excluded from both sides — it doesn't change with compression
// and would just inflate the denominator. Events without a baseline probe
// (count_tokens 4xx'd, or pre-feature replay) contribute to `requests`
// but not to the savings rollup; the operator sees a tighter `measured`
// count surface what's actually behind the headline.
const baseline = this.totals.baselineInputTokens;
// Real cache-aware saved%:
// per event: baseline_input_eff = cacheable × weight + cold_tail × 1.0
// where weight matches the actual response's cache class
// (cr>0 → 0.10, cc>0 → 1.25, neither → 1.0)
// actual_input_eff = input + cache_create×1.25 + cache_read×0.10
// saved_pct = Σ saved / Σ baseline_input_eff × 100
// Both sides input-only; output excluded because it's identical with
// or without compression. Events missing either probe stay out of the
// rollup — operator sees the requests counter but not phantom savings.
const baseline = this.totals.baselineInputWeighted;
const actual = this.totals.actualInputWeighted;
const saved = baseline - actual;
const pct = baseline > 0 ? (saved / baseline) * 100 : 0;
@@ -326,7 +355,7 @@ export class DashboardState {
const payload = {
requests: this.totals.requests,
compressed_requests: this.totals.compressedRequests,
baseline_input_tokens: Math.round(baseline),
baseline_input_weighted: Math.round(baseline),
actual_input_weighted: Math.round(actual),
saved_input_tokens: Math.round(saved),
saved_pct: round1(pct),
@@ -790,7 +819,7 @@ async function tick() {
// No estimation, no α, no range — just two real numbers subtracted.
document.getElementById('m_saved').textContent = numFmt(s.saved_input_tokens);
document.getElementById('m_saved_sub').textContent =
\`\${numFmt(s.actual_input_weighted)} paid · \${numFmt(s.baseline_input_tokens)} baseline\`;
\`\${numFmt(s.actual_input_weighted)} paid · \${numFmt(s.baseline_input_weighted)} baseline\`;
// $ saved card: saved input tokens × input rate. Single number, no range.
const inRate = s.pricing_assumptions && s.pricing_assumptions.input_per_mtok;
document.getElementById('m_usd').textContent = \`$\${(s.saved_usd || 0).toFixed(4)}\`;
@@ -864,14 +893,16 @@ function shortPath(p) {
// ---- savings math panels -------------------------------------------------
//
// Renders the "show calculation" blocks beneath each savings card. Every
// number on the headline cards is a straight subtraction or rate-multiply
// of two real measurements:
// baseline = /v1/messages/count_tokens(originalBody).input_tokens, summed
// actual = (input + cache_create·1.25 + cache_read·0.10), summed
// Output is excluded from both sides because it's identical with or
// without compression. No α, no fit, no estimation — operator can copy
// the values into a calculator and reproduce the headline byte-identically.
// Renders the "show calculation" blocks beneath each savings card. The
// baseline is CACHE-AWARE — per event we measure:
// baseline_input_eff = cacheable × weight + cold_tail × 1.0
// cacheable = count_tokens(originalBody truncated at last cache_control)
// cold_tail = count_tokens(originalBody) − cacheable
// weight = matches the actual response's cache class
// (cr>0 → 0.10, cc>0 → 1.25, neither → 1.0)
// actual_input_eff = input + cache_create×1.25 + cache_read×0.10
// Both probes are free. Output is excluded — identical with or without
// compression. No α, no fit, no estimation.
function renderSavingsMath(s) {
const pa = s.pricing_assumptions || {};
@@ -880,7 +911,7 @@ function renderSavingsMath(s) {
'<div><span class="k">formula:</span> <span class="v">saved = baseline − actual</span></div>'
+ '<div><span class="k">weights:</span> <span class="v">input ×1.0, cache_create ×1.25, cache_read ×0.10</span></div>'
+ '<div style="height:6px"></div>'
+ fmtRow('baseline', s.baseline_input_tokens, '(count_tokens on original body)')
+ fmtRow('baseline', s.baseline_input_weighted, '(cache-aware: cacheable·weight + cold_tail)')
+ fmtRow('actual', s.actual_input_weighted, '(input + cc·1.25 + cr·0.10 from usage)')
+ fmtRow('saved', s.saved_input_tokens, '<span class="op">=</span> baseline − actual')
+ '<span class="src">output excluded — identical with/without compression</span>';
@@ -891,7 +922,7 @@ function renderSavingsMath(s) {
'<div><span class="k">formula:</span> <span class="v">$ = saved × $'
+ inRate + '/Mtok</span></div>'
+ '<div style="height:6px"></div>'
+ fmtRow('saved_tokens', s.saved_input_tokens, '(input-side, weighted)')
+ fmtRow('saved_tokens', s.saved_input_tokens, '(cache-aware, input-side)')
+ fmtRow('input_rate', \`$\${inRate}/Mtok\`, '(Opus 4.7 base input)')
+ fmtRow('saved_usd',
\`$\${(s.saved_usd || 0).toFixed(4)}\`,
@@ -902,8 +933,10 @@ function renderSavingsMath(s) {
document.getElementById('m_pct_math').innerHTML =
'<div><span class="k">formula:</span> <span class="v">'
+ 'saved_pct = (baseline − actual) / baseline × 100</span></div>'
+ '<div><span class="k">per event:</span> <span class="v">'
+ 'baseline = cacheable·weight + cold_tail, weight matches actual cache class</span></div>'
+ '<div style="height:6px"></div>'
+ fmtRow('baseline', s.baseline_input_tokens, '(count_tokens on original body)')
+ fmtRow('baseline', s.baseline_input_weighted, '(cache-aware counterfactual)')
+ fmtRow('actual', s.actual_input_weighted, '(weighted upstream usage)')
+ fmtRow('saved', s.saved_input_tokens, '<span class="op">=</span> baseline − actual')
+ fmtRow('saved_pct', (s.saved_pct || 0).toFixed(1) + '%',
src/sessions.ts
+16 -9
View File
@@ -161,11 +161,13 @@ export async function aggregateSessions(
// Cling to whichever cwd we saw first; sessions that hop directories are
// rare and the first cwd is the most stable identifier.
if (s.project === undefined && ev.cwd) s.project = ev.cwd;
// Real per-session savings: only count events that carry both a
// count_tokens baseline AND an upstream usage block. Same input-only
// weighting as the headline cards (input + cc·1.25 + cr·0.10).
// Events missing either side contribute to requestCount but not the
// savings rollup — no estimation, no synthetic 4-char/tok shortcut.
// Real per-session savings, cache-aware (mirrors dashboard.update()):
// cacheable = baseline_cacheable_tokens or 0
// cold_tail = baseline_tokens − cacheable
// weight = cr>0 ? 0.10 : cc>0 ? 1.25 : 1.0
// baseline_eff = cacheable·weight + cold_tail
// actual_eff = input + cc·1.25 + cr·0.10
// Events missing either probe stay out of the rollup — no estimation.
const inp = ev.input_tokens ?? 0;
const cc = ev.cache_create_tokens ?? 0;
const cr = ev.cache_read_tokens ?? 0;
@@ -176,11 +178,16 @@ export async function aggregateSessions(
baseline > 0 &&
haveUsage
) {
const actualInputEff = inp + cc * 1.25 + cr * 0.1;
const tokensSaved = baseline - actualInputEff;
const cacheable = Math.min(
ev.baseline_cacheable_tokens ?? 0,
baseline,
);
const coldTail = baseline - cacheable;
const weight = cr > 0 ? 0.1 : cc > 0 ? 1.25 : 1.0;
const baselineEff = cacheable * weight + coldTail * 1.0;
const actualEff = inp + cc * 1.25 + cr * 0.1;
const tokensSaved = baselineEff - actualEff;
s.tokensSavedEst += Math.round(tokensSaved);
// charsSaved remains a coarse byte-equivalent; useful as a rough
// "we shaved X kB off the wire" callout but not load-bearing math.
s.charsSaved += Math.round(tokensSaved * 4);
}
if (typeof ev.cache_read_tokens === 'number') {
tests/proxy-usage.test.ts
+165
View File
@@ -423,10 +423,175 @@ describe('proxy usage extraction', () => {
await new Promise((r) => setTimeout(r, 20));
restore();
// No cache_control markers in SAMPLE_REQ_BODY → second probe is skipped.
// count_tokens hit exactly once, no other side paths.
expect(sidePaths).toEqual(['/v1/messages/count_tokens']);
});
// When the request body carries any `cache_control` marker, the proxy
// fires a SECOND count_tokens probe on the body truncated at the last
// marker. The difference between the two probe results is the
// cacheable-prefix vs cold-tail split that lets the dashboard compute
// a cache-aware baseline instead of a cold-every-time approximation.
it('fires a SECOND count_tokens probe when body has cache_control markers', async () => {
const bodiesSeen: string[] = [];
const restore = mockUpstream(async (req) => {
const url = new URL(req.url);
if (url.pathname === '/v1/messages/count_tokens') {
bodiesSeen.push(await req.text());
// Full body returns N; truncated returns M < N. The proxy doesn't
// care which response goes to which probe — it just attaches both.
const len = bodiesSeen.length;
return new Response(
JSON.stringify({ input_tokens: len === 1 ? 9000 : 6000 }),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
}
return new Response(
JSON.stringify({
id: 'msg_x',
type: 'message',
role: 'assistant',
content: [{ type: 'text', text: 'ok' }],
model: 'claude-opus-4-5',
stop_reason: 'end_turn',
usage: {
input_tokens: 1,
output_tokens: 1,
cache_creation_input_tokens: 5000,
cache_read_input_tokens: 0,
},
}),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
});
// Realistic shape: a long system prompt cached, then the user turn left
// uncacheable. Marker lives on the LAST system block — that's the
// canonical "cache everything above this line" layout Claude Code uses.
const bodyWithMarkers = JSON.stringify({
model: 'claude-3-5-haiku-latest',
system: [
{ type: 'text', text: 'You are helpful.' },
{
type: 'text',
text: 'A long preamble...',
cache_control: { type: 'ephemeral' },
},
],
messages: [{ role: 'user', content: 'hi' }],
});
let captured: ProxyEvent | undefined;
const proxy = createProxy({
upstream: 'http://mock',
onRequest: (e) => { captured = e; },
});
const res = await proxy(
new Request('http://proxy/v1/messages', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: bodyWithMarkers,
}),
);
await res.text();
await new Promise((r) => setTimeout(r, 30));
restore();
// Two probes fired (parallel, order indeterminate). At least one of the
// posted bodies must DIFFER from the other — the second probe is the
// truncated prefix, not a duplicate of the first.
expect(bodiesSeen).toHaveLength(2);
expect(bodiesSeen[0]).not.toBe(bodiesSeen[1]);
// Both numbers landed on info; baselineCacheableTokens is the smaller one
// (truncated body has fewer tokens than the full body). Whichever probe
// got the 9000 response is `baselineTokens`; the 6000 is `baselineCacheableTokens`.
expect(captured!.info?.baselineTokens).toBeDefined();
expect(captured!.info?.baselineCacheableTokens).toBeDefined();
const full = captured!.info!.baselineTokens!;
const cacheable = captured!.info!.baselineCacheableTokens!;
expect(new Set([full, cacheable])).toEqual(new Set([9000, 6000]));
});
// The two probes are independent. If the cacheable-prefix probe 4xx's
// (e.g. upstream rejects the synthesized sentinel message), the main
// forward succeeds and the FULL probe's baseline still lands. The
// dashboard's per-event math degrades cleanly to cold_tail = baseline.
it('survives cacheable-prefix probe failure without losing the full-body baseline', async () => {
let probeCount = 0;
const restore = mockUpstream((req) => {
const url = new URL(req.url);
if (url.pathname === '/v1/messages/count_tokens') {
probeCount += 1;
// First probe (full body) succeeds; second (truncated) fails.
// The proxy fires them in parallel so order matters — assume the
// longer body arrives first because it's queued first.
if (probeCount === 1) {
return new Response(JSON.stringify({ input_tokens: 7777 }), {
status: 200,
headers: { 'content-type': 'application/json' },
});
}
return new Response(JSON.stringify({ error: 'bad' }), {
status: 400,
headers: { 'content-type': 'application/json' },
});
}
return new Response(
JSON.stringify({
id: 'msg_x',
type: 'message',
role: 'assistant',
content: [{ type: 'text', text: 'ok' }],
model: 'claude-opus-4-5',
stop_reason: 'end_turn',
usage: { input_tokens: 1, output_tokens: 1 },
}),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
});
const bodyWithMarkers = JSON.stringify({
model: 'claude-3-5-haiku-latest',
system: [
{
type: 'text',
text: 'preamble',
cache_control: { type: 'ephemeral' },
},
],
messages: [{ role: 'user', content: 'hi' }],
});
let captured: ProxyEvent | undefined;
const proxy = createProxy({
upstream: 'http://mock',
onRequest: (e) => { captured = e; },
});
const res = await proxy(
new Request('http://proxy/v1/messages', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: bodyWithMarkers,
}),
);
expect(res.status).toBe(200);
await res.text();
await new Promise((r) => setTimeout(r, 30));
restore();
expect(probeCount).toBe(2);
// Whichever probe got the success response must have landed. Both
// succeed → both land. One fails → only the other lands. The contract
// we care about: ONE failure doesn't poison the OTHER.
// Probe order is parallel + non-deterministic in mock-fetch land,
// so just assert that at least one of the two baseline fields is set.
const haveFull = captured!.info?.baselineTokens !== undefined;
const haveCacheable = captured!.info?.baselineCacheableTokens !== undefined;
expect(haveFull || haveCacheable).toBe(true);
});
// baselineTokens from the count_tokens probe must land on info so the
// dashboard can roll it into the saved% denominator. This is the wiring
// that makes the headline number real instead of estimated.