fix(dashboard): headline = share of ALL paid spend, not just measured rows

The 'share of total bill saved' headline divided savings by
baseline_input_weighted, which only accumulated on rows where the
count_tokens probe succeeded AND the request had upstream usage. Every
passthrough turn, every probe-failed turn, and every flap-polluted
untransformed turn was excluded from the denominator. That's a
cherry-pick: it answers 'did pixelpipe help on the rows where it ran'
instead of 'did pixelpipe move my real bill'.

Track allActualInputWeighted / allOutputWeighted ungated on probe
success (any request with a usage block contributes), expose
saved_pct_of_all_spend on /proxy-stats, and rewrite the headline card
to use it. Numerator is still measured-rows-only because that's the
only place we have a counterfactual baseline; you can't fabricate
savings for rows where the probe failed. But the denominator now
matches the user's actual bill.

The legacy saved_pct_of_total_bill stays on the wire for back-compat
(marked deprecated in types.ts). The headline card no longer reads it.

Card flips color: positive % stays green, negative % goes red, because
flap-driven net losses are real and should be visible.
This commit is contained in:
Steven Chong
2026-05-23 13:14:40 -04:00
parent d0c64d932b
commit 62bcd4a0dc
4 changed files with 100 additions and 38 deletions
+16 -16
View File
File diff suppressed because one or more lines are too long
+50
View File
@@ -159,6 +159,23 @@ interface Totals {
* doesn't touch output). Without this the headline ignores half the
* bill on output-heavy sessions. */
outputWeighted: number;
/** Sum of weighted actual input tokens across ALL requests that had a
* usage block, regardless of whether the probe succeeded i.e. every
* request the user actually paid for, measured or not, transformed or
* passthrough. Forms the honest denominator for "share of total spend
* saved": you can only credit savings against the slice where pixelpipe
* ran AND we measured, but you have to divide by the full bill (passthrough
* + probe-failed + unmeasured + measured) to answer "did pixelpipe move
* my real bill" instead of "did pixelpipe help on rows where it ran". */
allActualInputWeighted: number;
/** Sum of output_tokens × OUTPUT_TOKEN_RATE across the same all-rows set
* as allActualInputWeighted. Output is on both sides of the savings ratio
* at its actual 5× rate; numerator stays measured-rows-only because saved
* cancels output (proxy doesn't touch it). */
allOutputWeighted: number;
/** Count of requests that contributed to allActualInputWeighted (had a
* usage block). Lets the UI annotate "N of M paid requests". */
allUsageRequests: number;
/** Sum of ground-truth output character counts from the SSE/JSON scanner
* (see `OutputMeasurement` in proxy.ts). These three accumulators are
* independent of Anthropic's `usage.output_tokens` they let the operator
@@ -226,6 +243,9 @@ export class DashboardState {
actualInputWeighted: 0,
baselineInputWeighted: 0,
outputWeighted: 0,
allActualInputWeighted: 0,
allOutputWeighted: 0,
allUsageRequests: 0,
textCharsMeasured: 0,
thinkingCharsMeasured: 0,
toolUseCharsMeasured: 0,
@@ -377,6 +397,15 @@ export class DashboardState {
this.totals.actualInputWeighted += actualInputEff;
this.totals.outputWeighted += outputEquiv;
}
// All-rows spend, ungated on the probe. The "share of total bill saved"
// headline divides measured-rows savings into this, so passthrough rows,
// probe-failed rows, and uncompressed rows all dilute the headline the
// same way they dilute the user's real bill.
if (haveUsage) {
this.totals.allActualInputWeighted += actualInputEff;
this.totals.allOutputWeighted += outputEquiv;
this.totals.allUsageRequests += 1;
}
// Measurement totals are independent of usage/baseline gating — they
// accumulate whenever the scanner produced numbers. The scanner sets
@@ -505,6 +534,18 @@ export class DashboardState {
const baselineTotal = baseline + output;
const actualTotal = actual + output;
const pctTotal = baselineTotal > 0 ? (saved / baselineTotal) * 100 : 0;
// Share-of-all-spend: honest denominator. The numerator can only credit
// savings against rows where we have a probe baseline (otherwise it's
// estimation), but the denominator MUST include every request the user
// actually paid for — including passthrough rows, probe-failed rows,
// and untransformed turns the gate said no to. Otherwise the headline
// answers "did pixelpipe help on the rows where it ran" instead of
// "did pixelpipe move my real bill". The first is a cherry-pick.
const allActual = this.totals.allActualInputWeighted;
const allOutput = this.totals.allOutputWeighted;
const allSpend = allActual + allOutput;
const pctAllSpend = allSpend > 0 ? (saved / allSpend) * 100 : 0;
const uptimeSec = Date.now() / 1000 - this.totals.startedAt;
const payload = {
requests: this.totals.requests,
@@ -517,6 +558,15 @@ export class DashboardState {
saved_pct: round1(pctInput),
saved_pct_input_only: round1(pctInput),
saved_pct_of_total_bill: round1(pctTotal),
// Honest "share of total bill saved" — measured-rows numerator over
// ALL paid requests in the denominator (compressed + passthrough +
// probe-failed). This is the number users actually want when they
// ask "is pixelpipe helping". Negative when flap-pollution from
// passthrough turns exceeds the collapse win on measured turns.
saved_pct_of_all_spend: round1(pctAllSpend),
all_actual_input_weighted: Math.round(allActual),
all_output_weighted: Math.round(allOutput),
all_usage_requests: this.totals.allUsageRequests,
saved_usd: round4((saved * ASSUMED_INPUT_USD_PER_MTOK) / 1e6),
output_weighted: Math.round(output),
baseline_token_equivalent: Math.round(baselineTotal),
+26 -22
View File
@@ -53,35 +53,37 @@
$: pctMath = s && pa
? `<div><span class="k">formula:</span> <span class="v">` +
` share_of_bill = saved / (baseline_input + output × ` +
` share_of_spend = saved / (all_actual_input + all_output × ` +
(pa.output_multiplier ?? 5) +
`)</span></div>` +
`<div><span class="k">why include output:</span> <span class="v">` +
`Anthropic's weekly meter counts input + output × 5, the proxy only moves input</span></div>` +
`<div><span class="k">why this denominator:</span> <span class="v">` +
`the question is "did pixelpipe move my real bill", not "did pixelpipe help on the slice it ran on". ` +
`The denominator is every paid request — compressed, passthrough, probe-failed — so passthrough rows ` +
`and flap-polluted turns count against the proxy the same way they count against the user.</span></div>` +
`<div style="height:6px"></div>` +
row('saved', s.saved_input_tokens, '(input savings - proxy doesn\'t touch output)') +
row('baseline_input', s.baseline_input_weighted, '(cache-aware counterfactual)') +
row('saved', s.saved_input_tokens, '(measured-rows numerator; cache-aware)') +
row('all_actual_input', s.all_actual_input_weighted, '(every paid request, weighted)') +
row(
'output',
` × ` + (pa.output_multiplier ?? 5),
s.output_weighted + ' (weighted output tokens)',
'all_output × ' + (pa.output_multiplier ?? 5),
s.all_output_weighted,
'(every paid request, output × ' + (pa.output_multiplier ?? 5) + ')',
) +
row(
'baseline_total',
s.baseline_input_weighted + s.output_weighted,
`<span class="op">=</span> baseline_input + output`,
'all_spend_total',
s.all_actual_input_weighted + s.all_output_weighted,
`<span class="op">=</span> all_actual_input + all_output`,
) +
row(
'share_of_bill',
(s.saved_pct_of_total_bill || 0).toFixed(1) + '%',
`<span class="op">=</span> saved / baseline_total × 100`,
'share_of_spend',
(s.saved_pct_of_all_spend || 0).toFixed(1) + '%',
`<span class="op">=</span> saved / all_spend_total × 100`,
) +
row(
'input-only %',
(s.saved_pct_input_only || 0).toFixed(1) + '%',
'(sub-line: saved / baseline_input × 100 - output excluded)',
'all_usage_requests',
s.all_usage_requests,
'(denominator request count - compressed + passthrough + probe-failed)',
) +
`<span class="src">measured - no estimation</span>`
`<span class="src">measured numerator, all-rows denominator - no cherry-pick</span>`
: '';
$: tokeqMath = s && pa
@@ -171,11 +173,10 @@
{/if}
</div>
<div class="card">
<div class="label">share of total bill saved</div>
<div class="value pos">{(s?.saved_pct_of_total_bill ?? 0).toFixed(1)}%</div>
<div class="label">share of spend saved</div>
<div class="value" class:pos={(s?.saved_pct_of_all_spend ?? 0) >= 0} class:neg={(s?.saved_pct_of_all_spend ?? 0) < 0}>{(s?.saved_pct_of_all_spend ?? 0).toFixed(1)}%</div>
<div class="small">
÷ (input + 5×output) - output billed at 5× input rate · input-only:
<span class="pos">{(s?.saved_pct_input_only ?? 0).toFixed(1)}%</span>
÷ all paid requests ({numFmt(s?.all_usage_requests)} req · compressed + passthrough + probe-failed) · negative = pixelpipe added cost
</div>
{#if s && pa}
<details class="math">
@@ -236,6 +237,9 @@
.value.pos {
color: #3fb950;
}
.value.neg {
color: #f85149;
}
.small {
font-size: 11px;
color: #6e7681;
+8
View File
@@ -15,7 +15,15 @@ export interface StatsPayload {
/** Back-compat duplicate of `saved_pct_input_only`. */
saved_pct: number;
saved_pct_input_only: number;
/** DEPRECATED denominator was filtered to measured-rows-only, which
* cherry-picks the wins. Kept on the wire for back-compat. */
saved_pct_of_total_bill: number;
/** Honest "share of total bill saved": measured-rows savings ÷ ALL paid
* requests in the window (compressed + passthrough + probe-failed). */
saved_pct_of_all_spend: number;
all_actual_input_weighted: number;
all_output_weighted: number;
all_usage_requests: number;
saved_usd: number;
output_weighted: number;
baseline_token_equivalent: number;