mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
fix(dashboard): headline = share of ALL paid spend, not just measured rows
The 'share of total bill saved' headline divided savings by baseline_input_weighted, which only accumulated on rows where the count_tokens probe succeeded AND the request had upstream usage. Every passthrough turn, every probe-failed turn, and every flap-polluted untransformed turn was excluded from the denominator. That's a cherry-pick: it answers 'did pixelpipe help on the rows where it ran' instead of 'did pixelpipe move my real bill'. Track allActualInputWeighted / allOutputWeighted ungated on probe success (any request with a usage block contributes), expose saved_pct_of_all_spend on /proxy-stats, and rewrite the headline card to use it. Numerator is still measured-rows-only because that's the only place we have a counterfactual baseline; you can't fabricate savings for rows where the probe failed. But the denominator now matches the user's actual bill. The legacy saved_pct_of_total_bill stays on the wire for back-compat (marked deprecated in types.ts). The headline card no longer reads it. Card flips color: positive % stays green, negative % goes red, because flap-driven net losses are real and should be visible.
This commit is contained in:
+16
-16
File diff suppressed because one or more lines are too long
@@ -159,6 +159,23 @@ interface Totals {
|
||||
* doesn't touch output). Without this the headline ignores half the
|
||||
* bill on output-heavy sessions. */
|
||||
outputWeighted: number;
|
||||
/** Sum of weighted actual input tokens across ALL requests that had a
|
||||
* usage block, regardless of whether the probe succeeded — i.e. every
|
||||
* request the user actually paid for, measured or not, transformed or
|
||||
* passthrough. Forms the honest denominator for "share of total spend
|
||||
* saved": you can only credit savings against the slice where pixelpipe
|
||||
* ran AND we measured, but you have to divide by the full bill (passthrough
|
||||
* + probe-failed + unmeasured + measured) to answer "did pixelpipe move
|
||||
* my real bill" instead of "did pixelpipe help on rows where it ran". */
|
||||
allActualInputWeighted: number;
|
||||
/** Sum of output_tokens × OUTPUT_TOKEN_RATE across the same all-rows set
|
||||
* as allActualInputWeighted. Output is on both sides of the savings ratio
|
||||
* at its actual 5× rate; numerator stays measured-rows-only because saved
|
||||
* cancels output (proxy doesn't touch it). */
|
||||
allOutputWeighted: number;
|
||||
/** Count of requests that contributed to allActualInputWeighted (had a
|
||||
* usage block). Lets the UI annotate "N of M paid requests". */
|
||||
allUsageRequests: number;
|
||||
/** Sum of ground-truth output character counts from the SSE/JSON scanner
|
||||
* (see `OutputMeasurement` in proxy.ts). These three accumulators are
|
||||
* independent of Anthropic's `usage.output_tokens` — they let the operator
|
||||
@@ -226,6 +243,9 @@ export class DashboardState {
|
||||
actualInputWeighted: 0,
|
||||
baselineInputWeighted: 0,
|
||||
outputWeighted: 0,
|
||||
allActualInputWeighted: 0,
|
||||
allOutputWeighted: 0,
|
||||
allUsageRequests: 0,
|
||||
textCharsMeasured: 0,
|
||||
thinkingCharsMeasured: 0,
|
||||
toolUseCharsMeasured: 0,
|
||||
@@ -377,6 +397,15 @@ export class DashboardState {
|
||||
this.totals.actualInputWeighted += actualInputEff;
|
||||
this.totals.outputWeighted += outputEquiv;
|
||||
}
|
||||
// All-rows spend, ungated on the probe. The "share of total bill saved"
|
||||
// headline divides measured-rows savings into this, so passthrough rows,
|
||||
// probe-failed rows, and uncompressed rows all dilute the headline the
|
||||
// same way they dilute the user's real bill.
|
||||
if (haveUsage) {
|
||||
this.totals.allActualInputWeighted += actualInputEff;
|
||||
this.totals.allOutputWeighted += outputEquiv;
|
||||
this.totals.allUsageRequests += 1;
|
||||
}
|
||||
|
||||
// Measurement totals are independent of usage/baseline gating — they
|
||||
// accumulate whenever the scanner produced numbers. The scanner sets
|
||||
@@ -505,6 +534,18 @@ export class DashboardState {
|
||||
const baselineTotal = baseline + output;
|
||||
const actualTotal = actual + output;
|
||||
const pctTotal = baselineTotal > 0 ? (saved / baselineTotal) * 100 : 0;
|
||||
|
||||
// Share-of-all-spend: honest denominator. The numerator can only credit
|
||||
// savings against rows where we have a probe baseline (otherwise it's
|
||||
// estimation), but the denominator MUST include every request the user
|
||||
// actually paid for — including passthrough rows, probe-failed rows,
|
||||
// and untransformed turns the gate said no to. Otherwise the headline
|
||||
// answers "did pixelpipe help on the rows where it ran" instead of
|
||||
// "did pixelpipe move my real bill". The first is a cherry-pick.
|
||||
const allActual = this.totals.allActualInputWeighted;
|
||||
const allOutput = this.totals.allOutputWeighted;
|
||||
const allSpend = allActual + allOutput;
|
||||
const pctAllSpend = allSpend > 0 ? (saved / allSpend) * 100 : 0;
|
||||
const uptimeSec = Date.now() / 1000 - this.totals.startedAt;
|
||||
const payload = {
|
||||
requests: this.totals.requests,
|
||||
@@ -517,6 +558,15 @@ export class DashboardState {
|
||||
saved_pct: round1(pctInput),
|
||||
saved_pct_input_only: round1(pctInput),
|
||||
saved_pct_of_total_bill: round1(pctTotal),
|
||||
// Honest "share of total bill saved" — measured-rows numerator over
|
||||
// ALL paid requests in the denominator (compressed + passthrough +
|
||||
// probe-failed). This is the number users actually want when they
|
||||
// ask "is pixelpipe helping". Negative when flap-pollution from
|
||||
// passthrough turns exceeds the collapse win on measured turns.
|
||||
saved_pct_of_all_spend: round1(pctAllSpend),
|
||||
all_actual_input_weighted: Math.round(allActual),
|
||||
all_output_weighted: Math.round(allOutput),
|
||||
all_usage_requests: this.totals.allUsageRequests,
|
||||
saved_usd: round4((saved * ASSUMED_INPUT_USD_PER_MTOK) / 1e6),
|
||||
output_weighted: Math.round(output),
|
||||
baseline_token_equivalent: Math.round(baselineTotal),
|
||||
|
||||
@@ -53,35 +53,37 @@
|
||||
|
||||
$: pctMath = s && pa
|
||||
? `<div><span class="k">formula:</span> <span class="v">` +
|
||||
` share_of_bill = saved / (baseline_input + output × ` +
|
||||
` share_of_spend = saved / (all_actual_input + all_output × ` +
|
||||
(pa.output_multiplier ?? 5) +
|
||||
`)</span></div>` +
|
||||
`<div><span class="k">why include output:</span> <span class="v">` +
|
||||
`Anthropic's weekly meter counts input + output × 5, the proxy only moves input</span></div>` +
|
||||
`<div><span class="k">why this denominator:</span> <span class="v">` +
|
||||
`the question is "did pixelpipe move my real bill", not "did pixelpipe help on the slice it ran on". ` +
|
||||
`The denominator is every paid request — compressed, passthrough, probe-failed — so passthrough rows ` +
|
||||
`and flap-polluted turns count against the proxy the same way they count against the user.</span></div>` +
|
||||
`<div style="height:6px"></div>` +
|
||||
row('saved', s.saved_input_tokens, '(input savings - proxy doesn\'t touch output)') +
|
||||
row('baseline_input', s.baseline_input_weighted, '(cache-aware counterfactual)') +
|
||||
row('saved', s.saved_input_tokens, '(measured-rows numerator; cache-aware)') +
|
||||
row('all_actual_input', s.all_actual_input_weighted, '(every paid request, weighted)') +
|
||||
row(
|
||||
'output',
|
||||
` × ` + (pa.output_multiplier ?? 5),
|
||||
s.output_weighted + ' (weighted output tokens)',
|
||||
'all_output × ' + (pa.output_multiplier ?? 5),
|
||||
s.all_output_weighted,
|
||||
'(every paid request, output × ' + (pa.output_multiplier ?? 5) + ')',
|
||||
) +
|
||||
row(
|
||||
'baseline_total',
|
||||
s.baseline_input_weighted + s.output_weighted,
|
||||
`<span class="op">=</span> baseline_input + output`,
|
||||
'all_spend_total',
|
||||
s.all_actual_input_weighted + s.all_output_weighted,
|
||||
`<span class="op">=</span> all_actual_input + all_output`,
|
||||
) +
|
||||
row(
|
||||
'share_of_bill',
|
||||
(s.saved_pct_of_total_bill || 0).toFixed(1) + '%',
|
||||
`<span class="op">=</span> saved / baseline_total × 100`,
|
||||
'share_of_spend',
|
||||
(s.saved_pct_of_all_spend || 0).toFixed(1) + '%',
|
||||
`<span class="op">=</span> saved / all_spend_total × 100`,
|
||||
) +
|
||||
row(
|
||||
'input-only %',
|
||||
(s.saved_pct_input_only || 0).toFixed(1) + '%',
|
||||
'(sub-line: saved / baseline_input × 100 - output excluded)',
|
||||
'all_usage_requests',
|
||||
s.all_usage_requests,
|
||||
'(denominator request count - compressed + passthrough + probe-failed)',
|
||||
) +
|
||||
`<span class="src">measured - no estimation</span>`
|
||||
`<span class="src">measured numerator, all-rows denominator - no cherry-pick</span>`
|
||||
: '';
|
||||
|
||||
$: tokeqMath = s && pa
|
||||
@@ -171,11 +173,10 @@
|
||||
{/if}
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="label">share of total bill saved</div>
|
||||
<div class="value pos">{(s?.saved_pct_of_total_bill ?? 0).toFixed(1)}%</div>
|
||||
<div class="label">share of spend saved</div>
|
||||
<div class="value" class:pos={(s?.saved_pct_of_all_spend ?? 0) >= 0} class:neg={(s?.saved_pct_of_all_spend ?? 0) < 0}>{(s?.saved_pct_of_all_spend ?? 0).toFixed(1)}%</div>
|
||||
<div class="small">
|
||||
÷ (input + 5×output) - output billed at 5× input rate · input-only:
|
||||
<span class="pos">{(s?.saved_pct_input_only ?? 0).toFixed(1)}%</span>
|
||||
÷ all paid requests ({numFmt(s?.all_usage_requests)} req · compressed + passthrough + probe-failed) · negative = pixelpipe added cost
|
||||
</div>
|
||||
{#if s && pa}
|
||||
<details class="math">
|
||||
@@ -236,6 +237,9 @@
|
||||
.value.pos {
|
||||
color: #3fb950;
|
||||
}
|
||||
.value.neg {
|
||||
color: #f85149;
|
||||
}
|
||||
.small {
|
||||
font-size: 11px;
|
||||
color: #6e7681;
|
||||
|
||||
@@ -15,7 +15,15 @@ export interface StatsPayload {
|
||||
/** Back-compat duplicate of `saved_pct_input_only`. */
|
||||
saved_pct: number;
|
||||
saved_pct_input_only: number;
|
||||
/** DEPRECATED — denominator was filtered to measured-rows-only, which
|
||||
* cherry-picks the wins. Kept on the wire for back-compat. */
|
||||
saved_pct_of_total_bill: number;
|
||||
/** Honest "share of total bill saved": measured-rows savings ÷ ALL paid
|
||||
* requests in the window (compressed + passthrough + probe-failed). */
|
||||
saved_pct_of_all_spend: number;
|
||||
all_actual_input_weighted: number;
|
||||
all_output_weighted: number;
|
||||
all_usage_requests: number;
|
||||
saved_usd: number;
|
||||
output_weighted: number;
|
||||
baseline_token_equivalent: number;
|
||||
|
||||
Reference in New Issue
Block a user