fix(cache): pin prefix-cache anchor to the byte-stable history image

Intermittent prefix-cache busts on long sessions (#11): the single cache
breakpoint relocated onto the LAST history image — the still-growing chunk
whose bytes change every window advance — so the slab+history prefix
re-created at 1.25x instead of being read.

- relocateAnchorToHistoryImage pins collapseHistory's carry-over ordinal
  (the last byte-stable chunk) instead of the last image.
- demoteProtectedHeadText keeps pxpipe's slab scaffolding (images +
  fact-sheet + the [End of rendered context.] boundary) verbatim and only
  quarantines the user's stale opening turn, so the relocator can still
  find the slab anchor (#14 demotion had clobbered the boundary it keys on).
- cachePrefixDigest joins parts with \x00 so block-boundary differences
  can't hash-collide into false prefix-stability.

520/520 green, tsc clean.
This commit is contained in:
teamchong
2026-06-26 00:17:10 -04:00
parent 6ca72260e8
commit 6d1c46dd20
4 changed files with 226 additions and 8 deletions
+89 -1
View File
@@ -102,6 +102,11 @@ export interface HistoryCollapseInfo {
/** Per-image pixel dims, parallel to collapsedPngs. The dashboard ring reads
* info.imageDims in lockstep with info.imagePngs, so these must be pushed together. */
collapsedImageDims: { width: number; height: number }[];
/** Ordinal (0-based, into the emitted history images) of the last byte-stable
* history image — the carry-over cache anchor. The relocator pins the cache
* breakpoint here so it survives window advances (#11). Undefined when history is
* too short to have a fully grid-aligned chunk before collapseLen. */
carryOverImageOrdinal?: number;
/** Why we didn't collapse — populated only when no collapse happened. */
reason?:
| 'no_history'
@@ -292,6 +297,75 @@ function compactPreview(text: string): string {
return compact.slice(0, LATEST_COLLAPSED_USER_PREVIEW_CHARS).trimEnd() + '...';
}
/**
* Demote request TEXT in the protected head (slab anchor) to a marked PRIOR-CONTEXT
* tombstone. The session's OPENING user turn rides in the SAME message as the slab
* images (transform.ts sets protectedPrefix = firstUserIdx + 1 to keep that message
* from collapsing into [image] placeholders). Protecting it for the cache anchor also
* passed its request text through as clean native text at the very TOP — ahead of the
* synthetic history block — where the model reads it as the LIVE request. It never is:
* the live request is always in the tail (tail = messages.slice(collapseLen),
* keepTail >= 1), so any text in the protected head is, by construction, stale.
*
* Image/tool blocks (the slab) pass through byte-identical so the cache anchor and any
* cache_control breakpoint survive; the demotion is a pure function of the message, so
* the protected prefix stays byte-stable across turns (one-time re-cache on deploy).
*/
function demoteProtectedHeadText(head: Message[]): Message[] {
return head.map((m, idx) => {
if (m.role !== 'user') return m;
const tomb = (preview: string, cc?: CacheControl): TextBlock => {
const t: TextBlock = {
type: 'text',
text:
`[Opening turn <user t="${idx}"> of this session — PRIOR CONTEXT ONLY, ` +
`superseded by later turns; NOT the current request and must not be acted ` +
`on. Preview: "${preview}"]`,
};
if (cc !== undefined) {
(t as TextBlock & { cache_control?: CacheControl }).cache_control = cc;
}
return t;
};
if (typeof m.content === 'string') {
const preview = compactPreview(m.content);
return preview ? { ...m, content: [tomb(preview)] } : m;
}
if (!Array.isArray(m.content)) return m;
// pxpipe's own slab scaffolding (the rendered images, the fact-sheet, and the
// '[End of rendered context.]' boundary) is NOT the user's request and must
// survive byte-identical: relocateAnchorToHistoryImage keys on that boundary
// text to locate the slab cache anchor. Only the user's stale opening turn —
// the blocks AFTER the boundary — gets demoted. With no boundary (the slab did
// not image) boundaryIdx is -1 and the whole message demotes, exactly as before.
const boundaryIdx = m.content.findIndex(
(b) =>
b && typeof b === 'object' &&
(b as { type?: string }).type === 'text' &&
(b as TextBlock).text === '[End of rendered context.]',
);
let changed = false;
const out: ContentBlock[] = [];
for (let i = 0; i < m.content.length; i++) {
const blk = m.content[i]!;
if (boundaryIdx >= 0 && i <= boundaryIdx) {
out.push(blk); // slab images + fact-sheet + boundary: proxy scaffolding, kept verbatim
continue;
}
if (blk && typeof blk === 'object' && (blk as { type?: string }).type === 'text') {
const preview = compactPreview((blk as TextBlock).text);
if (preview) {
out.push(tomb(preview, (blk as { cache_control?: CacheControl }).cache_control));
changed = true;
continue;
}
}
out.push(blk); // images / tool blocks (slab anchor) pass through byte-identical
}
return changed ? { ...m, content: out } : m;
});
}
function latestCollapsedUserPointer(
messages: Message[],
upToExclusive: number,
@@ -410,6 +484,13 @@ export async function collapseHistory(
ends.add(collapseLen);
const sortedEnds = [...ends].filter((e) => e > protectedPrefix && e <= collapseLen).sort((a, b) => a - b);
// Carry-over anchor end: the largest FULLY grid-aligned chunk boundary strictly
// before collapseLen. That chunk's bytes are frozen across window advances, unlike
// the newest partial chunk — so it's the stable place to pin the cache breakpoint (#11).
let carryOverEnd = -1;
for (let e = protectedPrefix + step; e < collapseLen; e += step) carryOverEnd = e;
let carryOverOrdinal = -1;
const imageBlocks: Array<ImageBlock & { cache_control?: CacheControl }> = [];
let chunkStart = protectedPrefix;
for (const chunkEnd of sortedEnds) {
@@ -474,6 +555,10 @@ export async function collapseHistory(
info.droppedCodepoints.set(cp, (info.droppedCodepoints.get(cp) ?? 0) + n);
}
}
// The carry-over chunk's LAST image is the newest byte-stable history image.
// Record its ordinal so the relocator pins the cache breakpoint here instead of
// on the still-growing newest chunk, which busts every window advance (#11).
if (chunkEnd === carryOverEnd) carryOverOrdinal = imageBlocks.length - 1;
}
if (imageBlocks.length === 0) {
info.reason = 'render_empty';
@@ -492,11 +577,14 @@ export async function collapseHistory(
role: 'user',
content: syntheticContent,
};
const head = messages.slice(0, protectedPrefix);
// Demote stale request text in the protected head so the session's opening turn
// can't surface as clean native text ahead of the history image and read as live.
const head = demoteProtectedHeadText(messages.slice(0, protectedPrefix));
const tail = messages.slice(collapseLen);
info.collapsedTurns = collapseLen - protectedPrefix;
info.collapsedChars = text.length;
info.collapsedImages = imageBlocks.length;
if (carryOverOrdinal >= 0) info.carryOverImageOrdinal = carryOverOrdinal;
// [slab, history image, live tail] — slab cache_control anchor stays at the front.
return { messages: [...head, syntheticUser, ...tail], info };
}
+14 -7
View File
@@ -745,7 +745,7 @@ async function historyImageSha8(
* Pure relocation: it acts only when a slab image already carries the anchor, so
* the total marker count never increases (pxpipe never *adds* only moves).
*/
function relocateAnchorToHistoryImage(messages: Message[] | undefined): void {
function relocateAnchorToHistoryImage(messages: Message[] | undefined, anchorOrdinal?: number): void {
if (!Array.isArray(messages)) return;
// The synthetic history message is identified by its banner text block.
@@ -754,13 +754,20 @@ function relocateAnchorToHistoryImage(messages: Message[] | undefined): void {
if (!Array.isArray(m.content)) continue;
const first = m.content[0] as TextBlock | undefined;
if (!first || first.type !== 'text' || first.text !== HISTORY_SYNTHETIC_INTRO) continue;
for (let i = m.content.length - 1; i >= 0; i--) {
const b = m.content[i];
// Collect this message's images in order, then pin the carry-over anchor (the last
// byte-stable history image) when collapseHistory provided its ordinal; otherwise
// fall back to the last image. Pinning the LAST image is the #11 bust: it's the
// newest, still-growing chunk and its bytes change on every window advance.
const imgsInMsg: Array<ImageBlock & { cache_control?: unknown }> = [];
for (const b of m.content) {
if (b && (b as ImageBlock).type === 'image') {
historyImg = b as ImageBlock & { cache_control?: unknown };
break;
imgsInMsg.push(b as ImageBlock & { cache_control?: unknown });
}
}
historyImg =
anchorOrdinal !== undefined && anchorOrdinal >= 0 && anchorOrdinal < imgsInMsg.length
? imgsInMsg[anchorOrdinal]
: imgsInMsg[imgsInMsg.length - 1];
break;
}
if (!historyImg) return;
@@ -831,7 +838,7 @@ async function cachePrefixDigest(
else if (Array.isArray(content))
for (const b of content) parts.push(typeof b === 'string' ? b : JSON.stringify(b));
}
const prefix = parts.join('');
const prefix = parts.join('\x00');
return { sha8: await sha8(prefix), bytes: prefix.length };
}
@@ -1889,7 +1896,7 @@ export async function transformRequest(
// Move the single cache anchor onto the history image so slab + history
// cache as one stable prefix (created once, then read), instead of the
// history image re-creating whenever the caller's downstream marker moves.
relocateAnchorToHistoryImage(req.messages);
relocateAnchorToHistoryImage(req.messages, histInfo.carryOverImageOrdinal);
} else if (histInfo.reason) {
info.historyReason = histInfo.reason;
}
+26
View File
@@ -268,6 +268,32 @@ describe('e2e cache alignment — Anthropic /v1/messages through the real proxy'
expect(countCacheControlMarkers(new TextEncoder().encode(cap1.main[0]!.body))).toBe(0);
});
it('CARRY-OVER (#11): the marked history image is a frozen page, not the partial tail, across an advance', async () => {
// Slab present (so the anchor relocates onto a history image) + enough history
// to collapse AND advance the window. The marker must land on a byte-FROZEN page
// (the carry-over anchor), never the last partial page — else the cached prefix
// busts on every advance (#11). Before the fix the marker sat on the LAST image.
const cap1 = await driveAnthropic(anthropicBody({ slabChars: 80_000, turns: turns(80, 4000) }));
cap1.restore();
const cap2 = await driveAnthropic(anthropicBody({ slabChars: 80_000, turns: turns(200, 4000) }));
cap2.restore();
const imgs1 = anthropicImages(cap1.main[0]!.body);
const imgs2 = anthropicImages(cap2.main[0]!.body);
const markedIdx1 = imgs1.findIndex((i) => i.marked);
// Conserved: exactly one marker, and it sits on an image.
expect(imgs1.filter((i) => i.marked)).toHaveLength(1);
expect(markedIdx1).toBeGreaterThanOrEqual(0);
// The advance is real (the window grew), so the byte-frozen check isn't vacuous.
expect(imgs2.length).toBeGreaterThan(imgs1.length);
// (1) NOT the last (partial, still-growing) page — that placement is the #11 bust.
expect(markedIdx1).toBeLessThan(imgs1.length - 1);
// (2) The marked page is byte-frozen: it reappears identically after the advance,
// so Anthropic cache_reads the prefix instead of re-creating it.
expect(imgs2.some((i) => i.data === imgs1[markedIdx1]!.data)).toBe(true);
});
it('relocates the single marker onto the HISTORY image once history collapses', async () => {
// The trickiest cache move (relocateAnchorToHistoryImage), end-to-end: with a
// tiny tail the caller marker stays on the SLAB image; once history collapses
+97
View File
@@ -874,3 +874,100 @@ describe('isCompressionProfitableAmortized — multi-turn horizon gate', () => {
expect(cold(text, 100, undefined, 1, 1.5, NaN)).toBe(cold(text, 100, undefined, 1, 1.5, 0));
});
});
// ---------------------------------------------------------------------------
// Regression (task #14): the session's OPENING user turn must never resurface
// as the LIVE request after collapse.
//
// transform.ts sets protectedPrefix = firstUserIdx + 1 so the opening turn —
// which carries BOTH the slab image AND the user's first request text in one
// message — is protected from collapsing into [image] placeholders (we keep the
// slab as the byte-stable cache anchor). The trap: protecting it used to pass
// the opening REQUEST TEXT through as clean native text at the very TOP, ahead
// of the synthetic history image, where the model reads it as the live request
// and re-actions a long-superseded ask ("add a Sonnet button" — already shipped).
// The live request is ALWAYS the last user turn (tail = slice(collapseLen),
// keepTail >= 1). Guard: slab survives byte-identical, opening text is demoted
// to a PRIOR-CONTEXT tombstone, live tail preserved verbatim.
// ---------------------------------------------------------------------------
describe('collapseHistory — opening-turn request quarantine (regression #14)', () => {
const SLAB_DATA = 'U0xBQg=='; // base64("SLAB") — the recognition / cache anchor
const OPENING_REQUEST = 'can you update the ux to add a sonnet button';
const LIVE_REQUEST = 'LIVE: enforce live=last-user invariant and fail closed';
it('demotes the opening request to a tombstone, keeps the slab image, preserves the live tail', async () => {
const msgs: Message[] = [
// turn 0 — opening turn: request text + slab image in ONE message.
usr([
{ type: 'text', text: OPENING_REQUEST },
{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: SLAB_DATA } },
]),
];
// filler turns 1..12 — comfortably past break-even at cols=100.
for (let i = 1; i <= 12; i++) {
const body = `turn ${i}: ` + 'x'.repeat(2800);
msgs.push(i % 2 === 1 ? asst(body) : usr(body));
}
// final user turn — the actual live request.
msgs.push(usr(LIVE_REQUEST));
const { messages: out, info } = await collapseHistory(msgs, isCompressionProfitable, {
keepTail: 1,
minCollapsePrefix: 5,
cols: 100,
collapseChunk: 0,
protectedPrefix: 1, // protect the opening (slab) turn — transform.ts uses firstUserIdx + 1
});
// Collapse fired: [demoted head, synthetic history, live tail].
expect(info.reason).toBe(undefined);
expect(info.collapsedTurns).toBe(12);
expect(info.collapsedImages).toBeGreaterThanOrEqual(1);
expect(out.length).toBe(3);
// (1) Opening request TEXT is quarantined behind the PRIOR-CONTEXT tombstone —
// never a clean native text block that could read as the live request.
const head = out[0]!;
expect(head.role).toBe('user');
const headContent = head.content as Array<Record<string, unknown>>;
const headText = headContent.filter((c) => c.type === 'text') as Array<{ text: string }>;
expect(headText).toHaveLength(1);
expect(headText[0]!.text).toContain('PRIOR CONTEXT ONLY');
expect(headText[0]!.text).toContain('must not be acted');
expect(headText[0]!.text).toContain('<user t="0">');
expect(headText[0]!.text).toContain('Preview:'); // the ask survives only as a marked preview
// The bare request string never appears as a standalone clean text block anywhere.
const cleanOpeningSomewhere = out.some(
(m) =>
Array.isArray(m.content) &&
(m.content as Array<Record<string, unknown>>).some(
(b) => b.type === 'text' && b.text === OPENING_REQUEST,
),
);
expect(cleanOpeningSomewhere).toBe(false);
// (2) The slab image survives byte-identical (data unchanged) in the head.
const headImgs = headContent.filter((c) => c.type === 'image');
expect(headImgs).toHaveLength(1);
expect(headImgs[0]).toMatchObject({
type: 'image',
source: { type: 'base64', media_type: 'image/png', data: SLAB_DATA },
});
// (3) The live request is the LAST message, byte-identical to the input ref.
const live = out[out.length - 1]!;
expect(live).toBe(msgs[msgs.length - 1]);
expect(live.content).toBe(LIVE_REQUEST);
// (4) Synthetic history sits BETWEEN head and live; its recency pointer/outro
// points at the live text and never resurrects the opening request.
const synth = out[1]!;
const synthText = (synth.content as Array<Record<string, unknown>>).filter(
(c) => c.type === 'text',
) as Array<{ text: string }>;
expect(synthText.some((t) => t.text.includes('current request is the live text'))).toBe(true);
for (const t of synthText) {
expect(t.text).not.toContain(OPENING_REQUEST);
}
});
});