fix(transform): relocate cache anchor onto history image (stop 1.25x re-create)

The history image sits after the slab in prefix order, but the single cache
breakpoint was anchored on the slab. So the largest block — the ~141k-token
history image — only cached when the caller's roaming downstream marker happened
to land after it. When it didn't, the whole history image re-created at the 1.25x
cache_create rate every turn instead of amortizing into 0.1x reads. On a warm
session that turned a real compression win into a net loss (the "411% more"
the dashboard correctly surfaced).

Fix is pure relocation: move pxpipe's single breakpoint from the slab onto the
LAST history image (which sits after the slab), so slab + history cache as one
stable prefix — created once, then read. Marker count is unchanged: pxpipe still
never ADDS a marker (Task #21), it only moves the caller's existing one. When no
slab imaged, the history path keeps its own relocation.

- transform.ts: relocateAnchorToHistoryImage on the collapse-success branch
- history.test.ts: two old "history carries no marker" invariants upgraded to
  assert the relocation (last history image carries it; exactly one marker total)
This commit is contained in:
teamchong
2026-06-19 16:39:46 -04:00
parent b20c976f61
commit 8b15525cb9
2 changed files with 110 additions and 14 deletions
+62
View File
@@ -693,6 +693,64 @@ async function historyImageSha8(
return concat ? sha8(concat) : undefined;
}
/**
* After a history collapse, move pxpipe's single relocated cache breakpoint off
* the slab image and onto the LAST history image.
*
* The history image sits AFTER the slab in prefix order, so one marker on it
* caches the WHOLE imaged prefix (slab + history) as a single stable segment —
* created once, then read at the ~0.1x rate every turn. Without this the history
* image (usually the largest block) only lands in a cached prefix when the
* caller's roaming downstream marker happens to fall after it; when it doesn't,
* the entire history image re-creates at the 1.25x rate turn after turn.
*
* Pure relocation: it acts only when a slab image already carries the anchor, so
* the total marker count never increases (pxpipe never *adds* — only moves).
*/
function relocateAnchorToHistoryImage(messages: Message[] | undefined): void {
if (!Array.isArray(messages)) return;
// The synthetic history message is identified by its banner text block.
let historyImg: (ImageBlock & { cache_control?: unknown }) | undefined;
for (const m of messages) {
if (!Array.isArray(m.content)) continue;
const first = m.content[0] as TextBlock | undefined;
if (!first || first.type !== 'text' || first.text !== '[Earlier in this conversation:]') continue;
for (let i = m.content.length - 1; i >= 0; i--) {
const b = m.content[i];
if (b && (b as ImageBlock).type === 'image') {
historyImg = b as ImageBlock & { cache_control?: unknown };
break;
}
}
break;
}
if (!historyImg) return;
// The slab anchor is the marked image BEFORE the '[End of rendered context.]'
// boundary in the slab-bearing message. Reminder/tool images sit after that
// boundary (or in other messages) and keep their own caller markers.
let slabAnchor: (ImageBlock & { cache_control?: unknown }) | undefined;
for (const m of messages) {
if (!Array.isArray(m.content)) continue;
const hasBoundary = m.content.some(
(b) => b && (b as TextBlock).type === 'text' && (b as TextBlock).text === '[End of rendered context.]',
);
if (!hasBoundary) continue;
for (const b of m.content) {
if (b && (b as TextBlock).type === 'text' && (b as TextBlock).text === '[End of rendered context.]') break;
if (b && (b as ImageBlock).type === 'image' && (b as { cache_control?: unknown }).cache_control !== undefined) {
slabAnchor = b as ImageBlock & { cache_control?: unknown };
}
}
break;
}
if (!slabAnchor) return; // nothing to relocate → never add a marker
historyImg.cache_control = slabAnchor.cache_control;
delete slabAnchor.cache_control;
}
/** Best-effort extraction of the CLAUDE.md slab from a system text (heuristic).
* Returns empty string if nothing CLAUDE.md-shaped is detected. */
export function extractClaudeMdSlab(staticText: string): string {
@@ -1852,6 +1910,10 @@ export async function transformRequest(
info.historyTextChars = histInfo.collapsedChars;
info.historyImageSha = await historyImageSha8(newMessages);
bumpBucket(info, 'history', histInfo.collapsedChars);
// Move the single cache anchor onto the history image so slab + history
// cache as one stable prefix (created once, then read), instead of the
// history image re-creating whenever the caller's downstream marker moves.
relocateAnchorToHistoryImage(req.messages);
} else if (histInfo.reason) {
info.historyReason = histInfo.reason;
}
+48 -14
View File
@@ -513,16 +513,18 @@ describe('transformRequest history compression (always-on)', () => {
// REGRESSION (slab survives collapse): messages[0] is the slab-bearing
// first user message, NOT the synthetic history. It must still carry a real
// image (the system prompt + tool docs) AND the cache_control anchor — if
// collapse had swept it in, the slab would be reduced to an `[image]`
// placeholder and the marker would be gone.
// image (the system prompt + tool docs) — if collapse had swept it in, the
// slab would be reduced to an `[image]` placeholder.
const slabMsg = reparsed.messages[0];
expect(slabMsg.role).toBe('user');
const slabImgs = slabMsg.content.filter((b: { type: string }) => b.type === 'image');
expect(slabImgs.length).toBeGreaterThanOrEqual(1);
expect(slabImgs.some((b: { cache_control?: unknown }) => b.cache_control !== undefined)).toBe(true);
// The cache anchor relocates onto the history image (the last imaged block
// in prefix order), so the slab image no longer carries it after a collapse.
expect(slabImgs.some((b: { cache_control?: unknown }) => b.cache_control !== undefined)).toBe(false);
// The synthetic history image is at messages[1], AFTER the slab anchor.
// The synthetic history image is at messages[1], AFTER the slab anchor, and
// now holds the sole relocated cache_control breakpoint.
expect(reparsed.messages[1].role).toBe('user');
const content = reparsed.messages[1].content;
expect(Array.isArray(content)).toBe(true);
@@ -531,6 +533,8 @@ describe('transformRequest history compression (always-on)', () => {
type: 'text',
text: '[End of earlier context.]',
});
const histImgs = content.filter((b: { type: string }) => b.type === 'image');
expect(histImgs[histImgs.length - 1].cache_control).toBeDefined();
});
it('sets historyReason=no_closed_prefix when an open tool_use precedes the tail', async () => {
@@ -620,22 +624,52 @@ describe('transformRequest history compression (always-on)', () => {
expect(explicit20.info.collapsedTurns).toBe(10);
});
it('history-image blocks carry NO cache_control (slab anchor is the sole breakpoint)', async () => {
it('relocates the sole cache anchor onto the last history image (one stable prefix, no added marker)', async () => {
// Why: the history image sits AFTER the slab in prefix order. Anchoring the
// single breakpoint on it caches slab + history as ONE stable segment
// (created once, then read). Left unanchored, the history image only caches
// when the caller's roaming downstream marker lands after it — otherwise the
// largest block re-creates at 1.25x every turn.
const msgs: Message[] = [];
for (let i = 0; i < 15; i++) {
const body = `turn ${i}: ` + bigPlain(3500);
msgs.push(i % 2 === 0 ? usr(body) : asst(body));
}
const { body, info } = await transformRequest(mkBody(msgs, bigPlain(80_000)));
// Marked system array (as real Claude Code sends) gives pxpipe exactly one
// caller marker to relocate.
const marked = new TextEncoder().encode(
JSON.stringify({
model: 'claude-3-5-sonnet',
system: [{ type: 'text', text: bigPlain(80_000), cache_control: { type: 'ephemeral' } }],
messages: msgs,
}),
);
const { body, info } = await transformRequest(marked);
expect(info.collapsedImages).toBeGreaterThanOrEqual(1);
const reparsed = JSON.parse(new TextDecoder().decode(body));
// Synthetic history is at messages[1], after the protected slab anchor.
const synth = reparsed.messages[1];
const imgs = synth.content.filter((b: { type: string }) => b.type === 'image');
expect(imgs.length).toBeGreaterThanOrEqual(1);
for (const img of imgs) {
expect(img.cache_control).toBeUndefined();
}
// Slab images survive but no longer carry the anchor.
const slabImgs = reparsed.messages[0].content.filter((b: { type: string }) => b.type === 'image');
expect(slabImgs.length).toBeGreaterThanOrEqual(1);
for (const img of slabImgs) expect(img.cache_control).toBeUndefined();
// The LAST history image carries the breakpoint; earlier ones do not.
const histImgs = reparsed.messages[1].content.filter((b: { type: string }) => b.type === 'image');
expect(histImgs.length).toBeGreaterThanOrEqual(1);
histImgs.forEach((img: { cache_control?: unknown }, i: number) => {
if (i === histImgs.length - 1) expect(img.cache_control).toBeDefined();
else expect(img.cache_control).toBeUndefined();
});
// Pure relocation: exactly one cache_control across the whole request — the
// caller sent one (on the system slab); pxpipe moved it, never added.
const all = [
...(Array.isArray(reparsed.system) ? reparsed.system : []),
...reparsed.messages.flatMap((m: { content?: unknown }) =>
Array.isArray(m.content) ? m.content : [],
),
];
expect(all.filter((b: { cache_control?: unknown }) => b && b.cache_control !== undefined).length).toBe(1);
});
});