fix(history): stop teaching the model to skip Reads

- staleFreshnessHints() rewrites the upstream "(file state is current in
  your context - no need to Read it back)" hint into a stale-state warning
  at history-serialization time, covering both collapsed same-session
  turns and slabs inherited by continuation sessions
- Edit/Write/NotebookEdit stubs gain a live-text same-session Read
  precondition (READ_FIRST_TOOLS), riding un-imaged in the tools array
- telemetry: churning_static_tags canary - FNV-1a fingerprint per
  session/tag flags slab tags whose content churns and busts the image
  cache (TAG_OBSERVATIONS_MAX-capped)
- 5 regression tests in tests/history.test.ts; baseline to beat: 97
  read-gate errors across 18 sessions
This commit is contained in:
teamchong
2026-07-03 19:00:39 -04:00
parent 9fa16eef54
commit 3294c0d15c
4 changed files with 177 additions and 10 deletions
+23 -1
View File
@@ -160,6 +160,28 @@ export function findClosedPrefixBoundary(
return lastClosed;
}
/**
* Claude Code appends "(file state is current in your context — no need to Read it
* back)" to Edit/Write tool_results. True when emitted; stale by the time the turn
* reaches this serializer: everything blocksToText feeds becomes collapsed/imaged
* HISTORY, the CLI's read-ledger resets on process restart, and the file may have
* changed in later turns anyway. Models trusting the hint from prior turns were the
* dominant cause of `File has not been read yet` gate errors (2026-07-03 audit,
* n=55 classified: 20 had a same-transcript Read invalidated by a restart while
* this hint said "current"; 34 edited from prior-session context with no Read at
* all). Rewriting at serialization time also cleans slabs inherited by future
* continuation sessions. Whitespace-tolerant match: 3 of ~2,125 logged instances
* wrap mid-hint.
*/
const FRESHNESS_HINT_RE =
/\(file state is current in your\s+context — no need to Read it back\)/g;
const STALE_FRESHNESS_NOTE =
'(state as of this PRIOR turn — the file may have changed since; Read it again before editing)';
export function staleFreshnessHints(text: string): string {
return text.replace(FRESHNESS_HINT_RE, STALE_FRESHNESS_NOTE);
}
/**
* Linearise content blocks to a single string. Drops thinking blocks (only the
* most-recent assistant turn needs bit-perfect thinking, and it's in the live tail).
@@ -208,7 +230,7 @@ export function blocksToText(content: string | ContentBlock[]): string {
innerText = '';
}
const errMark = tr.is_error === true ? ' (error)' : '';
parts.push(`[tool_result${errMark}]\n${innerText}`);
parts.push(`[tool_result${errMark}]\n${staleFreshnessHints(innerText)}`);
break;
}
case 'image':
+4
View File
@@ -65,6 +65,8 @@ export interface TrackEvent {
passthrough_reasons?: { below_threshold?: number; not_profitable?: number };
/** Unrecognized tag names in the static slab — canary for Claude Code releases adding new dynamic tags. */
unknown_static_tags?: string[];
/** Slab tags whose content changed within a session — proven per-turn dynamics busting the image cache. */
churning_static_tags?: string[];
/** Per-bucket TEXT chars through each gate call site (static_slab, reminder, tool_result_*, history).
* Undefined on uncompressed requests; enables per-bucket cpt regression. */
bucket_chars?: Partial<Record<
@@ -258,6 +260,8 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent {
if (info.cachePrefixBytes !== undefined) out.cache_prefix_bytes = info.cachePrefixBytes;
if (info.unknownStaticTags && info.unknownStaticTags.length > 0)
out.unknown_static_tags = info.unknownStaticTags;
if (info.churningStaticTags && info.churningStaticTags.length > 0)
out.churning_static_tags = info.churningStaticTags;
if (info.systemSha8) out.system_sha8 = info.systemSha8;
if (info.claudeMdSha8) out.claude_md_sha8 = info.claudeMdSha8;
if (info.firstUserSha8) out.first_user_sha8 = info.firstUserSha8;
+107 -9
View File
@@ -166,6 +166,10 @@ const CHARS_PER_TOKEN = 4;
* Slab-specific because reminders/tool_results have unknown shape; those stay at 4. */
export const SLAB_CHARS_PER_TOKEN = 2.0;
// Tools whose stub description keeps a live-text read-before-edit precondition
// when full docs move into the imaged Tool Reference (read-gate audit, 2026-07-03).
const READ_FIRST_TOOLS = new Set(['Edit', 'Write', 'NotebookEdit']);
/** Empirical cpt for the history-collapse path (same Opus 4.7 telemetry as SLAB_CHARS_PER_TOKEN).
* History is even denser (tool_use JSON dominates), so 2.0 is doubly conservative. */
export const HISTORY_CHARS_PER_TOKEN = 2.0;
@@ -518,6 +522,9 @@ export interface TransformInfo {
/** Tag-shaped blocks in the static slab not in DYNAMIC_BLOCK_TAGS.
* Canary: a new per-turn Claude Code tag would appear here before cache rate collapses. */
unknownStaticTags?: string[];
/** Static-slab tags whose content changed within a session proven dynamic,
* busting the image cache each turn. The real alert signal. */
churningStaticTags?: string[];
env?: EnvFields;
/** sha8 of static slab + tool docs (what goes in the image). Repeats across turns → cache hits. */
systemSha8?: string;
@@ -650,19 +657,45 @@ const DYNAMIC_BLOCK_TAGS = [
'system-reminder',
] as const;
// Known-static tags in the slab (part of Claude Code's built-in prompt, not per-turn).
// Listed here so the canary in splitStaticDynamic doesn't false-fire on them.
// Add a tag only after confirming it doesn't rotate per turn.
const KNOWN_STATIC_TAGS = ['types'] as const;
// Known-static slab tags — suppresses first-sighting `unknownStaticTags` noise
// only. Correctness doesn't depend on this list: observeStaticTagChurn catches
// a wrong entry on its second sighting.
const KNOWN_STATIC_TAGS = [
// Claude Code
'types',
// opencode (codex system prompts have no tag-shaped blocks)
'example',
'available_skills',
// beast.txt + title.txt
'examples',
'rules',
'task',
// copilot-gpt-5.txt
'codeSearchInstructions',
'codeSearchToolUseInstructions',
'communicationGuidelines',
'gptAgentInstructions',
'outputFormatting',
'structuredWorkflow',
'toolUseInstructions',
] as const;
function splitStaticDynamic(text: string): {
staticText: string;
dynamicText: string;
blockCount: number;
unknownTags: string[];
/** tag → concatenated inner content of same-named slab blocks. */
staticTagContents: Map<string, string>;
} {
if (!text)
return { staticText: '', dynamicText: '', blockCount: 0, unknownTags: [] };
return {
staticText: '',
dynamicText: '',
blockCount: 0,
unknownTags: [],
staticTagContents: new Map(),
};
const pattern = new RegExp(
`<(${DYNAMIC_BLOCK_TAGS.join('|')})(\\s[^>]*)?>[\\s\\S]*?</\\1>`,
'g',
@@ -683,13 +716,16 @@ function splitStaticDynamic(text: string): {
// surfacing the tag name lets us detect it within hours of a release.
const known = new Set<string>(DYNAMIC_BLOCK_TAGS);
const knownStatic = new Set<string>(KNOWN_STATIC_TAGS);
const sniffer = /<([a-zA-Z][a-zA-Z0-9_-]*)(?:\s[^>]*)?>[\s\S]*?<\/\1>/g;
const sniffer = /<([a-zA-Z][a-zA-Z0-9_-]*)(?:\s[^>]*)?>([\s\S]*?)<\/\1>/g;
const unknown = new Set<string>();
const staticTagContents = new Map<string, string>();
let s: RegExpExecArray | null;
while ((s = sniffer.exec(staticBuf)) !== null) {
const tag = s[1]!;
if (!known.has(tag) && !knownStatic.has(tag) && tag.length <= 64)
unknown.add(tag);
if (tag.length > 64) continue;
if (!known.has(tag) && !knownStatic.has(tag)) unknown.add(tag);
// Fold repeated tags (e.g. several <example>s) into one fingerprint.
staticTagContents.set(tag, (staticTagContents.get(tag) ?? '') + s[2]!);
}
return {
@@ -698,9 +734,49 @@ function splitStaticDynamic(text: string): {
dynamicText: dynamicParts.join('\n\n'),
blockCount: dynamicParts.length,
unknownTags: [...unknown],
staticTagContents,
};
}
/** FNV-1a 32-bit — cheap synchronous content fingerprint for churn detection. */
function fnv1a(text: string): number {
let h = 0x811c9dc5;
for (let i = 0; i < text.length; i++) {
h ^= text.charCodeAt(i);
h = Math.imul(h, 0x01000193);
}
return h >>> 0;
}
// Last content hash per (session, tag). Bounded LRU.
const TAG_OBSERVATIONS_MAX = 4096;
const tagObservations = new Map<string, number>();
/** Returns slab tags whose content changed since the last sighting in the same
* session proven per-turn dynamics, whatever the hardcoded lists say. */
function observeStaticTagChurn(
sessionKey: string,
tagContents: ReadonlyMap<string, string>,
): string[] {
const churned: string[] = [];
for (const [tag, inner] of tagContents) {
const key = `${sessionKey}\0${tag}`;
const hash = fnv1a(inner);
const prev = tagObservations.get(key);
if (prev !== undefined) {
if (prev !== hash) churned.push(tag);
tagObservations.delete(key); // refresh LRU position
}
tagObservations.set(key, hash);
}
while (tagObservations.size > TAG_OBSERVATIONS_MAX) {
const oldest = tagObservations.keys().next().value;
if (oldest === undefined) break;
tagObservations.delete(oldest);
}
return churned;
}
/** sha256[0..8] hex via Web Crypto (works in Node 18+ and Workers). 32-bit collision-safe. */
export async function sha8(text: string): Promise<string> {
const buf = new TextEncoder().encode(text);
@@ -1421,6 +1497,7 @@ export async function transformRequest(
dynamicText,
blockCount: dynBlocks,
unknownTags,
staticTagContents,
} = splitStaticDynamic(sysBody);
info.staticChars = staticText.length;
info.dynamicChars = dynamicText.length + envMarkdown.length;
@@ -1442,6 +1519,16 @@ export async function transformRequest(
if (claudeMdSha) info.claudeMdSha8 = claudeMdSha;
if (firstUserSha) info.firstUserSha8 = firstUserSha;
// Canary: slab tags whose content churns within a session bust the image
// cache every turn — report them regardless of the hardcoded lists.
if (staticTagContents.size > 0) {
const churning = observeStaticTagChurn(
firstUserSha ?? claudeMdSha ?? 'global',
staticTagContents,
);
if (churning.length > 0) info.churningStaticTags = churning;
}
// 2. Move tool docs into the imaged "Tool Reference", stubbing originals.
// Imaged (not text) because that IS the compression — descriptions and
// schema annotations ride at image token rates, mirroring the GPT path.
@@ -1470,6 +1557,17 @@ export async function transformRequest(
schema = stripped;
}
}
// Read-before-Edit precondition rides as LIVE TEXT, not imaged: the CLI
// rejects Edit/Write on any existing file not Read in THIS session's
// process, and the rule lost salience once full tool docs moved into the
// imaged reference (read-gate audit, 2026-07-03). Three tools only, no
// banned wording — stays clear of the per-tool-stub repetition pattern
// that tripped reasoning_extraction (see wording note below).
const readFirstNote = READ_FIRST_TOOLS.has(t.name ?? '')
? ' Requires a Read of the same file earlier in THIS session when the file' +
' already exists — the call is rejected otherwise; file content recalled' +
' from imaged or prior-session context does not satisfy this.'
: '';
return {
...t,
// Wording note (do NOT reintroduce "system prompt"/"authoritative" — same ban
@@ -1478,7 +1576,7 @@ export async function transformRequest(
// reasoning_extraction refusal (stop_reason: "refusal" → Claude Code fell back
// to claude-opus-4-8 immediately on cold start, 2026-07-02). The reference
// block's own header says where the docs live; the stub only needs the heading.
description: `ⓘ Full docs: see "## Tool: ${t.name ?? '?'}" in the Tool Reference section.`,
description: `ⓘ Full docs: see "## Tool: ${t.name ?? '?'}" in the Tool Reference section.${readFirstNote}`,
...(schema !== undefined ? { input_schema: schema } : {}),
};
});
+43
View File
@@ -19,6 +19,7 @@ import { describe, expect, it } from 'vitest';
import {
findClosedPrefixBoundary,
blocksToText,
staleFreshnessHints,
messagesToHistoryText,
collapseHistory,
HISTORY_DEFAULTS,
@@ -206,6 +207,48 @@ describe('blocksToText', () => {
});
});
describe('staleFreshnessHints (read-gate audit, 2026-07-03)', () => {
const HINT = '(file state is current in your context — no need to Read it back)';
const STALE =
'(state as of this PRIOR turn — the file may have changed since; Read it again before editing)';
it('rewrites the canonical Claude Code freshness hint', () => {
const input = `The file /tmp/x.ts has been updated successfully. ${HINT}`;
expect(staleFreshnessHints(input)).toBe(
`The file /tmp/x.ts has been updated successfully. ${STALE}`,
);
});
it('rewrites the line-wrapped variant (3 of ~2,125 logged instances)', () => {
const wrapped =
'(file state is current in your\n context — no need to Read it back)';
expect(staleFreshnessHints(`ok. ${wrapped}`)).toBe(`ok. ${STALE}`);
});
it('rewrites every occurrence, not just the first', () => {
const out = staleFreshnessHints(`${HINT} middle ${HINT}`);
expect(out).toBe(`${STALE} middle ${STALE}`);
expect(out).not.toContain('no need to Read it back');
});
it('leaves unrelated text untouched', () => {
const s = 'Edit rejected: File has not been read yet. Read it first.';
expect(staleFreshnessHints(s)).toBe(s);
});
it('applies inside blocksToText tool_result serialisation', () => {
const out = blocksToText([
{
type: 'tool_result',
tool_use_id: 'tx1',
content: `The file /a/b.ts has been updated successfully. ${HINT}`,
},
]);
expect(out).toContain(STALE);
expect(out).not.toContain('no need to Read it back');
});
});
describe('messagesToHistoryText', () => {
it('wraps each turn in <role> XML tags and joins with blank line', () => {
const msgs: Message[] = [usr('hi'), asst('hello')];