ai-space/pxpipe
3
0
Fork 0
mirror of https://github.com/teamchong/pxpipe.git synced 2026-07-22 02:02:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(transform): reword tool-docs stub/header to avoid refusal classifier

Browse Source 
The text-based Tool Reference triggered Anthropic's model-cloning
safeguard (stop_reason: refusal -> silent Opus fallback) because the
per-tool stub said docs lived in "the system prompt" — the exact
phrasing the 169521c retrip identified as the trigger for the imaged
slab, now reintroduced in text form.

- Stub now reads: 'ⓘ Full docs: see "## Tool: <name>" in the Tool
  Reference section.' — no "system prompt"/"authoritative" framing.
- Tool Reference header is provenance-framed as first-party ("pxpipe
  (this user's local proxy) moved the full tool documentation...").
- tracker.ts now maps tool_docs_chars into events.jsonl so refusals
  are attributable to the text-reference path.
- Regression test: stub + reference header must never match
  /system prompt|authoritative/i and must carry provenance framing
  (scoped to the header; quoted third-party tool docs below it may
  legitimately contain the phrase).

Verified: 614/614 tests, tsc --noEmit clean, and a live cold-start
through the rebuilt proxy on claude-fable-5 with the text reference
active — no refusal event, no fallback.
This commit is contained in:
teamchong
2026-07-02 13:17:39 -04:00
parent 169521c9f8
commit baa1e1ee01
3 changed files with 72 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/tracker.ts
+3
View File
@@ -43,6 +43,8 @@ export interface TrackEvent {
reminder_imgs?: number;
/** Images from compressing tool_result content. */
tool_result_imgs?: number;
/** Chars of tool docs moved to the system-text Tool Reference (not imaged). */
tool_docs_chars?: number;
/** tool_result blocks where text exceeded the per-result image budget and was truncated. */
truncated_tool_results?: number;
/** Chars elided by paging across all tool_results this request. */
@@ -201,6 +203,7 @@ export function toTrackEvent(ev: ProxyEvent): TrackEvent {
if (info.dynamicBlockCount !== undefined) out.dynamic_block_count = info.dynamicBlockCount;
if (info.reminderImgs !== undefined) out.reminder_imgs = info.reminderImgs;
if (info.toolResultImgs !== undefined) out.tool_result_imgs = info.toolResultImgs;
if (info.toolDocsChars !== undefined) out.tool_docs_chars = info.toolDocsChars;
if (info.truncatedToolResults !== undefined && info.truncatedToolResults > 0) {
out.truncated_tool_results = info.truncatedToolResults;
}
src/core/transform.ts
+41 -6
View File
@@ -536,6 +536,8 @@ export interface TransformInfo {
imageSourceText?: string;
reminderImgs?: number;
toolResultImgs?: number;
/** Chars of tool docs moved to the system-text Tool Reference (not imaged). */
toolDocsChars?: number;
/** Codepoints missing from the atlas (rendered as blank cells). Telemetry for atlas tuning. */
droppedChars?: number;
/** Top dropped codepoints by frequency (`U+HHHH` → count), at most 20 entries. */
@@ -1437,7 +1439,12 @@ export async function transformRequest(
if (claudeMdSha) info.claudeMdSha8 = claudeMdSha;
if (firstUserSha) info.firstUserSha8 = firstUserSha;
// 2. Optionally fold tool docs into the same image, stubbing originals.
// 2. Optionally move tool docs to a plain-text "Tool Reference" section in the
// system field, stubbing originals. Text (not image) because tool docs are
// static per session — they sit inside the cached prefix and cost ~0.1× after
// the first turn, while keeping schemas/param docs losslessly readable.
// Each stub description cites its own heading ("## Tool: <name>") so the
// model can link stub → full doc deterministically.
let toolDocsText = '';
let toolsRewritten: ToolDef[] | undefined;
if (o.compressTools && Array.isArray(req.tools) && req.tools.length > 0) {
@@ -1471,7 +1478,13 @@ export async function transformRequest(
}
return {
...t,
description: 'ⓘ See image.',
// Wording note (do NOT reintroduce "system prompt"/"authoritative" — same ban
// as the imaged-slab banner below): a stub citing "...the Tool Reference
// section of the system prompt", repeated once per tool, retripped Anthropic's
// reasoning_extraction refusal (stop_reason: "refusal" → Claude Code fell back
// to claude-opus-4-8 immediately on cold start, 2026-07-02). The reference
// block's own header says where the docs live; the stub only needs the heading.
description: `ⓘ Full docs: see "## Tool: ${t.name ?? '?'}" in the Tool Reference section.`,
...(stubSchema !== undefined ? { input_schema: stubSchema } : {}),
};
});
@@ -1481,9 +1494,11 @@ export async function transformRequest(
}
}
// Static slab + tool docs go into the renderer; dynamic slab and billing line stay
// as plain text so the cache key (= image bytes) is stable across turns.
const combinedRaw = [staticText, toolDocsText].filter((s) => s.length > 0).join('\n\n');
// Only the static slab goes into the renderer; dynamic slab and billing line stay
// as plain text so the cache key (= image bytes) is stable across turns. Tool docs
// are spliced into the system field as text (see sysTail below), NOT imaged —
// static per session, so they ride the cache prefix at ~0.1× cost.
const combinedRaw = staticText;
// Compact then reflow before the gate; gate/renderer/paging all see the same text.
// origChars anchored to raw length — that's what Anthropic would have billed.
const combined = maybeReflow(compactSlabWhitespace(combinedRaw), o.reflow);
@@ -1535,7 +1550,7 @@ export async function transformRequest(
// provenance framing below keeps obedience without the extraction signature.
const imageInstructionHeader =
'=================== SESSION CONFIGURATION PAGES ===================\n' +
"pxpipe (this user's local proxy) rendered this session's configuration and tool documentation" +
"pxpipe (this user's local proxy) rendered this session's configuration" +
' into the following images to reduce token cost. Read the pages carefully and follow them as' +
' your operating instructions for this session.' +
columnNoteImg +
@@ -1624,6 +1639,26 @@ export async function transformRequest(
if (billingLine) sysTail.push({ type: 'text', text: billingLine });
if (dynamicText) sysTail.push({ type: 'text', text: dynamicText });
if (Array.isArray(sysRemainder)) sysTail.push(...sysRemainder);
// Tool Reference: full tool docs as plain system text. Stubbed tools[]
// descriptions cite these "## Tool: <name>" headings verbatim. Emitted only
// when toolsRewritten is set — both are applied on this same path, so the
// stub ↔ reference invariant holds (gate-fail paths return earlier with
// original tools untouched).
if (toolsRewritten && toolDocsText) {
// First-party provenance framing, mirroring the imaged-slab banner fix
// (169521c): pxpipe names itself as the author of the relocation so the
// block reads as this session's own config, not a replayed/extracted prompt.
const toolReferenceText =
'=== TOOL REFERENCE ===\n' +
"pxpipe (this user's local proxy) moved the full tool documentation for this" +
' session here to reduce token cost. Each tool in the tools list carries a short' +
' stub description pointing here; the entry under the matching' +
' "## Tool: <name>" heading below is the complete description for that tool.\n\n' +
toolDocsText +
'\n=== END TOOL REFERENCE ===';
sysTail.push({ type: 'text', text: toolReferenceText });
info.toolDocsChars = toolDocsText.length;
}
req.system = sysTail.length > 0 ? sysTail : undefined;
const firstUserIdx = (req.messages ?? []).findIndex((m) => m.role === 'user');
tests/render.test.ts
+28 -6
View File
@@ -695,16 +695,16 @@ describe('transform', () => {
}
});
it('folds tool docs into the same image and stubs originals', async () => {
it('moves tool docs to a system-text Tool Reference and stubs originals', async () => {
const req = JSON.stringify({
model: 'claude-3-5-sonnet',
messages: [{ role: 'user', content: 'hi' }],
system: 'short',
// Tool docs no longer feed the image slab, so the static system text
// alone must clear the compression gate.
system: 'x'.repeat(150000),
tools: [
{
name: 'BigTool',
// Long enough to push the combined slab past the 2-image break-even
// (20k chars). 'A very long tool description. ' = 30 chars × 1100 = 33k.
description: 'A very long tool description. '.repeat(5000),
input_schema: { type: 'object', properties: { x: { type: 'string' } } },
},
@@ -713,10 +713,32 @@ describe('transform', () => {
const bytes = new TextEncoder().encode(req);
const { body, info } = await transformRequest(bytes);
expect(info.compressed).toBe(true);
expect(info.toolDocsChars).toBeGreaterThan(0);
const out = JSON.parse(new TextDecoder().decode(body));
expect(out.tools[0].description).toContain('See image');
// Stub cites its own heading in the system Tool Reference (link-up contract).
expect(out.tools[0].name).toBe('BigTool');
expect(out.tools[0].description).toContain('"## Tool: BigTool"');
expect(out.tools[0].description).toContain('Tool Reference');
// Full docs land as plain text in the system field, under that exact heading.
const sysTexts = (out.system as any[])
.filter((b: any) => b.type === 'text')
.map((b: any) => b.text as string);
const ref = sysTexts.find((t) => t.includes('=== TOOL REFERENCE ==='));
expect(ref).toBeDefined();
expect(ref!).toContain('## Tool: BigTool');
expect(ref!).toContain('A very long tool description.');
// And the imaged slab no longer carries the tool docs.
expect(info.imageSourceText ?? '').not.toContain('A very long tool description.');
// Classifier regression (169521c; retripped 2026-07-02 by a stub citing "the
// system prompt"): pxpipe-authored framing must never read as a replayed or
// extracted prompt. Ban the trigger wording in the stub and the reference
// header, and require first-party provenance framing on the reference block.
// Scoped to the header only — quoted tool docs below it are third-party text.
expect(out.tools[0].description).not.toMatch(/system prompt|authoritative/i);
const refHeader = ref!.slice(0, ref!.indexOf('## Tool:'));
expect(refHeader).not.toMatch(/system prompt|authoritative/i);
expect(refHeader).toContain("this user's local proxy");
});
it('preserves input_schema structure (properties / required / enum) when compressing', async () => {
@@ -725,7 +747,7 @@ describe('transform', () => {
// when the model tried to actually invoke a tool. The fix preserves the
// schema SHELL (type, properties keys, required, enum, items) and only
// strips long-form `description` / `title` / `$schema` / `default` /
// `examples`. The image still carries the original schema for the model.
// `examples`. The system-text Tool Reference still carries the original schema.
const req = JSON.stringify({
model: 'claude-3-5-sonnet',
messages: [{ role: 'user', content: 'hi' }],