mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
Danil Silantyev
69c3a51829
FINDINGS.md documents the main failure mode of imaged text: the reader (Opus especially) confabulates a plausible exact value instead of abstaining. Add abstention guidance to the rendered framing so the model defers exact identifiers to the factsheet / source rather than guess. - the imaged slab banner (imageInstructionHeader) and the history-transcript intro (HISTORY_SYNTHETIC_INTRO) now say: for exact identifiers, paths, hashes, version strings, and numbers, use the adjacent exact-value factsheet; if a value was only in an image and not in the factsheet, do not guess — say it is not safe to quote and re-read the source. - no geometry, model-allowlist, or gate change; the exact factsheet still rides after the image blocks unchanged. - test asserts the phrasing appears in info.imageSourceText for an applied transform and in the history intro, and NOT in a non-applied (compress=false) transform. Full suite green (633).
45 lines
1.8 KiB
TypeScript
45 lines
1.8 KiB
TypeScript
import { describe, it, expect } from 'vitest';
|
|
import { transformRequest } from '../src/core/transform.js';
|
|
import { HISTORY_SYNTHETIC_INTRO } from '../src/core/history.js';
|
|
|
|
const enc = new TextEncoder();
|
|
const dec = new TextDecoder();
|
|
const ABSTAIN = 'not safe to quote from the image';
|
|
|
|
// FINDINGS.md documents the main risk of imaged text: the reader confabulates a
|
|
// plausible exact value instead of abstaining. The rendered framing now tells the
|
|
// model to defer exact identifiers to the factsheet / source rather than guess.
|
|
describe('exact-string abstention framing', () => {
|
|
it('the imaged slab banner discourages guessing exact values from images', async () => {
|
|
const bigSystem = 'You are a coding agent with detailed operating instructions. '.repeat(300);
|
|
const { info } = await transformRequest(
|
|
enc.encode(JSON.stringify({
|
|
model: 'claude-fable-5',
|
|
system: bigSystem,
|
|
messages: [{ role: 'user', content: 'hi' }],
|
|
})),
|
|
{ compress: true, minCompressChars: 1, charsPerToken: 1 },
|
|
);
|
|
expect(info.compressed).toBe(true);
|
|
// imageSourceText is the exact text rendered into the slab image(s).
|
|
expect(info.imageSourceText ?? '').toContain(ABSTAIN);
|
|
});
|
|
|
|
it('the history-transcript framing carries the same abstention', () => {
|
|
expect(HISTORY_SYNTHETIC_INTRO).toContain('do not guess an exact value seen only in the image');
|
|
});
|
|
|
|
it('adds nothing when the transform is not applied', async () => {
|
|
const { body: out, info } = await transformRequest(
|
|
enc.encode(JSON.stringify({
|
|
model: 'claude-fable-5',
|
|
system: 'short',
|
|
messages: [{ role: 'user', content: 'hi' }],
|
|
})),
|
|
{ compress: false },
|
|
);
|
|
expect(info.compressed).toBe(false);
|
|
expect(dec.decode(out)).not.toContain(ABSTAIN);
|
|
});
|
|
});
|