pxpipe/tests/design-behavior-e2e.test.ts

/**
 * END-TO-END design-behaviour contract through the REAL proxy.
 *
 * After "don't bust the cache" (cache-stability-e2e) and "don't overclaim"
 * (savings-honesty / savings-math), this asserts the three things pxpipe is
 * actually FOR behave per design, end to end:
 *
 *   1. SYSTEM PROMPT  — the bulky system/slab is imaged out of the request
 *                       (no longer billed as text), live content preserved.
 *   2. HISTORY COLLAPSE — old closed turns become images; recent turns stay as
 *                         legible text (the working set the model still acts on).
 *   3. TOOLS IN RECENT TURNS — a tool_use/tool_result in the recent tail stays
 *                              text (usable), and no tool_result is ever orphaned
 *                              from its tool_use across the image boundary.
 *
 *   fake api  = upstream output (canned response + count_tokens)
 *   our input = pxpipe's transform of the request
 *
 * Run just this file:  pnpm vitest run tests/design-behavior-e2e.test.ts
 */
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import { createProxy } from '../src/core/proxy.js';

// Pin the model scope so these proxy-contract tests stay independent of the developer shell.
let ambientPxpipeModels: string | undefined;
beforeAll(() => {
  ambientPxpipeModels = process.env.PXPIPE_MODELS;
  process.env.PXPIPE_MODELS = 'claude-fable-5,gpt-5.6-sol';
});
afterAll(() => {
  if (ambientPxpipeModels === undefined) delete process.env.PXPIPE_MODELS;
  else process.env.PXPIPE_MODELS = ambientPxpipeModels;
});

function fakeUpstream() {
  const main: string[] = [];
  const real = globalThis.fetch;
  globalThis.fetch = (async (input: Request | string | URL, init?: RequestInit) => {
    const req = input instanceof Request ? input : new Request(String(input), init);
    const path = new URL(req.url).pathname;
    if (path.endsWith('/count_tokens')) {
      return new Response(JSON.stringify({ input_tokens: 9999 }), {
        status: 200,
        headers: { 'content-type': 'application/json' },
      });
    }
    main.push(await req.clone().text());
    if (path.includes('chat/completions')) {
      return new Response(
        JSON.stringify({
          id: 'c1', object: 'chat.completion',
          choices: [{ message: { role: 'assistant', content: 'ok' } }],
          usage: { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 },
        }),
        { status: 200, headers: { 'content-type': 'application/json' } },
      );
    }
    return new Response(
      JSON.stringify({
        id: 'm1', type: 'message', role: 'assistant',
        content: [{ type: 'text', text: 'ok' }],
        model: 'claude-fable-5', stop_reason: 'end_turn',
        usage: { input_tokens: 5, output_tokens: 2 },
      }),
      { status: 200, headers: { 'content-type': 'application/json' } },
    );
  }) as typeof fetch;
  return { main, restore: () => { globalThis.fetch = real; } };
}

const FORCE = { charsPerToken: 1, minCompressChars: 1 } as const;
const big = (n: number) => 'x'.repeat(n);
const APPENDED_SYSTEM_SENTINEL = 'APPENDED_SYSTEM_SENTINEL_keep_live_text';
const BASE_SYSTEM_SENTINEL = 'BASE_SYSTEM_SENTINEL_image_me';
const LARGE_SYSTEM_CHARS = 80_000;

async function drive(path: string, body: string): Promise<any> {
  const cap = fakeUpstream();
  const proxy = createProxy({
    upstream: 'http://anthropic.test', apiKey: 'sk-ant',
    openAIUpstream: 'https://openai.test', openAIApiKey: 'sk-oai',
    transform: FORCE, onRequest: () => {},
  });
  const res = await proxy(
    new Request(`http://localhost${path}`, {
      method: 'POST', headers: { 'content-type': 'application/json' }, body,
    }),
  );
  await res.text();
  cap.restore();
  return JSON.parse(cap.main[0]!);
}

const imageCount = (out: any, key: 'messages' | 'input' = 'messages'): number => {
  let n = 0;
  for (const m of out[key] ?? []) {
    if (!Array.isArray(m.content)) continue;
    for (const c of m.content) if (c?.type === 'image' || c?.type === 'image_url' || c?.type === 'input_image') n++;
  }
  return n;
};

// ===========================================================================
describe('design: SYSTEM PROMPT imaging (Anthropic)', () => {
  it('moves the bulky system slab into images and drops it from the request text', async () => {
    const out = await drive(
      '/v1/messages',
      JSON.stringify({
        model: 'claude-fable-5',
        max_tokens: 16,
        system: [{ type: 'text', text: 'SLAB_SECRET_' + big(LARGE_SYSTEM_CHARS), cache_control: { type: 'ephemeral' } }],
        messages: [{ role: 'user', content: 'LIVE_QUESTION here' }],
      }),
    );
    const hay = JSON.stringify(out);
    // The slab is no longer billed as text anywhere…
    expect(hay).not.toContain('SLAB_SECRET_');
    // …it became image(s), which land in the first user message.
    expect(imageCount(out)).toBeGreaterThan(0);
    // The system field no longer carries the slab (Anthropic forbids images there).
    expect(JSON.stringify(out.system ?? '')).not.toContain('SLAB_SECRET_');
    // The live turn is preserved verbatim and legible.
    expect(hay).toContain('LIVE_QUESTION here');
  });

  it('keeps non-cache-controlled appended system blocks as live text', async () => {
    const out = await drive(
      '/v1/messages',
      JSON.stringify({
        model: 'claude-fable-5',
        max_tokens: 16,
        system: [
          {
            type: 'text',
            text: BASE_SYSTEM_SENTINEL + big(LARGE_SYSTEM_CHARS),
            cache_control: { type: 'ephemeral' },
          },
          { type: 'text', text: APPENDED_SYSTEM_SENTINEL },
        ],
        messages: [{ role: 'user', content: 'LIVE_QUESTION here' }],
      }),
    );
    const hay = JSON.stringify(out);
    expect(imageCount(out)).toBeGreaterThan(0);
    expect(hay).not.toContain(BASE_SYSTEM_SENTINEL);
    expect(JSON.stringify(out.system ?? '')).toContain(APPENDED_SYSTEM_SENTINEL);
    expect(hay).toContain('LIVE_QUESTION here');
  });
});

// ===========================================================================
describe('design: HISTORY COLLAPSE (Anthropic)', () => {
  it('images OLD turns but keeps RECENT turns as legible text', async () => {
    const turns = Array.from({ length: 30 }, (_, i) => ({
      role: (i % 2 === 0 ? 'user' : 'assistant') as 'user' | 'assistant',
      content: `TURNMARK_${i} ${big(4000)}`,
    }));
    const out = await drive(
      '/v1/messages',
      JSON.stringify({ model: 'claude-fable-5', max_tokens: 16, system: 'short', messages: turns }),
    );
    const hay = JSON.stringify(out);
    expect(imageCount(out)).toBeGreaterThan(0);
    // Recent turns survive as legible text BODY (the working set the model still reasons over).
    expect(hay).toContain('TURNMARK_29 ' + 'x'.repeat(100));
    expect(hay).toContain('TURNMARK_28 ' + 'x'.repeat(100));
    // A mid-history turn's BODY was collapsed into an image → its content is not legible text.
    // Its bare identifier may appear in the verbatim fact-sheet beside the image (by design —
    // precision-critical tokens are preserved as text); the 4000-char body is not.
    expect(hay).not.toContain('TURNMARK_5 ' + 'x'.repeat(100));
  });
});

// ===========================================================================
describe('design: TOOLS IN RECENT TURNS (Anthropic)', () => {
  it('keeps a recent tool_use/tool_result as live text and never orphans a result', async () => {
    const msgs: any[] = [{ role: 'user', content: 'start' }];
    for (let i = 0; i < 16; i++) {
      msgs.push({ role: i % 2 === 0 ? 'assistant' : 'user', content: `FILL_${i} ${big(4000)}` });
    }
    // An OLD tool pair (deep in history) and a RECENT one (in the live tail).
    msgs.push({ role: 'assistant', content: [{ type: 'tool_use', id: 'tool_OLD', name: 'bash', input: { cmd: 'old' } }] });
    msgs.push({ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_OLD', content: 'OLDRESULT' }] });
    for (let i = 0; i < 10; i++) {
      msgs.push({ role: i % 2 === 0 ? 'assistant' : 'user', content: `MID_${i} ${big(4000)}` });
    }
    msgs.push({ role: 'assistant', content: [{ type: 'tool_use', id: 'tool_RECENT', name: 'bash', input: { cmd: 'recent' } }] });
    msgs.push({ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool_RECENT', content: 'RECENTRESULT' }] });
    msgs.push({ role: 'user', content: 'LIVE_ASK answer me' });

    const out = await drive(
      '/v1/messages',
      JSON.stringify({ model: 'claude-fable-5', max_tokens: 16, system: 'short', messages: msgs }),
    );
    const hay = JSON.stringify(out);

    expect(imageCount(out)).toBeGreaterThan(0);
    // The recent tool round-trip stays usable (text, not imaged).
    expect(hay).toContain('tool_RECENT');
    expect(hay).toContain('RECENTRESULT');
    // The live ask is preserved.
    expect(hay).toContain('LIVE_ASK answer me');

    // PAIRING INTEGRITY: every tool_result still present as TEXT must have its
    // tool_use present as TEXT too — the image boundary never splits a pair
    // (an orphaned tool_result is a 400 from Anthropic).
    const textToolUseIds = new Set<string>();
    const textToolResultIds: string[] = [];
    for (const m of out.messages ?? []) {
      if (!Array.isArray(m.content)) continue;
      for (const c of m.content) {
        if (c?.type === 'tool_use') textToolUseIds.add(c.id);
        if (c?.type === 'tool_result') textToolResultIds.push(c.tool_use_id);
      }
    }
    for (const id of textToolResultIds) expect(textToolUseIds.has(id)).toBe(true);
  });
});

// ===========================================================================
describe('design: RECENT REQUEST stays legible (GPT)', () => {
  it('images the system slab but keeps the most-recent user request as readable text', async () => {
    const turns = Array.from({ length: 12 }, (_, i) => ({
      role: (i % 2 === 0 ? 'user' : 'assistant') as 'user' | 'assistant',
      content: `gturn ${i} ${big(2000)}`,
    }));
    turns.push({ role: 'user', content: 'FINAL_REQUEST_MARKER please answer' });
    const out = await drive(
      '/v1/chat/completions',
      JSON.stringify({
        model: 'gpt-5.6-sol',
        messages: [{ role: 'system', content: 'SYS ' + big(60_000) }, ...turns],
      }),
    );
    const hay = JSON.stringify(out);
    expect(imageCount(out)).toBeGreaterThan(0); // system imaged
    // The agent's live request must remain legible text, never OCR-only.
    expect(hay).toContain('FINAL_REQUEST_MARKER please answer');
  });
});