mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
Steven Chong
1468 lines
51 KiB
TypeScript
1468 lines
51 KiB
TypeScript
import { afterAll, beforeAll, describe, it, expect } from 'vitest';
|
|
import { createProxy, type ProxyEvent } from '../src/core/proxy.js';
|
|
|
|
// Pin the model scope so these proxy-contract tests stay independent of the developer shell.
|
|
let ambientPxpipeModels: string | undefined;
|
|
beforeAll(() => {
|
|
ambientPxpipeModels = process.env.PXPIPE_MODELS;
|
|
process.env.PXPIPE_MODELS = 'claude-fable-5,gpt-5.6-sol';
|
|
});
|
|
afterAll(() => {
|
|
if (ambientPxpipeModels === undefined) delete process.env.PXPIPE_MODELS;
|
|
else process.env.PXPIPE_MODELS = ambientPxpipeModels;
|
|
});
|
|
|
|
/** Tiny in-process mock upstream — accepts any request and returns whatever
|
|
* the test fixture configured. Lets us assert that the proxy correctly
|
|
* extracts Anthropic's usage block from both SSE and JSON responses without
|
|
* touching the network. */
|
|
function mockUpstream(handler: (req: Request) => Promise<Response> | Response) {
|
|
// Patch globalThis.fetch for the duration of the test.
|
|
const real = globalThis.fetch;
|
|
globalThis.fetch = ((req: Request | string | URL, init?: RequestInit) => {
|
|
const r = req instanceof Request ? req : new Request(String(req), init);
|
|
return Promise.resolve(handler(r));
|
|
}) as typeof fetch;
|
|
return () => {
|
|
globalThis.fetch = real;
|
|
};
|
|
}
|
|
|
|
const SAMPLE_REQ_BODY = JSON.stringify({
|
|
model: 'claude-3-5-haiku-latest',
|
|
messages: [{ role: 'user', content: 'hi' }],
|
|
system: 'short',
|
|
});
|
|
|
|
describe('proxy usage extraction', () => {
|
|
it('extracts usage tokens from a non-stream JSON response', async () => {
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(
|
|
JSON.stringify({
|
|
id: 'msg_1',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [{ type: 'text', text: 'hello' }],
|
|
usage: {
|
|
input_tokens: 123,
|
|
output_tokens: 7,
|
|
cache_creation_input_tokens: 0,
|
|
cache_read_input_tokens: 100,
|
|
},
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
// Drain the client-side body so the tee is forced to finish.
|
|
await res.text();
|
|
// Give the onRequest callback a tick to fire (it's behind a void promise).
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.usage?.input_tokens).toBe(123);
|
|
expect(captured!.usage?.output_tokens).toBe(7);
|
|
expect(captured!.usage?.cache_read_input_tokens).toBe(100);
|
|
expect(captured!.firstByteMs).toBeTypeOf('number');
|
|
});
|
|
|
|
it('never calls Anthropic count_tokens for Sol Responses', async () => {
|
|
const upstreamRequests: Request[] = [];
|
|
const restore = mockUpstream(async (req) => {
|
|
upstreamRequests.push(req.clone());
|
|
return new Response(JSON.stringify({
|
|
id: 'resp_sol_1', object: 'response', status: 'completed',
|
|
output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'ok' }] }],
|
|
usage: { input_tokens: 400, output_tokens: 8, input_tokens_details: { cached_tokens: 300 } },
|
|
}), { status: 200, headers: { 'content-type': 'application/json' } });
|
|
});
|
|
const proxy = createProxy({
|
|
openAIUpstream: 'https://api.openai.test', openAIApiKey: 'sk-test',
|
|
transform: { charsPerToken: 1, minCompressChars: 1 },
|
|
});
|
|
const body = JSON.stringify({
|
|
model: 'gpt-5.6-sol',
|
|
instructions: 'System instruction. '.repeat(900),
|
|
input: [{ role: 'user', content: 'hi' }],
|
|
});
|
|
const res = await proxy(new Request('http://localhost/v1/responses', {
|
|
method: 'POST', headers: { 'content-type': 'application/json' }, body,
|
|
}));
|
|
await res.text();
|
|
restore();
|
|
expect(upstreamRequests).toHaveLength(1);
|
|
expect(upstreamRequests[0]!.url).toBe('https://api.openai.test/v1/responses');
|
|
expect(upstreamRequests.some((r) => r.url.includes('/count_tokens'))).toBe(false);
|
|
});
|
|
|
|
it('transforms OpenCode /anthropic/messages (no /v1) and records the model', async () => {
|
|
const upstreamRequests: Request[] = [];
|
|
const restore = mockUpstream(async (req) => {
|
|
upstreamRequests.push(req.clone());
|
|
const url = req.url;
|
|
if (url.endsWith('/count_tokens')) {
|
|
return new Response(JSON.stringify({ input_tokens: 9000 }), {
|
|
status: 200,
|
|
headers: { 'content-type': 'application/json' },
|
|
});
|
|
}
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'msg_1',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [{ type: 'text', text: 'hello' }],
|
|
usage: { input_tokens: 120, output_tokens: 7, cache_read_input_tokens: 0 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
upstream: 'http://ocproxy.test',
|
|
apiKey: 'sk-anthropic-test',
|
|
transform: { charsPerToken: 1, minCompressChars: 1 },
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const reqBody = JSON.stringify({
|
|
model: 'claude-fable-5',
|
|
max_tokens: 1,
|
|
system: 'System instruction. '.repeat(900),
|
|
messages: [{ role: 'user', content: 'hi' }],
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/anthropic/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json', 'x-api-key': 'sk-anthropic-test' },
|
|
body: reqBody,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
const main = upstreamRequests.find((r) => r.url === 'http://ocproxy.test/anthropic/messages');
|
|
expect(main).toBeDefined();
|
|
expect(captured?.model).toBe('claude-fable-5');
|
|
expect(captured?.info?.compressed).toBe(true);
|
|
// count_tokens probe mirrors the request path under the same prefix.
|
|
expect(
|
|
upstreamRequests.some((r) => r.url === 'http://ocproxy.test/anthropic/messages/count_tokens'),
|
|
).toBe(true);
|
|
});
|
|
|
|
it('passes Claude Code Messages + Sol through without Anthropic count_tokens', async () => {
|
|
const upstreamRequests: Request[] = [];
|
|
const restore = mockUpstream(async (req) => {
|
|
upstreamRequests.push(req.clone());
|
|
return new Response(JSON.stringify({
|
|
id: 'msg_sol', type: 'message', role: 'assistant',
|
|
content: [{ type: 'text', text: 'ok' }], model: 'gpt-5.6-sol',
|
|
usage: { input_tokens: 120, output_tokens: 7 },
|
|
}), { status: 200, headers: { 'content-type': 'application/json' } });
|
|
});
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
upstream: 'http://ocproxy.test', apiKey: 'sk-test',
|
|
transform: { charsPerToken: 1, minCompressChars: 1 },
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
const body = JSON.stringify({
|
|
model: 'gpt-5.6-sol', max_tokens: 16,
|
|
system: 'System instruction. '.repeat(900),
|
|
messages: [{ role: 'user', content: 'hi' }],
|
|
});
|
|
const res = await proxy(new Request('http://localhost/v1/messages', {
|
|
method: 'POST', headers: { 'content-type': 'application/json' }, body,
|
|
}));
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
expect(upstreamRequests).toHaveLength(1);
|
|
expect(upstreamRequests[0]!.url).toBe('http://ocproxy.test/v1/messages');
|
|
expect(upstreamRequests.some((r) => r.url.includes('/count_tokens'))).toBe(false);
|
|
expect(JSON.parse(await upstreamRequests[0]!.text())).toEqual(JSON.parse(body));
|
|
expect(captured?.info?.compressed).toBe(false);
|
|
expect(captured?.info?.reason).toBe('unsupported_model');
|
|
});
|
|
|
|
it('routes GPT 5.6 Sol chat completions to OpenAI, transforms once, and normalizes usage', async () => {
|
|
const upstreamRequests: Request[] = [];
|
|
const restore = mockUpstream(async (req) => {
|
|
upstreamRequests.push(req.clone());
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'chatcmpl_1',
|
|
object: 'chat.completion',
|
|
choices: [{ message: { role: 'assistant', content: 'hello' } }],
|
|
usage: { prompt_tokens: 55, completion_tokens: 7, total_tokens: 62 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
openAIUpstream: 'https://api.openai.test',
|
|
openAIApiKey: 'sk-test',
|
|
transform: { charsPerToken: 1, minCompressChars: 1 },
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const reqBody = JSON.stringify({
|
|
model: 'gpt-5.6-sol',
|
|
messages: [
|
|
{ role: 'system', content: 'System instruction. '.repeat(900) },
|
|
{ role: 'user', content: 'hi' },
|
|
],
|
|
tools: [{
|
|
type: 'function',
|
|
function: {
|
|
name: 'search',
|
|
description: 'Search files. '.repeat(100),
|
|
parameters: { type: 'object', properties: { query: { type: 'string' } } },
|
|
},
|
|
}],
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/chat/completions', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: reqBody,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
// OpenAI/Sol must never use Anthropic's /count_tokens endpoint. Its
|
|
// counterfactual is local o200k tokenization in src/core/openai.ts.
|
|
expect(upstreamRequests).toHaveLength(1);
|
|
expect(upstreamRequests.some((r) => r.url.includes('/count_tokens'))).toBe(false);
|
|
expect(upstreamRequests[0]!.url).toBe('https://api.openai.test/v1/chat/completions');
|
|
expect(upstreamRequests[0]!.headers.get('authorization')).toBe('Bearer sk-test');
|
|
const sent = JSON.parse(await upstreamRequests[0]!.text()) as any;
|
|
const firstUser = sent.messages.find((m: any) => m.role === 'user');
|
|
expect(firstUser.content[0].type).toBe('image_url');
|
|
expect(firstUser.content[0].image_url.url).toMatch(/^data:image\/png;base64,/);
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.usage?.input_tokens).toBe(55);
|
|
expect(captured!.usage?.output_tokens).toBe(7);
|
|
expect(captured!.info?.baselineProbeStatus).toBeUndefined();
|
|
});
|
|
|
|
it('transforms provider-prefixed OpenAI chat but forwards through the generic upstream', async () => {
|
|
const upstreamRequests: Request[] = [];
|
|
const restore = mockUpstream(async (req) => {
|
|
upstreamRequests.push(req.clone());
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'chatcmpl_1',
|
|
object: 'chat.completion',
|
|
choices: [{ message: { role: 'assistant', content: 'hello' } }],
|
|
usage: { prompt_tokens: 55, completion_tokens: 7, total_tokens: 62 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
|
|
const proxy = createProxy({
|
|
upstream: 'http://ocproxy.test',
|
|
openAIUpstream: 'https://api.openai.test',
|
|
transform: { charsPerToken: 1, minCompressChars: 1 },
|
|
});
|
|
|
|
const reqBody = JSON.stringify({
|
|
model: 'gpt-5.6-sol',
|
|
messages: [
|
|
{ role: 'system', content: 'System instruction. '.repeat(900) },
|
|
{ role: 'user', content: 'hi' },
|
|
],
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/openai/v1/chat/completions', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json', authorization: 'Bearer local-token' },
|
|
body: reqBody,
|
|
}),
|
|
);
|
|
await res.text();
|
|
restore();
|
|
|
|
expect(upstreamRequests).toHaveLength(1);
|
|
expect(upstreamRequests[0]!.url).toBe('http://ocproxy.test/openai/v1/chat/completions');
|
|
expect(upstreamRequests[0]!.headers.get('authorization')).toBe('Bearer local-token');
|
|
const sent = JSON.parse(await upstreamRequests[0]!.text()) as any;
|
|
const firstUser = sent.messages.find((m: any) => m.role === 'user');
|
|
expect(firstUser.content[0].type).toBe('image_url');
|
|
});
|
|
|
|
it('transforms OpenCode /openai/responses requests and records the model', async () => {
|
|
const upstreamRequests: Request[] = [];
|
|
const restore = mockUpstream(async (req) => {
|
|
upstreamRequests.push(req.clone());
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'resp_1',
|
|
object: 'response',
|
|
output: [{ type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'hello' }] }],
|
|
usage: { input_tokens: 55, output_tokens: 7, total_tokens: 62 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
upstream: 'http://ocproxy.test',
|
|
openAIUpstream: 'https://api.openai.test',
|
|
transform: { charsPerToken: 1, minCompressChars: 1 },
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const reqBody = JSON.stringify({
|
|
model: 'gpt-5.6-sol',
|
|
instructions: 'System instruction. '.repeat(900),
|
|
input: [{ role: 'user', content: 'hi' }],
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/openai/responses', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json', authorization: 'Bearer local-token' },
|
|
body: reqBody,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(upstreamRequests).toHaveLength(1);
|
|
expect(upstreamRequests[0]!.url).toBe('http://ocproxy.test/openai/responses');
|
|
expect(upstreamRequests[0]!.headers.get('authorization')).toBe('Bearer local-token');
|
|
const sent = JSON.parse(await upstreamRequests[0]!.text()) as any;
|
|
const firstUser = sent.input.find((m: any) => m.role === 'user');
|
|
expect(firstUser.content[0].type).toBe('input_image');
|
|
expect(captured?.model).toBe('gpt-5.6-sol');
|
|
expect(captured?.info?.compressed).toBe(true);
|
|
expect(captured?.info?.firstUserSha8).toMatch(/^[0-9a-f]{8}$/);
|
|
});
|
|
|
|
it('extracts usage tokens from an SSE stream (message_start event)', async () => {
|
|
const sseBody =
|
|
'event: message_start\n' +
|
|
'data: ' +
|
|
JSON.stringify({
|
|
type: 'message_start',
|
|
message: {
|
|
id: 'msg_2',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [],
|
|
usage: {
|
|
input_tokens: 42,
|
|
output_tokens: 0,
|
|
cache_creation_input_tokens: 5000,
|
|
cache_read_input_tokens: 0,
|
|
},
|
|
},
|
|
}) +
|
|
'\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"hi"}}\n\n' +
|
|
'event: message_stop\n' +
|
|
'data: {"type":"message_stop"}\n\n';
|
|
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(sseBody, {
|
|
status: 200,
|
|
headers: { 'content-type': 'text/event-stream' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.usage?.input_tokens).toBe(42);
|
|
expect(captured!.usage?.cache_creation_input_tokens).toBe(5000);
|
|
});
|
|
|
|
it('fires the event with undefined usage when the response is an error', async () => {
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(JSON.stringify({ error: { type: 'overloaded_error' } }), {
|
|
status: 529,
|
|
headers: { 'content-type': 'application/json' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.status).toBe(529);
|
|
expect(captured!.usage).toBeUndefined();
|
|
// 5xx: we synthesize our own message upstream, so no errorBody capture.
|
|
expect(captured!.errorBody).toBeUndefined();
|
|
});
|
|
|
|
it('captures upstream error body for 4xx responses (up to 2 KiB)', async () => {
|
|
const upstreamErr = {
|
|
type: 'error',
|
|
error: {
|
|
type: 'invalid_request_error',
|
|
message: 'messages.5.content.0.tool_use_id: unknown tool_use id',
|
|
},
|
|
};
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(JSON.stringify(upstreamErr), {
|
|
status: 400,
|
|
headers: { 'content-type': 'application/json' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
// Drain the client side so the tee can complete.
|
|
const clientBody = await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.status).toBe(400);
|
|
expect(captured!.usage).toBeUndefined();
|
|
expect(captured!.errorBody).toBe(JSON.stringify(upstreamErr));
|
|
// Client must still receive the full body unchanged.
|
|
expect(clientBody).toBe(JSON.stringify(upstreamErr));
|
|
});
|
|
|
|
it('caps the captured 4xx error body at ~2 KiB', async () => {
|
|
const huge = 'x'.repeat(10_000);
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(huge, {
|
|
status: 400,
|
|
headers: { 'content-type': 'text/plain' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.errorBody).toBeDefined();
|
|
expect(captured!.errorBody!.length).toBe(2048);
|
|
});
|
|
|
|
/** Decompress a gzip Uint8Array back to bytes — mirror of proxy's gzipBytes. */
|
|
async function gunzipBytes(buf: Uint8Array): Promise<Uint8Array> {
|
|
const stream = new Response(buf as BufferSource).body!.pipeThrough(
|
|
new DecompressionStream('gzip'),
|
|
);
|
|
return new Uint8Array(await new Response(stream).arrayBuffer());
|
|
}
|
|
|
|
it('captures the FULL gzipped transformed body on 4xx + sets reqBodySha8', async () => {
|
|
// Pair with errorBody so a future debugger can reconstruct
|
|
// "we sent X, Anthropic said Y" from the JSONL alone. We gzip the body
|
|
// so even a 170 KiB transformed payload fits inline once base64'd
|
|
// (typical PNG-heavy bodies compress to <10% of source).
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(JSON.stringify({ error: { type: 'bad' } }), {
|
|
status: 400,
|
|
headers: { 'content-type': 'application/json' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.status).toBe(400);
|
|
|
|
// Hash lands on every event, not just 4xx.
|
|
expect(captured!.reqBodySha8).toMatch(/^[0-9a-f]{8}$/);
|
|
|
|
// Gzipped body is present, has the gzip magic header, and decompresses
|
|
// back to the transformed JSON we sent upstream.
|
|
expect(captured!.reqBodyGz).toBeDefined();
|
|
expect(captured!.reqBodyGz![0]).toBe(0x1f);
|
|
expect(captured!.reqBodyGz![1]).toBe(0x8b);
|
|
|
|
const decoded = new TextDecoder().decode(
|
|
await gunzipBytes(captured!.reqBodyGz!),
|
|
);
|
|
const parsed = JSON.parse(decoded);
|
|
expect(parsed.model).toBe('claude-3-5-haiku-latest');
|
|
expect(parsed.messages[0].role).toBe('user');
|
|
});
|
|
|
|
it('does NOT gzip the request body on 2xx (but still sets reqBodySha8)', async () => {
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(JSON.stringify({
|
|
id: 'x', type: 'message', role: 'assistant',
|
|
content: [{ type: 'text', text: 'ok' }],
|
|
model: 'x', stop_reason: 'end_turn',
|
|
usage: { input_tokens: 1, output_tokens: 1 },
|
|
}), {
|
|
status: 200,
|
|
headers: { 'content-type': 'application/json' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => {
|
|
captured = e;
|
|
},
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.status).toBe(200);
|
|
// Hash lands on every event.
|
|
expect(captured!.reqBodySha8).toMatch(/^[0-9a-f]{8}$/);
|
|
// But the gzipped body itself is only captured on 4xx.
|
|
expect(captured!.reqBodyGz).toBeUndefined();
|
|
});
|
|
|
|
it('reqBodySha8 is identical across two requests with the same body', async () => {
|
|
// Correlation use-case: spot "same payload sometimes works, sometimes
|
|
// fails" patterns in events.jsonl.
|
|
let restore = mockUpstream(
|
|
() =>
|
|
new Response('{"x":1}', {
|
|
status: 400,
|
|
headers: { 'content-type': 'application/json' },
|
|
}),
|
|
);
|
|
|
|
const captures: ProxyEvent[] = [];
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => {
|
|
captures.push(e);
|
|
},
|
|
});
|
|
|
|
for (let i = 0; i < 2; i++) {
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
}
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captures.length).toBe(2);
|
|
expect(captures[0]!.reqBodySha8).toBeDefined();
|
|
expect(captures[0]!.reqBodySha8).toBe(captures[1]!.reqBodySha8);
|
|
});
|
|
|
|
// The proxy makes ONE parallel side call: /v1/messages/count_tokens on
|
|
// the PRE-COMPRESSION body. That number lands on the dashboard as the
|
|
// baseline against which we measure savings. count_tokens is free
|
|
// (no billing) and is the only side path we whitelist — any other
|
|
// endpoint would be an unexpected leak.
|
|
it('calls /v1/messages/count_tokens (baseline probe) and no other side endpoints', async () => {
|
|
const sidePaths: string[] = [];
|
|
const restore = mockUpstream((req) => {
|
|
const url = new URL(req.url);
|
|
if (url.pathname !== '/v1/messages') sidePaths.push(url.pathname);
|
|
// count_tokens response shape: { input_tokens: number }
|
|
if (url.pathname === '/v1/messages/count_tokens') {
|
|
return new Response(JSON.stringify({ input_tokens: 123 }), {
|
|
status: 200,
|
|
headers: { 'content-type': 'application/json' },
|
|
});
|
|
}
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'msg_x',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [{ type: 'text', text: 'ok' }],
|
|
model: 'claude-opus-4-5',
|
|
stop_reason: 'end_turn',
|
|
usage: { input_tokens: 1, output_tokens: 1 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
|
|
const proxy = createProxy({ upstream: 'http://mock', onRequest: () => {} });
|
|
const res = await proxy(
|
|
new Request('http://proxy/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
// No cache_control markers in SAMPLE_REQ_BODY → second probe is skipped.
|
|
// count_tokens hit exactly once, no other side paths.
|
|
expect(sidePaths).toEqual(['/v1/messages/count_tokens']);
|
|
});
|
|
|
|
// When the request body carries any `cache_control` marker, the proxy
|
|
// fires a SECOND count_tokens probe on the body truncated at the last
|
|
// marker. The difference between the two probe results is the
|
|
// cacheable-prefix vs cold-tail split that lets the dashboard compute
|
|
// a cache-aware baseline instead of a cold-every-time approximation.
|
|
it('fires a SECOND count_tokens probe when body has cache_control markers', async () => {
|
|
const bodiesSeen: string[] = [];
|
|
const restore = mockUpstream(async (req) => {
|
|
const url = new URL(req.url);
|
|
if (url.pathname === '/v1/messages/count_tokens') {
|
|
bodiesSeen.push(await req.text());
|
|
// Full body returns N; truncated returns M < N. The proxy doesn't
|
|
// care which response goes to which probe — it just attaches both.
|
|
const len = bodiesSeen.length;
|
|
return new Response(
|
|
JSON.stringify({ input_tokens: len === 1 ? 9000 : 6000 }),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
}
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'msg_x',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [{ type: 'text', text: 'ok' }],
|
|
model: 'claude-opus-4-5',
|
|
stop_reason: 'end_turn',
|
|
usage: {
|
|
input_tokens: 1,
|
|
output_tokens: 1,
|
|
cache_creation_input_tokens: 5000,
|
|
cache_read_input_tokens: 0,
|
|
},
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
|
|
// Realistic shape: a long system prompt cached, then the user turn left
|
|
// uncacheable. Marker lives on the LAST system block — that's the
|
|
// canonical "cache everything above this line" layout Claude Code uses.
|
|
const bodyWithMarkers = JSON.stringify({
|
|
model: 'claude-3-5-haiku-latest',
|
|
system: [
|
|
{ type: 'text', text: 'You are helpful.' },
|
|
{
|
|
type: 'text',
|
|
text: 'A long preamble...',
|
|
cache_control: { type: 'ephemeral' },
|
|
},
|
|
],
|
|
messages: [{ role: 'user', content: 'hi' }],
|
|
});
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
upstream: 'http://mock',
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
const res = await proxy(
|
|
new Request('http://proxy/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: bodyWithMarkers,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 30));
|
|
restore();
|
|
|
|
// Two probes fired (parallel, order indeterminate). At least one of the
|
|
// posted bodies must DIFFER from the other — the second probe is the
|
|
// truncated prefix, not a duplicate of the first.
|
|
expect(bodiesSeen).toHaveLength(2);
|
|
expect(bodiesSeen[0]).not.toBe(bodiesSeen[1]);
|
|
|
|
// Both numbers landed on info; baselineCacheableTokens is the smaller one
|
|
// (truncated body has fewer tokens than the full body). Whichever probe
|
|
// got the 9000 response is `baselineTokens`; the 6000 is `baselineCacheableTokens`.
|
|
expect(captured!.info?.baselineTokens).toBeDefined();
|
|
expect(captured!.info?.baselineCacheableTokens).toBeDefined();
|
|
const full = captured!.info!.baselineTokens!;
|
|
const cacheable = captured!.info!.baselineCacheableTokens!;
|
|
expect(new Set([full, cacheable])).toEqual(new Set([9000, 6000]));
|
|
});
|
|
|
|
// The two probes are independent. If the cacheable-prefix probe 4xx's
|
|
// (e.g. upstream rejects the synthesized sentinel message), the main
|
|
// forward succeeds and the FULL probe's baseline still lands. The
|
|
// dashboard's per-event math degrades cleanly to cold_tail = baseline.
|
|
it('survives cacheable-prefix probe failure without losing the full-body baseline', async () => {
|
|
let probeCount = 0;
|
|
const restore = mockUpstream((req) => {
|
|
const url = new URL(req.url);
|
|
if (url.pathname === '/v1/messages/count_tokens') {
|
|
probeCount += 1;
|
|
// First probe (full body) succeeds; second (truncated) fails.
|
|
// The proxy fires them in parallel so order matters — assume the
|
|
// longer body arrives first because it's queued first.
|
|
if (probeCount === 1) {
|
|
return new Response(JSON.stringify({ input_tokens: 7777 }), {
|
|
status: 200,
|
|
headers: { 'content-type': 'application/json' },
|
|
});
|
|
}
|
|
return new Response(JSON.stringify({ error: 'bad' }), {
|
|
status: 400,
|
|
headers: { 'content-type': 'application/json' },
|
|
});
|
|
}
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'msg_x',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [{ type: 'text', text: 'ok' }],
|
|
model: 'claude-opus-4-5',
|
|
stop_reason: 'end_turn',
|
|
usage: { input_tokens: 1, output_tokens: 1 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
|
|
const bodyWithMarkers = JSON.stringify({
|
|
model: 'claude-3-5-haiku-latest',
|
|
system: [
|
|
{
|
|
type: 'text',
|
|
text: 'preamble',
|
|
cache_control: { type: 'ephemeral' },
|
|
},
|
|
],
|
|
messages: [{ role: 'user', content: 'hi' }],
|
|
});
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
upstream: 'http://mock',
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
const res = await proxy(
|
|
new Request('http://proxy/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: bodyWithMarkers,
|
|
}),
|
|
);
|
|
expect(res.status).toBe(200);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 30));
|
|
restore();
|
|
|
|
expect(probeCount).toBe(2);
|
|
// Whichever probe got the success response must have landed. Both
|
|
// succeed → both land. One fails → only the other lands. The contract
|
|
// we care about: ONE failure doesn't poison the OTHER.
|
|
// Probe order is parallel + non-deterministic in mock-fetch land,
|
|
// so just assert that at least one of the two baseline fields is set.
|
|
const haveFull = captured!.info?.baselineTokens !== undefined;
|
|
const haveCacheable = captured!.info?.baselineCacheableTokens !== undefined;
|
|
expect(haveFull || haveCacheable).toBe(true);
|
|
});
|
|
|
|
// baselineTokens from the count_tokens probe must land on info so the
|
|
// dashboard can roll it into the saved% denominator. This is the wiring
|
|
// that makes the headline number real instead of estimated.
|
|
it('attaches baselineTokens from count_tokens probe to info', async () => {
|
|
const restore = mockUpstream((req) => {
|
|
const url = new URL(req.url);
|
|
if (url.pathname === '/v1/messages/count_tokens') {
|
|
return new Response(JSON.stringify({ input_tokens: 4242 }), {
|
|
status: 200,
|
|
headers: { 'content-type': 'application/json' },
|
|
});
|
|
}
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'msg_x',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [{ type: 'text', text: 'ok' }],
|
|
model: 'claude-opus-4-5',
|
|
stop_reason: 'end_turn',
|
|
usage: { input_tokens: 1, output_tokens: 1 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
upstream: 'http://mock',
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
const res = await proxy(
|
|
new Request('http://proxy/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.info?.baselineTokens).toBe(4242);
|
|
});
|
|
|
|
// count_tokens is best-effort. If the probe 4xx's (e.g. upstream rejects
|
|
// a malformed model field, or the field-whitelist drops something the
|
|
// user added), the main /v1/messages forward must still succeed and the
|
|
// dashboard event just won't carry a baselineTokens. No exception thrown
|
|
// to the caller.
|
|
it('survives count_tokens failure without breaking /v1/messages', async () => {
|
|
const restore = mockUpstream((req) => {
|
|
const url = new URL(req.url);
|
|
if (url.pathname === '/v1/messages/count_tokens') {
|
|
return new Response(JSON.stringify({ error: 'bad model' }), {
|
|
status: 400,
|
|
headers: { 'content-type': 'application/json' },
|
|
});
|
|
}
|
|
return new Response(
|
|
JSON.stringify({
|
|
id: 'msg_x',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [{ type: 'text', text: 'ok' }],
|
|
model: 'claude-opus-4-5',
|
|
stop_reason: 'end_turn',
|
|
usage: { input_tokens: 1, output_tokens: 1 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
);
|
|
});
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
upstream: 'http://mock',
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
const res = await proxy(
|
|
new Request('http://proxy/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
expect(res.status).toBe(200);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.info?.baselineTokens).toBeUndefined();
|
|
});
|
|
|
|
// ---- Ground-truth output measurement (Task #22) ----------------------
|
|
//
|
|
// The proxy scans the response stream for `text_delta` / `thinking_delta`
|
|
// chars and `redacted_thinking` block counts. These numbers are
|
|
// INDEPENDENT of Anthropic's `usage.output_tokens` — they're a raw ruler
|
|
// against the redacted_thinking-inflated bill we surfaced in the May-2026
|
|
// weekly-meter audit. The dashboard layer turns them into low/mid/high
|
|
// bands; the proxy layer just has to count honestly.
|
|
|
|
it('measures SSE text_delta chars across multiple delta events', async () => {
|
|
// Three text_delta events spanning a couple of code points each — the
|
|
// ruler must use STRING length (UTF-16 code units), matching what
|
|
// `JSON.stringify(text).length` would count if we re-serialized.
|
|
const sseBody =
|
|
'event: message_start\n' +
|
|
`data: ${JSON.stringify({
|
|
type: 'message_start',
|
|
message: {
|
|
id: 'msg_m1', type: 'message', role: 'assistant', content: [],
|
|
usage: { input_tokens: 10, output_tokens: 1 },
|
|
},
|
|
})}\n\n` +
|
|
'event: content_block_start\n' +
|
|
'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello "}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"world"}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"!"}}\n\n' +
|
|
'event: message_delta\n' +
|
|
'data: {"type":"message_delta","delta":{},"usage":{"output_tokens":42}}\n\n' +
|
|
'event: message_stop\n' +
|
|
'data: {"type":"message_stop"}\n\n';
|
|
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(sseBody, {
|
|
status: 200,
|
|
headers: { 'content-type': 'text/event-stream' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
// 'hello ' (6) + 'world' (5) + '!' (1) = 12 chars.
|
|
expect(captured!.measurement?.textChars).toBe(12);
|
|
expect(captured!.measurement?.thinkingChars).toBe(0);
|
|
expect(captured!.measurement?.toolUseChars).toBe(0);
|
|
expect(captured!.measurement?.redactedBlockCount).toBe(0);
|
|
// Final output_tokens from message_delta overrides message_start's 1.
|
|
expect(captured!.usage?.output_tokens).toBe(42);
|
|
});
|
|
|
|
it('measures SSE thinking_delta chars and counts redacted_thinking blocks', async () => {
|
|
// Extended thinking turn: a `thinking` block and a `redacted_thinking`
|
|
// block. The redacted block has no readable chars (server-encrypted
|
|
// bytes), so we just count the block — the dashboard surfaces it as
|
|
// an opaque low/mid/high estimate.
|
|
const sseBody =
|
|
'event: message_start\n' +
|
|
`data: ${JSON.stringify({
|
|
type: 'message_start',
|
|
message: {
|
|
id: 'msg_m2', type: 'message', role: 'assistant', content: [],
|
|
usage: { input_tokens: 100, output_tokens: 1 },
|
|
},
|
|
})}\n\n` +
|
|
'event: content_block_start\n' +
|
|
'data: {"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"step 1: "}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"reason carefully"}}\n\n' +
|
|
'event: content_block_start\n' +
|
|
'data: {"type":"content_block_start","index":1,"content_block":{"type":"redacted_thinking","data":"opaque"}}\n\n' +
|
|
'event: content_block_start\n' +
|
|
'data: {"type":"content_block_start","index":2,"content_block":{"type":"redacted_thinking","data":"alsoopaque"}}\n\n' +
|
|
'event: message_delta\n' +
|
|
'data: {"type":"message_delta","delta":{},"usage":{"output_tokens":500}}\n\n' +
|
|
'event: message_stop\n' +
|
|
'data: {"type":"message_stop"}\n\n';
|
|
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(sseBody, {
|
|
status: 200,
|
|
headers: { 'content-type': 'text/event-stream' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
// 'step 1: ' (8) + 'reason carefully' (16) = 24 chars.
|
|
expect(captured!.measurement?.thinkingChars).toBe(24);
|
|
expect(captured!.measurement?.textChars).toBe(0);
|
|
expect(captured!.measurement?.redactedBlockCount).toBe(2);
|
|
expect(captured!.usage?.output_tokens).toBe(500);
|
|
});
|
|
|
|
it('measures SSE tool_use chars via input_json_delta', async () => {
|
|
// tool_use blocks stream their `input` field as a JSON string assembled
|
|
// from `input_json_delta` events. We count the raw JSON-string length —
|
|
// that's the closest apples-to-apples we get against the billed body.
|
|
const sseBody =
|
|
'event: message_start\n' +
|
|
`data: ${JSON.stringify({
|
|
type: 'message_start',
|
|
message: {
|
|
id: 'msg_m3', type: 'message', role: 'assistant', content: [],
|
|
usage: { input_tokens: 50, output_tokens: 1 },
|
|
},
|
|
})}\n\n` +
|
|
'event: content_block_start\n' +
|
|
'data: {"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"t1","name":"bash","input":{}}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\\"cmd\\":"}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"\\"ls\\"}"}}\n\n' +
|
|
'event: message_delta\n' +
|
|
'data: {"type":"message_delta","delta":{},"usage":{"output_tokens":20}}\n\n' +
|
|
'event: message_stop\n' +
|
|
'data: {"type":"message_stop"}\n\n';
|
|
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(sseBody, {
|
|
status: 200,
|
|
headers: { 'content-type': 'text/event-stream' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
// '{"cmd":' (7) + '"ls"}' (5) = 12 chars.
|
|
expect(captured!.measurement?.toolUseChars).toBe(12);
|
|
expect(captured!.measurement?.textChars).toBe(0);
|
|
expect(captured!.measurement?.thinkingChars).toBe(0);
|
|
});
|
|
|
|
it('measures non-stream JSON response by walking content[]', async () => {
|
|
// Non-stream path: the whole body is one JSON object. Counter walks
|
|
// content[] and adds up text/thinking chars, tool_use input chars, and
|
|
// redacted_thinking blocks. Same shape as the SSE accumulator — the
|
|
// ruler must report the SAME numbers regardless of transport.
|
|
const responseBody = JSON.stringify({
|
|
id: 'msg_n1',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [
|
|
{ type: 'text', text: 'one two three' },
|
|
{ type: 'thinking', thinking: 'reasoning here' },
|
|
{ type: 'redacted_thinking', data: 'opaque1' },
|
|
{ type: 'tool_use', id: 't1', name: 'bash', input: { cmd: 'ls' } },
|
|
{ type: 'text', text: '!!' },
|
|
],
|
|
model: 'claude-opus-4-5',
|
|
stop_reason: 'end_turn',
|
|
usage: { input_tokens: 5, output_tokens: 99 },
|
|
});
|
|
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(responseBody, {
|
|
status: 200,
|
|
headers: { 'content-type': 'application/json' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
// 'one two three' (13) + '!!' (2) = 15 chars text.
|
|
expect(captured!.measurement?.textChars).toBe(15);
|
|
// 'reasoning here' = 14.
|
|
expect(captured!.measurement?.thinkingChars).toBe(14);
|
|
// tool_use input JSON.stringify({cmd:'ls'}) = '{"cmd":"ls"}' = 12 chars.
|
|
expect(captured!.measurement?.toolUseChars).toBe(12);
|
|
expect(captured!.measurement?.redactedBlockCount).toBe(1);
|
|
});
|
|
|
|
it('leaves measurement undefined on 5xx (no body to scan)', async () => {
|
|
// Upstream 5xx bails on usage AND measurement — the host synthesizes
|
|
// an error message and the body is whatever Anthropic returned, which
|
|
// by convention we don't try to parse. The dashboard event will just
|
|
// skip the row from output-honesty math.
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response('upstream broke', {
|
|
status: 503,
|
|
headers: { 'content-type': 'text/plain' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.status).toBe(503);
|
|
expect(captured!.measurement).toBeUndefined();
|
|
});
|
|
|
|
it('handles message_start with no usage gracefully (still measures content)', async () => {
|
|
// Defensive: if a future Anthropic release ships a message_start
|
|
// without `usage`, the proxy should still scan deltas and report
|
|
// measurement. Only the usage rollup degrades.
|
|
const sseBody =
|
|
'event: message_start\n' +
|
|
'data: {"type":"message_start","message":{"id":"x","type":"message","role":"assistant","content":[]}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hi there"}}\n\n' +
|
|
'event: message_stop\n' +
|
|
'data: {"type":"message_stop"}\n\n';
|
|
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(sseBody, {
|
|
status: 200,
|
|
headers: { 'content-type': 'text/event-stream' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.measurement?.textChars).toBe(8);
|
|
});
|
|
|
|
it('extracts stop_reason from the SSE message_delta event', async () => {
|
|
const sseBody =
|
|
'event: message_start\n' +
|
|
'data: {"type":"message_start","message":{"id":"x","type":"message","role":"assistant","content":[],"usage":{"input_tokens":10,"output_tokens":1}}}\n\n' +
|
|
'event: content_block_delta\n' +
|
|
'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}\n\n' +
|
|
'event: message_delta\n' +
|
|
'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":9}}\n\n' +
|
|
'event: message_stop\n' +
|
|
'data: {"type":"message_stop"}\n\n';
|
|
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(sseBody, {
|
|
status: 200,
|
|
headers: { 'content-type': 'text/event-stream' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.stopReason).toBe('end_turn');
|
|
// message_delta output_tokens must win over message_start's placeholder 1.
|
|
expect(captured!.usage?.output_tokens).toBe(9);
|
|
});
|
|
|
|
it('extracts stop_reason "refusal" from a non-stream JSON response', async () => {
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(
|
|
JSON.stringify({
|
|
id: 'msg_r',
|
|
type: 'message',
|
|
role: 'assistant',
|
|
content: [],
|
|
stop_reason: 'refusal',
|
|
usage: { input_tokens: 5, output_tokens: 2 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.stopReason).toBe('refusal');
|
|
});
|
|
|
|
it('extracts OpenAI choices[].finish_reason from a JSON body', async () => {
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(
|
|
JSON.stringify({
|
|
id: 'chatcmpl-1',
|
|
object: 'chat.completion',
|
|
choices: [
|
|
{
|
|
index: 0,
|
|
message: { role: 'assistant', content: '' },
|
|
finish_reason: 'content_filter',
|
|
},
|
|
],
|
|
usage: { prompt_tokens: 11, completion_tokens: 3 },
|
|
}),
|
|
{ status: 200, headers: { 'content-type': 'application/json' } },
|
|
),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.stopReason).toBe('content_filter');
|
|
});
|
|
|
|
it('leaves stopReason undefined when the stream never ships one', async () => {
|
|
const sseBody =
|
|
'event: message_start\n' +
|
|
'data: {"type":"message_start","message":{"id":"x","type":"message","role":"assistant","content":[],"usage":{"input_tokens":10,"output_tokens":1}}}\n\n' +
|
|
'event: message_stop\n' +
|
|
'data: {"type":"message_stop"}\n\n';
|
|
|
|
const restore = mockUpstream(
|
|
() =>
|
|
new Response(sseBody, {
|
|
status: 200,
|
|
headers: { 'content-type': 'text/event-stream' },
|
|
}),
|
|
);
|
|
|
|
let captured: ProxyEvent | undefined;
|
|
const proxy = createProxy({
|
|
transform: {},
|
|
onRequest: (e) => { captured = e; },
|
|
});
|
|
|
|
const res = await proxy(
|
|
new Request('http://localhost/v1/messages', {
|
|
method: 'POST',
|
|
headers: { 'content-type': 'application/json' },
|
|
body: SAMPLE_REQ_BODY,
|
|
}),
|
|
);
|
|
await res.text();
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
restore();
|
|
|
|
expect(captured).toBeDefined();
|
|
expect(captured!.stopReason).toBeUndefined();
|
|
});
|
|
});
|