mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
Expose pixelpipe library API
This commit is contained in:
@@ -456,6 +456,44 @@ npx wrangler secret put ANTHROPIC_API_KEY
|
||||
|
||||
If unset, the proxy forwards whatever `x-api-key` the client sent.
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Library API
|
||||
|
||||
Pixelpipe is also published as a runtime-agnostic transform library so another
|
||||
local proxy can own auth/routing while reusing the Opus 4.7 compression and
|
||||
measurement logic directly:
|
||||
|
||||
```ts
|
||||
import {
|
||||
transformAnthropicMessages,
|
||||
buildCountTokensBodies,
|
||||
isPixelpipeSupportedModel,
|
||||
} from "pixelpipe";
|
||||
|
||||
if (isPixelpipeSupportedModel(upstreamModel)) {
|
||||
// Run these probes with the host proxy's own auth/transport.
|
||||
const baseline = buildCountTokensBodies(originalMessagesBody);
|
||||
|
||||
const result = await transformAnthropicMessages({
|
||||
body: originalMessagesBody,
|
||||
model: upstreamModel,
|
||||
});
|
||||
|
||||
// result.body is the body to forward; result.cache.ownsCacheControl tells
|
||||
// the host not to stack a second cache injector on top of pixelpipe.
|
||||
}
|
||||
```
|
||||
|
||||
Stable public exports:
|
||||
|
||||
- `pixelpipe` — library API plus `createProxy` for standalone use.
|
||||
- `pixelpipe/transform` — `transformAnthropicMessages(...)`.
|
||||
- `pixelpipe/measurement` — count-token probe body builders.
|
||||
- `pixelpipe/applicability` — Opus 4.7 applicability helpers.
|
||||
- `pixelpipe/proxy` — standalone Web `fetch` proxy.
|
||||
|
||||
---
|
||||
|
||||
## Architecture
|
||||
@@ -467,6 +505,9 @@ src/
|
||||
│ ├── png.ts minimal grayscale PNG encoder
|
||||
│ ├── render.ts text → PNG bytes
|
||||
│ ├── transform.ts request body rewriter
|
||||
│ ├── library.ts public transform wrapper for host proxies
|
||||
│ ├── measurement.ts count_tokens probe body builders
|
||||
│ ├── applicability.ts Opus 4.7 eligibility helpers
|
||||
│ ├── proxy.ts the fetch handler
|
||||
│ └── types.ts Anthropic API types
|
||||
├── node.ts node:http adapter + CLI
|
||||
|
||||
@@ -8,6 +8,22 @@
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/core/index.d.ts",
|
||||
"import": "./dist/core/index.js"
|
||||
},
|
||||
"./transform": {
|
||||
"types": "./dist/core/library.d.ts",
|
||||
"import": "./dist/core/library.js"
|
||||
},
|
||||
"./measurement": {
|
||||
"types": "./dist/core/measurement.d.ts",
|
||||
"import": "./dist/core/measurement.js"
|
||||
},
|
||||
"./applicability": {
|
||||
"types": "./dist/core/applicability.d.ts",
|
||||
"import": "./dist/core/applicability.js"
|
||||
},
|
||||
"./proxy": {
|
||||
"types": "./dist/core/proxy.d.ts",
|
||||
"import": "./dist/core/proxy.js"
|
||||
},
|
||||
|
||||
+15
-4
@@ -1,11 +1,22 @@
|
||||
// Build the Node bundle. The Worker target is built by wrangler directly
|
||||
// from src/worker.ts (no separate build step needed).
|
||||
// Build library ESM + declarations with tsc, then overwrite the Node CLI
|
||||
// entry with a bundled executable. The Worker target can still be built by
|
||||
// wrangler directly from src/worker.ts, but dist/worker.js is also emitted for
|
||||
// package consumers via tsc.
|
||||
import { build } from 'esbuild';
|
||||
import { mkdir, copyFile } from 'node:fs/promises';
|
||||
import { mkdir, rm } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const OUT = 'dist';
|
||||
if (!existsSync(OUT)) await mkdir(OUT, { recursive: true });
|
||||
if (existsSync(OUT)) await rm(OUT, { recursive: true, force: true });
|
||||
await mkdir(OUT, { recursive: true });
|
||||
|
||||
const tsc = spawnSync('pnpm', ['exec', 'tsc', '-p', 'tsconfig.json'], {
|
||||
stdio: 'inherit',
|
||||
shell: false,
|
||||
});
|
||||
if (tsc.status !== 0) process.exit(tsc.status ?? 1);
|
||||
console.log('✓ emitted dist/ library modules + declarations');
|
||||
|
||||
await build({
|
||||
entryPoints: ['src/node.ts'],
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/** Applicability helpers for pixelpipe's production-safe model scope. */
|
||||
|
||||
export type PixelpipeApplicabilityReason =
|
||||
| 'eligible'
|
||||
| 'unsupported_model'
|
||||
| 'unsupported_method'
|
||||
| 'unsupported_path'
|
||||
| 'empty_body';
|
||||
|
||||
export interface PixelpipeApplicabilityInput {
|
||||
readonly model?: string | null;
|
||||
readonly method?: string | null;
|
||||
readonly path?: string | null;
|
||||
readonly bodyBytes?: number | null;
|
||||
}
|
||||
|
||||
/** Pixelpipe is a product path for Opus 4.7 only. Suffix aliases such as
|
||||
* `claude-opus-4-7-high` are accepted because hosts may check either the
|
||||
* client alias or resolved upstream model. */
|
||||
export function isPixelpipeSupportedModel(model: string | null | undefined): boolean {
|
||||
return typeof model === 'string' && /^claude-opus-4-7(?:-|$)/.test(model);
|
||||
}
|
||||
|
||||
export function shouldTransformAnthropicMessages(
|
||||
input: PixelpipeApplicabilityInput,
|
||||
): { eligible: boolean; reason: PixelpipeApplicabilityReason } {
|
||||
if (input.method !== undefined && input.method !== null && input.method.toUpperCase() !== 'POST') {
|
||||
return { eligible: false, reason: 'unsupported_method' };
|
||||
}
|
||||
if (input.path !== undefined && input.path !== null && !input.path.endsWith('/v1/messages')) {
|
||||
return { eligible: false, reason: 'unsupported_path' };
|
||||
}
|
||||
if (input.bodyBytes !== undefined && input.bodyBytes !== null && input.bodyBytes <= 0) {
|
||||
return { eligible: false, reason: 'empty_body' };
|
||||
}
|
||||
if (!isPixelpipeSupportedModel(input.model)) {
|
||||
return { eligible: false, reason: 'unsupported_model' };
|
||||
}
|
||||
return { eligible: true, reason: 'eligible' };
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
export {
|
||||
isPixelpipeSupportedModel,
|
||||
shouldTransformAnthropicMessages,
|
||||
type PixelpipeApplicabilityInput,
|
||||
type PixelpipeApplicabilityReason,
|
||||
} from './applicability.js';
|
||||
export {
|
||||
buildCountTokensBodies,
|
||||
buildBaselineCountTokensBody,
|
||||
buildCacheablePrefixCountTokensBody,
|
||||
countCacheControlMarkers,
|
||||
type CountTokensBodies,
|
||||
} from './measurement.js';
|
||||
export {
|
||||
transformAnthropicMessages,
|
||||
type PixelpipeOptions,
|
||||
type PixelpipeReason,
|
||||
type PixelpipeTransformInput,
|
||||
type PixelpipeTransformResult,
|
||||
} from './library.js';
|
||||
export {
|
||||
transformRequest,
|
||||
type TransformInfo as PixelpipeTransformInfo,
|
||||
type TransformOptions,
|
||||
} from './transform.js';
|
||||
export { createProxy, type ProxyConfig, type ProxyEvent } from './proxy.js';
|
||||
export {
|
||||
computeActualInputEff,
|
||||
computeBaselineInputEff,
|
||||
CACHE_CREATE_RATE,
|
||||
CACHE_READ_RATE,
|
||||
} from './baseline.js';
|
||||
@@ -0,0 +1,121 @@
|
||||
import { isPixelpipeSupportedModel } from './applicability.js';
|
||||
import { countCacheControlMarkers } from './measurement.js';
|
||||
import { transformRequest, type TransformInfo, type TransformOptions } from './transform.js';
|
||||
|
||||
export type BytesLike = Uint8Array | ArrayBuffer | ArrayBufferView;
|
||||
|
||||
export interface PixelpipeOptions extends Pick<TransformOptions, 'charsPerToken'> {
|
||||
/** Test/debug-only bypass. Product hosts should prefer their dashboard setting. */
|
||||
readonly compress?: boolean;
|
||||
}
|
||||
|
||||
export interface PixelpipeTransformInput {
|
||||
readonly body: BytesLike;
|
||||
/** Resolved upstream model when available; aliases are accepted for applicability checks. */
|
||||
readonly model?: string | null;
|
||||
readonly requestId?: string;
|
||||
readonly options?: PixelpipeOptions;
|
||||
}
|
||||
|
||||
export type PixelpipeReason =
|
||||
| 'applied'
|
||||
| 'unsupported_model'
|
||||
| 'parse_error'
|
||||
| 'below_min_chars'
|
||||
| 'not_profitable'
|
||||
| 'compress_disabled'
|
||||
| 'image_limit'
|
||||
| 'transform_error'
|
||||
| 'passthrough';
|
||||
|
||||
export interface PixelpipeTransformResult {
|
||||
readonly body: Uint8Array;
|
||||
readonly applied: boolean;
|
||||
readonly reason: PixelpipeReason;
|
||||
readonly detail?: string;
|
||||
readonly info: TransformInfo;
|
||||
readonly cache: {
|
||||
readonly ownsCacheControl: boolean;
|
||||
readonly markerCount: number;
|
||||
};
|
||||
}
|
||||
|
||||
function toUint8Array(bytes: BytesLike): Uint8Array {
|
||||
if (bytes instanceof Uint8Array) return bytes;
|
||||
if (bytes instanceof ArrayBuffer) return new Uint8Array(bytes);
|
||||
return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
}
|
||||
|
||||
function emptyInfo(reason: string): TransformInfo {
|
||||
return {
|
||||
compressed: false,
|
||||
reason,
|
||||
origChars: 0,
|
||||
compressedChars: 0,
|
||||
imageCount: 0,
|
||||
imageBytes: 0,
|
||||
staticChars: 0,
|
||||
dynamicChars: 0,
|
||||
dynamicBlockCount: 0,
|
||||
droppedChars: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function classifyReason(info: TransformInfo): PixelpipeReason {
|
||||
if (info.compressed) return 'applied';
|
||||
const r = info.reason ?? '';
|
||||
if (r.startsWith('parse_error')) return 'parse_error';
|
||||
if (r.startsWith('compress=false')) return 'compress_disabled';
|
||||
if (r.startsWith('below_min_chars')) return 'below_min_chars';
|
||||
if (r.startsWith('not_profitable')) return 'not_profitable';
|
||||
if (r.includes('image') && r.includes('limit')) return 'image_limit';
|
||||
return 'passthrough';
|
||||
}
|
||||
|
||||
/**
|
||||
* Library-first wrapper around pixelpipe's Anthropic Messages transformer.
|
||||
* It performs the Opus-4.7-only model gate, returns machine-readable reasons,
|
||||
* and reports cache_control ownership so hosts such as ocproxy do not stack a
|
||||
* second cache injector on top of pixelpipe's image breakpoint.
|
||||
*/
|
||||
export async function transformAnthropicMessages(
|
||||
input: PixelpipeTransformInput,
|
||||
): Promise<PixelpipeTransformResult> {
|
||||
const original = toUint8Array(input.body);
|
||||
if (!isPixelpipeSupportedModel(input.model)) {
|
||||
return {
|
||||
body: original,
|
||||
applied: false,
|
||||
reason: 'unsupported_model',
|
||||
detail: input.model ?? undefined,
|
||||
info: emptyInfo('unsupported_model'),
|
||||
cache: { ownsCacheControl: false, markerCount: countCacheControlMarkers(original) },
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const { body, info } = await transformRequest(original, input.options);
|
||||
const reason = classifyReason(info);
|
||||
const markerCount = countCacheControlMarkers(body);
|
||||
return {
|
||||
body,
|
||||
applied: info.compressed,
|
||||
reason,
|
||||
detail: info.reason,
|
||||
info,
|
||||
cache: {
|
||||
ownsCacheControl: info.compressed && markerCount > 0,
|
||||
markerCount,
|
||||
},
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
body: original,
|
||||
applied: false,
|
||||
reason: 'transform_error',
|
||||
detail: e instanceof Error ? e.message : String(e),
|
||||
info: emptyInfo(`transform_error: ${e instanceof Error ? e.message : String(e)}`),
|
||||
cache: { ownsCacheControl: false, markerCount: countCacheControlMarkers(original) },
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
/**
|
||||
* Helpers for measuring the uncompressed Anthropic Messages counterfactual.
|
||||
*
|
||||
* These are pure body-shaping utilities: no fetch, no auth, no Node APIs.
|
||||
* Hosts such as ocproxy can use them to run shadow /v1/messages/count_tokens
|
||||
* probes with their own transport/auth while pixelpipe transforms the real
|
||||
* request body.
|
||||
*/
|
||||
|
||||
export interface CountTokensBodies {
|
||||
/** Full original body, filtered to the strict count_tokens parameter set. */
|
||||
readonly fullBody: Uint8Array | null;
|
||||
/** Original body truncated at the latest cache_control marker, or null when no marker exists. */
|
||||
readonly cacheablePrefixBody: Uint8Array | null;
|
||||
}
|
||||
|
||||
/** /v1/messages/count_tokens accepts a strict subset of /v1/messages params.
|
||||
* Anything else (`stream`, `max_tokens`, `temperature`, `top_p`, `top_k`,
|
||||
* `stop_sequences`, `metadata`, `service_tier`) makes it 400 with
|
||||
* "Unknown parameter". Strip the verbatim body to the accepted fields.
|
||||
* Returns null if the body can't be parsed or is missing required fields. */
|
||||
const COUNT_TOKENS_FIELDS = new Set([
|
||||
'model',
|
||||
'messages',
|
||||
'system',
|
||||
'tools',
|
||||
'tool_choice',
|
||||
'thinking',
|
||||
'mcp_servers',
|
||||
]);
|
||||
|
||||
type BytesLike = Uint8Array | ArrayBuffer | ArrayBufferView;
|
||||
|
||||
function toUint8Array(bytes: BytesLike): Uint8Array {
|
||||
if (bytes instanceof Uint8Array) return bytes;
|
||||
if (bytes instanceof ArrayBuffer) return new Uint8Array(bytes);
|
||||
return new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
}
|
||||
|
||||
export function buildCountTokensBodies(bytes: BytesLike): CountTokensBodies {
|
||||
const b = toUint8Array(bytes);
|
||||
return {
|
||||
fullBody: buildBaselineCountTokensBody(b),
|
||||
cacheablePrefixBody: buildCacheablePrefixCountTokensBody(b),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildBaselineCountTokensBody(bytes: BytesLike): Uint8Array | null {
|
||||
const b = toUint8Array(bytes);
|
||||
try {
|
||||
const obj = JSON.parse(new TextDecoder().decode(b)) as Record<string, unknown>;
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(obj)) {
|
||||
if (COUNT_TOKENS_FIELDS.has(k)) out[k] = obj[k];
|
||||
}
|
||||
if (typeof out.model !== 'string' || !Array.isArray(out.messages)) return null;
|
||||
return new TextEncoder().encode(JSON.stringify(out));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Type guard for content blocks that may carry a cache_control marker.
|
||||
* Anthropic accepts `cache_control` on: any system block, any message
|
||||
* content block, any tool definition. We don't care about the marker's
|
||||
* value — only its presence/position. */
|
||||
function hasCacheControl(x: unknown): boolean {
|
||||
return (
|
||||
typeof x === 'object'
|
||||
&& x !== null
|
||||
&& (x as { cache_control?: unknown }).cache_control != null
|
||||
);
|
||||
}
|
||||
|
||||
/** Build a body that contains EXACTLY the tokens forming the longest
|
||||
* cacheable prefix on the unproxied path — everything up to and INCLUDING
|
||||
* the last `cache_control` marker in the original request, with everything
|
||||
* after that marker stripped. count_tokens on this body returns
|
||||
* `cacheable_prefix_tokens`; subtracting from the full count_tokens gives
|
||||
* `cold_tail_tokens` (always-cold input on both proxied and unproxied paths).
|
||||
*
|
||||
* Anthropic's cache-traversal order is tools → system → messages. We walk
|
||||
* messages first (latest), then system, then tools — the FIRST one we find
|
||||
* a marker in (walking backward) is the latest in cache order. Everything
|
||||
* after that marker in the same section is dropped; later sections are
|
||||
* dropped wholesale.
|
||||
*
|
||||
* Returns null when the original body has zero `cache_control` markers
|
||||
* anywhere — caller treats that as `cacheable_prefix_tokens = 0`. */
|
||||
export function buildCacheablePrefixCountTokensBody(bytes: BytesLike): Uint8Array | null {
|
||||
const b = toUint8Array(bytes);
|
||||
let obj: Record<string, unknown>;
|
||||
try {
|
||||
obj = JSON.parse(new TextDecoder().decode(b)) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (typeof obj.model !== 'string') return null;
|
||||
|
||||
const system = obj.system;
|
||||
const messages = obj.messages;
|
||||
const tools = obj.tools;
|
||||
|
||||
let truncated: Record<string, unknown> | null = null;
|
||||
if (Array.isArray(messages)) {
|
||||
for (let mi = messages.length - 1; mi >= 0 && truncated == null; mi--) {
|
||||
const msg = messages[mi] as { role?: unknown; content?: unknown };
|
||||
const content = msg?.content;
|
||||
if (Array.isArray(content)) {
|
||||
for (let bi = content.length - 1; bi >= 0; bi--) {
|
||||
if (hasCacheControl(content[bi])) {
|
||||
const truncatedMsg = { ...msg, content: content.slice(0, bi + 1) };
|
||||
const truncatedMessages = messages.slice(0, mi).concat([truncatedMsg]);
|
||||
truncated = {
|
||||
model: obj.model,
|
||||
messages: truncatedMessages,
|
||||
};
|
||||
if (system !== undefined) truncated.system = system;
|
||||
if (tools !== undefined) truncated.tools = tools;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (hasCacheControl(msg)) {
|
||||
truncated = {
|
||||
model: obj.model,
|
||||
messages: messages.slice(0, mi + 1),
|
||||
};
|
||||
if (system !== undefined) truncated.system = system;
|
||||
if (tools !== undefined) truncated.tools = tools;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (truncated == null && Array.isArray(system)) {
|
||||
for (let si = system.length - 1; si >= 0; si--) {
|
||||
if (hasCacheControl(system[si])) {
|
||||
truncated = {
|
||||
model: obj.model,
|
||||
system: system.slice(0, si + 1),
|
||||
messages: [{ role: 'user', content: 'x' }],
|
||||
};
|
||||
if (tools !== undefined) truncated.tools = tools;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (truncated == null && Array.isArray(tools)) {
|
||||
for (let ti = tools.length - 1; ti >= 0; ti--) {
|
||||
if (hasCacheControl(tools[ti])) {
|
||||
truncated = {
|
||||
model: obj.model,
|
||||
tools: tools.slice(0, ti + 1),
|
||||
messages: [{ role: 'user', content: 'x' }],
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (truncated == null) return null;
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(truncated)) {
|
||||
if (COUNT_TOKENS_FIELDS.has(k)) out[k] = truncated[k];
|
||||
}
|
||||
return new TextEncoder().encode(JSON.stringify(out));
|
||||
}
|
||||
|
||||
/** Count cache_control markers anywhere in a JSON-ish Anthropic Messages body. */
|
||||
export function countCacheControlMarkers(bytes: BytesLike): number {
|
||||
const b = toUint8Array(bytes);
|
||||
try {
|
||||
return countCacheControlValue(JSON.parse(new TextDecoder().decode(b)));
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function countCacheControlValue(value: unknown): number {
|
||||
if (!value || typeof value !== 'object') return 0;
|
||||
let n = hasCacheControl(value) ? 1 : 0;
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) n += countCacheControlValue(item);
|
||||
} else {
|
||||
for (const item of Object.values(value as Record<string, unknown>)) {
|
||||
n += countCacheControlValue(item);
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
+6
-152
@@ -8,6 +8,10 @@
|
||||
*/
|
||||
|
||||
import { transformRequest, type TransformOptions, type TransformInfo } from './transform.js';
|
||||
import {
|
||||
buildBaselineCountTokensBody,
|
||||
buildCacheablePrefixCountTokensBody,
|
||||
} from './measurement.js';
|
||||
import type { Usage } from './types.js';
|
||||
|
||||
export interface ProxyConfig {
|
||||
@@ -405,156 +409,6 @@ function filterHeaders(src: Headers, strip: Set<string>): Headers {
|
||||
return out;
|
||||
}
|
||||
|
||||
/** /v1/messages/count_tokens accepts a strict subset of /v1/messages params.
|
||||
* Anything else (`stream`, `max_tokens`, `temperature`, `top_p`, `top_k`,
|
||||
* `stop_sequences`, `metadata`, `service_tier`) makes it 400 with
|
||||
* "Unknown parameter". Strip the verbatim body to the accepted fields.
|
||||
* Returns null if the body can't be parsed or is missing required fields
|
||||
* (probe is skipped, baseline_tokens stays absent on the event). */
|
||||
const COUNT_TOKENS_FIELDS = new Set([
|
||||
'model',
|
||||
'messages',
|
||||
'system',
|
||||
'tools',
|
||||
'tool_choice',
|
||||
'thinking',
|
||||
'mcp_servers',
|
||||
]);
|
||||
|
||||
function buildCountTokensBody(bytes: Uint8Array): Uint8Array | null {
|
||||
try {
|
||||
const obj = JSON.parse(new TextDecoder().decode(bytes)) as Record<string, unknown>;
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(obj)) {
|
||||
if (COUNT_TOKENS_FIELDS.has(k)) out[k] = obj[k];
|
||||
}
|
||||
if (typeof out.model !== 'string' || !Array.isArray(out.messages)) return null;
|
||||
return new TextEncoder().encode(JSON.stringify(out));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Type guard for content blocks that may carry a cache_control marker.
|
||||
* Anthropic accepts `cache_control` on: any system block, any message
|
||||
* content block, any tool definition. We don't care about the marker's
|
||||
* value — only its presence/position. */
|
||||
function hasCacheControl(x: unknown): boolean {
|
||||
return (
|
||||
typeof x === 'object'
|
||||
&& x !== null
|
||||
&& (x as { cache_control?: unknown }).cache_control != null
|
||||
);
|
||||
}
|
||||
|
||||
/** Build a body that contains EXACTLY the tokens forming the longest
|
||||
* cacheable prefix on the unproxied path — everything up to and INCLUDING
|
||||
* the last `cache_control` marker in the original request, with everything
|
||||
* after that marker stripped. count_tokens on this body returns
|
||||
* `cacheable_prefix_tokens`; subtracting from the full count_tokens gives
|
||||
* `cold_tail_tokens` (always-cold input on both proxied and unproxied paths).
|
||||
*
|
||||
* Anthropic's cache-traversal order is tools → system → messages. We walk
|
||||
* messages first (latest), then system, then tools — the FIRST one we find
|
||||
* a marker in (walking backward) is the latest in cache order. Everything
|
||||
* after that marker in the same section is dropped; later sections are
|
||||
* dropped wholesale.
|
||||
*
|
||||
* Returns null when the original body has zero `cache_control` markers
|
||||
* anywhere — caller treats that as `cacheable_prefix_tokens = 0`. The
|
||||
* unproxied path doesn't cache anything in that case, so the full body
|
||||
* is cold input. */
|
||||
function buildCountTokensCacheablePrefix(bytes: Uint8Array): Uint8Array | null {
|
||||
let obj: Record<string, unknown>;
|
||||
try {
|
||||
obj = JSON.parse(new TextDecoder().decode(bytes)) as Record<string, unknown>;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (typeof obj.model !== 'string') return null;
|
||||
|
||||
const system = obj.system;
|
||||
const messages = obj.messages;
|
||||
const tools = obj.tools;
|
||||
|
||||
// Walk messages backward; for each message walk content blocks backward.
|
||||
// First block we hit with cache_control is the latest marker in the
|
||||
// entire request (messages come last in cache traversal order).
|
||||
let truncated: Record<string, unknown> | null = null;
|
||||
if (Array.isArray(messages)) {
|
||||
for (let mi = messages.length - 1; mi >= 0 && truncated == null; mi--) {
|
||||
const msg = messages[mi] as { role?: unknown; content?: unknown };
|
||||
const content = msg?.content;
|
||||
if (Array.isArray(content)) {
|
||||
for (let bi = content.length - 1; bi >= 0; bi--) {
|
||||
if (hasCacheControl(content[bi])) {
|
||||
// Truncate this message's content at and including the marker.
|
||||
const truncatedMsg = { ...msg, content: content.slice(0, bi + 1) };
|
||||
const truncatedMessages = messages.slice(0, mi).concat([truncatedMsg]);
|
||||
truncated = {
|
||||
model: obj.model,
|
||||
messages: truncatedMessages,
|
||||
};
|
||||
if (system !== undefined) truncated.system = system;
|
||||
if (tools !== undefined) truncated.tools = tools;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else if (hasCacheControl(msg)) {
|
||||
// String-content message with a marker — keep up to this message.
|
||||
truncated = {
|
||||
model: obj.model,
|
||||
messages: messages.slice(0, mi + 1),
|
||||
};
|
||||
if (system !== undefined) truncated.system = system;
|
||||
if (tools !== undefined) truncated.tools = tools;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No marker in messages — check system (next-earliest in cache order).
|
||||
// count_tokens requires non-empty messages, so we append a 1-token sentinel
|
||||
// user message and subtract its weight on the consumer side. Or simpler:
|
||||
// include a single-char user message and accept ~1 token of noise.
|
||||
if (truncated == null && Array.isArray(system)) {
|
||||
for (let si = system.length - 1; si >= 0; si--) {
|
||||
if (hasCacheControl(system[si])) {
|
||||
truncated = {
|
||||
model: obj.model,
|
||||
system: system.slice(0, si + 1),
|
||||
messages: [{ role: 'user', content: 'x' }],
|
||||
};
|
||||
if (tools !== undefined) truncated.tools = tools;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No marker in system — check tools (earliest in cache order). Same
|
||||
// sentinel-message trick to keep count_tokens happy.
|
||||
if (truncated == null && Array.isArray(tools)) {
|
||||
for (let ti = tools.length - 1; ti >= 0; ti--) {
|
||||
if (hasCacheControl(tools[ti])) {
|
||||
truncated = {
|
||||
model: obj.model,
|
||||
tools: tools.slice(0, ti + 1),
|
||||
messages: [{ role: 'user', content: 'x' }],
|
||||
};
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Filter through the same whitelist as the full-body probe to avoid 400s
|
||||
// from stray fields slipping into the truncated copy.
|
||||
if (truncated == null) return null;
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(truncated)) {
|
||||
if (COUNT_TOKENS_FIELDS.has(k)) out[k] = truncated[k];
|
||||
}
|
||||
return new TextEncoder().encode(JSON.stringify(out));
|
||||
}
|
||||
|
||||
/** POST /v1/messages/count_tokens with the given body. Returns the upstream's
|
||||
* `input_tokens` number or null on any failure. count_tokens is documented
|
||||
* as a free endpoint (no input-token billing) — we use it once per request
|
||||
@@ -701,7 +555,7 @@ export function createProxy(config: ProxyConfig = {}) {
|
||||
// main /v1/messages) overlap. Anthropic doesn't bill count_tokens, so
|
||||
// the cost is wall-clock only — typically ~30-80ms, fully hidden by
|
||||
// the main forward latency.
|
||||
const ctBody = buildCountTokensBody(bodyIn);
|
||||
const ctBody = buildBaselineCountTokensBody(bodyIn);
|
||||
if (ctBody) {
|
||||
const ctHeaders = filterHeaders(req.headers, STRIP_REQ_HEADERS);
|
||||
ctHeaders.set('content-type', 'application/json');
|
||||
@@ -710,7 +564,7 @@ export function createProxy(config: ProxyConfig = {}) {
|
||||
// Second probe: body truncated at the last cache_control marker.
|
||||
// Null body = no markers exist → cacheable=0 by definition, no
|
||||
// probe needed.
|
||||
const ctCacheableBody = buildCountTokensCacheablePrefix(bodyIn);
|
||||
const ctCacheableBody = buildCacheablePrefixCountTokensBody(bodyIn);
|
||||
if (ctCacheableBody) {
|
||||
baselineCacheablePromise = countTokensUpstream(
|
||||
upstream,
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildCountTokensBodies,
|
||||
isPixelpipeSupportedModel,
|
||||
shouldTransformAnthropicMessages,
|
||||
transformAnthropicMessages,
|
||||
} from '../src/core/index.js';
|
||||
|
||||
const enc = new TextEncoder();
|
||||
const dec = new TextDecoder();
|
||||
|
||||
describe('public library API', () => {
|
||||
it('recognizes Opus 4.7 and only Opus 4.7 as supported', () => {
|
||||
expect(isPixelpipeSupportedModel('claude-opus-4-7')).toBe(true);
|
||||
expect(isPixelpipeSupportedModel('claude-opus-4-7-high')).toBe(true);
|
||||
expect(isPixelpipeSupportedModel('claude-opus-4-6')).toBe(false);
|
||||
expect(isPixelpipeSupportedModel('claude-sonnet-4-5')).toBe(false);
|
||||
expect(isPixelpipeSupportedModel(null)).toBe(false);
|
||||
});
|
||||
|
||||
it('reports applicability with route/method/body gates', () => {
|
||||
expect(shouldTransformAnthropicMessages({
|
||||
model: 'claude-opus-4-7',
|
||||
method: 'POST',
|
||||
path: '/v1/messages',
|
||||
bodyBytes: 10,
|
||||
})).toEqual({ eligible: true, reason: 'eligible' });
|
||||
expect(shouldTransformAnthropicMessages({
|
||||
model: 'claude-opus-4-7',
|
||||
method: 'GET',
|
||||
path: '/v1/messages',
|
||||
bodyBytes: 10,
|
||||
}).reason).toBe('unsupported_method');
|
||||
expect(shouldTransformAnthropicMessages({
|
||||
model: 'claude-opus-4-7',
|
||||
method: 'POST',
|
||||
path: '/v1/messages/count_tokens',
|
||||
bodyBytes: 10,
|
||||
}).reason).toBe('unsupported_path');
|
||||
});
|
||||
|
||||
it('builds count_tokens probe bodies from a messages body', () => {
|
||||
const body = enc.encode(JSON.stringify({
|
||||
model: 'claude-opus-4-7',
|
||||
max_tokens: 1024,
|
||||
stream: true,
|
||||
system: [{ type: 'text', text: 'sys' }],
|
||||
tools: [{ name: 't', description: 'd', input_schema: { type: 'object' } }],
|
||||
messages: [
|
||||
{ role: 'user', content: 'hello' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'text', text: 'cached', cache_control: { type: 'ephemeral', ttl: '1h' } },
|
||||
{ type: 'text', text: 'tail' },
|
||||
],
|
||||
},
|
||||
],
|
||||
}));
|
||||
|
||||
const probes = buildCountTokensBodies(body);
|
||||
expect(probes.fullBody).toBeInstanceOf(Uint8Array);
|
||||
const full = JSON.parse(dec.decode(probes.fullBody!)) as Record<string, unknown>;
|
||||
expect(full.model).toBe('claude-opus-4-7');
|
||||
expect(full.max_tokens).toBeUndefined();
|
||||
expect(full.stream).toBeUndefined();
|
||||
expect(Array.isArray(full.messages)).toBe(true);
|
||||
|
||||
expect(probes.cacheablePrefixBody).toBeInstanceOf(Uint8Array);
|
||||
const prefix = JSON.parse(dec.decode(probes.cacheablePrefixBody!)) as { messages: Array<{ content: unknown }> };
|
||||
const last = prefix.messages.at(-1)!;
|
||||
expect(Array.isArray(last.content)).toBe(true);
|
||||
expect((last.content as unknown[])).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('wraps the transformer with model gating and cache ownership metadata', async () => {
|
||||
const unsupported = enc.encode(JSON.stringify({
|
||||
model: 'claude-opus-4-6',
|
||||
system: 'x'.repeat(20_000),
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
}));
|
||||
const skipped = await transformAnthropicMessages({ body: unsupported, model: 'claude-opus-4-6' });
|
||||
expect(skipped.applied).toBe(false);
|
||||
expect(skipped.reason).toBe('unsupported_model');
|
||||
expect(skipped.body).toBe(unsupported);
|
||||
|
||||
const supported = enc.encode(JSON.stringify({
|
||||
model: 'claude-opus-4-7',
|
||||
system: 'Important system instruction. '.repeat(1200),
|
||||
tools: [{
|
||||
name: 'read_file',
|
||||
description: 'Read a file from disk. '.repeat(200),
|
||||
input_schema: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] },
|
||||
}],
|
||||
messages: [{ role: 'user', content: 'hello' }],
|
||||
}));
|
||||
const transformed = await transformAnthropicMessages({ body: supported, model: 'claude-opus-4-7' });
|
||||
expect(transformed.applied).toBe(true);
|
||||
expect(transformed.reason).toBe('applied');
|
||||
expect(transformed.info.compressedChars).toBeGreaterThan(0);
|
||||
expect(transformed.info.imageCount).toBeGreaterThan(0);
|
||||
expect(transformed.cache.ownsCacheControl).toBe(true);
|
||||
expect(transformed.cache.markerCount).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user