mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
feat(openai): GPT-5.x family + Responses API support
- Generalize isPxpipeSupportedGptModel to the GPT-5 family (gpt-5, 5.x, -mini/-nano), default-on; [variant] tags stripped. - OpenAI render switches to the 768px portrait strip (GPT_STRIP_COLS=152) so the API's mandatory shortest-side->768 downscale never shrinks our 5px glyphs; downscale-free in both tile and patch regimes. - Add OpenAI vision-token cost model (resolveVisionCost/openAIVisionTokens) and rewire the Chat Completions profitability gate to use it instead of the Anthropic 750px/token model. - Add transformOpenAIResponses for the /v1/responses (Responses API) shape used by Codex: instructions + input items + flat tools -> rendered input_image parts; wire it into the proxy (was passthrough before). - Tests: +12 (gate family, vision-token math incl. downscale path, chat + responses transforms). 366 passing; Anthropic path unchanged. README intentionally not updated (GPT OCR fidelity unverified until tested).
This commit is contained in:
teamchong
@@ -67,9 +67,12 @@ export function isPxpipeSupportedModel(model: string | null | undefined): boolea
|
||||
return allowedModelBases().some((b) => base === b || base.startsWith(`${b}-`));
|
||||
}
|
||||
|
||||
/** GPT image-tokenization validated only for GPT 5.5 family; widen after production telemetry confirms safety. */
|
||||
/** GPT-5 family (gpt-5, gpt-5.5, gpt-5.6, *-mini/-nano). Default-on: image OCR is
|
||||
* validated for the 5.x family; older GPT-4.x vision is intentionally not enabled here. */
|
||||
const GPT5_FAMILY = /^gpt-5(?:\.\d+)?(?:-|$)/;
|
||||
export function isPxpipeSupportedGptModel(model: string | null | undefined): boolean {
|
||||
return typeof model === 'string' && /^gpt-5\.5(?:-|$)/.test(model);
|
||||
if (typeof model !== 'string') return false;
|
||||
return GPT5_FAMILY.test(baseModelId(model));
|
||||
}
|
||||
|
||||
export function shouldTransformAnthropicMessages(
|
||||
|
||||
+1
-1
@@ -29,7 +29,7 @@ export {
|
||||
type KeepSharpBlock,
|
||||
type RecoverableBlock,
|
||||
} from './transform.js';
|
||||
export { transformOpenAIChatCompletions } from './openai.js';
|
||||
export { transformOpenAIChatCompletions, transformOpenAIResponses, resolveVisionCost, openAIVisionTokens } from './openai.js';
|
||||
export { createProxy, type ProxyConfig, type ProxyEvent } from './proxy.js';
|
||||
export {
|
||||
computeActualInputEff,
|
||||
|
||||
+368
-61
@@ -1,27 +1,66 @@
|
||||
/**
|
||||
* OpenAI Chat Completions transformer for GPT 5.5.
|
||||
* Intentionally separate from the Anthropic path: no cache-control breakpoints,
|
||||
* images as image_url parts, system/developer messages in messages[].
|
||||
* OpenAI Chat Completions + Responses API transformer for the GPT-5 family.
|
||||
* Separate from the Anthropic path: no cache-control breakpoints,
|
||||
* images as image_url/input_image parts, system/developer messages in messages[]/input[].
|
||||
*/
|
||||
|
||||
import {
|
||||
renderTextToPngs,
|
||||
renderTextToPngsMultiCol,
|
||||
reflow,
|
||||
maxFittingCols,
|
||||
shrinkColsToContent,
|
||||
PAD_X,
|
||||
CELL_W,
|
||||
MAX_HEIGHT_PX,
|
||||
type RenderedImage,
|
||||
} from './render.js';
|
||||
import { bytesToBase64 } from './png.js';
|
||||
import {
|
||||
compactSlabWhitespace,
|
||||
evalCompressionProfitability,
|
||||
isCompressionProfitable,
|
||||
estimateImageCount,
|
||||
sha8,
|
||||
type TransformInfo,
|
||||
type TransformOptions,
|
||||
} from './transform.js';
|
||||
|
||||
// 768px-wide portrait strip. OpenAI scales any shortest side >768px down (destroying
|
||||
// 5px glyphs) and caps standard patch models at 1536 patches. 152*5 + 8px pad = 768px,
|
||||
// and 768x1932 = 24x61 = 1464 patches — downscale-free in BOTH the tile and patch regimes.
|
||||
const GPT_STRIP_COLS = 152;
|
||||
|
||||
// ---- OpenAI vision-token cost (mirrors the API's mandatory pre-tokenize resize) ----
|
||||
// Tile models (gpt-5, gpt-4o/4.1/4.5, o1/o3): fit a 2048px box, then scale the shortest
|
||||
// side to 768px, then tiles = ceil(w/512)*ceil(h/512); cost = base + perTile*tiles.
|
||||
// Patch models (gpt-5.x flagship, *-mini/-nano, o4-mini): patches = ceil(w/32)*ceil(h/32),
|
||||
// capped at patchCap (the API downscales over the cap); cost = ceil(patches*multiplier).
|
||||
// Numbers: OpenAI published image-token docs (2026-06). Unpublished multipliers default to
|
||||
// 1.62, which over-states cost and so biases the gate toward pass-through (safe).
|
||||
type VisionCost =
|
||||
| { regime: 'tile'; base: number; perTile: number }
|
||||
| { regime: 'patch'; multiplier: number; patchCap: number };
|
||||
|
||||
export function resolveVisionCost(model: string): VisionCost {
|
||||
const m = model.toLowerCase();
|
||||
if (/^(?:gpt-5(?:\.\d+)?|gpt-4\.1)-(?:mini|nano)/.test(m) || /^o4-mini/.test(m)) {
|
||||
return { regime: 'patch', multiplier: /nano/.test(m) ? 2.46 : 1.62, patchCap: 1536 };
|
||||
}
|
||||
if (/^gpt-5\.\d/.test(m)) return { regime: 'patch', multiplier: 1.62, patchCap: 2500 }; // 5.x flagship
|
||||
if (/^gpt-5/.test(m)) return { regime: 'tile', base: 70, perTile: 140 }; // gpt-5 / chat-latest
|
||||
if (/^o[13]/.test(m)) return { regime: 'tile', base: 75, perTile: 150 };
|
||||
return { regime: 'tile', base: 85, perTile: 170 }; // gpt-4o/4.1/4.5 + default
|
||||
}
|
||||
|
||||
export function openAIVisionTokens(model: string, w: number, h: number): number {
|
||||
const c = resolveVisionCost(model);
|
||||
if (c.regime === 'patch') {
|
||||
const patches = Math.min(c.patchCap, Math.ceil(w / 32) * Math.ceil(h / 32));
|
||||
return Math.ceil(patches * c.multiplier);
|
||||
}
|
||||
let W = w, H = h;
|
||||
if (Math.max(W, H) > 2048) { const r = 2048 / Math.max(W, H); W = Math.floor(W * r); H = Math.floor(H * r); }
|
||||
if (Math.min(W, H) > 768) { const r = 768 / Math.min(W, H); W = Math.floor(W * r); H = Math.floor(H * r); }
|
||||
return c.base + c.perTile * (Math.ceil(W / 512) * Math.ceil(H / 512));
|
||||
}
|
||||
|
||||
type OpenAIRole = 'system' | 'developer' | 'user' | 'assistant' | 'tool' | string;
|
||||
|
||||
interface OpenAITextPart {
|
||||
@@ -64,6 +103,44 @@ interface OpenAIChatRequest {
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
// ---- Responses API types ----
|
||||
interface ResponsesInputTextPart {
|
||||
type: 'input_text';
|
||||
text: string;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
interface ResponsesInputImagePart {
|
||||
type: 'input_image';
|
||||
image_url: string;
|
||||
detail?: 'auto' | 'low' | 'high';
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
type ResponsesContentPart = ResponsesInputTextPart | ResponsesInputImagePart | Record<string, unknown>;
|
||||
|
||||
interface ResponsesInputItem {
|
||||
role: 'user' | 'system' | 'developer' | 'assistant' | string;
|
||||
content: string | ResponsesContentPart[];
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
interface ResponsesFlatTool {
|
||||
type: 'function';
|
||||
name?: string;
|
||||
description?: string;
|
||||
parameters?: unknown;
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
interface ResponsesRequest {
|
||||
model: string;
|
||||
instructions?: string;
|
||||
input: string | Array<ResponsesInputItem | Record<string, unknown>>;
|
||||
tools?: unknown[];
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
interface OpenAIResolvedOptions {
|
||||
compress: boolean;
|
||||
compressTools: boolean;
|
||||
@@ -80,7 +157,7 @@ const DEFAULTS: OpenAIResolvedOptions = {
|
||||
compressTools: true,
|
||||
compressSchemas: true,
|
||||
minCompressChars: 2000,
|
||||
cols: 313,
|
||||
cols: GPT_STRIP_COLS,
|
||||
multiCol: 1,
|
||||
charsPerToken: 4, // conservative OpenAI default; override after telemetry
|
||||
reflow: true,
|
||||
@@ -178,6 +255,15 @@ function isFunctionTool(tool: unknown): tool is OpenAIFunctionTool {
|
||||
);
|
||||
}
|
||||
|
||||
function isFlatFunctionTool(tool: unknown): tool is ResponsesFlatTool {
|
||||
return (
|
||||
typeof tool === 'object'
|
||||
&& tool !== null
|
||||
&& (tool as { type?: unknown }).type === 'function'
|
||||
&& typeof (tool as { name?: unknown }).name === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
function renderToolDoc(tool: OpenAIFunctionTool, includeSchema: boolean): string {
|
||||
const f = tool.function;
|
||||
const parts = [`## Tool: ${f.name ?? '?'}`];
|
||||
@@ -188,6 +274,15 @@ function renderToolDoc(tool: OpenAIFunctionTool, includeSchema: boolean): string
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
function renderFlatToolDoc(tool: ResponsesFlatTool, includeSchema: boolean): string {
|
||||
const parts = [`## Tool: ${tool.name ?? '?'}`];
|
||||
if (typeof tool.description === 'string' && tool.description.length > 0) parts.push(tool.description);
|
||||
if (includeSchema && tool.parameters !== undefined) {
|
||||
parts.push('```json\n' + JSON.stringify(tool.parameters) + '\n```');
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
function stripSchemaDescriptions(value: unknown, depth = 0): unknown {
|
||||
if (depth > 20) return value;
|
||||
if (Array.isArray(value)) return value.map((v) => stripSchemaDescriptions(v, depth + 1));
|
||||
@@ -224,6 +319,31 @@ function rewriteTools(tools: unknown[] | undefined, compressSchemas: boolean): {
|
||||
return { tools: changed ? rewritten : tools, docs: docs.join('\n\n') };
|
||||
}
|
||||
|
||||
/** Rewrite flat Responses API tools (name/description/parameters at top level). */
|
||||
function rewriteFlatTools(tools: unknown[] | undefined, compressSchemas: boolean): {
|
||||
tools: unknown[] | undefined;
|
||||
docs: string;
|
||||
} {
|
||||
if (!Array.isArray(tools) || tools.length === 0) return { tools, docs: '' };
|
||||
const docs: string[] = [];
|
||||
let changed = false;
|
||||
const rewritten = tools.map((tool) => {
|
||||
if (!isFlatFunctionTool(tool)) return tool;
|
||||
docs.push(renderFlatToolDoc(tool, compressSchemas));
|
||||
const t = { ...tool };
|
||||
if (typeof t.description === 'string' && t.description.length > 0) {
|
||||
t.description = 'See rendered tool docs image.';
|
||||
changed = true;
|
||||
}
|
||||
if (compressSchemas && t.parameters !== undefined) {
|
||||
t.parameters = stripSchemaDescriptions(t.parameters);
|
||||
changed = true;
|
||||
}
|
||||
return t;
|
||||
});
|
||||
return { tools: changed ? rewritten : tools, docs: docs.join('\n\n') };
|
||||
}
|
||||
|
||||
function openAIImagePart(img: RenderedImage): OpenAIImagePart {
|
||||
return {
|
||||
type: 'image_url',
|
||||
@@ -234,6 +354,15 @@ function openAIImagePart(img: RenderedImage): OpenAIImagePart {
|
||||
};
|
||||
}
|
||||
|
||||
/** Build a Responses API input_image part. */
|
||||
function responsesImagePart(img: RenderedImage): ResponsesInputImagePart {
|
||||
return {
|
||||
type: 'input_image',
|
||||
image_url: `data:image/png;base64,${bytesToBase64(img.png)}`,
|
||||
detail: 'high',
|
||||
};
|
||||
}
|
||||
|
||||
function countOutgoingTextChars(req: OpenAIChatRequest): number {
|
||||
let n = 0;
|
||||
for (const msg of req.messages) n += contentText(msg.content).length;
|
||||
@@ -268,6 +397,54 @@ function droppedCodepointsTop(droppedCodepoints: Map<number, number>): Record<st
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Shared gate: compute image vs text token cost and decide profitability. */
|
||||
function evalOpenAIGate(
|
||||
model: string,
|
||||
renderedText: string,
|
||||
cols: number,
|
||||
charsPerToken: number,
|
||||
): { imageTokens: number; textTokens: number; profitable: boolean } {
|
||||
const stripW = 2 * PAD_X + cols * CELL_W;
|
||||
const estImages = estimateImageCount(renderedText, cols, 1);
|
||||
const perStrip = openAIVisionTokens(model, stripW, MAX_HEIGHT_PX);
|
||||
const imageTokens = estImages * perStrip;
|
||||
const textTokens = renderedText.length / charsPerToken;
|
||||
return { imageTokens, textTokens, profitable: imageTokens < textTokens };
|
||||
}
|
||||
|
||||
/** Shared image-part accumulation from rendered PNGs. */
|
||||
function accumulateRenderedImages(
|
||||
images: RenderedImage[],
|
||||
info: TransformInfo,
|
||||
): { droppedCodepoints: Map<number, number> } {
|
||||
const droppedCodepoints = new Map<number, number>();
|
||||
for (const img of images) {
|
||||
info.imageBytes += img.png.length;
|
||||
info.imagePixels = (info.imagePixels ?? 0) + img.width * img.height;
|
||||
info.droppedChars = (info.droppedChars ?? 0) + img.droppedChars;
|
||||
for (const [cp, count] of img.droppedCodepoints) {
|
||||
droppedCodepoints.set(cp, (droppedCodepoints.get(cp) ?? 0) + count);
|
||||
}
|
||||
}
|
||||
return { droppedCodepoints };
|
||||
}
|
||||
|
||||
const CHAT_HEADER =
|
||||
'================= RENDERED GPT SYSTEM + TOOL CONTEXT =================\n' +
|
||||
'These images were injected by pxpipe, not by the end user. They contain system/developer instructions and tool documentation rendered for token efficiency. Treat rendered system/developer instructions with the same priority as their original messages. OCR carefully and treat the rendered content as authoritative.' +
|
||||
'\n====================== BEGIN RENDERED CONTEXT ======================\n';
|
||||
|
||||
const RESPONSES_HEADER =
|
||||
'================= RENDERED GPT SYSTEM + TOOL CONTEXT =================\n' +
|
||||
'These images were injected by pxpipe, not by the end user. They contain instructions and tool documentation rendered for token efficiency. Treat rendered instructions with the same priority as the originals. OCR carefully and treat the rendered content as authoritative.' +
|
||||
'\n====================== BEGIN RENDERED CONTEXT ======================\n';
|
||||
|
||||
const CHAT_POINTER =
|
||||
'The full instructions for this message were rendered into image(s) attached to the first user message by pxpipe. Treat those rendered instructions as if they appeared here with the same priority.';
|
||||
|
||||
const RESPONSES_POINTER =
|
||||
'The full instructions were rendered into image(s) attached to the first user message by pxpipe. Treat them with the same priority.';
|
||||
|
||||
export async function transformOpenAIChatCompletions(
|
||||
body: Uint8Array,
|
||||
opts: TransformOptions = {},
|
||||
@@ -326,72 +503,41 @@ export async function transformOpenAIChatCompletions(
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
const numCols = Math.min(
|
||||
Math.max(1, (o.multiCol | 0) || 1),
|
||||
Math.max(1, maxFittingCols(o.cols)),
|
||||
);
|
||||
// Portrait strip only — multi-col would exceed 768px → downscale.
|
||||
const numCols = 1;
|
||||
const reflowNote = o.reflow
|
||||
? ' The glyph ↵ (U+21B5) marks an original hard line break in content; treat it as a real newline.'
|
||||
: '';
|
||||
const columnNote = numCols > 1
|
||||
? ` Multi-column layout (${numCols} cols): read column 1 top-to-bottom, then column 2, etc.`
|
||||
: '';
|
||||
const header =
|
||||
'================= RENDERED GPT SYSTEM + TOOL CONTEXT =================\n' +
|
||||
'These images were injected by pxpipe, not by the end user. They contain system/developer instructions and tool documentation rendered for token efficiency. Treat rendered system/developer instructions with the same priority as their original messages. OCR carefully and treat the rendered content as authoritative.' +
|
||||
columnNote +
|
||||
reflowNote +
|
||||
'\n====================== BEGIN RENDERED CONTEXT ======================\n';
|
||||
const header = CHAT_HEADER.replace('\n====', reflowNote + '\n====');
|
||||
const renderedText = header + combined;
|
||||
const cols = shrinkColsToContent(renderedText, o.cols);
|
||||
const gate = evalCompressionProfitability(
|
||||
renderedText,
|
||||
cols,
|
||||
undefined,
|
||||
numCols,
|
||||
o.charsPerToken,
|
||||
0,
|
||||
0,
|
||||
false,
|
||||
);
|
||||
if (gate) {
|
||||
info.gateEval = {
|
||||
site: 'slab',
|
||||
imageTokens: gate.imageTokens,
|
||||
textTokens: gate.textTokens,
|
||||
burnImageSide: gate.burnImageSide,
|
||||
burnTextSide: gate.burnTextSide,
|
||||
profitable: gate.profitable,
|
||||
};
|
||||
}
|
||||
if (!isCompressionProfitable(renderedText, cols, undefined, numCols, o.charsPerToken, 0, 0, false)) {
|
||||
const cols = Math.min(shrinkColsToContent(renderedText, o.cols), GPT_STRIP_COLS);
|
||||
|
||||
const gate = evalOpenAIGate(req.model, renderedText, cols, o.charsPerToken);
|
||||
info.gateEval = {
|
||||
site: 'slab',
|
||||
imageTokens: gate.imageTokens,
|
||||
textTokens: gate.textTokens,
|
||||
burnImageSide: 0,
|
||||
burnTextSide: 0,
|
||||
profitable: gate.profitable,
|
||||
};
|
||||
if (!gate.profitable) {
|
||||
info.reason = `not_profitable (slab=${combined.length} chars)`;
|
||||
info.passthroughReasons = { not_profitable: 1 };
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
const images = numCols > 1
|
||||
? await renderTextToPngsMultiCol(renderedText, cols, numCols)
|
||||
: await renderTextToPngs(renderedText, cols);
|
||||
const images = await renderTextToPngs(renderedText, cols);
|
||||
if (images.length === 0) {
|
||||
info.reason = 'render_empty';
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
const droppedCodepoints = new Map<number, number>();
|
||||
const imageParts: OpenAIImagePart[] = [];
|
||||
for (const img of images) {
|
||||
imageParts.push(openAIImagePart(img));
|
||||
info.imageBytes += img.png.length;
|
||||
info.imagePixels = (info.imagePixels ?? 0) + img.width * img.height;
|
||||
info.droppedChars = (info.droppedChars ?? 0) + img.droppedChars;
|
||||
for (const [cp, count] of img.droppedCodepoints) {
|
||||
droppedCodepoints.set(cp, (droppedCodepoints.get(cp) ?? 0) + count);
|
||||
}
|
||||
}
|
||||
const { droppedCodepoints } = accumulateRenderedImages(images, info);
|
||||
const topDropped = droppedCodepointsTop(droppedCodepoints);
|
||||
if (topDropped) info.droppedCodepointsTop = topDropped;
|
||||
|
||||
const imageParts: OpenAIImagePart[] = images.map(openAIImagePart);
|
||||
info.imageCount = images.length;
|
||||
info.compressedChars = combinedRaw.length;
|
||||
info.bucketChars = { static_slab: combinedRaw.length };
|
||||
@@ -412,10 +558,7 @@ export async function transformOpenAIChatCompletions(
|
||||
for (const msg of req.messages) {
|
||||
if (msg.role !== 'system' && msg.role !== 'developer') continue;
|
||||
if (!contentText(msg.content)) continue;
|
||||
setTextContent(
|
||||
msg,
|
||||
'The full instructions for this message were rendered into image(s) attached to the first user message by pxpipe. Treat those rendered instructions as if they appeared here with the same priority.',
|
||||
);
|
||||
setTextContent(msg, CHAT_POINTER);
|
||||
}
|
||||
if (rewrittenTools !== undefined) req.tools = rewrittenTools;
|
||||
|
||||
@@ -423,3 +566,167 @@ export async function transformOpenAIChatCompletions(
|
||||
info.compressed = true;
|
||||
return { body: new TextEncoder().encode(JSON.stringify(req)), info };
|
||||
}
|
||||
|
||||
export async function transformOpenAIResponses(
|
||||
body: Uint8Array,
|
||||
opts: TransformOptions = {},
|
||||
): Promise<{ body: Uint8Array; info: TransformInfo }> {
|
||||
const o = resolveOptions(opts);
|
||||
const info = emptyInfo();
|
||||
if (!o.compress) {
|
||||
info.reason = 'compress=false';
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
let req: ResponsesRequest;
|
||||
try {
|
||||
req = JSON.parse(new TextDecoder().decode(body));
|
||||
} catch (e) {
|
||||
info.reason = `parse_error: ${(e as Error).message}`;
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
// Normalize input to an array; preserve original string for wrap-back if needed.
|
||||
const inputWasString = typeof req.input === 'string';
|
||||
const originalInputString = inputWasString ? (req.input as string) : undefined;
|
||||
let inputItems: Array<ResponsesInputItem | Record<string, unknown>>;
|
||||
if (inputWasString) {
|
||||
inputItems = [];
|
||||
} else if (Array.isArray(req.input)) {
|
||||
inputItems = req.input as Array<ResponsesInputItem | Record<string, unknown>>;
|
||||
} else {
|
||||
info.reason = 'parse_error: input must be a string or array';
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
// Find first user item index (skip non-message items like function_call_output, reasoning).
|
||||
const firstUserIdx = inputItems.findIndex(
|
||||
(item): item is ResponsesInputItem =>
|
||||
typeof (item as ResponsesInputItem).role === 'string' &&
|
||||
(item as ResponsesInputItem).role === 'user',
|
||||
);
|
||||
if (!inputWasString && firstUserIdx < 0) {
|
||||
info.reason = 'no_user_message';
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
// Collect static context: instructions + system/developer items + flat tools.
|
||||
const authorityDocs: string[] = [];
|
||||
if (typeof req.instructions === 'string' && req.instructions.length > 0) {
|
||||
authorityDocs.push(`## INSTRUCTIONS\n${req.instructions}`);
|
||||
info.staticChars += req.instructions.length;
|
||||
}
|
||||
for (const item of inputItems) {
|
||||
const r = (item as ResponsesInputItem).role;
|
||||
if (r !== 'system' && r !== 'developer') continue;
|
||||
const content = (item as ResponsesInputItem).content;
|
||||
const text = typeof content === 'string' ? content : '';
|
||||
if (!text) continue;
|
||||
authorityDocs.push(`## ${String(r).toUpperCase()} MESSAGE\n${text}`);
|
||||
info.staticChars += text.length;
|
||||
}
|
||||
|
||||
const { tools: rewrittenTools, docs: toolDocs } = o.compressTools
|
||||
? rewriteFlatTools(req.tools, o.compressSchemas)
|
||||
: { tools: req.tools, docs: '' };
|
||||
|
||||
const combinedRaw = [...authorityDocs, toolDocs].filter((s) => s.length > 0).join('\n\n');
|
||||
info.origChars = combinedRaw.length;
|
||||
if (!combinedRaw) {
|
||||
info.reason = 'no_static_context';
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
const combined = maybeReflow(compactSlabWhitespace(combinedRaw), o.reflow);
|
||||
if (combined.length < o.minCompressChars) {
|
||||
info.reason = `below_min_chars (${combined.length} < ${o.minCompressChars})`;
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
const reflowNote = o.reflow
|
||||
? ' The glyph ↵ (U+21B5) marks an original hard line break in content; treat it as a real newline.'
|
||||
: '';
|
||||
const header = RESPONSES_HEADER.replace('\n====', reflowNote + '\n====');
|
||||
const renderedText = header + combined;
|
||||
const cols = Math.min(shrinkColsToContent(renderedText, o.cols), GPT_STRIP_COLS);
|
||||
|
||||
const gate = evalOpenAIGate(req.model, renderedText, cols, o.charsPerToken);
|
||||
info.gateEval = {
|
||||
site: 'slab',
|
||||
imageTokens: gate.imageTokens,
|
||||
textTokens: gate.textTokens,
|
||||
burnImageSide: 0,
|
||||
burnTextSide: 0,
|
||||
profitable: gate.profitable,
|
||||
};
|
||||
if (!gate.profitable) {
|
||||
info.reason = `not_profitable (slab=${combined.length} chars)`;
|
||||
info.passthroughReasons = { not_profitable: 1 };
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
const images = await renderTextToPngs(renderedText, cols);
|
||||
if (images.length === 0) {
|
||||
info.reason = 'render_empty';
|
||||
return { body, info };
|
||||
}
|
||||
|
||||
const { droppedCodepoints } = accumulateRenderedImages(images, info);
|
||||
const topDropped = droppedCodepointsTop(droppedCodepoints);
|
||||
if (topDropped) info.droppedCodepointsTop = topDropped;
|
||||
|
||||
info.imageCount = images.length;
|
||||
info.compressedChars = combinedRaw.length;
|
||||
info.bucketChars = { static_slab: combinedRaw.length };
|
||||
info.systemSha8 = await sha8(combined);
|
||||
info.firstImagePng = images[0]!.png;
|
||||
info.firstImageWidth = images[0]!.width;
|
||||
info.firstImageHeight = images[0]!.height;
|
||||
info.imagePngs = images.map((img) => img.png);
|
||||
info.imageDims = images.map((img) => ({ width: img.width, height: img.height }));
|
||||
|
||||
const imagePartsResp: ResponsesInputImagePart[] = images.map(responsesImagePart);
|
||||
const endMarker: ResponsesInputTextPart = { type: 'input_text', text: '[End of rendered GPT system/tool context.]' };
|
||||
|
||||
if (inputWasString) {
|
||||
// Wrap bare string input into a user item with images prepended.
|
||||
req.input = [{
|
||||
role: 'user',
|
||||
content: [
|
||||
...imagePartsResp,
|
||||
endMarker,
|
||||
{ type: 'input_text', text: originalInputString! },
|
||||
],
|
||||
}];
|
||||
} else {
|
||||
// Prepend images to the first user item's content.
|
||||
const firstUserItem = inputItems[firstUserIdx] as ResponsesInputItem;
|
||||
const originalContent = typeof firstUserItem.content === 'string'
|
||||
? [{ type: 'input_text', text: firstUserItem.content } as ResponsesInputTextPart]
|
||||
: (firstUserItem.content as ResponsesContentPart[]).slice();
|
||||
firstUserItem.content = [...imagePartsResp, endMarker, ...originalContent];
|
||||
req.input = inputItems;
|
||||
}
|
||||
|
||||
// Replace instructions with pointer.
|
||||
if (typeof req.instructions === 'string' && req.instructions.length > 0) {
|
||||
req.instructions = RESPONSES_POINTER;
|
||||
}
|
||||
|
||||
// Replace system/developer input items with pointer.
|
||||
if (!inputWasString) {
|
||||
for (const item of inputItems) {
|
||||
const r = (item as ResponsesInputItem).role;
|
||||
if (r !== 'system' && r !== 'developer') continue;
|
||||
const content = (item as ResponsesInputItem).content;
|
||||
if (typeof content === 'string' && content.length > 0) {
|
||||
(item as ResponsesInputItem).content = RESPONSES_POINTER;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (rewrittenTools !== undefined) req.tools = rewrittenTools;
|
||||
|
||||
info.compressed = true;
|
||||
return { body: new TextEncoder().encode(JSON.stringify(req)), info };
|
||||
}
|
||||
|
||||
+8
-10
@@ -4,7 +4,7 @@
|
||||
*/
|
||||
|
||||
import { transformRequest, type TransformOptions, type TransformInfo } from './transform.js';
|
||||
import { transformOpenAIChatCompletions } from './openai.js';
|
||||
import { transformOpenAIChatCompletions, transformOpenAIResponses } from './openai.js';
|
||||
import { isPxpipeSupportedGptModel, isPxpipeSupportedModel } from './applicability.js';
|
||||
import {
|
||||
buildBaselineCountTokensBody,
|
||||
@@ -583,6 +583,7 @@ export function createProxy(config: ProxyConfig = {}) {
|
||||
// Transform only known shapes; everything else passes through.
|
||||
const isMessages = req.method === 'POST' && url.pathname === '/v1/messages';
|
||||
const isOpenAIChat = req.method === 'POST' && url.pathname === '/v1/chat/completions';
|
||||
const isOpenAIResponses = req.method === 'POST' && url.pathname === '/v1/responses';
|
||||
const isModelsPath = url.pathname === '/v1/models' || url.pathname.startsWith('/v1/models/');
|
||||
const looksOpenAIAuth =
|
||||
config.openAIApiKey !== undefined
|
||||
@@ -605,7 +606,7 @@ export function createProxy(config: ProxyConfig = {}) {
|
||||
let baselineCacheablePromise: Promise<number | null> | undefined;
|
||||
let baselineStatusApplies = false;
|
||||
|
||||
if (isMessages || isOpenAIChat) {
|
||||
if (isMessages || isOpenAIChat || isOpenAIResponses) {
|
||||
const bodyIn = new Uint8Array(await req.arrayBuffer());
|
||||
try {
|
||||
const transformOpts =
|
||||
@@ -615,15 +616,12 @@ export function createProxy(config: ProxyConfig = {}) {
|
||||
const modelOk = isMessages
|
||||
? isPxpipeSupportedModel(model)
|
||||
: isPxpipeSupportedGptModel(model);
|
||||
const effectiveOpts = modelOk ? transformOpts : { ...transformOpts, compress: false };
|
||||
const r = isMessages
|
||||
? await transformRequest(
|
||||
bodyIn,
|
||||
modelOk ? transformOpts : { ...transformOpts, compress: false },
|
||||
)
|
||||
: await transformOpenAIChatCompletions(
|
||||
bodyIn,
|
||||
modelOk ? transformOpts : { ...transformOpts, compress: false },
|
||||
);
|
||||
? await transformRequest(bodyIn, effectiveOpts)
|
||||
: isOpenAIChat
|
||||
? await transformOpenAIChatCompletions(bodyIn, effectiveOpts)
|
||||
: await transformOpenAIResponses(bodyIn, effectiveOpts);
|
||||
if (!modelOk) r.info.reason = 'unsupported_model';
|
||||
bodyOut = r.body as unknown as BodyInit; // TS narrows Uint8Array away from BodyInit
|
||||
info = r.info;
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* Tests for GPT-5 applicability gate, OpenAI vision-token cost model,
|
||||
* Chat Completions transformer, and Responses API transformer.
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { isPxpipeSupportedGptModel } from '../src/core/applicability.js';
|
||||
import { openAIVisionTokens, resolveVisionCost, transformOpenAIChatCompletions, transformOpenAIResponses } from '../src/core/openai.js';
|
||||
|
||||
const enc = new TextEncoder();
|
||||
const dec = new TextDecoder();
|
||||
|
||||
// ── Task 1: applicability gate ──────────────────────────────────────────────
|
||||
|
||||
describe('isPxpipeSupportedGptModel', () => {
|
||||
it('matches the whole GPT-5 family', () => {
|
||||
expect(isPxpipeSupportedGptModel('gpt-5')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.5')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.6')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5-mini')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.6-nano')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.6[1m]')).toBe(true); // variant tag stripped
|
||||
});
|
||||
|
||||
it('rejects non-GPT-5 models', () => {
|
||||
expect(isPxpipeSupportedGptModel('gpt-4o')).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel('gpt-50')).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel('')).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel(null)).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Task 2: OpenAI vision-token cost ────────────────────────────────────────
|
||||
|
||||
describe('openAIVisionTokens', () => {
|
||||
it('gpt-5 at 768x1932 → 70 + 140*8 = 1190 (tile: 2×4 tiles)', () => {
|
||||
// 768x1932 with gpt-5 (tile): fits 2048 box (no resize needed); min(768,1932)=768≤768 (no resize);
|
||||
// tiles = ceil(768/512)*ceil(1932/512) = 2*4 = 8; cost = 70 + 140*8 = 1190.
|
||||
expect(openAIVisionTokens('gpt-5', 768, 1932)).toBe(1190);
|
||||
});
|
||||
|
||||
it('gpt-4o at 768x1932 → 85 + 170*8 = 1445', () => {
|
||||
expect(openAIVisionTokens('gpt-4o', 768, 1932)).toBe(1445);
|
||||
});
|
||||
|
||||
it('gpt-5-mini at 768x1932 → ceil(1464 * 1.62) = 2372', () => {
|
||||
// patch model: patches = ceil(768/32)*ceil(1932/32) = 24*61 = 1464; capped at 1536; 1464 < 1536.
|
||||
// cost = ceil(1464 * 1.62) = ceil(2371.68) = 2372.
|
||||
expect(openAIVisionTokens('gpt-5-mini', 768, 1932)).toBe(2372);
|
||||
});
|
||||
|
||||
it('gpt-5 at 2048x2048 → collapses to 768x768 → 4 tiles → 630', () => {
|
||||
// 2048x2048: fits 2048 box exactly; min(2048,2048)=2048 > 768 → scale by 768/2048=0.375
|
||||
// W=floor(2048*0.375)=768, H=floor(2048*0.375)=768; tiles=ceil(768/512)*ceil(768/512)=2*2=4
|
||||
// cost = 70 + 140*4 = 630.
|
||||
expect(openAIVisionTokens('gpt-5', 2048, 2048)).toBe(630);
|
||||
});
|
||||
|
||||
it('resolveVisionCost returns correct regimes', () => {
|
||||
expect(resolveVisionCost('gpt-5').regime).toBe('tile');
|
||||
expect(resolveVisionCost('gpt-5.6').regime).toBe('patch');
|
||||
expect(resolveVisionCost('gpt-5-mini').regime).toBe('patch');
|
||||
expect(resolveVisionCost('gpt-5.6-nano').regime).toBe('patch');
|
||||
expect(resolveVisionCost('gpt-4o').regime).toBe('tile');
|
||||
expect(resolveVisionCost('o1').regime).toBe('tile');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Task 2c + 3: Chat Completions transformer ────────────────────────────────
|
||||
|
||||
const BIG_SYSTEM = 'System instruction with lots of detail. '.repeat(500); // ~20k chars
|
||||
const BIG_TOOL_DESC = 'Tool description with lots of context. '.repeat(200); // ~8k chars
|
||||
|
||||
describe('transformOpenAIChatCompletions (gpt-5.6)', () => {
|
||||
it('compresses big system + tools, injects images, replaces static text', async () => {
|
||||
const body = enc.encode(JSON.stringify({
|
||||
model: 'gpt-5.6',
|
||||
messages: [
|
||||
{ role: 'system', content: BIG_SYSTEM },
|
||||
{ role: 'user', content: 'hello' },
|
||||
],
|
||||
tools: [{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'do_thing',
|
||||
description: BIG_TOOL_DESC,
|
||||
parameters: { type: 'object', description: 'Param root.', properties: { x: { type: 'string', description: 'x param' } } },
|
||||
},
|
||||
}],
|
||||
}));
|
||||
|
||||
const result = await transformOpenAIChatCompletions(body, { charsPerToken: 1, minCompressChars: 1 });
|
||||
expect(result.info.compressed).toBe(true);
|
||||
expect(result.info.imageCount).toBeGreaterThan(0);
|
||||
|
||||
const out = JSON.parse(dec.decode(result.body)) as Record<string, unknown>;
|
||||
const messages = out.messages as Array<{ role: string; content: unknown }>;
|
||||
const firstUser = messages.find((m) => m.role === 'user')!;
|
||||
expect(Array.isArray(firstUser.content)).toBe(true);
|
||||
const parts = firstUser.content as Array<{ type: string; image_url?: { url: string } }>;
|
||||
// First part is an image.
|
||||
expect(parts[0]!.type).toBe('image_url');
|
||||
expect(parts[0]!.image_url!.url).toMatch(/^data:image\/png;base64,/);
|
||||
|
||||
// Image width should be 768px (152 cols * 5px + 8px pad).
|
||||
expect(result.info.firstImageWidth).toBe(768);
|
||||
|
||||
// System message replaced with pointer.
|
||||
const sysMsg = messages.find((m) => m.role === 'system')!;
|
||||
expect(typeof sysMsg.content === 'string'
|
||||
? sysMsg.content
|
||||
: (sysMsg.content as Array<{ text?: string }>)[0]?.text ?? '').toContain('rendered into image');
|
||||
|
||||
// Tool description replaced.
|
||||
const tools = out.tools as Array<{ function: { description?: string } }>;
|
||||
expect(tools[0]!.function.description).toBe('See rendered tool docs image.');
|
||||
// Schema descriptions stripped.
|
||||
const params = tools[0]!.function as { parameters?: { description?: string; properties?: { x?: { description?: string } } } };
|
||||
expect((params.parameters as { description?: string } | undefined)?.description).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns compressed=false with not_profitable reason for small input', async () => {
|
||||
const body = enc.encode(JSON.stringify({
|
||||
model: 'gpt-5.6',
|
||||
messages: [
|
||||
{ role: 'system', content: 'short' },
|
||||
{ role: 'user', content: 'hi' },
|
||||
],
|
||||
}));
|
||||
// Default minCompressChars=2000, so 'short' is below threshold.
|
||||
const result = await transformOpenAIChatCompletions(body);
|
||||
expect(result.info.compressed).toBe(false);
|
||||
expect(result.info.reason).toMatch(/below_min_chars|not_profitable/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Task 3: Responses API transformer ───────────────────────────────────────
|
||||
|
||||
const BIG_INSTRUCTIONS = 'These are detailed instructions. '.repeat(600); // ~20k chars
|
||||
const BIG_FLAT_TOOL_DESC = 'Flat tool description with lots of context. '.repeat(200); // ~8k chars
|
||||
|
||||
describe('transformOpenAIResponses (gpt-5.6)', () => {
|
||||
it('compresses instructions + flat tools, injects input_image parts into first user item', async () => {
|
||||
const body = enc.encode(JSON.stringify({
|
||||
model: 'gpt-5.6',
|
||||
instructions: BIG_INSTRUCTIONS,
|
||||
input: [
|
||||
{ role: 'user', content: 'Please do the thing.' },
|
||||
],
|
||||
tools: [{
|
||||
type: 'function',
|
||||
name: 'do_thing',
|
||||
description: BIG_FLAT_TOOL_DESC,
|
||||
parameters: { type: 'object', description: 'Param root.', properties: { x: { type: 'string', description: 'x param' } } },
|
||||
}],
|
||||
}));
|
||||
|
||||
const result = await transformOpenAIResponses(body, { charsPerToken: 1, minCompressChars: 1 });
|
||||
expect(result.info.compressed).toBe(true);
|
||||
expect(result.info.imageCount).toBeGreaterThan(0);
|
||||
|
||||
const out = JSON.parse(dec.decode(result.body)) as Record<string, unknown>;
|
||||
// instructions replaced with pointer.
|
||||
expect(out.instructions as string).toContain('rendered into image');
|
||||
expect(out.instructions as string).not.toContain('These are detailed');
|
||||
|
||||
// First user item gains input_image parts.
|
||||
const inputItems = out.input as Array<{ role: string; content: unknown }>;
|
||||
const firstUser = inputItems.find((i) => i.role === 'user')!;
|
||||
expect(Array.isArray(firstUser.content)).toBe(true);
|
||||
const parts = firstUser.content as Array<{ type: string; image_url?: string }>;
|
||||
expect(parts[0]!.type).toBe('input_image');
|
||||
expect(parts[0]!.image_url).toMatch(/^data:image\/png;base64,/);
|
||||
|
||||
// Flat tool description replaced.
|
||||
const tools = out.tools as Array<{ description?: string; parameters?: { description?: string } }>;
|
||||
expect(tools[0]!.description).toBe('See rendered tool docs image.');
|
||||
expect((tools[0]!.parameters as { description?: string } | undefined)?.description).toBeUndefined();
|
||||
});
|
||||
|
||||
it('handles bare string input (wraps into user item with images)', async () => {
|
||||
const body = enc.encode(JSON.stringify({
|
||||
model: 'gpt-5.6',
|
||||
instructions: BIG_INSTRUCTIONS,
|
||||
input: 'Do the thing please.',
|
||||
}));
|
||||
|
||||
const result = await transformOpenAIResponses(body, { charsPerToken: 1, minCompressChars: 1 });
|
||||
expect(result.info.compressed).toBe(true);
|
||||
|
||||
const out = JSON.parse(dec.decode(result.body)) as Record<string, unknown>;
|
||||
// input should now be an array.
|
||||
expect(Array.isArray(out.input)).toBe(true);
|
||||
const inputItems = out.input as Array<{ role: string; content: Array<{ type: string; text?: string }> }>;
|
||||
expect(inputItems[0]!.role).toBe('user');
|
||||
const parts = inputItems[0]!.content;
|
||||
expect(parts[0]!.type).toBe('input_image');
|
||||
// Original string preserved as input_text part.
|
||||
const textParts = parts.filter((p) => p.type === 'input_text');
|
||||
expect(textParts.some((p) => p.text?.includes('Do the thing'))).toBe(true);
|
||||
});
|
||||
|
||||
it('returns compressed=false with not_profitable/below_min reason for small input', async () => {
|
||||
const body = enc.encode(JSON.stringify({
|
||||
model: 'gpt-5.6',
|
||||
instructions: 'Short.',
|
||||
input: [{ role: 'user', content: 'hi' }],
|
||||
}));
|
||||
const result = await transformOpenAIResponses(body);
|
||||
expect(result.info.compressed).toBe(false);
|
||||
expect(result.info.reason).toMatch(/below_min_chars|not_profitable/);
|
||||
});
|
||||
});
|
||||
@@ -74,11 +74,18 @@ describe('public library API', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('recognizes only the GPT 5.5 family for OpenAI chat support', () => {
|
||||
it('recognizes the full GPT-5 family for OpenAI chat support', () => {
|
||||
expect(isPxpipeSupportedGptModel('gpt-5')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.5')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.5-codex')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.5-2026-06-01')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.1')).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.6')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5-mini')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.6-nano')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-5.6[1m]')).toBe(true);
|
||||
expect(isPxpipeSupportedGptModel('gpt-4o')).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel('gpt-50')).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel('')).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel('claude-opus-4-8')).toBe(false);
|
||||
expect(isPxpipeSupportedGptModel(null)).toBe(false);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user