mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
feat(gate): config-driven model scope (PXPIPE_MODELS) + Opus 4.8 re-test
- applicability: isPxpipeSupportedModel reads PXPIPE_MODELS (comma-sep, default claude-fable-5 only) and strips bracket/variant tags like [1m]; Opus becomes opt-in with no code change. Default behavior unchanged. - FINDINGS: Opus 4.8 re-test - arithmetic 98/100 (-2pp), verbatim 6/15 (was 93/100, 0/15): improved but still taxed -> default stays Fable-only. - FINDINGS + eval/glyph-matrix/sweep: root cause of the read tax is per-glyph RESOLUTION, not font shape (confusions scale with px and vanish by 10x16). Opus needs ~4x glyph area to read reliably; the 1568px ceiling makes that break-even, so there is no profitable Opus operating point.
This commit is contained in:
@@ -14,16 +14,48 @@ export interface PxpipeApplicabilityInput {
|
||||
readonly bodyBytes?: number | null;
|
||||
}
|
||||
|
||||
/** Pxpipe's validated production scope: Fable 5 only.
|
||||
* Measured 2026-06-09: Fable 5 reads pxpipe renders at 100/100 on the
|
||||
* novel-arithmetic eval (Opus 4.8: 93/100) and bills the same image tokens
|
||||
* (w·h/750, same tokenizer as Opus 4.7+). Opus is disabled — its ~7% read
|
||||
* tax is the wrong trade now that a tax-free model exists. Mythos 5 is
|
||||
* unmeasured (no access). Suffix aliases such as `claude-fable-5-high` are
|
||||
* accepted because hosts may check either the client alias or the resolved
|
||||
* upstream model. */
|
||||
/** Bracketed variant tags that don't change reading behavior and so must not
|
||||
* affect the gate — e.g. the context-window tag in `claude-opus-4-8[1m]`.
|
||||
* Stripped before matching so a base model and its `[1m]` form gate alike. */
|
||||
const VARIANT_TAG = /\[[^\]]*\]/g;
|
||||
|
||||
function baseModelId(model: string): string {
|
||||
return model.replace(VARIANT_TAG, '');
|
||||
}
|
||||
|
||||
/** Base model ids pxpipe is allowed to transform, from `PXPIPE_MODELS`
|
||||
* (comma-separated). Defaults to Fable 5 only — the validated production
|
||||
* scope. Read per call so the scope can be widened/narrowed by env alone,
|
||||
* no rebuild or restart.
|
||||
*
|
||||
* Validated 2026-06-09: Fable 5 reads pxpipe renders at 100/100 on the
|
||||
* novel-arithmetic eval (Opus 4.8: 93/100) and bills the same image tokens
|
||||
* (w·h/750, same tokenizer as Opus 4.7+). Opus was the original scope but
|
||||
* carried a ~7% read tax, so it was dropped once a tax-free model existed.
|
||||
* Re-tested 2026-06-16 on a newer Opus 4.8 snapshot: improved to 98/100
|
||||
* (-2pp) on arithmetic and 6/15 (was 0/15) on dense-hex recall, but still
|
||||
* taxed and still silently confabulating vs Fable's 100/100 and 13/15 — so
|
||||
* the default stays Fable-only.
|
||||
* To re-enable it (e.g. while re-evaluating a newer Opus snapshot):
|
||||
* PXPIPE_MODELS=claude-fable-5,claude-opus-4-8
|
||||
* Mythos 5 is unmeasured (no access). */
|
||||
function allowedModelBases(): string[] {
|
||||
const raw = process.env.PXPIPE_MODELS;
|
||||
return (raw && raw.trim() ? raw : 'claude-fable-5')
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** True when pxpipe is allowed to transform requests for this model. A model
|
||||
* matches an allowed base when it equals the base or extends it with a
|
||||
* `-suffix` alias (`claude-fable-5-high`) — hosts may send either the client
|
||||
* alias or the resolved upstream id. Bracketed variant tags (`[1m]`) are
|
||||
* stripped first so `claude-opus-4-8[1m]` matches its base. */
|
||||
export function isPxpipeSupportedModel(model: string | null | undefined): boolean {
|
||||
return typeof model === 'string' && /^claude-fable-5(?:-|$)/.test(model);
|
||||
if (typeof model !== 'string') return false;
|
||||
const base = baseModelId(model);
|
||||
return allowedModelBases().some((b) => base === b || base.startsWith(`${b}-`));
|
||||
}
|
||||
|
||||
/** GPT image-tokenization has not been validated across the whole OpenAI
|
||||
|
||||
Reference in New Issue
Block a user