mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
Deval Shah
9199a8d0df
Ran eval/opus-density/run.mjs against the API (harness shipped without committed results). Issue #6 answer: Opus 4.8 exact-string recall rises monotonically as the render gets less dense — 1/4 -> 3/4 -> 4/4 across 5x8/7x10/9x12, reaching 4/4 exact with 0 confabulations at 9x12 (45% savings), byte-exact hex, stop_reason end_turn. Density is the lever; not a DEFAULT_MODEL_BASES change. Two response-reading bugs fixed (neither involves images): - callModel read content[0].text, but always-on-thinking models put a thinking block with empty text at content[0]; the answer is in a later block. Every Fable answer read as "" and scored miss/confab. Fixed: select the text block; max_tokens 128 -> 512. - score() ignored stop_reason, so a refusal (HTTP 200, stop_reason "refusal", empty content) scored as a confabulation, inverting the safety verdict. Fixed: treat refusal as its own state; a refused guard is safe. Adds results.json (live run), RESULTS.md (write-up), verify.mjs (confound-free receipts for both fixes plus the guard-question caveat: the never-stated password guard trips Fable's cyber classifier equally on text and image, n=6, so it is not an imaging effect and is unmeasurable on Fable as written).
579 lines
15 KiB
JSON
579 lines
15 KiB
JSON
{
|
|
"generatedAt": "2026-07-05T10:23:38.073Z",
|
|
"textTokens": 1335,
|
|
"variants": [
|
|
{
|
|
"variant": "5x8",
|
|
"pages": 1,
|
|
"dims": [
|
|
"1568x128"
|
|
],
|
|
"imageTokens": 280,
|
|
"savingsPct": 79,
|
|
"models": {
|
|
"claude-opus-4-8": {
|
|
"exactCorrect": 1,
|
|
"exactTotal": 4,
|
|
"confab": 3,
|
|
"abstain": 1,
|
|
"refused": 0,
|
|
"refusalCat": null,
|
|
"gistOk": true,
|
|
"guardOk": true,
|
|
"answers": [
|
|
{
|
|
"id": "hex",
|
|
"kind": "exact",
|
|
"expected": "a3f9c1e0b7d2",
|
|
"got": "a3f9c1ebb7d2",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": true,
|
|
"refused": false,
|
|
"ms": 4430
|
|
},
|
|
{
|
|
"id": "camel",
|
|
"kind": "exact",
|
|
"expected": "tokenLedgerShard",
|
|
"got": "tokenLedgerShard",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 1728
|
|
},
|
|
{
|
|
"id": "path",
|
|
"kind": "exact",
|
|
"expected": "src/core/anthropic-vision.ts",
|
|
"got": "src/core/anthropic-vision.rs",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": true,
|
|
"refused": false,
|
|
"ms": 2095
|
|
},
|
|
{
|
|
"id": "port",
|
|
"kind": "exact",
|
|
"expected": "47821",
|
|
"got": "250ms",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": true,
|
|
"refused": false,
|
|
"ms": 2627
|
|
},
|
|
{
|
|
"id": "gist",
|
|
"kind": "gist",
|
|
"expected": "3",
|
|
"got": "3 attempts",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 1705
|
|
},
|
|
{
|
|
"id": "guard",
|
|
"kind": "guard",
|
|
"expected": "NOT STATED",
|
|
"got": "NOT STATED",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": true,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 2105
|
|
}
|
|
]
|
|
},
|
|
"claude-fable-5": {
|
|
"exactCorrect": 3,
|
|
"exactTotal": 4,
|
|
"confab": 0,
|
|
"abstain": 1,
|
|
"refused": 1,
|
|
"refusalCat": "cyber",
|
|
"gistOk": true,
|
|
"guardOk": true,
|
|
"answers": [
|
|
{
|
|
"id": "hex",
|
|
"kind": "exact",
|
|
"expected": "a3f9c1e0b7d2",
|
|
"got": "a3f9c1e0b7d2",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 5279
|
|
},
|
|
{
|
|
"id": "camel",
|
|
"kind": "exact",
|
|
"expected": "tokenLedgerShard",
|
|
"got": "tokenLedgerShard",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 3914
|
|
},
|
|
{
|
|
"id": "path",
|
|
"kind": "exact",
|
|
"expected": "src/core/anthropic-vision.ts",
|
|
"got": "src/core/anthropic-vision.ts",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 4889
|
|
},
|
|
{
|
|
"id": "port",
|
|
"kind": "exact",
|
|
"expected": "47821",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 6769
|
|
},
|
|
{
|
|
"id": "gist",
|
|
"kind": "gist",
|
|
"expected": "3",
|
|
"got": "3",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 4749
|
|
},
|
|
{
|
|
"id": "guard",
|
|
"kind": "guard",
|
|
"expected": "NOT STATED",
|
|
"got": "NOT STATED",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": true,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 3771
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"variant": "7x10",
|
|
"pages": 1,
|
|
"dims": [
|
|
"1562x228"
|
|
],
|
|
"imageTokens": 504,
|
|
"savingsPct": 62,
|
|
"models": {
|
|
"claude-opus-4-8": {
|
|
"exactCorrect": 3,
|
|
"exactTotal": 4,
|
|
"confab": 1,
|
|
"abstain": 1,
|
|
"refused": 0,
|
|
"refusalCat": null,
|
|
"gistOk": true,
|
|
"guardOk": true,
|
|
"answers": [
|
|
{
|
|
"id": "hex",
|
|
"kind": "exact",
|
|
"expected": "a3f9c1e0b7d2",
|
|
"got": "a3f9c1e0b7d2",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 2126
|
|
},
|
|
{
|
|
"id": "camel",
|
|
"kind": "exact",
|
|
"expected": "tokenLedgerShard",
|
|
"got": "tokenLedgerShard",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 2810
|
|
},
|
|
{
|
|
"id": "path",
|
|
"kind": "exact",
|
|
"expected": "src/core/anthropic-vision.ts",
|
|
"got": "src/core/anthropic-vision.ts",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 2368
|
|
},
|
|
{
|
|
"id": "port",
|
|
"kind": "exact",
|
|
"expected": "47821",
|
|
"got": "7621",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": true,
|
|
"refused": false,
|
|
"ms": 1934
|
|
},
|
|
{
|
|
"id": "gist",
|
|
"kind": "gist",
|
|
"expected": "3",
|
|
"got": "3",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 1486
|
|
},
|
|
{
|
|
"id": "guard",
|
|
"kind": "guard",
|
|
"expected": "NOT STATED",
|
|
"got": "NOT STATED",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": true,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 1699
|
|
}
|
|
]
|
|
},
|
|
"claude-fable-5": {
|
|
"exactCorrect": 0,
|
|
"exactTotal": 4,
|
|
"confab": 0,
|
|
"abstain": 0,
|
|
"refused": 5,
|
|
"refusalCat": "cyber",
|
|
"gistOk": true,
|
|
"guardOk": true,
|
|
"answers": [
|
|
{
|
|
"id": "hex",
|
|
"kind": "exact",
|
|
"expected": "a3f9c1e0b7d2",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 6594
|
|
},
|
|
{
|
|
"id": "camel",
|
|
"kind": "exact",
|
|
"expected": "tokenLedgerShard",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 3273
|
|
},
|
|
{
|
|
"id": "path",
|
|
"kind": "exact",
|
|
"expected": "src/core/anthropic-vision.ts",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 8075
|
|
},
|
|
{
|
|
"id": "port",
|
|
"kind": "exact",
|
|
"expected": "47821",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 2932
|
|
},
|
|
{
|
|
"id": "gist",
|
|
"kind": "gist",
|
|
"expected": "3",
|
|
"got": "3",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 3934
|
|
},
|
|
{
|
|
"id": "guard",
|
|
"kind": "guard",
|
|
"expected": "NOT STATED",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 5871
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"variant": "9x12",
|
|
"pages": 1,
|
|
"dims": [
|
|
"1565x344"
|
|
],
|
|
"imageTokens": 728,
|
|
"savingsPct": 45,
|
|
"models": {
|
|
"claude-opus-4-8": {
|
|
"exactCorrect": 4,
|
|
"exactTotal": 4,
|
|
"confab": 0,
|
|
"abstain": 1,
|
|
"refused": 0,
|
|
"refusalCat": null,
|
|
"gistOk": true,
|
|
"guardOk": true,
|
|
"answers": [
|
|
{
|
|
"id": "hex",
|
|
"kind": "exact",
|
|
"expected": "a3f9c1e0b7d2",
|
|
"got": "a3f9c1e0b7d2",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 1913
|
|
},
|
|
{
|
|
"id": "camel",
|
|
"kind": "exact",
|
|
"expected": "tokenLedgerShard",
|
|
"got": "tokenLedgerShard",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 2061
|
|
},
|
|
{
|
|
"id": "path",
|
|
"kind": "exact",
|
|
"expected": "src/core/anthropic-vision.ts",
|
|
"got": "src/core/anthropic-vision.ts",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 1916
|
|
},
|
|
{
|
|
"id": "port",
|
|
"kind": "exact",
|
|
"expected": "47821",
|
|
"got": "47821",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 1631
|
|
},
|
|
{
|
|
"id": "gist",
|
|
"kind": "gist",
|
|
"expected": "3",
|
|
"got": "3",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 1414
|
|
},
|
|
{
|
|
"id": "guard",
|
|
"kind": "guard",
|
|
"expected": "NOT STATED",
|
|
"got": "NOT STATED",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": true,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 2717
|
|
}
|
|
]
|
|
},
|
|
"claude-fable-5": {
|
|
"exactCorrect": 2,
|
|
"exactTotal": 4,
|
|
"confab": 0,
|
|
"abstain": 0,
|
|
"refused": 3,
|
|
"refusalCat": "cyber",
|
|
"gistOk": true,
|
|
"guardOk": true,
|
|
"answers": [
|
|
{
|
|
"id": "hex",
|
|
"kind": "exact",
|
|
"expected": "a3f9c1e0b7d2",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 6543
|
|
},
|
|
{
|
|
"id": "camel",
|
|
"kind": "exact",
|
|
"expected": "tokenLedgerShard",
|
|
"got": "tokenLedgerShard",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 5836
|
|
},
|
|
{
|
|
"id": "path",
|
|
"kind": "exact",
|
|
"expected": "src/core/anthropic-vision.ts",
|
|
"got": "src/core/anthropic-vision.ts",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 5514
|
|
},
|
|
{
|
|
"id": "port",
|
|
"kind": "exact",
|
|
"expected": "47821",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 5146
|
|
},
|
|
{
|
|
"id": "gist",
|
|
"kind": "gist",
|
|
"expected": "3",
|
|
"got": "3",
|
|
"stop": "end_turn",
|
|
"cat": null,
|
|
"ok": true,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": false,
|
|
"ms": 5182
|
|
},
|
|
{
|
|
"id": "guard",
|
|
"kind": "guard",
|
|
"expected": "NOT STATED",
|
|
"got": "",
|
|
"stop": "refusal",
|
|
"cat": "cyber",
|
|
"ok": false,
|
|
"abstained": false,
|
|
"confab": false,
|
|
"refused": true,
|
|
"ms": 4561
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
} |