ai-space/pxpipe
3
0
Fork 0
mirror of https://github.com/teamchong/pxpipe.git synced 2026-07-22 02:02:51 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
pxpipe/eval/grok-density/results.json
T
Steven Chong 056f970881 test(eval): measure Grok image recall and the factsheet contract 
Grok was imaging on the Responses path with no readability data. Add
eval/grok-density: density ladder, climb receipts, and a live
image-vs-factsheet comparison on grok-4.5.

Findings locked into the receipts:
- Production 5x8 image-only confabulates exact IDs (0/4).
- Image + factsheet clears 4/4 exact, 0 confab, gist+guard ok (~70%
  fixture savings).
- Looser pure-image packing can clear exact recall but only ~30%
  savings — not worth the default.

Supports keeping Grok opt-in with dense 5x8 + factsheet when enabled.
2026-07-09 12:44:14 -04:00

314 lines
7.7 KiB
JSON
Raw Permalink Blame History

{
"generatedAt": "2026-07-09T11:36:45.973Z",
"model": "grok-4.5",
"profile": {
"vision": {
"regime": "tile",
"base": 85,
"perTile": 170
},
"stripCols": 152,
"maxHeightPx": 1932
},
"textTokens": 1168,
"live": true,
"variants": [
{
"variant": "5x8",
"cols": 152,
"pages": 1,
"dims": [
"768x360"
],
"imageTokens": 425,
"savingsPct": 64,
"model": {
"exactCorrect": 0,
"exactTotal": 4,
"confab": 4,
"abstain": 1,
"refused": 0,
"gistOk": true,
"guardOk": true,
"answers": [
{
"id": "hex",
"kind": "exact",
"expected": "a3f9c1e0b7d2",
"got": "5c5eacb0a2",
"status": "completed",
"incomplete": null,
"ok": false,
"abstained": false,
"confab": true,
"refused": false,
"ms": 9158
},
{
"id": "camel",
"kind": "exact",
"expected": "tokenLedgerShard",
"got": "tokenBudget",
"status": "completed",
"incomplete": null,
"ok": false,
"abstained": false,
"confab": true,
"refused": false,
"ms": 20045
},
{
"id": "path",
"kind": "exact",
"expected": "src/core/anthropic-vision.ts",
"got": "pro/core/anthropic-client.ts",
"status": "completed",
"incomplete": null,
"ok": false,
"abstained": false,
"confab": true,
"refused": false,
"ms": 3344
},
{
"id": "port",
"kind": "exact",
"expected": "47821",
"got": "97821",
"status": "completed",
"incomplete": null,
"ok": false,
"abstained": false,
"confab": true,
"refused": false,
"ms": 2370
},
{
"id": "gist",
"kind": "gist",
"expected": "3",
"got": "3",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 2100
},
{
"id": "guard",
"kind": "guard",
"expected": "NOT STATED",
"got": "NOT STATED",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": true,
"confab": false,
"refused": false,
"ms": 5222
}
]
}
},
{
"variant": "7x10",
"cols": 108,
"pages": 1,
"dims": [
"764x458"
],
"imageTokens": 425,
"savingsPct": 64,
"model": {
"exactCorrect": 3,
"exactTotal": 4,
"confab": 1,
"abstain": 1,
"refused": 0,
"gistOk": true,
"guardOk": true,
"answers": [
{
"id": "hex",
"kind": "exact",
"expected": "a3f9c1e0b7d2",
"got": "a03c1e0b7d2",
"status": "completed",
"incomplete": null,
"ok": false,
"abstained": false,
"confab": true,
"refused": false,
"ms": 37783
},
{
"id": "camel",
"kind": "exact",
"expected": "tokenLedgerShard",
"got": "tokenLedgerShard",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 4407
},
{
"id": "path",
"kind": "exact",
"expected": "src/core/anthropic-vision.ts",
"got": "src/core/anthropic-vision.ts",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 3398
},
{
"id": "port",
"kind": "exact",
"expected": "47821",
"got": "47821",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 16378
},
{
"id": "gist",
"kind": "gist",
"expected": "3",
"got": "3",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 2461
},
{
"id": "guard",
"kind": "guard",
"expected": "NOT STATED",
"got": "NOT STATED",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": true,
"confab": false,
"refused": false,
"ms": 7416
}
]
}
},
{
"variant": "9x12",
"cols": 84,
"pages": 1,
"dims": [
"764x1064"
],
"imageTokens": 1105,
"savingsPct": 5,
"model": {
"exactCorrect": 4,
"exactTotal": 4,
"confab": 0,
"abstain": 1,
"refused": 0,
"gistOk": true,
"guardOk": true,
"answers": [
{
"id": "hex",
"kind": "exact",
"expected": "a3f9c1e0b7d2",
"got": "a3f9c1e0b7d2",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 7167
},
{
"id": "camel",
"kind": "exact",
"expected": "tokenLedgerShard",
"got": "tokenLedgerShard",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 725
},
{
"id": "path",
"kind": "exact",
"expected": "src/core/anthropic-vision.ts",
"got": "src/core/anthropic-vision.ts",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 899
},
{
"id": "port",
"kind": "exact",
"expected": "47821",
"got": "47821",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 4805
},
{
"id": "gist",
"kind": "gist",
"expected": "3",
"got": "3",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": false,
"confab": false,
"refused": false,
"ms": 831
},
{
"id": "guard",
"kind": "guard",
"expected": "NOT STATED",
"got": "NOT STATED",
"status": "completed",
"incomplete": null,
"ok": true,
"abstained": true,
"confab": false,
"refused": false,
"ms": 10402
}
]
}
}
]
}
Reference in New Issue View Git Blame Copy Permalink