ai-space/pxpipe
3
0
Fork 0
mirror of https://github.com/teamchong/pxpipe.git synced 2026-07-22 02:02:51 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
pxpipe/eval/sol-profile/results.json
T

325 lines
16 KiB
JSON
Raw Permalink Blame History

{
"generatedAt": "2026-07-09T21:56:55.765Z",
"model": "gpt-5.6-sol",
"live": true,
"preflightSha256": "78288b351deea4bce2e21f018c255904770e78864040a8771ed6e08f94e58179",
"plannedCalls": 4,
"completedCalls": 3,
"stoppedEarly": true,
"stopReason": "clear acceptance failure on alpha/old_shared",
"calls": [
{
"sequence": 1,
"fixture": "alpha",
"profile": "current_sol",
"imageDimensions": [
"764x1724"
],
"estimatedImageTokens": 1296,
"estimatedSavingsVsTextPct": 73.5,
"latencyMs": 13857,
"httpStatus": 200,
"responseHeaders": {
"x-request-id": "5dbb1379-3a2b-49c1-88e6-2ef4baa5aded",
"openai-processing-ms": "13292",
"cf-ray": "a18a94dc3f3efce9-AUS-DOG"
},
"transportError": null,
"usage": {
"input_tokens": 1748,
"input_tokens_details": {
"cache_write_tokens": 1745,
"cached_tokens": 0
},
"output_tokens": 512,
"output_tokens_details": {
"reasoning_tokens": 512
},
"total_tokens": 2260
},
"responseId": "resp_0413981843da1074016a501928820c8190a2a8c656f84e1c75",
"responseModel": "gpt-5.6-sol",
"rawResponseBody": "{\n \"id\": \"resp_0413981843da1074016a501928820c8190a2a8c656f84e1c75\",\n \"object\": \"response\",\n \"created_at\": 1783634216,\n \"status\": \"incomplete\",\n \"background\": false,\n \"billing\": {\n \"payer\": \"developer\"\n },\n \"completed_at\": null,\n \"error\": null,\n \"frequency_penalty\": 0.0,\n \"incomplete_details\": {\n \"reason\": \"max_output_tokens\"\n },\n \"instructions\": null,\n \"max_output_tokens\": 512,\n \"max_tool_calls\": null,\n \"model\": \"gpt-5.6-sol\",\n \"moderation\": null,\n \"output\": [\n {\n \"id\": \"rs_0413981843da1074016a50192976748190bb8cf1789c294ae6\",\n \"type\": \"reasoning\",\n \"content\": [],\n \"summary\": []\n }\n ],\n \"parallel_tool_calls\": true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\": \"in_memory\",\n \"reasoning\": {\n \"context\": \"all_turns\",\n \"effort\": \"low\",\n \"mode\": \"standard\",\n \"summary\": null\n },\n \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": false,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": \"auto\",\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": 0,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"output_tokens\": 0,\n \"output_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"total_tokens\": 0\n },\n \"web_search\": {\n \"num_requests\": 0\n }\n },\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 0.98,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": 1748,\n \"input_tokens_details\": {\n \"cache_write_tokens\": 1745,\n \"cached_tokens\": 0\n },\n \"output_tokens\": 512,\n \"output_tokens_details\": {\n \"reasoning_tokens\": 512\n },\n \"total_tokens\": 2260\n },\n \"user\": null,\n \"metadata\": {}\n}",
"rawResponseSha256": "07afb8a3fb7d685a8d480c5efa31697a9b51bb0ecb09720858cdcee7e7dbec4a",
"score": {
"responseStatus": "incomplete",
"incompleteReason": "max_output_tokens",
"outputText": "",
"refusals": [],
"parseError": "empty output",
"strictJson": false,
"schemaValid": false,
"exact": {
"fingerprint": {
"expected": "c7a1e90b4d2f",
"got": "",
"ok": false
},
"camelCase": {
"expected": "retryBudgetSeconds",
"got": "",
"ok": false
},
"path": {
"expected": "/srv/sol-pilot/releases/alpha-07/config/runtime-map.json",
"got": "",
"ok": false
},
"port": {
"expected": "47831",
"got": "",
"ok": false
}
},
"exactCorrect": 0,
"exactTotal": 4,
"gist": {
"expected": "B",
"got": "",
"ok": false
},
"guard": {
"expected": "NOT STATED",
"got": "",
"ok": false
},
"confabulations": [],
"abstentions": [],
"pass": false
},
"reasoningEffort": "low"
},
{
"sequence": 2,
"fixture": "alpha",
"profile": "current_sol",
"imageDimensions": [
"764x1724"
],
"estimatedImageTokens": 1296,
"estimatedSavingsVsTextPct": 73.5,
"latencyMs": 3294,
"httpStatus": 200,
"responseHeaders": {
"x-request-id": "fd696db0-302f-4065-9714-209dbcaab732",
"openai-processing-ms": "2948",
"cf-ray": "a18a9751a965fce9-AUS-DOG"
},
"transportError": null,
"usage": {
"input_tokens": 1748,
"input_tokens_details": {
"cache_write_tokens": 1745,
"cached_tokens": 0
},
"output_tokens": 52,
"output_tokens_details": {
"reasoning_tokens": 0
},
"total_tokens": 1800
},
"responseId": "resp_020acbb2cc7a89da016a50198d3364819387c192a9f4b370b9",
"responseModel": "gpt-5.6-sol",
"reasoningEffort": "none",
"rawResponseBody": "{\n \"id\": \"resp_020acbb2cc7a89da016a50198d3364819387c192a9f4b370b9\",\n \"object\": \"response\",\n \"created_at\": 1783634317,\n \"status\": \"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1783634320,\n \"error\": null,\n \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": null,\n \"max_output_tokens\": 512,\n \"max_tool_calls\": null,\n \"model\": \"gpt-5.6-sol\",\n \"moderation\": null,\n \"output\": [\n {\n \"id\": \"msg_020acbb2cc7a89da016a50198edbdc819383c8352f6b908521\",\n \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": [\n {\n \"type\": \"output_text\",\n \"annotations\": [],\n \"logprobs\": [],\n \"text\": \"{\\\"fingerprint\\\":\\\"c0ffee0913a7\\\",\\\"camelCase\\\":\\\"matrixField\\\",\\\"path\\\":\\\"/srv/app/releases/alpha-07/config/runtime.yaml\\\",\\\"port\\\":\\\"41702\\\",\\\"gist\\\":\\\"B\\\",\\\"unstated\\\":\\\"NOT STATED\\\"}\"\n }\n ],\n \"phase\": \"final_answer\",\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\": \"in_memory\",\n \"reasoning\": {\n \"context\": \"all_turns\",\n \"effort\": \"none\",\n \"mode\": \"standard\",\n \"summary\": null\n },\n \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": false,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": \"text\"\n },\n \"verbosity\": \"low\"\n },\n \"tool_choice\": \"auto\",\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": 0,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"output_tokens\": 0,\n \"output_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"total_tokens\": 0\n },\n \"web_search\": {\n \"num_requests\": 0\n }\n },\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 0.98,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": 1748,\n \"input_tokens_details\": {\n \"cache_write_tokens\": 1745,\n \"cached_tokens\": 0\n },\n \"output_tokens\": 52,\n \"output_tokens_details\": {\n \"reasoning_tokens\": 0\n },\n \"total_tokens\": 1800\n },\n \"user\": null,\n \"metadata\": {}\n}",
"rawResponseSha256": "492e540e233eb5dd21caf87a6246cec0c62d866fc8dca6e57b8cca019918dad9",
"score": {
"responseStatus": "completed",
"incompleteReason": null,
"outputText": "{\"fingerprint\":\"c0ffee0913a7\",\"camelCase\":\"matrixField\",\"path\":\"/srv/app/releases/alpha-07/config/runtime.yaml\",\"port\":\"41702\",\"gist\":\"B\",\"unstated\":\"NOT STATED\"}",
"refusals": [],
"parseError": null,
"strictJson": true,
"schemaValid": true,
"exact": {
"fingerprint": {
"expected": "c7a1e90b4d2f",
"got": "c0ffee0913a7",
"ok": false
},
"camelCase": {
"expected": "retryBudgetSeconds",
"got": "matrixField",
"ok": false
},
"path": {
"expected": "/srv/sol-pilot/releases/alpha-07/config/runtime-map.json",
"got": "/srv/app/releases/alpha-07/config/runtime.yaml",
"ok": false
},
"port": {
"expected": "47831",
"got": "41702",
"ok": false
}
},
"exactCorrect": 0,
"exactTotal": 4,
"gist": {
"expected": "B",
"got": "B",
"ok": true
},
"guard": {
"expected": "NOT STATED",
"got": "NOT STATED",
"ok": true
},
"confabulations": [
{
"field": "fingerprint",
"value": "c0ffee0913a7"
},
{
"field": "camelCase",
"value": "matrixField"
},
{
"field": "path",
"value": "/srv/app/releases/alpha-07/config/runtime.yaml"
},
{
"field": "port",
"value": "41702"
}
],
"abstentions": [],
"pass": false
}
},
{
"sequence": 3,
"fixture": "alpha",
"profile": "old_shared",
"imageDimensions": [
"768x1040"
],
"estimatedImageTokens": 792,
"estimatedSavingsVsTextPct": 83.8,
"latencyMs": 2429,
"httpStatus": 200,
"responseHeaders": {
"x-request-id": "ff15ed67-d378-4e1b-ad9a-d2d3d9c2be9d",
"openai-processing-ms": "2034",
"cf-ray": "a18a9b58c937fce9-AUS-DOG"
},
"transportError": null,
"usage": {
"input_tokens": 1143,
"input_tokens_details": {
"cache_write_tokens": 1140,
"cached_tokens": 0
},
"output_tokens": 59,
"output_tokens_details": {
"reasoning_tokens": 0
},
"total_tokens": 1202
},
"responseId": "resp_083fde5dff13ac2c016a501a3242348197882ae33f8b15e29a",
"responseModel": "gpt-5.6-sol",
"reasoningEffort": "none",
"rawResponseBody": "{\n \"id\": \"resp_083fde5dff13ac2c016a501a3242348197882ae33f8b15e29a\",\n \"object\": \"response\",\n \"created_at\": 1783634482,\n \"status\": \"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1783634484,\n \"error\": null,\n \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": null,\n \"max_output_tokens\": 512,\n \"max_tool_calls\": null,\n \"model\": \"gpt-5.6-sol\",\n \"moderation\": null,\n \"output\": [\n {\n \"id\": \"msg_083fde5dff13ac2c016a501a3331948197afa081cd8f933ab0\",\n \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": [\n {\n \"type\": \"output_text\",\n \"annotations\": [],\n \"logprobs\": [],\n \"text\": \"{\\\"fingerprint\\\":\\\"7ac089b47c21\\\",\\\"camelCase\\\":\\\"fluxLimiterCeiling\\\",\\\"path\\\":\\\"/srv/releases/2025-04-18/edge-router/manifest.json\\\",\\\"port\\\":\\\"41820\\\",\\\"gist\\\":\\\"A\\\",\\\"unstated\\\":\\\"NOT STATED\\\"}\"\n }\n ],\n \"phase\": \"final_answer\",\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": null,\n \"prompt_cache_retention\": \"in_memory\",\n \"reasoning\": {\n \"context\": \"all_turns\",\n \"effort\": \"none\",\n \"mode\": \"standard\",\n \"summary\": null\n },\n \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": false,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": \"text\"\n },\n \"verbosity\": \"low\"\n },\n \"tool_choice\": \"auto\",\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": 0,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"output_tokens\": 0,\n \"output_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"total_tokens\": 0\n },\n \"web_search\": {\n \"num_requests\": 0\n }\n },\n \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 0.98,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": 1143,\n \"input_tokens_details\": {\n \"cache_write_tokens\": 1140,\n \"cached_tokens\": 0\n },\n \"output_tokens\": 59,\n \"output_tokens_details\": {\n \"reasoning_tokens\": 0\n },\n \"total_tokens\": 1202\n },\n \"user\": null,\n \"metadata\": {}\n}",
"rawResponseSha256": "5ede01fba8670a28b80d9f7ce3bb7b60ae4d86ef50826afc8c787f59b45dfbdc",
"score": {
"responseStatus": "completed",
"incompleteReason": null,
"outputText": "{\"fingerprint\":\"7ac089b47c21\",\"camelCase\":\"fluxLimiterCeiling\",\"path\":\"/srv/releases/2025-04-18/edge-router/manifest.json\",\"port\":\"41820\",\"gist\":\"A\",\"unstated\":\"NOT STATED\"}",
"refusals": [],
"parseError": null,
"strictJson": true,
"schemaValid": true,
"exact": {
"fingerprint": {
"expected": "c7a1e90b4d2f",
"got": "7ac089b47c21",
"ok": false
},
"camelCase": {
"expected": "retryBudgetSeconds",
"got": "fluxLimiterCeiling",
"ok": false
},
"path": {
"expected": "/srv/sol-pilot/releases/alpha-07/config/runtime-map.json",
"got": "/srv/releases/2025-04-18/edge-router/manifest.json",
"ok": false
},
"port": {
"expected": "47831",
"got": "41820",
"ok": false
}
},
"exactCorrect": 0,
"exactTotal": 4,
"gist": {
"expected": "B",
"got": "A",
"ok": false
},
"guard": {
"expected": "NOT STATED",
"got": "NOT STATED",
"ok": true
},
"confabulations": [
{
"field": "fingerprint",
"value": "7ac089b47c21"
},
{
"field": "camelCase",
"value": "fluxLimiterCeiling"
},
{
"field": "path",
"value": "/srv/releases/2025-04-18/edge-router/manifest.json"
},
{
"field": "port",
"value": "41820"
}
],
"abstentions": [],
"pass": false
}
}
],
"resume": {
"reason": "first attempt returned only hidden reasoning and hit max_output_tokens",
"reviewedAt": "2026-07-09T21:58:36.732Z",
"priorAttemptsCountAgainstCap": true,
"changedReasoningEffort": "none",
"remainingOrder": [
{
"fixture": "alpha",
"profile": "current_sol"
},
{
"fixture": "alpha",
"profile": "old_shared"
},
{
"fixture": "beta",
"profile": "current_sol"
}
]
},
"retune": {
"reason": "current Sol 6x11 profile returned 0/4 exact with four confabulations",
"reviewedAt": "2026-07-09T22:01:21.706Z",
"candidate": "Sol-only fallback to old shared Spleen 5x8 / 152 columns",
"remainingOrder": [
{
"fixture": "alpha",
"profile": "old_shared"
},
{
"fixture": "beta",
"profile": "old_shared"
}
]
}
}
Reference in New Issue View Git Blame Copy Permalink