mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
teamchong
7af6c200c3
eval: - Add eval/lib/cci.py: drop-in interactive Claude session shim (pexpect/PTY) replacing headless `claude -p`. Drives the real TUI on Max-subscription auth (no ANTHROPIC_API_KEY), scrapes the reply + /context + /cost, and is argv/stdin/--output-format json compatible with the old call sites. - Wire every call site to cci.py: gist-recall (run/run2/run3), needle-haystack (sweep/crux), gsm8k (bench), swe-bench (run_pilot), swe-bench-pro (rep_nav/run_pro), and eval/lib/anthropic-client.mjs. - Isolation per call: --setting-sources project --strict-mcp-config --dangerously-skip-permissions. Recall evals keep --allowedTools Read; swe-bench keeps the full agentic toolset. Each trial is a fresh process. demo: - Resolve the `opus` shorthand to the full claude-opus-4-8[1m] model in cost-ab and effective-context a.sh/b.sh. setup.sh intentionally keeps the base id: the proxy scope gate strips [1m] from the request before matching, so [1m]-on-launch + base-in-scope is the correct pairing. docs: - FINDINGS.md: drop the 2026-06-16 Opus 4.8 re-test sections.
70 lines
2.6 KiB
Python
70 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
import os, sys, subprocess, secrets, re
|
|
from PIL import Image, ImageDraw, ImageFont
|
|
|
|
FONT = "/System/Library/Fonts/SFNSMono.ttf"
|
|
WORK = "/tmp/needle_eval/sweep"
|
|
os.makedirs(WORK, exist_ok=True)
|
|
CCI = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "lib", "cci.py")
|
|
W, H = 1568, 1276 # pxpipe-equivalent dims -> ~2668 image tokens
|
|
IMG_TOKENS = round(W*H/750)
|
|
|
|
FILLER = ("the quick brown fox jumps over the lazy dog while the cache layer "
|
|
"evicts stale keys and the scheduler drains the queue in order ").split()
|
|
|
|
def needle(): return secrets.token_hex(6)
|
|
|
|
def make_lines(pt, n_needle_line, needle_val):
|
|
font = ImageFont.truetype(FONT, pt)
|
|
# chars per line at this font
|
|
cw = font.getbbox("M")[2]
|
|
cpl = max(8, (W-40)//cw)
|
|
lh = int(pt*1.35)
|
|
nlines = (H-40)//lh
|
|
words = (FILLER*200)
|
|
lines, wi = [], 0
|
|
for _ in range(nlines):
|
|
ln = ""
|
|
while wi < len(words) and len(ln)+len(words[wi])+1 <= cpl:
|
|
ln += words[wi]+" "; wi += 1
|
|
lines.append(ln.rstrip())
|
|
mid = nlines//2
|
|
lines[mid] = f"SECRET_TOKEN = {needle_val}"[:cpl]
|
|
chars = sum(len(l) for l in lines)
|
|
return font, lines, lh, chars
|
|
|
|
def render(pt, path, needle_val):
|
|
font, lines, lh, chars = make_lines(pt, None, needle_val)
|
|
img = Image.new("RGB",(W,H),"white"); d=ImageDraw.Draw(img)
|
|
y=20
|
|
for ln in lines:
|
|
d.text((20,y),ln,fill="black",font=font); y+=lh
|
|
img.save(path)
|
|
return chars
|
|
|
|
def ask(path):
|
|
p=(f"Read the image at {path}. What is the value of SECRET_TOKEN? "
|
|
"Reply with ONLY the value, no other words.")
|
|
out=subprocess.run([sys.executable,CCI,"--model","claude-opus-4-8",
|
|
"--allowedTools","Read",p],capture_output=True,text=True,
|
|
timeout=120,cwd=WORK,env=dict(os.environ,CCI_TIMEOUT="100")).stdout
|
|
m=re.search(r'[0-9a-f]{12}',out); return m.group(0) if m else ""
|
|
|
|
def run(pt,n):
|
|
ok=0; chars=0
|
|
for i in range(1,n+1):
|
|
nd=needle(); path=f"{WORK}/{pt}_{i}.png"
|
|
chars=render(pt,path,nd)
|
|
got=ask(path); hit=int(got==nd); ok+=hit
|
|
print(f" {pt}pt trial{i}: {nd} -> {got or '-'} {'OK' if hit else 'x'}")
|
|
# text-equivalent tokens of the chars packed in one image (cc empirical ~3.5 c/tok generic)
|
|
txt_tok=round(chars/3.5)
|
|
ratio=txt_tok/IMG_TOKENS
|
|
print(f">>> {pt}pt: {ok}/{n} ({100*ok/n:.0f}%) | chars/img={chars} "
|
|
f"| img_tok={IMG_TOKENS} txt_tok~{txt_tok} | compression={ratio:.2f}x")
|
|
|
|
if __name__=="__main__":
|
|
n=int(sys.argv[2]) if len(sys.argv)>2 else 6
|
|
for pt in [int(x) for x in sys.argv[1].split(",")]:
|
|
run(pt,n)
|