Files
teamchong 7af6c200c3 eval+demo: route harnesses through interactive cci.py; opus → claude-opus-4-8[1m]
eval:
- Add eval/lib/cci.py: drop-in interactive Claude session shim (pexpect/PTY)
  replacing headless `claude -p`. Drives the real TUI on Max-subscription auth
  (no ANTHROPIC_API_KEY), scrapes the reply + /context + /cost, and is
  argv/stdin/--output-format json compatible with the old call sites.
- Wire every call site to cci.py: gist-recall (run/run2/run3),
  needle-haystack (sweep/crux), gsm8k (bench), swe-bench (run_pilot),
  swe-bench-pro (rep_nav/run_pro), and eval/lib/anthropic-client.mjs.
- Isolation per call: --setting-sources project --strict-mcp-config
  --dangerously-skip-permissions. Recall evals keep --allowedTools Read;
  swe-bench keeps the full agentic toolset. Each trial is a fresh process.

demo:
- Resolve the `opus` shorthand to the full claude-opus-4-8[1m] model in
  cost-ab and effective-context a.sh/b.sh. setup.sh intentionally keeps the
  base id: the proxy scope gate strips [1m] from the request before matching,
  so [1m]-on-launch + base-in-scope is the correct pairing.

docs:
- FINDINGS.md: drop the 2026-06-16 Opus 4.8 re-test sections.
2026-06-18 23:43:37 -04:00

292 lines
11 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
cci.py - interactive Claude session shim. Drop-in replacement for `claude -p`.
Drives the REAL interactive TUI (Max-subscription auth via keychain, NO API key)
instead of headless -p, then scrapes results off the screen with a real terminal
emulator (pyte), not naive ANSI strip.
What we read off the live session (no transcript - this isolated mode never
writes one):
- reply : the last assistant '' block on screen
- $ : /cost -> "Total cost: $X" <- the ONLY server-truth number
- tokens : /context -> "N/200k tokens" + per-category breakdown (ESTIMATE,
as the panel itself states). The /cost per-model token breakdown
is a local estimate too and is intentionally NOT used.
Both /context and /cost are captured BEFORE /quit.
ISOLATION (the demo's proven combo; --bare is NOT usable - it forces
ANTHROPIC_API_KEY and never reads OAuth/keychain, breaking Max auth):
--setting-sources project --strict-mcp-config --dangerously-skip-permissions
=> no global/project CLAUDE.md, no MCP, OAuth/Max intact.
TRUST GATE: --dangerously-skip-permissions does NOT bypass the workspace-trust
dialog on a PTY (only skipped when stdout is not a TTY). When the gate appears
we accept it by pressing "1" + Enter ("Yes, I trust this folder").
ENV passes straight through (harness owns ANTHROPIC_BASE_URL / proxy on-off).
ARGV is `claude -p`-style; -p / --output-format / --strict-mcp-config /
--no-session-persistence are absorbed. Prompt = positional arg, else stdin.
"""
import sys, os, time, json, re
CLAUDE = os.environ.get("CCI_CLAUDE_BIN", os.path.expanduser("~/.claude/local/claude"))
TIMEOUT = float(os.environ.get("CCI_TIMEOUT", "300"))
READY_TIMEOUT = float(os.environ.get("CCI_READY_TIMEOUT", "60"))
QUIET_S = float(os.environ.get("CCI_QUIET_S", "4.0"))
DEBUG = os.environ.get("CCI_DEBUG")
ROWS = int(os.environ.get("CCI_ROWS", "60")) # tall buffer (e.g. 1500) for long replies
COLS = int(os.environ.get("CCI_COLS", "200"))
def parse_argv(argv):
model = None; output_format = "text"; allowed = None; prompt = None
takes_val = {"--model", "--output-format", "--allowedTools", "--allowed-tools",
"--add-dir", "--mcp-config", "--permission-mode",
"--append-system-prompt", "--system-prompt", "--settings"}
i = 0
while i < len(argv):
a = argv[i]
if a == "--model":
model = argv[i + 1]; i += 2; continue
if a == "--output-format":
output_format = argv[i + 1]; i += 2; continue
if a in ("--allowedTools", "--allowed-tools"):
allowed = argv[i + 1]; i += 2; continue
if a in ("-p", "--print", "--no-session-persistence", "--strict-mcp-config",
"--dangerously-skip-permissions", "--verbose"):
i += 1; continue
if a in takes_val:
i += 2; continue
if a.startswith("-"):
i += 1; continue
prompt = a; i += 1
if prompt is None and not sys.stdin.isatty():
prompt = sys.stdin.read()
return model, output_format, allowed, prompt
def pnum(s):
"""'517'->517, '38.5k'->38500, '1.2m'->1200000, '12,345'->12345."""
s = s.strip().replace(",", "")
m = re.match(r"([\d.]+)\s*([kKmM]?)", s)
if not m:
return 0
v = float(m.group(1)); suf = m.group(2).lower()
if suf == "k":
v *= 1_000
elif suf == "m":
v *= 1_000_000
return int(round(v))
def parse_cost_total(text):
"""The ONLY server-truth value on /cost: Total cost in USD (or None)."""
m = re.search(r"Total cost:\s*\$([\d.]+)", text)
return float(m.group(1)) if m else None
def parse_context(text):
"""/context: total context tokens + per-category estimate breakdown."""
out = {"tokens": None, "max": None, "pct": None, "breakdown": {}}
mt = re.search(r"([\d.,]+\s*[kKmM]?)\s*/\s*([\d.,]+\s*[kKmM]?)\s*tokens"
r"(?:\s*\((\d+(?:\.\d+)?)%\))?", text)
if mt:
out["tokens"] = pnum(mt.group(1))
out["max"] = pnum(mt.group(2))
if mt.group(3):
out["pct"] = float(mt.group(3))
for m in re.finditer(r"([A-Za-z][A-Za-z ]+?):\s*([\d.,]+\s*[kKmM]?)\s*"
r"(?:tokens?)?\s*\(([\d.]+)%\)", text):
name = m.group(1).strip()
if name.lower() in ("total cost",):
continue
out["breakdown"][name] = {"tokens": pnum(m.group(2)), "pct": float(m.group(3))}
return out
def extract_reply(lines):
"""Reply = the last assistant bullet block (lines under a leading '')."""
out = []; cap = False
for t in lines:
s = t.strip()
if s.startswith(""):
cap = True
out = [s.lstrip("").strip()]
elif cap:
if (not s) or s.startswith("") or s.startswith("") \
or s.startswith("") or "bypass permissions" in s \
or "esc to interrupt" in s or set(s) <= set("─╌╭╮╰╯│ "):
break
out.append(s)
return "\n".join(x for x in out if x).strip()
def main():
try:
import pexpect, pyte
except ImportError as e:
sys.stderr.write("cci: missing dependency (%s). Install: pip install pexpect pyte\n" % e)
sys.exit(3)
model, output_format, allowed, prompt = parse_argv(sys.argv[1:])
if not prompt:
sys.stderr.write("cci: no prompt (positional arg or stdin)\n")
sys.exit(2)
args = ["--setting-sources", "project", "--strict-mcp-config",
"--dangerously-skip-permissions"]
if model:
args += ["--model", model]
if allowed:
args += ["--allowedTools", allowed]
env = dict(os.environ)
env["TERM"] = "xterm-256color"
for k in list(env):
if "VSCODE" in k or k in ("TERM_PROGRAM", "CLAUDE_CODE_SSE_PORT", "CLAUDECODE"):
env.pop(k, None)
screen = pyte.Screen(COLS, ROWS)
stream = pyte.Stream(screen)
child = pexpect.spawn(CLAUDE, args, cwd=os.getcwd(), env=env,
encoding="utf-8", codec_errors="replace",
dimensions=(ROWS, COLS), timeout=TIMEOUT)
def feed(slice_s=0.3):
try:
stream.feed(child.read_nonblocking(65536, timeout=slice_s))
return True
except pexpect.TIMEOUT:
return False
except pexpect.EOF:
return None
def disp():
return [ln.rstrip() for ln in screen.display]
def scr():
return "\n".join(disp())
def settle(label, seconds=7):
"""Send a slash command and drain the screen for `seconds`."""
screen.reset()
child.send(label + "\r")
end = time.time() + seconds
while time.time() < end:
if feed(0.3) is None:
break
return scr()
# "working" = any live spinner state, not just the footer string. The spinner
# glyph + its elapsed-second counter ("(12s · esc to interrupt)") refresh every
# ~1s, so this stays TRUE across the whole turn — including the preamble->tool
# gap where the bare "esc to interrupt" footer briefly drops. That keeps the
# QUIET_S window as a pure slow-render cushion, never the thing deciding "done".
working_re = re.compile(
r"esc to interrupt" # footer affordance while busy
r"|\(\s*\d+\s*s\b" # elapsed-second counter, ticks every ~1s
r"|^\s*[✻✶✳✽✢⋆∗·*]\s+\S+…", # animated spinner glyph + verb + ellipsis
re.I | re.M)
trust_re = re.compile(r"trust this folder|Quick safety check|created or one you trust", re.I)
ready_re = re.compile(r"Welcome back|Try \"|bypass permissions")
# 1) wait for ready; accept trust gate by pressing 1 + Enter
deadline = time.time() + READY_TIMEOUT
ready = False
while time.time() < deadline:
g = feed(0.3)
if g is None:
break
s = scr()
if trust_re.search(s):
child.send("1\r"); time.sleep(0.8); continue
if ready_re.search(s) and not working_re.search(s):
ready = True; break
if not ready:
sys.stderr.write("cci: never reached ready\n--- screen ---\n" + scr() + "\n")
child.close(force=True); sys.exit(1)
time.sleep(0.4)
# 2) submit prompt, then retry Enter until the turn PROVABLY starts. A single
# Enter can no-op while the input box is still catching up to the welcome
# render, which would leave the prompt unsubmitted.
if "\n" in prompt:
child.send("\x1b[200~" + prompt + "\x1b[201~") # bracketed paste: newlines don't submit early
else:
child.send(prompt)
time.sleep(0.6)
def turn_started():
s = scr()
if working_re.search(s):
return True
return any(ln.strip().startswith("") for ln in disp())
started = False
for _ in range(6):
child.send("\r")
t0 = time.time()
while time.time() - t0 < 3:
feed(0.3)
if turn_started():
started = True
break
if started:
break
# 3) wait for completion: once started, done when neither the working
# indicator nor new output has appeared for QUIET_S (spinner animation
# counts as activity, so tool calls don't look idle mid-turn).
hard = time.time() + TIMEOUT
last_active = time.time()
seen = started
time.sleep(0.5)
while time.time() < hard:
g = feed(0.3)
if g is None:
break
if working_re.search(scr()):
seen = True; last_active = time.time(); continue
if g:
last_active = time.time(); continue
if seen and (time.time() - last_active) > QUIET_S:
break
reply = extract_reply(disp())
if DEBUG:
sys.stderr.write("\n--- REPLY SCREEN ---\n" + scr() + "\n")
# 4) capture BOTH panels BEFORE quitting
context_raw = settle("/context")
ctx = parse_context(context_raw)
cost_raw = settle("/cost")
total_cost = parse_cost_total(cost_raw)
if DEBUG:
sys.stderr.write("\n--- /context ---\n" + context_raw +
"\n--- /cost ---\n" + cost_raw + "\n")
# 5) quit
try:
child.send("/quit\r"); time.sleep(0.5)
child.sendcontrol("c"); child.sendcontrol("c")
except Exception:
pass
child.close(force=True)
if output_format == "json":
out = {"type": "result", "subtype": "success", "result": reply,
"total_cost_usd": total_cost, # server-truth ($)
"context_tokens": ctx["tokens"], # estimate
"context_max": ctx["max"],
"context_pct": ctx["pct"],
"context_breakdown": ctx["breakdown"]} # estimate, per category
sys.stdout.write(json.dumps(out))
else:
sys.stdout.write(reply)
sys.stdout.flush()
sys.exit(0)
if __name__ == "__main__":
main()