diff --git a/README.md b/README.md index 2188ba3..4b991cb 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,10 @@ Recent turns do; a dedicated verbatim-risk guard is not built yet. **Does it break real work?** Not in what we measured: a 10-instance SWE-bench Lite pilot (the easy subset) resolved **10/10 on both arms** — -pxpipe ON at $27 vs OFF at $54 token-equivalent. Small n, parity not -superiority; details and caveats in the benchmarks below. +pxpipe ON at $27 vs OFF at $54 token-equivalent, and a SWE-bench Pro +single-pair smoke test (harder, long-horizon) resolved 1/1 both arms at +-66% per-request. Small n, parity not superiority; details and caveats +in the benchmarks below. **Savings are workload-dependent.** It wins on token-dense content (~1 char/token: code, JSON, hashes) and *loses money* on sparse English prose @@ -92,6 +94,10 @@ The −65% is per-request (`count_tokens` probe of each body before compression), so it has no turn-count confound. n=10/arm, Lite skews easy. Run totals, receipts, caveats: [`eval/swe-bench/`](eval/swe-bench/). +A SWE-bench Pro pilot (harder, long-horizon tasks) is at n=1: both arms +resolved 1/1 with −66% per-request, consistent with Lite and the +production log. Receipts: [`eval/swe-bench-pro/`](eval/swe-bench-pro/). + We also ran GSM8K: 96% imaged. But GSM8K is in training data, so the model recalls memorized answers through its own misreads, inflating the score, so we lead with the clean novel-number eval instead. Reproduce: diff --git a/eval/swe-bench-pro/instances.json b/eval/swe-bench-pro/instances.json new file mode 100644 index 0000000..aa63b70 --- /dev/null +++ b/eval/swe-bench-pro/instances.json @@ -0,0 +1,12 @@ +[ + "instance_future-architect__vuls-36456cb151894964ba1683ce7da5c35ada789970", + "instance_flipt-io__flipt-2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63", + "instance_element-hq__element-web-923ad4323b2006b2b180544429455ffe7d4a6cc3-vnan", + "instance_protonmail__webclients-32ff10999a06455cb2147f6873d627456924ae13", + "instance_qutebrowser__qutebrowser-0833b5f6f140d04200ec91605f88704dd18e2970-v059c6fdc75567943479b23ebca7c07b5e9a7f34c", + "instance_tutao__tutanota-b4934a0f3c34d9d7649e944b183137e8fad3e859-vbc0d9ba8f0071fbe982809910959a6ff8884dbbf", + "instance_navidrome__navidrome-677d9947f302c9f7bba8c08c788c3dc99f235f39", + "instance_NodeBB__NodeBB-0e07f3c9bace416cbab078a30eae972868c0a8a3-vf2cf3cbd463b7ad942381f1c6d077626485a1e9e", + "instance_gravitational__teleport-89f0432ad5dc70f1f6a30ec3a8363d548371a718", + "instance_internetarchive__openlibrary-e8084193a895d8ee81200f49093389a3887479ce-ve8c8d62a2b60610a3c4631f5f23ed866bada9818" +] \ No newline at end of file diff --git a/eval/swe-bench-pro/run_pro.py b/eval/swe-bench-pro/run_pro.py new file mode 100644 index 0000000..2c42813 --- /dev/null +++ b/eval/swe-bench-pro/run_pro.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Paired SWE-bench Pro runs: pxpipe ON vs OFF. Generation only. + +Measurement hygiene (the rules that make the numbers trustworthy): +- Bench-dedicated proxies on fresh ports (ON 47823, OFF 47824) with their own + PXPIPE_LOG files. The operator's own Claude session (47821) can never + pollute the bench logs - separation by construction, not by time-window. +- Docker grading does NOT involve the proxy at all: containers run the + repo's test suite only, make no model calls, and get no ANTHROPIC_BASE_URL. +- Per-request savings come from the proxy's own `count_tokens` counterfactual + probe (uncompressed body vs sent body), so they have no turn-count confound. + +Quota safety (resume-by-default): +- Instances run sequentially; both arms of a pair run in parallel. +- Every patch/meta is written to disk the moment a run finishes. +- A pair whose patch file already exists is skipped, so if the weekly limit + hits mid-bench, just re-run this script after the reset. +- If a run's transcript smells like a rate/usage-limit error, the script + stops cleanly instead of burning the remaining instances on garbage. +""" +import json, os, re, subprocess, sys, time +from concurrent.futures import ThreadPoolExecutor + +HERE = os.path.dirname(os.path.abspath(__file__)) +WORK = os.path.expanduser("~/swe-pro-bench") +CACHE = os.path.join(WORK, "cache") +CLAUDE = os.path.expanduser("~/.claude/local/claude") +MODEL = "claude-fable-5" +ARMS = {"on": 47823, "off": 47824} +LOGS = {a: os.path.expanduser(f"~/.pxpipe/events-bench-{a}.jsonl") for a in ARMS} +TIMEOUT = 1800 # 30 min hard cap per run; Pro tasks are long-horizon +QUOTA_RE = re.compile(r"rate.?limit|usage limit|exceed.*limit|quota", re.I) + +PROMPT = """You are implementing a change in this repository ({repo}). + +# Issue +{problem} + +# Requirements +{requirements} + +# Interface +{interface} + +Make the minimal source change to satisfy the requirements exactly (names, +signatures, types). Do NOT modify test files. The project's full test +environment is NOT installed here - do not run the test suite or install +dependencies; write the source code only. Leave your changes in the working +tree (do not commit).""" + + +def sh(cmd, **kw): + return subprocess.run(cmd, shell=True, capture_output=True, text=True, **kw) + + +def ensure_proxies(): + """Bench proxies only. Compression: ON arm default-on, OFF arm forced off.""" + for arm, port in ARMS.items(): + if sh(f"curl -sf -o /dev/null http://127.0.0.1:{port}/").returncode != 0: + env = f"PORT={port} PXPIPE_LOG={LOGS[arm]}" + sh(f"cd {os.path.join(HERE, '..', '..')} && nohup env {env} node bin/cli.js " + f">> /tmp/pxpipe-bench-{arm}.log 2>&1 & disown") + time.sleep(2) + if arm == "off": + sh(f"curl -s -X POST http://127.0.0.1:{port}/api/compression " + f"-H 'content-type: application/json' -d '{{\"enabled\":false}}'") + for arm, port in ARMS.items(): + r = sh(f"curl -s http://127.0.0.1:{port}/api/compression " + f"-X POST -H 'content-type: application/json' " + f"-d '{{\"enabled\":{'true' if arm == 'on' else 'false'}}}'") + print(f"[proxy] {arm} :{port} -> {r.stdout.strip()}") + + +def load_instances(): + with open(os.path.join(HERE, "instances.json")) as f: + picks = json.load(f) + import pyarrow.parquet as pq + t = pq.read_table("/tmp/swepro.parquet").to_pylist() + by_id = {r["instance_id"]: r for r in t} + return [by_id[i] for i in picks] + + +def run_one(inst, arm): + iid = inst["instance_id"] + d = os.path.join(WORK, iid, arm) + pdir = os.path.join(WORK, iid) + patch_file = os.path.join(pdir, f"patch_{arm}.diff") + if os.path.exists(patch_file): + return iid, arm, "cached" + os.makedirs(pdir, exist_ok=True) + repo = inst["repo"] + cache = os.path.join(CACHE, repo.replace("/", "_") + ".git") + if not os.path.exists(cache): + r = sh(f"git clone -q --bare https://github.com/{repo}.git {cache}") + if r.returncode != 0: + return iid, arm, "cache-clone-fail" + sh(f"rm -rf {d}") + r = sh(f"git clone -q {cache} {d} && git -C {d} checkout -q {inst['base_commit']}") + if r.returncode != 0: + return iid, arm, "checkout-fail" + if inst.get("before_repo_set_cmd"): + sh(inst["before_repo_set_cmd"], cwd=d) + prompt = PROMPT.format( + repo=repo, + problem=(inst.get("problem_statement") or "").strip(), + requirements=(inst.get("requirements") or "").strip() or "(see issue)", + interface=(inst.get("interface") or "").strip() or "No new interfaces are introduced.", + ) + env = dict(os.environ, ANTHROPIC_BASE_URL=f"http://127.0.0.1:{ARMS[arm]}") + t0 = time.time() + try: + r = subprocess.run( + [CLAUDE, "-p", "--model", MODEL, "--dangerously-skip-permissions", prompt], + cwd=d, env=env, capture_output=True, text=True, timeout=TIMEOUT) + rc, tail = r.returncode, ((r.stdout or "") + (r.stderr or ""))[-3000:] + except subprocess.TimeoutExpired: + rc, tail = -9, "TIMEOUT" + dur = round(time.time() - t0, 1) + diff = sh(f"git -C {d} diff", cwd=d).stdout + open(patch_file, "w").write(diff) + json.dump({"rc": rc, "dur_s": dur, "patch_bytes": len(diff), "tail": tail[-800:]}, + open(os.path.join(pdir, f"meta_{arm}.json"), "w"), indent=1) + if rc != 0 and QUOTA_RE.search(tail): + return iid, arm, "QUOTA" + return iid, arm, f"done rc={rc} dur={dur}s patch={len(diff)}b" + + +def main(): + os.makedirs(CACHE, exist_ok=True) + ensure_proxies() + insts = load_instances() + print(f"{len(insts)} instances; resume = skip existing patches\n") + for inst in insts: + with ThreadPoolExecutor(max_workers=2) as ex: + results = list(ex.map(lambda a: run_one(inst, a), ARMS)) + for iid, arm, status in results: + print(f"[{iid[:48]}] {arm}: {status}", flush=True) + if any(s == "QUOTA" for _, _, s in results): + print("\nUSAGE LIMIT HIT - stopping cleanly. Completed pairs are on " + "disk; re-run this script after the reset to resume.") + sys.exit(2) + # predictions for the grader + for arm in ARMS: + preds = [] + for inst in insts: + pf = os.path.join(WORK, inst["instance_id"], f"patch_{arm}.diff") + if os.path.exists(pf): + preds.append({"instance_id": inst["instance_id"], + "patch": open(pf).read(), "prefix": ""}) + out = os.path.join(WORK, f"preds_{arm}.json") + json.dump(preds, open(out, "w")) + print(f"wrote {out} ({len(preds)} patches)") + + +if __name__ == "__main__": + main()