mirror of
https://github.com/teamchong/pxpipe.git
synced 2026-07-22 02:02:51 +02:00
feat(eval/swe-bench-pro): resumable 10-pair runner on bench-dedicated proxies
- ON 47823 / OFF 47824 with own PXPIPE_LOG files: operator session can never pollute bench measurement (separation by construction) - resume = skip existing patch files; clean stop on quota errors - Docker grading documented as proxy-free (no model calls in containers) - README: mention Pro pilot alongside Lite
This commit is contained in:
teamchong
@@ -51,8 +51,10 @@ Recent turns do; a dedicated verbatim-risk guard is not built yet.
|
||||
|
||||
**Does it break real work?** Not in what we measured: a 10-instance
|
||||
SWE-bench Lite pilot (the easy subset) resolved **10/10 on both arms** —
|
||||
pxpipe ON at $27 vs OFF at $54 token-equivalent. Small n, parity not
|
||||
superiority; details and caveats in the benchmarks below.
|
||||
pxpipe ON at $27 vs OFF at $54 token-equivalent, and a SWE-bench Pro
|
||||
single-pair smoke test (harder, long-horizon) resolved 1/1 both arms at
|
||||
-66% per-request. Small n, parity not superiority; details and caveats
|
||||
in the benchmarks below.
|
||||
|
||||
**Savings are workload-dependent.** It wins on token-dense content
|
||||
(~1 char/token: code, JSON, hashes) and *loses money* on sparse English prose
|
||||
@@ -92,6 +94,10 @@ The −65% is per-request (`count_tokens` probe of each body before
|
||||
compression), so it has no turn-count confound. n=10/arm, Lite skews easy.
|
||||
Run totals, receipts, caveats: [`eval/swe-bench/`](eval/swe-bench/).
|
||||
|
||||
A SWE-bench Pro pilot (harder, long-horizon tasks) is at n=1: both arms
|
||||
resolved 1/1 with −66% per-request, consistent with Lite and the
|
||||
production log. Receipts: [`eval/swe-bench-pro/`](eval/swe-bench-pro/).
|
||||
|
||||
<sub>We also ran GSM8K: 96% imaged. But GSM8K is in training data, so the model
|
||||
recalls memorized answers through its own misreads, inflating the score, so we
|
||||
lead with the clean novel-number eval instead. Reproduce:
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
[
|
||||
"instance_future-architect__vuls-36456cb151894964ba1683ce7da5c35ada789970",
|
||||
"instance_flipt-io__flipt-2ce8a0331e8a8f63f2c1b555db8277ffe5aa2e63",
|
||||
"instance_element-hq__element-web-923ad4323b2006b2b180544429455ffe7d4a6cc3-vnan",
|
||||
"instance_protonmail__webclients-32ff10999a06455cb2147f6873d627456924ae13",
|
||||
"instance_qutebrowser__qutebrowser-0833b5f6f140d04200ec91605f88704dd18e2970-v059c6fdc75567943479b23ebca7c07b5e9a7f34c",
|
||||
"instance_tutao__tutanota-b4934a0f3c34d9d7649e944b183137e8fad3e859-vbc0d9ba8f0071fbe982809910959a6ff8884dbbf",
|
||||
"instance_navidrome__navidrome-677d9947f302c9f7bba8c08c788c3dc99f235f39",
|
||||
"instance_NodeBB__NodeBB-0e07f3c9bace416cbab078a30eae972868c0a8a3-vf2cf3cbd463b7ad942381f1c6d077626485a1e9e",
|
||||
"instance_gravitational__teleport-89f0432ad5dc70f1f6a30ec3a8363d548371a718",
|
||||
"instance_internetarchive__openlibrary-e8084193a895d8ee81200f49093389a3887479ce-ve8c8d62a2b60610a3c4631f5f23ed866bada9818"
|
||||
]
|
||||
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Paired SWE-bench Pro runs: pxpipe ON vs OFF. Generation only.
|
||||
|
||||
Measurement hygiene (the rules that make the numbers trustworthy):
|
||||
- Bench-dedicated proxies on fresh ports (ON 47823, OFF 47824) with their own
|
||||
PXPIPE_LOG files. The operator's own Claude session (47821) can never
|
||||
pollute the bench logs - separation by construction, not by time-window.
|
||||
- Docker grading does NOT involve the proxy at all: containers run the
|
||||
repo's test suite only, make no model calls, and get no ANTHROPIC_BASE_URL.
|
||||
- Per-request savings come from the proxy's own `count_tokens` counterfactual
|
||||
probe (uncompressed body vs sent body), so they have no turn-count confound.
|
||||
|
||||
Quota safety (resume-by-default):
|
||||
- Instances run sequentially; both arms of a pair run in parallel.
|
||||
- Every patch/meta is written to disk the moment a run finishes.
|
||||
- A pair whose patch file already exists is skipped, so if the weekly limit
|
||||
hits mid-bench, just re-run this script after the reset.
|
||||
- If a run's transcript smells like a rate/usage-limit error, the script
|
||||
stops cleanly instead of burning the remaining instances on garbage.
|
||||
"""
|
||||
import json, os, re, subprocess, sys, time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
WORK = os.path.expanduser("~/swe-pro-bench")
|
||||
CACHE = os.path.join(WORK, "cache")
|
||||
CLAUDE = os.path.expanduser("~/.claude/local/claude")
|
||||
MODEL = "claude-fable-5"
|
||||
ARMS = {"on": 47823, "off": 47824}
|
||||
LOGS = {a: os.path.expanduser(f"~/.pxpipe/events-bench-{a}.jsonl") for a in ARMS}
|
||||
TIMEOUT = 1800 # 30 min hard cap per run; Pro tasks are long-horizon
|
||||
QUOTA_RE = re.compile(r"rate.?limit|usage limit|exceed.*limit|quota", re.I)
|
||||
|
||||
PROMPT = """You are implementing a change in this repository ({repo}).
|
||||
|
||||
# Issue
|
||||
{problem}
|
||||
|
||||
# Requirements
|
||||
{requirements}
|
||||
|
||||
# Interface
|
||||
{interface}
|
||||
|
||||
Make the minimal source change to satisfy the requirements exactly (names,
|
||||
signatures, types). Do NOT modify test files. The project's full test
|
||||
environment is NOT installed here - do not run the test suite or install
|
||||
dependencies; write the source code only. Leave your changes in the working
|
||||
tree (do not commit)."""
|
||||
|
||||
|
||||
def sh(cmd, **kw):
|
||||
return subprocess.run(cmd, shell=True, capture_output=True, text=True, **kw)
|
||||
|
||||
|
||||
def ensure_proxies():
|
||||
"""Bench proxies only. Compression: ON arm default-on, OFF arm forced off."""
|
||||
for arm, port in ARMS.items():
|
||||
if sh(f"curl -sf -o /dev/null http://127.0.0.1:{port}/").returncode != 0:
|
||||
env = f"PORT={port} PXPIPE_LOG={LOGS[arm]}"
|
||||
sh(f"cd {os.path.join(HERE, '..', '..')} && nohup env {env} node bin/cli.js "
|
||||
f">> /tmp/pxpipe-bench-{arm}.log 2>&1 & disown")
|
||||
time.sleep(2)
|
||||
if arm == "off":
|
||||
sh(f"curl -s -X POST http://127.0.0.1:{port}/api/compression "
|
||||
f"-H 'content-type: application/json' -d '{{\"enabled\":false}}'")
|
||||
for arm, port in ARMS.items():
|
||||
r = sh(f"curl -s http://127.0.0.1:{port}/api/compression "
|
||||
f"-X POST -H 'content-type: application/json' "
|
||||
f"-d '{{\"enabled\":{'true' if arm == 'on' else 'false'}}}'")
|
||||
print(f"[proxy] {arm} :{port} -> {r.stdout.strip()}")
|
||||
|
||||
|
||||
def load_instances():
|
||||
with open(os.path.join(HERE, "instances.json")) as f:
|
||||
picks = json.load(f)
|
||||
import pyarrow.parquet as pq
|
||||
t = pq.read_table("/tmp/swepro.parquet").to_pylist()
|
||||
by_id = {r["instance_id"]: r for r in t}
|
||||
return [by_id[i] for i in picks]
|
||||
|
||||
|
||||
def run_one(inst, arm):
|
||||
iid = inst["instance_id"]
|
||||
d = os.path.join(WORK, iid, arm)
|
||||
pdir = os.path.join(WORK, iid)
|
||||
patch_file = os.path.join(pdir, f"patch_{arm}.diff")
|
||||
if os.path.exists(patch_file):
|
||||
return iid, arm, "cached"
|
||||
os.makedirs(pdir, exist_ok=True)
|
||||
repo = inst["repo"]
|
||||
cache = os.path.join(CACHE, repo.replace("/", "_") + ".git")
|
||||
if not os.path.exists(cache):
|
||||
r = sh(f"git clone -q --bare https://github.com/{repo}.git {cache}")
|
||||
if r.returncode != 0:
|
||||
return iid, arm, "cache-clone-fail"
|
||||
sh(f"rm -rf {d}")
|
||||
r = sh(f"git clone -q {cache} {d} && git -C {d} checkout -q {inst['base_commit']}")
|
||||
if r.returncode != 0:
|
||||
return iid, arm, "checkout-fail"
|
||||
if inst.get("before_repo_set_cmd"):
|
||||
sh(inst["before_repo_set_cmd"], cwd=d)
|
||||
prompt = PROMPT.format(
|
||||
repo=repo,
|
||||
problem=(inst.get("problem_statement") or "").strip(),
|
||||
requirements=(inst.get("requirements") or "").strip() or "(see issue)",
|
||||
interface=(inst.get("interface") or "").strip() or "No new interfaces are introduced.",
|
||||
)
|
||||
env = dict(os.environ, ANTHROPIC_BASE_URL=f"http://127.0.0.1:{ARMS[arm]}")
|
||||
t0 = time.time()
|
||||
try:
|
||||
r = subprocess.run(
|
||||
[CLAUDE, "-p", "--model", MODEL, "--dangerously-skip-permissions", prompt],
|
||||
cwd=d, env=env, capture_output=True, text=True, timeout=TIMEOUT)
|
||||
rc, tail = r.returncode, ((r.stdout or "") + (r.stderr or ""))[-3000:]
|
||||
except subprocess.TimeoutExpired:
|
||||
rc, tail = -9, "TIMEOUT"
|
||||
dur = round(time.time() - t0, 1)
|
||||
diff = sh(f"git -C {d} diff", cwd=d).stdout
|
||||
open(patch_file, "w").write(diff)
|
||||
json.dump({"rc": rc, "dur_s": dur, "patch_bytes": len(diff), "tail": tail[-800:]},
|
||||
open(os.path.join(pdir, f"meta_{arm}.json"), "w"), indent=1)
|
||||
if rc != 0 and QUOTA_RE.search(tail):
|
||||
return iid, arm, "QUOTA"
|
||||
return iid, arm, f"done rc={rc} dur={dur}s patch={len(diff)}b"
|
||||
|
||||
|
||||
def main():
|
||||
os.makedirs(CACHE, exist_ok=True)
|
||||
ensure_proxies()
|
||||
insts = load_instances()
|
||||
print(f"{len(insts)} instances; resume = skip existing patches\n")
|
||||
for inst in insts:
|
||||
with ThreadPoolExecutor(max_workers=2) as ex:
|
||||
results = list(ex.map(lambda a: run_one(inst, a), ARMS))
|
||||
for iid, arm, status in results:
|
||||
print(f"[{iid[:48]}] {arm}: {status}", flush=True)
|
||||
if any(s == "QUOTA" for _, _, s in results):
|
||||
print("\nUSAGE LIMIT HIT - stopping cleanly. Completed pairs are on "
|
||||
"disk; re-run this script after the reset to resume.")
|
||||
sys.exit(2)
|
||||
# predictions for the grader
|
||||
for arm in ARMS:
|
||||
preds = []
|
||||
for inst in insts:
|
||||
pf = os.path.join(WORK, inst["instance_id"], f"patch_{arm}.diff")
|
||||
if os.path.exists(pf):
|
||||
preds.append({"instance_id": inst["instance_id"],
|
||||
"patch": open(pf).read(), "prefix": ""})
|
||||
out = os.path.join(WORK, f"preds_{arm}.json")
|
||||
json.dump(preds, open(out, "w"))
|
||||
print(f"wrote {out} ({len(preds)} patches)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user