Restructure: archive Python+Rust, prep for TypeScript rewrite

- Move Python proxy + scripts to legacy/python/ (parity oracle during port)
- Delete Rust port (src/rust/) - history preserved in commits 482c967, f094649
- Delete Zig atlas builder (src/zig/) - will regenerate from TTF via Node
- Move JetBrainsMono-Regular.ttf to assets/
- Update .gitignore for Node/TS/Wrangler

Next: scaffold TypeScript with dual Node + Cloudflare Workers targets.
This commit is contained in:
teamchong
2026-05-18 15:22:41 -04:00
parent f20207cd7b
commit 437afa4dfe
27 changed files with 46 additions and 3804 deletions
-136
View File
@@ -1,136 +0,0 @@
#!/usr/bin/env python3
"""
Diff harness: compare Python render_chunks vs Rust render_chunks pixel-by-pixel.
Strategy:
1. For each sample text, call Python render_chunks() -> list[PNG bytes]
2. Call Rust diff_render binary -> writes PNGs to disk
3. Decode both sides' PNGs and compare pixel arrays
4. Report: pass / fail with first diff coordinates
Run:
python3 scripts/diff_render.py
"""
import hashlib
import json
import subprocess
import sys
import tempfile
from pathlib import Path
# Make src/proxy.py importable
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "src"))
import proxy as py_proxy # noqa: E402
from PIL import Image # noqa: E402
import io # noqa: E402
RUST_BIN = ROOT / "src" / "rust" / "target" / "release" / "examples" / "diff_render"
SAMPLES = {
"short_ascii": "hello world\n",
"multiline_code": (
"def fibonacci(n):\n"
" if n <= 1: return n\n"
" return fibonacci(n-1) + fibonacci(n-2)\n"
"print(fibonacci(10))\n"
),
"long_with_blanks": (
"# Header\n\n"
+ "\n".join(f"line {i}: " + "x" * (i % 60) for i in range(200))
+ "\n\n# Footer\n"
),
"many_short_lines": "\n".join(f"L{i}" for i in range(500)) + "\n",
"wide_lines": "\n".join("." * 80 for _ in range(100)) + "\n",
"tools_like_json": json.dumps(
{"tools": [{"name": f"tool_{i}", "input_schema": {"type": "object",
"properties": {f"p{j}": {"type": "string"} for j in range(5)}}}
for i in range(20)]}, indent=2),
}
def png_to_pixels(png_bytes: bytes):
"""Decode PNG to (width, height, raw_grayscale_bytes)."""
img = Image.open(io.BytesIO(png_bytes))
img = img.convert("L") # force 8-bit grayscale
return img.width, img.height, img.tobytes()
def run_rust(samples: dict, out_dir: Path) -> dict:
"""Invoke rust diff_render binary, returns name -> list[png_bytes]."""
spec = {name: text for name, text in samples.items()}
spec_path = out_dir / "spec.json"
spec_path.write_text(json.dumps(spec))
res = subprocess.run(
[str(RUST_BIN), str(spec_path), str(out_dir)],
capture_output=True, text=True,
)
if res.returncode != 0:
print("RUST FAILED:", res.stderr, file=sys.stderr)
sys.exit(2)
manifest = json.loads((out_dir / "manifest.json").read_text())
out = {}
for name, files in manifest.items():
out[name] = [(out_dir / f).read_bytes() for f in files]
return out
def diff_pixels(py_bytes: bytes, rs_bytes: bytes):
"""Return None if pixel-identical, else (reason, details)."""
pw, ph, pp = png_to_pixels(py_bytes)
rw, rh, rp = png_to_pixels(rs_bytes)
if (pw, ph) != (rw, rh):
return ("dim_mismatch", f"py={pw}x{ph} rs={rw}x{rh}")
if pp == rp:
return None
# Find first diff
for i, (a, b) in enumerate(zip(pp, rp)):
if a != b:
x = i % pw
y = i // pw
return ("pixel_mismatch", f"first diff at ({x},{y}): py={a} rs={b}")
return ("byte_len_mismatch", f"py_len={len(pp)} rs_len={len(rp)}")
def main():
if not RUST_BIN.exists():
print(f"build rust first: cd src/rust && cargo build --release --example diff_render")
print(f"missing: {RUST_BIN}")
sys.exit(1)
with tempfile.TemporaryDirectory() as td:
out_dir = Path(td)
rust_out = run_rust(SAMPLES, out_dir)
total = 0
fail = 0
for name, text in SAMPLES.items():
py_pngs = py_proxy.render_chunks(text)
rs_pngs = rust_out.get(name, [])
total += 1
print(f"\n[{name}] py_chunks={len(py_pngs)} rs_chunks={len(rs_pngs)}")
if len(py_pngs) != len(rs_pngs):
fail += 1
print(f" FAIL: chunk count differs")
continue
any_fail = False
for i, (a, b) in enumerate(zip(py_pngs, rs_pngs)):
ah = hashlib.sha256(a).hexdigest()[:12]
bh = hashlib.sha256(b).hexdigest()[:12]
diff = diff_pixels(a, b)
if diff is None:
print(f" chunk {i}: OK py={ah} rs={bh} bytes={len(a)}/{len(b)}")
else:
any_fail = True
print(f" chunk {i}: FAIL {diff[0]}: {diff[1]} py={ah} rs={bh}")
if any_fail:
fail += 1
print(f"\n=== {total - fail}/{total} samples pixel-identical ===")
sys.exit(0 if fail == 0 else 1)
if __name__ == "__main__":
main()
-213
View File
@@ -1,213 +0,0 @@
#!/usr/bin/env python3
"""
Diff harness at the TRANSFORM layer: synthesize a realistic Anthropic
/v1/messages body, run it through Python proxy.transform_request() AND
the Rust transform_only binary, then diff what each emits.
This is the validation we need before switching :47821 to Rust. It
catches divergences in:
- Field structure (system field rewriting, cache_control placement)
- Image block emission (number of imgs, dims)
- Tools/schemas/reminders compression behavior
- JSON serialization order (Anthropic cache hashes on bytes)
Run:
python3 scripts/diff_transform.py
"""
import io
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "src"))
import proxy as py_proxy # noqa: E402
RUST_BIN = ROOT / "src" / "rust" / "target" / "release" / "examples" / "transform_only"
def synthesize_body() -> bytes:
"""Build a realistic Claude Code-shaped /v1/messages body.
Shape based on what real Anthropic API requests look like:
- long `system` field (system prompt content)
- `tools` array with name/description/input_schema each
- `messages` array
- `max_tokens`, `model`, etc.
"""
# Real-ish system prompt: load CLAUDE.md + some extra padding to clear MIN_COMPRESS_CHARS=2000
sysprompt_parts = []
try:
sysprompt_parts.append((ROOT / "CLAUDE.md").read_text())
except Exception:
pass
try:
sysprompt_parts.append((ROOT / "HANDOFF.md").read_text())
except Exception:
pass
sysprompt_parts.append(
"\n## Additional padding to ensure >2000 chars\n"
+ "Line of fake system instructions blah blah blah.\n" * 30
)
sysprompt = "\n\n".join(sysprompt_parts)
body = {
"model": "claude-opus-4-5-20250929",
"max_tokens": 8192,
"system": [
{"type": "text", "text": sysprompt},
],
"tools": [
{
"name": "Bash",
"description": "Execute a bash command with optional timeout.\n\nThe command runs in a fresh subshell. Use this for git, builds, file ops.",
"input_schema": {
"type": "object",
"properties": {
"command": {"type": "string", "description": "The bash command to execute."},
"timeout": {"type": "integer", "description": "Timeout in milliseconds. Defaults to 120000."},
"description": {"type": "string", "description": "A short description of what the command does."},
},
"required": ["command"],
},
},
{
"name": "Read",
"description": "Read a file from disk.\n\nReturns up to 2000 lines starting from offset. Long lines truncated at 2000 chars.",
"input_schema": {
"type": "object",
"properties": {
"file_path": {"type": "string", "description": "Absolute path to the file."},
"offset": {"type": "integer", "description": "1-indexed line to start at."},
"limit": {"type": "integer", "description": "Max number of lines to read."},
},
"required": ["file_path"],
},
},
{
"name": "Edit",
"description": "Apply an exact-string find/replace in a file.\n\nold_string must match exactly including whitespace. Use replace_all for global.",
"input_schema": {
"type": "object",
"properties": {
"file_path": {"type": "string"},
"old_string": {"type": "string"},
"new_string": {"type": "string"},
"replace_all": {"type": "boolean"},
},
"required": ["file_path", "old_string", "new_string"],
},
},
],
"messages": [
{
"role": "user",
"content": "Run `git status` and tell me what's modified.",
},
],
}
return json.dumps(body).encode()
def deep_compare(a, b, path=""):
"""Walk two JSON structures and yield (path, kind, py_repr, rs_repr) diffs."""
if type(a) is not type(b):
yield (path, "type", type(a).__name__, type(b).__name__)
return
if isinstance(a, dict):
keys_only_py = set(a) - set(b)
keys_only_rs = set(b) - set(a)
for k in keys_only_py:
yield (f"{path}.{k}", "missing_in_rs", short(a[k]), "<missing>")
for k in keys_only_rs:
yield (f"{path}.{k}", "missing_in_py", "<missing>", short(b[k]))
for k in set(a) & set(b):
yield from deep_compare(a[k], b[k], f"{path}.{k}")
return
if isinstance(a, list):
if len(a) != len(b):
yield (path, "list_len", str(len(a)), str(len(b)))
for i, (x, y) in enumerate(zip(a, b)):
yield from deep_compare(x, y, f"{path}[{i}]")
return
if a != b:
yield (path, "value", short(a), short(b))
def short(v) -> str:
s = json.dumps(v) if not isinstance(v, str) else v
if len(s) > 80:
return s[:77] + "..."
return s
def main():
if not RUST_BIN.exists():
print(f"build rust first: cd src/rust && cargo build --release --example transform_only")
sys.exit(1)
body = synthesize_body()
print(f"input body: {len(body)} bytes")
# Python transform
py_out, py_info = py_proxy.transform_request(body)
print(f"python transform: out={len(py_out)}B info_keys={sorted(py_info.keys())}")
# Rust transform via subprocess
rust_dir = Path("/tmp/cip_xform_rs")
rust_dir.mkdir(exist_ok=True)
body_path = rust_dir / "body_in.json"
body_path.write_bytes(body)
r = subprocess.run(
[str(RUST_BIN), str(body_path), str(rust_dir)],
capture_output=True, text=True,
)
if r.returncode != 0:
print("RUST FAILED:", r.stderr); sys.exit(2)
print("rust transform stderr:", r.stderr.strip())
rs_out = (rust_dir / "body_out.json").read_bytes()
rs_info = json.loads((rust_dir / "info.json").read_text())
print(f"rust transform: out={len(rs_out)}B info_keys={sorted(rs_info.keys())}")
print(f"\n=== body bytes: py={len(py_out)} rs={len(rs_out)} delta={len(rs_out)-len(py_out):+d} ===")
# Decode and structural diff
try:
py_json = json.loads(py_out)
rs_json = json.loads(rs_out)
except json.JSONDecodeError as e:
print(f"FAIL: invalid JSON output: {e}")
sys.exit(3)
diffs = list(deep_compare(py_json, rs_json, "$"))
if not diffs:
print("body JSON is STRUCTURALLY IDENTICAL")
else:
print(f"\nfound {len(diffs)} structural diffs (showing first 30):")
for path, kind, pv, rv in diffs[:30]:
print(f" [{kind:18s}] {path}")
print(f" py: {pv}")
print(f" rs: {rv}")
if len(diffs) > 30:
print(f" ... and {len(diffs)-30} more")
# Info side-by-side
print("\n=== TransformInfo comparison ===")
common = sorted(set(py_info.keys()) | set(rs_info.keys()))
print(f"{'field':30s} {'python':>14s} {'rust':>14s}")
for k in common:
pv = py_info.get(k, "<missing>")
rv = rs_info.get(k, "<missing>")
if isinstance(pv, list): pv = f"list[{len(pv)}]"
if isinstance(rv, list): rv = f"list[{len(rv)}]"
print(f"{k:30s} {str(pv):>14s} {str(rv):>14s}")
sys.exit(0 if not diffs else 1)
if __name__ == "__main__":
main()
-75
View File
@@ -1,75 +0,0 @@
"""Pre-render Menlo 5pt glyph atlas + metrics for Zig consumption.
Format (binary, little-endian):
Header (16 bytes):
magic 4 "MNAT"
version 2 u16 = 1
cell_w 2 u16 max glyph width
cell_h 2 u16 cell height (asc+desc)
asc 2 u16
desc 2 u16
first_char 1 u8 = 32 (space)
last_char 1 u8 = 126 (~)
Per-glyph (N glyphs = last-first+1):
advance_px 2 u16
-- packed 1-bit bitmap, cell_w*cell_h bits, row-major
"""
import struct
from PIL import Image, ImageDraw, ImageFont
FONT_PATH = "/System/Library/Fonts/Menlo.ttc"
FONT_SIZE = 5
FIRST = 32 # space
LAST = 126 # ~
font = ImageFont.truetype(FONT_PATH, FONT_SIZE)
asc, desc = font.getmetrics()
cell_h = asc + desc
# Find max glyph width across our range
max_w = 0
for code in range(FIRST, LAST + 1):
ch = chr(code)
w = int(font.getlength(ch)) + 1
if w > max_w:
max_w = w
cell_w = max_w
print(f"Atlas: cell={cell_w}x{cell_h}, asc={asc}, desc={desc}, chars={LAST-FIRST+1}")
with open("/tmp/menlo5_atlas.bin", "wb") as f:
# Header
f.write(b"MNAT")
f.write(struct.pack("<H", 1)) # version
f.write(struct.pack("<H", cell_w))
f.write(struct.pack("<H", cell_h))
f.write(struct.pack("<H", asc))
f.write(struct.pack("<H", desc))
f.write(struct.pack("<B", FIRST))
f.write(struct.pack("<B", LAST))
# Glyphs
for code in range(FIRST, LAST + 1):
ch = chr(code)
advance = int(font.getlength(ch))
img = Image.new("L", (cell_w, cell_h), 255)
d = ImageDraw.Draw(img)
d.text((0, -desc // 2), ch, fill=0, font=font)
# Threshold to 1-bit (anti-aliased pixels -> any non-white -> black)
pix = img.load()
f.write(struct.pack("<H", advance))
# Pack row-major bits: bit=1 means BLACK (ink)
bits = []
for y in range(cell_h):
for x in range(cell_w):
bits.append(1 if pix[x, y] < 200 else 0)
# Pack 8 bits per byte
for i in range(0, len(bits), 8):
byte = 0
for j in range(8):
if i + j < len(bits) and bits[i + j]:
byte |= 1 << (7 - j)
f.write(struct.pack("<B", byte))
import os
print(f"Atlas size: {os.path.getsize('/tmp/menlo5_atlas.bin')} bytes")
-50
View File
@@ -1,50 +0,0 @@
#!/usr/bin/env node
/**
* Postinstall: verify Python 3.8+ and install Pillow + httpx (the only deps).
* Idempotent — safe to run on reinstall.
*/
"use strict";
const { spawnSync } = require("child_process");
function findPython() {
const candidates = ["python3", "python", "python3.12", "python3.11", "python3.10", "python3.9", "python3.8"];
for (const c of candidates) {
const r = spawnSync(c, ["--version"], { stdio: ["ignore", "pipe", "pipe"] });
if (r.status === 0) {
const ver = (r.stdout || r.stderr || Buffer.from("")).toString();
const m = ver.match(/Python (\d+)\.(\d+)/);
if (m && parseInt(m[1], 10) >= 3 && parseInt(m[2], 10) >= 8) return c;
}
}
return null;
}
function hasModule(py, mod) {
const r = spawnSync(py, ["-c", `import ${mod}`], { stdio: ["ignore", "pipe", "pipe"] });
return r.status === 0;
}
const py = findPython();
if (!py) {
console.warn("\n[pixelpipe] WARNING: Python 3.8+ not found.");
console.warn(" Install Python 3 (https://www.python.org/downloads/) then run:");
console.warn(" python3 -m pip install Pillow httpx");
console.warn(" Otherwise the proxy will fail to start.\n");
process.exit(0); // don't break npm install
}
const need = [];
if (!hasModule(py, "PIL")) need.push("Pillow");
if (!hasModule(py, "httpx")) need.push("httpx");
if (need.length === 0) {
console.log("[pixelpipe] Python deps already installed.");
process.exit(0);
}
console.log(`[pixelpipe] Installing Python deps: ${need.join(", ")}`);
const args = ["-m", "pip", "install", "--quiet", "--user", ...need];
const r = spawnSync(py, args, { stdio: "inherit" });
if (r.status !== 0) {
console.warn("\n[pixelpipe] WARNING: failed to auto-install Python deps.");
console.warn(` Please run manually: ${py} -m pip install ${need.join(" ")}`);
}
-150
View File
@@ -1,150 +0,0 @@
#!/usr/bin/env python3
"""
Render the same text through Python and Rust, save PNGs side-by-side, and
also stitch a single comparison image with labels for easy visual inspection.
Usage:
python3 scripts/visual_compare.py [out_dir]
Output:
<out_dir>/python_<sample>.png
<out_dir>/rust_<sample>.png
<out_dir>/compare_<sample>.png (side-by-side with labels)
<out_dir>/README.txt (dim + byte size summary)
"""
import io
import json
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT / "src"))
import proxy as py_proxy # noqa: E402
from PIL import Image, ImageDraw, ImageFont # noqa: E402
RUST_BIN = ROOT / "src" / "rust" / "target" / "release" / "examples" / "diff_render"
# Three representative samples
SAMPLES = {
"tiny": "echo hello world\n",
"code": (
"def render_chunks(text):\n"
" \"\"\"Render text into 5pt PNG, multi-column.\"\"\"\n"
" font = get_font(FONT_SIZE)\n"
" lines = wrap(text, FIXED_COL_CHARS)\n"
" return paint(font, lines)\n"
),
"tools_schema": json.dumps({
"tools": [
{
"name": "Bash",
"description": "Execute a bash command",
"input_schema": {
"type": "object",
"properties": {
"command": {"type": "string", "description": "the command"},
"timeout": {"type": "integer", "description": "timeout in ms"},
},
"required": ["command"],
},
},
{
"name": "Read",
"description": "Read a file from disk",
"input_schema": {
"type": "object",
"properties": {
"file_path": {"type": "string"},
"offset": {"type": "integer"},
"limit": {"type": "integer"},
},
"required": ["file_path"],
},
},
]
}, indent=2),
}
def label_image(img: Image.Image, label: str) -> Image.Image:
"""Add a black bar at top with the label in white."""
bar_h = 24
out = Image.new("RGB", (img.width, img.height + bar_h), (0, 0, 0))
out.paste(img.convert("RGB"), (0, bar_h))
d = ImageDraw.Draw(out)
try:
f = ImageFont.truetype("/System/Library/Fonts/Menlo.ttc", 14)
except Exception:
f = ImageFont.load_default()
d.text((6, 4), label, fill=(255, 255, 255), font=f)
return out
def main():
out_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/tmp/cip_compare")
out_dir.mkdir(parents=True, exist_ok=True)
if not RUST_BIN.exists():
print(f"build rust first: cd src/rust && cargo build --release --example diff_render")
sys.exit(1)
# Run rust
spec_path = out_dir / "spec.json"
spec_path.write_text(json.dumps(SAMPLES))
rust_outdir = out_dir / "_rust"
rust_outdir.mkdir(exist_ok=True)
r = subprocess.run(
[str(RUST_BIN), str(spec_path), str(rust_outdir)],
capture_output=True, text=True,
)
if r.returncode != 0:
print("RUST FAILED:", r.stderr); sys.exit(2)
rust_manifest = json.loads((rust_outdir / "manifest.json").read_text())
summary_lines = ["Visual compare: Python proxy.py vs Rust src/rust/", ""]
for name, text in SAMPLES.items():
# Python
py_pngs = py_proxy.render_chunks(text)
assert len(py_pngs) == 1, f"unexpected multi-chunk python output for {name}"
py_path = out_dir / f"python_{name}.png"
py_path.write_bytes(py_pngs[0])
py_img = Image.open(io.BytesIO(py_pngs[0]))
# Rust
rs_files = rust_manifest.get(name, [])
assert len(rs_files) == 1, f"unexpected multi-chunk rust output for {name}"
rs_bytes = (rust_outdir / rs_files[0]).read_bytes()
rs_path = out_dir / f"rust_{name}.png"
rs_path.write_bytes(rs_bytes)
rs_img = Image.open(io.BytesIO(rs_bytes))
# Side-by-side compare
py_labeled = label_image(py_img, f"PYTHON {py_img.width}x{py_img.height} ({len(py_pngs[0])}B)")
rs_labeled = label_image(rs_img, f"RUST {rs_img.width}x{rs_img.height} ({len(rs_bytes)}B)")
gap = 8
cmp_w = max(py_labeled.width, rs_labeled.width)
cmp_h = py_labeled.height + gap + rs_labeled.height
cmp = Image.new("RGB", (cmp_w, cmp_h), (40, 40, 40))
cmp.paste(py_labeled, (0, 0))
cmp.paste(rs_labeled, (0, py_labeled.height + gap))
cmp.save(out_dir / f"compare_{name}.png", "PNG")
line = (
f"[{name:14s}] py={py_img.width:4d}x{py_img.height:<4d} ({len(py_pngs[0]):>6d}B) "
f"rs={rs_img.width:4d}x{rs_img.height:<4d} ({len(rs_bytes):>6d}B) "
f"text_len={len(text)}"
)
print(line)
summary_lines.append(line)
(out_dir / "README.txt").write_text("\n".join(summary_lines) + "\n")
print(f"\nwrote PNGs to {out_dir}/")
if __name__ == "__main__":
main()