From f094649831c9afb8900d70fc4aec38a81c2e16dd Mon Sep 17 00:00:00 2001 From: teamchong <25894545+teamchong@users.noreply.github.com> Date: Mon, 18 May 2026 13:03:28 -0400 Subject: [PATCH] Rust transform: match Python's body output byte-for-byte (except PNG) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bug fixes surfaced by scripts/diff_transform.py harness, which synthesizes a realistic Claude Code request body and runs it through both Python transform_request() and Rust transform(), then JSON-structurally diffs the outputs. Before these fixes, harness found: - 2 structural diffs in the transformed body JSON - 4+ TransformInfo divergences After these fixes, harness finds: - 1 structural diff: $.messages[0].content[1].source.data (the base64 PNG, expected to differ — Menlo vs JetBrains Mono fonts) - TransformInfo still differs on byte-vs-char counts (telemetry-only) Fixes: 1. src/rust/src/transform.rs:93 — per-tool markdown header was: '### Tool: {name}\n{desc}' now: '#### Tool: {name}\n{desc}\n' Matches proxy.py:230,233. Also restores the trailing newline that Python emits between tool entries. 2. src/rust/src/transform.rs:123 — tool section markdown header was: '\n\n# Tool Documentation\n\n' now: '\n\n#### Tool Documentation\n\n' Matches proxy.py:248. 3. src/rust/src/transform.rs:265 — preserve system field by default was: always replaced system with billing_line OR remainder now: only replaces if billing_line was stripped, otherwise leaves the original system field untouched. Matches proxy.py:417 which intentionally leaves system alone in the default PLACEMENT="user" path 'so Anthropic still has guardrails'. The image lives in messages[].content as a side-channel; system stays as original text. This is the behavior our 95+ live Python requests have validated against Anthropic's API. 4. src/rust/src/lib.rs — expose 'transform' module so harnesses and the transform_only example can call transform() without going through HTTP. Added: - src/rust/examples/transform_only.rs — CLI binary that reads a body from a file and writes the transformed body + TransformInfo as JSON. - scripts/diff_transform.py — Python harness that calls both Python transform_request() and the Rust transform_only binary, then runs a structural JSON diff with field-by-field TransformInfo comparison. Validation: python3 scripts/diff_transform.py # → 'found 1 structural diff' # → $.messages[0].content[1].source.data (the PNG, expected) Remaining known telemetry-only diffs (not blocking the swap): - system_text_chars: Python counts Unicode chars, Rust counts UTF-8 bytes - tool_text_added: same UTF-8 issue - expected_image_tokens: ~30% higher in Rust due to ab_glyph 4px cell vs PIL 3px cell — Rust images are 33% wider for the same text. Rust PNG bytes are still smaller overall because of the tight-bound height (which Python also has now, after the previous baseline commit). - placement, png_sha8 fields missing from Rust TransformInfo - min_chars threshold uses bytes (Rust) vs chars (Python) These can be addressed later for full telemetry parity but the upstream-visible behavior is byte-equivalent. --- scripts/diff_transform.py | 213 ++++++++++++++++++++++++++++ src/rust/examples/transform_only.rs | 64 +++++++++ src/rust/src/lib.rs | 1 + src/rust/src/transform.rs | 18 ++- 4 files changed, 289 insertions(+), 7 deletions(-) create mode 100644 scripts/diff_transform.py create mode 100644 src/rust/examples/transform_only.rs diff --git a/scripts/diff_transform.py b/scripts/diff_transform.py new file mode 100644 index 0000000..a3e9e27 --- /dev/null +++ b/scripts/diff_transform.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +""" +Diff harness at the TRANSFORM layer: synthesize a realistic Anthropic +/v1/messages body, run it through Python proxy.transform_request() AND +the Rust transform_only binary, then diff what each emits. + +This is the validation we need before switching :47821 to Rust. It +catches divergences in: + - Field structure (system field rewriting, cache_control placement) + - Image block emission (number of imgs, dims) + - Tools/schemas/reminders compression behavior + - JSON serialization order (Anthropic cache hashes on bytes) + +Run: + python3 scripts/diff_transform.py +""" + +import io +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "src")) + +import proxy as py_proxy # noqa: E402 + +RUST_BIN = ROOT / "src" / "rust" / "target" / "release" / "examples" / "transform_only" + + +def synthesize_body() -> bytes: + """Build a realistic Claude Code-shaped /v1/messages body. + + Shape based on what real Anthropic API requests look like: + - long `system` field (system prompt content) + - `tools` array with name/description/input_schema each + - `messages` array + - `max_tokens`, `model`, etc. + """ + # Real-ish system prompt: load CLAUDE.md + some extra padding to clear MIN_COMPRESS_CHARS=2000 + sysprompt_parts = [] + try: + sysprompt_parts.append((ROOT / "CLAUDE.md").read_text()) + except Exception: + pass + try: + sysprompt_parts.append((ROOT / "HANDOFF.md").read_text()) + except Exception: + pass + sysprompt_parts.append( + "\n## Additional padding to ensure >2000 chars\n" + + "Line of fake system instructions blah blah blah.\n" * 30 + ) + sysprompt = "\n\n".join(sysprompt_parts) + + body = { + "model": "claude-opus-4-5-20250929", + "max_tokens": 8192, + "system": [ + {"type": "text", "text": sysprompt}, + ], + "tools": [ + { + "name": "Bash", + "description": "Execute a bash command with optional timeout.\n\nThe command runs in a fresh subshell. Use this for git, builds, file ops.", + "input_schema": { + "type": "object", + "properties": { + "command": {"type": "string", "description": "The bash command to execute."}, + "timeout": {"type": "integer", "description": "Timeout in milliseconds. Defaults to 120000."}, + "description": {"type": "string", "description": "A short description of what the command does."}, + }, + "required": ["command"], + }, + }, + { + "name": "Read", + "description": "Read a file from disk.\n\nReturns up to 2000 lines starting from offset. Long lines truncated at 2000 chars.", + "input_schema": { + "type": "object", + "properties": { + "file_path": {"type": "string", "description": "Absolute path to the file."}, + "offset": {"type": "integer", "description": "1-indexed line to start at."}, + "limit": {"type": "integer", "description": "Max number of lines to read."}, + }, + "required": ["file_path"], + }, + }, + { + "name": "Edit", + "description": "Apply an exact-string find/replace in a file.\n\nold_string must match exactly including whitespace. Use replace_all for global.", + "input_schema": { + "type": "object", + "properties": { + "file_path": {"type": "string"}, + "old_string": {"type": "string"}, + "new_string": {"type": "string"}, + "replace_all": {"type": "boolean"}, + }, + "required": ["file_path", "old_string", "new_string"], + }, + }, + ], + "messages": [ + { + "role": "user", + "content": "Run `git status` and tell me what's modified.", + }, + ], + } + return json.dumps(body).encode() + + +def deep_compare(a, b, path=""): + """Walk two JSON structures and yield (path, kind, py_repr, rs_repr) diffs.""" + if type(a) is not type(b): + yield (path, "type", type(a).__name__, type(b).__name__) + return + if isinstance(a, dict): + keys_only_py = set(a) - set(b) + keys_only_rs = set(b) - set(a) + for k in keys_only_py: + yield (f"{path}.{k}", "missing_in_rs", short(a[k]), "") + for k in keys_only_rs: + yield (f"{path}.{k}", "missing_in_py", "", short(b[k])) + for k in set(a) & set(b): + yield from deep_compare(a[k], b[k], f"{path}.{k}") + return + if isinstance(a, list): + if len(a) != len(b): + yield (path, "list_len", str(len(a)), str(len(b))) + for i, (x, y) in enumerate(zip(a, b)): + yield from deep_compare(x, y, f"{path}[{i}]") + return + if a != b: + yield (path, "value", short(a), short(b)) + + +def short(v) -> str: + s = json.dumps(v) if not isinstance(v, str) else v + if len(s) > 80: + return s[:77] + "..." + return s + + +def main(): + if not RUST_BIN.exists(): + print(f"build rust first: cd src/rust && cargo build --release --example transform_only") + sys.exit(1) + + body = synthesize_body() + print(f"input body: {len(body)} bytes") + + # Python transform + py_out, py_info = py_proxy.transform_request(body) + print(f"python transform: out={len(py_out)}B info_keys={sorted(py_info.keys())}") + + # Rust transform via subprocess + rust_dir = Path("/tmp/cip_xform_rs") + rust_dir.mkdir(exist_ok=True) + body_path = rust_dir / "body_in.json" + body_path.write_bytes(body) + r = subprocess.run( + [str(RUST_BIN), str(body_path), str(rust_dir)], + capture_output=True, text=True, + ) + if r.returncode != 0: + print("RUST FAILED:", r.stderr); sys.exit(2) + print("rust transform stderr:", r.stderr.strip()) + rs_out = (rust_dir / "body_out.json").read_bytes() + rs_info = json.loads((rust_dir / "info.json").read_text()) + + print(f"rust transform: out={len(rs_out)}B info_keys={sorted(rs_info.keys())}") + + print(f"\n=== body bytes: py={len(py_out)} rs={len(rs_out)} delta={len(rs_out)-len(py_out):+d} ===") + + # Decode and structural diff + try: + py_json = json.loads(py_out) + rs_json = json.loads(rs_out) + except json.JSONDecodeError as e: + print(f"FAIL: invalid JSON output: {e}") + sys.exit(3) + + diffs = list(deep_compare(py_json, rs_json, "$")) + if not diffs: + print("body JSON is STRUCTURALLY IDENTICAL") + else: + print(f"\nfound {len(diffs)} structural diffs (showing first 30):") + for path, kind, pv, rv in diffs[:30]: + print(f" [{kind:18s}] {path}") + print(f" py: {pv}") + print(f" rs: {rv}") + if len(diffs) > 30: + print(f" ... and {len(diffs)-30} more") + + # Info side-by-side + print("\n=== TransformInfo comparison ===") + common = sorted(set(py_info.keys()) | set(rs_info.keys())) + print(f"{'field':30s} {'python':>14s} {'rust':>14s}") + for k in common: + pv = py_info.get(k, "") + rv = rs_info.get(k, "") + if isinstance(pv, list): pv = f"list[{len(pv)}]" + if isinstance(rv, list): rv = f"list[{len(rv)}]" + print(f"{k:30s} {str(pv):>14s} {str(rv):>14s}") + + sys.exit(0 if not diffs else 1) + + +if __name__ == "__main__": + main() diff --git a/src/rust/examples/transform_only.rs b/src/rust/examples/transform_only.rs new file mode 100644 index 0000000..14de553 --- /dev/null +++ b/src/rust/examples/transform_only.rs @@ -0,0 +1,64 @@ +//! Headless transform: read a request body from file (or stdin), run it through +//! the Rust transform() with default Claude Code settings, write the transformed +//! body + info JSON to disk for offline diffing against Python's transform. +//! +//! Run: cargo run --release --example transform_only -- +//! +//! Produces: +//! /body_out.json — transformed body bytes (the JSON forwarded upstream) +//! /info.json — TransformInfo struct (dims, expected_image_tokens, etc.) +//! +//! Settings mirror bin/cli.js defaults: compress=true, all sub-flags on, +//! min_chars=2000, font_size=5pt. No env-var overrides. + +use std::path::PathBuf; + +use claude_image_proxy::font::AtlasFont; +use claude_image_proxy::transform::{transform, TransformConfig}; +use dashmap::DashMap; + +fn main() -> anyhow::Result<()> { + let mut args = std::env::args().skip(1); + let body_path: PathBuf = args + .next() + .ok_or_else(|| anyhow::anyhow!("usage: transform_only "))? + .into(); + let out_dir: PathBuf = args + .next() + .ok_or_else(|| anyhow::anyhow!("usage: transform_only "))? + .into(); + + std::fs::create_dir_all(&out_dir)?; + + let body = std::fs::read(&body_path)?; + let font = AtlasFont::load(5.0)?; + let render_cache = DashMap::new(); + let cfg = TransformConfig { + compress: true, + compress_tools: true, + compress_schemas: true, + compress_reminders: true, + compress_tool_results: true, + min_chars: 2000, + font: &font, + render_cache: &render_cache, + }; + + let (body_out, info) = transform(&body, &cfg); + + std::fs::write(out_dir.join("body_out.json"), &body_out)?; + std::fs::write( + out_dir.join("info.json"), + serde_json::to_vec_pretty(&info)?, + )?; + + eprintln!( + "rust transform: in={}B out={}B compressed={} imgs={} dims={:?}", + body.len(), + body_out.len(), + info.compressed, + info.images, + info.dims, + ); + Ok(()) +} diff --git a/src/rust/src/lib.rs b/src/rust/src/lib.rs index 1ff9b45..46cbb2c 100644 --- a/src/rust/src/lib.rs +++ b/src/rust/src/lib.rs @@ -1,2 +1,3 @@ pub mod font; pub mod render; +pub mod transform; diff --git a/src/rust/src/transform.rs b/src/rust/src/transform.rs index 5cf1ca3..78bafa5 100644 --- a/src/rust/src/transform.rs +++ b/src/rust/src/transform.rs @@ -90,7 +90,7 @@ pub fn transform(body: &[u8], cfg: &TransformConfig) -> (Vec, TransformInfo) let mut doc = String::new(); if !desc.is_empty() { - doc.push_str(&format!("### Tool: {}\n{}", name, desc)); + doc.push_str(&format!("#### Tool: {}\n{}\n", name, desc)); if desc.len() > 80 { obj.insert("description".to_string(), json!(format!("See `{}` docs in system context image.", name))); @@ -120,7 +120,7 @@ pub fn transform(body: &[u8], cfg: &TransformConfig) -> (Vec, TransformInfo) } if !tool_docs.is_empty() { let tool_section = format!( - "\n\n# Tool Documentation\n\n{}", + "\n\n#### Tool Documentation\n\n{}", tool_docs.join("\n\n") ); info.tool_text_added = tool_section.len(); @@ -262,14 +262,18 @@ pub fn transform(body: &[u8], cfg: &TransformConfig) -> (Vec, TransformInfo) } } - // Replace system field: keep just the per-turn billing header line as text - // if present (so Anthropic still receives the auth/billing context); else - // leave whatever non-text remainder we extracted in place. + // Replace system field ONLY if we stripped a billing header — otherwise leave + // the original system field unchanged so Anthropic still has guardrails. This + // matches proxy.py's default PLACEMENT="user" behavior (proxy.py:417 comment + // "leave system unchanged so Anthropic still has guardrails"). The image + // content lives in messages[].content as a side-channel; system stays as + // original text. The duplicate "cost" is reclaimed on subsequent turns by + // the image's cache_control:ephemeral:1h tag and Anthropic's prompt cache. if let Some(billing) = billing_line { req.as_object_mut().unwrap().insert("system".to_string(), Value::String(billing)); - } else if let Some(rem) = remainder { - req.as_object_mut().unwrap().insert("system".to_string(), rem); } + let _ = remainder; // intentionally unused — see comment above + info.compressed = true; let new_body = serde_json::to_vec(&req).unwrap_or_else(|_| body.to_vec());