diff --git a/.github/workflows/windows-memory-benchmark.yml b/.github/workflows/windows-memory-benchmark.yml index 2d306e5..b888cb9 100644 --- a/.github/workflows/windows-memory-benchmark.yml +++ b/.github/workflows/windows-memory-benchmark.yml @@ -2,11 +2,6 @@ name: Windows memory benchmark on: workflow_dispatch: - push: - branches: ['test/windows-memory-benchmark'] - paths: - - 'validation/windows_positive/**' - - '.github/workflows/windows-memory-benchmark.yml' permissions: contents: read diff --git a/README.md b/README.md index 66f272e..42767ca 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,20 @@ vol -p plugins -s symbols -f linux.lime -o output-linux linux.mcptrace.MCPTrace - Omit `--pid` to scan all processes, or pass multiple PIDs: `--pid 4242 5252`. - `--dump` exports per-process JSONL files to the directory specified by `-o`. - Use `vol -p plugins windows.mcptrace.MCPTrace -h` (or the Linux equivalent) for all options. + +## Benchmark + +Windows Server 2022, Python MCP SDK over stdio: **3 independent RAM captures**, +64 tool calls / 133 messages per condition per capture. Results combine both endpoints. + +| Condition | Messages / 133: mean (range) | Tool pairs / 64: mean (range) | +| --- | ---: | ---: | +| No retained-frame list | 59.7 (55-66) | 18.3 (15-21) | +| Retained-frame control | 133.0 (133-133) | 64.0 (64-64) | + +CLI median: **53.95 s** (53.32-54.48 s, 9 scans of 4 endpoint processes). +Acquisition and initial symbol setup are excluded; scans are not cold-cache measurements. + +**2/3 captures hit candidate search limits** (4 and 3 events); these limited results are included. +Ground-truth logging affects memory, and retained frames are a favorable control. +This is fixture-specific recovery, not general recall. [Methodology and full results](docs/windows-memory-benchmark.md). diff --git a/docs/benchmarks/windows-memory-benchmark-2026-10-27.json b/docs/benchmarks/windows-memory-benchmark-2026-10-27.json new file mode 100644 index 0000000..3380357 --- /dev/null +++ b/docs/benchmarks/windows-memory-benchmark-2026-10-27.json @@ -0,0 +1,373 @@ +{ + "schema_version": 1, + "independent_captures": 3, + "captures_with_scan_limits": 2, + "commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89", + "environment": { + "platform": "Windows-2022Server-10.0.20348-SP0", + "python": "3.12.10", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 4 + }, + "tool_calls_per_case": 64, + "scan_repeats_per_capture": 3, + "modes": { + "natural": { + "totals": { + "expected_logical_messages": 399, + "distinct_expected_messages_found": 179, + "expected_tool_pairs": 192, + "complete_tool_pairs_any_endpoint": 55, + "artifact_records": 485, + "records_matching_logged_traffic": 485, + "distinct_unsent_controls_found": 0, + "records_matching_unsent_controls": 0, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 800, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0 + }, + "logical_message_coverage": 0.44862155388471175, + "messages_found_per_capture": { + "n": 3, + "mean": 59.666666666666664, + "median": 58, + "min": 55, + "max": 66 + }, + "tool_pairs_per_capture": { + "n": 3, + "mean": 18.333333333333332, + "median": 19, + "min": 15, + "max": 21 + } + }, + "retained": { + "totals": { + "expected_logical_messages": 399, + "distinct_expected_messages_found": 399, + "expected_tool_pairs": 192, + "complete_tool_pairs_any_endpoint": 192, + "artifact_records": 854, + "records_matching_logged_traffic": 851, + "distinct_unsent_controls_found": 3, + "records_matching_unsent_controls": 3, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 1680, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0 + }, + "logical_message_coverage": 1.0, + "messages_found_per_capture": { + "n": 3, + "mean": 133, + "median": 133, + "min": 133, + "max": 133 + }, + "tool_pairs_per_capture": { + "n": 3, + "mean": 64, + "median": 64, + "min": 64, + "max": 64 + } + } + }, + "plugin_cli_seconds": { + "n": 9, + "mean": 53.94062956666671, + "median": 53.95398000000006, + "min": 53.317625600000156, + "max": 54.475639200000046 + }, + "per_capture_cli_median_seconds": { + "n": 3, + "mean": 53.88407783333334, + "median": 54.121342500000026, + "min": 53.36027970000009, + "max": 54.170611299999905 + }, + "artifact_records_byte_verified": 1339, + "artifact_bytes_verified": 562402, + "runs": [ + { + "schema_version": 1, + "replica": 1, + "status": "passed", + "commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89", + "environment": { + "platform": "Windows-2022Server-10.0.20348-SP0", + "python": "3.12.10", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 4 + }, + "tool_calls_per_case": 64, + "scan_repeats": 3, + "report_scan_limits": true, + "scan_diagnostics": { + "processes": 4, + "limited_processes": 0, + "error_processes": 0, + "skipped_processes": 0, + "invalid_regions": 0, + "anchor_limit_hits": 0, + "candidate_limit_hits": 0, + "record_budget_processes": 0, + "byte_budget_processes": 0 + }, + "cases": { + "natural": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 66, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 21, + "artifact_records": 173, + "records_matching_logged_traffic": 173, + "distinct_unsent_controls_found": 0, + "records_matching_unsent_controls": 0, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 328, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 14 + }, + "retained": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 133, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 64, + "artifact_records": 299, + "records_matching_logged_traffic": 298, + "distinct_unsent_controls_found": 1, + "records_matching_unsent_controls": 1, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 620, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 64 + } + }, + "capture_sha256": "177c6afd7886aa05c05408c74f8c108caafa6cfbfb181c73f90afe81be931aa8", + "physical_ram_bytes": 17174360064, + "image_logical_bytes": 20401094656, + "artifact_records_byte_verified": 472, + "artifact_bytes_verified": 189113, + "scan_totals": { + "bytes_read": 193032192, + "unreadable_bytes": 17689243648, + "regions_scanned": 576, + "candidate_records": 472 + }, + "timings_seconds": { + "acquisition": 39.56291780000001, + "symbols_and_info": 117.4078207, + "native_byte_verification": 10.376795300000026, + "pipeline_total": 427.56338089999997, + "plugin_cli": [ + 54.121342500000026, + 54.475639200000046, + 53.95398000000006 + ] + }, + "ready_to_capture_seconds": { + "natural": 0.0543968, + "retained": 0.0543968 + }, + "checks": { + "full_endpoint_scans": true, + "native_bytes_verified": true, + "repeated_exports_identical": true, + "image_hash_unchanged": true, + "raw_memory_deleted": true + } + }, + { + "schema_version": 1, + "replica": 2, + "status": "completed_with_limits", + "commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89", + "environment": { + "platform": "Windows-2022Server-10.0.20348-SP0", + "python": "3.12.10", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 4 + }, + "tool_calls_per_case": 64, + "scan_repeats": 3, + "report_scan_limits": true, + "scan_diagnostics": { + "processes": 4, + "limited_processes": 1, + "error_processes": 0, + "skipped_processes": 0, + "invalid_regions": 0, + "anchor_limit_hits": 0, + "candidate_limit_hits": 4, + "record_budget_processes": 0, + "byte_budget_processes": 0 + }, + "cases": { + "natural": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 58, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 19, + "artifact_records": 150, + "records_matching_logged_traffic": 150, + "distinct_unsent_controls_found": 0, + "records_matching_unsent_controls": 0, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 228, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 14 + }, + "retained": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 133, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 64, + "artifact_records": 273, + "records_matching_logged_traffic": 272, + "distinct_unsent_controls_found": 1, + "records_matching_unsent_controls": 1, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 496, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 64 + } + }, + "capture_sha256": "a533b6ca43572eafa8dac01fd36fd9b585fe67bda122ff25bc1b8bb3147b67e6", + "physical_ram_bytes": 17174360064, + "image_logical_bytes": 20401094656, + "artifact_records_byte_verified": 423, + "artifact_bytes_verified": 194685, + "scan_totals": { + "bytes_read": 221855744, + "unreadable_bytes": 17660420096, + "regions_scanned": 576, + "candidate_records": 423 + }, + "timings_seconds": { + "acquisition": 38.52607280000001, + "symbols_and_info": 116.74145750000002, + "native_byte_verification": 11.414818200000013, + "pipeline_total": 428.09518140000006, + "plugin_cli": [ + 54.38048649999996, + 53.882765600000084, + 54.170611299999905 + ] + }, + "ready_to_capture_seconds": { + "natural": 0.0827794, + "retained": 0.0827794 + }, + "checks": { + "full_endpoint_scans": false, + "native_bytes_verified": true, + "repeated_exports_identical": true, + "image_hash_unchanged": true, + "raw_memory_deleted": true + } + }, + { + "schema_version": 1, + "replica": 3, + "status": "completed_with_limits", + "commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89", + "environment": { + "platform": "Windows-2022Server-10.0.20348-SP0", + "python": "3.12.10", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 4 + }, + "tool_calls_per_case": 64, + "scan_repeats": 3, + "report_scan_limits": true, + "scan_diagnostics": { + "processes": 4, + "limited_processes": 1, + "error_processes": 0, + "skipped_processes": 0, + "invalid_regions": 0, + "anchor_limit_hits": 0, + "candidate_limit_hits": 3, + "record_budget_processes": 0, + "byte_budget_processes": 0 + }, + "cases": { + "natural": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 55, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 15, + "artifact_records": 162, + "records_matching_logged_traffic": 162, + "distinct_unsent_controls_found": 0, + "records_matching_unsent_controls": 0, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 244, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 12 + }, + "retained": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 133, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 64, + "artifact_records": 282, + "records_matching_logged_traffic": 281, + "distinct_unsent_controls_found": 1, + "records_matching_unsent_controls": 1, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 564, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 64 + } + }, + "capture_sha256": "ff0627c99a912c71b7c905e30f0cba894bffefa9cfdc5fdc4a40a083349616e3", + "physical_ram_bytes": 17174360064, + "image_logical_bytes": 20401094656, + "artifact_records_byte_verified": 444, + "artifact_bytes_verified": 178604, + "scan_totals": { + "bytes_read": 211656704, + "unreadable_bytes": 17670619136, + "regions_scanned": 576, + "candidate_records": 444 + }, + "timings_seconds": { + "acquisition": 39.40428389999988, + "symbols_and_info": 178.97452040000007, + "native_byte_verification": 10.353987799999686, + "pipeline_total": 487.8709704999999, + "plugin_cli": [ + 53.802935700000035, + 53.36027970000009, + 53.317625600000156 + ] + }, + "ready_to_capture_seconds": { + "natural": 0.0931878, + "retained": 0.0931878 + }, + "checks": { + "full_endpoint_scans": false, + "native_bytes_verified": true, + "repeated_exports_identical": true, + "image_hash_unchanged": true, + "raw_memory_deleted": true + } + } + ] +} diff --git a/docs/windows-memory-benchmark.md b/docs/windows-memory-benchmark.md new file mode 100644 index 0000000..34684d6 --- /dev/null +++ b/docs/windows-memory-benchmark.md @@ -0,0 +1,159 @@ +# Windows RAM benchmark + +Measured on 2026-10-27 with MCPTrace 0.2.0 at commit +`d59f51e118480cb3be08ef96b04f94d8328bcc89`. +[Actions run](https://github.com/Kowntaewook/MCPTrace/actions/runs/40472181802). + +Three fresh Windows Server 2022 jobs each ran two independent MCP SDK stdio sessions: +one with ground-truth logging but no retained-frame list (`natural`), and one with +the same logging plus an explicit retained-frame list (`retained`). Each session +completed 64 echo tool calls and logged 133 distinct logical messages, including +initialization, notification, and tool discovery. One deliberately unsent JSON +control was held in each retained client. No messages were injected into RAM images. + +The eight payload patterns include ASCII, Korean/emoji, quotes/backslashes/newlines, +a 2 KiB string, repeated text with distinct sequence numbers, and whitespace. This +is a bounded SDK fixture, not a sample of real users' conversations or attacks. + +## Recovery + +| Capture | Condition | Messages / 133 | Tool pairs / 64 | Best same-PID pairs | Exported records | +| --- | --- | ---: | ---: | ---: | ---: | +| 1 | natural | 66 | 21 | 14 | 173 | +| 1 | retained | 133 | 64 | 64 | 299 | +| 2 | natural | 58 | 19 | 14 | 150 | +| 2 | retained | 133 | 64 | 64 | 273 | +| 3 | natural | 55 | 15 | 12 | 162 | +| 3 | retained | 133 | 64 | 64 | 282 | + +| Condition | Pooled messages | Mean messages (range) | Pooled tool pairs | Mean tool pairs (range) | +| --- | ---: | ---: | ---: | ---: | +| natural | 179/399 (44.9%) | 59.7 (55-66) | 55/192 | 18.3 (15-21) | +| retained | 399/399 (100.0%) | 133.0 (133-133) | 192/192 | 64.0 (64-64) | + +**Captures 2 and 3 are `completed_with_limits`: one process in each reported 4 and 3 +candidate-backtracking limit events, respectively. Capture 1 had no limit events. +The means and pooled counts include all three captures; limited results were not discarded.** +These counters are search events, not a count of distinct missing messages. The aggregate +does not identify which mode contained the limited process, so the events are not attributed +to either mode. No process-read errors, skipped targets, invalid regions, anchor limits, +record budgets, or byte budgets were observed. + +Message and pair counts combine both endpoints within one capture and deduplicate +logical messages. Matching normalizes parsed JSON objects; incoming SDK ground truth +is reserialized, not a byte-for-byte pipe capture. Exported raw bytes are separately +verified against the memory image. Same-image scan repeats are not additional recovery observations. +There are three independent observations per mode: 399 expected messages and 192 +expected tool-call pairs in total. A pair counted in the endpoint union may have +its request and response in different processes. Per-capture best same-PID pair +counts remain available in the aggregate JSON. + +All **1,339 exported records / 562,402 bytes** passed native +address/length/SHA-256/VAD-bound checks. Repeat scans produced byte-identical JSONL for +each image. The RAM hash remained unchanged and deletion completed for all captures. +The three retained runs recovered all three deliberately unsent controls. Across +2,480 evaluable directional candidate links, incorrect and unverifiable links were +both zero. All remaining exported records matched the logged fixture messages; +there were no unmatched records in this small controlled set. + +Unsent controls are not traffic. Unmatched artifacts are not automatically labeled +false positives. Known directional candidate-link comparisons include duplicate +memory copies and are not a count of unique tool calls or proven sessions. + +## Runtime and scan scope + +| Capture | CLI scan 1 (s) | Scan 2 (s) | Scan 3 (s) | VADs | Read MiB | +| --- | ---: | ---: | ---: | ---: | ---: | +| 1 | 54.121 | 54.476 | 53.954 | 576 | 184.09 | +| 2 | 54.380 | 53.883 | 54.171 | 576 | 211.58 | +| 3 | 53.803 | 53.360 | 53.318 | 576 | 201.85 | + +Across nine scans: mean **53.94 s**, median **53.95 s**, range +**53.32-54.48 s**. Per-capture scan medians are +54.12 s, 54.17 s, 53.36 s. These nine scans are nested within three images, +not nine independent acquisition experiments. + +| Capture | Acquisition (s) | Symbols/info (s) | Native verification (s) | Full pipeline (s) | +| --- | ---: | ---: | ---: | ---: | +| 1 | 39.56 | 117.41 | 10.38 | 427.56 | +| 2 | 38.53 | 116.74 | 11.41 | 428.10 | +| 3 | 39.40 | 178.97 | 10.35 | 487.87 | + +Each of the three images was scanned three times by separate CLI processes using +the same symbol cache, four endpoint PIDs, and default plugin limits. Any exhausted +search limit is retained in the counters and +marked `completed_with_limits`, never presented as a complete scan. Timing includes +startup, Volatility automagic, selected-process scanning, JSON rendering, and JSONL +export. Acquisition, initial symbol preparation, output comparison, image hashing, +and native byte verification are excluded from CLI timing. Acquisition, initial +symbol preparation, and native verification have their own reported durations. +The image was hashed and inspected before the first scan. These are not cold-cache +or full-image/all-process throughput measurements. + +All three runners reported `Windows-2022Server-10.0.20348-SP0`, 4 logical CPUs, +17,174,360,064 bytes of physical RAM (approximately 16 GiB), Python +`3.12.10`, MCP SDK `1.26.0`, and Volatility `2.28.2`. +Each sparse image had a logical size of 20,401,094,656 bytes. The +image includes physical-address holes; this is not the size of scanned process data. +Workload-ready to capture-start delay was 0.054-0.093 s. +These are shared hosted runners, not a dedicated hardware performance guarantee. + +## Reproduction and evidence + +The [benchmark procedure](../validation/windows_positive/BENCHMARK.md) documents the +fixed sample size, workload, commands, and interpretation. The +[workflow](../.github/workflows/windows-memory-benchmark.yml) runs the three fresh +VM captures on demand. The machine-readable result is +[windows-memory-benchmark-2026-10-27.json](benchmarks/windows-memory-benchmark-2026-10-27.json). + +Only allowlisted measurement counts, timings, version/environment information, +image hashes, and validation flags are published. Full RAM images and message +contents are not included. RAM was deleted after analysis, so reproduction means +running the capture harness again rather than reanalyzing these exact images. +The aggregate rejects missing/failed captures, duplicate image hashes, and mismatched +source versions or workload settings. No failed run is silently dropped. + +## Initial attempt and counter correction + +The [initial three-capture attempt](https://github.com/Kowntaewook/MCPTrace/actions/runs/40470688699) +at commit `b4524239bd2c1be12b4c83803fd76908e5a3f833` failed its incomplete-scan gate. +Per-process limit diagnostics were not preserved in that first aggregate, so the +exact gate inputs for those images cannot be reconstructed from the published files. +No recovery or timing values from that failed attempt are included in the final +three-capture measurement. + +A local reproduction using all 133 SDK messages exposed premature limit accounting: +all 133 messages were found, yet 228 candidate-limit events were reported. The +counter incremented when the preceding window contained more than 64 object starts, +even if a valid message was found before search exhaustion. The counter now increments +only when that bounded search fails without examining the older candidates. Search +bounds and extraction logic remain unchanged. UTF-8/UTF-16LE regression cases verify +the correction, and the existing truly exhausted-backtrack test still passes. + +The harness now preserves diagnostics before evaluating limits. Benchmark mode can +measure default-limit output with an explicit `completed_with_limits` status and +published limit counters; read errors or missing endpoints still fail. Strict +positive validation continues rejecting limited scans by default. All three fresh +VM captures were rerun at the corrected commit; they are not selectively chosen +successful members of the original attempt. + +The 135-test suite and Ruff checks passed. The existing public Linux MCP sample was +also rescanned after the counter correction: both JSONL exports were byte-identical +to the earlier result, retaining all 41 records. This is a regression check, not an +independent Linux recovery benchmark; the sample's inferred physical-layout caveat +remains as described in [the Linux validation report](real-memory-validation.md). + +## Limits + +These statistics describe three captures of one Python SDK/stdin-stdout fixture. +Ground-truth logging changes memory, and explicitly retained frames are a favorable +control. Memory acquisition takes time; it is not an atomic snapshot. No deliberate +post-call delay or extra memory-pressure workload was introduced. Results may vary +with residency, acquisition, workload, and runtime state; this experiment does not +separate those causes. + +This benchmark does not establish general recall/precision, attack detection, +Windows 11 or Linux performance, other client/SDK compatibility, HTTP/SSE support, +or survival over longer collection delays. The earlier +[8-call positive validation](windows-positive-memory-validation.md) used a different +workload size and should not be compared as a controlled before/after experiment. diff --git a/validation/windows_positive/BENCHMARK.md b/validation/windows_positive/BENCHMARK.md index f78dd40..53f7e07 100644 --- a/validation/windows_positive/BENCHMARK.md +++ b/validation/windows_positive/BENCHMARK.md @@ -58,3 +58,5 @@ traffic. Mean/min/max describe only these three captures. Logging changes memory and retained frames are a favorable control. No general recall, detection precision, latency/pressure survival curve, or Windows 11/Linux benchmark is established by this Windows Server 2022 SDK/stdio experiment. + +Measured results: [Windows RAM benchmark](../../docs/windows-memory-benchmark.md).