diff --git a/.github/workflows/windows-memory-benchmark.yml b/.github/workflows/windows-memory-benchmark.yml new file mode 100644 index 0000000..b888cb9 --- /dev/null +++ b/.github/workflows/windows-memory-benchmark.yml @@ -0,0 +1,50 @@ +name: Windows memory benchmark + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: windows-memory-benchmark-${{ github.ref }} + cancel-in-progress: false + +jobs: + capture: + name: Capture ${{ matrix.replica }} + runs-on: windows-2022 + timeout-minutes: 25 + strategy: + fail-fast: false + max-parallel: 3 + matrix: + replica: [1, 2, 3] + env: + PYTHONUTF8: '1' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + - name: Install plugin and pinned SDK + run: python -m pip install -e . -r validation/windows_positive/requirements.txt + - name: Acquire real RAM and benchmark three scans + shell: pwsh + run: | + python validation/windows_positive/run.py --workdir "$env:RUNNER_TEMP/mcp-benchmark-work" --reportdir "$env:RUNNER_TEMP/mcp-benchmark-report" --tool-calls 64 --scan-repeats 3 --report-scan-limits + - name: Extract aggregate measurements only + if: always() + shell: pwsh + run: | + python validation/windows_positive/benchmark.py summarize --reportdir "$env:RUNNER_TEMP/mcp-benchmark-report" --replica ${{ matrix.replica }} --output "$env:RUNNER_TEMP/benchmark-${{ matrix.replica }}.json" + - name: Upload aggregate measurements only + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: windows-benchmark-${{ github.run_id }}-${{ matrix.replica }} + path: ${{ runner.temp }}/benchmark-${{ matrix.replica }}.json + if-no-files-found: warn + retention-days: 30 diff --git a/README.md b/README.md index 66f272e..42767ca 100644 --- a/README.md +++ b/README.md @@ -37,3 +37,20 @@ vol -p plugins -s symbols -f linux.lime -o output-linux linux.mcptrace.MCPTrace - Omit `--pid` to scan all processes, or pass multiple PIDs: `--pid 4242 5252`. - `--dump` exports per-process JSONL files to the directory specified by `-o`. - Use `vol -p plugins windows.mcptrace.MCPTrace -h` (or the Linux equivalent) for all options. + +## Benchmark + +Windows Server 2022, Python MCP SDK over stdio: **3 independent RAM captures**, +64 tool calls / 133 messages per condition per capture. Results combine both endpoints. + +| Condition | Messages / 133: mean (range) | Tool pairs / 64: mean (range) | +| --- | ---: | ---: | +| No retained-frame list | 59.7 (55-66) | 18.3 (15-21) | +| Retained-frame control | 133.0 (133-133) | 64.0 (64-64) | + +CLI median: **53.95 s** (53.32-54.48 s, 9 scans of 4 endpoint processes). +Acquisition and initial symbol setup are excluded; scans are not cold-cache measurements. + +**2/3 captures hit candidate search limits** (4 and 3 events); these limited results are included. +Ground-truth logging affects memory, and retained frames are a favorable control. +This is fixture-specific recovery, not general recall. [Methodology and full results](docs/windows-memory-benchmark.md). diff --git a/docs/benchmarks/windows-memory-benchmark-2026-10-27.json b/docs/benchmarks/windows-memory-benchmark-2026-10-27.json new file mode 100644 index 0000000..3380357 --- /dev/null +++ b/docs/benchmarks/windows-memory-benchmark-2026-10-27.json @@ -0,0 +1,373 @@ +{ + "schema_version": 1, + "independent_captures": 3, + "captures_with_scan_limits": 2, + "commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89", + "environment": { + "platform": "Windows-2022Server-10.0.20348-SP0", + "python": "3.12.10", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 4 + }, + "tool_calls_per_case": 64, + "scan_repeats_per_capture": 3, + "modes": { + "natural": { + "totals": { + "expected_logical_messages": 399, + "distinct_expected_messages_found": 179, + "expected_tool_pairs": 192, + "complete_tool_pairs_any_endpoint": 55, + "artifact_records": 485, + "records_matching_logged_traffic": 485, + "distinct_unsent_controls_found": 0, + "records_matching_unsent_controls": 0, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 800, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0 + }, + "logical_message_coverage": 0.44862155388471175, + "messages_found_per_capture": { + "n": 3, + "mean": 59.666666666666664, + "median": 58, + "min": 55, + "max": 66 + }, + "tool_pairs_per_capture": { + "n": 3, + "mean": 18.333333333333332, + "median": 19, + "min": 15, + "max": 21 + } + }, + "retained": { + "totals": { + "expected_logical_messages": 399, + "distinct_expected_messages_found": 399, + "expected_tool_pairs": 192, + "complete_tool_pairs_any_endpoint": 192, + "artifact_records": 854, + "records_matching_logged_traffic": 851, + "distinct_unsent_controls_found": 3, + "records_matching_unsent_controls": 3, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 1680, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0 + }, + "logical_message_coverage": 1.0, + "messages_found_per_capture": { + "n": 3, + "mean": 133, + "median": 133, + "min": 133, + "max": 133 + }, + "tool_pairs_per_capture": { + "n": 3, + "mean": 64, + "median": 64, + "min": 64, + "max": 64 + } + } + }, + "plugin_cli_seconds": { + "n": 9, + "mean": 53.94062956666671, + "median": 53.95398000000006, + "min": 53.317625600000156, + "max": 54.475639200000046 + }, + "per_capture_cli_median_seconds": { + "n": 3, + "mean": 53.88407783333334, + "median": 54.121342500000026, + "min": 53.36027970000009, + "max": 54.170611299999905 + }, + "artifact_records_byte_verified": 1339, + "artifact_bytes_verified": 562402, + "runs": [ + { + "schema_version": 1, + "replica": 1, + "status": "passed", + "commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89", + "environment": { + "platform": "Windows-2022Server-10.0.20348-SP0", + "python": "3.12.10", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 4 + }, + "tool_calls_per_case": 64, + "scan_repeats": 3, + "report_scan_limits": true, + "scan_diagnostics": { + "processes": 4, + "limited_processes": 0, + "error_processes": 0, + "skipped_processes": 0, + "invalid_regions": 0, + "anchor_limit_hits": 0, + "candidate_limit_hits": 0, + "record_budget_processes": 0, + "byte_budget_processes": 0 + }, + "cases": { + "natural": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 66, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 21, + "artifact_records": 173, + "records_matching_logged_traffic": 173, + "distinct_unsent_controls_found": 0, + "records_matching_unsent_controls": 0, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 328, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 14 + }, + "retained": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 133, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 64, + "artifact_records": 299, + "records_matching_logged_traffic": 298, + "distinct_unsent_controls_found": 1, + "records_matching_unsent_controls": 1, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 620, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 64 + } + }, + "capture_sha256": "177c6afd7886aa05c05408c74f8c108caafa6cfbfb181c73f90afe81be931aa8", + "physical_ram_bytes": 17174360064, + "image_logical_bytes": 20401094656, + "artifact_records_byte_verified": 472, + "artifact_bytes_verified": 189113, + "scan_totals": { + "bytes_read": 193032192, + "unreadable_bytes": 17689243648, + "regions_scanned": 576, + "candidate_records": 472 + }, + "timings_seconds": { + "acquisition": 39.56291780000001, + "symbols_and_info": 117.4078207, + "native_byte_verification": 10.376795300000026, + "pipeline_total": 427.56338089999997, + "plugin_cli": [ + 54.121342500000026, + 54.475639200000046, + 53.95398000000006 + ] + }, + "ready_to_capture_seconds": { + "natural": 0.0543968, + "retained": 0.0543968 + }, + "checks": { + "full_endpoint_scans": true, + "native_bytes_verified": true, + "repeated_exports_identical": true, + "image_hash_unchanged": true, + "raw_memory_deleted": true + } + }, + { + "schema_version": 1, + "replica": 2, + "status": "completed_with_limits", + "commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89", + "environment": { + "platform": "Windows-2022Server-10.0.20348-SP0", + "python": "3.12.10", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 4 + }, + "tool_calls_per_case": 64, + "scan_repeats": 3, + "report_scan_limits": true, + "scan_diagnostics": { + "processes": 4, + "limited_processes": 1, + "error_processes": 0, + "skipped_processes": 0, + "invalid_regions": 0, + "anchor_limit_hits": 0, + "candidate_limit_hits": 4, + "record_budget_processes": 0, + "byte_budget_processes": 0 + }, + "cases": { + "natural": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 58, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 19, + "artifact_records": 150, + "records_matching_logged_traffic": 150, + "distinct_unsent_controls_found": 0, + "records_matching_unsent_controls": 0, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 228, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 14 + }, + "retained": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 133, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 64, + "artifact_records": 273, + "records_matching_logged_traffic": 272, + "distinct_unsent_controls_found": 1, + "records_matching_unsent_controls": 1, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 496, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 64 + } + }, + "capture_sha256": "a533b6ca43572eafa8dac01fd36fd9b585fe67bda122ff25bc1b8bb3147b67e6", + "physical_ram_bytes": 17174360064, + "image_logical_bytes": 20401094656, + "artifact_records_byte_verified": 423, + "artifact_bytes_verified": 194685, + "scan_totals": { + "bytes_read": 221855744, + "unreadable_bytes": 17660420096, + "regions_scanned": 576, + "candidate_records": 423 + }, + "timings_seconds": { + "acquisition": 38.52607280000001, + "symbols_and_info": 116.74145750000002, + "native_byte_verification": 11.414818200000013, + "pipeline_total": 428.09518140000006, + "plugin_cli": [ + 54.38048649999996, + 53.882765600000084, + 54.170611299999905 + ] + }, + "ready_to_capture_seconds": { + "natural": 0.0827794, + "retained": 0.0827794 + }, + "checks": { + "full_endpoint_scans": false, + "native_bytes_verified": true, + "repeated_exports_identical": true, + "image_hash_unchanged": true, + "raw_memory_deleted": true + } + }, + { + "schema_version": 1, + "replica": 3, + "status": "completed_with_limits", + "commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89", + "environment": { + "platform": "Windows-2022Server-10.0.20348-SP0", + "python": "3.12.10", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 4 + }, + "tool_calls_per_case": 64, + "scan_repeats": 3, + "report_scan_limits": true, + "scan_diagnostics": { + "processes": 4, + "limited_processes": 1, + "error_processes": 0, + "skipped_processes": 0, + "invalid_regions": 0, + "anchor_limit_hits": 0, + "candidate_limit_hits": 3, + "record_budget_processes": 0, + "byte_budget_processes": 0 + }, + "cases": { + "natural": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 55, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 15, + "artifact_records": 162, + "records_matching_logged_traffic": 162, + "distinct_unsent_controls_found": 0, + "records_matching_unsent_controls": 0, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 244, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 12 + }, + "retained": { + "expected_logical_messages": 133, + "distinct_expected_messages_found": 133, + "expected_tool_pairs": 64, + "complete_tool_pairs_any_endpoint": 64, + "artifact_records": 282, + "records_matching_logged_traffic": 281, + "distinct_unsent_controls_found": 1, + "records_matching_unsent_controls": 1, + "unmatched_artifact_records": 0, + "evaluated_candidate_links": 564, + "incorrect_candidate_links": 0, + "unverifiable_candidate_links": 0, + "best_single_pid_tool_pairs": 64 + } + }, + "capture_sha256": "ff0627c99a912c71b7c905e30f0cba894bffefa9cfdc5fdc4a40a083349616e3", + "physical_ram_bytes": 17174360064, + "image_logical_bytes": 20401094656, + "artifact_records_byte_verified": 444, + "artifact_bytes_verified": 178604, + "scan_totals": { + "bytes_read": 211656704, + "unreadable_bytes": 17670619136, + "regions_scanned": 576, + "candidate_records": 444 + }, + "timings_seconds": { + "acquisition": 39.40428389999988, + "symbols_and_info": 178.97452040000007, + "native_byte_verification": 10.353987799999686, + "pipeline_total": 487.8709704999999, + "plugin_cli": [ + 53.802935700000035, + 53.36027970000009, + 53.317625600000156 + ] + }, + "ready_to_capture_seconds": { + "natural": 0.0931878, + "retained": 0.0931878 + }, + "checks": { + "full_endpoint_scans": false, + "native_bytes_verified": true, + "repeated_exports_identical": true, + "image_hash_unchanged": true, + "raw_memory_deleted": true + } + } + ] +} diff --git a/docs/windows-memory-benchmark.md b/docs/windows-memory-benchmark.md new file mode 100644 index 0000000..34684d6 --- /dev/null +++ b/docs/windows-memory-benchmark.md @@ -0,0 +1,159 @@ +# Windows RAM benchmark + +Measured on 2026-10-27 with MCPTrace 0.2.0 at commit +`d59f51e118480cb3be08ef96b04f94d8328bcc89`. +[Actions run](https://github.com/Kowntaewook/MCPTrace/actions/runs/40472181802). + +Three fresh Windows Server 2022 jobs each ran two independent MCP SDK stdio sessions: +one with ground-truth logging but no retained-frame list (`natural`), and one with +the same logging plus an explicit retained-frame list (`retained`). Each session +completed 64 echo tool calls and logged 133 distinct logical messages, including +initialization, notification, and tool discovery. One deliberately unsent JSON +control was held in each retained client. No messages were injected into RAM images. + +The eight payload patterns include ASCII, Korean/emoji, quotes/backslashes/newlines, +a 2 KiB string, repeated text with distinct sequence numbers, and whitespace. This +is a bounded SDK fixture, not a sample of real users' conversations or attacks. + +## Recovery + +| Capture | Condition | Messages / 133 | Tool pairs / 64 | Best same-PID pairs | Exported records | +| --- | --- | ---: | ---: | ---: | ---: | +| 1 | natural | 66 | 21 | 14 | 173 | +| 1 | retained | 133 | 64 | 64 | 299 | +| 2 | natural | 58 | 19 | 14 | 150 | +| 2 | retained | 133 | 64 | 64 | 273 | +| 3 | natural | 55 | 15 | 12 | 162 | +| 3 | retained | 133 | 64 | 64 | 282 | + +| Condition | Pooled messages | Mean messages (range) | Pooled tool pairs | Mean tool pairs (range) | +| --- | ---: | ---: | ---: | ---: | +| natural | 179/399 (44.9%) | 59.7 (55-66) | 55/192 | 18.3 (15-21) | +| retained | 399/399 (100.0%) | 133.0 (133-133) | 192/192 | 64.0 (64-64) | + +**Captures 2 and 3 are `completed_with_limits`: one process in each reported 4 and 3 +candidate-backtracking limit events, respectively. Capture 1 had no limit events. +The means and pooled counts include all three captures; limited results were not discarded.** +These counters are search events, not a count of distinct missing messages. The aggregate +does not identify which mode contained the limited process, so the events are not attributed +to either mode. No process-read errors, skipped targets, invalid regions, anchor limits, +record budgets, or byte budgets were observed. + +Message and pair counts combine both endpoints within one capture and deduplicate +logical messages. Matching normalizes parsed JSON objects; incoming SDK ground truth +is reserialized, not a byte-for-byte pipe capture. Exported raw bytes are separately +verified against the memory image. Same-image scan repeats are not additional recovery observations. +There are three independent observations per mode: 399 expected messages and 192 +expected tool-call pairs in total. A pair counted in the endpoint union may have +its request and response in different processes. Per-capture best same-PID pair +counts remain available in the aggregate JSON. + +All **1,339 exported records / 562,402 bytes** passed native +address/length/SHA-256/VAD-bound checks. Repeat scans produced byte-identical JSONL for +each image. The RAM hash remained unchanged and deletion completed for all captures. +The three retained runs recovered all three deliberately unsent controls. Across +2,480 evaluable directional candidate links, incorrect and unverifiable links were +both zero. All remaining exported records matched the logged fixture messages; +there were no unmatched records in this small controlled set. + +Unsent controls are not traffic. Unmatched artifacts are not automatically labeled +false positives. Known directional candidate-link comparisons include duplicate +memory copies and are not a count of unique tool calls or proven sessions. + +## Runtime and scan scope + +| Capture | CLI scan 1 (s) | Scan 2 (s) | Scan 3 (s) | VADs | Read MiB | +| --- | ---: | ---: | ---: | ---: | ---: | +| 1 | 54.121 | 54.476 | 53.954 | 576 | 184.09 | +| 2 | 54.380 | 53.883 | 54.171 | 576 | 211.58 | +| 3 | 53.803 | 53.360 | 53.318 | 576 | 201.85 | + +Across nine scans: mean **53.94 s**, median **53.95 s**, range +**53.32-54.48 s**. Per-capture scan medians are +54.12 s, 54.17 s, 53.36 s. These nine scans are nested within three images, +not nine independent acquisition experiments. + +| Capture | Acquisition (s) | Symbols/info (s) | Native verification (s) | Full pipeline (s) | +| --- | ---: | ---: | ---: | ---: | +| 1 | 39.56 | 117.41 | 10.38 | 427.56 | +| 2 | 38.53 | 116.74 | 11.41 | 428.10 | +| 3 | 39.40 | 178.97 | 10.35 | 487.87 | + +Each of the three images was scanned three times by separate CLI processes using +the same symbol cache, four endpoint PIDs, and default plugin limits. Any exhausted +search limit is retained in the counters and +marked `completed_with_limits`, never presented as a complete scan. Timing includes +startup, Volatility automagic, selected-process scanning, JSON rendering, and JSONL +export. Acquisition, initial symbol preparation, output comparison, image hashing, +and native byte verification are excluded from CLI timing. Acquisition, initial +symbol preparation, and native verification have their own reported durations. +The image was hashed and inspected before the first scan. These are not cold-cache +or full-image/all-process throughput measurements. + +All three runners reported `Windows-2022Server-10.0.20348-SP0`, 4 logical CPUs, +17,174,360,064 bytes of physical RAM (approximately 16 GiB), Python +`3.12.10`, MCP SDK `1.26.0`, and Volatility `2.28.2`. +Each sparse image had a logical size of 20,401,094,656 bytes. The +image includes physical-address holes; this is not the size of scanned process data. +Workload-ready to capture-start delay was 0.054-0.093 s. +These are shared hosted runners, not a dedicated hardware performance guarantee. + +## Reproduction and evidence + +The [benchmark procedure](../validation/windows_positive/BENCHMARK.md) documents the +fixed sample size, workload, commands, and interpretation. The +[workflow](../.github/workflows/windows-memory-benchmark.yml) runs the three fresh +VM captures on demand. The machine-readable result is +[windows-memory-benchmark-2026-10-27.json](benchmarks/windows-memory-benchmark-2026-10-27.json). + +Only allowlisted measurement counts, timings, version/environment information, +image hashes, and validation flags are published. Full RAM images and message +contents are not included. RAM was deleted after analysis, so reproduction means +running the capture harness again rather than reanalyzing these exact images. +The aggregate rejects missing/failed captures, duplicate image hashes, and mismatched +source versions or workload settings. No failed run is silently dropped. + +## Initial attempt and counter correction + +The [initial three-capture attempt](https://github.com/Kowntaewook/MCPTrace/actions/runs/40470688699) +at commit `b4524239bd2c1be12b4c83803fd76908e5a3f833` failed its incomplete-scan gate. +Per-process limit diagnostics were not preserved in that first aggregate, so the +exact gate inputs for those images cannot be reconstructed from the published files. +No recovery or timing values from that failed attempt are included in the final +three-capture measurement. + +A local reproduction using all 133 SDK messages exposed premature limit accounting: +all 133 messages were found, yet 228 candidate-limit events were reported. The +counter incremented when the preceding window contained more than 64 object starts, +even if a valid message was found before search exhaustion. The counter now increments +only when that bounded search fails without examining the older candidates. Search +bounds and extraction logic remain unchanged. UTF-8/UTF-16LE regression cases verify +the correction, and the existing truly exhausted-backtrack test still passes. + +The harness now preserves diagnostics before evaluating limits. Benchmark mode can +measure default-limit output with an explicit `completed_with_limits` status and +published limit counters; read errors or missing endpoints still fail. Strict +positive validation continues rejecting limited scans by default. All three fresh +VM captures were rerun at the corrected commit; they are not selectively chosen +successful members of the original attempt. + +The 135-test suite and Ruff checks passed. The existing public Linux MCP sample was +also rescanned after the counter correction: both JSONL exports were byte-identical +to the earlier result, retaining all 41 records. This is a regression check, not an +independent Linux recovery benchmark; the sample's inferred physical-layout caveat +remains as described in [the Linux validation report](real-memory-validation.md). + +## Limits + +These statistics describe three captures of one Python SDK/stdin-stdout fixture. +Ground-truth logging changes memory, and explicitly retained frames are a favorable +control. Memory acquisition takes time; it is not an atomic snapshot. No deliberate +post-call delay or extra memory-pressure workload was introduced. Results may vary +with residency, acquisition, workload, and runtime state; this experiment does not +separate those causes. + +This benchmark does not establish general recall/precision, attack detection, +Windows 11 or Linux performance, other client/SDK compatibility, HTTP/SSE support, +or survival over longer collection delays. The earlier +[8-call positive validation](windows-positive-memory-validation.md) used a different +workload size and should not be compared as a controlled before/after experiment. diff --git a/mcptrace/core.py b/mcptrace/core.py index abae57b..3973799 100644 --- a/mcptrace/core.py +++ b/mcptrace/core.py @@ -225,8 +225,8 @@ def _carve_buffer( anchor = match.start() low = bisect.bisect_left(starts, max(0, anchor - maximum + 1)) high = bisect.bisect_right(starts, anchor) - if high - low > 64: - stats.candidate_limit_hits += 1 + limited = high - low > 64 + if limited: low = high - 64 found = False for index in range(high - 1, low - 1, -1): @@ -254,6 +254,8 @@ def _carve_buffer( found = True break if not found: + if limited: + stats.candidate_limit_hits += 1 unmatched.append(anchor) yield from complete.values() if partial: diff --git a/tests/test_benchmark_reporting.py b/tests/test_benchmark_reporting.py new file mode 100644 index 0000000..75045a4 --- /dev/null +++ b/tests/test_benchmark_reporting.py @@ -0,0 +1,172 @@ +"""Benchmark publication must keep denominators, failures, and content boundaries.""" + +import copy +import importlib.util +import json +from pathlib import Path + +import pytest + +SPEC = importlib.util.spec_from_file_location( + "benchmark", Path(__file__).resolve().parents[1] / "validation/windows_positive/benchmark.py" +) +benchmark = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(benchmark) + + +def report_files(tmp_path): + manifest = { + "status": "passed", + "commit": "test-commit", + "platform": "Windows-2022Server", + "python": "3.12", + "mcp_sdk": "1.26.0", + "volatility": "2.28.2", + "logical_cpus": 2, + "tool_calls_per_case": 64, + "scan_repeats": 3, + "raw_memory_deleted": True, + "image_hash_unchanged_after_analysis": True, + "repeated_exports_identical": True, + "image_sha256": "1" * 64, + "physical_ram_bytes": 8 * 1024**3, + "image_bytes": 11 * 1024**3, + "capture_started_ns": 1_000_000_000, + "cases": [ + {"mode": mode, "ready_ns": 900_000_000, "client_pid": 12345} + for mode in ("natural", "retained") + ], + "timings_seconds": { + "acquisition": 10.0, + "symbols_and_info": 20.0, + "native_byte_verification": 2.0, + "pipeline_total": 50.0, + "plugin_cli_seconds": [4.0, 3.0, 2.0], + }, + "unrelated_secret": "DO_NOT_PUBLISH", + } + cases = [] + for mode in ("natural", "retained"): + case = dict.fromkeys(benchmark.CASE_FIELDS, 0) + case.update( + mode=mode, + expected_logical_messages=133, + expected_tool_pairs=64, + distinct_expected_messages_found=10, + complete_tool_pairs_any_endpoint=2, + artifact_records=10, + records_matching_logged_traffic=10, + per_pid={"12345": {"complete_tool_pairs": 2}}, + recovered_message="DO_NOT_PUBLISH", + ) + cases.append(case) + scans = { + str(pid): { + "error": None, + "limits_hit": [], + "skipped_reason": None, + "anchor_limit_hits": 0, + "candidate_limit_hits": 0, + "invalid_regions": 0, + "displayed_records": 5, + "candidate_records": 5, + "bytes_read": 1024, + "unreadable_bytes": 2048, + "regions_scanned": 2, + } + for pid in range(4) + } + for name, data in ( + ("manifest.json", manifest), + ("scan-summaries.json", scans), + ( + "comparison.json", + {"cases": cases, "artifact_records_byte_verified": 20, "artifact_bytes_verified": 500}, + ), + ): + (tmp_path / name).write_text(json.dumps(data)) + return manifest + + +def test_summary_excludes_raw_content_and_identifiers(tmp_path): + report_files(tmp_path) + summary = benchmark.summarize_run(tmp_path, 1) + encoded = json.dumps(summary) + assert "DO_NOT_PUBLISH" not in encoded and "12345" not in encoded + assert summary["scan_totals"]["bytes_read"] == 4096 + assert summary["cases"]["natural"]["expected_logical_messages"] == 133 + assert summary["checks"]["raw_memory_deleted"] + + +def test_aggregation_uses_capture_denominators_not_scan_repeats(tmp_path): + report_files(tmp_path) + runs = [] + for replica, found in enumerate([10, 12, 14], 1): + run = benchmark.summarize_run(tmp_path, replica) + run["capture_sha256"] = str(replica) * 64 + run["cases"]["natural"]["distinct_expected_messages_found"] = found + runs.append(run) + combined = benchmark.aggregate_runs(runs) + natural = combined["modes"]["natural"] + assert natural["totals"]["expected_logical_messages"] == 399 + assert natural["totals"]["distinct_expected_messages_found"] == 36 + assert natural["logical_message_coverage"] == 36 / 399 + assert natural["messages_found_per_capture"]["mean"] == 12 + assert combined["plugin_cli_seconds"]["n"] == 9 + with pytest.raises(ValueError, match="exactly replicas"): + benchmark.aggregate_runs(runs[:2]) + bad = copy.deepcopy(runs) + bad[2]["status"] = "failed" + with pytest.raises(ValueError, match="failed capture"): + benchmark.aggregate_runs(bad) + bad = copy.deepcopy(runs) + bad[2]["capture_sha256"] = bad[0]["capture_sha256"] + with pytest.raises(ValueError, match="independently"): + benchmark.aggregate_runs(bad) + bad = copy.deepcopy(runs) + bad[2]["tool_calls_per_case"] = 8 + with pytest.raises(ValueError, match="Incompatible"): + benchmark.aggregate_runs(bad) + + +def test_failed_run_remains_a_failure_without_partial_metrics(tmp_path): + manifest = report_files(tmp_path) + manifest["status"] = "failed" + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + summary = benchmark.summarize_run(tmp_path, 1) + assert summary["status"] == "failed" + assert "cases" not in summary + assert summary["scan_diagnostics"]["processes"] == 4 + + +def test_scan_limits_are_never_hidden_behind_a_passed_status(tmp_path): + manifest = report_files(tmp_path) + path = tmp_path / "scan-summaries.json" + scans = json.loads(path.read_text()) + scans["0"]["candidate_limit_hits"] = 7 + path.write_text(json.dumps(scans)) + with pytest.raises(ValueError, match="explicitly marked"): + benchmark.summarize_run(tmp_path, 1) + manifest.update(status="completed_with_limits", report_scan_limits=True) + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + summary = benchmark.summarize_run(tmp_path, 1) + assert summary["status"] == "completed_with_limits" + assert not summary["checks"]["full_endpoint_scans"] + assert summary["scan_diagnostics"]["candidate_limit_hits"] == 7 + + +@pytest.mark.parametrize("problem", ["cleanup", "record_count", "timings"]) +def test_incomplete_validation_cannot_be_published_as_success(tmp_path, problem): + manifest = report_files(tmp_path) + if problem == "cleanup": + manifest["raw_memory_deleted"] = False + elif problem == "timings": + manifest["timings_seconds"]["plugin_cli_seconds"] = [1.0] + else: + path = tmp_path / "comparison.json" + data = json.loads(path.read_text()) + data["artifact_records_byte_verified"] = 999 + path.write_text(json.dumps(data)) + (tmp_path / "manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError): + benchmark.summarize_run(tmp_path, 1) diff --git a/tests/test_core.py b/tests/test_core.py index 0359585..db8d049 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -169,6 +169,16 @@ def test_backtrack_limit_is_observable(): assert stats.candidate_limit_hits > 0 +@pytest.mark.parametrize("encoding", ["utf-8", "utf-16-le"]) +def test_dense_successful_messages_do_not_report_backtrack_exhaustion(encoding): + objects = [request(rid=i) for i in range(133)] + raw = "\n".encode(encoding).join(encode(obj, encoding) for obj in objects) + stats = ScanStats() + records = list(scan_chunks([(0, raw)], stats=stats)) + assert [r.obj for r in records] == objects + assert stats.candidate_limit_hits == 0 + + def test_candidate_addresses_are_bounded_without_dropping_artifacts(): req = carve(request()) responses = [carve({"jsonrpc": "2.0", "id": 1, "result": {}}) for _ in range(300)] diff --git a/validation/windows_positive/BENCHMARK.md b/validation/windows_positive/BENCHMARK.md new file mode 100644 index 0000000..53f7e07 --- /dev/null +++ b/validation/windows_positive/BENCHMARK.md @@ -0,0 +1,62 @@ +# Repeated Windows RAM benchmark + +The benchmark uses three independently acquired Windows Server 2022 VM images. +Each VM runs two official Python MCP SDK stdio client/server pairs: `natural` +(ground-truth logging without a separate retained-frame list) and `retained` +(the same logging plus a retained-frame list and one deliberately unsent control). + +Each pair completes 64 echo tool calls, cycling through the existing eight payload +patterns with a unique sequence and request ID. Initialization, notification, and +tool discovery bring the ground truth to 133 distinct logical messages per mode +per capture. The three-capture denominator is 399 messages and 192 tool-call pairs +per mode. These sample counts are fixed before execution. + +Each capture is analyzed three times by separate MCPTrace CLI processes. All four +selected endpoint PIDs are scanned together. Exported JSONL must be byte-identical +across the three scans. One set of exports is independently re-read through native +Volatility Windows process layers to check bytes, SHA-256, addresses, and VAD bounds. +The original RAM image is hashed before and after analysis, then deleted. + +CLI timing includes process startup, Volatility automagic, scanning, JSON rendering, +and JSONL export. It excludes acquisition, initial `windows.info`/symbol preparation, +export comparison, and native byte verification. The image has already been hashed +and inspected before the first scan; no scan is claimed to be a cold-cache result. +Repeated scans measure timing/repeatability, not additional independent recovery +samples. Runtime covers four selected processes, not all processes in the RAM image. + +## Reproduce + +Use a fresh administrative Windows VM for each replica. Install the repository and +`validation/windows_positive/requirements.txt`, then run from the repository root: + +```powershell +python validation/windows_positive/run.py --workdir C:\MCPBenchmark\work --reportdir C:\MCPBenchmark\report --tool-calls 64 --scan-repeats 3 --report-scan-limits +python validation/windows_positive/benchmark.py summarize --reportdir C:\MCPBenchmark\report --replica 1 --output benchmark-1.json +``` + +Repeat in two other fresh VMs using replicas 2 and 3, then aggregate all three files: + +```powershell +python validation/windows_positive/benchmark.py aggregate benchmark-1.json benchmark-2.json benchmark-3.json --output benchmark.json +``` + +The GitHub Actions `Windows memory benchmark` workflow performs the three captures. +It uploads only allowlisted counts, durations, environment/version information, image +hashes, and validation flags. RAM, recovered messages, ground truth, process IDs, +and arbitrary log text are excluded. Failed or missing captures cannot be silently +dropped from the three-capture aggregate. Read errors and missing endpoints remain +failures. `--report-scan-limits` allows bounded-search results to be measured, but +marks them `completed_with_limits`, preserves every limit counter, and sets +`full_endpoint_scans=false`; the aggregate reports how many captures were limited. +The normal positive-validation command still rejects limited scans by default. + +## Interpretation + +Recovery counts deduplicate messages within each capture and combine both endpoints. +Candidate links remain candidates; unsent controls are counted separately from logged +traffic. Mean/min/max describe only these three captures. Logging changes memory, +and retained frames are a favorable control. No general recall, detection precision, +latency/pressure survival curve, or Windows 11/Linux benchmark is established by +this Windows Server 2022 SDK/stdio experiment. + +Measured results: [Windows RAM benchmark](../../docs/windows-memory-benchmark.md). diff --git a/validation/windows_positive/README.md b/validation/windows_positive/README.md index 0ed388b..5a7e1f7 100644 --- a/validation/windows_positive/README.md +++ b/validation/windows_positive/README.md @@ -57,3 +57,11 @@ GitHub Actions의 `Windows positive memory validation` 워크플로도 같은 전체 실험은 실행 가능성을 확인한 것으로 기록되므로 **반드시 natural/retained 수치를 나눠 읽어야 합니다.** 실제로 수행한 Windows Server 2022 캡처와 조건별 수치는 [양성 검증 보고서](../../docs/windows-positive-memory-validation.md)에 기록했습니다. + +`--tool-calls`로 조건별 도구 호출 수를, `--scan-repeats`로 동일 이미지의 CLI 분석 반복 수를 +지정할 수 있습니다. 기본값은 기존 검증과 같은 8회/1회입니다. 별도 VM 3개에서 64회 호출과 +3회 분석을 수행하는 [반복 벤치마크 절차](BENCHMARK.md)도 제공합니다. + +벤치마크의 `--report-scan-limits`는 실제 탐색 제한에 도달한 결과를 +`completed_with_limits`로 구분해 집계합니다. 기본 양성 검증은 제한 도달을 계속 실패로 처리하며, +읽기 오류나 누락된 대상 프로세스는 벤치마크에서도 실패입니다. diff --git a/validation/windows_positive/benchmark.py b/validation/windows_positive/benchmark.py new file mode 100644 index 0000000..7fa9112 --- /dev/null +++ b/validation/windows_positive/benchmark.py @@ -0,0 +1,246 @@ +"""Publish allowlisted measurements and aggregate three independent Windows captures.""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +from pathlib import Path + +CASE_FIELDS = ( + "expected_logical_messages", + "distinct_expected_messages_found", + "expected_tool_pairs", + "complete_tool_pairs_any_endpoint", + "artifact_records", + "records_matching_logged_traffic", + "distinct_unsent_controls_found", + "records_matching_unsent_controls", + "unmatched_artifact_records", + "evaluated_candidate_links", + "incorrect_candidate_links", + "unverifiable_candidate_links", +) +ENVIRONMENT_FIELDS = ("platform", "python", "mcp_sdk", "volatility", "logical_cpus") +COMPLETED = {"passed", "completed_with_limits"} + + +def read_json(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def scan_diagnostics(scans): + return { + "processes": len(scans), + "limited_processes": sum( + bool(s["limits_hit"] or s["anchor_limit_hits"] or s["candidate_limit_hits"]) + for s in scans.values() + ), + "error_processes": sum(bool(s["error"]) for s in scans.values()), + "skipped_processes": sum(bool(s["skipped_reason"]) for s in scans.values()), + **{ + key: sum(s[key] for s in scans.values()) + for key in ("invalid_regions", "anchor_limit_hits", "candidate_limit_hits") + }, + "record_budget_processes": sum("max-records" in s["limits_hit"] for s in scans.values()), + "byte_budget_processes": sum("max-bytes" in s["limits_hit"] for s in scans.values()), + } + + +def summarize_run(report, replica): + """Exclude recovered content, ground truth, PIDs, paths, and arbitrary log text.""" + path = report / "manifest.json" + manifest = read_json(path) if path.exists() else {} + output = { + "schema_version": 1, + "replica": replica, + "status": manifest["status"] if manifest.get("status") in COMPLETED else "failed", + "commit": manifest.get("commit"), + "environment": {key: manifest.get(key) for key in ENVIRONMENT_FIELDS}, + "tool_calls_per_case": manifest.get("tool_calls_per_case"), + "scan_repeats": manifest.get("scan_repeats"), + "report_scan_limits": manifest.get("report_scan_limits", False), + } + scan_path = report / "scan-summaries.json" + scans = read_json(scan_path) if scan_path.exists() else {} + if scans: + output["scan_diagnostics"] = scan_diagnostics(scans) + if output["status"] not in COMPLETED: + return output + for key in ( + "raw_memory_deleted", + "image_hash_unchanged_after_analysis", + "repeated_exports_identical", + ): + if manifest.get(key) is not True: + raise ValueError(f"Completed capture is missing validation: {key}") + comparison = read_json(report / "comparison.json") + if len(scans) != 4 or any( + s["error"] or s["skipped_reason"] or s["invalid_regions"] for s in scans.values() + ): + raise ValueError("Benchmark requires four complete endpoint scans") + limited = output["scan_diagnostics"]["limited_processes"] > 0 + if limited and ( + output["status"] != "completed_with_limits" or not output["report_scan_limits"] + ): + raise ValueError("Limited scans must be explicitly marked completed_with_limits") + if not limited and output["status"] != "passed": + raise ValueError("Completed status does not agree with scan diagnostics") + cases = {} + for case in comparison["cases"]: + counts = {key: case[key] for key in CASE_FIELDS} + if any(type(value) is not int or value < 0 for value in counts.values()): + raise ValueError("Case measurements must be nonnegative integer counts") + if ( + counts["expected_tool_pairs"] != output["tool_calls_per_case"] + or counts["expected_logical_messages"] != 5 + 2 * output["tool_calls_per_case"] + or counts["distinct_expected_messages_found"] > counts["expected_logical_messages"] + or counts["complete_tool_pairs_any_endpoint"] > counts["expected_tool_pairs"] + or counts["incorrect_candidate_links"] + ): + raise ValueError("Inconsistent ground truth or incorrect candidate links") + counts["best_single_pid_tool_pairs"] = max( + p["complete_tool_pairs"] for p in case["per_pid"].values() + ) + if case["mode"] in cases: + raise ValueError("Duplicate benchmark mode") + cases[case["mode"]] = counts + if set(cases) != {"natural", "retained"}: + raise ValueError("Both benchmark modes must be present") + verified = comparison["artifact_records_byte_verified"] + if verified != sum(c["artifact_records"] for c in cases.values()) or verified != sum( + s["displayed_records"] for s in scans.values() + ): + raise ValueError("Verified and exported record counts differ") + timings = manifest["timings_seconds"] + durations = timings["plugin_cli_seconds"] + if len(durations) != output["scan_repeats"] or any( + not math.isfinite(s) or s <= 0 for s in durations + ): + raise ValueError("Missing or invalid CLI timing samples") + output.update( + cases=cases, + capture_sha256=manifest["image_sha256"], + physical_ram_bytes=manifest["physical_ram_bytes"], + image_logical_bytes=manifest["image_bytes"], + artifact_records_byte_verified=verified, + artifact_bytes_verified=comparison["artifact_bytes_verified"], + scan_totals={ + key: sum(s[key] for s in scans.values()) + for key in ("bytes_read", "unreadable_bytes", "regions_scanned", "candidate_records") + }, + timings_seconds={ + **{ + key: timings[key] + for key in ( + "acquisition", + "symbols_and_info", + "native_byte_verification", + "pipeline_total", + ) + }, + "plugin_cli": durations, + }, + ready_to_capture_seconds={ + c["mode"]: (manifest["capture_started_ns"] - c["ready_ns"]) / 1e9 + for c in manifest["cases"] + }, + checks={ + "full_endpoint_scans": not limited, + "native_bytes_verified": True, + "repeated_exports_identical": True, + "image_hash_unchanged": True, + "raw_memory_deleted": True, + }, + ) + return output + + +def distribution(values): + return { + "n": len(values), + "mean": statistics.mean(values), + "median": statistics.median(values), + "min": min(values), + "max": max(values), + } + + +def aggregate_runs(runs): + """Require the full predeclared sample; never silently discard a failed capture.""" + if len(runs) != 3 or {r["replica"] for r in runs} != {1, 2, 3}: + raise ValueError("Require exactly replicas 1, 2, and 3") + if any(r["status"] not in COMPLETED for r in runs): + raise ValueError("Cannot aggregate a failed capture as a successful benchmark") + if len({r["capture_sha256"] for r in runs}) != 3: + raise ValueError("Require three independently acquired images") + for key in ( + "schema_version", + "commit", + "environment", + "tool_calls_per_case", + "scan_repeats", + "report_scan_limits", + ): + if any(r[key] != runs[0][key] for r in runs): + raise ValueError(f"Incompatible benchmark conditions: {key}") + modes = {} + for mode in ("natural", "retained"): + cases = [r["cases"][mode] for r in runs] + totals = {key: sum(c[key] for c in cases) for key in CASE_FIELDS} + modes[mode] = { + "totals": totals, + "logical_message_coverage": ( + totals["distinct_expected_messages_found"] / totals["expected_logical_messages"] + ), + "messages_found_per_capture": distribution( + [c["distinct_expected_messages_found"] for c in cases] + ), + "tool_pairs_per_capture": distribution( + [c["complete_tool_pairs_any_endpoint"] for c in cases] + ), + } + return { + "schema_version": 1, + "independent_captures": 3, + "captures_with_scan_limits": sum(r["status"] == "completed_with_limits" for r in runs), + "commit": runs[0]["commit"], + "environment": runs[0]["environment"], + "tool_calls_per_case": runs[0]["tool_calls_per_case"], + "scan_repeats_per_capture": runs[0]["scan_repeats"], + "modes": modes, + "plugin_cli_seconds": distribution( + [s for r in runs for s in r["timings_seconds"]["plugin_cli"]] + ), + "per_capture_cli_median_seconds": distribution( + [statistics.median(r["timings_seconds"]["plugin_cli"]) for r in runs] + ), + "artifact_records_byte_verified": sum(r["artifact_records_byte_verified"] for r in runs), + "artifact_bytes_verified": sum(r["artifact_bytes_verified"] for r in runs), + "runs": sorted(runs, key=lambda r: r["replica"]), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="action", required=True) + summarize = subparsers.add_parser("summarize") + summarize.add_argument("--reportdir", type=Path, required=True) + summarize.add_argument("--replica", type=int, choices=[1, 2, 3], required=True) + summarize.add_argument("--output", type=Path, required=True) + aggregate = subparsers.add_parser("aggregate") + aggregate.add_argument("inputs", nargs=3, type=Path) + aggregate.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + result = ( + summarize_run(args.reportdir, args.replica) + if args.action == "summarize" + else aggregate_runs([read_json(path) for path in args.inputs]) + ) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n", encoding="utf-8") + print(f"Wrote {args.output.name}") + + +if __name__ == "__main__": + main() diff --git a/validation/windows_positive/run.py b/validation/windows_positive/run.py index c5d8366..cfb9747 100644 --- a/validation/windows_positive/run.py +++ b/validation/windows_positive/run.py @@ -8,6 +8,7 @@ import ctypes import hashlib import importlib.metadata import json +import os import platform import shutil import subprocess @@ -40,12 +41,14 @@ def write_json(path, value): def command(args, stdout, stderr, timeout=300): + started = time.perf_counter() with stdout.open("wb") as out, stderr.open("wb") as err: completed = subprocess.run(args, stdout=out, stderr=err, timeout=timeout, check=False) if completed.returncode: raise RuntimeError( f"Command failed ({completed.returncode}); see {stdout.name}/{stderr.name}" ) + return time.perf_counter() - started def physical_ram(): @@ -78,14 +81,21 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--workdir", required=True, type=Path) parser.add_argument("--reportdir", required=True, type=Path) + parser.add_argument("--tool-calls", type=int, default=8) + parser.add_argument("--scan-repeats", type=int, default=1) + parser.add_argument("--report-scan-limits", action="store_true") args = parser.parse_args() + if not 1 <= args.tool_calls <= 512 or not 1 <= args.scan_repeats <= 5: + parser.error("Require 1..512 tool calls and 1..5 scan repeats") if sys.platform != "win32": parser.error("Run this acquisition harness on Windows as administrator") work, report = args.workdir.resolve(), args.reportdir.resolve() work.mkdir(parents=True, exist_ok=False) report.mkdir(parents=True, exist_ok=False) repo = Path(__file__).resolve().parents[2] + pipeline_started = time.perf_counter() image = work / "memory.raw" + timings = {"plugin_cli_seconds": []} manifest = { "status": "running", "stage": "preflight", @@ -103,6 +113,11 @@ def main(): "physical_ram_bytes": physical_ram(), "free_disk_bytes": shutil.disk_usage(work).free, "raw_memory_uploaded": False, + "logical_cpus": os.cpu_count(), + "tool_calls_per_case": args.tool_calls, + "scan_repeats": args.scan_repeats, + "report_scan_limits": args.report_scan_limits, + "timings_seconds": timings, } children, cases = [], [] @@ -144,6 +159,8 @@ def main(): str(output), "--mode", mode, + "--tool-calls", + str(args.tool_calls), ], stdout=subprocess.DEVNULL, stderr=error_log, @@ -160,7 +177,10 @@ def main(): raise TimeoutError(f"{output.name}: MCP workload did not become ready") time.sleep(0.1) ready = json.loads((output / "ready.json").read_text(encoding="utf-8")) - if ready["tool_calls_completed"] != 8 or ready["logged_messages"] != 21: + if ( + ready["tool_calls_completed"] != args.tool_calls + or ready["logged_messages"] != 5 + 2 * args.tool_calls + ): raise RuntimeError("Incomplete MCP ground truth") cases.append(ready) pids = [c[key] for c in cases for key in ["client_pid", "server_pid"]] @@ -169,7 +189,7 @@ def main(): manifest["cases"] = cases manifest["capture_started_ns"] = time.time_ns() checkpoint("physical-ram-acquisition") - command( + timings["acquisition"] = command( [str(tool), "acquire", str(image)], report / "capture.txt", report / "capture.stderr", @@ -203,7 +223,7 @@ def main(): str(image), ] # Windows symbols can be fetched during info; subsequent steps are offline. - command( + timings["symbols_and_info"] = command( common + ["-r", "json", "windows.info.Info"], report / "info.json", report / "info.stderr", @@ -225,22 +245,51 @@ def main(): "--dump", ] manifest["plugin_command"] = trace_cmd - command(trace_cmd, work / "all-findings.json", report / "mcptrace.stderr") - checkpoint("byte-verification-and-ground-truth-comparison") + timings["plugin_cli_seconds"].append( + command(trace_cmd, work / "all-findings.json", report / "mcptrace.stderr") + ) exports = [export / f"mcptrace.pid-{pid}.jsonl" for pid in pids] + expected_exports = {path.name: sha256(path) for path in exports} + for repeat in range(1, args.scan_repeats): + repeated_output = work / f"exports-{repeat + 1}" + repeated_output.mkdir() + repeated_cmd = trace_cmd.copy() + repeated_cmd[repeated_cmd.index("-o") + 1] = str(repeated_output) + repeated_cmd[repeated_cmd.index("--save-config") + 1] = str( + work / f"mcptrace-config-{repeat + 1}.json" + ) + timings["plugin_cli_seconds"].append( + command( + repeated_cmd, + work / f"all-findings-{repeat + 1}.json", + report / f"mcptrace-{repeat + 1}.stderr", + ) + ) + if any( + sha256(repeated_output / name) != digest + for name, digest in expected_exports.items() + ): + raise RuntimeError("Repeated scans of the same image produced different JSONL") + manifest["repeated_exports_identical"] = True + checkpoint("byte-verification-and-ground-truth-comparison") + verification_started = time.perf_counter() artifacts, summaries = verify_exports(image, symbols, cache, config, exports) + timings["native_byte_verification"] = time.perf_counter() - verification_started + # Persist diagnostics before deciding whether a scan can be scored. + write_json(report / "scan-summaries.json", summaries) + limited = False for summary in summaries.values(): + if summary["error"] or summary["skipped_reason"] or summary["invalid_regions"]: + raise RuntimeError("An endpoint scan failed; inspect scan summaries") if ( - summary["error"] - or summary["limits_hit"] - or summary["skipped_reason"] + summary["limits_hit"] or summary["anchor_limit_hits"] or summary["candidate_limit_hits"] ): - raise RuntimeError( - "A selected endpoint scan was incomplete; inspect scan summaries" - ) - write_json(report / "scan-summaries.json", summaries) + limited = True + manifest["scan_complete"] = not limited + if limited and not args.report_scan_limits: + raise RuntimeError("A selected endpoint scan was incomplete; inspect scan summaries") scores, publish_keys = [], set() for case in cases: mode = case["mode"] @@ -282,7 +331,7 @@ def main(): raise RuntimeError("No complete known tool request/response pair recovered") if any(s["incorrect_candidate_links"] for s in scores): raise RuntimeError("Incorrect candidate links observed in controlled traffic") - manifest["analysis_status"] = "passed" + manifest["analysis_status"] = "completed_with_limits" if limited else "passed" checkpoint("analysis-completed") print(json.dumps(result, ensure_ascii=True, indent=2), flush=True) except BaseException as exc: @@ -310,7 +359,8 @@ def main(): else: manifest["raw_memory_deleted"] = True write_json(report / "manifest.json", manifest) - manifest["status"] = "passed" + manifest["status"] = manifest["analysis_status"] + timings["pipeline_total"] = time.perf_counter() - pipeline_started checkpoint("completed") diff --git a/validation/windows_positive/workload.py b/validation/windows_positive/workload.py index ee4a977..e3c96f7 100644 --- a/validation/windows_positive/workload.py +++ b/validation/windows_positive/workload.py @@ -116,7 +116,8 @@ async def client(args): "last message", ] server_pid = None - for sequence, text in enumerate(samples): + for sequence in range(args.tool_calls): + text = samples[sequence % len(samples)] result = await session.call_tool( "echo_marker", {"run_id": args.run_id, "sequence": sequence, "text": text}, @@ -159,7 +160,7 @@ async def client(args): "run_id": args.run_id, "client_pid": os.getpid(), "server_pid": server_pid, - "tool_calls_completed": len(samples), + "tool_calls_completed": args.tool_calls, "logged_messages": journal.count, "retained_transport_frames": len(journal.frames), "retained_unsent_controls": len(retained_decoys), @@ -186,7 +187,10 @@ def main(): parser.add_argument("--output", type=Path) parser.add_argument("--mode", choices=["natural", "retained"], default="natural") parser.add_argument("--hold-seconds", type=int, default=900) + parser.add_argument("--tool-calls", type=int, default=8) args = parser.parse_args() + if not 1 <= args.tool_calls <= 512: + parser.error("--tool-calls must be between 1 and 512") if args.role == "server": server(args.run_id) elif args.output is None: