ai-space/MCPTrace
3
0
Fork 0
mirror of https://github.com/Kowntaewook/MCPTrace.git synced 2026-11-02 04:45:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6 from Kowntaewook/test/windows-memory-benchmark

Browse Source 
Benchmark Windows RAM recovery and fix premature search-limit warnings
This commit is contained in:
kwonta3wook
2026-10-27 17:45:35 +09:00
committed by GitHub
parent 33d74db1a0 caffed4dcc
commit d23379664e
12 changed files with 1171 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/windows-memory-benchmark.yml
+50
View File
@@ -0,0 +1,50 @@
name: Windows memory benchmark
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: windows-memory-benchmark-${{ github.ref }}
cancel-in-progress: false
jobs:
capture:
name: Capture ${{ matrix.replica }}
runs-on: windows-2022
timeout-minutes: 25
strategy:
fail-fast: false
max-parallel: 3
matrix:
replica: [1, 2, 3]
env:
PYTHONUTF8: '1'
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: '3.12'
- name: Install plugin and pinned SDK
run: python -m pip install -e . -r validation/windows_positive/requirements.txt
- name: Acquire real RAM and benchmark three scans
shell: pwsh
run: |
python validation/windows_positive/run.py --workdir "$env:RUNNER_TEMP/mcp-benchmark-work" --reportdir "$env:RUNNER_TEMP/mcp-benchmark-report" --tool-calls 64 --scan-repeats 3 --report-scan-limits
- name: Extract aggregate measurements only
if: always()
shell: pwsh
run: |
python validation/windows_positive/benchmark.py summarize --reportdir "$env:RUNNER_TEMP/mcp-benchmark-report" --replica ${{ matrix.replica }} --output "$env:RUNNER_TEMP/benchmark-${{ matrix.replica }}.json"
- name: Upload aggregate measurements only
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: windows-benchmark-${{ github.run_id }}-${{ matrix.replica }}
path: ${{ runner.temp }}/benchmark-${{ matrix.replica }}.json
if-no-files-found: warn
retention-days: 30
README.md
+17
View File
@@ -37,3 +37,20 @@ vol -p plugins -s symbols -f linux.lime -o output-linux linux.mcptrace.MCPTrace
- Omit `--pid` to scan all processes, or pass multiple PIDs: `--pid 4242 5252`.
- `--dump` exports per-process JSONL files to the directory specified by `-o`.
- Use `vol -p plugins windows.mcptrace.MCPTrace -h` (or the Linux equivalent) for all options.
## Benchmark
Windows Server 2022, Python MCP SDK over stdio: **3 independent RAM captures**,
64 tool calls / 133 messages per condition per capture. Results combine both endpoints.
| Condition | Messages / 133: mean (range) | Tool pairs / 64: mean (range) |
| --- | ---: | ---: |
| No retained-frame list | 59.7 (55-66) | 18.3 (15-21) |
| Retained-frame control | 133.0 (133-133) | 64.0 (64-64) |
CLI median: **53.95 s** (53.32-54.48 s, 9 scans of 4 endpoint processes).
Acquisition and initial symbol setup are excluded; scans are not cold-cache measurements.
**2/3 captures hit candidate search limits** (4 and 3 events); these limited results are included.
Ground-truth logging affects memory, and retained frames are a favorable control.
This is fixture-specific recovery, not general recall. [Methodology and full results](docs/windows-memory-benchmark.md).
docs/benchmarks/windows-memory-benchmark-2026-10-27.json
+373
View File
@@ -0,0 +1,373 @@
{
"schema_version": 1,
"independent_captures": 3,
"captures_with_scan_limits": 2,
"commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89",
"environment": {
"platform": "Windows-2022Server-10.0.20348-SP0",
"python": "3.12.10",
"mcp_sdk": "1.26.0",
"volatility": "2.28.2",
"logical_cpus": 4
},
"tool_calls_per_case": 64,
"scan_repeats_per_capture": 3,
"modes": {
"natural": {
"totals": {
"expected_logical_messages": 399,
"distinct_expected_messages_found": 179,
"expected_tool_pairs": 192,
"complete_tool_pairs_any_endpoint": 55,
"artifact_records": 485,
"records_matching_logged_traffic": 485,
"distinct_unsent_controls_found": 0,
"records_matching_unsent_controls": 0,
"unmatched_artifact_records": 0,
"evaluated_candidate_links": 800,
"incorrect_candidate_links": 0,
"unverifiable_candidate_links": 0
},
"logical_message_coverage": 0.44862155388471175,
"messages_found_per_capture": {
"n": 3,
"mean": 59.666666666666664,
"median": 58,
"min": 55,
"max": 66
},
"tool_pairs_per_capture": {
"n": 3,
"mean": 18.333333333333332,
"median": 19,
"min": 15,
"max": 21
}
},
"retained": {
"totals": {
"expected_logical_messages": 399,
"distinct_expected_messages_found": 399,
"expected_tool_pairs": 192,
"complete_tool_pairs_any_endpoint": 192,
"artifact_records": 854,
"records_matching_logged_traffic": 851,
"distinct_unsent_controls_found": 3,
"records_matching_unsent_controls": 3,
"unmatched_artifact_records": 0,
"evaluated_candidate_links": 1680,
"incorrect_candidate_links": 0,
"unverifiable_candidate_links": 0
},
"logical_message_coverage": 1.0,
"messages_found_per_capture": {
"n": 3,
"mean": 133,
"median": 133,
"min": 133,
"max": 133
},
"tool_pairs_per_capture": {
"n": 3,
"mean": 64,
"median": 64,
"min": 64,
"max": 64
}
}
},
"plugin_cli_seconds": {
"n": 9,
"mean": 53.94062956666671,
"median": 53.95398000000006,
"min": 53.317625600000156,
"max": 54.475639200000046
},
"per_capture_cli_median_seconds": {
"n": 3,
"mean": 53.88407783333334,
"median": 54.121342500000026,
"min": 53.36027970000009,
"max": 54.170611299999905
},
"artifact_records_byte_verified": 1339,
"artifact_bytes_verified": 562402,
"runs": [
{
"schema_version": 1,
"replica": 1,
"status": "passed",
"commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89",
"environment": {
"platform": "Windows-2022Server-10.0.20348-SP0",
"python": "3.12.10",
"mcp_sdk": "1.26.0",
"volatility": "2.28.2",
"logical_cpus": 4
},
"tool_calls_per_case": 64,
"scan_repeats": 3,
"report_scan_limits": true,
"scan_diagnostics": {
"processes": 4,
"limited_processes": 0,
"error_processes": 0,
"skipped_processes": 0,
"invalid_regions": 0,
"anchor_limit_hits": 0,
"candidate_limit_hits": 0,
"record_budget_processes": 0,
"byte_budget_processes": 0
},
"cases": {
"natural": {
"expected_logical_messages": 133,
"distinct_expected_messages_found": 66,
"expected_tool_pairs": 64,
"complete_tool_pairs_any_endpoint": 21,
"artifact_records": 173,
"records_matching_logged_traffic": 173,
"distinct_unsent_controls_found": 0,
"records_matching_unsent_controls": 0,
"unmatched_artifact_records": 0,
"evaluated_candidate_links": 328,
"incorrect_candidate_links": 0,
"unverifiable_candidate_links": 0,
"best_single_pid_tool_pairs": 14
},
"retained": {
"expected_logical_messages": 133,
"distinct_expected_messages_found": 133,
"expected_tool_pairs": 64,
"complete_tool_pairs_any_endpoint": 64,
"artifact_records": 299,
"records_matching_logged_traffic": 298,
"distinct_unsent_controls_found": 1,
"records_matching_unsent_controls": 1,
"unmatched_artifact_records": 0,
"evaluated_candidate_links": 620,
"incorrect_candidate_links": 0,
"unverifiable_candidate_links": 0,
"best_single_pid_tool_pairs": 64
}
},
"capture_sha256": "177c6afd7886aa05c05408c74f8c108caafa6cfbfb181c73f90afe81be931aa8",
"physical_ram_bytes": 17174360064,
"image_logical_bytes": 20401094656,
"artifact_records_byte_verified": 472,
"artifact_bytes_verified": 189113,
"scan_totals": {
"bytes_read": 193032192,
"unreadable_bytes": 17689243648,
"regions_scanned": 576,
"candidate_records": 472
},
"timings_seconds": {
"acquisition": 39.56291780000001,
"symbols_and_info": 117.4078207,
"native_byte_verification": 10.376795300000026,
"pipeline_total": 427.56338089999997,
"plugin_cli": [
54.121342500000026,
54.475639200000046,
53.95398000000006
]
},
"ready_to_capture_seconds": {
"natural": 0.0543968,
"retained": 0.0543968
},
"checks": {
"full_endpoint_scans": true,
"native_bytes_verified": true,
"repeated_exports_identical": true,
"image_hash_unchanged": true,
"raw_memory_deleted": true
}
},
{
"schema_version": 1,
"replica": 2,
"status": "completed_with_limits",
"commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89",
"environment": {
"platform": "Windows-2022Server-10.0.20348-SP0",
"python": "3.12.10",
"mcp_sdk": "1.26.0",
"volatility": "2.28.2",
"logical_cpus": 4
},
"tool_calls_per_case": 64,
"scan_repeats": 3,
"report_scan_limits": true,
"scan_diagnostics": {
"processes": 4,
"limited_processes": 1,
"error_processes": 0,
"skipped_processes": 0,
"invalid_regions": 0,
"anchor_limit_hits": 0,
"candidate_limit_hits": 4,
"record_budget_processes": 0,
"byte_budget_processes": 0
},
"cases": {
"natural": {
"expected_logical_messages": 133,
"distinct_expected_messages_found": 58,
"expected_tool_pairs": 64,
"complete_tool_pairs_any_endpoint": 19,
"artifact_records": 150,
"records_matching_logged_traffic": 150,
"distinct_unsent_controls_found": 0,
"records_matching_unsent_controls": 0,
"unmatched_artifact_records": 0,
"evaluated_candidate_links": 228,
"incorrect_candidate_links": 0,
"unverifiable_candidate_links": 0,
"best_single_pid_tool_pairs": 14
},
"retained": {
"expected_logical_messages": 133,
"distinct_expected_messages_found": 133,
"expected_tool_pairs": 64,
"complete_tool_pairs_any_endpoint": 64,
"artifact_records": 273,
"records_matching_logged_traffic": 272,
"distinct_unsent_controls_found": 1,
"records_matching_unsent_controls": 1,
"unmatched_artifact_records": 0,
"evaluated_candidate_links": 496,
"incorrect_candidate_links": 0,
"unverifiable_candidate_links": 0,
"best_single_pid_tool_pairs": 64
}
},
"capture_sha256": "a533b6ca43572eafa8dac01fd36fd9b585fe67bda122ff25bc1b8bb3147b67e6",
"physical_ram_bytes": 17174360064,
"image_logical_bytes": 20401094656,
"artifact_records_byte_verified": 423,
"artifact_bytes_verified": 194685,
"scan_totals": {
"bytes_read": 221855744,
"unreadable_bytes": 17660420096,
"regions_scanned": 576,
"candidate_records": 423
},
"timings_seconds": {
"acquisition": 38.52607280000001,
"symbols_and_info": 116.74145750000002,
"native_byte_verification": 11.414818200000013,
"pipeline_total": 428.09518140000006,
"plugin_cli": [
54.38048649999996,
53.882765600000084,
54.170611299999905
]
},
"ready_to_capture_seconds": {
"natural": 0.0827794,
"retained": 0.0827794
},
"checks": {
"full_endpoint_scans": false,
"native_bytes_verified": true,
"repeated_exports_identical": true,
"image_hash_unchanged": true,
"raw_memory_deleted": true
}
},
{
"schema_version": 1,
"replica": 3,
"status": "completed_with_limits",
"commit": "d59f51e118480cb3be08ef96b04f94d8328bcc89",
"environment": {
"platform": "Windows-2022Server-10.0.20348-SP0",
"python": "3.12.10",
"mcp_sdk": "1.26.0",
"volatility": "2.28.2",
"logical_cpus": 4
},
"tool_calls_per_case": 64,
"scan_repeats": 3,
"report_scan_limits": true,
"scan_diagnostics": {
"processes": 4,
"limited_processes": 1,
"error_processes": 0,
"skipped_processes": 0,
"invalid_regions": 0,
"anchor_limit_hits": 0,
"candidate_limit_hits": 3,
"record_budget_processes": 0,
"byte_budget_processes": 0
},
"cases": {
"natural": {
"expected_logical_messages": 133,
"distinct_expected_messages_found": 55,
"expected_tool_pairs": 64,
"complete_tool_pairs_any_endpoint": 15,
"artifact_records": 162,
"records_matching_logged_traffic": 162,
"distinct_unsent_controls_found": 0,
"records_matching_unsent_controls": 0,
"unmatched_artifact_records": 0,
"evaluated_candidate_links": 244,
"incorrect_candidate_links": 0,
"unverifiable_candidate_links": 0,
"best_single_pid_tool_pairs": 12
},
"retained": {
"expected_logical_messages": 133,
"distinct_expected_messages_found": 133,
"expected_tool_pairs": 64,
"complete_tool_pairs_any_endpoint": 64,
"artifact_records": 282,
"records_matching_logged_traffic": 281,
"distinct_unsent_controls_found": 1,
"records_matching_unsent_controls": 1,
"unmatched_artifact_records": 0,
"evaluated_candidate_links": 564,
"incorrect_candidate_links": 0,
"unverifiable_candidate_links": 0,
"best_single_pid_tool_pairs": 64
}
},
"capture_sha256": "ff0627c99a912c71b7c905e30f0cba894bffefa9cfdc5fdc4a40a083349616e3",
"physical_ram_bytes": 17174360064,
"image_logical_bytes": 20401094656,
"artifact_records_byte_verified": 444,
"artifact_bytes_verified": 178604,
"scan_totals": {
"bytes_read": 211656704,
"unreadable_bytes": 17670619136,
"regions_scanned": 576,
"candidate_records": 444
},
"timings_seconds": {
"acquisition": 39.40428389999988,
"symbols_and_info": 178.97452040000007,
"native_byte_verification": 10.353987799999686,
"pipeline_total": 487.8709704999999,
"plugin_cli": [
53.802935700000035,
53.36027970000009,
53.317625600000156
]
},
"ready_to_capture_seconds": {
"natural": 0.0931878,
"retained": 0.0931878
},
"checks": {
"full_endpoint_scans": false,
"native_bytes_verified": true,
"repeated_exports_identical": true,
"image_hash_unchanged": true,
"raw_memory_deleted": true
}
}
]
}
docs/windows-memory-benchmark.md
+159
View File
@@ -0,0 +1,159 @@
# Windows RAM benchmark
Measured on 2026-10-27 with MCPTrace 0.2.0 at commit
`d59f51e118480cb3be08ef96b04f94d8328bcc89`.
[Actions run](https://github.com/Kowntaewook/MCPTrace/actions/runs/40472181802).
Three fresh Windows Server 2022 jobs each ran two independent MCP SDK stdio sessions:
one with ground-truth logging but no retained-frame list (`natural`), and one with
the same logging plus an explicit retained-frame list (`retained`). Each session
completed 64 echo tool calls and logged 133 distinct logical messages, including
initialization, notification, and tool discovery. One deliberately unsent JSON
control was held in each retained client. No messages were injected into RAM images.
The eight payload patterns include ASCII, Korean/emoji, quotes/backslashes/newlines,
a 2 KiB string, repeated text with distinct sequence numbers, and whitespace. This
is a bounded SDK fixture, not a sample of real users' conversations or attacks.
## Recovery
| Capture | Condition | Messages / 133 | Tool pairs / 64 | Best same-PID pairs | Exported records |
| --- | --- | ---: | ---: | ---: | ---: |
| 1 | natural | 66 | 21 | 14 | 173 |
| 1 | retained | 133 | 64 | 64 | 299 |
| 2 | natural | 58 | 19 | 14 | 150 |
| 2 | retained | 133 | 64 | 64 | 273 |
| 3 | natural | 55 | 15 | 12 | 162 |
| 3 | retained | 133 | 64 | 64 | 282 |
| Condition | Pooled messages | Mean messages (range) | Pooled tool pairs | Mean tool pairs (range) |
| --- | ---: | ---: | ---: | ---: |
| natural | 179/399 (44.9%) | 59.7 (55-66) | 55/192 | 18.3 (15-21) |
| retained | 399/399 (100.0%) | 133.0 (133-133) | 192/192 | 64.0 (64-64) |
**Captures 2 and 3 are `completed_with_limits`: one process in each reported 4 and 3
candidate-backtracking limit events, respectively. Capture 1 had no limit events.
The means and pooled counts include all three captures; limited results were not discarded.**
These counters are search events, not a count of distinct missing messages. The aggregate
does not identify which mode contained the limited process, so the events are not attributed
to either mode. No process-read errors, skipped targets, invalid regions, anchor limits,
record budgets, or byte budgets were observed.
Message and pair counts combine both endpoints within one capture and deduplicate
logical messages. Matching normalizes parsed JSON objects; incoming SDK ground truth
is reserialized, not a byte-for-byte pipe capture. Exported raw bytes are separately
verified against the memory image. Same-image scan repeats are not additional recovery observations.
There are three independent observations per mode: 399 expected messages and 192
expected tool-call pairs in total. A pair counted in the endpoint union may have
its request and response in different processes. Per-capture best same-PID pair
counts remain available in the aggregate JSON.
All **1,339 exported records / 562,402 bytes** passed native
address/length/SHA-256/VAD-bound checks. Repeat scans produced byte-identical JSONL for
each image. The RAM hash remained unchanged and deletion completed for all captures.
The three retained runs recovered all three deliberately unsent controls. Across
2,480 evaluable directional candidate links, incorrect and unverifiable links were
both zero. All remaining exported records matched the logged fixture messages;
there were no unmatched records in this small controlled set.
Unsent controls are not traffic. Unmatched artifacts are not automatically labeled
false positives. Known directional candidate-link comparisons include duplicate
memory copies and are not a count of unique tool calls or proven sessions.
## Runtime and scan scope
| Capture | CLI scan 1 (s) | Scan 2 (s) | Scan 3 (s) | VADs | Read MiB |
| --- | ---: | ---: | ---: | ---: | ---: |
| 1 | 54.121 | 54.476 | 53.954 | 576 | 184.09 |
| 2 | 54.380 | 53.883 | 54.171 | 576 | 211.58 |
| 3 | 53.803 | 53.360 | 53.318 | 576 | 201.85 |
Across nine scans: mean **53.94 s**, median **53.95 s**, range
**53.32-54.48 s**. Per-capture scan medians are
54.12 s, 54.17 s, 53.36 s. These nine scans are nested within three images,
not nine independent acquisition experiments.
| Capture | Acquisition (s) | Symbols/info (s) | Native verification (s) | Full pipeline (s) |
| --- | ---: | ---: | ---: | ---: |
| 1 | 39.56 | 117.41 | 10.38 | 427.56 |
| 2 | 38.53 | 116.74 | 11.41 | 428.10 |
| 3 | 39.40 | 178.97 | 10.35 | 487.87 |
Each of the three images was scanned three times by separate CLI processes using
the same symbol cache, four endpoint PIDs, and default plugin limits. Any exhausted
search limit is retained in the counters and
marked `completed_with_limits`, never presented as a complete scan. Timing includes
startup, Volatility automagic, selected-process scanning, JSON rendering, and JSONL
export. Acquisition, initial symbol preparation, output comparison, image hashing,
and native byte verification are excluded from CLI timing. Acquisition, initial
symbol preparation, and native verification have their own reported durations.
The image was hashed and inspected before the first scan. These are not cold-cache
or full-image/all-process throughput measurements.
All three runners reported `Windows-2022Server-10.0.20348-SP0`, 4 logical CPUs,
17,174,360,064 bytes of physical RAM (approximately 16 GiB), Python
`3.12.10`, MCP SDK `1.26.0`, and Volatility `2.28.2`.
Each sparse image had a logical size of 20,401,094,656 bytes. The
image includes physical-address holes; this is not the size of scanned process data.
Workload-ready to capture-start delay was 0.054-0.093 s.
These are shared hosted runners, not a dedicated hardware performance guarantee.
## Reproduction and evidence
The [benchmark procedure](../validation/windows_positive/BENCHMARK.md) documents the
fixed sample size, workload, commands, and interpretation. The
[workflow](../.github/workflows/windows-memory-benchmark.yml) runs the three fresh
VM captures on demand. The machine-readable result is
[windows-memory-benchmark-2026-10-27.json](benchmarks/windows-memory-benchmark-2026-10-27.json).
Only allowlisted measurement counts, timings, version/environment information,
image hashes, and validation flags are published. Full RAM images and message
contents are not included. RAM was deleted after analysis, so reproduction means
running the capture harness again rather than reanalyzing these exact images.
The aggregate rejects missing/failed captures, duplicate image hashes, and mismatched
source versions or workload settings. No failed run is silently dropped.
## Initial attempt and counter correction
The [initial three-capture attempt](https://github.com/Kowntaewook/MCPTrace/actions/runs/40470688699)
at commit `b4524239bd2c1be12b4c83803fd76908e5a3f833` failed its incomplete-scan gate.
Per-process limit diagnostics were not preserved in that first aggregate, so the
exact gate inputs for those images cannot be reconstructed from the published files.
No recovery or timing values from that failed attempt are included in the final
three-capture measurement.
A local reproduction using all 133 SDK messages exposed premature limit accounting:
all 133 messages were found, yet 228 candidate-limit events were reported. The
counter incremented when the preceding window contained more than 64 object starts,
even if a valid message was found before search exhaustion. The counter now increments
only when that bounded search fails without examining the older candidates. Search
bounds and extraction logic remain unchanged. UTF-8/UTF-16LE regression cases verify
the correction, and the existing truly exhausted-backtrack test still passes.
The harness now preserves diagnostics before evaluating limits. Benchmark mode can
measure default-limit output with an explicit `completed_with_limits` status and
published limit counters; read errors or missing endpoints still fail. Strict
positive validation continues rejecting limited scans by default. All three fresh
VM captures were rerun at the corrected commit; they are not selectively chosen
successful members of the original attempt.
The 135-test suite and Ruff checks passed. The existing public Linux MCP sample was
also rescanned after the counter correction: both JSONL exports were byte-identical
to the earlier result, retaining all 41 records. This is a regression check, not an
independent Linux recovery benchmark; the sample's inferred physical-layout caveat
remains as described in [the Linux validation report](real-memory-validation.md).
## Limits
These statistics describe three captures of one Python SDK/stdin-stdout fixture.
Ground-truth logging changes memory, and explicitly retained frames are a favorable
control. Memory acquisition takes time; it is not an atomic snapshot. No deliberate
post-call delay or extra memory-pressure workload was introduced. Results may vary
with residency, acquisition, workload, and runtime state; this experiment does not
separate those causes.
This benchmark does not establish general recall/precision, attack detection,
Windows 11 or Linux performance, other client/SDK compatibility, HTTP/SSE support,
or survival over longer collection delays. The earlier
[8-call positive validation](windows-positive-memory-validation.md) used a different
workload size and should not be compared as a controlled before/after experiment.
mcptrace/core.py
+4 -2
View File
@@ -225,8 +225,8 @@ def _carve_buffer(
anchor = match.start()
low = bisect.bisect_left(starts, max(0, anchor - maximum + 1))
high = bisect.bisect_right(starts, anchor)
if high - low > 64:
stats.candidate_limit_hits += 1
limited = high - low > 64
if limited:
low = high - 64
found = False
for index in range(high - 1, low - 1, -1):
@@ -254,6 +254,8 @@ def _carve_buffer(
found = True
break
if not found:
if limited:
stats.candidate_limit_hits += 1
unmatched.append(anchor)
yield from complete.values()
if partial:
tests/test_benchmark_reporting.py
+172
View File
@@ -0,0 +1,172 @@
"""Benchmark publication must keep denominators, failures, and content boundaries."""
import copy
import importlib.util
import json
from pathlib import Path
import pytest
SPEC = importlib.util.spec_from_file_location(
"benchmark", Path(__file__).resolve().parents[1] / "validation/windows_positive/benchmark.py"
)
benchmark = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(benchmark)
def report_files(tmp_path):
manifest = {
"status": "passed",
"commit": "test-commit",
"platform": "Windows-2022Server",
"python": "3.12",
"mcp_sdk": "1.26.0",
"volatility": "2.28.2",
"logical_cpus": 2,
"tool_calls_per_case": 64,
"scan_repeats": 3,
"raw_memory_deleted": True,
"image_hash_unchanged_after_analysis": True,
"repeated_exports_identical": True,
"image_sha256": "1" * 64,
"physical_ram_bytes": 8 * 1024**3,
"image_bytes": 11 * 1024**3,
"capture_started_ns": 1_000_000_000,
"cases": [
{"mode": mode, "ready_ns": 900_000_000, "client_pid": 12345}
for mode in ("natural", "retained")
],
"timings_seconds": {
"acquisition": 10.0,
"symbols_and_info": 20.0,
"native_byte_verification": 2.0,
"pipeline_total": 50.0,
"plugin_cli_seconds": [4.0, 3.0, 2.0],
},
"unrelated_secret": "DO_NOT_PUBLISH",
}
cases = []
for mode in ("natural", "retained"):
case = dict.fromkeys(benchmark.CASE_FIELDS, 0)
case.update(
mode=mode,
expected_logical_messages=133,
expected_tool_pairs=64,
distinct_expected_messages_found=10,
complete_tool_pairs_any_endpoint=2,
artifact_records=10,
records_matching_logged_traffic=10,
per_pid={"12345": {"complete_tool_pairs": 2}},
recovered_message="DO_NOT_PUBLISH",
)
cases.append(case)
scans = {
str(pid): {
"error": None,
"limits_hit": [],
"skipped_reason": None,
"anchor_limit_hits": 0,
"candidate_limit_hits": 0,
"invalid_regions": 0,
"displayed_records": 5,
"candidate_records": 5,
"bytes_read": 1024,
"unreadable_bytes": 2048,
"regions_scanned": 2,
}
for pid in range(4)
}
for name, data in (
("manifest.json", manifest),
("scan-summaries.json", scans),
(
"comparison.json",
{"cases": cases, "artifact_records_byte_verified": 20, "artifact_bytes_verified": 500},
),
):
(tmp_path / name).write_text(json.dumps(data))
return manifest
def test_summary_excludes_raw_content_and_identifiers(tmp_path):
report_files(tmp_path)
summary = benchmark.summarize_run(tmp_path, 1)
encoded = json.dumps(summary)
assert "DO_NOT_PUBLISH" not in encoded and "12345" not in encoded
assert summary["scan_totals"]["bytes_read"] == 4096
assert summary["cases"]["natural"]["expected_logical_messages"] == 133
assert summary["checks"]["raw_memory_deleted"]
def test_aggregation_uses_capture_denominators_not_scan_repeats(tmp_path):
report_files(tmp_path)
runs = []
for replica, found in enumerate([10, 12, 14], 1):
run = benchmark.summarize_run(tmp_path, replica)
run["capture_sha256"] = str(replica) * 64
run["cases"]["natural"]["distinct_expected_messages_found"] = found
runs.append(run)
combined = benchmark.aggregate_runs(runs)
natural = combined["modes"]["natural"]
assert natural["totals"]["expected_logical_messages"] == 399
assert natural["totals"]["distinct_expected_messages_found"] == 36
assert natural["logical_message_coverage"] == 36 / 399
assert natural["messages_found_per_capture"]["mean"] == 12
assert combined["plugin_cli_seconds"]["n"] == 9
with pytest.raises(ValueError, match="exactly replicas"):
benchmark.aggregate_runs(runs[:2])
bad = copy.deepcopy(runs)
bad[2]["status"] = "failed"
with pytest.raises(ValueError, match="failed capture"):
benchmark.aggregate_runs(bad)
bad = copy.deepcopy(runs)
bad[2]["capture_sha256"] = bad[0]["capture_sha256"]
with pytest.raises(ValueError, match="independently"):
benchmark.aggregate_runs(bad)
bad = copy.deepcopy(runs)
bad[2]["tool_calls_per_case"] = 8
with pytest.raises(ValueError, match="Incompatible"):
benchmark.aggregate_runs(bad)
def test_failed_run_remains_a_failure_without_partial_metrics(tmp_path):
manifest = report_files(tmp_path)
manifest["status"] = "failed"
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
summary = benchmark.summarize_run(tmp_path, 1)
assert summary["status"] == "failed"
assert "cases" not in summary
assert summary["scan_diagnostics"]["processes"] == 4
def test_scan_limits_are_never_hidden_behind_a_passed_status(tmp_path):
manifest = report_files(tmp_path)
path = tmp_path / "scan-summaries.json"
scans = json.loads(path.read_text())
scans["0"]["candidate_limit_hits"] = 7
path.write_text(json.dumps(scans))
with pytest.raises(ValueError, match="explicitly marked"):
benchmark.summarize_run(tmp_path, 1)
manifest.update(status="completed_with_limits", report_scan_limits=True)
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
summary = benchmark.summarize_run(tmp_path, 1)
assert summary["status"] == "completed_with_limits"
assert not summary["checks"]["full_endpoint_scans"]
assert summary["scan_diagnostics"]["candidate_limit_hits"] == 7
@pytest.mark.parametrize("problem", ["cleanup", "record_count", "timings"])
def test_incomplete_validation_cannot_be_published_as_success(tmp_path, problem):
manifest = report_files(tmp_path)
if problem == "cleanup":
manifest["raw_memory_deleted"] = False
elif problem == "timings":
manifest["timings_seconds"]["plugin_cli_seconds"] = [1.0]
else:
path = tmp_path / "comparison.json"
data = json.loads(path.read_text())
data["artifact_records_byte_verified"] = 999
path.write_text(json.dumps(data))
(tmp_path / "manifest.json").write_text(json.dumps(manifest))
with pytest.raises(ValueError):
benchmark.summarize_run(tmp_path, 1)
tests/test_core.py
+10
View File
@@ -169,6 +169,16 @@ def test_backtrack_limit_is_observable():
assert stats.candidate_limit_hits > 0
@pytest.mark.parametrize("encoding", ["utf-8", "utf-16-le"])
def test_dense_successful_messages_do_not_report_backtrack_exhaustion(encoding):
objects = [request(rid=i) for i in range(133)]
raw = "\n".encode(encoding).join(encode(obj, encoding) for obj in objects)
stats = ScanStats()
records = list(scan_chunks([(0, raw)], stats=stats))
assert [r.obj for r in records] == objects
assert stats.candidate_limit_hits == 0
def test_candidate_addresses_are_bounded_without_dropping_artifacts():
req = carve(request())
responses = [carve({"jsonrpc": "2.0", "id": 1, "result": {}}) for _ in range(300)]
validation/windows_positive/BENCHMARK.md
+62
View File
@@ -0,0 +1,62 @@
# Repeated Windows RAM benchmark
The benchmark uses three independently acquired Windows Server 2022 VM images.
Each VM runs two official Python MCP SDK stdio client/server pairs: `natural`
(ground-truth logging without a separate retained-frame list) and `retained`
(the same logging plus a retained-frame list and one deliberately unsent control).
Each pair completes 64 echo tool calls, cycling through the existing eight payload
patterns with a unique sequence and request ID. Initialization, notification, and
tool discovery bring the ground truth to 133 distinct logical messages per mode
per capture. The three-capture denominator is 399 messages and 192 tool-call pairs
per mode. These sample counts are fixed before execution.
Each capture is analyzed three times by separate MCPTrace CLI processes. All four
selected endpoint PIDs are scanned together. Exported JSONL must be byte-identical
across the three scans. One set of exports is independently re-read through native
Volatility Windows process layers to check bytes, SHA-256, addresses, and VAD bounds.
The original RAM image is hashed before and after analysis, then deleted.
CLI timing includes process startup, Volatility automagic, scanning, JSON rendering,
and JSONL export. It excludes acquisition, initial `windows.info`/symbol preparation,
export comparison, and native byte verification. The image has already been hashed
and inspected before the first scan; no scan is claimed to be a cold-cache result.
Repeated scans measure timing/repeatability, not additional independent recovery
samples. Runtime covers four selected processes, not all processes in the RAM image.
## Reproduce
Use a fresh administrative Windows VM for each replica. Install the repository and
`validation/windows_positive/requirements.txt`, then run from the repository root:
```powershell
python validation/windows_positive/run.py --workdir C:\MCPBenchmark\work --reportdir C:\MCPBenchmark\report --tool-calls 64 --scan-repeats 3 --report-scan-limits
python validation/windows_positive/benchmark.py summarize --reportdir C:\MCPBenchmark\report --replica 1 --output benchmark-1.json
```
Repeat in two other fresh VMs using replicas 2 and 3, then aggregate all three files:
```powershell
python validation/windows_positive/benchmark.py aggregate benchmark-1.json benchmark-2.json benchmark-3.json --output benchmark.json
```
The GitHub Actions `Windows memory benchmark` workflow performs the three captures.
It uploads only allowlisted counts, durations, environment/version information, image
hashes, and validation flags. RAM, recovered messages, ground truth, process IDs,
and arbitrary log text are excluded. Failed or missing captures cannot be silently
dropped from the three-capture aggregate. Read errors and missing endpoints remain
failures. `--report-scan-limits` allows bounded-search results to be measured, but
marks them `completed_with_limits`, preserves every limit counter, and sets
`full_endpoint_scans=false`; the aggregate reports how many captures were limited.
The normal positive-validation command still rejects limited scans by default.
## Interpretation
Recovery counts deduplicate messages within each capture and combine both endpoints.
Candidate links remain candidates; unsent controls are counted separately from logged
traffic. Mean/min/max describe only these three captures. Logging changes memory,
and retained frames are a favorable control. No general recall, detection precision,
latency/pressure survival curve, or Windows 11/Linux benchmark is established by
this Windows Server 2022 SDK/stdio experiment.
Measured results: [Windows RAM benchmark](../../docs/windows-memory-benchmark.md).
validation/windows_positive/README.md
+8
View File
@@ -57,3 +57,11 @@ GitHub Actions의 `Windows positive memory validation` 워크플로도 같은
전체 실험은 실행 가능성을 확인한 것으로 기록되므로 **반드시 natural/retained 수치를 나눠 읽어야 합니다.**
실제로 수행한 Windows Server 2022 캡처와 조건별 수치는
[양성 검증 보고서](../../docs/windows-positive-memory-validation.md)에 기록했습니다.
`--tool-calls`로 조건별 도구 호출 수를, `--scan-repeats`로 동일 이미지의 CLI 분석 반복 수를
지정할 수 있습니다. 기본값은 기존 검증과 같은 8회/1회입니다. 별도 VM 3개에서 64회 호출과
3회 분석을 수행하는 [반복 벤치마크 절차](BENCHMARK.md)도 제공합니다.
벤치마크의 `--report-scan-limits`는 실제 탐색 제한에 도달한 결과를
`completed_with_limits`로 구분해 집계합니다. 기본 양성 검증은 제한 도달을 계속 실패로 처리하며,
읽기 오류나 누락된 대상 프로세스는 벤치마크에서도 실패입니다.
validation/windows_positive/benchmark.py
+246
View File
@@ -0,0 +1,246 @@
"""Publish allowlisted measurements and aggregate three independent Windows captures."""
from __future__ import annotations
import argparse
import json
import math
import statistics
from pathlib import Path
CASE_FIELDS = (
"expected_logical_messages",
"distinct_expected_messages_found",
"expected_tool_pairs",
"complete_tool_pairs_any_endpoint",
"artifact_records",
"records_matching_logged_traffic",
"distinct_unsent_controls_found",
"records_matching_unsent_controls",
"unmatched_artifact_records",
"evaluated_candidate_links",
"incorrect_candidate_links",
"unverifiable_candidate_links",
)
ENVIRONMENT_FIELDS = ("platform", "python", "mcp_sdk", "volatility", "logical_cpus")
COMPLETED = {"passed", "completed_with_limits"}
def read_json(path):
return json.loads(path.read_text(encoding="utf-8"))
def scan_diagnostics(scans):
return {
"processes": len(scans),
"limited_processes": sum(
bool(s["limits_hit"] or s["anchor_limit_hits"] or s["candidate_limit_hits"])
for s in scans.values()
),
"error_processes": sum(bool(s["error"]) for s in scans.values()),
"skipped_processes": sum(bool(s["skipped_reason"]) for s in scans.values()),
**{
key: sum(s[key] for s in scans.values())
for key in ("invalid_regions", "anchor_limit_hits", "candidate_limit_hits")
},
"record_budget_processes": sum("max-records" in s["limits_hit"] for s in scans.values()),
"byte_budget_processes": sum("max-bytes" in s["limits_hit"] for s in scans.values()),
}
def summarize_run(report, replica):
"""Exclude recovered content, ground truth, PIDs, paths, and arbitrary log text."""
path = report / "manifest.json"
manifest = read_json(path) if path.exists() else {}
output = {
"schema_version": 1,
"replica": replica,
"status": manifest["status"] if manifest.get("status") in COMPLETED else "failed",
"commit": manifest.get("commit"),
"environment": {key: manifest.get(key) for key in ENVIRONMENT_FIELDS},
"tool_calls_per_case": manifest.get("tool_calls_per_case"),
"scan_repeats": manifest.get("scan_repeats"),
"report_scan_limits": manifest.get("report_scan_limits", False),
}
scan_path = report / "scan-summaries.json"
scans = read_json(scan_path) if scan_path.exists() else {}
if scans:
output["scan_diagnostics"] = scan_diagnostics(scans)
if output["status"] not in COMPLETED:
return output
for key in (
"raw_memory_deleted",
"image_hash_unchanged_after_analysis",
"repeated_exports_identical",
):
if manifest.get(key) is not True:
raise ValueError(f"Completed capture is missing validation: {key}")
comparison = read_json(report / "comparison.json")
if len(scans) != 4 or any(
s["error"] or s["skipped_reason"] or s["invalid_regions"] for s in scans.values()
):
raise ValueError("Benchmark requires four complete endpoint scans")
limited = output["scan_diagnostics"]["limited_processes"] > 0
if limited and (
output["status"] != "completed_with_limits" or not output["report_scan_limits"]
):
raise ValueError("Limited scans must be explicitly marked completed_with_limits")
if not limited and output["status"] != "passed":
raise ValueError("Completed status does not agree with scan diagnostics")
cases = {}
for case in comparison["cases"]:
counts = {key: case[key] for key in CASE_FIELDS}
if any(type(value) is not int or value < 0 for value in counts.values()):
raise ValueError("Case measurements must be nonnegative integer counts")
if (
counts["expected_tool_pairs"] != output["tool_calls_per_case"]
or counts["expected_logical_messages"] != 5 + 2 * output["tool_calls_per_case"]
or counts["distinct_expected_messages_found"] > counts["expected_logical_messages"]
or counts["complete_tool_pairs_any_endpoint"] > counts["expected_tool_pairs"]
or counts["incorrect_candidate_links"]
):
raise ValueError("Inconsistent ground truth or incorrect candidate links")
counts["best_single_pid_tool_pairs"] = max(
p["complete_tool_pairs"] for p in case["per_pid"].values()
)
if case["mode"] in cases:
raise ValueError("Duplicate benchmark mode")
cases[case["mode"]] = counts
if set(cases) != {"natural", "retained"}:
raise ValueError("Both benchmark modes must be present")
verified = comparison["artifact_records_byte_verified"]
if verified != sum(c["artifact_records"] for c in cases.values()) or verified != sum(
s["displayed_records"] for s in scans.values()
):
raise ValueError("Verified and exported record counts differ")
timings = manifest["timings_seconds"]
durations = timings["plugin_cli_seconds"]
if len(durations) != output["scan_repeats"] or any(
not math.isfinite(s) or s <= 0 for s in durations
):
raise ValueError("Missing or invalid CLI timing samples")
output.update(
cases=cases,
capture_sha256=manifest["image_sha256"],
physical_ram_bytes=manifest["physical_ram_bytes"],
image_logical_bytes=manifest["image_bytes"],
artifact_records_byte_verified=verified,
artifact_bytes_verified=comparison["artifact_bytes_verified"],
scan_totals={
key: sum(s[key] for s in scans.values())
for key in ("bytes_read", "unreadable_bytes", "regions_scanned", "candidate_records")
},
timings_seconds={
**{
key: timings[key]
for key in (
"acquisition",
"symbols_and_info",
"native_byte_verification",
"pipeline_total",
)
},
"plugin_cli": durations,
},
ready_to_capture_seconds={
c["mode"]: (manifest["capture_started_ns"] - c["ready_ns"]) / 1e9
for c in manifest["cases"]
},
checks={
"full_endpoint_scans": not limited,
"native_bytes_verified": True,
"repeated_exports_identical": True,
"image_hash_unchanged": True,
"raw_memory_deleted": True,
},
)
return output
def distribution(values):
return {
"n": len(values),
"mean": statistics.mean(values),
"median": statistics.median(values),
"min": min(values),
"max": max(values),
}
def aggregate_runs(runs):
"""Require the full predeclared sample; never silently discard a failed capture."""
if len(runs) != 3 or {r["replica"] for r in runs} != {1, 2, 3}:
raise ValueError("Require exactly replicas 1, 2, and 3")
if any(r["status"] not in COMPLETED for r in runs):
raise ValueError("Cannot aggregate a failed capture as a successful benchmark")
if len({r["capture_sha256"] for r in runs}) != 3:
raise ValueError("Require three independently acquired images")
for key in (
"schema_version",
"commit",
"environment",
"tool_calls_per_case",
"scan_repeats",
"report_scan_limits",
):
if any(r[key] != runs[0][key] for r in runs):
raise ValueError(f"Incompatible benchmark conditions: {key}")
modes = {}
for mode in ("natural", "retained"):
cases = [r["cases"][mode] for r in runs]
totals = {key: sum(c[key] for c in cases) for key in CASE_FIELDS}
modes[mode] = {
"totals": totals,
"logical_message_coverage": (
totals["distinct_expected_messages_found"] / totals["expected_logical_messages"]
),
"messages_found_per_capture": distribution(
[c["distinct_expected_messages_found"] for c in cases]
),
"tool_pairs_per_capture": distribution(
[c["complete_tool_pairs_any_endpoint"] for c in cases]
),
}
return {
"schema_version": 1,
"independent_captures": 3,
"captures_with_scan_limits": sum(r["status"] == "completed_with_limits" for r in runs),
"commit": runs[0]["commit"],
"environment": runs[0]["environment"],
"tool_calls_per_case": runs[0]["tool_calls_per_case"],
"scan_repeats_per_capture": runs[0]["scan_repeats"],
"modes": modes,
"plugin_cli_seconds": distribution(
[s for r in runs for s in r["timings_seconds"]["plugin_cli"]]
),
"per_capture_cli_median_seconds": distribution(
[statistics.median(r["timings_seconds"]["plugin_cli"]) for r in runs]
),
"artifact_records_byte_verified": sum(r["artifact_records_byte_verified"] for r in runs),
"artifact_bytes_verified": sum(r["artifact_bytes_verified"] for r in runs),
"runs": sorted(runs, key=lambda r: r["replica"]),
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="action", required=True)
summarize = subparsers.add_parser("summarize")
summarize.add_argument("--reportdir", type=Path, required=True)
summarize.add_argument("--replica", type=int, choices=[1, 2, 3], required=True)
summarize.add_argument("--output", type=Path, required=True)
aggregate = subparsers.add_parser("aggregate")
aggregate.add_argument("inputs", nargs=3, type=Path)
aggregate.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
result = (
summarize_run(args.reportdir, args.replica)
if args.action == "summarize"
else aggregate_runs([read_json(path) for path in args.inputs])
)
args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n", encoding="utf-8")
print(f"Wrote {args.output.name}")
if __name__ == "__main__":
main()
validation/windows_positive/run.py
+64 -14
View File
@@ -8,6 +8,7 @@ import ctypes
import hashlib
import importlib.metadata
import json
import os
import platform
import shutil
import subprocess
@@ -40,12 +41,14 @@ def write_json(path, value):
def command(args, stdout, stderr, timeout=300):
started = time.perf_counter()
with stdout.open("wb") as out, stderr.open("wb") as err:
completed = subprocess.run(args, stdout=out, stderr=err, timeout=timeout, check=False)
if completed.returncode:
raise RuntimeError(
f"Command failed ({completed.returncode}); see {stdout.name}/{stderr.name}"
)
return time.perf_counter() - started
def physical_ram():
@@ -78,14 +81,21 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--workdir", required=True, type=Path)
parser.add_argument("--reportdir", required=True, type=Path)
parser.add_argument("--tool-calls", type=int, default=8)
parser.add_argument("--scan-repeats", type=int, default=1)
parser.add_argument("--report-scan-limits", action="store_true")
args = parser.parse_args()
if not 1 <= args.tool_calls <= 512 or not 1 <= args.scan_repeats <= 5:
parser.error("Require 1..512 tool calls and 1..5 scan repeats")
if sys.platform != "win32":
parser.error("Run this acquisition harness on Windows as administrator")
work, report = args.workdir.resolve(), args.reportdir.resolve()
work.mkdir(parents=True, exist_ok=False)
report.mkdir(parents=True, exist_ok=False)
repo = Path(__file__).resolve().parents[2]
pipeline_started = time.perf_counter()
image = work / "memory.raw"
timings = {"plugin_cli_seconds": []}
manifest = {
"status": "running",
"stage": "preflight",
@@ -103,6 +113,11 @@ def main():
"physical_ram_bytes": physical_ram(),
"free_disk_bytes": shutil.disk_usage(work).free,
"raw_memory_uploaded": False,
"logical_cpus": os.cpu_count(),
"tool_calls_per_case": args.tool_calls,
"scan_repeats": args.scan_repeats,
"report_scan_limits": args.report_scan_limits,
"timings_seconds": timings,
}
children, cases = [], []
@@ -144,6 +159,8 @@ def main():
str(output),
"--mode",
mode,
"--tool-calls",
str(args.tool_calls),
],
stdout=subprocess.DEVNULL,
stderr=error_log,
@@ -160,7 +177,10 @@ def main():
raise TimeoutError(f"{output.name}: MCP workload did not become ready")
time.sleep(0.1)
ready = json.loads((output / "ready.json").read_text(encoding="utf-8"))
if ready["tool_calls_completed"] != 8 or ready["logged_messages"] != 21:
if (
ready["tool_calls_completed"] != args.tool_calls
or ready["logged_messages"] != 5 + 2 * args.tool_calls
):
raise RuntimeError("Incomplete MCP ground truth")
cases.append(ready)
pids = [c[key] for c in cases for key in ["client_pid", "server_pid"]]
@@ -169,7 +189,7 @@ def main():
manifest["cases"] = cases
manifest["capture_started_ns"] = time.time_ns()
checkpoint("physical-ram-acquisition")
command(
timings["acquisition"] = command(
[str(tool), "acquire", str(image)],
report / "capture.txt",
report / "capture.stderr",
@@ -203,7 +223,7 @@ def main():
str(image),
]
# Windows symbols can be fetched during info; subsequent steps are offline.
command(
timings["symbols_and_info"] = command(
common + ["-r", "json", "windows.info.Info"],
report / "info.json",
report / "info.stderr",
@@ -225,22 +245,51 @@ def main():
"--dump",
]
manifest["plugin_command"] = trace_cmd
command(trace_cmd, work / "all-findings.json", report / "mcptrace.stderr")
checkpoint("byte-verification-and-ground-truth-comparison")
timings["plugin_cli_seconds"].append(
command(trace_cmd, work / "all-findings.json", report / "mcptrace.stderr")
)
exports = [export / f"mcptrace.pid-{pid}.jsonl" for pid in pids]
expected_exports = {path.name: sha256(path) for path in exports}
for repeat in range(1, args.scan_repeats):
repeated_output = work / f"exports-{repeat + 1}"
repeated_output.mkdir()
repeated_cmd = trace_cmd.copy()
repeated_cmd[repeated_cmd.index("-o") + 1] = str(repeated_output)
repeated_cmd[repeated_cmd.index("--save-config") + 1] = str(
work / f"mcptrace-config-{repeat + 1}.json"
)
timings["plugin_cli_seconds"].append(
command(
repeated_cmd,
work / f"all-findings-{repeat + 1}.json",
report / f"mcptrace-{repeat + 1}.stderr",
)
)
if any(
sha256(repeated_output / name) != digest
for name, digest in expected_exports.items()
):
raise RuntimeError("Repeated scans of the same image produced different JSONL")
manifest["repeated_exports_identical"] = True
checkpoint("byte-verification-and-ground-truth-comparison")
verification_started = time.perf_counter()
artifacts, summaries = verify_exports(image, symbols, cache, config, exports)
timings["native_byte_verification"] = time.perf_counter() - verification_started
# Persist diagnostics before deciding whether a scan can be scored.
write_json(report / "scan-summaries.json", summaries)
limited = False
for summary in summaries.values():
if summary["error"] or summary["skipped_reason"] or summary["invalid_regions"]:
raise RuntimeError("An endpoint scan failed; inspect scan summaries")
if (
summary["error"]
or summary["limits_hit"]
or summary["skipped_reason"]
summary["limits_hit"]
or summary["anchor_limit_hits"]
or summary["candidate_limit_hits"]
):
raise RuntimeError(
"A selected endpoint scan was incomplete; inspect scan summaries"
)
write_json(report / "scan-summaries.json", summaries)
limited = True
manifest["scan_complete"] = not limited
if limited and not args.report_scan_limits:
raise RuntimeError("A selected endpoint scan was incomplete; inspect scan summaries")
scores, publish_keys = [], set()
for case in cases:
mode = case["mode"]
@@ -282,7 +331,7 @@ def main():
raise RuntimeError("No complete known tool request/response pair recovered")
if any(s["incorrect_candidate_links"] for s in scores):
raise RuntimeError("Incorrect candidate links observed in controlled traffic")
manifest["analysis_status"] = "passed"
manifest["analysis_status"] = "completed_with_limits" if limited else "passed"
checkpoint("analysis-completed")
print(json.dumps(result, ensure_ascii=True, indent=2), flush=True)
except BaseException as exc:
@@ -310,7 +359,8 @@ def main():
else:
manifest["raw_memory_deleted"] = True
write_json(report / "manifest.json", manifest)
manifest["status"] = "passed"
manifest["status"] = manifest["analysis_status"]
timings["pipeline_total"] = time.perf_counter() - pipeline_started
checkpoint("completed")
validation/windows_positive/workload.py
+6 -2
View File
@@ -116,7 +116,8 @@ async def client(args):
"last message",
]
server_pid = None
for sequence, text in enumerate(samples):
for sequence in range(args.tool_calls):
text = samples[sequence % len(samples)]
result = await session.call_tool(
"echo_marker",
{"run_id": args.run_id, "sequence": sequence, "text": text},
@@ -159,7 +160,7 @@ async def client(args):
"run_id": args.run_id,
"client_pid": os.getpid(),
"server_pid": server_pid,
"tool_calls_completed": len(samples),
"tool_calls_completed": args.tool_calls,
"logged_messages": journal.count,
"retained_transport_frames": len(journal.frames),
"retained_unsent_controls": len(retained_decoys),
@@ -186,7 +187,10 @@ def main():
parser.add_argument("--output", type=Path)
parser.add_argument("--mode", choices=["natural", "retained"], default="natural")
parser.add_argument("--hold-seconds", type=int, default=900)
parser.add_argument("--tool-calls", type=int, default=8)
args = parser.parse_args()
if not 1 <= args.tool_calls <= 512:
parser.error("--tool-calls must be between 1 and 512")
if args.role == "server":
server(args.run_id)
elif args.output is None: