From 7ac706e94dd8a8d9151a236a7e05e6d79569504f Mon Sep 17 00:00:00 2001 From: Afloat <132809511+Afloat16@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:47:22 +0700 Subject: [PATCH] Redact declared secrets from retained HAR metadata (R3.13) (from #603) --- crates/analysis/src/web_har.rs | 79 ++++++++++++++----- .../analysis/tests/regression_har_metadata.rs | 25 ++++++ .../regression_har_normalized_metadata.rs | 60 ++++++++++++++ 3 files changed, 144 insertions(+), 20 deletions(-) create mode 100644 crates/analysis/tests/regression_har_metadata.rs create mode 100644 crates/analysis/tests/regression_har_normalized_metadata.rs diff --git a/crates/analysis/src/web_har.rs b/crates/analysis/src/web_har.rs index 7f616344..56cae11b 100644 --- a/crates/analysis/src/web_har.rs +++ b/crates/analysis/src/web_har.rs @@ -87,7 +87,7 @@ impl Redactor { let mut needles: Vec = Vec::new(); for v in values.iter().filter(|v| !v.is_empty()) { let json_escaped = serde_json::to_string(v).ok().and_then(|q| q.get(1..q.len().saturating_sub(1)).map(String::from)).unwrap_or_default(); - for form in [v.clone(), percent_encode(v), percent_encode(v).to_ascii_lowercase(), json_escaped, v.replace(' ', "+")] { + for form in [v.clone(), v.to_ascii_lowercase(), percent_encode(v), percent_encode(v).to_ascii_lowercase(), json_escaped, v.replace(' ', "+")] { if !form.is_empty() && !needles.contains(&form) { needles.push(form); } @@ -140,6 +140,27 @@ pub(crate) fn split_url(url: &str) -> (String, String, String, Vec<(String, Stri (scheme.to_ascii_lowercase(), host.to_ascii_lowercase(), if path.is_empty() { "/".into() } else { path.to_string() }, pairs, userinfo) } +/// Decode a query name once for privacy checks; malformed escapes stay literal. +fn decode_query_name(name: &str) -> String { + let mut decoded = Vec::with_capacity(name.len()); + let mut bytes = name.as_bytes().iter().copied().peekable(); + while let Some(b) = bytes.next() { + if b == b'%' { + let mut next = bytes.clone(); + let hex = next.next().and_then(|hi| next.next().map(|lo| [hi, lo])); + if let Some(value) = + hex.filter(|h| h.iter().all(u8::is_ascii_hexdigit)).and_then(|h| std::str::from_utf8(&h).ok().and_then(|s| u8::from_str_radix(s, 16).ok())) + { + decoded.push(value); + bytes = next; + continue; + } + } + decoded.push(if b == b'+' { b' ' } else { b }); + } + String::from_utf8_lossy(&decoded).into_owned() +} + pub(crate) fn is_credential_param(k: &str) -> bool { let l = k.to_ascii_lowercase(); CREDENTIAL_PARAMS.contains(&l.as_str()) || l.ends_with("_token") || l.ends_with("token") || l.contains("password") || l.contains("secret") @@ -154,6 +175,10 @@ fn safe_url(url: &str, pointer: &str, red: &mut Redactor) -> (String, Vec Vec { h.iter() .take(200) .enumerate() - .map(|(i, (n, v))| { + .filter_map(|(i, (n, v))| { + if red.contains(n) { + red.log.push(json!({"pointer": pointer, "reason": "secret-value", "scope": "header-name"})); + return None; + } let l = n.to_ascii_lowercase(); let p = format!("{pointer}/{i}/value"); if CREDENTIAL_HEADERS.contains(&l.as_str()) { red.log.push(json!({"pointer": p, "reason": "auth-header"})); - json!({"name": n, "value": null, "redacted": "auth-header"}) + Some(json!({"name": n, "value": null, "redacted": "auth-header"})) } else if matches!(l.as_str(), "referer" | "origin" | "location") { let (u, _) = safe_url(v, &p, red); - json!({"name": n, "value": u}) + Some(json!({"name": n, "value": u})) } else { - json!({"name": n, "value": red.text(&p, v)}) + Some(json!({"name": n, "value": red.text(&p, v)})) } }) .collect() @@ -361,27 +390,27 @@ fn entry_view(e: &HarEntry, red: &mut Redactor) -> Value { } json!({ "ordinal": e.ordinal, - "started": e.started, + "started": e.started.as_deref().map(|value| red.text(&format!("{base}/started"), value)), "time_ms": e.time_ms, - "resource_type": e.resource_type, - "method": e.request.method, + "resource_type": e.resource_type.as_deref().map(|value| red.text(&format!("{base}/resource_type"), value)), + "method": red.text(&format!("{base}/request/method"), &e.request.method), "url": url, - "host": host, - "path_template": path_template(&path), + "host": red.text(&format!("{base}/request/host"), &host), + "path_template": red.text(&format!("{base}/request/path_template"), &path_template(&path)), "query_parameters": query_names, - "http_version": e.request.http_version, + "http_version": e.request.http_version.as_deref().map(|value| red.text(&format!("{base}/http_version"), value)), "status": e.response.status, - "status_text": e.response.status_text, + "status_text": e.response.status_text.as_deref().map(|value| red.text(&format!("{base}/status_text"), value)), "request_headers": headers_view(&e.request.headers, &format!("{base}/request/headers"), red), "response_headers": headers_view(&e.response.headers, &format!("{base}/response/headers"), red), "request_cookie_names": e.request.cookie_names.iter().filter(|n| !red.contains(n)).collect::>(), "response_cookie_names": e.response.cookie_names.iter().filter(|n| !red.contains(n)).collect::>(), "request_body": req_body, "response_body": res_body, - "response_mime": e.response.mime, + "response_mime": e.response.mime.as_deref().map(|value| red.text(&format!("{base}/response_mime"), value)), "response_bytes": e.response.content_size, "redirect_url": redirect, - "timings": e.timings.iter().map(|(k, v)| (k.clone(), json!(v))).collect::>(), + "timings": e.timings.iter().filter(|(k, _)| !red.contains(k)).map(|(k, v)| (k.clone(), json!(v))).collect::>(), }) } @@ -401,11 +430,11 @@ pub fn inspect_har(bytes: &[u8], opts: &HarOptions) -> Result { let mut views = Vec::new(); for e in &har.entries { let view = entry_view(e, &mut red); - *methods.entry(e.request.method.to_ascii_uppercase()).or_default() += 1; + *methods.entry(view["method"].as_str().unwrap_or_default().to_ascii_uppercase()).or_default() += 1; *statuses.entry(format!("{}xx", e.response.status / 100)).or_default() += 1; let host = view["host"].as_str().unwrap_or("").to_string(); *hosts.entry(red.text("/hosts", &host)).or_default() += 1; - let mime = e.response.mime.as_deref().unwrap_or("").split(';').next().unwrap_or("").trim().to_string(); + let mime = view["response_mime"].as_str().unwrap_or("").split(';').next().unwrap_or("").trim().to_string(); *mimes.entry(mime.clone()).or_default() += 1; total_bytes = total_bytes.saturating_add(e.response.body_size.unwrap_or(0).max(e.response.content_size.unwrap_or(0)).max(0)); if let Some(s) = &e.started { @@ -421,7 +450,13 @@ pub fn inspect_har(bytes: &[u8], opts: &HarOptions) -> Result { } // API surface. let (scheme, _, _, _, _) = split_url(&e.request.url); - let key = format!("{} {}://{}{}", e.request.method.to_ascii_uppercase(), scheme, host, view["path_template"].as_str().unwrap_or("")); + let key = format!( + "{} {}://{}{}", + view["method"].as_str().unwrap_or_default().to_ascii_uppercase(), + red.text("/scheme", &scheme), + host, + view["path_template"].as_str().unwrap_or("") + ); let key = red.text("/api_surface", &key); let g = surface.entry(key.clone()).or_insert_with(|| json!({"operation": key, "count": 0, "statuses": [], "query_parameters": [], "request_outline": null, "response_outline": null, "response_mimes": []})); if let Some(o) = g.as_object_mut() { @@ -455,12 +490,16 @@ pub fn inspect_har(bytes: &[u8], opts: &HarOptions) -> Result { None => views.iter().skip(opts.offset).take(opts.limit.max(1)).cloned().collect(), }; let ops: Vec = surface.into_values().collect(); + let version = har.version.as_deref().map(|v| red.text("/har_version", v)); + let creator = har.creator.as_deref().map(|v| red.text("/creator", v)); + let first = first.as_deref().map(|v| red.text("/summary/first_request", v)); + let last = last.as_deref().map(|v| red.text("/summary/last_request", v)); let mut redactions = red.log; redactions.truncate(5000); Ok(json!({ "format": "har", - "har_version": har.version, - "creator": har.creator, + "har_version": version, + "creator": creator, "pages": har.pages, "summary": { "entries": har.entries.len(), diff --git a/crates/analysis/tests/regression_har_metadata.rs b/crates/analysis/tests/regression_har_metadata.rs new file mode 100644 index 00000000..84c1c345 --- /dev/null +++ b/crates/analysis/tests/regression_har_metadata.rs @@ -0,0 +1,25 @@ +use craft_anything_analysis::web_har::{HarOptions, inspect_har}; +use serde_json::json; + +#[test] +fn regression_declared_values_do_not_leak_through_metadata() { + let secret = "privatevalue"; + let bytes = serde_json::to_vec(&json!({"log": { + "version": secret, "creator": {"name": secret, "version": secret}, + "entries": [{"startedDateTime": secret, "_resourceType": secret, + "request": {"method": secret, "url": format!("https://{secret}.test/{secret}?{secret}=safe&%70rivatevalue=safe&%70r%69vatevalue=safe&keep=safe"), + "httpVersion": secret, "headers": [{"name": secret, "value": "ordinary"}], + "postData": {"mimeType": format!("text/{secret}"), "text": "ordinary"}}, + "response": {"status": 200, "statusText": secret, "content": {"mimeType": format!("text/{secret}"), "text": "ordinary"}}, + "timings": {(secret): 1.0}}] + }})) + .unwrap(); + let report = inspect_har(&bytes, &HarOptions { hidden_values: vec![secret.into()], limit: 10, max_entries: 10, ..Default::default() }).unwrap(); + let text = report.to_string().to_ascii_lowercase(); + assert!(!text.contains(secret), "{text}"); + assert!(!text.contains("%70rivatevalue") && !text.contains("%70r%69vatevalue"), "{text}"); + assert_eq!(report["entries"][0]["query_parameters"], json!(["keep"])); + assert_eq!(report["summary"]["entries"], 1); + assert_eq!(report["entries"][0]["status"], 200); + assert!(report["entries"][0]["request_headers"].as_array().unwrap().is_empty()); +} diff --git a/crates/analysis/tests/regression_har_normalized_metadata.rs b/crates/analysis/tests/regression_har_normalized_metadata.rs new file mode 100644 index 00000000..86920407 --- /dev/null +++ b/crates/analysis/tests/regression_har_normalized_metadata.rs @@ -0,0 +1,60 @@ +use craft_anything_analysis::web_har::{HarOptions, inspect_har}; +use serde_json::{Value, json}; + +fn report(entries: Value, hidden: &[&str]) -> Value { + let bytes = serde_json::to_vec(&json!({"log": {"entries": entries}})).unwrap(); + let options = HarOptions { hidden_values: hidden.iter().map(|s| s.to_string()).collect(), limit: 10, max_entries: 10, ..Default::default() }; + inspect_har(&bytes, &options).unwrap() +} + +#[test] +fn normalized_ipv6_hosts_do_not_reveal_declared_values() { + let secret = "[2001:DB8::BEEF]"; + let output = report( + json!([ + {"request": {"method": "GET", "url": format!("https://{secret}/items")}, "response": {"status": 200}}, + {"request": {"method": "GET", "url": "https://PUBLIC.Example.test/items"}, "response": {"status": 204}} + ]), + &[secret], + ); + assert_eq!(output["entries"][1]["host"], "public.example.test"); + assert_eq!(output["entries"][1]["url"], "https://public.example.test/items"); + assert_eq!(output["entries"][0]["host"], "[redacted]"); + assert_eq!(output["entries"][0]["url"], "https://[redacted]/items"); + assert!(!output.to_string().to_ascii_lowercase().contains(&secret.to_ascii_lowercase())); +} + +#[test] +fn normalized_mime_parameters_do_not_reveal_declared_values() { + let secret = "Secret=Value"; + let mime = format!("text/plain; x-private={secret}"); + let output = report( + json!([{ + "request": {"method": "POST", "url": "https://example.test/items", "postData": {"mimeType": mime, "text": "public"}}, + "response": {"status": 200, "content": {"mimeType": mime, "text": "public"}} + }]), + &[secret], + ); + for side in ["request_body", "response_body"] { + assert_eq!(output["entries"][0][side]["kind"], "text/plain; x-private=[redacted]"); + assert_eq!(output["entries"][0][side]["bytes"], 6); + } + assert!(!output.to_string().to_ascii_lowercase().contains("secret=value")); +} + +#[test] +fn redacted_mime_values_still_select_json_form_and_multipart_outlines() { + let output = report( + json!([ + {"request": {"method": "POST", "url": "https://example.test/json", "postData": {"mimeType": "application/json", "text": "{\"label\":\"public\"}"}}, "response": {"status": 200}}, + {"request": {"method": "POST", "url": "https://example.test/form", "postData": {"mimeType": "application/x-www-form-urlencoded", "text": "label=public"}}, "response": {"status": 200}}, + {"request": {"method": "POST", "url": "https://example.test/multipart", "postData": {"mimeType": "multipart/form-data", "params": [{"name": "label", "value": "public"}]}}, "response": {"status": 200}} + ]), + &["application/json", "application/x-www-form-urlencoded", "multipart/form-data"], + ); + for (index, kind) in ["json", "form", "multipart"].into_iter().enumerate() { + assert_eq!(output["entries"][index]["request_body"]["kind"], kind); + } + assert_eq!(output["entries"][1]["request_body"]["fields"], json!(["label"])); + assert_eq!(output["entries"][2]["request_body"]["fields"], json!(["label"])); +}