ai-space/craft-anything
3
0
Fork 0
mirror of https://github.com/storytold/craft-anything.git synced 2026-11-05 20:47:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Redact declared secrets from retained HAR metadata (R3.13) (from #603)

Browse Source
This commit is contained in:
Afloat
2026-11-03 19:47:22 +07:00
committed by bflatastic
parent ee72fbede9
commit 7ac706e94d
3 changed files with 144 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/analysis/src/web_har.rs
+59 -20
View File
@@ -87,7 +87,7 @@ impl Redactor {
let mut needles: Vec<String> = Vec::new();
for v in values.iter().filter(|v| !v.is_empty()) {
let json_escaped = serde_json::to_string(v).ok().and_then(|q| q.get(1..q.len().saturating_sub(1)).map(String::from)).unwrap_or_default();
for form in [v.clone(), percent_encode(v), percent_encode(v).to_ascii_lowercase(), json_escaped, v.replace(' ', "+")] {
for form in [v.clone(), v.to_ascii_lowercase(), percent_encode(v), percent_encode(v).to_ascii_lowercase(), json_escaped, v.replace(' ', "+")] {
if !form.is_empty() && !needles.contains(&form) {
needles.push(form);
}
@@ -140,6 +140,27 @@ pub(crate) fn split_url(url: &str) -> (String, String, String, Vec<(String, Stri
(scheme.to_ascii_lowercase(), host.to_ascii_lowercase(), if path.is_empty() { "/".into() } else { path.to_string() }, pairs, userinfo)
}
/// Decode a query name once for privacy checks; malformed escapes stay literal.
fn decode_query_name(name: &str) -> String {
let mut decoded = Vec::with_capacity(name.len());
let mut bytes = name.as_bytes().iter().copied().peekable();
while let Some(b) = bytes.next() {
if b == b'%' {
let mut next = bytes.clone();
let hex = next.next().and_then(|hi| next.next().map(|lo| [hi, lo]));
if let Some(value) =
hex.filter(|h| h.iter().all(u8::is_ascii_hexdigit)).and_then(|h| std::str::from_utf8(&h).ok().and_then(|s| u8::from_str_radix(s, 16).ok()))
{
decoded.push(value);
bytes = next;
continue;
}
}
decoded.push(if b == b'+' { b' ' } else { b });
}
String::from_utf8_lossy(&decoded).into_owned()
}
pub(crate) fn is_credential_param(k: &str) -> bool {
let l = k.to_ascii_lowercase();
CREDENTIAL_PARAMS.contains(&l.as_str()) || l.ends_with("_token") || l.ends_with("token") || l.contains("password") || l.contains("secret")
@@ -154,6 +175,10 @@ fn safe_url(url: &str, pointer: &str, red: &mut Redactor) -> (String, Vec<String
let mut q = Vec::new();
let mut names = Vec::new();
for (k, v) in pairs {
if red.contains(&k) || red.contains(&decode_query_name(&k)) {
red.log.push(json!({"pointer": pointer, "reason": "secret-value", "scope": "query-name"}));
continue;
}
names.push(k.clone());
if is_credential_param(&k) && !v.is_empty() {
red.log.push(json!({"pointer": pointer, "reason": "credential-parameter", "parameter": k}));
@@ -324,24 +349,28 @@ fn body_outline(
if decoded.is_none() {
return Value::Null;
}
json!({"kind": if mime_l.is_empty() { "unknown".to_string() } else { mime_l }, "bytes": bytes})
json!({"kind": if mime_l.is_empty() { "unknown".to_string() } else { red.text(&format!("{pointer}/mime"), &mime_l) }, "bytes": bytes})
}
fn headers_view(h: &[(String, String)], pointer: &str, red: &mut Redactor) -> Vec<Value> {
h.iter()
.take(200)
.enumerate()
.map(|(i, (n, v))| {
.filter_map(|(i, (n, v))| {
if red.contains(n) {
red.log.push(json!({"pointer": pointer, "reason": "secret-value", "scope": "header-name"}));
return None;
}
let l = n.to_ascii_lowercase();
let p = format!("{pointer}/{i}/value");
if CREDENTIAL_HEADERS.contains(&l.as_str()) {
red.log.push(json!({"pointer": p, "reason": "auth-header"}));
json!({"name": n, "value": null, "redacted": "auth-header"})
Some(json!({"name": n, "value": null, "redacted": "auth-header"}))
} else if matches!(l.as_str(), "referer" | "origin" | "location") {
let (u, _) = safe_url(v, &p, red);
json!({"name": n, "value": u})
Some(json!({"name": n, "value": u}))
} else {
json!({"name": n, "value": red.text(&p, v)})
Some(json!({"name": n, "value": red.text(&p, v)}))
}
})
.collect()
@@ -361,27 +390,27 @@ fn entry_view(e: &HarEntry, red: &mut Redactor) -> Value {
}
json!({
"ordinal": e.ordinal,
"started": e.started,
"started": e.started.as_deref().map(|value| red.text(&format!("{base}/started"), value)),
"time_ms": e.time_ms,
"resource_type": e.resource_type,
"method": e.request.method,
"resource_type": e.resource_type.as_deref().map(|value| red.text(&format!("{base}/resource_type"), value)),
"method": red.text(&format!("{base}/request/method"), &e.request.method),
"url": url,
"host": host,
"path_template": path_template(&path),
"host": red.text(&format!("{base}/request/host"), &host),
"path_template": red.text(&format!("{base}/request/path_template"), &path_template(&path)),
"query_parameters": query_names,
"http_version": e.request.http_version,
"http_version": e.request.http_version.as_deref().map(|value| red.text(&format!("{base}/http_version"), value)),
"status": e.response.status,
"status_text": e.response.status_text,
"status_text": e.response.status_text.as_deref().map(|value| red.text(&format!("{base}/status_text"), value)),
"request_headers": headers_view(&e.request.headers, &format!("{base}/request/headers"), red),
"response_headers": headers_view(&e.response.headers, &format!("{base}/response/headers"), red),
"request_cookie_names": e.request.cookie_names.iter().filter(|n| !red.contains(n)).collect::<Vec<_>>(),
"response_cookie_names": e.response.cookie_names.iter().filter(|n| !red.contains(n)).collect::<Vec<_>>(),
"request_body": req_body,
"response_body": res_body,
"response_mime": e.response.mime,
"response_mime": e.response.mime.as_deref().map(|value| red.text(&format!("{base}/response_mime"), value)),
"response_bytes": e.response.content_size,
"redirect_url": redirect,
"timings": e.timings.iter().map(|(k, v)| (k.clone(), json!(v))).collect::<Map<String, Value>>(),
"timings": e.timings.iter().filter(|(k, _)| !red.contains(k)).map(|(k, v)| (k.clone(), json!(v))).collect::<Map<String, Value>>(),
})
}
@@ -401,11 +430,11 @@ pub fn inspect_har(bytes: &[u8], opts: &HarOptions) -> Result<Value, String> {
let mut views = Vec::new();
for e in &har.entries {
let view = entry_view(e, &mut red);
*methods.entry(e.request.method.to_ascii_uppercase()).or_default() += 1;
*methods.entry(view["method"].as_str().unwrap_or_default().to_ascii_uppercase()).or_default() += 1;
*statuses.entry(format!("{}xx", e.response.status / 100)).or_default() += 1;
let host = view["host"].as_str().unwrap_or("").to_string();
*hosts.entry(red.text("/hosts", &host)).or_default() += 1;
let mime = e.response.mime.as_deref().unwrap_or("").split(';').next().unwrap_or("").trim().to_string();
let mime = view["response_mime"].as_str().unwrap_or("").split(';').next().unwrap_or("").trim().to_string();
*mimes.entry(mime.clone()).or_default() += 1;
total_bytes = total_bytes.saturating_add(e.response.body_size.unwrap_or(0).max(e.response.content_size.unwrap_or(0)).max(0));
if let Some(s) = &e.started {
@@ -421,7 +450,13 @@ pub fn inspect_har(bytes: &[u8], opts: &HarOptions) -> Result<Value, String> {
}
// API surface.
let (scheme, _, _, _, _) = split_url(&e.request.url);
let key = format!("{} {}://{}{}", e.request.method.to_ascii_uppercase(), scheme, host, view["path_template"].as_str().unwrap_or(""));
let key = format!(
"{} {}://{}{}",
view["method"].as_str().unwrap_or_default().to_ascii_uppercase(),
red.text("/scheme", &scheme),
host,
view["path_template"].as_str().unwrap_or("")
);
let key = red.text("/api_surface", &key);
let g = surface.entry(key.clone()).or_insert_with(|| json!({"operation": key, "count": 0, "statuses": [], "query_parameters": [], "request_outline": null, "response_outline": null, "response_mimes": []}));
if let Some(o) = g.as_object_mut() {
@@ -455,12 +490,16 @@ pub fn inspect_har(bytes: &[u8], opts: &HarOptions) -> Result<Value, String> {
None => views.iter().skip(opts.offset).take(opts.limit.max(1)).cloned().collect(),
};
let ops: Vec<Value> = surface.into_values().collect();
let version = har.version.as_deref().map(|v| red.text("/har_version", v));
let creator = har.creator.as_deref().map(|v| red.text("/creator", v));
let first = first.as_deref().map(|v| red.text("/summary/first_request", v));
let last = last.as_deref().map(|v| red.text("/summary/last_request", v));
let mut redactions = red.log;
redactions.truncate(5000);
Ok(json!({
"format": "har",
"har_version": har.version,
"creator": har.creator,
"har_version": version,
"creator": creator,
"pages": har.pages,
"summary": {
"entries": har.entries.len(),
crates/analysis/tests/regression_har_metadata.rs
+25
View File
@@ -0,0 +1,25 @@
use craft_anything_analysis::web_har::{HarOptions, inspect_har};
use serde_json::json;
#[test]
fn regression_declared_values_do_not_leak_through_metadata() {
let secret = "privatevalue";
let bytes = serde_json::to_vec(&json!({"log": {
"version": secret, "creator": {"name": secret, "version": secret},
"entries": [{"startedDateTime": secret, "_resourceType": secret,
"request": {"method": secret, "url": format!("https://{secret}.test/{secret}?{secret}=safe&%70rivatevalue=safe&%70r%69vatevalue=safe&keep=safe"),
"httpVersion": secret, "headers": [{"name": secret, "value": "ordinary"}],
"postData": {"mimeType": format!("text/{secret}"), "text": "ordinary"}},
"response": {"status": 200, "statusText": secret, "content": {"mimeType": format!("text/{secret}"), "text": "ordinary"}},
"timings": {(secret): 1.0}}]
}}))
.unwrap();
let report = inspect_har(&bytes, &HarOptions { hidden_values: vec![secret.into()], limit: 10, max_entries: 10, ..Default::default() }).unwrap();
let text = report.to_string().to_ascii_lowercase();
assert!(!text.contains(secret), "{text}");
assert!(!text.contains("%70rivatevalue") && !text.contains("%70r%69vatevalue"), "{text}");
assert_eq!(report["entries"][0]["query_parameters"], json!(["keep"]));
assert_eq!(report["summary"]["entries"], 1);
assert_eq!(report["entries"][0]["status"], 200);
assert!(report["entries"][0]["request_headers"].as_array().unwrap().is_empty());
}
crates/analysis/tests/regression_har_normalized_metadata.rs
+60
View File
@@ -0,0 +1,60 @@
use craft_anything_analysis::web_har::{HarOptions, inspect_har};
use serde_json::{Value, json};
fn report(entries: Value, hidden: &[&str]) -> Value {
let bytes = serde_json::to_vec(&json!({"log": {"entries": entries}})).unwrap();
let options = HarOptions { hidden_values: hidden.iter().map(|s| s.to_string()).collect(), limit: 10, max_entries: 10, ..Default::default() };
inspect_har(&bytes, &options).unwrap()
}
#[test]
fn normalized_ipv6_hosts_do_not_reveal_declared_values() {
let secret = "[2001:DB8::BEEF]";
let output = report(
json!([
{"request": {"method": "GET", "url": format!("https://{secret}/items")}, "response": {"status": 200}},
{"request": {"method": "GET", "url": "https://PUBLIC.Example.test/items"}, "response": {"status": 204}}
]),
&[secret],
);
assert_eq!(output["entries"][1]["host"], "public.example.test");
assert_eq!(output["entries"][1]["url"], "https://public.example.test/items");
assert_eq!(output["entries"][0]["host"], "[redacted]");
assert_eq!(output["entries"][0]["url"], "https://[redacted]/items");
assert!(!output.to_string().to_ascii_lowercase().contains(&secret.to_ascii_lowercase()));
}
#[test]
fn normalized_mime_parameters_do_not_reveal_declared_values() {
let secret = "Secret=Value";
let mime = format!("text/plain; x-private={secret}");
let output = report(
json!([{
"request": {"method": "POST", "url": "https://example.test/items", "postData": {"mimeType": mime, "text": "public"}},
"response": {"status": 200, "content": {"mimeType": mime, "text": "public"}}
}]),
&[secret],
);
for side in ["request_body", "response_body"] {
assert_eq!(output["entries"][0][side]["kind"], "text/plain; x-private=[redacted]");
assert_eq!(output["entries"][0][side]["bytes"], 6);
}
assert!(!output.to_string().to_ascii_lowercase().contains("secret=value"));
}
#[test]
fn redacted_mime_values_still_select_json_form_and_multipart_outlines() {
let output = report(
json!([
{"request": {"method": "POST", "url": "https://example.test/json", "postData": {"mimeType": "application/json", "text": "{\"label\":\"public\"}"}}, "response": {"status": 200}},
{"request": {"method": "POST", "url": "https://example.test/form", "postData": {"mimeType": "application/x-www-form-urlencoded", "text": "label=public"}}, "response": {"status": 200}},
{"request": {"method": "POST", "url": "https://example.test/multipart", "postData": {"mimeType": "multipart/form-data", "params": [{"name": "label", "value": "public"}]}}, "response": {"status": 200}}
]),
&["application/json", "application/x-www-form-urlencoded", "multipart/form-data"],
);
for (index, kind) in ["json", "form", "multipart"].into_iter().enumerate() {
assert_eq!(output["entries"][index]["request_body"]["kind"], kind);
}
assert_eq!(output["entries"][1]["request_body"]["fields"], json!(["label"]));
assert_eq!(output["entries"][2]["request_body"]["fields"], json!(["label"]));
}