From f0a4c0f68cb534564d35e87cd4bb4242f8beb43f Mon Sep 17 00:00:00 2001 From: Afloat <132809511+Afloat16@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:20:52 +0700 Subject: [PATCH] Bound CPIO checksum work across firmware probes (R3.12) (from #676) --- crates/formats/src/firmware.rs | 69 ++++++++- crates/formats/src/tarcpio.rs | 26 +++- crates/formats/tests/cpio_crc_scan_limits.rs | 144 +++++++++++++++++++ 3 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 crates/formats/tests/cpio_crc_scan_limits.rs diff --git a/crates/formats/src/firmware.rs b/crates/formats/src/firmware.rs index 9fb5652f..55d1b9c8 100644 --- a/crates/formats/src/firmware.rs +++ b/crates/formats/src/firmware.rs @@ -7,6 +7,7 @@ use crate::bytes::Reader; use crate::common::entropy; +use crate::tarcpio::CpioChecksum; use miniz_oxide::inflate::stream::{InflateState, inflate as mz_inflate}; use miniz_oxide::{DataFormat, MZError, MZFlush, MZStatus}; use serde::Serialize; @@ -311,6 +312,10 @@ struct Scanner<'a> { probes: BTreeMap<&'static str, usize>, opts: &'a ScanOptions, inflate_left: u64, + checksum_left: usize, + checksum_exhausted: bool, + stop: &'a dyn Fn() -> bool, + cancelled: bool, exhausted: Vec<&'static str>, } @@ -791,22 +796,53 @@ impl<'a> Scanner<'a> { if !matches!(self.r.u8(i + 5)?, b'1' | b'2' | b'7') || !self.probe("cpio") { return None; } - let l = crate::tarcpio::cpio(self.data.get(i..)?, 1_000_000)?; + let data = self.data; + let before = self.checksum_left; + let mut skipped = false; + let l = crate::tarcpio::cpio_with_checksum(data.get(i..)?, 1_000_000, &mut |body| { + let result = self.cpio_checksum(body); + skipped |= matches!(result, CpioChecksum::Skipped); + result + })?; if l.members.is_empty() && !l.complete { return None; } + let mut details = json!({"members": l.members.len(), "complete": l.complete, "warnings": l.warnings}); + if l.format == "cpio-crc" { + details["checksum_validation_skipped"] = json!(skipped); + details["checksum_bytes_visited"] = json!(before.saturating_sub(self.checksum_left)); + } Some(reg( i, Some(l.length), l.format, Family::Archive, format!("cpio archive ({}), {} member(s)", l.format.trim_start_matches("cpio-"), l.members.len()), - if l.complete { "high" } else { "medium" }, + if l.complete && !skipped { "high" } else { "medium" }, Unpack::Members, - json!({"members": l.members.len(), "complete": l.complete, "warnings": l.warnings}), + details, )) } + /// Sum at most one input's worth of CRC member bytes across the entire scan. + fn cpio_checksum(&mut self, body: &[u8]) -> CpioChecksum { + let mut sum = 0u32; + for chunk in body.chunks(64 << 10) { + if (self.stop)() { + self.cancelled = true; + return CpioChecksum::Cancelled; + } + let count = chunk.len().min(self.checksum_left); + sum = chunk.iter().take(count).fold(sum, |sum, &byte| sum.wrapping_add(u32::from(byte))); + self.checksum_left = self.checksum_left.saturating_sub(count); + if count < chunk.len() { + self.checksum_exhausted = true; + return CpioChecksum::Skipped; + } + } + CpioChecksum::Value(sum) + } + fn squashfs(&mut self, i: usize, be: bool) -> Option { let r = self.r; let u16f = |o: usize| if be { r.u16_be(i + o) } else { r.u16_le(i + o) }; @@ -1499,11 +1535,23 @@ impl<'a> Scanner<'a> { } } -/// Scan `data` for recognised regions. `stop` is polled every MiB; when it returns true the scan -/// ends early with `cancelled` set. A region with a confirmed length hides its interior from the -/// scan (nested content is found by scanning the region again after extraction). +/// Scan `data` for recognised regions. `stop` is polled every MiB and every 64 KiB during CPIO +/// checksum sums; when it returns true the scan ends early with `cancelled` set. CRC member sums +/// share a byte allowance equal to the input length; skipped verification is reported. A region +/// with a confirmed length hides its interior (nested content is found after extraction). pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanResult { - let mut s = Scanner { data, r: Reader::new(data), probes: BTreeMap::new(), opts, inflate_left: opts.inflate_budget, exhausted: Vec::new() }; + let mut s = Scanner { + data, + r: Reader::new(data), + probes: BTreeMap::new(), + opts, + inflate_left: opts.inflate_budget, + checksum_left: data.len(), + checksum_exhausted: false, + stop, + cancelled: false, + exhausted: Vec::new(), + }; let mut out = ScanResult::default(); let mut skip_until = 0usize; let mut i = 0usize; @@ -1513,6 +1561,10 @@ pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanRes break; } let found = s.try_backref(i).filter(|r| r.offset as usize >= skip_until).or_else(|| s.try_at(i)); + if s.cancelled { + out.cancelled = true; + break; + } if let Some(r) = found { if out.regions.len() >= opts.max_regions { out.truncated = true; @@ -1538,6 +1590,9 @@ pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanRes if !out.probes_exhausted.is_empty() { out.warnings.push(format!("probe budget spent for: {} (later candidates were not checked)", out.probes_exhausted.join(", "))); } + if s.checksum_exhausted { + out.warnings.push("cpio CRC checksum byte budget spent (some member data was not verified)".into()); + } out } diff --git a/crates/formats/src/tarcpio.rs b/crates/formats/src/tarcpio.rs index 6039691d..5e7fe4e9 100644 --- a/crates/formats/src/tarcpio.rs +++ b/crates/formats/src/tarcpio.rs @@ -275,6 +275,19 @@ fn mode_kind(mode: u32) -> MemberKind { /// Walk a cpio archive (newc `070701`, crc `070702`, odc `070707`). `None` when the first header /// is not one of these. pub fn cpio(data: &[u8], max_members: usize) -> Option { + cpio_with_checksum(data, max_members, &mut |body| CpioChecksum::Value(body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte))))) +} + +/// A checksum sum, or the reason a caller stopped verification before completing it. +#[derive(Clone, Copy)] +pub(crate) enum CpioChecksum { + Value(u32), + Skipped, + Cancelled, +} + +/// Let a scanner share its checksum work allowance and cancellation across overlapping probes. +pub(crate) fn cpio_with_checksum(data: &[u8], max_members: usize, checksum: &mut dyn FnMut(&[u8]) -> CpioChecksum) -> Option { let r = Reader::new(data); let format = match r.slice(0, 6)? { b"070701" => "cpio-newc", @@ -341,12 +354,17 @@ pub fn cpio(data: &[u8], max_members: usize) -> Option { break; } if format == "cpio-crc" { - match h.get(102..110).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) { + match h.get(102..110).filter(|field| field.iter().all(u8::is_ascii_hexdigit)).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) { Some(expected) => { if let Some(body) = r.slice(data_off, size) { - let actual = body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte))); - if actual != expected { - out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})")); + match checksum(body) { + CpioChecksum::Value(actual) => { + if actual != expected { + out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})")); + } + } + CpioChecksum::Skipped => out.warnings.push(format!("{name}: data checksum not verified (scan byte budget exhausted)")), + CpioChecksum::Cancelled => return None, } } } diff --git a/crates/formats/tests/cpio_crc_scan_limits.rs b/crates/formats/tests/cpio_crc_scan_limits.rs new file mode 100644 index 00000000..d161de30 --- /dev/null +++ b/crates/formats/tests/cpio_crc_scan_limits.rs @@ -0,0 +1,144 @@ +use std::cell::Cell; + +use craft_anything_formats::firmware::{ScanOptions, scan}; +use craft_anything_formats::tarcpio::cpio; +use craft_anything_testkit::firmware as fx; + +const DATA_OFFSET: usize = 112; // 110-byte newc header plus the two-byte name "x\0". + +fn crc_archive(payload: &[u8]) -> Vec { + let mut archive = fx::cpio_newc(&[("x", payload, 0o100644)]); + assert_eq!(&archive[DATA_OFFSET..DATA_OFFSET + payload.len()], payload); + let trailer = (DATA_OFFSET + payload.len() + 3) & !3; + let sum = payload.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte))); + archive[..6].copy_from_slice(b"070702"); + archive[102..110].copy_from_slice(format!("{sum:08x}").as_bytes()); + archive[trailer..trailer + 6].copy_from_slice(b"070702"); + archive +} + +fn odc_archive() -> Vec { + let mut archive = Vec::new(); + for (name, payload) in [("plain", b"data".as_slice()), ("TRAILER!!!", b"".as_slice())] { + let header = format!( + "070707{dev:06o}{ino:06o}{mode:06o}{uid:06o}{gid:06o}{nlink:06o}{rdev:06o}{mtime:011o}{namesize:06o}{filesize:011o}", + dev = 0, + ino = 1, + mode = 0o100644, + uid = 0, + gid = 0, + nlink = 1, + rdev = 0, + mtime = 0, + namesize = name.len() + 1, + filesize = payload.len(), + ); + assert_eq!(header.len(), 76); + archive.extend_from_slice(header.as_bytes()); + archive.extend_from_slice(name.as_bytes()); + archive.push(0); + archive.extend_from_slice(payload); + } + archive +} + +#[test] +fn crc_member_checksum_requires_eight_ascii_hex_digits() { + for payload in [b"".as_slice(), &[0, 0]] { + for field in [b"+0000000", b"0000000+", b"0000000g", b" 0000000", b"0000000\0"] { + let mut archive = crc_archive(payload); + archive[102..110].copy_from_slice(field); + let listing = cpio(&archive, 10).unwrap(); + assert!(listing.complete); + assert_eq!(listing.members.len(), 1); + assert!(listing.warnings.iter().any(|warning| warning.contains("unreadable data checksum")), "{field:?}: {:?}", listing.warnings); + assert!(!listing.warnings.iter().any(|warning| warning.contains("checksum mismatch")), "invalid syntax is not a computed mismatch"); + } + } +} + +#[test] +fn crc_checksum_hex_case_and_zero_length_members_remain_valid() { + for field in [b"000001ba", b"000001BA"] { + let mut archive = crc_archive(&[255, 171, 16]); + archive[102..110].copy_from_slice(field); + let listing = cpio(&archive, 10).unwrap(); + assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}"); + } + let empty = crc_archive(&[]); + let listing = cpio(&empty, 10).unwrap(); + assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}"); +} + +#[test] +fn non_crc_variants_do_not_interpret_a_crc_field() { + let mut newc = fx::cpio_newc(&[("plain", b"data", 0o100644)]); + newc[102..110].copy_from_slice(b"+0000000"); + for archive in [newc, odc_archive()] { + let listing = cpio(&archive, 10).unwrap(); + assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}"); + let result = scan(&archive, &ScanOptions::default(), &|| false); + let region = result.regions.iter().find(|region| region.offset == 0).unwrap(); + assert_eq!(region.confidence, "high"); + assert!(result.warnings.is_empty()); + } +} + +#[test] +fn overlapping_crc_probes_share_the_scan_byte_budget() { + // Both checksums are correct. The outer member has no trailer, so its medium-confidence + // region does not hide the complete inner archive from the scanner. + let inner = crc_archive(&vec![0x5a; 8192]); + let mut image = crc_archive(&inner); + image.truncate((DATA_OFFSET + inner.len() + 3) & !3); + let plain_offset = image.len(); + image.extend_from_slice(&fx::cpio_newc(&[("plain", b"data", 0o100644)])); + let empty_offset = image.len(); + image.extend_from_slice(&crc_archive(&[])); + + let complete = cpio(&inner, 10).unwrap(); + assert!(complete.complete && complete.warnings.is_empty(), "the direct parser still checks the entire inner archive"); + let result = scan(&image, &ScanOptions::default(), &|| false); + assert!(!result.cancelled && !result.truncated); + assert!(result.warnings.iter().any(|warning| warning.contains("checksum byte budget")), "{:?}", result.warnings); + let nested = result.regions.iter().find(|region| region.offset == DATA_OFFSET as u64 && region.kind == "cpio-crc").unwrap(); + assert_eq!(nested.details["complete"], true, "the trailer is still present"); + assert_eq!(nested.confidence, "medium", "unverified member data must not gain high confidence"); + assert_eq!(nested.details["checksum_validation_skipped"], true); + let warnings = nested.details["warnings"].as_array().unwrap(); + assert!(warnings.iter().any(|warning| warning.as_str().unwrap().contains("not verified"))); + assert!(!warnings.iter().any(|warning| warning.as_str().unwrap().contains("checksum mismatch"))); + let visited: u64 = + result.regions.iter().filter(|region| region.kind == "cpio-crc").map(|region| region.details["checksum_bytes_visited"].as_u64().unwrap()).sum(); + assert_eq!(visited, image.len() as u64, "the partial probe consumes only the remaining shared allowance"); + + let plain = result.regions.iter().find(|region| region.offset == plain_offset as u64).unwrap(); + assert_eq!(plain.kind, "cpio-newc"); + assert_eq!(plain.confidence, "high"); + let empty = result.regions.iter().find(|region| region.offset == empty_offset as u64).unwrap(); + assert_eq!(empty.kind, "cpio-crc"); + assert_eq!(empty.confidence, "high"); + assert_eq!(empty.details["checksum_validation_skipped"], false, "a zero-byte member still verifies with no byte allowance left"); + assert_eq!(empty.details["checksum_bytes_visited"], 0); + assert!(empty.details["warnings"].as_array().unwrap().is_empty()); +} + +#[test] +fn crc_body_cancellation_does_not_publish_a_completed_region() { + let archive = crc_archive(&vec![7; 3 * 64 * 1024]); + let calls = Cell::new(0usize); + let cancelled = scan(&archive, &ScanOptions::default(), &|| { + calls.set(calls.get() + 1); + calls.get() >= 3 + }); + assert!(cancelled.cancelled, "the stop callback must be polled inside the checksum body"); + assert_eq!(calls.get(), 3); + assert!(cancelled.regions.is_empty(), "a cancelled archive probe must not be reported as complete"); + + let finished = scan(&archive, &ScanOptions::default(), &|| false); + assert!(!finished.cancelled); + let region = finished.regions.iter().find(|region| region.offset == 0).unwrap(); + assert_eq!(region.confidence, "high"); + assert_eq!(region.details["complete"], true); + assert!(region.details["warnings"].as_array().unwrap().is_empty()); +}