Bound CPIO checksum work across firmware probes (R3.12) (from #676)

crates/formats/src/firmware.rs

@@ -7,6 +7,7 @@
 
 use crate::bytes::Reader;
 use crate::common::entropy;
+use crate::tarcpio::CpioChecksum;
 use miniz_oxide::inflate::stream::{InflateState, inflate as mz_inflate};
 use miniz_oxide::{DataFormat, MZError, MZFlush, MZStatus};
 use serde::Serialize;
@@ -311,6 +312,10 @@ struct Scanner<'a> {
     probes: BTreeMap<&'static str, usize>,
     opts: &'a ScanOptions,
     inflate_left: u64,
+    checksum_left: usize,
+    checksum_exhausted: bool,
+    stop: &'a dyn Fn() -> bool,
+    cancelled: bool,
     exhausted: Vec<&'static str>,
 }
 
@@ -791,22 +796,53 @@ impl<'a> Scanner<'a> {
         if !matches!(self.r.u8(i + 5)?, b'1' | b'2' | b'7') || !self.probe("cpio") {
             return None;
         }
-        let l = crate::tarcpio::cpio(self.data.get(i..)?, 1_000_000)?;
+        let data = self.data;
+        let before = self.checksum_left;
+        let mut skipped = false;
+        let l = crate::tarcpio::cpio_with_checksum(data.get(i..)?, 1_000_000, &mut |body| {
+            let result = self.cpio_checksum(body);
+            skipped |= matches!(result, CpioChecksum::Skipped);
+            result
+        })?;
         if l.members.is_empty() && !l.complete {
             return None;
         }
+        let mut details = json!({"members": l.members.len(), "complete": l.complete, "warnings": l.warnings});
+        if l.format == "cpio-crc" {
+            details["checksum_validation_skipped"] = json!(skipped);
+            details["checksum_bytes_visited"] = json!(before.saturating_sub(self.checksum_left));
+        }
         Some(reg(
             i,
             Some(l.length),
             l.format,
             Family::Archive,
             format!("cpio archive ({}), {} member(s)", l.format.trim_start_matches("cpio-"), l.members.len()),
-            if l.complete { "high" } else { "medium" },
+            if l.complete && !skipped { "high" } else { "medium" },
             Unpack::Members,
-            json!({"members": l.members.len(), "complete": l.complete, "warnings": l.warnings}),
+            details,
         ))
     }
 
+    /// Sum at most one input's worth of CRC member bytes across the entire scan.
+    fn cpio_checksum(&mut self, body: &[u8]) -> CpioChecksum {
+        let mut sum = 0u32;
+        for chunk in body.chunks(64 << 10) {
+            if (self.stop)() {
+                self.cancelled = true;
+                return CpioChecksum::Cancelled;
+            }
+            let count = chunk.len().min(self.checksum_left);
+            sum = chunk.iter().take(count).fold(sum, |sum, &byte| sum.wrapping_add(u32::from(byte)));
+            self.checksum_left = self.checksum_left.saturating_sub(count);
+            if count < chunk.len() {
+                self.checksum_exhausted = true;
+                return CpioChecksum::Skipped;
+            }
+        }
+        CpioChecksum::Value(sum)
+    }
+
     fn squashfs(&mut self, i: usize, be: bool) -> Option<Region> {
         let r = self.r;
         let u16f = |o: usize| if be { r.u16_be(i + o) } else { r.u16_le(i + o) };
@@ -1499,11 +1535,23 @@ impl<'a> Scanner<'a> {
     }
 }
 
-/// Scan `data` for recognised regions. `stop` is polled every MiB; when it returns true the scan
-/// ends early with `cancelled` set. A region with a confirmed length hides its interior from the
-/// scan (nested content is found by scanning the region again after extraction).
+/// Scan `data` for recognised regions. `stop` is polled every MiB and every 64 KiB during CPIO
+/// checksum sums; when it returns true the scan ends early with `cancelled` set. CRC member sums
+/// share a byte allowance equal to the input length; skipped verification is reported. A region
+/// with a confirmed length hides its interior (nested content is found after extraction).
 pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanResult {
-    let mut s = Scanner { data, r: Reader::new(data), probes: BTreeMap::new(), opts, inflate_left: opts.inflate_budget, exhausted: Vec::new() };
+    let mut s = Scanner {
+        data,
+        r: Reader::new(data),
+        probes: BTreeMap::new(),
+        opts,
+        inflate_left: opts.inflate_budget,
+        checksum_left: data.len(),
+        checksum_exhausted: false,
+        stop,
+        cancelled: false,
+        exhausted: Vec::new(),
+    };
     let mut out = ScanResult::default();
     let mut skip_until = 0usize;
     let mut i = 0usize;
@@ -1513,6 +1561,10 @@ pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanRes
             break;
         }
         let found = s.try_backref(i).filter(|r| r.offset as usize >= skip_until).or_else(|| s.try_at(i));
+        if s.cancelled {
+            out.cancelled = true;
+            break;
+        }
         if let Some(r) = found {
             if out.regions.len() >= opts.max_regions {
                 out.truncated = true;
@@ -1538,6 +1590,9 @@ pub fn scan(data: &[u8], opts: &ScanOptions, stop: &dyn Fn() -> bool) -> ScanRes
     if !out.probes_exhausted.is_empty() {
         out.warnings.push(format!("probe budget spent for: {} (later candidates were not checked)", out.probes_exhausted.join(", ")));
     }
+    if s.checksum_exhausted {
+        out.warnings.push("cpio CRC checksum byte budget spent (some member data was not verified)".into());
+    }
     out
 }
 

crates/formats/src/tarcpio.rs

@@ -275,6 +275,19 @@ fn mode_kind(mode: u32) -> MemberKind {
 /// Walk a cpio archive (newc `070701`, crc `070702`, odc `070707`). `None` when the first header
 /// is not one of these.
 pub fn cpio(data: &[u8], max_members: usize) -> Option<Listing> {
+    cpio_with_checksum(data, max_members, &mut |body| CpioChecksum::Value(body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)))))
+}
+
+/// A checksum sum, or the reason a caller stopped verification before completing it.
+#[derive(Clone, Copy)]
+pub(crate) enum CpioChecksum {
+    Value(u32),
+    Skipped,
+    Cancelled,
+}
+
+/// Let a scanner share its checksum work allowance and cancellation across overlapping probes.
+pub(crate) fn cpio_with_checksum(data: &[u8], max_members: usize, checksum: &mut dyn FnMut(&[u8]) -> CpioChecksum) -> Option<Listing> {
     let r = Reader::new(data);
     let format = match r.slice(0, 6)? {
         b"070701" => "cpio-newc",
@@ -341,12 +354,17 @@ pub fn cpio(data: &[u8], max_members: usize) -> Option<Listing> {
             break;
         }
         if format == "cpio-crc" {
-            match h.get(102..110).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) {
+            match h.get(102..110).filter(|field| field.iter().all(u8::is_ascii_hexdigit)).and_then(hex_field).and_then(|n| u32::try_from(n).ok()) {
                 Some(expected) => {
                     if let Some(body) = r.slice(data_off, size) {
-                        let actual = body.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)));
-                        if actual != expected {
-                            out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})"));
+                        match checksum(body) {
+                            CpioChecksum::Value(actual) => {
+                                if actual != expected {
+                                    out.warnings.push(format!("{name}: data checksum mismatch (stored {expected:08x}, computed {actual:08x})"));
+                                }
+                            }
+                            CpioChecksum::Skipped => out.warnings.push(format!("{name}: data checksum not verified (scan byte budget exhausted)")),
+                            CpioChecksum::Cancelled => return None,
                         }
                     }
                 }

crates/formats/tests/cpio_crc_scan_limits.rs

@@ -0,0 +1,144 @@
+use std::cell::Cell;
+
+use craft_anything_formats::firmware::{ScanOptions, scan};
+use craft_anything_formats::tarcpio::cpio;
+use craft_anything_testkit::firmware as fx;
+
+const DATA_OFFSET: usize = 112; // 110-byte newc header plus the two-byte name "x\0".
+
+fn crc_archive(payload: &[u8]) -> Vec<u8> {
+    let mut archive = fx::cpio_newc(&[("x", payload, 0o100644)]);
+    assert_eq!(&archive[DATA_OFFSET..DATA_OFFSET + payload.len()], payload);
+    let trailer = (DATA_OFFSET + payload.len() + 3) & !3;
+    let sum = payload.iter().fold(0u32, |sum, &byte| sum.wrapping_add(u32::from(byte)));
+    archive[..6].copy_from_slice(b"070702");
+    archive[102..110].copy_from_slice(format!("{sum:08x}").as_bytes());
+    archive[trailer..trailer + 6].copy_from_slice(b"070702");
+    archive
+}
+
+fn odc_archive() -> Vec<u8> {
+    let mut archive = Vec::new();
+    for (name, payload) in [("plain", b"data".as_slice()), ("TRAILER!!!", b"".as_slice())] {
+        let header = format!(
+            "070707{dev:06o}{ino:06o}{mode:06o}{uid:06o}{gid:06o}{nlink:06o}{rdev:06o}{mtime:011o}{namesize:06o}{filesize:011o}",
+            dev = 0,
+            ino = 1,
+            mode = 0o100644,
+            uid = 0,
+            gid = 0,
+            nlink = 1,
+            rdev = 0,
+            mtime = 0,
+            namesize = name.len() + 1,
+            filesize = payload.len(),
+        );
+        assert_eq!(header.len(), 76);
+        archive.extend_from_slice(header.as_bytes());
+        archive.extend_from_slice(name.as_bytes());
+        archive.push(0);
+        archive.extend_from_slice(payload);
+    }
+    archive
+}
+
+#[test]
+fn crc_member_checksum_requires_eight_ascii_hex_digits() {
+    for payload in [b"".as_slice(), &[0, 0]] {
+        for field in [b"+0000000", b"0000000+", b"0000000g", b" 0000000", b"0000000\0"] {
+            let mut archive = crc_archive(payload);
+            archive[102..110].copy_from_slice(field);
+            let listing = cpio(&archive, 10).unwrap();
+            assert!(listing.complete);
+            assert_eq!(listing.members.len(), 1);
+            assert!(listing.warnings.iter().any(|warning| warning.contains("unreadable data checksum")), "{field:?}: {:?}", listing.warnings);
+            assert!(!listing.warnings.iter().any(|warning| warning.contains("checksum mismatch")), "invalid syntax is not a computed mismatch");
+        }
+    }
+}
+
+#[test]
+fn crc_checksum_hex_case_and_zero_length_members_remain_valid() {
+    for field in [b"000001ba", b"000001BA"] {
+        let mut archive = crc_archive(&[255, 171, 16]);
+        archive[102..110].copy_from_slice(field);
+        let listing = cpio(&archive, 10).unwrap();
+        assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");
+    }
+    let empty = crc_archive(&[]);
+    let listing = cpio(&empty, 10).unwrap();
+    assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");
+}
+
+#[test]
+fn non_crc_variants_do_not_interpret_a_crc_field() {
+    let mut newc = fx::cpio_newc(&[("plain", b"data", 0o100644)]);
+    newc[102..110].copy_from_slice(b"+0000000");
+    for archive in [newc, odc_archive()] {
+        let listing = cpio(&archive, 10).unwrap();
+        assert!(listing.complete && listing.warnings.is_empty(), "{listing:?}");
+        let result = scan(&archive, &ScanOptions::default(), &|| false);
+        let region = result.regions.iter().find(|region| region.offset == 0).unwrap();
+        assert_eq!(region.confidence, "high");
+        assert!(result.warnings.is_empty());
+    }
+}
+
+#[test]
+fn overlapping_crc_probes_share_the_scan_byte_budget() {
+    // Both checksums are correct. The outer member has no trailer, so its medium-confidence
+    // region does not hide the complete inner archive from the scanner.
+    let inner = crc_archive(&vec![0x5a; 8192]);
+    let mut image = crc_archive(&inner);
+    image.truncate((DATA_OFFSET + inner.len() + 3) & !3);
+    let plain_offset = image.len();
+    image.extend_from_slice(&fx::cpio_newc(&[("plain", b"data", 0o100644)]));
+    let empty_offset = image.len();
+    image.extend_from_slice(&crc_archive(&[]));
+
+    let complete = cpio(&inner, 10).unwrap();
+    assert!(complete.complete && complete.warnings.is_empty(), "the direct parser still checks the entire inner archive");
+    let result = scan(&image, &ScanOptions::default(), &|| false);
+    assert!(!result.cancelled && !result.truncated);
+    assert!(result.warnings.iter().any(|warning| warning.contains("checksum byte budget")), "{:?}", result.warnings);
+    let nested = result.regions.iter().find(|region| region.offset == DATA_OFFSET as u64 && region.kind == "cpio-crc").unwrap();
+    assert_eq!(nested.details["complete"], true, "the trailer is still present");
+    assert_eq!(nested.confidence, "medium", "unverified member data must not gain high confidence");
+    assert_eq!(nested.details["checksum_validation_skipped"], true);
+    let warnings = nested.details["warnings"].as_array().unwrap();
+    assert!(warnings.iter().any(|warning| warning.as_str().unwrap().contains("not verified")));
+    assert!(!warnings.iter().any(|warning| warning.as_str().unwrap().contains("checksum mismatch")));
+    let visited: u64 =
+        result.regions.iter().filter(|region| region.kind == "cpio-crc").map(|region| region.details["checksum_bytes_visited"].as_u64().unwrap()).sum();
+    assert_eq!(visited, image.len() as u64, "the partial probe consumes only the remaining shared allowance");
+
+    let plain = result.regions.iter().find(|region| region.offset == plain_offset as u64).unwrap();
+    assert_eq!(plain.kind, "cpio-newc");
+    assert_eq!(plain.confidence, "high");
+    let empty = result.regions.iter().find(|region| region.offset == empty_offset as u64).unwrap();
+    assert_eq!(empty.kind, "cpio-crc");
+    assert_eq!(empty.confidence, "high");
+    assert_eq!(empty.details["checksum_validation_skipped"], false, "a zero-byte member still verifies with no byte allowance left");
+    assert_eq!(empty.details["checksum_bytes_visited"], 0);
+    assert!(empty.details["warnings"].as_array().unwrap().is_empty());
+}
+
+#[test]
+fn crc_body_cancellation_does_not_publish_a_completed_region() {
+    let archive = crc_archive(&vec![7; 3 * 64 * 1024]);
+    let calls = Cell::new(0usize);
+    let cancelled = scan(&archive, &ScanOptions::default(), &|| {
+        calls.set(calls.get() + 1);
+        calls.get() >= 3
+    });
+    assert!(cancelled.cancelled, "the stop callback must be polled inside the checksum body");
+    assert_eq!(calls.get(), 3);
+    assert!(cancelled.regions.is_empty(), "a cancelled archive probe must not be reported as complete");
+
+    let finished = scan(&archive, &ScanOptions::default(), &|| false);
+    assert!(!finished.cancelled);
+    let region = finished.regions.iter().find(|region| region.offset == 0).unwrap();
+    assert_eq!(region.confidence, "high");
+    assert_eq!(region.details["complete"], true);
+    assert!(region.details["warnings"].as_array().unwrap().is_empty());
+}