diff --git a/Cargo.lock b/Cargo.lock index 11648c2..0138456 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aes" version = "0.8.4" @@ -132,6 +138,7 @@ dependencies = [ "liblzma", "lz4_flex", "memchr", + "miniz_oxide 0.8.0", "num-bigint-dig", "num-traits", "phf", @@ -613,7 +620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.7.4", ] [[package]] @@ -953,6 +960,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "multimap" version = "0.10.0" diff --git a/avbroot/Cargo.toml b/avbroot/Cargo.toml index c0b9fa3..d4c5d99 100644 --- a/avbroot/Cargo.toml +++ b/avbroot/Cargo.toml @@ -27,6 +27,7 @@ hex = { version = "0.4.3", features = ["serde"] } liblzma = "0.3.0" lz4_flex = "0.11.1" memchr = "2.6.0" +miniz_oxide = "0.8.0" num-bigint-dig = "0.8.4" num-traits = "0.2.16" phf = { version = "0.11.2", features = ["macros"] } diff --git a/avbroot/src/cli/ota.rs b/avbroot/src/cli/ota.rs index d0eb9bf..27c6ab8 100644 --- a/avbroot/src/cli/ota.rs +++ b/avbroot/src/cli/ota.rs @@ -34,7 +34,7 @@ use crate::{ avb::{self, Descriptor, Header}, ota::{self, SigningWriter, ZipEntry}, padding, - payload::{self, PayloadHeader, PayloadWriter}, + payload::{self, PayloadHeader, PayloadWriter, VabcAlgo}, }, patch::{ boot::{ @@ -676,12 +676,11 @@ pub fn compress_image( // Otherwise, compress the entire image. If VABC is enabled, we need to // update the CoW size estimate or else the CoW block device may run out of // space during flashing. - let need_cow = partition.estimate_cow_size.is_some(); - if need_cow { + let vabc_algo = if partition.estimate_cow_size.is_some() { info!("Needs updated CoW size estimate: {name}"); - // Only CoW v2 + lz4 seems to exist in the wild currently, so that is - // all we support. + // Only CoW v2 seems to exist in the wild currently, so that is all we + // support. let Some(dpm) = &header.manifest.dynamic_partition_metadata else { bail!("Dynamic partition metadata is missing"); }; @@ -696,13 +695,17 @@ pub fn compress_image( } let compression = dpm.vabc_compression_param(); - if compression != "lz4" { + let Some(vabc_algo) = VabcAlgo::new(compression) else { bail!("Unsupported VABC compression: {compression}"); - } - } + }; + + Some(vabc_algo) + } else { + None + }; let (partition_info, operations, cow_estimate) = - payload::compress_image(&*file, &writer, name, block_size, need_cow, cancel_signal)?; + payload::compress_image(&*file, &writer, name, block_size, vabc_algo, cancel_signal)?; partition.new_partition_info = Some(partition_info); partition.operations = operations; diff --git a/avbroot/src/format/payload.rs b/avbroot/src/format/payload.rs index da827c0..29e0683 100644 --- a/avbroot/src/format/payload.rs +++ b/avbroot/src/format/payload.rs @@ -5,6 +5,7 @@ use std::{ collections::{HashMap, HashSet}, + fmt, io::{self, Cursor, Read, Seek, SeekFrom, Write}, ops::Range, sync::atomic::AtomicBool, @@ -887,19 +888,50 @@ fn compress_chunk(raw_data: &[u8], cancel_signal: &AtomicBool) -> Result<(Vec u64 { - let mut total = 0; +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +pub enum VabcAlgo { + Lz4, + Gzip, +} - while !raw_data.is_empty() { - let n = raw_data.len().min(block_size as usize); - let compressed = lz4_flex::block::compress(&raw_data[..n]); - - total += compressed.len().min(n) as u64; - - raw_data = &raw_data[n..]; +impl VabcAlgo { + pub fn new(name: &str) -> Option { + match name { + "lz4" => Some(Self::Lz4), + "gz" => Some(Self::Gzip), + _ => None, + } } - total + fn compressed_size(&self, mut raw_data: &[u8], block_size: u32) -> u64 { + let mut total = 0; + + while !raw_data.is_empty() { + let n = raw_data.len().min(block_size as usize); + let compressed = match self { + Self::Lz4 => lz4_flex::block::compress(&raw_data[..n]), + // We use the miniz_oxide backend for flate2, but flate2 doesn't + // expose a nice function for compressing to a vec, so just use + // miniz_oxide directly. + Self::Gzip => miniz_oxide::deflate::compress_to_vec_zlib(&raw_data[..n], 9), + }; + + total += compressed.len().min(n) as u64; + + raw_data = &raw_data[n..]; + } + + total + } +} + +impl fmt::Display for VabcAlgo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Lz4 => f.write_str("lz4"), + Self::Gzip => f.write_str("gz"), + } + } } /// Compress the image and return the corresponding information to insert into @@ -910,18 +942,18 @@ fn compress_cow_size(mut raw_data: &[u8], block_size: u32) -> u64 { /// update [`InstallOperation::data_offset`] in each operation manually because /// the initial values are relative to 0. /// -/// If `need_cow_estimate` is true, the VABC CoW v2 + lz4 size estimate will be -/// computed. The caller must update [`PartitionUpdate::estimate_cow_size`] with -/// this value or else update_engine may fail to flash the partition due to -/// running out of space on the CoW block device. CoW v2 + other algorithms and -/// also CoW v3 are currently unsupported because there currently are no known -/// OTAs that use those configurations. +/// If `vabc_algo` is set, the VABC CoW v2 size estimate will be computed. The +/// caller must update [`PartitionUpdate::estimate_cow_size`] with this value or +/// else update_engine may fail to flash the partition due to running out of +/// space on the CoW block device. CoW v2 + other algorithms and also CoW v3 are +/// currently unsupported because there currently are no known OTAs that use +/// those configurations. pub fn compress_image( input: &(dyn ReadSeekReopen + Sync), output: &(dyn WriteSeekReopen + Sync), partition_name: &str, block_size: u32, - need_cow_estimate: bool, + vabc_algo: Option, cancel_signal: &AtomicBool, ) -> Result<(PartitionInfo, Vec, Option)> { const CHUNK_SIZE: u64 = 2 * 1024 * 1024; @@ -980,8 +1012,8 @@ pub fn compress_image( .map( |(raw_offset, raw_data)| -> Result<(Vec, InstallOperation, u64)> { let (data, digest_compressed) = compress_chunk(&raw_data, cancel_signal)?; - let cow_size = if need_cow_estimate { - compress_cow_size(&raw_data, block_size) + let cow_size = if let Some(algo) = vabc_algo { + algo.compressed_size(&raw_data, block_size) } else { 0 }; @@ -1028,11 +1060,25 @@ pub fn compress_image( hash: Some(digest_uncompressed.as_ref().to_vec()), }; - let cow_estimate = if need_cow_estimate { - // Because lz4_flex compresses better than official lz4. - let fudge = cow_estimate / 100; + let cow_estimate = if vabc_algo.is_some() { + // lz4_flex and miniz_oxide usually compress better than the lz4 and + // zlib implementations used by libsnapshot_cow. Make up for this by + // adding percentage-based overhead. + cow_estimate += cow_estimate / 100; - Some(cow_estimate + fudge) + // We also need to account for constant overhead, especially with + // smaller partitions. We can match what delta_generator normally adds + // in CowWriterV2::InitPos() exactly. Since we only ever create full + // OTAs, we can assume that all CoW operations are kCowReplaceOp. + + // sizeof(CowHeader). + cow_estimate += 38; + // header_.buffer_size (equal to BUFFER_REGION_DEFAULT_SIZE). + cow_estimate += 2 * 1024 * 1024; + // CowOptions::cluster_ops * sizeof(CowOperationV2). + cow_estimate += 200 * 20; + + Some(cow_estimate) } else { None }; diff --git a/e2e/e2e.toml b/e2e/e2e.toml index 0222166..6a4da1a 100644 --- a/e2e/e2e.toml +++ b/e2e/e2e.toml @@ -12,6 +12,9 @@ security_patch_level = "2024-01-01" # Google Pixel 7 Pro # What's unique: init_boot (boot v4) + vendor_boot (vendor v4) +[profile.pixel_v4_gki] +vabc_algo = "Lz4" + [profile.pixel_v4_gki.partitions.boot] avb.signed = true data.type = "boot" @@ -46,12 +49,15 @@ data.version = "vendor_v4" data.ramdisks = [["otacerts", "first_stage", "dsu_key_dir"]] [profile.pixel_v4_gki.hashes] -original = "6b140c378d21eae2fa4fc581bce13a689b21bd32f5fba865698d1fd322f2f8c6" -patched = "a68b3a44c0cf015a225f92837c05fd0dec6e159584c5880eff30d12daa7123ff" +original = "c00f891f941f3dddb28966f7b07f3acea773bee104dace82b37c2d1341f09422" +patched = "ce9d8ee97828d233809742a5d3f23aa27b042675b1935ca9e3df0592c55788fd" # Google Pixel 6a # What's unique: boot (boot v4, no ramdisk) + vendor_boot (vendor v4, 2 ramdisks) +[profile.pixel_v4_non_gki] +vabc_algo = "Lz4" + [profile.pixel_v4_non_gki.partitions.boot] avb.signed = true data.type = "boot" @@ -80,12 +86,15 @@ data.version = "vendor_v4" data.ramdisks = [["init", "otacerts", "first_stage", "dsu_key_dir"], ["dlkm"]] [profile.pixel_v4_non_gki.hashes] -original = "31963e6f81986c6686111f50e36b89e4d85ee5c02bc8e5ecd560528bc98d6fe7" -patched = "a13173a006ad94b25db682bb14fde2e36891417727d6541bdf5f9bca57d26751" +original = "4d692bc777b568b0626d3c08d2e6f83f1b472db5ad903486daaec6a78d0cc26e" +patched = "e27673e4f30933710c11d51f0e73849068cbe9bc9f54e6076bdd93f9a5c8ea0a" # Google Pixel 4a 5G # What's unique: boot (boot v3) + vendor_boot (vendor v3) +[profile.pixel_v3] +vabc_algo = "Lz4" + [profile.pixel_v3.partitions.boot] avb.signed = true data.type = "boot" @@ -115,12 +124,15 @@ data.version = "vendor_v3" data.ramdisks = [["otacerts", "first_stage", "dsu_key_dir"]] [profile.pixel_v3.hashes] -original = "e684aacb54464098c1b8e3f499efe35dff10ea792e89d71a83404620d0108b3e" -patched = "49e810ae76154bccf2dc7d810b9eec19d23a5b8fa32381469660488d0a25121b" +original = "f432dc7931520feb238474aa707dd5299747562ffe6129f3f763b5f11ac473ab" +patched = "3850a2e73bd783a1ec4a70c59f37d2374e017c20df7ab4b591182b14d187c18e" # Google Pixel 4a # What's unique: boot (boot v2) +[profile.pixel_v2] +vabc_algo = "Gzip" + [profile.pixel_v2.partitions.boot] avb.signed = false data.type = "boot" @@ -144,5 +156,5 @@ data.type = "vbmeta" data.deps = ["system"] [profile.pixel_v2.hashes] -original = "ee9568797d9195985f14753b89949d8ebb08c8863a32eceeeec6e8d94661b1cf" -patched = "e413f1f87ee5ba53d402edad03b0df8af451b5b0323202a9d32b8327d433d340" +original = "1b45235b58054009cc496f6c3ee11d3dc16ed5c388c861761e26a6fce83103a0" +patched = "193b2dc70dd465d686f35c7b7f74d2cc1b06a55e48cf5c2e4df0f667e03032fc" diff --git a/e2e/src/config.rs b/e2e/src/config.rs index 779b206..522aa89 100644 --- a/e2e/src/config.rs +++ b/e2e/src/config.rs @@ -6,6 +6,7 @@ use std::{collections::BTreeMap, fs, path::Path}; use anyhow::{Context, Result}; +use avbroot::format::payload::VabcAlgo; use serde::{Deserialize, Serialize}; use toml_edit::DocumentMut; @@ -109,6 +110,7 @@ pub struct Partition { #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Profile { + pub vabc_algo: Option, pub partitions: BTreeMap, pub hashes: Hashes, } diff --git a/e2e/src/main.rs b/e2e/src/main.rs index 5a9e1e9..bae8a94 100644 --- a/e2e/src/main.rs +++ b/e2e/src/main.rs @@ -577,6 +577,7 @@ fn create_payload( partitions: &BTreeMap, inputs: &BTreeMap, ota_info: &OtaInfo, + profile: &Profile, key_ota: &RsaSigningKey, cancel_signal: &AtomicBool, ) -> Result<(String, u64)> { @@ -594,14 +595,14 @@ fn create_payload( .map(PSeekFile::new) .with_context(|| format!("Failed to create temp file for: {name}"))?; - let (partition_info, operations, cow_estimate) = payload::compress_image( - file, - &writer, - name, - 4096, - dynamic_partitions_names.contains(name), - cancel_signal, - )?; + let vabc_algo = if dynamic_partitions_names.contains(name) { + profile.vabc_algo + } else { + None + }; + + let (partition_info, operations, cow_estimate) = + payload::compress_image(file, &writer, name, 4096, vabc_algo, cancel_signal)?; compressed.insert(name, writer); @@ -645,7 +646,7 @@ fn create_payload( }], snapshot_enabled: Some(true), vabc_enabled: Some(true), - vabc_compression_param: Some("lz4".to_owned()), + vabc_compression_param: profile.vabc_algo.map(|a| a.to_string()), cow_version: Some(2), vabc_feature_set: None, }), @@ -746,6 +747,7 @@ fn create_ota( &profile.partitions, &inputs, ota_info, + profile, key_ota, cancel_signal, )