diff --git a/avbroot/src/cli/ota.rs b/avbroot/src/cli/ota.rs index 240ba3f..f4bf0db 100644 --- a/avbroot/src/cli/ota.rs +++ b/avbroot/src/cli/ota.rs @@ -32,7 +32,7 @@ use crate::{ avb::{self, Descriptor, Header}, ota::{self, SigningWriter, ZipEntry, ZipMode}, padding, - payload::{self, PayloadHeader, PayloadWriter, VabcAlgo}, + payload::{self, CowVersion, PayloadHeader, PayloadWriter, VabcAlgo, VabcParams}, }, patch::{ boot::{ @@ -576,9 +576,10 @@ fn update_metadata_descriptors(parent_header: &mut Header, child_header: &Header } } -/// Get the VABC algorithm from the payload header. This will fail if an -/// unsupported VABC algorithm is specified, but not if VABC is disabled. -fn get_vabc_algo(header: &PayloadHeader) -> Result> { +/// Get the VABC parameters from the payload header. This will fail if an +/// unsupported VABC algorithm or CoW version is specified, but not if VABC is +/// disabled. +fn get_vabc_params(header: &PayloadHeader) -> Result> { // Only CoW v2 seems to exist in the wild currently, so that is all we // support. let Some(dpm) = &header.manifest.dynamic_partition_metadata else { @@ -589,17 +590,32 @@ fn get_vabc_algo(header: &PayloadHeader) -> Result> { return Ok(None); } - let cow_version = dpm.cow_version(); - if dpm.cow_version() != 2 { - bail!("Unsupported CoW version: {cow_version}"); - } + let cow_version = match dpm.cow_version() { + 2 => CowVersion::V2, + 3 => CowVersion::V3, + v => bail!("Unsupported CoW version: {v}"), + }; let compression = dpm.vabc_compression_param(); let Ok(vabc_algo) = VabcAlgo::from_str(compression, false) else { bail!("Unsupported VABC compression: {compression}"); }; - Ok(Some(vabc_algo)) + // This is unused by v2, but delta_generator sets it anyway. + let Some(compression_factor) = dpm.compression_factor else { + bail!("No CoW compression factor specified"); + }; + let Ok(compression_factor) = u32::try_from(compression_factor) else { + bail!("CoW compression factor is too large: {compression_factor}"); + }; + + let vabc_params = VabcParams { + version: cow_version, + algo: vabc_algo, + compression_factor, + }; + + Ok(Some(vabc_params)) } /// Set the VABC algorithm in the payload header and return whether it was @@ -731,7 +747,7 @@ pub fn compress_image( .map(PSeekFile::new) .with_context(|| format!("Failed to create temp file for: {name}"))?; - let vabc_algo = get_vabc_algo(header)?; + let vabc_params = get_vabc_params(header)?; let block_size = header.manifest.block_size(); let partition = header .manifest @@ -742,14 +758,17 @@ pub fn compress_image( // If VABC is enabled, we need to update the CoW size estimate or else the // CoW block device may run out of space during flashing. - let vabc_algo = if partition.estimate_cow_size.is_some() { - let Some(vabc_algo) = vabc_algo else { + let vabc_params = if partition.estimate_cow_size.is_some() { + let Some(vabc_params) = vabc_params else { bail!("Partition has CoW estimate, but VABC is disabled: {name}"); }; - info!("Needs updated {vabc_algo} CoW size estimate: {name}"); + info!( + "Needs updated {} CoW size estimate: {name}", + vabc_params.algo, + ); - Some(vabc_algo) + Some(vabc_params) } else { None }; @@ -770,16 +789,19 @@ pub fn compress_image( // The changes we make usually aren't any less compressible, but // we'll still recompute the CoW size estimate to handle the // case where the user requested a different algorithm. - if let Some(vabc_algo) = vabc_algo { + if let Some(vabc_params) = vabc_params { let cow_estimate = payload::compute_cow_estimate( &*file, + partition.operations.len() as u64, name, block_size, - vabc_algo, + vabc_params, cancel_signal, )?; - partition.estimate_cow_size = Some(cow_estimate); + partition.estimate_cow_size = Some(cow_estimate.size); + partition.estimate_op_count_max = + (vabc_params.version == CowVersion::V3).then_some(cow_estimate.num_ops); } *file = writer; @@ -797,12 +819,22 @@ pub fn compress_image( info!("Compressing full image: {name}"); - let (partition_info, operations, cow_estimate) = - payload::compress_image(&*file, &writer, name, block_size, vabc_algo, cancel_signal)?; + let (partition_info, operations, cow_estimate) = payload::compress_image( + &*file, + &writer, + name, + block_size, + vabc_params, + cancel_signal, + )?; partition.new_partition_info = Some(partition_info); partition.operations = operations; - partition.estimate_cow_size = cow_estimate; + partition.estimate_cow_size = cow_estimate.map(|e| e.size); + let is_v3 = vabc_params + .map(|p| p.version == CowVersion::V3) + .unwrap_or_default(); + partition.estimate_op_count_max = cow_estimate.and_then(|e| is_v3.then_some(e.num_ops)); *file = writer; @@ -822,7 +854,7 @@ fn recow_image( file.rewind()?; - let vabc_algo = get_vabc_algo(header)?; + let vabc_params = get_vabc_params(header)?; let block_size = header.manifest.block_size(); let partition = header .manifest @@ -835,16 +867,24 @@ fn recow_image( bail!("Partition has no original CoW estimate: {name}"); }; - let Some(vabc_algo) = vabc_algo else { + let Some(vabc_params) = vabc_params else { bail!("Partition has CoW estimate, but VABC is disabled: {name}"); }; - info!("Recomputing {vabc_algo} CoW size estimate: {name}"); + info!("Recomputing {} CoW size estimate: {name}", vabc_params.algo); - let cow_estimate = - payload::compute_cow_estimate(&*file, name, block_size, vabc_algo, cancel_signal)?; + let cow_estimate = payload::compute_cow_estimate( + &*file, + partition.operations.len() as u64, + name, + block_size, + vabc_params, + cancel_signal, + )?; - partition.estimate_cow_size = Some(cow_estimate); + partition.estimate_cow_size = Some(cow_estimate.size); + partition.estimate_op_count_max = + (vabc_params.version == CowVersion::V3).then_some(cow_estimate.num_ops); Ok(()) } diff --git a/avbroot/src/format/payload.rs b/avbroot/src/format/payload.rs index 94d8ce9..d68d3bc 100644 --- a/avbroot/src/format/payload.rs +++ b/avbroot/src/format/payload.rs @@ -5,7 +5,8 @@ use std::{ collections::{HashMap, HashSet}, fmt, io::{self, Cursor, Read, Seek, SeekFrom, Write}, - ops::Range, + num::NonZeroU32, + ops::{Add, Range}, sync::atomic::AtomicBool, }; @@ -19,6 +20,7 @@ use liblzma::{ write::XzDecoder, write::XzEncoder, }; +use num_traits::CheckedAdd; use prost::Message; use rayon::{ iter::{IndexedParallelIterator, IntoParallelRefMutIterator}, @@ -49,7 +51,9 @@ const PAYLOAD_VERSION: u64 = 2; const MANIFEST_MAX_SIZE: usize = 4 * 1024 * 1024; -/// Size of each extent. This matches what AOSP's delta_generator does. +/// Size of each extent. This matches what AOSP's delta_generator does. We also +/// require this to be a multiple of the block size and a multiple of the +/// maximum CoW compression chunk size. const CHUNK_SIZE: u64 = 2 * 1024 * 1024; #[derive(Debug, Error)] @@ -81,6 +85,10 @@ pub enum Error { expected: Option, actual: String, }, + #[error("Invalid block size: {0}")] + InvalidBlockSize(u32), + #[error("Invalid maximum CoW compression chunk size: {0}")] + InvalidMaxCompressionChunkSize(u32), #[error("Size of {name} ({size}) is not aligned to the block size ({block_size})")] InvalidPartitionSize { name: String, @@ -926,6 +934,8 @@ pub fn extract_images<'a>( .collect() } +/// Compress raw data into a chunk to be used with a [`Type::ReplaceXz`] +/// [`InstallOperation`]. fn compress_chunk(raw_data: &[u8], cancel_signal: &AtomicBool) -> Result<(Vec, Digest)> { let reader = Cursor::new(raw_data); let writer = Cursor::new(Vec::new()); @@ -949,6 +959,170 @@ fn compress_chunk(raw_data: &[u8], cancel_signal: &AtomicBool) -> Result<(Vec u64 { + const BUFFER_REGION_DEFAULT_SIZE: u64 = 2 * 1024 * 1024; + const CLUSTER_OPS: u64 = 200; + const NUM_RESUME_POINTS: u64 = 4; + const SIZEOF_COW_FOOTER_V2: u64 = 84; + const SIZEOF_COW_HEADER_V2: u64 = 38; + const SIZEOF_COW_HEADER_V3: u64 = 40; + const SIZEOF_COW_OPERATION_V2: u64 = 20; + const SIZEOF_COW_OPERATION_V3: u64 = 16; + const SIZEOF_RESUME_POINT_V3: u64 = 16; + + let mut overhead = 0; + + match self { + Self::V2 => { + // sizeof(CowHeader). + // AOSP: CowWriterV2::InitPos() + overhead += SIZEOF_COW_HEADER_V2; + + // header_.buffer_size. update_engine uses the default value. + // AOSP: CowWriterV2::InitPos() + overhead += BUFFER_REGION_DEFAULT_SIZE; + + // Add an operation header (sizeof(CowOperationV2)) for each + // kCowReplaceOp for the raw data. + // AOSP: CowWriterV2::EmitClusterIfNeeded() + overhead += cow_replace_ops * SIZEOF_COW_OPERATION_V2; + + // Add an operation header (sizeof(CowOperationV2)) for each + // kCowLabelOp. delta_generator adds one for each install + // operation. + // AOSP: CowDryRun() + overhead += payload_install_ops * SIZEOF_COW_OPERATION_V2; + + // Add an operation header (sizeof(CowOperationV2)) for each + // kCowClusterOp, which is emitted for each cluster of cow + // operations. The cluster size used to be 200, but changed to + // 1024 in 5e8e488c13cbff9e0a305ce7c22fd6a13aabb886. We'll use + // the smaller value because it's better to overestimate. + // AOSP: CowWriterV2::EmitClusterIfNeeded() + let num_ops = (cow_replace_ops + payload_install_ops).div_ceil(CLUSTER_OPS - 1); + overhead += num_ops * SIZEOF_COW_OPERATION_V2; + + // sizeof(CowFooter). + // AOSP: CowWriterV2::GetCowSizeInfo() + overhead += SIZEOF_COW_FOOTER_V2; + } + Self::V3 => { + // AOSP: CowWriterV3::OpenForWrite() -> GetDataOffset() + overhead += SIZEOF_COW_HEADER_V3; + overhead += BUFFER_REGION_DEFAULT_SIZE; + overhead += NUM_RESUME_POINTS * SIZEOF_RESUME_POINT_V3; + + // Add an operation header (sizeof(CowOperationV3)) for each + // chunk of compressed data. + // AOSP: CowWriterV3::WriteOperation() + overhead += cow_replace_ops * SIZEOF_COW_OPERATION_V3; + } + } + + overhead + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ChunkingMethod { + /// Compress data in block sized chunks each iteration. + Exact, + /// Compress data in chunks where each chunk is sized at the largest power + /// of 2 that's `<=` the specified size and the remaining input size. + MaxPowerOf2(NonZeroU32), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct ChunkingParams { + block_size: u32, + method: ChunkingMethod, +} + +impl ChunkingParams { + fn chunk_size(&self, num_blocks: u64) -> u32 { + match self.method { + ChunkingMethod::Exact => self.block_size, + ChunkingMethod::MaxPowerOf2(max_chunk_size) => { + assert!( + max_chunk_size.is_power_of_two() && max_chunk_size.get() % self.block_size == 0 + ); + + let mut chunk_size = max_chunk_size.get(); + while chunk_size > self.block_size { + let min_blocks = chunk_size / self.block_size; + if num_blocks >= u64::from(min_blocks) { + return chunk_size; + } + + chunk_size >>= 1; + } + + self.block_size + } + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CowEstimate { + /// Size of estimate in bytes. + pub size: u64, + /// Number of CoW operations (v3 only). + pub num_ops: u64, +} + +impl CowEstimate { + /// Add fudge factor to account for overhead. + fn fudged( + &self, + payload_install_ops: u64, + cow_version: CowVersion, + vabc_algo: VabcAlgo, + ) -> Option { + let version_overhead = cow_version.size_overhead(self.num_ops, payload_install_ops); + let algo_overhead = vabc_algo.size_overhead(self.size); + + let size = self + .size + .checked_add(version_overhead) + .and_then(|e| e.checked_add(algo_overhead))?; + + Some(Self { + size, + num_ops: self.num_ops, + }) + } +} + +impl Add for CowEstimate { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self { + size: self.size + rhs.size, + num_ops: self.num_ops + rhs.num_ops, + } + } +} + +impl CheckedAdd for CowEstimate { + fn checked_add(&self, rhs: &Self) -> Option { + Some(Self { + size: self.size.checked_add(rhs.size)?, + num_ops: self.num_ops.checked_add(rhs.num_ops)?, + }) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, ValueEnum)] pub enum VabcAlgo { Lz4, @@ -956,12 +1130,30 @@ pub enum VabcAlgo { } impl VabcAlgo { - fn compressed_size(self, mut raw_data: &[u8], block_size: u32) -> Result { - let mut total = 0; + /// Compute the size overhead to account for differences in compression + /// level between our compression libraries and AOSP's. + fn size_overhead(self, estimate: u64) -> u64 { + // lz4_flex and zlib-rs usually compress better than the lz4 and zlib + // implementations used by libsnapshot_cow. Make up for this by adding + // percentage-based overhead. + estimate / 100 + } + + /// Compute the compressed size of the raw data when split into chunks based + /// on the specified [`ChunkingParams`]. The length of `raw_data` must be a + /// multiple of the block size or else this will panic. The compressed data + /// for each chunk is temporarily stored in memory, but discarded after each + /// loop iteration. + fn compressed_size(self, mut raw_data: &[u8], chunking: ChunkingParams) -> Result { + assert!(raw_data.len() as u64 % u64::from(chunking.block_size) == 0); + + let mut size = 0; + let mut num_ops = 0; while !raw_data.is_empty() { - let n = raw_data.len().min(block_size as usize); - let (chunk, remaining) = raw_data.split_at(n); + let num_blocks = raw_data.len() as u64 / u64::from(chunking.block_size); + let chunk_size = chunking.chunk_size(num_blocks) as usize; + let (chunk, remaining) = raw_data.split_at(chunk_size); // This should match CompressWorker::GetDefaultCompressionLevel() in // AOSP's libsnapshot. @@ -974,12 +1166,17 @@ impl VabcAlgo { } }; - total += compressed.len().min(n) as u64; + // CoW v3 uses the raw data instead of the compressed data if the + // raw data is smaller. Because we use a different implementation of + // the compression algorithms, we don't implement this. It's safer + // to just overestimate and use the (larger) compressed size. + size += compressed.len().min(chunk_size) as u64; + num_ops += 1; raw_data = remaining; } - Ok(total) + Ok(CowEstimate { size, num_ops }) } } @@ -989,47 +1186,36 @@ impl fmt::Display for VabcAlgo { } } -/// Add fudge factor to CoW estimate to account for overhead. -fn fudge_cow_estimate(mut estimate: u64) -> Option { - // lz4_flex and zlib-rs usually compress better than the lz4 and zlib - // implementations used by libsnapshot_cow. Make up for this by adding - // percentage-based overhead. - estimate = estimate.checked_add(estimate / 100)?; - - // We also need to account for constant overhead, especially with smaller - // partitions. We can match what delta_generator normally adds in - // CowWriterV2::InitPos() exactly. Since we only ever create full OTAs, we - // can assume that all CoW operations are kCowReplaceOp. - - // sizeof(CowHeader). - estimate = estimate.checked_add(38)?; - // header_.buffer_size (equal to BUFFER_REGION_DEFAULT_SIZE). - estimate = estimate.checked_add(2 * 1024 * 1024)?; - // CowOptions::cluster_ops * sizeof(CowOperationV2). - estimate = estimate.checked_add(200 * 20)?; - - Some(estimate) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct VabcParams { + /// CoW on-disk format version. + pub version: CowVersion, + /// CoW compression algorithm. + pub algo: VabcAlgo, + /// The maximum number of bytes to compress at a time. + pub compression_factor: u32, } -/// Compute the VABC CoW v2 size estimate. The caller must update -/// [`PartitionUpdate::estimate_cow_size`] with this value or else update_engine -/// may fail to flash the partition due to running out of space on the CoW block -/// device. CoW v2 + other algorithms and also CoW v3 are currently unsupported -/// because there currently are no known OTAs that use those configurations. -pub fn compute_cow_estimate( - input: &(dyn ReadSeekReopen + Sync), +/// Ensure that the partition size is aligned to the block size and that the +/// block size and compression factor are factors of our [`CHUNK_SIZE`]. +fn validate_partition_size( partition_name: &str, + file_size: u64, block_size: u32, - vabc_algo: VabcAlgo, - cancel_signal: &AtomicBool, -) -> Result { - let file_size = input - .reopen_boxed() - .and_then(|mut r| r.seek(SeekFrom::End(0))) - .map_err(|e| Error::InputOpen(partition_name.to_owned(), e))?; - let final_chunk_different = file_size % CHUNK_SIZE != 0; + compression_factor: u32, +) -> Result<()> { + if block_size == 0 || !block_size.is_power_of_two() || CHUNK_SIZE % u64::from(block_size) != 0 { + return Err(Error::InvalidBlockSize(block_size)); + } - if file_size % u64::from(block_size) != 0 || CHUNK_SIZE % u64::from(block_size) != 0 { + if compression_factor == 0 + || !compression_factor.is_power_of_two() + || CHUNK_SIZE % u64::from(compression_factor) != 0 + { + return Err(Error::InvalidMaxCompressionChunkSize(compression_factor)); + } + + if file_size % u64::from(block_size) != 0 { return Err(Error::InvalidPartitionSize { name: partition_name.to_owned(), size: file_size, @@ -1037,11 +1223,51 @@ pub fn compute_cow_estimate( }); } + Ok(()) +} + +/// Compute the VABC CoW size estimate. For a more accurate size estimate with +/// CoW version 2, `payload_install_ops` must be equal to the number of +/// [`InstallOperation`]s in the payload. The caller must update +/// [`PartitionUpdate::estimate_cow_size`] and +/// [`PartitionUpdate::estimate_op_count_max`] or else update_engine may fail to +/// flash the partition due to running out of space on the CoW block device. +pub fn compute_cow_estimate( + input: &(dyn ReadSeekReopen + Sync), + payload_install_ops: u64, + partition_name: &str, + block_size: u32, + vabc_params: VabcParams, + cancel_signal: &AtomicBool, +) -> Result { + let file_size = input + .reopen_boxed() + .and_then(|mut r| r.seek(SeekFrom::End(0))) + .map_err(|e| Error::InputOpen(partition_name.to_owned(), e))?; + let final_chunk_different = file_size % CHUNK_SIZE != 0; + + validate_partition_size( + partition_name, + file_size, + block_size, + vabc_params.compression_factor, + )?; + + let chunking = ChunkingParams { + block_size, + method: match vabc_params.version { + CowVersion::V2 => ChunkingMethod::Exact, + CowVersion::V3 => { + ChunkingMethod::MaxPowerOf2(vabc_params.compression_factor.try_into().unwrap()) + } + }, + }; + let chunks_total = file_size.div_ceil(CHUNK_SIZE); - let cow_estimate = (0..chunks_total) + let initial_estimate = (0..chunks_total) .into_par_iter() - .map(|chunk| -> Result { + .map(|chunk| -> Result { let data = (|| { let mut reader = input.reopen_boxed()?; reader.seek(SeekFrom::Start(chunk * CHUNK_SIZE))?; @@ -1057,26 +1283,25 @@ pub fn compute_cow_estimate( })() .map_err(Error::ChunkRead)?; - vabc_algo.compressed_size(&data, block_size) + vabc_params.algo.compressed_size(&data, chunking) }) .try_fold( - || 0u64, - |total, chunk_estimate| -> Result { + CowEstimate::default, + |total, chunk_estimate| -> Result { total - .checked_add(chunk_estimate?) - .ok_or(Error::IntOverflow("cow_estimate")) + .checked_add(&chunk_estimate?) + .ok_or(Error::IntOverflow("initial_estimate")) }, ) - .try_reduce( - || 0u64, - |total, partial| { - total - .checked_add(partial) - .ok_or(Error::IntOverflow("cow_estimate")) - }, - )?; + .try_reduce(CowEstimate::default, |total, partial| { + total + .checked_add(&partial) + .ok_or(Error::IntOverflow("initial_estimate")) + })?; - fudge_cow_estimate(cow_estimate).ok_or(Error::IntOverflow("cow_estimate_fudged")) + initial_estimate + .fudged(payload_install_ops, vabc_params.version, vabc_params.algo) + .ok_or(Error::IntOverflow("fudged_estimate")) } /// Compress the image and return the corresponding information to insert into @@ -1095,9 +1320,9 @@ pub fn compress_image( output: &(dyn WriteSeekReopen + Sync), partition_name: &str, block_size: u32, - vabc_algo: Option, + vabc_params: Option, cancel_signal: &AtomicBool, -) -> Result<(PartitionInfo, Vec, Option)> { +) -> Result<(PartitionInfo, Vec, Option)> { const CHUNK_GROUP: u64 = 32; let file_size = input @@ -1106,18 +1331,23 @@ pub fn compress_image( .map_err(|e| Error::InputOpen(partition_name.to_owned(), e))?; let final_chunk_different = file_size % CHUNK_SIZE != 0; - if file_size % u64::from(block_size) != 0 || CHUNK_SIZE % u64::from(block_size) != 0 { - return Err(Error::InvalidPartitionSize { - name: partition_name.to_owned(), - size: file_size, - block_size, - }); - } + let compression_factor = vabc_params.map_or(block_size, |p| p.compression_factor); + validate_partition_size(partition_name, file_size, block_size, compression_factor)?; + + let chunking = ChunkingParams { + block_size, + method: match vabc_params.map(|p| p.version) { + Some(CowVersion::V3) => { + ChunkingMethod::MaxPowerOf2(compression_factor.try_into().unwrap()) + } + _ => ChunkingMethod::Exact, + }, + }; let chunks_total = file_size.div_ceil(CHUNK_SIZE); let mut bytes_compressed = 0u64; let mut context_uncompressed = Context::new(&ring::digest::SHA256); - let mut cow_estimate = 0u64; + let mut initial_estimate = CowEstimate::default(); let mut operations = vec![]; // Read the file one group at a time. This allows for some parallelization @@ -1154,12 +1384,12 @@ pub fn compress_image( let mut compressed_data_group = uncompressed_data_group .into_par_iter() .map( - |(raw_offset, raw_data)| -> Result<(Vec, InstallOperation, u64)> { + |(raw_offset, raw_data)| -> Result<(Vec, InstallOperation, CowEstimate)> { let (data, digest_compressed) = compress_chunk(&raw_data, cancel_signal)?; - let cow_size = vabc_algo - .map(|a| a.compressed_size(&raw_data, block_size)) + let cow_estimate = vabc_params + .map(|p| p.algo.compressed_size(&raw_data, chunking)) .transpose()? - .unwrap_or(0); + .unwrap_or_default(); let extent = Extent { start_block: Some(raw_offset / u64::from(block_size)), @@ -1172,19 +1402,19 @@ pub fn compress_image( operation.dst_extents.push(extent); operation.data_sha256_hash = Some(digest_compressed.as_ref().to_vec()); - Ok((data, operation, cow_size)) + Ok((data, operation, cow_estimate)) }, ) .collect::>>()?; - for (data, operation, cow_size) in &mut compressed_data_group { + for (data, operation, cow_estimate) in &mut compressed_data_group { operation.data_offset = Some(bytes_compressed); bytes_compressed = bytes_compressed .checked_add(data.len() as u64) .ok_or(Error::IntOverflow("bytes_compressed"))?; - cow_estimate = cow_estimate - .checked_add(*cow_size) - .ok_or(Error::IntOverflow("cow_estimate"))?; + initial_estimate = initial_estimate + .checked_add(cow_estimate) + .ok_or(Error::IntOverflow("initial_estimate"))?; } let group_operations = compressed_data_group @@ -1208,8 +1438,12 @@ pub fn compress_image( hash: Some(digest_uncompressed.as_ref().to_vec()), }; - let cow_estimate = if vabc_algo.is_some() { - Some(fudge_cow_estimate(cow_estimate).ok_or(Error::IntOverflow("cow_estimate_fudged"))?) + let cow_estimate = if let Some(p) = vabc_params { + Some( + initial_estimate + .fudged(operations.len() as u64, p.version, p.algo) + .ok_or(Error::IntOverflow("fudged_estimate"))?, + ) } else { None }; diff --git a/e2e/e2e.toml b/e2e/e2e.toml index 94b647e..fab4853 100644 --- a/e2e/e2e.toml +++ b/e2e/e2e.toml @@ -12,8 +12,10 @@ security_patch_level = "2024-01-01" # Google Pixel 7 Pro # What's unique: init_boot (boot v4) + vendor_boot (vendor v4) -[profile.pixel_v4_gki] -vabc_algo = "Lz4" +[profile.pixel_v4_gki.vabc] +# CoW v3 is used starting with the Google Pixel 9a. +version = "V3" +algo = "Lz4" [profile.pixel_v4_gki.partitions.boot] avb.signed = true @@ -49,18 +51,19 @@ data.version = "vendor_v4" data.ramdisks = [["otacerts", "first_stage", "dsu_key_dir"]] [profile.pixel_v4_gki.hashes_streaming] -original = "c00f891f941f3dddb28966f7b07f3acea773bee104dace82b37c2d1341f09422" -patched = "6c27ffb07f4497af8539f8283e506066af9417580230c3209c9875fc15d5069d" +original = "0615b681742f243aae0a022aeba6c180a3920bf6f0175a1a17129f13c7a2a214" +patched = "8e9506f447585e54b060d8b0ccb6d705a6370ebd2425dffd5bc21a438be5a454" [profile.pixel_v4_gki.hashes_seekable] -original = "96a6c366b5de1c3b10d4d6cb4ca503c83ac4cd9ca952a965cceb041990ba7022" -patched = "e4fc12523ffc312796b92210bc1e3bbb70dd60797a47daae76c8b5852e48b382" +original = "dda797dfbf8b2a9dfe103d4050a2b801e9e58c52bdd7606a8b73e8846a5cffec" +patched = "361f2337db95e29366aa473a618d7839d7c43b02f9a374d223546230dd794096" # Google Pixel 6a # What's unique: boot (boot v4, no ramdisk) + vendor_boot (vendor v4, 2 ramdisks) -[profile.pixel_v4_non_gki] -vabc_algo = "Lz4" +[profile.pixel_v4_non_gki.vabc] +version = "V2" +algo = "Lz4" [profile.pixel_v4_non_gki.partitions.boot] avb.signed = true @@ -90,18 +93,19 @@ data.version = "vendor_v4" data.ramdisks = [["init", "otacerts", "first_stage", "dsu_key_dir"], ["dlkm"]] [profile.pixel_v4_non_gki.hashes_streaming] -original = "4d692bc777b568b0626d3c08d2e6f83f1b472db5ad903486daaec6a78d0cc26e" -patched = "6832ded3e98a14edc8c5ea7284fcea0b958fa710ebf222c27116faec8dfefe2e" +original = "7b7acb994f938b256f07248f0b05dd160a44ed069c4bf8520db8cdc1a4441d8a" +patched = "593d7b7969c018575c3d5f9c9764354750d0677abc8340e37ac3c500f9f78532" [profile.pixel_v4_non_gki.hashes_seekable] -original = "ea27ecd9718c17b63400b2548680bb3cee93ce63b4fc44ff9654ca0d9c5372a8" -patched = "114f8936e917d7e4a71bc1521adb3c8e676de3a738f8c7c505b86464d20bd95c" +original = "23036d93a2b28aedff59390b6850469eb4ba57cf6bc0f80826cfef290a5e0246" +patched = "73e3852db9a0f6848f83599802fbf790a0d130068a0fe6341641547fe9c68cfc" # Google Pixel 4a 5G # What's unique: boot (boot v3) + vendor_boot (vendor v3) -[profile.pixel_v3] -vabc_algo = "Lz4" +[profile.pixel_v3.vabc] +version = "V2" +algo = "Lz4" [profile.pixel_v3.partitions.boot] avb.signed = true @@ -132,18 +136,19 @@ data.version = "vendor_v3" data.ramdisks = [["otacerts", "first_stage", "dsu_key_dir"]] [profile.pixel_v3.hashes_streaming] -original = "f432dc7931520feb238474aa707dd5299747562ffe6129f3f763b5f11ac473ab" -patched = "1f28d9210a17e233cd5da4af55b07db764b19eeab991394514170b405240464f" +original = "ca09a0158beae3f0ad215f9f381f61b28bc27377507664b79ab106c173cea700" +patched = "e59884857ac5773f66e666ada6c10382220ce4e4d3f602c3f07e6c570e159a1d" [profile.pixel_v3.hashes_seekable] -original = "7d29ecc6780953c22052a576b8dc85066c8667a875e918a786a08ff4545b47d1" -patched = "27b80c7be9c1e527ea26abe3dabde245c580e6f26ec084204278fbfd81a39f83" +original = "b10d9708c1d1baade25aa07a045e24846b0cd7520f2d6ede94be32f93b32a5da" +patched = "1e7cedcd4631327531451b32349eb13d8bb8cbc374f587a80bdac0985dd2812b" # Google Pixel 4a # What's unique: boot (boot v2) -[profile.pixel_v2] -vabc_algo = "Gz" +[profile.pixel_v2.vabc] +version = "V2" +algo = "Gz" [profile.pixel_v2.partitions.boot] avb.signed = false @@ -168,9 +173,9 @@ data.type = "vbmeta" data.deps = ["system"] [profile.pixel_v2.hashes_streaming] -original = "bd2f19cf3d2285e35e8b36d44f75ed910e8e0be44c3ebd29f17a812521ba754b" -patched = "cf65d5b90500af54cd1204a646379bb852825061bc7c3f973b7a042f353f75ad" +original = "6c5f1d534bafb687fd1829c42f12f381fc58d3f6c905b09de2ee9c34aa05bfc5" +patched = "c738e1ecc5e4a99cdc9387c050254eee35045cf1c800aa04942b213bf77a6eb7" [profile.pixel_v2.hashes_seekable] -original = "7f96ebf7366e0b60c91ac1e5f196a2189ffdb0bbc73f77804a736466fcab7315" -patched = "c2d9d60d73c038da39f82073ffadb459c96b901d66db7af11f59da58e0dd53e4" +original = "bdec93bcffdeb8fa9c0e540aeef1bc0df79bfa65baba76363262c5520e0ce6b5" +patched = "69d768a14580b789050375709f6eba176e117c81871b5f8ae44cfd491f9f30fa" diff --git a/e2e/src/config.rs b/e2e/src/config.rs index ce73699..65c9fdc 100644 --- a/e2e/src/config.rs +++ b/e2e/src/config.rs @@ -1,14 +1,14 @@ -// SPDX-FileCopyrightText: 2023-2024 Andrew Gunnerson +// SPDX-FileCopyrightText: 2023-2025 Andrew Gunnerson // SPDX-License-Identifier: GPL-3.0-only use std::{collections::BTreeMap, fs, path::Path}; use anyhow::{Context, Result}; -use avbroot::format::payload::VabcAlgo; +use avbroot::format::payload::{CowVersion, VabcAlgo}; use serde::{Deserialize, Serialize}; use toml_edit::DocumentMut; -#[derive(Serialize, Deserialize)] +#[derive(Clone, Copy, Serialize, Deserialize)] pub struct Sha256Hash( #[serde( serialize_with = "hex::serialize", @@ -17,7 +17,7 @@ pub struct Sha256Hash( pub [u8; 32], ); -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct OtaInfo { pub device: String, @@ -29,7 +29,7 @@ pub struct OtaInfo { pub security_patch_level: String, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Copy, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Avb { pub signed: bool, @@ -55,7 +55,7 @@ pub enum BootVersion { VendorV4, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct BootData { pub version: BootVersion, @@ -71,19 +71,19 @@ pub enum DmVerityContent { SystemOtacerts, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Copy, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct DmVerityData { pub content: DmVerityContent, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct VbmetaData { pub deps: Vec, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum Data { Boot(BootData), @@ -91,30 +91,37 @@ pub enum Data { Vbmeta(VbmetaData), } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Copy, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Hashes { pub original: Sha256Hash, pub patched: Sha256Hash, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Partition { pub avb: Avb, pub data: Data, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Copy, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VabcSettings { + pub version: CowVersion, + pub algo: VabcAlgo, +} + +#[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Profile { - pub vabc_algo: Option, + pub vabc: Option, pub partitions: BTreeMap, pub hashes_streaming: Hashes, pub hashes_seekable: Hashes, } -#[derive(Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct Config { pub ota_info: OtaInfo, diff --git a/e2e/src/main.rs b/e2e/src/main.rs index 6d3a8fe..6606427 100644 --- a/e2e/src/main.rs +++ b/e2e/src/main.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023-2024 Andrew Gunnerson +// SPDX-FileCopyrightText: 2023-2025 Andrew Gunnerson // SPDX-FileCopyrightText: 2023 Pascal Roeleven // SPDX-License-Identifier: GPL-3.0-only @@ -36,7 +36,7 @@ use avbroot::{ cpio::{self, CpioEntry, CpioEntryData}, ota::{self, SigningWriter, ZipEntry, ZipMode}, padding, - payload::{self, PayloadHeader, PayloadWriter}, + payload::{self, CowVersion, PayloadHeader, PayloadWriter, VabcParams}, }, patch::otacert::{self, OtaCertBuildFlags}, protobuf::{ @@ -580,6 +580,8 @@ fn create_payload( key_ota: &RsaSigningKey, cancel_signal: &AtomicBool, ) -> Result<(String, u64)> { + const COMPRESSION_FACTOR: u32 = 64 * 1024; + let dynamic_partitions_names = partitions .iter() .filter(|(_, p)| matches!(&p.data, Data::DmVerity(_))) @@ -594,17 +596,26 @@ fn create_payload( .map(PSeekFile::new) .with_context(|| format!("Failed to create temp file for: {name}"))?; - let vabc_algo = if dynamic_partitions_names.contains(name) { - profile.vabc_algo + let vabc_params = if dynamic_partitions_names.contains(name) { + profile.vabc.map(|v| VabcParams { + version: v.version, + algo: v.algo, + compression_factor: COMPRESSION_FACTOR, + }) } else { None }; let (partition_info, operations, cow_estimate) = - payload::compress_image(file, &writer, name, 4096, vabc_algo, cancel_signal)?; + payload::compress_image(file, &writer, name, 4096, vabc_params, cancel_signal)?; compressed.insert(name, writer); + let is_v3 = profile + .vabc + .map(|e| e.version == CowVersion::V3) + .unwrap_or_default(); + payload_partitions.push(PartitionUpdate { partition_name: name.clone(), run_postinstall: None, @@ -624,8 +635,8 @@ fn create_payload( fec_roots: None, version: None, merge_operations: vec![], - estimate_cow_size: cow_estimate, - estimate_op_count_max: None, + estimate_cow_size: cow_estimate.map(|e| e.size), + estimate_op_count_max: cow_estimate.and_then(|e| is_v3.then_some(e.num_ops)), }); } @@ -646,10 +657,10 @@ fn create_payload( }], snapshot_enabled: Some(true), vabc_enabled: Some(true), - vabc_compression_param: profile.vabc_algo.map(|a| a.to_string()), + vabc_compression_param: profile.vabc.map(|v| v.algo.to_string()), cow_version: Some(2), vabc_feature_set: None, - compression_factor: None, + compression_factor: Some(COMPRESSION_FACTOR.into()), }), partial_update: None, apex_info: vec![],