awesome-android/avbroot
3
0
Fork 0
mirror of https://github.com/chenxiaolong/avbroot.git synced 2026-07-03 14:05:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

payload: Add support for CoW version 3

Browse Source 
AOSP has long supported CoW version 3, but it wasn't used by the stock
OS on any Pixel devices until the new Pixel 9a.

CoW version 3 is fundamentally similar to version 2, though with
differences in the main header and how the operation headers are stored.
The compression is no longer done in fixed-size chunks equal to the
block size. Instead, the payload specifies a "compression factor", which
is the maximum chunk size to pass to the compressor. The actual chunk
size is the largest power of 2 <= the compression factor and the
remaining input size. Additionally, for version 3, the payload stores an
additional estimate_op_count_max field containing the number of CoW
operations.

While working on support for version 3, a few bugs in the version 2
estimation logic were found and fixed:

* The cluster_ops * sizeof(CowOperationV2) overhead incorrectly
  assumed that cluster_ops was a constant 200 instead of the actual
  number of CoW operations.
* The overhead did not account for kCowLabelOp headers, which
  delta_generator emits once for every InstallOperation in the payload.
* The overhead did not account for kCowClusterOp headers, which batch
  CoW operations into groups of 200.
* The overhead did not account for the CowFooter.

The version 3 overhead is much simpler and easier to calculate compared
to version 2.

Fixes: #441

Signed-off-by: Andrew Gunnerson <accounts+github@chiller3.com>
This commit is contained in:
Andrew Gunnerson
2025-05-03 20:02:15 -04:00
parent 7d786150cf
commit 6fee5346bb
5 changed files with 451 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
avbroot/src/cli/ota.rs
+66 -26
View File
@@ -32,7 +32,7 @@ use crate::{
avb::{self, Descriptor, Header},
ota::{self, SigningWriter, ZipEntry, ZipMode},
padding,
payload::{self, PayloadHeader, PayloadWriter, VabcAlgo},
payload::{self, CowVersion, PayloadHeader, PayloadWriter, VabcAlgo, VabcParams},
},
patch::{
boot::{
@@ -576,9 +576,10 @@ fn update_metadata_descriptors(parent_header: &mut Header, child_header: &Header
}
}
/// Get the VABC algorithm from the payload header. This will fail if an
/// unsupported VABC algorithm is specified, but not if VABC is disabled.
fn get_vabc_algo(header: &PayloadHeader) -> Result<Option<VabcAlgo>> {
/// Get the VABC parameters from the payload header. This will fail if an
/// unsupported VABC algorithm or CoW version is specified, but not if VABC is
/// disabled.
fn get_vabc_params(header: &PayloadHeader) -> Result<Option<VabcParams>> {
// Only CoW v2 seems to exist in the wild currently, so that is all we
// support.
let Some(dpm) = &header.manifest.dynamic_partition_metadata else {
@@ -589,17 +590,32 @@ fn get_vabc_algo(header: &PayloadHeader) -> Result<Option<VabcAlgo>> {
return Ok(None);
}
let cow_version = dpm.cow_version();
if dpm.cow_version() != 2 {
bail!("Unsupported CoW version: {cow_version}");
}
let cow_version = match dpm.cow_version() {
2 => CowVersion::V2,
3 => CowVersion::V3,
v => bail!("Unsupported CoW version: {v}"),
};
let compression = dpm.vabc_compression_param();
let Ok(vabc_algo) = VabcAlgo::from_str(compression, false) else {
bail!("Unsupported VABC compression: {compression}");
};
Ok(Some(vabc_algo))
// This is unused by v2, but delta_generator sets it anyway.
let Some(compression_factor) = dpm.compression_factor else {
bail!("No CoW compression factor specified");
};
let Ok(compression_factor) = u32::try_from(compression_factor) else {
bail!("CoW compression factor is too large: {compression_factor}");
};
let vabc_params = VabcParams {
version: cow_version,
algo: vabc_algo,
compression_factor,
};
Ok(Some(vabc_params))
}
/// Set the VABC algorithm in the payload header and return whether it was
@@ -731,7 +747,7 @@ pub fn compress_image(
.map(PSeekFile::new)
.with_context(|| format!("Failed to create temp file for: {name}"))?;
let vabc_algo = get_vabc_algo(header)?;
let vabc_params = get_vabc_params(header)?;
let block_size = header.manifest.block_size();
let partition = header
.manifest
@@ -742,14 +758,17 @@ pub fn compress_image(
// If VABC is enabled, we need to update the CoW size estimate or else the
// CoW block device may run out of space during flashing.
let vabc_algo = if partition.estimate_cow_size.is_some() {
let Some(vabc_algo) = vabc_algo else {
let vabc_params = if partition.estimate_cow_size.is_some() {
let Some(vabc_params) = vabc_params else {
bail!("Partition has CoW estimate, but VABC is disabled: {name}");
};
info!("Needs updated {vabc_algo} CoW size estimate: {name}");
info!(
"Needs updated {} CoW size estimate: {name}",
vabc_params.algo,
);
Some(vabc_algo)
Some(vabc_params)
} else {
None
};
@@ -770,16 +789,19 @@ pub fn compress_image(
// The changes we make usually aren't any less compressible, but
// we'll still recompute the CoW size estimate to handle the
// case where the user requested a different algorithm.
if let Some(vabc_algo) = vabc_algo {
if let Some(vabc_params) = vabc_params {
let cow_estimate = payload::compute_cow_estimate(
&*file,
partition.operations.len() as u64,
name,
block_size,
vabc_algo,
vabc_params,
cancel_signal,
)?;
partition.estimate_cow_size = Some(cow_estimate);
partition.estimate_cow_size = Some(cow_estimate.size);
partition.estimate_op_count_max =
(vabc_params.version == CowVersion::V3).then_some(cow_estimate.num_ops);
}
*file = writer;
@@ -797,12 +819,22 @@ pub fn compress_image(
info!("Compressing full image: {name}");
let (partition_info, operations, cow_estimate) =
payload::compress_image(&*file, &writer, name, block_size, vabc_algo, cancel_signal)?;
let (partition_info, operations, cow_estimate) = payload::compress_image(
&*file,
&writer,
name,
block_size,
vabc_params,
cancel_signal,
)?;
partition.new_partition_info = Some(partition_info);
partition.operations = operations;
partition.estimate_cow_size = cow_estimate;
partition.estimate_cow_size = cow_estimate.map(|e| e.size);
let is_v3 = vabc_params
.map(|p| p.version == CowVersion::V3)
.unwrap_or_default();
partition.estimate_op_count_max = cow_estimate.and_then(|e| is_v3.then_some(e.num_ops));
*file = writer;
@@ -822,7 +854,7 @@ fn recow_image(
file.rewind()?;
let vabc_algo = get_vabc_algo(header)?;
let vabc_params = get_vabc_params(header)?;
let block_size = header.manifest.block_size();
let partition = header
.manifest
@@ -835,16 +867,24 @@ fn recow_image(
bail!("Partition has no original CoW estimate: {name}");
};
let Some(vabc_algo) = vabc_algo else {
let Some(vabc_params) = vabc_params else {
bail!("Partition has CoW estimate, but VABC is disabled: {name}");
};
info!("Recomputing {vabc_algo} CoW size estimate: {name}");
info!("Recomputing {} CoW size estimate: {name}", vabc_params.algo);
let cow_estimate =
payload::compute_cow_estimate(&*file, name, block_size, vabc_algo, cancel_signal)?;
let cow_estimate = payload::compute_cow_estimate(
&*file,
partition.operations.len() as u64,
name,
block_size,
vabc_params,
cancel_signal,
)?;
partition.estimate_cow_size = Some(cow_estimate);
partition.estimate_cow_size = Some(cow_estimate.size);
partition.estimate_op_count_max =
(vabc_params.version == CowVersion::V3).then_some(cow_estimate.num_ops);
Ok(())
}
avbroot/src/format/payload.rs
+315 -81
View File
@@ -5,7 +5,8 @@ use std::{
collections::{HashMap, HashSet},
fmt,
io::{self, Cursor, Read, Seek, SeekFrom, Write},
ops::Range,
num::NonZeroU32,
ops::{Add, Range},
sync::atomic::AtomicBool,
};
@@ -19,6 +20,7 @@ use liblzma::{
write::XzDecoder,
write::XzEncoder,
};
use num_traits::CheckedAdd;
use prost::Message;
use rayon::{
iter::{IndexedParallelIterator, IntoParallelRefMutIterator},
@@ -49,7 +51,9 @@ const PAYLOAD_VERSION: u64 = 2;
const MANIFEST_MAX_SIZE: usize = 4 * 1024 * 1024;
/// Size of each extent. This matches what AOSP's delta_generator does.
/// Size of each extent. This matches what AOSP's delta_generator does. We also
/// require this to be a multiple of the block size and a multiple of the
/// maximum CoW compression chunk size.
const CHUNK_SIZE: u64 = 2 * 1024 * 1024;
#[derive(Debug, Error)]
@@ -81,6 +85,10 @@ pub enum Error {
expected: Option<String>,
actual: String,
},
#[error("Invalid block size: {0}")]
InvalidBlockSize(u32),
#[error("Invalid maximum CoW compression chunk size: {0}")]
InvalidMaxCompressionChunkSize(u32),
#[error("Size of {name} ({size}) is not aligned to the block size ({block_size})")]
InvalidPartitionSize {
name: String,
@@ -926,6 +934,8 @@ pub fn extract_images<'a>(
.collect()
}
/// Compress raw data into a chunk to be used with a [`Type::ReplaceXz`]
/// [`InstallOperation`].
fn compress_chunk(raw_data: &[u8], cancel_signal: &AtomicBool) -> Result<(Vec<u8>, Digest)> {
let reader = Cursor::new(raw_data);
let writer = Cursor::new(Vec::new());
@@ -949,6 +959,170 @@ fn compress_chunk(raw_data: &[u8], cancel_signal: &AtomicBool) -> Result<(Vec<u8
Ok((data, digest_compressed))
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
pub enum CowVersion {
V2,
V3,
}
impl CowVersion {
/// Compute the size overhead required to store the headers and footers
/// needed for this version of the on-disk CoW format.
fn size_overhead(self, cow_replace_ops: u64, payload_install_ops: u64) -> u64 {
const BUFFER_REGION_DEFAULT_SIZE: u64 = 2 * 1024 * 1024;
const CLUSTER_OPS: u64 = 200;
const NUM_RESUME_POINTS: u64 = 4;
const SIZEOF_COW_FOOTER_V2: u64 = 84;
const SIZEOF_COW_HEADER_V2: u64 = 38;
const SIZEOF_COW_HEADER_V3: u64 = 40;
const SIZEOF_COW_OPERATION_V2: u64 = 20;
const SIZEOF_COW_OPERATION_V3: u64 = 16;
const SIZEOF_RESUME_POINT_V3: u64 = 16;
let mut overhead = 0;
match self {
Self::V2 => {
// sizeof(CowHeader).
// AOSP: CowWriterV2::InitPos()
overhead += SIZEOF_COW_HEADER_V2;
// header_.buffer_size. update_engine uses the default value.
// AOSP: CowWriterV2::InitPos()
overhead += BUFFER_REGION_DEFAULT_SIZE;
// Add an operation header (sizeof(CowOperationV2)) for each
// kCowReplaceOp for the raw data.
// AOSP: CowWriterV2::EmitClusterIfNeeded()
overhead += cow_replace_ops * SIZEOF_COW_OPERATION_V2;
// Add an operation header (sizeof(CowOperationV2)) for each
// kCowLabelOp. delta_generator adds one for each install
// operation.
// AOSP: CowDryRun()
overhead += payload_install_ops * SIZEOF_COW_OPERATION_V2;
// Add an operation header (sizeof(CowOperationV2)) for each
// kCowClusterOp, which is emitted for each cluster of cow
// operations. The cluster size used to be 200, but changed to
// 1024 in 5e8e488c13cbff9e0a305ce7c22fd6a13aabb886. We'll use
// the smaller value because it's better to overestimate.
// AOSP: CowWriterV2::EmitClusterIfNeeded()
let num_ops = (cow_replace_ops + payload_install_ops).div_ceil(CLUSTER_OPS - 1);
overhead += num_ops * SIZEOF_COW_OPERATION_V2;
// sizeof(CowFooter).
// AOSP: CowWriterV2::GetCowSizeInfo()
overhead += SIZEOF_COW_FOOTER_V2;
}
Self::V3 => {
// AOSP: CowWriterV3::OpenForWrite() -> GetDataOffset()
overhead += SIZEOF_COW_HEADER_V3;
overhead += BUFFER_REGION_DEFAULT_SIZE;
overhead += NUM_RESUME_POINTS * SIZEOF_RESUME_POINT_V3;
// Add an operation header (sizeof(CowOperationV3)) for each
// chunk of compressed data.
// AOSP: CowWriterV3::WriteOperation()
overhead += cow_replace_ops * SIZEOF_COW_OPERATION_V3;
}
}
overhead
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum ChunkingMethod {
/// Compress data in block sized chunks each iteration.
Exact,
/// Compress data in chunks where each chunk is sized at the largest power
/// of 2 that's `<=` the specified size and the remaining input size.
MaxPowerOf2(NonZeroU32),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct ChunkingParams {
block_size: u32,
method: ChunkingMethod,
}
impl ChunkingParams {
fn chunk_size(&self, num_blocks: u64) -> u32 {
match self.method {
ChunkingMethod::Exact => self.block_size,
ChunkingMethod::MaxPowerOf2(max_chunk_size) => {
assert!(
max_chunk_size.is_power_of_two() && max_chunk_size.get() % self.block_size == 0
);
let mut chunk_size = max_chunk_size.get();
while chunk_size > self.block_size {
let min_blocks = chunk_size / self.block_size;
if num_blocks >= u64::from(min_blocks) {
return chunk_size;
}
chunk_size >>= 1;
}
self.block_size
}
}
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct CowEstimate {
/// Size of estimate in bytes.
pub size: u64,
/// Number of CoW operations (v3 only).
pub num_ops: u64,
}
impl CowEstimate {
/// Add fudge factor to account for overhead.
fn fudged(
&self,
payload_install_ops: u64,
cow_version: CowVersion,
vabc_algo: VabcAlgo,
) -> Option<Self> {
let version_overhead = cow_version.size_overhead(self.num_ops, payload_install_ops);
let algo_overhead = vabc_algo.size_overhead(self.size);
let size = self
.size
.checked_add(version_overhead)
.and_then(|e| e.checked_add(algo_overhead))?;
Some(Self {
size,
num_ops: self.num_ops,
})
}
}
impl Add for CowEstimate {
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self {
size: self.size + rhs.size,
num_ops: self.num_ops + rhs.num_ops,
}
}
}
impl CheckedAdd for CowEstimate {
fn checked_add(&self, rhs: &Self) -> Option<Self> {
Some(Self {
size: self.size.checked_add(rhs.size)?,
num_ops: self.num_ops.checked_add(rhs.num_ops)?,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, ValueEnum)]
pub enum VabcAlgo {
Lz4,
@@ -956,12 +1130,30 @@ pub enum VabcAlgo {
}
impl VabcAlgo {
fn compressed_size(self, mut raw_data: &[u8], block_size: u32) -> Result<u64> {
let mut total = 0;
/// Compute the size overhead to account for differences in compression
/// level between our compression libraries and AOSP's.
fn size_overhead(self, estimate: u64) -> u64 {
// lz4_flex and zlib-rs usually compress better than the lz4 and zlib
// implementations used by libsnapshot_cow. Make up for this by adding
// percentage-based overhead.
estimate / 100
}
/// Compute the compressed size of the raw data when split into chunks based
/// on the specified [`ChunkingParams`]. The length of `raw_data` must be a
/// multiple of the block size or else this will panic. The compressed data
/// for each chunk is temporarily stored in memory, but discarded after each
/// loop iteration.
fn compressed_size(self, mut raw_data: &[u8], chunking: ChunkingParams) -> Result<CowEstimate> {
assert!(raw_data.len() as u64 % u64::from(chunking.block_size) == 0);
let mut size = 0;
let mut num_ops = 0;
while !raw_data.is_empty() {
let n = raw_data.len().min(block_size as usize);
let (chunk, remaining) = raw_data.split_at(n);
let num_blocks = raw_data.len() as u64 / u64::from(chunking.block_size);
let chunk_size = chunking.chunk_size(num_blocks) as usize;
let (chunk, remaining) = raw_data.split_at(chunk_size);
// This should match CompressWorker::GetDefaultCompressionLevel() in
// AOSP's libsnapshot.
@@ -974,12 +1166,17 @@ impl VabcAlgo {
}
};
total += compressed.len().min(n) as u64;
// CoW v3 uses the raw data instead of the compressed data if the
// raw data is smaller. Because we use a different implementation of
// the compression algorithms, we don't implement this. It's safer
// to just overestimate and use the (larger) compressed size.
size += compressed.len().min(chunk_size) as u64;
num_ops += 1;
raw_data = remaining;
}
Ok(total)
Ok(CowEstimate { size, num_ops })
}
}
@@ -989,47 +1186,36 @@ impl fmt::Display for VabcAlgo {
}
}
/// Add fudge factor to CoW estimate to account for overhead.
fn fudge_cow_estimate(mut estimate: u64) -> Option<u64> {
// lz4_flex and zlib-rs usually compress better than the lz4 and zlib
// implementations used by libsnapshot_cow. Make up for this by adding
// percentage-based overhead.
estimate = estimate.checked_add(estimate / 100)?;
// We also need to account for constant overhead, especially with smaller
// partitions. We can match what delta_generator normally adds in
// CowWriterV2::InitPos() exactly. Since we only ever create full OTAs, we
// can assume that all CoW operations are kCowReplaceOp.
// sizeof(CowHeader).
estimate = estimate.checked_add(38)?;
// header_.buffer_size (equal to BUFFER_REGION_DEFAULT_SIZE).
estimate = estimate.checked_add(2 * 1024 * 1024)?;
// CowOptions::cluster_ops * sizeof(CowOperationV2).
estimate = estimate.checked_add(200 * 20)?;
Some(estimate)
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct VabcParams {
/// CoW on-disk format version.
pub version: CowVersion,
/// CoW compression algorithm.
pub algo: VabcAlgo,
/// The maximum number of bytes to compress at a time.
pub compression_factor: u32,
}
/// Compute the VABC CoW v2 size estimate. The caller must update
/// [`PartitionUpdate::estimate_cow_size`] with this value or else update_engine
/// may fail to flash the partition due to running out of space on the CoW block
/// device. CoW v2 + other algorithms and also CoW v3 are currently unsupported
/// because there currently are no known OTAs that use those configurations.
pub fn compute_cow_estimate(
input: &(dyn ReadSeekReopen + Sync),
/// Ensure that the partition size is aligned to the block size and that the
/// block size and compression factor are factors of our [`CHUNK_SIZE`].
fn validate_partition_size(
partition_name: &str,
file_size: u64,
block_size: u32,
vabc_algo: VabcAlgo,
cancel_signal: &AtomicBool,
) -> Result<u64> {
let file_size = input
.reopen_boxed()
.and_then(|mut r| r.seek(SeekFrom::End(0)))
.map_err(|e| Error::InputOpen(partition_name.to_owned(), e))?;
let final_chunk_different = file_size % CHUNK_SIZE != 0;
compression_factor: u32,
) -> Result<()> {
if block_size == 0 || !block_size.is_power_of_two() || CHUNK_SIZE % u64::from(block_size) != 0 {
return Err(Error::InvalidBlockSize(block_size));
}
if file_size % u64::from(block_size) != 0 || CHUNK_SIZE % u64::from(block_size) != 0 {
if compression_factor == 0
|| !compression_factor.is_power_of_two()
|| CHUNK_SIZE % u64::from(compression_factor) != 0
{
return Err(Error::InvalidMaxCompressionChunkSize(compression_factor));
}
if file_size % u64::from(block_size) != 0 {
return Err(Error::InvalidPartitionSize {
name: partition_name.to_owned(),
size: file_size,
@@ -1037,11 +1223,51 @@ pub fn compute_cow_estimate(
});
}
Ok(())
}
/// Compute the VABC CoW size estimate. For a more accurate size estimate with
/// CoW version 2, `payload_install_ops` must be equal to the number of
/// [`InstallOperation`]s in the payload. The caller must update
/// [`PartitionUpdate::estimate_cow_size`] and
/// [`PartitionUpdate::estimate_op_count_max`] or else update_engine may fail to
/// flash the partition due to running out of space on the CoW block device.
pub fn compute_cow_estimate(
input: &(dyn ReadSeekReopen + Sync),
payload_install_ops: u64,
partition_name: &str,
block_size: u32,
vabc_params: VabcParams,
cancel_signal: &AtomicBool,
) -> Result<CowEstimate> {
let file_size = input
.reopen_boxed()
.and_then(|mut r| r.seek(SeekFrom::End(0)))
.map_err(|e| Error::InputOpen(partition_name.to_owned(), e))?;
let final_chunk_different = file_size % CHUNK_SIZE != 0;
validate_partition_size(
partition_name,
file_size,
block_size,
vabc_params.compression_factor,
)?;
let chunking = ChunkingParams {
block_size,
method: match vabc_params.version {
CowVersion::V2 => ChunkingMethod::Exact,
CowVersion::V3 => {
ChunkingMethod::MaxPowerOf2(vabc_params.compression_factor.try_into().unwrap())
}
},
};
let chunks_total = file_size.div_ceil(CHUNK_SIZE);
let cow_estimate = (0..chunks_total)
let initial_estimate = (0..chunks_total)
.into_par_iter()
.map(|chunk| -> Result<u64> {
.map(|chunk| -> Result<CowEstimate> {
let data = (|| {
let mut reader = input.reopen_boxed()?;
reader.seek(SeekFrom::Start(chunk * CHUNK_SIZE))?;
@@ -1057,26 +1283,25 @@ pub fn compute_cow_estimate(
})()
.map_err(Error::ChunkRead)?;
vabc_algo.compressed_size(&data, block_size)
vabc_params.algo.compressed_size(&data, chunking)
})
.try_fold(
|| 0u64,
|total, chunk_estimate| -> Result<u64> {
CowEstimate::default,
|total, chunk_estimate| -> Result<CowEstimate> {
total
.checked_add(chunk_estimate?)
.ok_or(Error::IntOverflow("cow_estimate"))
.checked_add(&chunk_estimate?)
.ok_or(Error::IntOverflow("initial_estimate"))
},
)
.try_reduce(
|| 0u64,
|total, partial| {
total
.checked_add(partial)
.ok_or(Error::IntOverflow("cow_estimate"))
},
)?;
.try_reduce(CowEstimate::default, |total, partial| {
total
.checked_add(&partial)
.ok_or(Error::IntOverflow("initial_estimate"))
})?;
fudge_cow_estimate(cow_estimate).ok_or(Error::IntOverflow("cow_estimate_fudged"))
initial_estimate
.fudged(payload_install_ops, vabc_params.version, vabc_params.algo)
.ok_or(Error::IntOverflow("fudged_estimate"))
}
/// Compress the image and return the corresponding information to insert into
@@ -1095,9 +1320,9 @@ pub fn compress_image(
output: &(dyn WriteSeekReopen + Sync),
partition_name: &str,
block_size: u32,
vabc_algo: Option<VabcAlgo>,
vabc_params: Option<VabcParams>,
cancel_signal: &AtomicBool,
) -> Result<(PartitionInfo, Vec<InstallOperation>, Option<u64>)> {
) -> Result<(PartitionInfo, Vec<InstallOperation>, Option<CowEstimate>)> {
const CHUNK_GROUP: u64 = 32;
let file_size = input
@@ -1106,18 +1331,23 @@ pub fn compress_image(
.map_err(|e| Error::InputOpen(partition_name.to_owned(), e))?;
let final_chunk_different = file_size % CHUNK_SIZE != 0;
if file_size % u64::from(block_size) != 0 || CHUNK_SIZE % u64::from(block_size) != 0 {
return Err(Error::InvalidPartitionSize {
name: partition_name.to_owned(),
size: file_size,
block_size,
});
}
let compression_factor = vabc_params.map_or(block_size, |p| p.compression_factor);
validate_partition_size(partition_name, file_size, block_size, compression_factor)?;
let chunking = ChunkingParams {
block_size,
method: match vabc_params.map(|p| p.version) {
Some(CowVersion::V3) => {
ChunkingMethod::MaxPowerOf2(compression_factor.try_into().unwrap())
}
_ => ChunkingMethod::Exact,
},
};
let chunks_total = file_size.div_ceil(CHUNK_SIZE);
let mut bytes_compressed = 0u64;
let mut context_uncompressed = Context::new(&ring::digest::SHA256);
let mut cow_estimate = 0u64;
let mut initial_estimate = CowEstimate::default();
let mut operations = vec![];
// Read the file one group at a time. This allows for some parallelization
@@ -1154,12 +1384,12 @@ pub fn compress_image(
let mut compressed_data_group = uncompressed_data_group
.into_par_iter()
.map(
|(raw_offset, raw_data)| -> Result<(Vec<u8>, InstallOperation, u64)> {
|(raw_offset, raw_data)| -> Result<(Vec<u8>, InstallOperation, CowEstimate)> {
let (data, digest_compressed) = compress_chunk(&raw_data, cancel_signal)?;
let cow_size = vabc_algo
.map(|a| a.compressed_size(&raw_data, block_size))
let cow_estimate = vabc_params
.map(|p| p.algo.compressed_size(&raw_data, chunking))
.transpose()?
.unwrap_or(0);
.unwrap_or_default();
let extent = Extent {
start_block: Some(raw_offset / u64::from(block_size)),
@@ -1172,19 +1402,19 @@ pub fn compress_image(
operation.dst_extents.push(extent);
operation.data_sha256_hash = Some(digest_compressed.as_ref().to_vec());
Ok((data, operation, cow_size))
Ok((data, operation, cow_estimate))
},
)
.collect::<Result<Vec<_>>>()?;
for (data, operation, cow_size) in &mut compressed_data_group {
for (data, operation, cow_estimate) in &mut compressed_data_group {
operation.data_offset = Some(bytes_compressed);
bytes_compressed = bytes_compressed
.checked_add(data.len() as u64)
.ok_or(Error::IntOverflow("bytes_compressed"))?;
cow_estimate = cow_estimate
.checked_add(*cow_size)
.ok_or(Error::IntOverflow("cow_estimate"))?;
initial_estimate = initial_estimate
.checked_add(cow_estimate)
.ok_or(Error::IntOverflow("initial_estimate"))?;
}
let group_operations = compressed_data_group
@@ -1208,8 +1438,12 @@ pub fn compress_image(
hash: Some(digest_uncompressed.as_ref().to_vec()),
};
let cow_estimate = if vabc_algo.is_some() {
Some(fudge_cow_estimate(cow_estimate).ok_or(Error::IntOverflow("cow_estimate_fudged"))?)
let cow_estimate = if let Some(p) = vabc_params {
Some(
initial_estimate
.fudged(operations.len() as u64, p.version, p.algo)
.ok_or(Error::IntOverflow("fudged_estimate"))?,
)
} else {
None
};
e2e/e2e.toml
+29 -24
View File
@@ -12,8 +12,10 @@ security_patch_level = "2024-01-01"
# Google Pixel 7 Pro
# What's unique: init_boot (boot v4) + vendor_boot (vendor v4)
[profile.pixel_v4_gki]
vabc_algo = "Lz4"
[profile.pixel_v4_gki.vabc]
# CoW v3 is used starting with the Google Pixel 9a.
version = "V3"
algo = "Lz4"
[profile.pixel_v4_gki.partitions.boot]
avb.signed = true
@@ -49,18 +51,19 @@ data.version = "vendor_v4"
data.ramdisks = [["otacerts", "first_stage", "dsu_key_dir"]]
[profile.pixel_v4_gki.hashes_streaming]
original = "c00f891f941f3dddb28966f7b07f3acea773bee104dace82b37c2d1341f09422"
patched = "6c27ffb07f4497af8539f8283e506066af9417580230c3209c9875fc15d5069d"
original = "0615b681742f243aae0a022aeba6c180a3920bf6f0175a1a17129f13c7a2a214"
patched = "8e9506f447585e54b060d8b0ccb6d705a6370ebd2425dffd5bc21a438be5a454"
[profile.pixel_v4_gki.hashes_seekable]
original = "96a6c366b5de1c3b10d4d6cb4ca503c83ac4cd9ca952a965cceb041990ba7022"
patched = "e4fc12523ffc312796b92210bc1e3bbb70dd60797a47daae76c8b5852e48b382"
original = "dda797dfbf8b2a9dfe103d4050a2b801e9e58c52bdd7606a8b73e8846a5cffec"
patched = "361f2337db95e29366aa473a618d7839d7c43b02f9a374d223546230dd794096"
# Google Pixel 6a
# What's unique: boot (boot v4, no ramdisk) + vendor_boot (vendor v4, 2 ramdisks)
[profile.pixel_v4_non_gki]
vabc_algo = "Lz4"
[profile.pixel_v4_non_gki.vabc]
version = "V2"
algo = "Lz4"
[profile.pixel_v4_non_gki.partitions.boot]
avb.signed = true
@@ -90,18 +93,19 @@ data.version = "vendor_v4"
data.ramdisks = [["init", "otacerts", "first_stage", "dsu_key_dir"], ["dlkm"]]
[profile.pixel_v4_non_gki.hashes_streaming]
original = "4d692bc777b568b0626d3c08d2e6f83f1b472db5ad903486daaec6a78d0cc26e"
patched = "6832ded3e98a14edc8c5ea7284fcea0b958fa710ebf222c27116faec8dfefe2e"
original = "7b7acb994f938b256f07248f0b05dd160a44ed069c4bf8520db8cdc1a4441d8a"
patched = "593d7b7969c018575c3d5f9c9764354750d0677abc8340e37ac3c500f9f78532"
[profile.pixel_v4_non_gki.hashes_seekable]
original = "ea27ecd9718c17b63400b2548680bb3cee93ce63b4fc44ff9654ca0d9c5372a8"
patched = "114f8936e917d7e4a71bc1521adb3c8e676de3a738f8c7c505b86464d20bd95c"
original = "23036d93a2b28aedff59390b6850469eb4ba57cf6bc0f80826cfef290a5e0246"
patched = "73e3852db9a0f6848f83599802fbf790a0d130068a0fe6341641547fe9c68cfc"
# Google Pixel 4a 5G
# What's unique: boot (boot v3) + vendor_boot (vendor v3)
[profile.pixel_v3]
vabc_algo = "Lz4"
[profile.pixel_v3.vabc]
version = "V2"
algo = "Lz4"
[profile.pixel_v3.partitions.boot]
avb.signed = true
@@ -132,18 +136,19 @@ data.version = "vendor_v3"
data.ramdisks = [["otacerts", "first_stage", "dsu_key_dir"]]
[profile.pixel_v3.hashes_streaming]
original = "f432dc7931520feb238474aa707dd5299747562ffe6129f3f763b5f11ac473ab"
patched = "1f28d9210a17e233cd5da4af55b07db764b19eeab991394514170b405240464f"
original = "ca09a0158beae3f0ad215f9f381f61b28bc27377507664b79ab106c173cea700"
patched = "e59884857ac5773f66e666ada6c10382220ce4e4d3f602c3f07e6c570e159a1d"
[profile.pixel_v3.hashes_seekable]
original = "7d29ecc6780953c22052a576b8dc85066c8667a875e918a786a08ff4545b47d1"
patched = "27b80c7be9c1e527ea26abe3dabde245c580e6f26ec084204278fbfd81a39f83"
original = "b10d9708c1d1baade25aa07a045e24846b0cd7520f2d6ede94be32f93b32a5da"
patched = "1e7cedcd4631327531451b32349eb13d8bb8cbc374f587a80bdac0985dd2812b"
# Google Pixel 4a
# What's unique: boot (boot v2)
[profile.pixel_v2]
vabc_algo = "Gz"
[profile.pixel_v2.vabc]
version = "V2"
algo = "Gz"
[profile.pixel_v2.partitions.boot]
avb.signed = false
@@ -168,9 +173,9 @@ data.type = "vbmeta"
data.deps = ["system"]
[profile.pixel_v2.hashes_streaming]
original = "bd2f19cf3d2285e35e8b36d44f75ed910e8e0be44c3ebd29f17a812521ba754b"
patched = "cf65d5b90500af54cd1204a646379bb852825061bc7c3f973b7a042f353f75ad"
original = "6c5f1d534bafb687fd1829c42f12f381fc58d3f6c905b09de2ee9c34aa05bfc5"
patched = "c738e1ecc5e4a99cdc9387c050254eee35045cf1c800aa04942b213bf77a6eb7"
[profile.pixel_v2.hashes_seekable]
original = "7f96ebf7366e0b60c91ac1e5f196a2189ffdb0bbc73f77804a736466fcab7315"
patched = "c2d9d60d73c038da39f82073ffadb459c96b901d66db7af11f59da58e0dd53e4"
original = "bdec93bcffdeb8fa9c0e540aeef1bc0df79bfa65baba76363262c5520e0ce6b5"
patched = "69d768a14580b789050375709f6eba176e117c81871b5f8ae44cfd491f9f30fa"
e2e/src/config.rs
+21 -14
View File
@@ -1,14 +1,14 @@
// SPDX-FileCopyrightText: 2023-2024 Andrew Gunnerson
// SPDX-FileCopyrightText: 2023-2025 Andrew Gunnerson
// SPDX-License-Identifier: GPL-3.0-only
use std::{collections::BTreeMap, fs, path::Path};
use anyhow::{Context, Result};
use avbroot::format::payload::VabcAlgo;
use avbroot::format::payload::{CowVersion, VabcAlgo};
use serde::{Deserialize, Serialize};
use toml_edit::DocumentMut;
#[derive(Serialize, Deserialize)]
#[derive(Clone, Copy, Serialize, Deserialize)]
pub struct Sha256Hash(
#[serde(
serialize_with = "hex::serialize",
@@ -17,7 +17,7 @@ pub struct Sha256Hash(
pub [u8; 32],
);
#[derive(Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OtaInfo {
pub device: String,
@@ -29,7 +29,7 @@ pub struct OtaInfo {
pub security_patch_level: String,
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Copy, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Avb {
pub signed: bool,
@@ -55,7 +55,7 @@ pub enum BootVersion {
VendorV4,
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BootData {
pub version: BootVersion,
@@ -71,19 +71,19 @@ pub enum DmVerityContent {
SystemOtacerts,
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Copy, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DmVerityData {
pub content: DmVerityContent,
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VbmetaData {
pub deps: Vec<String>,
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Data {
Boot(BootData),
@@ -91,30 +91,37 @@ pub enum Data {
Vbmeta(VbmetaData),
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Copy, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Hashes {
pub original: Sha256Hash,
pub patched: Sha256Hash,
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Partition {
pub avb: Avb,
pub data: Data,
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Copy, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct VabcSettings {
pub version: CowVersion,
pub algo: VabcAlgo,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Profile {
pub vabc_algo: Option<VabcAlgo>,
pub vabc: Option<VabcSettings>,
pub partitions: BTreeMap<String, Partition>,
pub hashes_streaming: Hashes,
pub hashes_seekable: Hashes,
}
#[derive(Serialize, Deserialize)]
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Config {
pub ota_info: OtaInfo,
e2e/src/main.rs
+20 -9
View File
@@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: 2023-2024 Andrew Gunnerson
// SPDX-FileCopyrightText: 2023-2025 Andrew Gunnerson
// SPDX-FileCopyrightText: 2023 Pascal Roeleven
// SPDX-License-Identifier: GPL-3.0-only
@@ -36,7 +36,7 @@ use avbroot::{
cpio::{self, CpioEntry, CpioEntryData},
ota::{self, SigningWriter, ZipEntry, ZipMode},
padding,
payload::{self, PayloadHeader, PayloadWriter},
payload::{self, CowVersion, PayloadHeader, PayloadWriter, VabcParams},
},
patch::otacert::{self, OtaCertBuildFlags},
protobuf::{
@@ -580,6 +580,8 @@ fn create_payload(
key_ota: &RsaSigningKey,
cancel_signal: &AtomicBool,
) -> Result<(String, u64)> {
const COMPRESSION_FACTOR: u32 = 64 * 1024;
let dynamic_partitions_names = partitions
.iter()
.filter(|(_, p)| matches!(&p.data, Data::DmVerity(_)))
@@ -594,17 +596,26 @@ fn create_payload(
.map(PSeekFile::new)
.with_context(|| format!("Failed to create temp file for: {name}"))?;
let vabc_algo = if dynamic_partitions_names.contains(name) {
profile.vabc_algo
let vabc_params = if dynamic_partitions_names.contains(name) {
profile.vabc.map(|v| VabcParams {
version: v.version,
algo: v.algo,
compression_factor: COMPRESSION_FACTOR,
})
} else {
None
};
let (partition_info, operations, cow_estimate) =
payload::compress_image(file, &writer, name, 4096, vabc_algo, cancel_signal)?;
payload::compress_image(file, &writer, name, 4096, vabc_params, cancel_signal)?;
compressed.insert(name, writer);
let is_v3 = profile
.vabc
.map(|e| e.version == CowVersion::V3)
.unwrap_or_default();
payload_partitions.push(PartitionUpdate {
partition_name: name.clone(),
run_postinstall: None,
@@ -624,8 +635,8 @@ fn create_payload(
fec_roots: None,
version: None,
merge_operations: vec![],
estimate_cow_size: cow_estimate,
estimate_op_count_max: None,
estimate_cow_size: cow_estimate.map(|e| e.size),
estimate_op_count_max: cow_estimate.and_then(|e| is_v3.then_some(e.num_ops)),
});
}
@@ -646,10 +657,10 @@ fn create_payload(
}],
snapshot_enabled: Some(true),
vabc_enabled: Some(true),
vabc_compression_param: profile.vabc_algo.map(|a| a.to_string()),
vabc_compression_param: profile.vabc.map(|v| v.algo.to_string()),
cow_version: Some(2),
vabc_feature_set: None,
compression_factor: None,
compression_factor: Some(COMPRESSION_FACTOR.into()),
}),
partial_update: None,
apex_info: vec![],