From 4651b755bab0cede5b1f29dcec3c3d54e01432ad Mon Sep 17 00:00:00 2001 From: Andrew Gunnerson Date: Sat, 30 Sep 2023 22:17:16 -0400 Subject: [PATCH] Switch to prost for protobuf encoding/decoding There are several things we had to work around with quick-protobuf, like forcing no_std mode to use BTreeMaps and avoiding helper functions to read and write non-size-delimited messages. In addition, the pb-rs code generator doesn't support adding #[derive]s to enums and the existing support for adding derives to structs is broken due to incorrect string concatenation. Prost doesn't have these limitations and bugs. I originally avoided it because prost_build required the external `protoc` binary, but now that the protox library exists, the code generation can be done entirely in Rust without external tools. Prost also fully supports adding custom attributes to structs, enums, and fields. This will be helpful for future payload unpack and pack commands where the payload manifest would have to be serialized to TOML. The e2e checksums had to be updated because prost's serialized bytes on the wire differ from quick-protobuf, despite having the same semantic meaning. Since all checksums need to be updated anyway, the Magisk apk and OTA images have all been updated to the latest versions. Signed-off-by: Andrew Gunnerson --- Cargo.lock | 257 ++++++++++++++++++++++++++++++---- avbroot/Cargo.toml | 6 +- avbroot/build.rs | 16 +-- avbroot/src/cli/ota.rs | 4 +- avbroot/src/format/ota.rs | 20 +-- avbroot/src/format/payload.rs | 64 ++++----- avbroot/src/protobuf.rs | 12 +- avbroot/src/util.rs | 15 -- e2e/e2e.toml | 122 ++++++++-------- e2e/src/main.rs | 7 +- 10 files changed, 349 insertions(+), 174 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d8be1a..fb4f1ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,10 +143,11 @@ dependencies = [ "memchr", "num-bigint-dig", "num-traits", - "pb-rs", "phf", "pkcs8", - "quick-protobuf", + "prost", + "prost-build", + "protox", "rand", "rayon", "regex", @@ -178,6 +179,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bitflags" version = "1.3.2" @@ -624,6 +631,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flagset" version = "0.4.4" @@ -763,6 +776,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "honggfuzz" version = "0.5.55" @@ -850,6 +872,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -898,6 +929,38 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "logos" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax 0.6.29", + "syn 2.0.37", +] + +[[package]] +name = "logos-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lz4_flex" version = "0.11.1" @@ -948,10 +1011,27 @@ dependencies = [ ] [[package]] -name = "minimal-lexical" -version = "0.2.1" +name = "miette" +version = "5.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "5.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] [[package]] name = "miniz_oxide" @@ -962,6 +1042,12 @@ dependencies = [ "adler", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "native-tls" version = "0.2.11" @@ -991,16 +1077,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -1100,16 +1176,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "pb-rs" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "354a34df9c65b596152598001c0fe3393379ec2db03ae30b9985659422e2607e" -dependencies = [ - "log", - "nom", -] - [[package]] name = "pbkdf2" version = "0.12.2" @@ -1135,6 +1201,16 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.11.2" @@ -1227,6 +1303,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +dependencies = [ + "proc-macro2", + "syn 2.0.37", +] + [[package]] name = "proc-macro2" version = "1.0.67" @@ -1237,12 +1323,97 @@ dependencies = [ ] [[package]] -name = "quick-protobuf" -version = "0.8.1" +name = "prost" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" dependencies = [ - "byteorder", + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.37", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.37", +] + +[[package]] +name = "prost-reflect" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" +dependencies = [ + "logos", + "miette", + "once_cell", + "prost", + "prost-types", +] + +[[package]] +name = "prost-types" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +dependencies = [ + "prost", +] + +[[package]] +name = "protox" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66eb3a834c1ffe362107daab84dd87cfc1e1d2beda30e2eb8e4801f262839364" +dependencies = [ + "bytes", + "miette", + "prost", + "prost-reflect", + "prost-types", + "protox-parse", + "thiserror", +] + +[[package]] +name = "protox-parse" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4581f441c58863525a3e6bec7b8de98188cf75239a56c725a3e7288450a33f" +dependencies = [ + "logos", + "miette", + "prost-types", + "thiserror", ] [[package]] @@ -1322,7 +1493,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax", + "regex-syntax 0.7.5", ] [[package]] @@ -1333,9 +1504,15 @@ checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.5", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.7.5" @@ -1741,6 +1918,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" + [[package]] name = "untrusted" version = "0.7.1" @@ -1865,6 +2048,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/avbroot/Cargo.toml b/avbroot/Cargo.toml index c54d353..580e0be 100644 --- a/avbroot/Cargo.toml +++ b/avbroot/Cargo.toml @@ -29,7 +29,7 @@ num-bigint-dig = "0.8.4" num-traits = "0.2.16" phf = { version = "0.11.2", features = ["macros"] } pkcs8 = { version = "0.10.2", features = ["encryption", "pem"] } -quick-protobuf = "0.8.1" +prost = "0.12.1" rand = "0.8.5" rayon = "1.7.0" regex = { version = "1.9.4", default-features = false, features = ["perf", "std"] } @@ -77,8 +77,8 @@ features = ["deflate"] rustix = { version = "0.38.9", default-features = false, features = ["process"] } [build-dependencies] -# Disable the clap feature since it pulls in an ancient version of clap. -pb-rs = { version = "0.10.0", default-features = false } +prost-build = "0.12.1" +protox = "0.5.0" [dev-dependencies] assert_matches = "1.5.0" diff --git a/avbroot/build.rs b/avbroot/build.rs index 9d9671c..73f68cc 100644 --- a/avbroot/build.rs +++ b/avbroot/build.rs @@ -5,8 +5,6 @@ use std::{env, ffi::OsStr, fs, io, path::Path}; -use pb_rs::{types::FileDescriptor, ConfigBuilder}; - fn main() { let out_dir = Path::new(&env::var("OUT_DIR").unwrap()).join("protobuf"); let in_dir = Path::new(&env::var("CARGO_MANIFEST_DIR").unwrap()).join("protobuf"); @@ -30,14 +28,10 @@ fn main() { fs::create_dir_all(&out_dir).unwrap(); - let config = ConfigBuilder::new(&protos, None, Some(&out_dir), &[in_dir]) - .unwrap() - .dont_use_cow(true) - // We're using this as a means to force quick-protobuf to use BTreeMap - // instead of HashMap so that the serialized messages are reproducible. - // https://github.com/tafia/quick-protobuf/issues/251 - .nostd(true) - .build(); + let file_descriptors = protox::compile(&protos, [&in_dir]).unwrap(); - FileDescriptor::run(&config).unwrap(); + prost_build::Config::new() + .btree_map(["."]) + .compile_fds(file_descriptors) + .unwrap(); } diff --git a/avbroot/src/cli/ota.rs b/avbroot/src/cli/ota.rs index d370e4c..d4e8b59 100644 --- a/avbroot/src/cli/ota.rs +++ b/avbroot/src/cli/ota.rs @@ -535,7 +535,7 @@ fn patch_ota_payload( &mut vbmeta_order, clear_vbmeta_flags, key_avb, - header_locked.manifest.block_size.into(), + header_locked.manifest.block_size().into(), )?; status!( @@ -543,7 +543,7 @@ fn patch_ota_payload( joined(sorted(input_streams.keys())), ); - let block_size = header_locked.manifest.block_size; + let block_size = header_locked.manifest.block_size(); drop(header_locked); input_streams diff --git a/avbroot/src/format/ota.rs b/avbroot/src/format/ota.rs index d03259b..42242a7 100644 --- a/avbroot/src/format/ota.rs +++ b/avbroot/src/format/ota.rs @@ -13,6 +13,7 @@ use std::{ use cms::signed_data::SignedData; use const_oid::{db::rfc5912, ObjectIdentifier}; use memchr::memmem; +use prost::Message; use ring::digest::Context; use rsa::{Pkcs1v15Sign, RsaPrivateKey}; use sha1::Sha1; @@ -24,9 +25,8 @@ use zip::{result::ZipError, write::FileOptions, CompressionMethod, ZipArchive, Z use crate::{ crypto, format::payload::{self, PayloadHeader}, - protobuf::build::tools::releasetools::{mod_OtaMetadata::OtaType, OtaMetadata}, + protobuf::build::tools::releasetools::{ota_metadata::OtaType, OtaMetadata}, stream::{self, FromReader, HashingReader, HashingWriter}, - util, }; pub const PATH_METADATA: &str = "META-INF/com/android/metadata"; @@ -76,8 +76,8 @@ pub enum Error { CmsSign(#[from] crypto::Error), #[error("Payload error")] Payload(#[from] payload::Error), - #[error("Protobuf error")] - Protobuf(#[from] quick_protobuf::Error), + #[error("Failed to decode protobuf message")] + ProtobufDecode(#[from] prost::DecodeError), #[error("SPKI error")] Spki(#[from] pkcs8::spki::Error), #[error("x509 DER error")] @@ -99,11 +99,11 @@ fn serialize_metadata(metadata: &OtaMetadata) -> Result<(String, Vec)> { let mut pairs = BTreeMap::::new(); - match metadata.type_pb { - OtaType::AB => { + match metadata.r#type() { + OtaType::Ab => { pairs.insert("ota-type".to_owned(), "AB".to_owned()); } - OtaType::BLOCK => { + OtaType::Block => { pairs.insert("ota-type".to_owned(), "BLOCK".to_owned()); } _ => {} @@ -161,7 +161,7 @@ fn serialize_metadata(metadata: &OtaMetadata) -> Result<(String, Vec)> { .into_iter() .map(|(k, v)| format!("{k}={v}\n")) .collect::(); - let modern_metadata = util::write_protobuf(metadata)?; + let modern_metadata = metadata.encode_to_vec(); Ok((legacy_metadata, modern_metadata)) } @@ -299,7 +299,7 @@ pub fn add_metadata( metadata_pb_raw: &[u8], payload_metadata_size: u64, ) -> Result { - let mut metadata: OtaMetadata = util::read_protobuf(metadata_pb_raw)?; + let mut metadata = OtaMetadata::decode(metadata_pb_raw)?; let options = FileOptions::default().compression_method(CompressionMethod::Stored); let mut zip_entries = zip_entries.to_owned(); @@ -523,7 +523,7 @@ pub fn parse_zip_ota_info( let mut entry = zip.by_name(PATH_METADATA_PB)?; let mut buf = Vec::new(); entry.read_to_end(&mut buf)?; - util::read_protobuf::(&buf)? + OtaMetadata::decode(buf.as_slice())? }; let certificate = { diff --git a/avbroot/src/format/payload.rs b/avbroot/src/format/payload.rs index f21af3d..f582259 100644 --- a/avbroot/src/format/payload.rs +++ b/avbroot/src/format/payload.rs @@ -14,7 +14,7 @@ use base64::Engine; use byteorder::{BigEndian, ReadBytesExt}; use bzip2::write::BzDecoder; use num_traits::ToPrimitive; -use quick_protobuf::MessageWrite; +use prost::Message; use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use ring::digest::{Context, Digest}; use rsa::{traits::PublicKeyParts, Pkcs1v15Sign, RsaPrivateKey}; @@ -30,14 +30,13 @@ use xz2::{ use crate::{ crypto, protobuf::chromeos_update_engine::{ - mod_InstallOperation, mod_Signatures::Signature, DeltaArchiveManifest, Extent, + install_operation::Type, signatures::Signature, DeltaArchiveManifest, Extent, InstallOperation, PartitionInfo, PartitionUpdate, Signatures, }, stream::{ self, CountingReader, CountingWriter, FromReader, HashingWriter, ReadDiscardExt, ReadSeek, SharedCursor, WriteSeek, }, - util, }; const OTA_MAGIC: &[u8; 4] = b"CrAU"; @@ -68,7 +67,7 @@ pub enum Error { actual: String, }, #[error("Unsupported partition operation: {0:?}")] - UnsupportedOperation(mod_InstallOperation::Type), + UnsupportedOperation(Type), #[error("Expected sha256 {expected:?}, but have {actual:?}")] MismatchedDigest { expected: Option, @@ -90,8 +89,8 @@ pub enum Error { FieldOutOfBounds(&'static str), #[error("Crypto error")] Crypto(#[from] crypto::Error), - #[error("Protobuf error")] - Protobuf(#[from] quick_protobuf::Error), + #[error("Failed to decode protobuf message")] + ProtobufDecode(#[from] prost::DecodeError), #[error("XZ stream error")] XzStream(#[from] xz2::stream::Error), #[error("RSA error")] @@ -153,7 +152,7 @@ impl FromReader for PayloadHeader { let mut manifest_raw = vec![0u8; manifest_size]; reader.read_exact(&mut manifest_raw)?; - let manifest: DeltaArchiveManifest = util::read_protobuf(&manifest_raw)?; + let manifest = DeltaArchiveManifest::decode(manifest_raw.as_slice())?; // Skip manifest signatures. reader.read_discard_exact(metadata_signature_size.into())?; @@ -184,6 +183,7 @@ fn sign_digest(digest: &[u8], key: &RsaPrivateKey) -> Result { data: Some(digest_signed), // Always fits in even a u16. unpadded_signature_size: Some(unpadded_size as u32), + ..Default::default() }; let signatures = Signatures { @@ -329,14 +329,14 @@ impl PayloadWriter { ring::digest::digest(&ring::digest::SHA256, b"").as_ref(), &key, )?; - let dummy_sig_size = dummy_sig.get_size(); + let dummy_sig_size = dummy_sig.encoded_len(); // Fill out the new payload signature information. header.manifest.signatures_offset = Some(blob_size); header.manifest.signatures_size = Some(dummy_sig_size as u64); // Build new manifest. - let manifest_raw_new = util::write_protobuf(&header.manifest)?; + let manifest_raw_new = header.manifest.encode_to_vec(); // Excludes signatures (hashes are for signing). let mut h_partial = Context::new(&ring::digest::SHA256); @@ -364,7 +364,7 @@ impl PayloadWriter { // in the payload hash. let metadata_hash = h_partial.clone().finish(); let metadata_sig = sign_digest(metadata_hash.as_ref(), &key)?; - let metadata_sig_raw = util::write_protobuf(&metadata_sig)?; + let metadata_sig_raw = metadata_sig.encode_to_vec(); write_hash!(inner, [h_full], &metadata_sig_raw)?; Ok(Self { @@ -392,7 +392,7 @@ impl PayloadWriter { // Append payload signature. let payload_partial_hash = self.h_partial.clone().finish(); let payload_sig = sign_digest(payload_partial_hash.as_ref(), &self.key)?; - let payload_sig_raw = util::write_protobuf(&payload_sig)?; + let payload_sig_raw = payload_sig.encode_to_vec(); write_hash!(self.inner, [self.h_full], &payload_sig_raw)?; // Everything before the blob. @@ -600,18 +600,12 @@ impl CompressedPartitionWriter { num_blocks: Some(self.written / u64::from(self.block_size)), }; - let operation = InstallOperation { - type_pb: mod_InstallOperation::Type::REPLACE_XZ, - // Must be manually updated by the caller. - data_offset: None, - data_length: Some(size_compressed), - src_extents: vec![], - src_length: None, - dst_extents: vec![extent], - dst_length: None, - data_sha256_hash: Some(digest_compressed.as_ref().to_vec()), - src_sha256_hash: None, - }; + // data_offset must be manually updated by the caller. + let mut operation = InstallOperation::default(); + operation.set_type(Type::ReplaceXz); + operation.data_length = Some(size_compressed); + operation.dst_extents.push(extent); + operation.data_sha256_hash = Some(digest_compressed.as_ref().to_vec()); partition.operations.clear(); partition.operations.push(operation); @@ -685,7 +679,7 @@ pub fn verify_payload( )?; let buf = writer.into_inner(); - util::read_protobuf::(&buf)? + Signatures::decode(buf.as_slice())? }; // Check the metadata signatures. @@ -737,7 +731,7 @@ pub fn verify_payload( )?; let buf = writer.into_inner(); - util::read_protobuf::(&buf)? + Signatures::decode(buf.as_slice())? }; // Check the payload signatures. @@ -797,10 +791,10 @@ pub fn apply_operation( let mut hasher = Context::new(&ring::digest::SHA256); - match op.type_pb { + match op.r#type() { // Handle ZERO/DISCARD specially since they don't require access to // the payload blob. - mod_InstallOperation::Type::ZERO | mod_InstallOperation::Type::DISCARD => { + Type::Zero | Type::Discard => { stream::copy_n_inspect( io::repeat(0), &mut writer, @@ -823,7 +817,7 @@ pub fn apply_operation( reader.seek(SeekFrom::Start(in_offset))?; match other { - mod_InstallOperation::Type::REPLACE => { + Type::Replace => { stream::copy_n_inspect( &mut reader, &mut writer, @@ -832,7 +826,7 @@ pub fn apply_operation( cancel_signal, )?; } - mod_InstallOperation::Type::REPLACE_BZ => { + Type::ReplaceBz => { let mut decoder = BzDecoder::new(&mut writer); stream::copy_n_inspect( &mut reader, @@ -843,7 +837,7 @@ pub fn apply_operation( )?; decoder.finish()?; } - mod_InstallOperation::Type::REPLACE_XZ => { + Type::ReplaceXz => { let mut decoder = XzDecoder::new(&mut writer); stream::copy_n_inspect( &mut reader, @@ -854,7 +848,7 @@ pub fn apply_operation( )?; decoder.finish()?; } - _ => return Err(Error::UnsupportedOperation(op.type_pb)), + _ => return Err(Error::UnsupportedOperation(op.r#type())), } } } @@ -862,9 +856,7 @@ pub fn apply_operation( let expected_digest = op.data_sha256_hash.as_deref(); let digest = hasher.finish(); - if expected_digest != Some(digest.as_ref()) - && op.type_pb != mod_InstallOperation::Type::ZERO - { + if expected_digest != Some(digest.as_ref()) && op.r#type() != Type::Zero { return Err(Error::MismatchedDigest { expected: expected_digest.map(hex::encode), actual: hex::encode(digest.as_ref()), @@ -902,7 +894,7 @@ pub fn extract_image_to_memory( apply_operation( reader, writer, - header.manifest.block_size, + header.manifest.block_size(), header.blob_offset, op, cancel_signal, @@ -952,7 +944,7 @@ pub fn extract_images<'a>( apply_operation( reader, writer, - header.manifest.block_size, + header.manifest.block_size(), header.blob_offset, op, cancel_signal, diff --git a/avbroot/src/protobuf.rs b/avbroot/src/protobuf.rs index a257ec9..a8c858f 100644 --- a/avbroot/src/protobuf.rs +++ b/avbroot/src/protobuf.rs @@ -1 +1,11 @@ -include!(concat!(env!("OUT_DIR"), "/protobuf/mod.rs")); +pub mod build { + pub mod tools { + pub mod releasetools { + include!(concat!(env!("OUT_DIR"), "/build.tools.releasetools.rs")); + } + } +} + +pub mod chromeos_update_engine { + include!(concat!(env!("OUT_DIR"), "/chromeos_update_engine.rs")); +} diff --git a/avbroot/src/util.rs b/avbroot/src/util.rs index a7c8c61..bd80432 100644 --- a/avbroot/src/util.rs +++ b/avbroot/src/util.rs @@ -6,7 +6,6 @@ use std::{fmt, path::Path}; use num_traits::PrimInt; -use quick_protobuf::{BytesReader, MessageRead, MessageWrite, Writer}; pub const ZEROS: [u8; 16384] = [0u8; 16384]; @@ -38,20 +37,6 @@ pub fn is_zero(mut buf: &[u8]) -> bool { true } -/// Read a protobuf message with no leading size field. -pub fn read_protobuf<'a, M: MessageRead<'a>>(data: &'a [u8]) -> quick_protobuf::Result { - let mut reader = BytesReader::from_bytes(data); - M::from_reader(&mut reader, data) -} - -/// Write a protobuf message with no leading size field. -pub fn write_protobuf(message: &M) -> quick_protobuf::Result> { - let mut buf = Vec::with_capacity(message.get_size()); - let mut writer = Writer::new(&mut buf); - message.write_message(&mut writer)?; - Ok(buf) -} - /// Get the non-empty parent of a path. If the path has no parent in the string, /// then `.` is returned. This does not perform any filesystem operations. pub fn parent_path(path: &Path) -> &Path { diff --git a/e2e/e2e.toml b/e2e/e2e.toml index 9edf87f..d568a89 100644 --- a/e2e/e2e.toml +++ b/e2e/e2e.toml @@ -1,86 +1,86 @@ [magisk] -"url" = "https://github.com/topjohnwu/Magisk/releases/download/v26.0/Magisk-v26.0.apk" -"hash" = "9e14d3d3ca1f1a2765f8ca215ebbf35ea5fd2896fb147eea581fcaa3b4e77d25" +"url" = "https://github.com/topjohnwu/Magisk/releases/download/v26.3/Magisk.v26.3.apk" +"hash" = "30ff6ec0709412adfcd0b735c0eb1f61cd9d589af4bdef4cf03c09b986b5acce" # Google Pixel 7 Pro # What's unique: init_boot (boot v4) + vendor_boot (vendor v4) [device.cheetah] -url = "https://dl.google.com/dl/android/aosp/cheetah-ota-tq2a.230305.008.c1-6ac5ff2e.zip" +url = "https://dl.google.com/dl/android/aosp/cheetah-ota-tq3a.230901.001-6b881553.zip" sections = [ - { start = 0, end = 151715 }, - { start = 21683150, end = 23179365 }, - { start = 2043499985, end = 2043508325 }, - { start = 2315331822, end = 2333060126 }, - { start = 2344084950, end = 2344090066 }, + { start = 0, end = 152523 }, + { start = 21750700, end = 23251455 }, + { start = 2044485458, end = 2044493782 }, + { start = 2316042420, end = 2333812318 }, + { start = 2344911202, end = 2344916318 }, ] -hash.original.full = "6ac5ff2e14dc16755ea4ea30e6dbe25103b889a36a465194ef943bd0d665b91c" -hash.original.stripped = "549522015f0369a3b89385f532ab62235b47c2c39540bd0adaaf6acc81fdda94" -hash.patched.full = "b380720852e2a1e994bcf38064f577bac68b18e799cf8166cb6a7edb8a661cb4" -hash.patched.stripped = "560cdf4d7b25fb5bc650f96ae5a3c7593ac229a364bb39887287cfb734f6b377" -hash.avb_images."init_boot.img" = "fac9305ce22b897fbfb193968d5346f4c70f6c18c3060bb106226f134ce5f433" -hash.avb_images."vbmeta.img" = "0b3d719b751dd43bbec95d02b1bf57b5dae62e42a52502895892a207b773e77d" -hash.avb_images."vbmeta_system.img" = "cf8c77dcf0a4474d49b5bdc2a44bdb3646464d5212fbe12aa5d3c5f531742f4f" -hash.avb_images."vbmeta_vendor.img" = "660d8f61acd95a4f8ad416b4cbe126e9c039706462b4236ad723953c72ac49a8" -hash.avb_images."vendor_boot.img" = "c788d4d8eb7926ad1bfa2a9c000343c3c73cedc580449f5270a711feb620f033" +hash.original.full = "6b881553f012d582080642d660e1cf5c9e6fe41e9f1c6ab12ae87fab7894e307" +hash.original.stripped = "9befd7887a125ebd8e9ae0555469dababe6bc04b0aa41aa2562036782a6d87e0" +hash.patched.full = "9a16823737ec3586977106dbddfce3dc64acfcff4d7bc275989c6b78bd5b096e" +hash.patched.stripped = "0d3e5610a0215c94183a6ff15e4a44ceef20d476b3c1d3399fb1498e615c4623" +hash.avb_images."init_boot.img" = "a0f057ef2c2085a8fff2f4df98753e6db7721eb5f06e4011acdb3d10b1f75f85" +hash.avb_images."vbmeta.img" = "488d6a2e9941de44abd4bdf74dec463d5d6fdbf879f6dd3445a58bb7872a412e" +hash.avb_images."vbmeta_system.img" = "dbb63e08f26f46ccda501d99058d513ff71e3d6302c14d587442b666ff08862a" +hash.avb_images."vbmeta_vendor.img" = "6ffa0a10e72c3371653be80de1380832b4d7f8bbf38a2bd861d44a4097a57117" +hash.avb_images."vendor_boot.img" = "9662194df18e78549f3fb212e26926ecac8baecaa5a5e63af7ae12a9d56b87fd" # Google Pixel 6a # What's unique: boot (boot v4, no ramdisk) + vendor_boot (vendor v4, 2 ramdisks) [device.bluejay] -url = "https://dl.google.com/dl/android/aosp/bluejay-ota-tq2a.230305.008.e1-915f9087.zip" +url = "https://dl.google.com/dl/android/aosp/bluejay-ota-tq3a.230901.001-1f1f0abe.zip" sections = [ - { start = 0, end = 140787 }, - { start = 1060207, end = 21612852 }, - { start = 1886150700, end = 1886158844 }, - { start = 2069112987, end = 2092260102 }, - { start = 2098778558, end = 2098783674 }, + { start = 0, end = 142587 }, + { start = 1062619, end = 21678441 }, + { start = 1920893927, end = 1920902079 }, + { start = 2103596124, end = 2126776858 }, + { start = 2133324030, end = 2133329146 }, ] -hash.original.full = "915f9087b627b6961be9bb447dc63a7a1083b536753a78715e98641eaeb9c9d1" -hash.original.stripped = "a3ee5b6e39e687665c31790118ab9f47715b0b8285ae9847dbf81307f963db14" -hash.patched.full = "bfa7b26d90bdc889a7a199439e1564b219e5e2dbfddc1657bc1c6b73229be67e" -hash.patched.stripped = "4e56ee4a8554f2b08ffc2f1470ad60b9b63d2b6fb469ed1f7c4b1204bbf8ad7d" -hash.avb_images."boot.img" = "a1a705092e7034d20b83c94d78291418e13c343b1573c1b11e0fc884fc00ae62" -hash.avb_images."vbmeta.img" = "3c123705be57ab142d2b43beef9b123eaad129df17a523b59b1d63b9122d28b0" -hash.avb_images."vbmeta_system.img" = "285b83e4290f3257dc3678f0c3191794830bb2d72fb0969b69fc8f09d7ddff12" -hash.avb_images."vbmeta_vendor.img" = "981f736586b91a9f4c93c4208a0d191a35ff15118c6fa505d755ee7fda8b2477" -hash.avb_images."vendor_boot.img" = "ceffcb4fdb33aa3bb2c70621060acd5af612206b21cc413d5c2d39fee25144ab" +hash.original.full = "1f1f0abe67a6f6f47287be6dafec2c12628de6a715b82ca7beddaf67ad22aca5" +hash.original.stripped = "38b15f5efdc7e056bc799859ba72ef9a73e93c61292c59f85fb4b9c31acc5f82" +hash.patched.full = "5904f5eaeccd8bb6bb62919eeedef9bacbafd0261d88d2f86844cce5aac14da0" +hash.patched.stripped = "e3facf3f17fb35733f8e53611926f436880fa85d62a1655da8cee1aa6a672da4" +hash.avb_images."boot.img" = "2ecab6d2c261dd52f55b9d275429343472be6bdc360efe14ef3336a9fdce3778" +hash.avb_images."vbmeta.img" = "0ad8baf243194e8cd1b52a498a3b4c8c0c5e2bb5a399af616c171406eac2c5b6" +hash.avb_images."vbmeta_system.img" = "98a050f0d53a016fbb78147b1b4a9bca3fde615aa4da34bf62c2e07a395104b5" +hash.avb_images."vbmeta_vendor.img" = "fac530f47f237e76f3c7c3cdfe96308170dd8e8f0b227d81114a489c69ba763c" +hash.avb_images."vendor_boot.img" = "f4691800d32dc12eb1e2a9ace9d1e036b844043953ab56b986ce9c04054d3656" # Google Pixel 4a 5G # What's unique: boot (boot v3) + vendor_boot (vendor v3) [device.bramble] -url = "https://dl.google.com/dl/android/aosp/bramble-ota-tq2a.230305.008.c1-a925dd09.zip" +url = "https://dl.google.com/dl/android/aosp/bramble-ota-tq3a.230901.001-6d107ffa.zip" sections = [ - { start = 0, end = 140531 }, - { start = 496187, end = 11655082 }, - { start = 1650283561, end = 1650287993 }, - { start = 1884739919, end = 1908081011 }, - { start = 1910894027, end = 1910899144 }, + { start = 0, end = 142225 }, + { start = 497945, end = 11647087 }, + { start = 1683478307, end = 1683482723 }, + { start = 1917281683, end = 1940628772 }, + { start = 1943442548, end = 1943447665 }, ] -hash.original.full = "a925dd09c8d613d46cf72677c16f4fadee18bc21734d57047c6ccf31f672507b" -hash.original.stripped = "79322b0b417359e8f072032de676d7e5bd2715a3b3554c48ed5cc9e9a25c6866" -hash.patched.full = "cf79cd60acd3635f5085d8bb411d4a8dc4d7e62440f62a14e7ea12f5a9b7cd8a" -hash.patched.stripped = "b684a78fe08014f1633e74d1f380e137823ba77a47ddf428bc649364f5548b75" -hash.avb_images."boot.img" = "2bbf2c6d2f82d454426b26ac3b4887b26ca0591b458e6cf137207ecbb8f7649d" -hash.avb_images."vbmeta.img" = "ab3b2487671b3fc28e163898621d6c068164d335b6ddd9b94e4fde9471c95d66" -hash.avb_images."vbmeta_system.img" = "2fcd52d7462916a8510bbb07f2f5a14200afe2de97568396fe75e04c5c283152" -hash.avb_images."vendor_boot.img" = "e27f157c4ebf4e958165997a4b87d4de1cfc34f4dea423a153327a27b053cac7" +hash.original.full = "6d107ffac1cd3da2c972112acc75957ed725e5c13d57ca724d9bcca5404fcebd" +hash.original.stripped = "5b889bdab3bb12ddcd3c243a56e1c58bedada8831069f49d56fe5098fb141e35" +hash.patched.full = "d101bb5376eea859b69ea78cb4162753b69cff81c14b4f244021f27ec0ce58d0" +hash.patched.stripped = "04c4d8f2e5dc744fd5a8295a00349e661260ccd2dd0750201dedc1e2294b7962" +hash.avb_images."boot.img" = "e6bd56ad12013ed2e4cf26451d5e769c40bab05feee39f9a741d15ecf272cfbe" +hash.avb_images."vbmeta.img" = "f5079baa3aec734c3f8b346958a3fce9e7641e3ea2f1f9bbf13031b81d2a00a6" +hash.avb_images."vbmeta_system.img" = "9a7c6fd654e7a92aeffbdbd55ea0d87eee36f4c235e1b505423ad8a13a751a00" +hash.avb_images."vendor_boot.img" = "3cc7f110e69d3b32e33ecaac654f4189807d6cb8ad479b05054cec5c0372fafe" # Google Pixel 4a # What's unique: boot (boot v2) [device.sunfish] -url = "https://dl.google.com/dl/android/aosp/sunfish-ota-tq2a.230305.008.c1-174fd16b.zip" +url = "https://dl.google.com/dl/android/aosp/sunfish-ota-tq3a.230805.001-01fd34b2.zip" sections = [ - { start = 0, end = 129700 }, - { start = 476996, end = 34035261 }, - { start = 1624751947, end = 1624756371 }, - { start = 1823132563, end = 1823137581 }, + { start = 0, end = 131325 }, + { start = 478553, end = 34014760 }, + { start = 1658403115, end = 1658407531 }, + { start = 1855778567, end = 1855783585 }, ] -hash.original.full = "174fd16b47ef994ea8f3cb0f3fb456df2654b0aa1f9ea6fb8e54e5c6319f2601" -hash.original.stripped = "943ce3ae2aac8a0ccd4a7e9d4e38a9c495c39639f2621c6853be4c7a3fa0fc26" -hash.patched.full = "cdfffa731f0aca0ab9eff1e7d7bfee8c4716054d64cf0a47e50f50da6cbdb849" -hash.patched.stripped = "35b720058c460dd28b00f40bed4bcd1d133522d42d00ba048172b3092a7444ee" -hash.avb_images."boot.img" = "22182f2efc7043f35d79e71abb400ce919c6b3b8419405e16469644932367ee6" -hash.avb_images."vbmeta.img" = "4cbe171bb37515f59cd4082cc2d982ca1edfeba9db09dd62c8959dd318423888" -hash.avb_images."vbmeta_system.img" = "7cdb590bfc1056a5a8c7606ff05e99eb344efe108296682698b5cfe83905e0cd" +hash.original.full = "01fd34b206152a3559039161c9874ab03df37da4268b86a9e0be899de5fc0af7" +hash.original.stripped = "cc311b5bd46e06cfdefbade794d33aa9bc3ceda4ad4f38bfe9f0dfc17033d207" +hash.patched.full = "e06bef2fbf5d58d058d4b181a7f54cba6bc19b66986eb4c11519381f34b37622" +hash.patched.stripped = "709d2de4463df439349b85567bcda4a440685f5b82472ad098dbbcbdfefd05c6" +hash.avb_images."boot.img" = "ac504c793b095a66d79638295f2a6458c3ea775328238c8bb363a913d62d412a" +hash.avb_images."vbmeta.img" = "52a0b55a46ece965e58cbea199baa888f42d0789f3ddb24e740487d606732870" +hash.avb_images."vbmeta_system.img" = "1d3efa00fd1d44a594c7317072468fa95c23d83d2759d6d6e757783ceeabc594" # OnePlus 10 Pro # Build NE2215_11_C.26 @@ -107,9 +107,9 @@ sections = [ ] hash.original.full = "929f892fbd70699cf7f118a119aac1ae1b86351e1ada17715666fa4401e63472" hash.original.stripped = "4eabaf79b6c2b5df305e3ecdc2b9570c0dd27350b4e8d6434584000c4989ff3d" -hash.patched.full = "3453b2eda97ca1ba7367fa9d0dd37b21a5065e2e57cc3bbe9a6f135349d42d0b" -hash.patched.stripped = "1a41e435bdbf4302761e719e218e8682145644c0fe276739c3265f392adbd74d" -hash.avb_images."boot.img" = "f5dc3b147c54589be8db00ca15257a3688424a9eb88bd9c2cec82ebf4f6bf859" +hash.patched.full = "ca06e7d2905c421935e177d82535cee8e346facfa060b9dd5394387a758f8fdb" +hash.patched.stripped = "a1e2af8c50682b7cc02bd7ecc9b303cfc2f7dc540d5b5d43380075d13f54de0a" +hash.avb_images."boot.img" = "5bdf0f13343e52711eee493aeada0281c17a3b5204808228a564aa45c3a1871d" hash.avb_images."recovery.img" = "a42c0bf4f023cd24394184a33ee113783a9c89a7cc4c0c582a5f72cc23b72309" hash.avb_images."vbmeta.img" = "c022cf79da301a8430af5c49704944c490707fa0306031fe3ea22c39ce4734f6" hash.avb_images."vbmeta_system.img" = "749616b7f04487c05e9e363ad2071a0ab3bae29d497daf1f1a7695f7c8cfa82a" diff --git a/e2e/src/main.rs b/e2e/src/main.rs index f224d79..f6728ae 100644 --- a/e2e/src/main.rs +++ b/e2e/src/main.rs @@ -26,6 +26,7 @@ use anyhow::{anyhow, bail, Context, Result}; use avbroot::{ cli::ota::{ExtractCli, PatchCli, VerifyCli}, format::{ota, payload::PayloadHeader}, + protobuf::chromeos_update_engine::install_operation::Type, stream::{self, FromReader, HashingReader, PSeekFile, SectionReader}, }; use clap::Parser; @@ -121,13 +122,11 @@ fn strip_image( .collect::>(); let mut data_holes = vec![]; - use avbroot::protobuf::chromeos_update_engine::mod_InstallOperation::Type; - for p in &header.manifest.partitions { if !required_images.contains(&p.partition_name) { for op in &p.operations { - match op.type_pb { - Type::ZERO | Type::DISCARD => continue, + match op.r#type() { + Type::Zero | Type::Discard => continue, _ => { let start = payload_offset + header.blob_offset