Version 3.31.0

Signed-off-by: Andrew Gunnerson <accounts+github@chiller3.com>

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
             android_api: '31'
     steps:
       - name: Check out repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           # For git describe
           fetch-depth: 0
@@ -160,7 +160,7 @@ jobs:
         run: cp LICENSE README.md target/output/
 
       - name: Archive executable
-        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: avbroot-${{ steps.get_version.outputs.version }}-${{ matrix.artifact.name }}
           path: |

.github/workflows/deny.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
 
       - name: Run cargo-deny
-        uses: EmbarkStudios/cargo-deny-action@3fd3802e88374d3fe9159b834c7714ec57d6c979 # v2.0.15
+        uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20

.github/workflows/release.yml

@@ -24,10 +24,10 @@ jobs:
           echo "version=${version}" >> "${GITHUB_OUTPUT}"
 
       - name: Check out repository
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
 
       - name: Create release
-        uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2.6.1
+        uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0
         with:
           tag_name: v${{ steps.get_version.outputs.version }}
           name: Version ${{ steps.get_version.outputs.version }}

CHANGELOG.md

@@ -7,6 +7,27 @@
     to update the actual links at the bottom of the file.
 -->
 
+### Version 3.31.0
+
+* Update OTA metadata and payload manifest protobufs from AOSP `android-17.0.0_r1` release ([PR #613])
+
+### Version 3.30.2
+
+* Fix the symlink target path being incorrectly XZ-compressed when the Magisk patcher replaces a symlink with a regular file ([Issue #610], [PR #611])
+
+### Version 3.30.1
+
+* Fix `avbroot payload pack` and `avbroot zip pack --payload` for non-VABC devices ([Issue #607], [PR #608])
+
+### Version 3.30.0
+
+* Add `avbroot ota list` subcommand to list all partitions in an OTA ([Issue #603], [PR #606])
+* Update dependencies ([PR #605])
+
+### Version 3.29.1
+
+* Increase memory limit for payload manifest size to 16 MiB ([Issue #593], [PR #594])
+
 ### Version 3.29.0
 
 * Add support for unpacking/packing payloads at the same time as the OTA in `avbroot zip {unpack,pack}` ([Discussion #582], [PR #586])
@@ -490,6 +511,10 @@ Behind-the-scenes changes:
 [Issue #554]: https://github.com/chenxiaolong/avbroot/issues/554
 [Issue #569]: https://github.com/chenxiaolong/avbroot/issues/569
 [Issue #578]: https://github.com/chenxiaolong/avbroot/issues/578
+[Issue #593]: https://github.com/chenxiaolong/avbroot/issues/593
+[Issue #603]: https://github.com/chenxiaolong/avbroot/issues/603
+[Issue #607]: https://github.com/chenxiaolong/avbroot/issues/607
+[Issue #610]: https://github.com/chenxiaolong/avbroot/issues/610
 [PR #130]: https://github.com/chenxiaolong/avbroot/pull/130
 [PR #132]: https://github.com/chenxiaolong/avbroot/pull/132
 [PR #133]: https://github.com/chenxiaolong/avbroot/pull/133
@@ -720,3 +745,9 @@ Behind-the-scenes changes:
 [PR #584]: https://github.com/chenxiaolong/avbroot/pull/584
 [PR #586]: https://github.com/chenxiaolong/avbroot/pull/586
 [PR #587]: https://github.com/chenxiaolong/avbroot/pull/587
+[PR #594]: https://github.com/chenxiaolong/avbroot/pull/594
+[PR #605]: https://github.com/chenxiaolong/avbroot/pull/605
+[PR #606]: https://github.com/chenxiaolong/avbroot/pull/606
+[PR #608]: https://github.com/chenxiaolong/avbroot/pull/608
+[PR #611]: https://github.com/chenxiaolong/avbroot/pull/611
+[PR #613]: https://github.com/chenxiaolong/avbroot/pull/613

Cargo.toml

@@ -4,7 +4,7 @@ members = ["avbroot", "e2e", "fuzz", "xtask"]
 resolver = "2"
 
 [workspace.package]
-version = "3.29.0"
+version = "3.31.0"
 license = "GPL-3.0-only"
 edition = "2024"
 repository = "https://github.com/chenxiaolong/avbroot"

README.md

@@ -134,9 +134,26 @@ The commands above are provided for convenience. avbroot is compatible with any
 
 If you lose your AVB or OTA signing key, you will no longer be able to sign new OTA zips. You will have to generate new signing keys and unlock your bootloader again (triggering a data wipe). Follow the [Usage section](#usage) as if doing an initial setup.
 
+### AOSP keys
+
+If you're building Android from source, this is how avbroot's keys correspond to AOSP's keys:
+
+| avbroot        | AOSP                  |
+|----------------|-----------------------|
+| `avb.key`      | `avb.pem`             |
+| `avb_pkmd.bin` | `avb_pkmd.bin`        |
+| `ota.key`      | `releasekey.pk8`      |
+| `ota.crt`      | `releasekey.x509.pem` |
+
+To use an AOSP `releasekey.pk8` key with avbroot, it must be converted from DER to PEM first:
+
+```bash
+openssl pkcs8 -topk8 -scrypt -in releasekey.pk8 -out ota.key -outform PEM
+```
+
 ## Initial setup
 
-1. Make sure that the version of fastboot is 34 or newer. Older versions have bugs that prevent the `fastboot flashall` command (required later) from working properly.
+1. Make sure that the version of fastboot is 35 or newer. Older versions have bugs that prevent the `fastboot flashall` command (required later) from working properly.
 
     ```bash
     fastboot --version
@@ -503,6 +520,16 @@ By default, this only extracts the images that could potentially be patched by a
 
 This command also supports extracting the embedded OTA certificate and AVB public key using the `--cert-ota` and `--public-key-avb` options. To extract only these components, pass in `--none` to skip extracting partition images.
 
+### Listing partitions in OTA
+
+To list all partitions in an OTA, run:
+
+```bash
+avbroot ota list --input ota.zip
+```
+
+The output format is one partition per line with no formatting.
+
 ### Zip write mode
 
 By default, avbroot uses streaming writes for the output OTA during patching. This means it computes the sha256 digest for the digital signature as the file is being written. This mode causes the zip file to contain data descriptors, which is part of the zip standard and works on the vast majority of devices. However, some devices may have broken zip file parsers and fail to properly read OTA zip files containing data descriptors. If this is the case, pass in `--zip-mode seekable` when patching.
@@ -614,4 +641,4 @@ Contributions are welcome! However, I'm unlikely to accept changes for supportin
 
 ## License
 
-avbroot is licensed under GPLv3. Please see [`LICENSE`](./LICENSE) for the full license text.
+avbroot is licensed under GPL-3.0-only. Please see [`LICENSE`](./LICENSE) for the full license text.

README.ru.md

@@ -134,7 +134,7 @@ avbroot совместим с любым стандартным 4096-битны
 
 ## Первоначальная настройка
 
-1. Убедитесь, что вы используете утилиту fastboot версии 34 или новее. Предыдущие версии содержат баги, что не позволяют команде `fastboot flashall` (которая понадобится по ходу инструкции) работать правильно.
+1. Убедитесь, что вы используете утилиту fastboot версии 35 или новее. Предыдущие версии содержат баги, что не позволяют команде `fastboot flashall` (которая понадобится по ходу инструкции) работать правильно.
 
     ```bash
     fastboot --version
@@ -419,6 +419,16 @@ avbroot поддерживает подмену целых образов в OTA
 
 Это не влияет на ход применения пачтей. Например, при использовании Magisk, патч получения root-прав применяется к загрузочному образу одинаково, независимо от того, был ли он получен из оригинального `payload.bin` или это файл, указанный через `--replace`.
 
+### Переподписание разделов
+
+avbroot автоматически переподписывает все разделы в OTA, которые он изменяет. Однако и те разделы, что не были затронуты, также можно переподписать с помощью параметра `--re-sign <имя раздела>`. Это полезно, например, когда OTA содержит разделы, подписанные публичным тестовым ключом AOSP (test key).
+
+### Загрузка подписанных GSI
+
+Функция [динамических обновлений системы (DSU)](https://developer.android.com/topic/dsu?hl=ru) в Android использует корень доверия, отличный от используемого обычной системой. Вместо `avb_custom_key` в загрузчике, она получает доверенные ключи из файлов `first_stage_ramdisk/avb/*.avbpubkey` в ramdisk'е разделов `init_boot` или `vendor_boot`. Эти файлы закодированы в том же двоичном формате, что и файл `avb_pkmd.bin`.
+
+avbroot может добавить пользовательский публичный ключ AVB в эту директорию с помощью параметра `--dsu` при патчинге OTA. Это позволяет запускать [общие образы системы (GSI)](https://developer.android.com/topic/generic-system-image?hl=ru), подписанные пользовательским ключом AVB.
+
 ### Очистка флагов vbmeta
 
 Некоторые сборки Android-прошивок могут поставляться с образом `vbmeta`, в котором флаги установлены таким образом, что AVB фактически отключен. Если avbroot сталкивается с такими образом, процесс патчинга завершается ошибкой с сообщением следующего типа:
@@ -598,5 +608,5 @@ cargo android build --release --target aarch64-linux-android
 
 ## Лицензия
 
-avbroot распространяется по лицензии GPLv3. Полный текст лицензии см. в [`LICENSE`.](./LICENSE)
+avbroot распространяется по лицензии GPL-3.0-only. Полный текст лицензии см. в [`LICENSE`.](./LICENSE)
 

avbroot/Cargo.toml

@@ -34,7 +34,7 @@ memchr = "2.6.0"
 num-bigint-dig = "0.8.4"
 num-traits = "0.2.16"
 passterm = "2.0.3"
-phf = { version = "0.13.1", features = ["macros"] }
+phf = { version = "0.14.0", features = ["macros"] }
 pkcs8 = { version = "0.10.2", features = ["encryption", "pem"] }
 prost = "0.14.1"
 # We can't upgrade to 0.9.0 until rsa updates its rand_core dependency.
@@ -66,7 +66,7 @@ libc = "0.2.158"
 rustix = { version = "1.0.3", default-features = false, features = ["process"] }
 
 [target.'cfg(target_os = "android")'.dependencies]
-system-properties = { git = "https://github.com/chenxiaolong/system-properties", tag = "v0.2.1" }
+system-properties = { git = "https://github.com/chenxiaolong/system-properties", tag = "v0.3.0" }
 
 [build-dependencies]
 constcat = "0.6.0"

avbroot/protobuf/ota_metadata.proto

@@ -104,9 +104,10 @@ message OtaMetadata {
   // The expected device state after the update.
   DeviceState postcondition = 6;
 
-  // True if the ota that updates a device to support dynamic partitions, where
-  // the source build doesn't support it.
-  bool retrofit_dynamic_partitions = 7;
+  // This was previously a boolean flag called retrofit_dynamic_partitions, but
+  // is no longer used or supported.
+  reserved 7;
+
   // The required size of the cache partition, only valid for non-A/B update.
   int64 required_cache = 8;
 

avbroot/protobuf/update_metadata.proto

@@ -82,6 +82,8 @@
 // - REPLACE_XZ: Replace the dst_extents with the contents of the attached
 //   xz file after decompression. The xz file should only use crc32 or no crc at
 //   all to be compatible with xz-embedded.
+// - REPLACE_ZSTD: Replace the dst_extents with the contents of the attached
+//   zstd file after decompression.
 // - PUFFDIFF: Read the data in src_extents in the old partition, perform
 //   puffpatch with the attached data and write the new data to dst_extents in
 //   the new partition.
@@ -174,6 +176,9 @@ message InstallOperation {
     // On minor version 9 or newer, these operations are supported:
     LZ4DIFF_BSDIFF = 12;
     LZ4DIFF_PUFFDIFF = 13;
+
+    // On minor version 10 or newer, these operations are supported:
+    REPLACE_ZSTD = 14;
   }
   required Type type = 1;
 
@@ -376,6 +381,10 @@ message DynamicPartitionMetadata {
   // Max bytes to be compressed at once during ota. Options: 4k, 8k, 16k, 32k,
   // 64k, 128k
   optional uint64 compression_factor = 7;
+
+  // Whether to disable UBLK for OTA. This will force dm-user as OTA backend
+  // choice even if device was configured for UBLK based snapshots.
+  optional bool disable_ublk = 8;
 }
 
 // Definition has been duplicated from

avbroot/src/cli/ota.rs

@@ -2254,11 +2254,58 @@ pub fn verify_subcommand(cli: &VerifyCli, cancel_signal: &AtomicBool) -> Result<
     Ok(())
 }
 
+pub fn list_subcommand(cli: &ListCli) -> Result<()> {
+    let raw_reader = File::open(&cli.input)
+        .with_context(|| format!("Failed to open for reading: {:?}", cli.input))?;
+    let mut buffer = vec![0u8; RECOMMENDED_BUFFER_SIZE];
+    let zip = ZipArchive::from_read_at(&raw_reader, &mut buffer)
+        .with_context(|| format!("Failed to read zip: {:?}", cli.input))?;
+    let mut entries = zip.entries_safe(&mut buffer);
+
+    while let Some((cd_entry, _)) =
+        entries.next_entry().context("Failed to list zip entries")?
+    {
+        let path = cd_entry
+            .file_path_utf8()
+            .context("Zip contains non-UTF-8 paths")?;
+
+        if path != ota::PATH_PAYLOAD {
+            continue;
+        }
+
+        let entry_reader = zip
+            .get_entry(cd_entry.wayfinder())
+            .and_then(|e| zip::verifying_reader(&e, cd_entry.compression_method()))
+            .with_context(|| format!("Failed to open zip entry: {path}"))?;
+        let header = PayloadHeader::from_reader(entry_reader)
+            .context("Failed to load OTA payload header")?;
+
+        for partition in header.manifest.partitions {
+            let name = &partition.partition_name;
+
+            util::path_ensure_single(name)?;
+
+            // Ensure the output cannot be misparsed since we use newlines as a
+            // separator.
+            if name.contains('\n') {
+                bail!("Partition name contains newline: {name:?}");
+            }
+
+            println!("{name}");
+        }
+
+        return Ok(());
+    }
+
+    bail!("Zip entry not found: {}", ota::PATH_PAYLOAD);
+}
+
 pub fn ota_main(cli: &OtaCli, cancel_signal: &AtomicBool) -> Result<()> {
     match &cli.command {
         OtaCommand::Patch(c) => patch_subcommand(c, cancel_signal),
         OtaCommand::Extract(c) => extract_subcommand(c, cancel_signal),
         OtaCommand::Verify(c) => verify_subcommand(c, cancel_signal),
+        OtaCommand::List(c) => list_subcommand(c),
     }
 }
 
@@ -2612,12 +2659,22 @@ pub struct VerifyCli {
     fail_if_missing: bool,
 }
 
+/// List partitions contained in an OTA.
+#[derive(Debug, Parser)]
+pub struct ListCli {
+    /// Path to OTA zip.
+    #[arg(short, long, value_name = "FILE", value_parser)]
+    pub input: PathBuf,
+}
+
+
 #[allow(clippy::large_enum_variant)]
 #[derive(Debug, Subcommand)]
 enum OtaCommand {
     Patch(PatchCli),
     Extract(ExtractCli),
     Verify(VerifyCli),
+    List(ListCli),
 }
 
 /// Patch or extract OTA images.

avbroot/src/cli/payload.rs

@@ -219,8 +219,11 @@ pub fn pack_payload(
 
     for (name, input_file) in &input_files {
         let Some(dpm) = &header.manifest.dynamic_partition_metadata else {
-            continue;
+            break;
         };
+        if !dpm.vabc_enabled() {
+            break;
+        }
 
         if !dpm.groups.iter().any(|g| g.partition_names.contains(name)) {
             continue;

avbroot/src/format/ota.rs

@@ -172,9 +172,6 @@ pub fn parse_legacy_metadata(data: &str) -> Result<OtaMetadata> {
                 }
             }
             "ota-wipe" => metadata.wipe = parse_yes()?,
-            "ota-retrofit-dynamic-partitions" => {
-                metadata.retrofit_dynamic_partitions = parse_yes()?;
-            }
             "ota-downgrade" => metadata.downgrade = parse_yes()?,
             "ota-required-cache" => {
                 metadata.required_cache = value.parse().map_err(|_| unsupported())?;
@@ -240,12 +237,6 @@ fn serialize_metadata(metadata: &OtaMetadata) -> (String, Vec<u8>) {
     if metadata.wipe {
         pairs.insert("ota-wipe".to_owned(), "yes".to_owned());
     }
-    if metadata.retrofit_dynamic_partitions {
-        pairs.insert(
-            "ota-retrofit-dynamic-partitions".to_owned(),
-            "yes".to_owned(),
-        );
-    }
     if metadata.downgrade {
         pairs.insert("ota-downgrade".to_owned(), "yes".to_owned());
     }

avbroot/src/format/payload.rs

@@ -45,7 +45,7 @@ use crate::{
 const PAYLOAD_MAGIC: &[u8; 4] = b"CrAU";
 const PAYLOAD_VERSION: u64 = 2;
 
-const MANIFEST_MAX_SIZE: usize = 4 * 1024 * 1024;
+const MANIFEST_MAX_SIZE: usize = 16 * 1024 * 1024;
 
 /// Size of each extent. This matches what AOSP's delta_generator does. We also
 /// require this to be a multiple of the block size and a multiple of the
@@ -1459,7 +1459,7 @@ pub fn compress_image(
             .collect::<io::Result<Vec<_>>>()
             .map_err(Error::ChunkWrite)?;
 
-        operations.extend(group_operations.into_iter());
+        operations.extend(group_operations);
     }
 
     let digest_uncompressed = context_uncompressed.finish();

avbroot/src/patch/boot.rs

@@ -32,7 +32,7 @@ use crate::{
         avb::{self, AppendedDescriptorMut, Footer, Header},
         bootimage::{self, BootImage, BootImageExt, RamdiskMeta},
         compression::{self, CompressedFormat, CompressedReader, CompressedWriter},
-        cpio::{self, CpioEntry, CpioEntryData},
+        cpio::{self, CpioEntry, CpioEntryData, CpioEntryType},
         zip::{self, ZipEntriesSafeExt, ZipFileHeaderRecordExt, ZipSliceEntriesSafeExt},
     },
     patch::otacert::{self, OtaCertBuildFlags},
@@ -407,7 +407,10 @@ impl MagiskRootPatcher {
             let mut new_path = b".backup/".to_vec();
             new_path.extend(&old_entry.path);
 
-            let new_data = if xz_compress && let CpioEntryData::Data(data) = &old_entry.data {
+            let new_data = if xz_compress
+                && old_entry.file_type == CpioEntryType::Regular
+                && let CpioEntryData::Data(data) = &old_entry.data
+            {
                 new_path.extend(b".xz");
 
                 let reader = Cursor::new(data);
@@ -1415,7 +1418,7 @@ pub fn patch_boot_images<'a>(
 
     // Regroup data so we can parallelize by target.
     let mut groups = HashMap::<&str, (BootImageInfo, Vec<&Box<dyn BootImagePatch + Sync>>)>::new();
-    for (patcher, targets) in patchers.iter().zip(all_targets.into_iter()) {
+    for (patcher, targets) in patchers.iter().zip(all_targets) {
         for target in targets {
             groups
                 .entry(target)

e2e/e2e.toml

@@ -53,11 +53,11 @@ data.ramdisks = [["otacerts", "first_stage", "dsu_key_dir"]]
 
 [profile.pixel_v4_gki.hashes_streaming]
 original = "ea96196191e3a4133db4aff45d47aa3468514e29e0a724faac8081cbf4adf808"
-patched = "c073cbb271099aaf338a8c70f3565ec70db47a87b4c9bb3efffd96ccee1a0c27"
+patched = "6b88761a7927ea4a3fdd953cbb7ffb78361e51648f5ab7a34b0dffad2bc163d5"
 
 [profile.pixel_v4_gki.hashes_seekable]
 original = "f6615ae355eba38689d24aa535981d09175d4832e7c65c04cdd89aa95d21f09d"
-patched = "65ce91a95574a308ffc5279696efe7f6f7e4e847795dc9ae43d9ffd012a7ce3f"
+patched = "e9c0b85a0f9384d3e022f3c83a4653511cd03bfbfa26e660b35cf52986dec888"
 
 # Google Pixel 6a
 # What's unique: boot (boot v4, no ramdisk) + vendor_boot (vendor v4, 2 ramdisks)
@@ -141,11 +141,11 @@ data.ramdisks = [["otacerts", "first_stage", "dsu_key_dir"]]
 
 [profile.pixel_v3.hashes_streaming]
 original = "4b864b906a13ec98b1d6c3440a5eca7404ec63d073080e80a3b84afabcbd2fa2"
-patched = "da3b9a4d6a58bb5aaadfc63006cb36b7af9c1c5945e81b38587c245bd5ad1f5a"
+patched = "cdb9f92e357d8fdf43640704cf7c73917f026e0f759991871ccbab4bfd8c303b"
 
 [profile.pixel_v3.hashes_seekable]
 original = "e0e93ee11de56992f3a5f543858c1d11858b4deb76a91a4caf4fb0f3f34be855"
-patched = "71682a4f31a5920784a0e2e38a6ee5fe3c65213436c7bf1ce347b121cd01a367"
+patched = "bd57432a97feb62c5e4116f0d5bc3b5db61dd46ffa6e436403d197f091a19a55"
 
 # Google Pixel 4a
 # What's unique: boot (boot v2)
@@ -174,8 +174,8 @@ data.deps = ["system"]
 
 [profile.pixel_v2.hashes_streaming]
 original = "abc0ad7f80020018101437c2494d6564a877df663d208a123267fc100734c175"
-patched = "132fb4d90e3c19a7a5ee0da2082767228255228ea5fb66811df1ccd5ff600bb5"
+patched = "deea7ebf412d413743f14792fbf5605dea1a2a099114b33b9e05d464d90e1228"
 
 [profile.pixel_v2.hashes_seekable]
 original = "57d4ac5ab7d362a593f3c02f3d2044b5400c10db0f732741a2d18e025d4231ec"
-patched = "b365eb32f463ac4ba13f7bed4e6535a68483379640dc6e842c591f194db042c2"
+patched = "71f49a27c801b837a219927430b41ba37126084d88d141485263a011d2fb9d70"

e2e/src/main.rs

@@ -664,6 +664,7 @@ fn create_payload(
                     CowVersion::V2 => v.force_compression_factor.then_some(64 * 1024),
                     CowVersion::V3 { compression_factor } => Some(compression_factor.into()),
                 }),
+                disable_ublk: None,
             }),
             partial_update: None,
             apex_info: vec![],
@@ -820,7 +821,6 @@ fn create_ota(
             security_patch_level: ota_info.security_patch_level.clone(),
             partition_state: vec![],
         }),
-        retrofit_dynamic_partitions: false,
         required_cache: 0,
         spl_downgrade: false,
     };