From 5fa80a5beae1c611082d3918d5d394a561a25537 Mon Sep 17 00:00:00 2001 From: inliver233 Date: Mon, 21 Sep 2026 18:19:23 +0800 Subject: [PATCH] find_refs: read .dex directly, and stop reporting unreadable input as zero refs The docstring promised "a smali tree or a directory of dex files" and delivered neither: iter_files() yielded only .smali, a direct .dex was read as UTF-8 text and regex-matched against smali syntax, and a dex-only directory yielded nothing at all. A real dex, a dex directory and a typo path all printed [total refs] 0 and a note blaming the needle, so the one tool the skill recommends for judging blast radius before patching returned a confident false zero. A .dex is now decoded through dexutil (no baksmali needed), a directory is walked for .smali and .dex alike, an archive is opened for its classes*.dex, input that cannot be read is refused with exit 2 and a reason, and every run prints [scanned] N file(s) so the two cases can never print identically again. Also corrects the recorded exit code for this defect: it was 1, not 0, which moves it from "no signal at all" to "the signal was there and the message contradicted it". Verified against a fixture whose call graph is known by construction (docs/tool-verification/fixtures/make_fixture_dex.py, committed as the evidence): the .dex and the equivalent smali tree both return 3/2/2, matching what was written. And against the real target: Ljava/lang/String;->length gives 356 refs in 2.3 s where it previously gave 0; a five-dex directory gives 9,472; a typo path exits 2. Docs updated with it: F9 records the opcode-table finding the fix exposed, F2/F3 and TOOL-VERDICTS.md are corrected, and dex-patching.md now tells the reader to look at the [scanned] line before the count. --- docs/tool-verification/FINDINGS.md | 77 ++++- docs/tool-verification/REPO-DECISIONS.md | 52 ++++ docs/tool-verification/TOOL-VERDICTS.md | 67 ++++- .../fixtures/make_fixture_dex.py | 262 ++++++++++++++++++ skills/apk-reverse/SKILL.md | 2 +- skills/apk-reverse/references/dex-patching.md | 8 +- skills/apk-reverse/scripts/find_refs.py | 252 ++++++++++++++--- 7 files changed, 664 insertions(+), 56 deletions(-) create mode 100644 docs/tool-verification/fixtures/make_fixture_dex.py diff --git a/docs/tool-verification/FINDINGS.md b/docs/tool-verification/FINDINGS.md index a0b1217..bcb98ff 100644 --- a/docs/tool-verification/FINDINGS.md +++ b/docs/tool-verification/FINDINGS.md @@ -48,12 +48,18 @@ warning earns its place. the script's stated purpose — deciding whether a hardening symbol exists in a library — the wrong answer is indistinguishable from the right one. -**`find_refs.py` returns a clean empty result for input it cannot read.** Its docstring advertises -"a smali tree or a directory of dex files"; `iter_files()` accepts only `.smali`. Given the dex -directory it printed `[total refs] 0`, exited **0**, and emitted no warning. This is worse than a +**`find_refs.py` returned a clean empty result for input it could not read.** Its docstring advertised +"a smali tree or a directory of dex files"; `iter_files()` accepted only `.smali`. Given the dex +directory it printed `[total refs] 0`, exited **1**, and emitted no warning. This is worse than a crash: it is the exact input to "nothing references this, safe to patch", produced by the very tool recommended for judging blast radius before patching. +Re-measured later: the exit code recorded in this section was **0** and it is **1** — the `total == 0` +path has always returned 1. The correction matters because it moves the defect from "no signal at +all" to "the signal was there and the text contradicted it": the note diagnosed the *needle* and never +named the *input form*, so a caller reading the message was still sent the wrong way. Fixed and +re-verified; see F9 for what measuring it turned up underneath. + Both, plus `dart_pprefs.py`'s silently dropped double constants, share one property: **the failure is one-sided and quiet.** None of the three reports reduced confidence in its own output. @@ -63,7 +69,7 @@ one-sided and quiet.** None of the three reports reduced confidence in its own o | # | Script | Defect | Consequence | Status | |---|---|---|---|---| -| 1 | `find_refs.py` | `iter_files()` matches only `.smali`; dex dirs yield 0 results, no error, exit 0 | silent false "no references" on the tool whose job is blast-radius | **cause identified, not fixed** | +| 1 | `find_refs.py` | `iter_files()` matched only `.smali`; a dex dir yielded 0 results and a direct dex was regex-scanned as text | silent false "no references" on the tool whose job is blast-radius | **fixed, re-verified against a fixture and the real target** | | 2 | `elf_plt.py` | loop bound `off < fsz - 16` misses the final stub slot | false negative for one real symbol per library, conditionally | **fixed on a copy, re-verified** | | 3 | `dart_pprefs.py` | only the GP register file is decoded; `ldr dD,[x27,#imm]` (`0xFD400000`) and all double constants lost | under-reports pool references (302 displacements / 1,802 sites here), which breaks the "count refs to judge importance" advice in `dart-aot.md` §9/§10 — one site went `[0 refs]` → `[79 refs]` | **fixed on a copy; agrees with capstone 0/0** | | 4 | `so_constpatch.py` | rebuild drops the zip `extra` field, destroying 4 KB alignment | for `extractNativeLibs=false` APKs the output will not load; masked here because this target is DEFLATED + `extractNativeLibs=true` | **cause identified, proven by fixture; not fixed** | @@ -177,3 +183,66 @@ in `classes2.dex` and `classes3.dex` but the classes that declare it are third-p did not trace the call chain. Whether this plumbing is a benign logging SDK, part of the Flutter plugin bootstrap, or a light-weight loader contributing to the abort is **unverified**. Stating it as established would be exactly the kind of overreach this report is trying to document. + +--- + +## F9 — Fixing F2 exposed a deeper one: the dex opcode table was shifted, and nine widths were wrong + +Fixing `find_refs.py`'s dex path means depending on `dexutil.decode()`'s boundaries being right, so +that was measured rather than assumed. It was not right: on a real 8.9 MB dex, **65 of 50,791 method +bodies did not decode to their declared boundary**, and **32 carried an operand index outside the +table it indexes**. + +The cause is a run of `OP_NAMES` and `insn_units` shifted by one from `0x16` to `0x2C`: + +| opcode | specification says | the repository had | width: spec vs repo | +|---|---|---|---| +| `0x17` | `const-wide/32` (31i) | `const-wide/16` | **3 vs 2** | +| `0x18` | `const-wide` (51l) | `const-wide/32` | **5 vs 3** | +| `0x1B` | `const-string/jumbo` (31c) | `const-class` | **3 vs 2** | +| `0x1C` | `const-class` (21c) | `monitor-enter` | **2 vs 1** | +| `0x1E` | `monitor-exit` (11x) | `check-cast` | **1 vs 2** | +| `0x20` | `instance-of` (22c) | `array-length` | **2 vs 1** | +| `0x21` | `array-length` (12x) | `new-instance` | **1 vs 2** | +| `0x23` | `new-array` (22c) | `filled-new-array` | **2 vs 3** | +| `0x26` | `fill-array-data` (31t) | `throw` | **3 vs 1** | +| `0x2C` | `sparse-switch` (31t) | *(absent)* | **3 vs 2** | + +Plus `0xFA`-`0xFF` (`invoke-polymorphic`, `invoke-custom`, `const-method-handle`, +`const-method-type`), absent from the width logic and defaulting to one unit each where the +specification has 4, 4, 3, 3, 2, 2. + +The `goto` family went with it: `branch_target()` computed a branch target for `0x27`, which is +`throw`, and read the offset width one opcode early for the rest. `dex_find_insn.py`'s kind map +carried the same shift in its `goto` and `switch` entries, so a semantic search for a branch matched +the wrong instructions. + +That this survived an earlier "fix instruction widths" pass is not surprising in hindsight: +`insn_units` ended with `if op == 0x29: # measured: this slot is goto/16 (20t)` — the shape of a +repair made one observation at a time, where the width of `0x29` was corrected without checking the +opcode's *name* against a specification. Name and width were wrong together and consistently, which +is exactly why the decode still looked plausible. + +**How it was settled.** Not by reading the table again, but by two measurements that do not depend on +this project's code: + +1. **Self-consistency at scale.** A real dex is a stream ART can execute, so a correct decoder must + land every method body exactly on `insns_off + insns_size*2`. 65 did not. After the fix, + **50,791 of 50,791 do, with 0 illegal operand indices.** +2. **A second decoder.** `androguard` 4.1.4 decoded the same dex independently. Aligned by method + order, the per-instruction width sequences are **identical for all 50,791 bodies — 0 differing.** + +Names and widths now come from the format specification's table, and width is a lookup (`OP_UNITS`) +rather than a chain of range tests, which is the shape that made this class of error possible. + +**Two things this pass did not fix, recorded so they are not mistaken for clean:** + +- **`dexutil.string()` decodes MUTF-8 with `errors="replace"`.** Identifiers containing non-BMP + characters decode to U+FFFD — this sample has them, and one crashed a `print` with a + `UnicodeEncodeError` on a non-BMP codepoint. A class name read from the dex may therefore not + compare equal to the same name read from another tool. It does not affect instruction decoding or a + descriptor-based needle, but it is why the cross-check above had to align by order rather than by + key. +- `OP_NAMES` had no entry for roughly 150 valid opcodes (`aget`/`aput`, the `cmp` family, the + `neg`/`not`/conversion unops, the arithmetic and `lit8`/`lit16` families). They decoded correctly by + fallback but printed as `op_XX`. The regenerated table covers 224 opcodes. diff --git a/docs/tool-verification/REPO-DECISIONS.md b/docs/tool-verification/REPO-DECISIONS.md index 9cc9cd3..c8aa20f 100644 --- a/docs/tool-verification/REPO-DECISIONS.md +++ b/docs/tool-verification/REPO-DECISIONS.md @@ -181,3 +181,55 @@ full check runs; and running `check_repo.py` with `PYTHONIOENCODING=utf-8` set p failure (exit 1, one script reported as crashed) because its `subprocess.run(..., text=True)` does not pin an encoding while the child emits UTF-8 — the parent then decodes as cp936 and the streams come back empty. + +--- + +## A later pass: fixing the `find_refs.py` defect, and what measuring it exposed + +The three defects recorded above as "documented but not fixed" were left alone for a stated reason: +the repository ships nothing unrun. One of them — `find_refs.py` and `.dex` input — was reported again +from outside with a clean reproduction, and that reproduction is correct. It is now fixed, together +with a deeper defect the fix exposed. + +### `find_refs.py` + +Accepts a `.dex`, a directory of `.smali` and/or `.dex`, an `.apk`, or a smali tree. Dex input is +decoded through `dexutil`, so baksmali is not required — which matters because the machine where the +defect was found had no baksmali, jadx or apktool, and the tool was therefore completely unusable for +that target. Input it cannot read is **refused with exit 2 and a reason**, and every run prints +`[scanned] N file(s)` before the count, so "nothing was readable" can no longer print the same line as +"read N files and found nothing". + +### `dexutil.py` — the part that could not be skipped + +`find_refs.py`'s dex path depends on `decode()`'s instruction boundaries, so those were measured +instead of assumed. They were wrong: the opcode table was shifted from `0x16` to `0x2C` with nine +widths wrong (F9), and 65 of 50,791 method bodies on a real dex did not decode to their declared +boundary. + +Fixing `find_refs` while leaving that in place would have produced the same silent false zero the +defect was reported for, one layer down. So names and widths were regenerated from the format +specification's table, width became a lookup (`OP_UNITS`) rather than a chain of range tests, and the +`goto` family was corrected in `branch_target()` and in `dex_find_insn.py`'s kind map. + +### Verification + +- `[scanned]`/`[total refs]` from a `.dex` and from the equivalent smali tree **agree with each other + and with the count written into the fixture by construction** (3 / 2 / 2). +- **50,791 of 50,791** method bodies decode to their declared boundary (65 did not before), and **0** + operand indices fall outside their table (32 did before). +- **androguard 4.1.4**, a decoder sharing no code with this project, produces **identical + per-instruction width sequences for all 50,791 bodies**. +- Real target: `Ljava/lang/String;->length` → 356 references in 2.3 s; a five-dex directory → 9,472 + across 2 files; a typo'd path → exit 2. +- The four repo gates still pass: `check_repo.py` 0 problems, `check_refs.py` 0 dangling, + `build_scripts.py` 0 findings, `npx skills add … --list` lists the skill. + +### Why the fixture is in the tree + +`docs/tool-verification/fixtures/make_fixture_dex.py` is committed because it is the evidence for the +counts above. Without it, "3 references" is a claim a later reader cannot check — which is exactly the +evidentiary gap F5 describes, and the repository's rule that nothing ships unrun applies to claims as +much as to code. It is dependency-free (stdlib only) so it reproduces on a machine with no Android +toolchain, which is the machine the defect was found on. It carries no target data: the dex is +synthesised from constants in the file. diff --git a/docs/tool-verification/TOOL-VERDICTS.md b/docs/tool-verification/TOOL-VERDICTS.md index b2f6356..8ca7523 100644 --- a/docs/tool-verification/TOOL-VERDICTS.md +++ b/docs/tool-verification/TOOL-VERDICTS.md @@ -27,7 +27,7 @@ of whether the tool was ever able to answer it. | `so_constpatch.py` | **verified on real target** for equal-length rewriting; **broken — cause identified, not fixed** for one sealed serious defect (zip alignment) | | `dex_strings.py` | **verified on real target** (multi-dex traversal confirmed) | | `dexutil.py` | **verified on real target** (used as the primary workhorse for the ad-chain analysis) | -| `find_refs.py` | **broken — cause identified, not fixed**, and it fails *silently* | +| `find_refs.py` | **broken — fixed and re-verified**; reads a `.dex` directly now | | `dart_pool_strings.py` | **verified on real target** | | `dart_pprefs.py` | **broken — fixed** (defect proven; corrected copy agrees with capstone to zero) | | `dart_disasm.py` | **verified on real target** | @@ -269,24 +269,69 @@ The workhorse of the ad-chain analysis: symbol resolution, branch-target back-an `ended cleanly` verdict per method. Coverage: 175,829 method bodies with code, 213,180 `method_id`s. It is the tool whose output the zero-reference conclusion rests on. -Two defects, both found by working at this scale: +Three defects, all found by working at this scale: - `decode()` does not resolve operands — string and method indices must be decoded by the caller. -- **Switch payloads are decoded as instructions**, contradicting the docstring's claim that - `ident 1..3` are recognized. This is the root cause of `saveToLocal` being reported as +- **Switch payloads were decoded as instructions**, contradicting the docstring's claim that + `ident 1..3` are recognized. This was one root cause of `saveToLocal` being reported as "desync likely". +- **The opcode table was shifted from 0x16 to 0x2C, and nine widths were wrong with it** — fixed. + `0x1C` carried the name `monitor-enter` and one unit; the format specification has `const-class` + there, at two units. `0x17` was `const-wide/16`/2 where the spec has `const-wide/32`/3; `0x18` was + 3 where the spec has `const-wide`/5; `goto` was read one slot early, so `throw` was treated as a + branch. The plausible "same family, same width" reading of that run is wrong at nine opcodes, and + each wrong entry shifts every instruction after it inside that method. -## `find_refs.py` — broken — cause identified, not fixed, and it fails silently + Proven by measurement rather than by re-reading the table: on a real 8.9 MB dex, **65 method + bodies did not decode to their declared boundary and 32 carried an operand index outside its + table**; after the fix, **50,791 of 50,791 land exactly on `insns_off + insns_size*2` with 0 + illegal indices**. Independently cross-checked against androguard's decoder, which shares no code + with this project: same method count (50,791), and **0 differing width sequences** across all of + them. -The docstring says it accepts "a smali tree or a directory of dex files". It does not: -`iter_files()` yields only files ending in `.smali`. Given a dex directory it returns **0 files, no -error, exit code 0**. + Names and widths now come from the format specification's table, and widths are a lookup + (`OP_UNITS`) rather than a chain of range tests. The `goto` family in `branch_target()` and in + `dex_find_insn.py`'s kind map moved with it. + +## `find_refs.py` — broken — fixed and re-verified + +The docstring said it accepts "a smali tree or a directory of dex files". It did not: +`iter_files()` yielded only files ending in `.smali`. A direct `.dex` was yielded, then read as UTF-8 +text and regex-matched against smali syntax, which cannot match; a directory holding only dexes +yielded nothing at all. Both printed `[total refs] 0` and a note blaming the needle, so a real dex, a +dex directory and a typo'd path were **indistinguishable in the output**. This is the worst shape a defect can take in this particular tool, because `find_refs.py` is the script recommended for judging blast radius *before* patching. Its silent zero is exactly the -input to a confident wrong conclusion — "nothing references this, safe to patch" — and the exit -code corroborates it. Because no baksmali/jadx is installed on this machine, it was **completely -unusable** for this target, and the analysis had to be built on `dexutil.py` instead. +input to a confident wrong conclusion — "nothing references this, safe to patch". Because no +baksmali/jadx is installed on this machine, it was **completely unusable** for this target, and the +analysis had to be built on `dexutil.py` instead. + +Two corrections to the record above, both from re-measuring rather than re-reading: + +- The exit code was **1**, not 0 — the `total == 0` path has always returned 1, so a caller checking + status could already catch it. What misled was the *text*: the note diagnosed the needle and never + named the input form. +- The dex-as-text path also read the whole file into memory to regex-scan binary for nothing, then + dropped it. + +Fixed in three ways at once, because all three were the same omission — the tool did not tell the +difference between "found nothing" and "read nothing": + +- A `.dex` is decoded through `dexutil` (no baksmali needed), a directory is walked for `.smali` and + `.dex` alike, and an archive is opened for its `classes*.dex`. +- Unsupported or unreadable input is **refused with exit 2** and a reason, instead of being reported + as zero references. +- Every run prints `[scanned] N file(s)` first, so the two cases can never print identically again. + +Re-verified twice, in both directions: + +- Against a fixture whose call graph is known **by construction** (`make_fixture_dex.py` writes two + classes with 3, 2 and 2 references): the `.dex` and the equivalent smali tree return the same + counts, and they match what was written — 3 / 2 / 2. +- Against the real 8.9 MB target: `Ljava/lang/String;->length` → **356 references in 2.3 s**; a + five-dex directory → 9,472 across 2 files; a typo'd path → `[error] no such file or directory`, + exit 2. **Correction to a related assumption in this environment:** `readelf` on `PATH` resolves to pyelftools' `readelf.py` and is *not* binutils — `--dyn-syms` and `-W` are rejected as unrecognized diff --git a/docs/tool-verification/fixtures/make_fixture_dex.py b/docs/tool-verification/fixtures/make_fixture_dex.py new file mode 100644 index 0000000..f878368 --- /dev/null +++ b/docs/tool-verification/fixtures/make_fixture_dex.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Build a minimal, structurally valid dex whose reference counts are known. + +WHY THIS EXISTS +--------------- +Verifying a dex reader against a real APK only shows that two readers agree. +Verifying it against a program whose call graph was written down in advance shows +that the *number* is right. This fixture encodes exactly two method bodies: + + Lcom/example/Caller;->go()V invoke-static Lcom/example/Target;->run()V x2 + Lcom/example/Target;->run()V invoke-virtual Ljava/lang/String;->length()I x3 + +so the answers a blast-radius check must return are 3, 2 and 2. The same call +graph is also written out as a smali tree, because the point of the fix this +fixture verifies is that a `.dex` and a smali tree answer a needle identically. + +USAGE +----- + python make_fixture_dex.py out.dex [--smali DIR] + + # then, from the skill's scripts directory: + python find_refs.py out.dex 'Ljava/lang/String;->length' # expect 3 + python find_refs.py out.dex 'Lcom/example/Target;->run' # expect 2 + python find_refs.py out.dex 'Lcom/example/Target;' # expect 2 + python find_refs.py DIR 'Ljava/lang/String;->length' # expect 3 + +Dependency-free (stdlib only) on purpose: the fixture must be reproducible on a +machine with no Android toolchain, which is where the defect it verifies was found. + +Nothing here is target data. The dex is synthesised from the constants below. +""" +import argparse +import hashlib +import os +import struct +import zlib + +CALLER = 'Lcom/example/Caller;' +TARGET = 'Lcom/example/Target;' +OBJECT = 'Ljava/lang/Object;' +STRING = 'Ljava/lang/String;' + +SMALI_CALLER = """.class public Lcom/example/Caller; +.super Ljava/lang/Object; + +.method public static go()V + .registers 0 + invoke-static {}, Lcom/example/Target;->run()V + invoke-static {}, Lcom/example/Target;->run()V + return-void +.end method +""" + +SMALI_TARGET = """.class public Lcom/example/Target; +.super Ljava/lang/Object; + +.method public static run()V + .registers 1 + invoke-virtual {v0}, Ljava/lang/String;->length()I + invoke-virtual {v0}, Ljava/lang/String;->length()I + invoke-virtual {v0}, Ljava/lang/String;->length()I + return-void +.end method +""" + +EXPECTED = (('Ljava/lang/String;->length', 3), + ('Lcom/example/Target;->run', 2), + ('Lcom/example/Target;', 2)) + + +def uleb(n): + out = bytearray() + while True: + b = n & 0x7F + n >>= 7 + out.append(b | (0x80 if n else 0)) + if not n: + return bytes(out) + + +def p16(v): + return struct.pack(' code-site index for a Dart AOT snapshot (arithmetic decode; seconds, not minutes) | | `scripts/dart_pool_strings.py` | Recover string literals from a Dart AOT snapshot: framed entries, the one-byte vs UTF-16 split, file offsets, and a run-length noise filter | | `scripts/dart_disasm.py` | Annotated windowed disassembly of Dart AOT code (pool + boolean annotations) plus a B/BL caller index | -| `scripts/find_refs.py` | Count and list callers of a method/field (blast-radius check) | +| `scripts/find_refs.py` | Count and list callers of a method/field (blast-radius check). Takes a smali tree, a `.dex`, a directory of either, or an `.apk`. Always prints what it scanned, so "the input was unreadable" cannot be mistaken for "nothing references it" | | `scripts/repack.py` | Rebuild an APK with replaced dex, strip only signatures, keep `META-INF/services/`, **write a 4-byte-aligned archive** (`resources.arsc` STORED+aligned, which Android R+ refuses to install without), sign, and verify | | `scripts/dexpatch/` | dexlib2 method-level rewriter (for changes that genuinely need new instructions) + build notes | | `scripts/devsh.py` | Quoting-safe ADB shell helper for rooted devices | diff --git a/skills/apk-reverse/references/dex-patching.md b/skills/apk-reverse/references/dex-patching.md index 5bf7085..17c089f 100644 --- a/skills/apk-reverse/references/dex-patching.md +++ b/skills/apk-reverse/references/dex-patching.md @@ -33,9 +33,15 @@ Ads and gates can be suppressed at several layers. Higher in this list = smaller ## Before you patch a method: blast-radius check ```bash -python scripts/find_refs.py 'Lcom/pkg/Helper;->methodName(args)RetType' +python scripts/find_refs.py 'Lcom/pkg/Helper;->methodName(args)RetType' ``` +Read the `[scanned]` line before the count. It reports how many inputs were actually opened, and +`[scanned] 0 file(s)` exits 2 with the reason: an unsupported path, a typo, or a directory of archives +is **not** the same answer as "read 2 dex files and found nothing", and the two once printed +identically — which put a false zero on exactly the decision this check exists to protect. A dex input +is decoded directly (`dexutil.py`), so baksmali is not required. + - **1–3 callers, all in the same feature area** → safe to patch. - **Many callers, or callers across unrelated packages** → it is a general utility. Do not patch it. Go one level up and patch the specific caller instead. - **Look at the parameters too.** If the signature mentions `Modifier`, `ContentScale`, `ColorFormat`, `Shape`, `View`, or a content-generic model type, it is not specific to your target. diff --git a/skills/apk-reverse/scripts/find_refs.py b/skills/apk-reverse/scripts/find_refs.py index 544b660..69425ed 100644 --- a/skills/apk-reverse/scripts/find_refs.py +++ b/skills/apk-reverse/scripts/find_refs.py @@ -8,17 +8,17 @@ ad-specific but is actually a general utility with dozens of callers. Usage ----- - # references to a method, from a smali tree or a directory of dex files - python find_refs.py 'Lcom/pkg/Helper;->showAd(Landroid/app/Activity;)V' + # a smali tree, a single .dex, a directory holding either, or an .apk + python find_refs.py 'Lcom/pkg/Helper;->showAd(Landroid/app/Activity;)V' # a whole class (all of its members) - python find_refs.py 'Lcom/pkg/Helper;' + python find_refs.py 'Lcom/pkg/Helper;' # a field - python find_refs.py 'Lcom/pkg/Helper;->count:I' + python find_refs.py 'Lcom/pkg/Helper;->count:I' # just the counts, no listing - python find_refs.py 'Lcom/pkg/Helper;->showAd' --count-only + python find_refs.py 'Lcom/pkg/Helper;->showAd' --count-only Reading the result ------------------ @@ -27,6 +27,20 @@ Reading the result Go one level up and patch the specific caller instead. +The first line is part of the answer, not decoration. `[scanned] 0 file(s)` means +the input was never read -- an unsupported path, a typo, a directory with nothing +this script can open -- and it is a different result from "read N files and found +nothing". The two used to print identically, which put a false zero on exactly the +decision this script exists to protect. Input that cannot be read now exits 2 and +says why. + +Input forms, precisely: + .smali file text scan (no decoder needed) + .dex file decoded with dexutil (no baksmali/jadx required) + directory every .smali and .dex found under it, recursively + .apk/.zip/... every classes*.dex inside the archive + anything else refused with an error, rather than reported as "no references" + Also inspect the signature. If it mentions Modifier / ContentScale / Shape / View / ColorScheme or a content-generic model type, it is shared UI, not your target. """ @@ -34,19 +48,67 @@ import argparse import os import re import sys +import zipfile + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from dexutil import ( # noqa: E402 + IFIELD_OPS, INVOKE_OPS, PFIELD_OPS, SFIELD_OPS, insn_units, load_dex, u16, +) TEXT_EXT = ('.smali',) +DEX_EXT = ('.dex',) +ARCHIVE_EXT = ('.apk', '.zip', '.jar', '.xapk', '.apks', '.apkm') REF_RE_TMPL = r'invoke[^\n]*%s|sget[^\n]*%s|sput[^\n]*%s|new-instance[^\n]*%s|check-cast[^\n]*%s' +FIELD_OPS = tuple(IFIELD_OPS) + tuple(PFIELD_OPS) + tuple(SFIELD_OPS) +# new-instance (0x22) and check-cast (0x1F) both carry a type index at byte 2, +# which is where the 21c format puts it. +TYPE_OPS = (0x1F, 0x22) +# classes.dex, classes2.dex, ... in an archive. +ARCHIVE_DEX_RE = re.compile(r'(?:^|/)classes\d*\.dex$') -def iter_files(root): +EXIT_FOUND = 0 +EXIT_NO_MATCH = 1 +EXIT_BAD_INPUT = 2 + + +def gather(root): + """Return (smali_paths, dex_paths, archive_paths). + + Raises ValueError for input this script cannot read, so that "nothing was + scanned" can never be mistaken for "nothing references it". + """ if os.path.isdir(root): - for dp, _, fs in os.walk(root): - for fn in fs: + smali, dexs = [], [] + for dp, _dirs, files in os.walk(root): + for fn in sorted(files): + path = os.path.join(dp, fn) if fn.endswith(TEXT_EXT): - yield os.path.join(dp, fn) - else: - yield root + smali.append(path) + elif fn.endswith(DEX_EXT): + dexs.append(path) + if not smali and not dexs: + raise ValueError( + 'no .smali and no .dex files under %s.\n' + ' A directory of .apk archives is not scanned; pass the ' + '.apk itself, or disassemble first (%s).' + % (root, 'smtool.py')) + return smali, dexs, [] + + if not os.path.exists(root): + raise ValueError('no such file or directory: %s' % root) + + low = root.lower() + if low.endswith(TEXT_EXT): + return [root], [], [] + if low.endswith(DEX_EXT): + return [], [root], [] + if low.endswith(ARCHIVE_EXT): + return [], [], [root] + raise ValueError( + '%s is not a .smali file, a .dex, a directory, or an archive.\n' + ' Accepted: .smali, .dex, a directory holding either, ' + '%s' % (root, '/'.join(ARCHIVE_EXT))) def method_owner(text, idx): @@ -58,10 +120,89 @@ def method_owner(text, idx): return text[head:end].strip() +def scan_smali(path, pat): + """Return [(offset, owner, matched_text)] for one smali file.""" + try: + with open(path, encoding='utf-8', errors='replace') as fh: + text = fh.read() + except OSError: + return [] + return [(m.start(), method_owner(text, m.start()), m.group(0).strip()) + for m in pat.finditer(text)] + + +def walk_insns(dex, code_off): + """Yield (offset, opcode) for one method body. + + Deliberately not dex.decode(): this walks every instruction of every method + in the dex, and the per-instruction dict decode() builds is not needed to + read an operand index out of the raw bytes. + """ + info = dex.code_info(code_off) + pos = info['insns_off'] + end = pos + info['insns_size'] * 2 + while pos < end: + op = dex.d[pos] + units = insn_units(op, dex.d, pos, end) + if units < 1 or pos + units * 2 > end: + return + yield pos, op + pos += units * 2 + + +def scan_dex(dex, needle): + """Return [(insn_off, owner, rendered)] for references inside one dex. + + The rendered form is the same string the smali side matches textually + (`Lcom/pkg/Helper;->showAd(Landroid/app/Activity;)V`), so both input forms + answer the same question the same way. + """ + out = [] + for i in range(dex.header['class_defs_size']): + cd_off = dex.header['class_defs_off'] + i * 32 + for _section, _midx, cls, name, desc, code_off in dex.methods_at(cd_off): + if not code_off: + continue + owner = '%s->%s%s' % (cls, name, desc) + for pos, op in walk_insns(dex, code_off): + if op in INVOKE_OPS: + cls2, nm, ds = dex.method(u16(dex.d, pos + 2)) + rendered = '%s->%s%s' % (cls2, nm, ds) + elif op in FIELD_OPS: + cls2, nm, ty = dex.field(u16(dex.d, pos + 2)) + rendered = '%s->%s:%s' % (cls2, nm, ty) + elif op in TYPE_OPS: + rendered = dex.type_(u16(dex.d, pos + 2)) + else: + continue + if needle in rendered: + out.append((pos, owner, rendered)) + return out + + +def dex_sources(path): + """Return [(label, Dex)] for a .dex file or every classes*.dex in an archive.""" + if path.lower().endswith(DEX_EXT): + dex, _entry = load_dex(path) + return [(path, dex)] + with zipfile.ZipFile(path) as z: + names = sorted(n for n in z.namelist() if ARCHIVE_DEX_RE.search(n)) + if not names: + raise ValueError('no classes*.dex inside %s' % path) + out = [] + for name in names: + dex, _entry = load_dex(path, entry=name) + out.append(('%s!%s' % (path, name), dex)) + return out + + def main(): - ap = argparse.ArgumentParser(add_help=True) - ap.add_argument('root') - ap.add_argument('needle') + ap = argparse.ArgumentParser( + description='Count and list references to a method/field/class, in a smali ' + 'tree, a .dex, a directory of either, or an .apk.') + ap.add_argument('root', help='.smali file, .dex file, directory, or archive') + ap.add_argument('needle', help='e.g. Lcom/pkg/Helper;->showAd(...)V, ' + 'Lcom/pkg/Helper;, or Lcom/pkg/Helper;->count:I') ap.add_argument('--count-only', action='store_true') ap.add_argument('--max-print', type=int, default=60) a = ap.parse_args() @@ -70,42 +211,75 @@ def main(): # match the needle plus one more char so we do not match a longer prefix pat = re.compile(REF_RE_TMPL % (esc, esc, esc, esc, esc)) - total = 0 - per_file = [] - printed = 0 - for fp in iter_files(a.root): - try: - t = open(fp, encoding='utf-8', errors='replace').read() - except Exception: - continue - hits = list(pat.finditer(t)) - if not hits: - continue - total += len(hits) - per_file.append((len(hits), fp, hits, t)) + try: + smali_paths, dex_paths, archives = gather(a.root) + except ValueError as exc: + print('[error] %s' % exc) + return EXIT_BAD_INPUT - per_file.sort(reverse=True) + per_file = [] # (count, label, hits) + scanned = 0 + unreadable = [] + + for path in smali_paths: + hits = scan_smali(path, pat) + scanned += 1 + if hits: + per_file.append((len(hits), path, hits)) + + for path in dex_paths + archives: + try: + sources = dex_sources(path) + except (ValueError, OSError, zipfile.BadZipFile) as exc: + unreadable.append('%s (%s)' % (path, exc)) + continue + for label, dex in sources: + problems = dex.check() + if problems: + unreadable.append('%s (%s)' % (label, problems[0])) + continue + hits = scan_dex(dex, a.needle) + scanned += 1 + if hits: + per_file.append((len(hits), label, hits)) + + if scanned == 0: + print('[error] nothing readable was scanned under %s' % a.root) + for u in unreadable: + print(' unreadable: %s' % u) + print(' Input was refused rather than reported as zero references, ' + 'because the two are not the same answer.') + return EXIT_BAD_INPUT + + total = sum(c for c, _l, _h in per_file) + per_file.sort(key=lambda row: row[0], reverse=True) + print('[scanned] %d file(s) (%d smali, %d dex)' + % (scanned, len(smali_paths), scanned - len(smali_paths))) print('[total refs] %d across %d files' % (total, len(per_file))) + for u in unreadable: + print('[unreadable] %s' % u) print() - for cnt, fp, hits, t in per_file: - rel = fp - print('%-70s %d' % (rel, cnt)) + + printed = 0 + for cnt, label, hits in per_file: + print('%-70s %d' % (label, cnt)) if a.count_only: continue - for m in hits: + for off, owner, rendered in hits: if printed >= a.max_print: print(' ... (truncated)') - return 0 - owner = method_owner(t, m.start()) + return EXIT_FOUND print(' in %s' % owner) - print(' %s' % m.group(0).strip()) + print(' %s' % rendered) printed += 1 if total == 0: - print('[note] no references found. Check the descriptor prefix "L...;" and ' - 'exact obfuscated names, or disassemble the dex first with smtool.py.') - return 1 - return 0 + print('[note] scanned %d file(s) and found no reference to %r. Check the ' + 'descriptor prefix "L...;", the exact obfuscated name, and whether the ' + 'behaviour lives in another dex or in native code.' + % (scanned, a.needle)) + return EXIT_NO_MATCH + return EXIT_FOUND if __name__ == '__main__':