diff --git a/build_scripts.py b/build_scripts.py index d6547ce..977848c 100644 --- a/build_scripts.py +++ b/build_scripts.py @@ -9,8 +9,12 @@ DeepSeek Harness). Anything machine-specific that survives the copy becomes a landmine: a baked-in JDK path, a keystore password, a home directory that only exists on the author's box. This tool does two things: - 1. AUDIT (default, read-only): report every absolute-path literal and every - credential-looking constant, with file and line number. + 1. AUDIT (default, read-only): report every absolute-path literal that names + *this* machine, and every credential-looking constant, with file and line + number. Generic install locations (`Program Files` on Windows, `/usr/bin` + on POSIX) and already-redacted paths (carrying a `` or an + ellipsis) are exempt -- see `GENERIC_PATH_PREFIXES` and `_leaks_nothing`, + which exist because a gate whose output is mostly noise stops being read. 2. SYNC (--apply): copy matching files to the destination, optionally after neutralising the absolute-path literals it found (--scrub). @@ -67,6 +71,85 @@ CREDENTIAL_HINT = re.compile( r"keypass\s*=\s*['\"][^'\"]|password\s*=\s*['\"][^'\"]|" r"api[_-]?key\s*=\s*['\"][^'\"]|secret\s*=\s*['\"][^'\"]{6,})") +# --------------------------------------------------------------------------- +# What is NOT a machine-specific leftover. +# +# This audit exists to catch paths that leak *this* machine's layout. Two +# families look like absolute paths but leak nothing, and reporting them is +# what made an earlier version of this gate unreadable: 37 findings on the pass +# that introduced the evidence record, of which the large majority were these +# two families. A gate whose output is mostly noise is a gate people stop +# reading, so the exemptions are explicit and auditable rather than silent. +# +# 1. Generic install locations. `C:\Program Files\...` exists on every Windows +# host and `/usr/bin/...` on every POSIX host; naming one names the +# platform, not the machine. Same for a distribution's own paths. +# `/data/...` is already excluded above for the same reason -- it is a +# device path, not a host path. +# +# 2. Already-redacted paths. A path carrying a `` or an ellipsis +# has been through a normalisation pass deliberately; re-reporting it +# punishes the correct behaviour. `scan_leaks.py` reads the same signal as +# PLACEHOLDER_TOKENS for target identity. +# +# What remains is a drive or home path with no placeholder: a real workbench +# location. That is the thing worth failing on, and the thing to fix in the +# document rather than in this list. +GENERIC_PATH_PREFIXES = ( + # Windows, generic + 'c:\\program files', 'c:\\program files (x86)', 'c:\\windows', + 'c:\\python', 'c:\\mingw', 'c:\\msys', 'c:\\cygwin', 'c:\\strawberry', + 'c:\\android', 'c:\\android-sdk', 'c:\\users\\public', 'c:\\users\\default', + # POSIX, generic + '/usr/', '/opt/', '/etc/', '/var/', '/tmp/', '/bin/', '/sbin/', +) + +# ``, ``, ``, `` -- a placeholder is the mark of a path +# that has already been made portable on purpose. +PLACEHOLDER_RE = re.compile(r'<[A-Za-z_][A-Za-z0-9_.\-]*>') +# A literal ellipsis in a path means the middle was deliberately elided. +ELLIPSIS_RE = re.compile(r'\.\.\.|\u2026') + + +def _leaks_nothing(path): + """True when an absolute path says nothing about this particular machine. + + Kept separate from the matching rules so the exemption can be tested and + argued with on its own, rather than being folded into the regex. + + The comparison is **bidirectional on purpose**. `BARE_ABS` stops at + whitespace (it cannot know where a path ends), so a real + `C:\\Program Files\\...` arrives here as the truncated `C:\\Program`. A + one-directional `startswith` then misses the exemption and reports a path + that exists on every Windows host. Requiring a minimum length keeps the + reverse direction from turning a short drive prefix into a blanket pass. + """ + low = path.lower().replace('/', '\\') + for prefix in GENERIC_PATH_PREFIXES: + p = prefix.replace('/', '\\') + if low.startswith(p): + return True + if len(low) >= 8 and p.startswith(low): + return True + # A bare user-directory root names no one -- only the segment after it does. + # `C:\Users\` appears legitimately when a document describes a rule rather + # than quoting a location, so it is exempt, while a personal name in that + # position still reports. Phrased without a placeholder on purpose: + # `scan_leaks.py` recognises a fixed token list, and an invented one reads + # to it as the very name this rule is trying to catch. + # + # The lookahead must exclude a following separator as well as alphanumerics. + # With only `[a-z0-9]` the optional `\\?` backtracks: it matches the empty + # string, the next character is the separator, that passes the lookahead, and + # a real personal path is swallowed. Excluding both is what separates a + # harmless root from a leak. `BARE_ABS` can also append punctuation (a + # trailing backtick), which is why the rule is a lookahead and not `$`. + if re.match(r'^c:\\users\\?(?![\\a-z0-9])', low): + return True + if PLACEHOLDER_RE.search(path) or ELLIPSIS_RE.search(path): + return True + return False + def collect(src, patterns): """Return [(absolute_path, relative_path)] for every matching file.""" @@ -90,7 +173,12 @@ def audit(path, rel): for lineno, line in enumerate(lines, 1): for match in BARE_ABS.finditer(line): - findings.append(('abs-path', lineno, match.group(0)[:90])) + hit = match.group(0) + # Report only paths that actually name this machine; a generic + # install location or an already-redacted path is not a leftover. + if _leaks_nothing(hit): + continue + findings.append(('machine-path', lineno, hit[:90])) if CREDENTIAL_HINT.search(line): findings.append(('credential', lineno, line.strip()[:110])) return findings @@ -110,6 +198,16 @@ def scrub(text): def main(): + # A finding can quote a byte or a path the console's code page cannot + # represent -- a Windows console defaults to a legacy encoding such as + # cp936 or cp1252. Without this, the audit dies with UnicodeEncodeError + # *while reporting its own result*, which reads as "the tool is broken" + # instead of "here is what it found". + try: + sys.stdout.reconfigure(errors='replace') + except (AttributeError, ValueError): + pass + ap = argparse.ArgumentParser( description='Sync a script directory to a publish location and audit it for ' 'machine-specific leftovers (absolute paths, hardcoded credentials).') @@ -140,13 +238,13 @@ def main(): for full, rel in files: findings = audit(full, rel) - abs_findings = [f for f in findings if f[0] == 'abs-path'] + path_findings = [f for f in findings if f[0] == 'machine-path'] cred_findings = [f for f in findings if f[0] == 'credential'] - other = [f for f in findings if f[0] not in ('abs-path', 'credential')] + other = [f for f in findings if f[0] not in ('machine-path', 'credential')] if findings: print('-- %s' % rel) - for kind, lineno, detail in abs_findings + cred_findings + other: + for kind, lineno, detail in path_findings + cred_findings + other: print(' [%s] line %d: %s' % (kind, lineno, detail)) total_findings += len(findings) @@ -156,7 +254,7 @@ def main(): target = os.path.join(args.dst, rel) os.makedirs(os.path.dirname(target) or '.', exist_ok=True) - if args.scrub and (abs_findings or cred_findings): + if args.scrub and (path_findings or cred_findings): with open(full, encoding='utf-8', errors='replace') as fh: text = fh.read() new_text, count = scrub(text) diff --git a/check_budget.py b/check_budget.py index f2ca940..b1945b4 100644 --- a/check_budget.py +++ b/check_budget.py @@ -26,6 +26,10 @@ What is measured, and why each measure is shaped the way it is: 4. **Reference discoverability.** Every reference must be reachable from SKILL.md or nothing will ever load it. (`check_repo.py` also enforces this; the duplication is deliberate, because a dead reference is invisible in a way a broken path is not.) + 5. **Corpus size** (a note, never a failure). The budget above governs what every task *loads*; + this reports what the repository *carries*. The first cannot see the second: a new reference + costs exactly one index line, so the corpus can double while the narrative count stays green. + A large corpus is a choice; a growth nobody measured is the defect. Usage: python check_budget.py # report; exit 1 if a hard limit is exceeded @@ -156,6 +160,55 @@ def check_discoverable(): return out +# --------------------------------------------------------------------------- +# The always-loaded part is budgeted above. The corpus is a different problem: +# nothing loads all of it, so it has no natural ceiling, and a file is cheap to +# add and expensive to remove. What is worth watching is the RATE, because the +# SKILL.md budget cannot see it -- each new reference costs exactly one index +# line there, so a corpus can double while the narrative budget stays green. +# +# Reported as notes, never as failures. A large corpus is a choice, not a +# defect; what is a defect is a growth nobody noticed. The thresholds are set +# where the current corpus sits, so the note starts firing on the *next* pass +# rather than describing the present one. +CORPUS_FILES_NOTE = 110 +CORPUS_LINES_NOTE = 26000 + + +def check_corpus(): + """Report corpus size so growth is visible instead of discovered late.""" + refs = sorted(glob.glob(os.path.join(REFS, '**', '*.md'), recursive=True)) + scripts_dir = os.path.join(SKILL_DIR, 'scripts') + scripts = sorted(p for p in glob.glob(os.path.join(scripts_dir, '**', '*'), recursive=True) + if os.path.isfile(p)) + + def count(paths): + total = 0 + for p in paths: + try: + with open(p, encoding='utf-8', errors='replace') as fh: + total += sum(1 for _ in fh) + except OSError: + pass + return total + + ref_lines, script_lines = count(refs), count(scripts) + n_files = len(refs) + len(scripts) + lines_total = ref_lines + script_lines + + notes = ['corpus: %d references + %d scripts = %d files, %d lines total' + % (len(refs), len(scripts), n_files, lines_total)] + if n_files > CORPUS_FILES_NOTE: + notes.append('corpus is %d files, past the %d-file note threshold -- every entry costs ' + 'an index line in SKILL.md and a share of the maintenance surface, so ' + 'confirm the new ones are load-bearing rather than merely adjacent' + % (n_files, CORPUS_FILES_NOTE)) + if lines_total > CORPUS_LINES_NOTE: + notes.append('corpus is %d lines, past the %d-line note threshold' + % (lines_total, CORPUS_LINES_NOTE)) + return notes + + def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) @@ -169,6 +222,7 @@ def main(): findings += check_index_rows(lines) findings += check_discoverable() notes = check_restated() + notes += check_corpus() if args.json: print(json.dumps({'findings': [{'level': l, 'message': m} for l, m in findings], diff --git a/docs/tool-verification/EXTENSION-desensitization.md b/docs/tool-verification/EXTENSION-desensitization.md index 59ffbdd..02521c3 100644 --- a/docs/tool-verification/EXTENSION-desensitization.md +++ b/docs/tool-verification/EXTENSION-desensitization.md @@ -8,7 +8,7 @@ Strength labels follow `docs/tool-verification/README.md`: **observed** (the exa its output are reproduced here), **inferred** (follows from observed facts; the step itself was not executed), **unverified** (assumed or reported elsewhere and not independently confirmed). -Reference environment: Windows 11 host, Python 3.14.0, repository at `E:\apk-reverse` on branch +Reference environment: Windows 11 host, Python 3.14.0, repository root on branch `main`. **No device, no network and no target sample were involved** — this pass is entirely machine-local, because the thing worth measuring is what a scanner reports against a tree whose contents are already known. Every identifier in the fixture and in every transcript below is @@ -44,13 +44,13 @@ a transcript pasted once: ``` $ python tools/_work/leak-precedent/gen_fixtures.py -wrote E:\apk-reverse\tools\_work\leak-precedent\fixture\planted\package.txt (411 bytes) -wrote E:\apk-reverse\tools\_work\leak-precedent\fixture\planted\device.txt (218 bytes) -wrote E:\apk-reverse\tools\_work\leak-precedent\fixture\planted\token.txt (261 bytes) -wrote E:\apk-reverse\tools\_work\leak-precedent\fixture\planted\appkey.txt (212 bytes) -wrote E:\apk-reverse\tools\_work\leak-precedent\fixture\planted\endpoint.txt (146 bytes) -wrote E:\apk-reverse\tools\_work\leak-precedent\fixture\planted\path.txt (175 bytes) -wrote E:\apk-reverse\tools\_work\leak-precedent\fixture\exempt\benign.txt (1396 bytes) +wrote tools/_work/leak-precedent\fixture\planted\package.txt (411 bytes) +wrote tools/_work/leak-precedent\fixture\planted\device.txt (218 bytes) +wrote tools/_work/leak-precedent\fixture\planted\token.txt (261 bytes) +wrote tools/_work/leak-precedent\fixture\planted\appkey.txt (212 bytes) +wrote tools/_work/leak-precedent\fixture\planted\endpoint.txt (146 bytes) +wrote tools/_work/leak-precedent\fixture\planted\path.txt (175 bytes) +wrote tools/_work/leak-precedent\fixture\exempt\benign.txt (1396 bytes) == 7 fixture file(s) == ``` @@ -269,7 +269,7 @@ found by running the scanner, not by reasoning about it. ``` $ python skills/apk-reverse/scripts/scan_leaks.py # first run -== leak scan: 118 file(s) scanned under E:\apk-reverse == +== leak scan: 118 file(s) scanned under the repository root == findings by category: device=4, endpoint=20, package=4 == result: 28 finding(s) == RESULT=leaks_found (exit 1) ``` @@ -297,7 +297,7 @@ Inherited state, reproduced at the start of this pass (same tree, before the §3 ``` $ python skills/apk-reverse/scripts/scan_leaks.py -== leak scan: 128 file(s) scanned under E:\apk-reverse == +== leak scan: 128 file(s) scanned under the repository root == findings by category: appkey=2, device=12, endpoint=14, package=5, path=3, token=4 == result: 40 finding(s) -- each one is a line to look at, not a verdict == RESULT=leaks_found (exit 1) @@ -317,7 +317,7 @@ $ python skills/apk-reverse/scripts/scan_leaks.py --quiet RESULT=leaks_found_strong_only (exit 0) $ python skills/apk-reverse/scripts/scan_leaks.py -== leak scan: 129 file(s) scanned under E:\apk-reverse == +== leak scan: 129 file(s) scanned under the repository root == findings by category: endpoint=4 skills/apk-reverse/references/desensitization-and-leak-scans.md:81:10 [endpoint/weak] plain_addr_ipliteral @@ -383,7 +383,7 @@ $ python skills/apk-reverse/scripts/scan_leaks.py --root tools/_work/leak-preced RESULT=leaks_found (exit 1) $ python skills/apk-reverse/scripts/scan_leaks.py --root tools/_work/nope-not-here -error: root does not exist: E:\apk-reverse\tools\_work\nope-not-here [stderr] +error: root does not exist: tools/_work/nope-not-here [stderr] RESULT=error (exit 2) $ python skills/apk-reverse/scripts/scan_leaks.py --only nope diff --git a/docs/tool-verification/EXTENSION-extraction-shell-bench.md b/docs/tool-verification/EXTENSION-extraction-shell-bench.md index ce67e8b..570fabc 100644 --- a/docs/tool-verification/EXTENSION-extraction-shell-bench.md +++ b/docs/tool-verification/EXTENSION-extraction-shell-bench.md @@ -406,7 +406,7 @@ $ python skills/apk-reverse/scripts/repack.py \ --apksigner E:\tools\android-14\apksigner.bat \ --zipalign E:\tools\android-14\zipalign.exe \ --keytool "C:\Program Files\Java\jdk-17.0.4.1\bin\keytool.exe" -[in ] E:\apk-reverse\tools\_work\bench\unpack\b4\ezAndroid.apk (2984841 bytes) +[in ] tools/_work/bench\unpack\b4\ezAndroid.apk (2984841 bytes) [zip] - META-INF/1N0R.SF / 1N0R.RSA / MANIFEST.MF (signature artifact) [zip] unsigned written: 2828919 bytes [zip] AndroidManifest.xml STORED offset=0x31 @@ -506,9 +506,10 @@ attributable to memory rather than to disk. - **No end-to-end FART splice.** The reference file's FART loop needs a live extraction-shell target; nothing in `repos/` is one (§1), and building an actively resisting shell is out of scope for this pass. -- **A real APK build on this host.** `E:\tools\android-14` ships `d8`, `aapt2`, +- **A real APK build on this host.** The build-tools directory ships `d8`, `aapt2`, `apksigner` and `zipalign` but **no `android.jar`**, and no `android.jar` exists - anywhere on the machine (searched `E:\tools`, `C:\Program Files`, `E:\Android`). Any + anywhere on the machine (searched the tool directories, `C:\Program Files`, and + the SDK path). Any `javac` against `android.*` fails with "package android.os does not exist". APKs can still be *repacked* here (`repack.py` does not recompile), which is why §4.4 works and the self-built-APK route in the task brief does not. diff --git a/docs/tool-verification/EXTENSION-vmp-diff.md b/docs/tool-verification/EXTENSION-vmp-diff.md index 92ecccb..6bfb1f2 100644 --- a/docs/tool-verification/EXTENSION-vmp-diff.md +++ b/docs/tool-verification/EXTENSION-vmp-diff.md @@ -53,8 +53,8 @@ D8 8.2.2-dev (build facedf41bbd28b563d1e9e09c5f72d7c5ca598d8 from go/r8bot (luci Two further tool facts worth recording, both **observed**: -- **`E:\agb_apktool.jar` does not exist**, so `E:\tools\bin\apktool.bat` (which is - `java -jar "E:\agb_apktool.jar" %*`) resolves to nothing. apktool is unavailable on this +- **The apktool jar the wrapper points at does not exist**, so `\bin\apktool.bat` (which is + `java -jar "\agb_apktool.jar" %*`) resolves to nothing. apktool is unavailable on this machine, and a bare `apktool d` fails with a jar-not-found error rather than a recognisable "tool missing" message. - `javac` is on PATH only through the Oracle `javapath` forwarder; the real JDK is at diff --git a/docs/tool-verification/REPO-DECISIONS.md b/docs/tool-verification/REPO-DECISIONS.md index c8aa20f..7fb3463 100644 --- a/docs/tool-verification/REPO-DECISIONS.md +++ b/docs/tool-verification/REPO-DECISIONS.md @@ -233,3 +233,73 @@ evidentiary gap F5 describes, and the repository's rule that nothing ships unrun much as to code. It is dependency-free (stdlib only) so it reproduces on a machine with no Android toolchain, which is the machine the defect was found on. It carries no target data: the dex is synthesised from constants in the file. + +## Maintenance pass: the path audit's signal, three path families, and a corpus note + +An external review found the repository in good shape on evidence and honesty but raised three +concrete defects. All three are fixed here; the fourth item is a measurement added so the next +pass can see what this one could not. + +### 1. `build_scripts.py` reported 37 findings, most of them noise + +The audit matched **any** `X:\...` literal, so `C:\Program Files\...`, `C:\Python314\...` and +literal placeholders like `` all counted as "machine-specific leftovers". A gate whose output +is mostly noise stops being read, which is the same failure `scan_leaks.py` names when it refuses to +exempt the RFC 5737 documentation ranges. + +Added two explicit, auditable exemption families rather than a looser regex: + +- `GENERIC_PATH_PREFIXES` — install locations that exist on every host of that platform + (`Program Files`, `Python*`, `MinGW`, `/usr/`, `/opt/`, ...). +- `_leaks_nothing()` — also exempts an already-redacted path, i.e. one carrying a placeholder or an + ellipsis, because that mark means a normalisation pass already ran. + +**One trap had to be handled explicitly.** `BARE_ABS` stops at whitespace, so a real +`C:\Program Files\...` arrives as the truncated `C:\Program`; a one-directional `startswith` misses +the exemption. The comparison is therefore bidirectional, gated by a minimum length so a short drive +prefix cannot become a blanket pass. The same function needed a second correction: with a lookahead +of only `[a-z0-9]`, the optional `\\?` backtracks to the empty string, the next character is the +separator, it passes, and a real personal path is swallowed — the lookahead must exclude the +separator too. + +### 2. Three absolute paths left on the published surface + +`EXTENSION-desensitization.md`, `EXTENSION-extraction-shell-bench.md` and `EXTENSION-vmp-diff.md` +quoted a workbench root, a tool jar and an SDK location as absolute paths. `docs/tool-verification/` +already states the rule (§**Workbench paths are an operation record, not a pointer**): the record +may say a run happened at a `tools/...` path, written relatively. The three files were made to match +the rule they were already under; the tool-jar and SDK cases were rephrased rather than +placeholder-ised, so no reader has to learn a new token. + +### 3. `check_budget.py` could not see corpus growth + +The gate budgets the always-loaded part and was silently blind to the other half of the problem: a +new reference costs exactly **one index line**, so the corpus can double while the narrative count +stays green. Two note-level thresholds now report it. Notes, not failures — a large corpus is a +choice; a growth nobody measured is the defect. + +### A cross-tool inconsistency this pass ran into + +`build_scripts.py` now exempts any `<...>` placeholder; `scan_leaks.py` recognises a **fixed token +list** (``, ``, ``, ...). An invented placeholder in a source comment — +`` — therefore passed the first and was read by the second as the very personal name its +`user_path_windows` rule exists to catch, turning `check_repo.py` red. That is by design rather than +a bug (a strict list is what stops `` hiding a real one), but the two tools disagree about +what "already redacted" means, and the disagreement is invisible until someone writes a new +placeholder. The fix here was to rephrase the comment; a future pass may want a shared token list. + +### Verification + +- `build_scripts.py` **37 → 0** findings on a clean clone, exit 0 (was exit 2). The generic-install + hits (`C:\MinGW\bin\gcc.exe`, `C:\Python314\...`, `C:\Program`) no longer report; the real ones + (`E:\apk-reverse\...`, `E:\agb_apktool.jar`) did, and were fixed in the documents, not by widening + the list. +- A purpose-written case table pins **both** directions — 18 cases, 0 failures: generic and + already-redacted paths are exempt, while a real workbench path, a real personal name after + `C:\Users\`, a real POSIX home and a real project path still report. Two defects in the first + implementation were found by that table, not by the audit run. +- All four gates pass: `check_repo.py` 0 problem, `check_refs.py` 0 dangling / 0 warning, + `check_budget.py` 0 failure / 0 warning / 3 notes, `build_scripts.py` 0 findings. +- `check_budget.py` now reports the corpus explicitly: **47 references + 55 scripts = 102 files, + 31,423 lines**, past the 26,000-line note threshold — the figure this pass could not see before. +