apk-reverse/tests/integration/test_check_commands_gate.py

"""Regression tests for the documented-command gate itself.

`check_commands.py` is the gate that proves other documentation is runnable, so a blind spot in it
is worth more than a blind spot elsewhere: it turns "146 commands checked, 0 drift" into a claim the
repository cannot back. One such blind spot is pinned here, exactly as it was found.

**What was wrong.** `resolve_script()` tried the *document's own directory* first when resolving a
relative path. A bare `python analyze_pair.py` written inside `docs/tool-verification/` therefore
resolved to `tools/_work/bench/unpack/b3/analyze_pair.py` -- a real file, inside the gitignored
workbench -- and scored `ok`. Seven commands in two evidence files used that form while the same
files wrote the correct path elsewhere, and a reader copying one from the repository root gets
"No such file". The gate said the documentation was clean.

The tests below drive the resolver and the finding classifier directly rather than through the CLI,
so they stay fast and do not depend on the current state of the documentation.
"""
from __future__ import annotations

import sys
from pathlib import Path

import pytest

from conftest import REPO_ROOT

pytestmark = pytest.mark.integration

CHECK_COMMANDS = REPO_ROOT / "check_commands.py"


@pytest.fixture(scope="module")
def gate():
    """The gate as a module, so its internals can be driven without a subprocess."""
    sys.path.insert(0, str(REPO_ROOT))
    import importlib

    return importlib.import_module("check_commands")


def _workbench_script_name(gate):
    """A `.py` basename that exists only under tools/, or skip if the workbench is absent."""
    names = sorted(gate._workbench_names())
    if not names:
        pytest.skip("no workbench present (tools/ is gitignored and may not exist in a checkout)")
    shipped = {p.name for p in (REPO_ROOT / "skills" / "apk-reverse" / "scripts").glob("*.py")}
    root = {p.name for p in REPO_ROOT.glob("*.py")}
    for name in names:
        if name not in shipped and name not in root:
            return name
    pytest.skip("every workbench script name is also shipped; nothing to test with")


def test_a_bare_name_that_only_exists_in_the_workbench_is_drift(gate):
    """The exact defect: the name exists *somewhere*, which used to be enough to pass it."""
    name = _workbench_script_name(gate)
    doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-extraction-shell-bench.md")
    path, why = gate.resolve_script(name, doc, include_workbench=False)
    assert path is None
    assert why == "unqualified-workbench", why

    text = "```\n$ python %s --flag\n```\n" % name
    findings = gate.command_findings(doc, text, {}, False)
    kinds = [f["kind"] for f in findings]
    assert "unqualified-workbench" in kinds, findings


def test_a_path_qualified_workbench_reference_is_a_skip_not_drift(gate):
    """`tools/_work/...` is deliberate in an evidence record: a skip, reported with its reason."""
    name = _workbench_script_name(gate)
    doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-extraction-shell-bench.md")
    _path, why = gate.resolve_script("tools/_work/bench/unpack/b3/%s" % name, doc,
                                     include_workbench=False)
    assert why == "workbench"


@pytest.fixture
def fake_repo(gate, tmp_path, monkeypatch):
    """A repository-shaped tree with its own workbench, so resolution can be tested anywhere.

    This exists because the workbench-dependent tests above cannot run on a clean checkout: the
    `tools/` tree is gitignored, so on CI they skip and the gate's most important property goes
    unverified exactly where it matters. The tree built here has the layout the resolver assumes
    (a root, a skill dir, a gitignored `tools/`) and nothing else.
    """
    root = tmp_path / "repo"
    (root / "skills" / "apk-reverse" / "scripts").mkdir(parents=True)
    (root / "docs" / "tool-verification").mkdir(parents=True)
    (root / "tools" / "_work" / "bench").mkdir(parents=True)
    (root / "README.md").write_text("# root\n", encoding="utf-8")
    (root / "skills" / "apk-reverse" / "scripts" / "shipped.py").write_text(
        '"""capability: x\n"""\nimport argparse\np = argparse.ArgumentParser()\n'
        'p.add_argument("--flag")\n', encoding="utf-8")
    (root / "tools" / "_work" / "bench" / "solo_in_workbench.py").write_text(
        "print('x')\n", encoding="utf-8")
    (root / "docs" / "tool-verification" / "record.md").write_text("# record\n", encoding="utf-8")

    monkeypatch.setattr(gate, "ROOT", str(root))
    monkeypatch.setattr(gate, "SKILL_DIR", str(root / "skills" / "apk-reverse"))
    monkeypatch.setattr(gate, "SCRIPT_DIR", str(root / "skills" / "apk-reverse" / "scripts"))
    gate._WORKBENCH_INDEX.clear()
    yield root
    gate._WORKBENCH_INDEX.clear()


def test_resolution_order_is_root_then_skill_then_document(gate, fake_repo):
    """The corrected order, asserted against a tree that exists on any machine."""
    doc = str(fake_repo / "docs" / "tool-verification" / "record.md")
    path, why = gate.resolve_script("README.md", doc, include_workbench=False)
    assert why == "ok" and path.endswith("README.md")
    path, why = gate.resolve_script("shipped.py", doc, include_workbench=False)
    assert why == "ok" and path.endswith("shipped.py")


def test_a_workbench_only_name_is_drift_and_a_qualified_one_is_not(gate, fake_repo):
    doc = str(fake_repo / "docs" / "tool-verification" / "record.md")
    path, why = gate.resolve_script("solo_in_workbench.py", doc, include_workbench=False)
    assert (path, why) == (None, "unqualified-workbench")
    path, why = gate.resolve_script("tools/_work/bench/solo_in_workbench.py", doc,
                                    include_workbench=False)
    assert (path, why) == (None, "workbench")


def test_a_bare_name_the_document_paths_elsewhere_is_a_skip(gate, fake_repo):
    """The fix that made the gate behave the same on CI as on a populated machine.

    `solo_in_workbench.py` appears once bare and once with a path in the same document. The
    document has already told the reader where it lives, so the bare line is a shorthand -- and
    that verdict must not depend on the gitignored tree being visible, which is how the first
    version of this fix passed locally and failed on CI.
    """
    doc = str(fake_repo / "docs" / "tool-verification" / "record.md")
    text = ("Path: `tools/_work/bench/solo_in_workbench.py` explains it.\n\n"
            "```\n$ python solo_in_workbench.py\n```\n")
    findings = gate.command_findings(doc, text, {}, False)
    kinds = [f["kind"] for f in findings]
    assert "unqualified-workbench" not in kinds, findings
    assert "missing-script" not in kinds, findings
    assert "skipped" in kinds, findings


def test_a_bare_name_after_cd_with_no_resolution_is_skipped_not_drift(gate, fake_repo):
    """`cd <somewhere cloned>` then a bare name: unverifiable, and reported as such."""
    doc = str(fake_repo / "docs" / "tool-verification" / "record.md")
    text = "```\n$ cd tools/_work/bench/repos/elsewhere\n$ python upstream_tool.py --x\n```\n"
    findings = gate.command_findings(doc, text, {}, False)
    kinds = [f["kind"] for f in findings]
    assert "missing-script" not in kinds, findings
    assert "after-cd-unresolved" in kinds, findings


def test_the_committed_tree_reports_no_drift(gate, tmp_path):
    """The gate's own verdict on this repository, run in-process so it cannot drift silently."""
    docs = gate.collect_docs(None)
    findings = []
    cache = {}
    for doc in docs:
        try:
            text = Path(doc).read_text(encoding="utf-8")
        except OSError:
            continue
        findings.extend(gate.command_findings(doc, text, cache, False))
    drift = [f for f in findings
             if f["kind"] in ("unknown-flag", "missing-script", "unqualified-workbench")]
    detail = "\n".join("  %s:%s %s (%s)" % (f["doc"], f["line"], f["command"], f["kind"])
                       for f in drift[:20])
    assert not drift, "documented commands that cannot run:\n%s" % detail