apk-reverse/tests/integration/test_check_commands_gate.py

"""Regression tests for the documented-command gate itself.

`check_commands.py` is the gate that proves other documentation is runnable, so a blind spot in it
is worth more than a blind spot elsewhere: it turns "146 commands checked, 0 drift" into a claim the
repository cannot back. One such blind spot is pinned here, exactly as it was found.

**What was wrong.** `resolve_script()` tried the *document's own directory* first when resolving a
relative path. A bare `python analyze_pair.py` written inside `docs/tool-verification/` therefore
resolved to `tools/_work/bench/unpack/b3/analyze_pair.py` -- a real file, inside the gitignored
workbench -- and scored `ok`. Seven commands in two evidence files used that form while the same
files wrote the correct path elsewhere, and a reader copying one from the repository root gets
"No such file". The gate said the documentation was clean.

The tests below drive the resolver and the finding classifier directly rather than through the CLI,
so they stay fast and do not depend on the current state of the documentation.
"""
from __future__ import annotations

import sys
from pathlib import Path

import pytest

from conftest import REPO_ROOT

pytestmark = pytest.mark.integration

CHECK_COMMANDS = REPO_ROOT / "check_commands.py"


@pytest.fixture(scope="module")
def gate():
    """The gate as a module, so its internals can be driven without a subprocess."""
    sys.path.insert(0, str(REPO_ROOT))
    import importlib

    return importlib.import_module("check_commands")


def _workbench_script_name(gate):
    """A `.py` basename that exists only under tools/, or skip if the workbench is absent."""
    names = sorted(gate._workbench_names())
    if not names:
        pytest.skip("no workbench present (tools/ is gitignored and may not exist in a checkout)")
    shipped = {p.name for p in (REPO_ROOT / "skills" / "apk-reverse" / "scripts").glob("*.py")}
    root = {p.name for p in REPO_ROOT.glob("*.py")}
    for name in names:
        if name not in shipped and name not in root:
            return name
    pytest.skip("every workbench script name is also shipped; nothing to test with")


def test_a_bare_name_that_only_exists_in_the_workbench_is_drift(gate):
    """The exact defect: the name exists *somewhere*, which used to be enough to pass it."""
    name = _workbench_script_name(gate)
    doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-extraction-shell-bench.md")
    path, why = gate.resolve_script(name, doc, include_workbench=False)
    assert path is None
    assert why == "unqualified-workbench", why

    text = "```\n$ python %s --flag\n```\n" % name
    findings = gate.command_findings(doc, text, {}, False)
    kinds = [f["kind"] for f in findings]
    assert "unqualified-workbench" in kinds, findings


def test_a_path_qualified_workbench_reference_is_a_skip_not_drift(gate):
    """`tools/_work/...` is deliberate in an evidence record: a skip, reported with its reason."""
    name = _workbench_script_name(gate)
    doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-extraction-shell-bench.md")
    _path, why = gate.resolve_script("tools/_work/somewhere/%s" % name, doc, include_workbench=False)
    assert why == "workbench"


def test_a_bare_name_after_a_cd_in_the_same_block_is_not_drift(gate):
    """A block that says `cd tools/_work/bench/repos/dcc` tells the reader where to stand."""
    name = _workbench_script_name(gate)
    doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-java2c.md")
    text = "```\n$ cd tools/_work/bench/repos/dcc\n$ python %s --no-build\n```\n" % name
    findings = gate.command_findings(doc, text, {}, False)
    kinds = [f["kind"] for f in findings]
    assert "unqualified-workbench" not in kinds, findings
    assert "workbench-after-cd" in kinds, findings


def test_a_shipped_script_resolves_from_the_repository_root(gate):
    """The order that matters: root, then the skill's scripts, then the document's own directory."""
    doc = str(REPO_ROOT / "docs" / "tool-verification" / "README.md")
    path, why = gate.resolve_script("check_repo.py", doc, include_workbench=False)
    assert why == "ok" and path and path.endswith("check_repo.py")
    path, why = gate.resolve_script("skills/apk-reverse/scripts/scan_leaks.py", doc,
                                    include_workbench=False)
    assert why == "ok" and path and path.endswith("scan_leaks.py")


def test_the_committed_tree_reports_no_drift(gate, tmp_path):
    """The gate's own verdict on this repository, run in-process so it cannot drift silently."""
    docs = gate.collect_docs(None)
    findings = []
    cache = {}
    for doc in docs:
        try:
            text = Path(doc).read_text(encoding="utf-8")
        except OSError:
            continue
        findings.extend(gate.command_findings(doc, text, cache, False))
    drift = [f for f in findings
             if f["kind"] in ("unknown-flag", "missing-script", "unqualified-workbench")]
    detail = "\n".join("  %s:%s %s (%s)" % (f["doc"], f["line"], f["command"], f["kind"])
                       for f in drift[:20])
    assert not drift, "documented commands that cannot run:\n%s" % detail