mirror of
https://codeberg.org/reverse-skill-lab/apk-reverse
synced 2026-11-15 20:44:22 +08:00
Fix the command gate's blind spot, and make --json actually be JSON
A reviewer reported seven drifting commands in the evidence record. Reproduced, the report split
three ways, and the middle category was the useful one: the gate had been scoring them clean.
Not reproducible, with the measurement:
tests/run_tests.py -q runs. The wrapper forwards its arguments to pytest
(`args = list(argv[1:]) or ["-q"]`), and documents that in its own usage.
dcc.py already written as `cd tools/_work/bench/repos/dcc` followed by
`python dcc.py ...` -- an external checkout, not a shipped script.
Real, and fixed here: seven commands in two evidence records named a **workbench** script with no
path at all (`analyze_pair.py`, `make_skeleton.py`, `ttd_attach.py`, `b2_measure.py`). The files
exist under `tools/`, which .gitignore excludes, so a reader copying one from the repository root
gets "No such file" -- and two of those documents wrote the correct path elsewhere, so they
disagreed with themselves.
**The actual defect was in `check_commands.py`, not in the seven lines.** `resolve_script()` tried
the document's own directory first, so a bare `analyze_pair.py` inside `docs/tool-verification/`
resolved to `tools/_work/bench/unpack/b3/analyze_pair.py` -- a real file -- and passed. Resolution
now follows the order a reader would try (repository root, then the skill's scripts, then the
document's own directory, which is what `../x.py` in a nested README legitimately means); a bare
name that only resolves under `tools/` is `unqualified-workbench`; and that class counts as **drift,
not as a skip**, because it is a one-token fix that a reader cannot run. The first attempt classified
it as a low-confidence warning, which would have left all seven in place.
One exception is honoured and pinned: a code block opening with `cd tools/_work/bench/repos/dcc` is
telling the reader where to stand, so a bare `python dcc.py` after it is correct.
`documented_commands()` now reports whether a `cd` preceded the command in the same block, and that
case is skipped as `workbench-after-cd`.
Two defects of this pass's own, found while fixing it:
--json emitted the RESULT= token after the JSON document, so `--json | jq` failed with "Extra
data". The token is gone from that path; the status is a field in the document, and the exit code
carries the same meaning.
The fix's own substitution left four literal backslashes before `$` in the two files it edited --
a character a reader would copy by mistake. Removed, and verified absent.
`tests/integration/test_check_commands_gate.py` pins the resolution order, the drift classification,
the `cd` exception, and a clean pass over the committed tree, so this cannot come back silently.
Gates: check_commands 153 checked / 0 drift, check_repo 0 problem, check_refs 0 dangling / 0 warning,
check_routing coherent, check_budget 0 failure / 0 warning, pytest 613 passed / 158 skipped / 25
xfailed.
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
"""Regression tests for the documented-command gate itself.
|
||||
|
||||
`check_commands.py` is the gate that proves other documentation is runnable, so a blind spot in it
|
||||
is worth more than a blind spot elsewhere: it turns "146 commands checked, 0 drift" into a claim the
|
||||
repository cannot back. One such blind spot is pinned here, exactly as it was found.
|
||||
|
||||
**What was wrong.** `resolve_script()` tried the *document's own directory* first when resolving a
|
||||
relative path. A bare `python analyze_pair.py` written inside `docs/tool-verification/` therefore
|
||||
resolved to `tools/_work/bench/unpack/b3/analyze_pair.py` -- a real file, inside the gitignored
|
||||
workbench -- and scored `ok`. Seven commands in two evidence files used that form while the same
|
||||
files wrote the correct path elsewhere, and a reader copying one from the repository root gets
|
||||
"No such file". The gate said the documentation was clean.
|
||||
|
||||
The tests below drive the resolver and the finding classifier directly rather than through the CLI,
|
||||
so they stay fast and do not depend on the current state of the documentation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import REPO_ROOT
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
CHECK_COMMANDS = REPO_ROOT / "check_commands.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def gate():
|
||||
"""The gate as a module, so its internals can be driven without a subprocess."""
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
import importlib
|
||||
|
||||
return importlib.import_module("check_commands")
|
||||
|
||||
|
||||
def _workbench_script_name(gate):
|
||||
"""A `.py` basename that exists only under tools/, or skip if the workbench is absent."""
|
||||
names = sorted(gate._workbench_names())
|
||||
if not names:
|
||||
pytest.skip("no workbench present (tools/ is gitignored and may not exist in a checkout)")
|
||||
shipped = {p.name for p in (REPO_ROOT / "skills" / "apk-reverse" / "scripts").glob("*.py")}
|
||||
root = {p.name for p in REPO_ROOT.glob("*.py")}
|
||||
for name in names:
|
||||
if name not in shipped and name not in root:
|
||||
return name
|
||||
pytest.skip("every workbench script name is also shipped; nothing to test with")
|
||||
|
||||
|
||||
def test_a_bare_name_that_only_exists_in_the_workbench_is_drift(gate):
|
||||
"""The exact defect: the name exists *somewhere*, which used to be enough to pass it."""
|
||||
name = _workbench_script_name(gate)
|
||||
doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-extraction-shell-bench.md")
|
||||
path, why = gate.resolve_script(name, doc, include_workbench=False)
|
||||
assert path is None
|
||||
assert why == "unqualified-workbench", why
|
||||
|
||||
text = "```\n$ python %s --flag\n```\n" % name
|
||||
findings = gate.command_findings(doc, text, {}, False)
|
||||
kinds = [f["kind"] for f in findings]
|
||||
assert "unqualified-workbench" in kinds, findings
|
||||
|
||||
|
||||
def test_a_path_qualified_workbench_reference_is_a_skip_not_drift(gate):
|
||||
"""`tools/_work/...` is deliberate in an evidence record: a skip, reported with its reason."""
|
||||
name = _workbench_script_name(gate)
|
||||
doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-extraction-shell-bench.md")
|
||||
_path, why = gate.resolve_script("tools/_work/somewhere/%s" % name, doc, include_workbench=False)
|
||||
assert why == "workbench"
|
||||
|
||||
|
||||
def test_a_bare_name_after_a_cd_in_the_same_block_is_not_drift(gate):
|
||||
"""A block that says `cd tools/_work/bench/repos/dcc` tells the reader where to stand."""
|
||||
name = _workbench_script_name(gate)
|
||||
doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-java2c.md")
|
||||
text = "```\n$ cd tools/_work/bench/repos/dcc\n$ python %s --no-build\n```\n" % name
|
||||
findings = gate.command_findings(doc, text, {}, False)
|
||||
kinds = [f["kind"] for f in findings]
|
||||
assert "unqualified-workbench" not in kinds, findings
|
||||
assert "workbench-after-cd" in kinds, findings
|
||||
|
||||
|
||||
def test_a_shipped_script_resolves_from_the_repository_root(gate):
|
||||
"""The order that matters: root, then the skill's scripts, then the document's own directory."""
|
||||
doc = str(REPO_ROOT / "docs" / "tool-verification" / "README.md")
|
||||
path, why = gate.resolve_script("check_repo.py", doc, include_workbench=False)
|
||||
assert why == "ok" and path and path.endswith("check_repo.py")
|
||||
path, why = gate.resolve_script("skills/apk-reverse/scripts/scan_leaks.py", doc,
|
||||
include_workbench=False)
|
||||
assert why == "ok" and path and path.endswith("scan_leaks.py")
|
||||
|
||||
|
||||
def test_the_committed_tree_reports_no_drift(gate, tmp_path):
|
||||
"""The gate's own verdict on this repository, run in-process so it cannot drift silently."""
|
||||
docs = gate.collect_docs(None)
|
||||
findings = []
|
||||
cache = {}
|
||||
for doc in docs:
|
||||
try:
|
||||
text = Path(doc).read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
findings.extend(gate.command_findings(doc, text, cache, False))
|
||||
drift = [f for f in findings
|
||||
if f["kind"] in ("unknown-flag", "missing-script", "unqualified-workbench")]
|
||||
detail = "\n".join(" %s:%s %s (%s)" % (f["doc"], f["line"], f["command"], f["kind"])
|
||||
for f in drift[:20])
|
||||
assert not drift, "documented commands that cannot run:\n%s" % detail
|
||||
Reference in New Issue
Block a user