diff --git a/README.md b/README.md index bd911d0..533ab68 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,10 @@ measurements. execution on the host, or a live function service-ified over Frida RPC. - Reading **instruction-level execution evidence** when a native function has been flattened into a state machine by OLLVM — including the two ways Stalker was measured to bite back on a real device. +- Reaching for the **hardware layer** when every software tracer modifies what it observes: arm64 + hardware watchpoints (no code change, but an honest `TracerPid` ledger), ETE/TRBE differential + tracing with its five gates, and kernel-armed injection templates that are labelled unverified + until a device with the matching gate fires them. - Recognising when **userspace hooking cannot reach the check at all** (raw `svc` syscalls, `init_array`-early detection), what the next layer up and down can actually do, and when escalating is the wrong answer. @@ -269,6 +273,25 @@ references/ loaded on demand, one topic each userspace hooking provably cannot reach the check: raw `svc`, init_array-early detection, what each root scheme hides, the kernel-route map with its version gate, and when to stop escalating + hardware-watchpoints.md invisible memory watchpoints on arm64: the hardware debug slots, the + ptrace route with its honest `TracerPid` ledger, the 8-byte-window + alignment rules, the traps (per-thread slots, signal forwarding, the + breakpoint step-over dance), and the control run every hit is owed + ete-diff-tracing.md hardware execution tracing (CoreSight ETE/TRBE) plus the differential + method: the five gates and how each fails silently, per-thread `cs_etm` + capture, and two one-bit-different runs turned into the input-dependent + block set — with the two open flattening families' state machines + read from source, and the xvmp IR-VM shape + hardware-sampling-and-crash.md + data flow without a known address (SPE: PC + data address per + sample), crash forensics when the stack is not trustworthy (BRBE + branch records / the ETE tail), and runtime addresses turned into + symbols and static offsets (`trace_addr2sym.py`, PIE vaddr/file-offset + rules) + kernel-event-injection.md the driver as your hands inside the target: watchpoints armed from the + kernel with no tracer relationship, register rewrites at the trap, the + primitive map per route (KPM/LKM/eBPF/Zygisk), and the composition + recipes that stack this layer with the other two on-device-tooling.md working from the phone itself: MT Manager edit/repack/sign and its APK MCP, LSPosed Manager, Termux+frida, on-device data inspection java2c-and-jni-sinking.md Java2C and JNI sinking, the two hardening shapes most easily confused @@ -392,6 +415,31 @@ scripts/ parameterized, path-agnostic kernelsu_syscall_mask.py generate a KernelSU/APatch syscall-masking scaffold: an installable userspace module skeleton plus KPM/LKM/eBPF kernel-side templates, each with its version gate and an explicit unverified label + hw_watch.py invisible memory watchpoints / execution breakpoints on arm64: plan + (WCR/BCR encoders with the 8-byte-window rules), emit (a self-contained + on-device C helper — compiles clean as a static aarch64 ELF), gates, + run (build+push+stream hits) and decode (module+offset per hit) + trace_diff.py differential trace reduction: reduce one trace to runs+histogram, or + diff two traces into first-divergence / divergent regions / blocks + unique to each side — producer-agnostic (Stalker logs or any + address-per-line stream, incl. decoded cs_etm) + ete_capture.py CoreSight ETE capture with gates first: the five gates (kernel era, + coresight sysfs, cs_etm PMU, perf-with-CORESIGHT, permission), the + perf/OpenCSD build recipe, and the per-thread record wrapper whose + `--control` validation run must pass before any real capture + kernel_instrument_scaffold.py + generate the kernel-side instrumentation scaffold: KPM/LKM watchpoint + + register-rewrite templates and an honest userspace carrier, each with + its gate and a `NOT COMPILED AND NOT LOADED` banner that `verify` + enforces + trace_addr2sym.py runtime addresses -> symbols + static offsets, zero-dependency + ELF64 (PT_LOAD vaddr/file-off conversion, .dynsym/.symtab, stripped + fallback): accepts hw_watch hits, trace mod+off keys or raw address + lists; host-verified against a real cross-compiled aarch64 .so + vm_handler_slice.py de-virtualize an interpreter trace: dispatcher detection, slicing + into handler segments at dispatch entries, signature clustering, and + the guest-opcode trajectory via an optional handler->opcode map — + calibrated against the open xvmp IR virtualizer rasc_build.py build and verify rasc, the Rust ASC re-implementation: --check what is present, --build clone plus cargo, --verify an APK against droidasc and fail on any class-set difference @@ -513,7 +561,11 @@ targets under those routes (`tests/benchmark.md`): it added **Java2C discriminat misdiagnosis that sends an agent hunting a decrypted DEX that never exists), **split APK / App Bundle handling**, **schema-free protobuf decoding**, a **Dex-VMP differential** harness, and **kernel-module templates with their version gates** — and it corrected two earlier claims -whose measurements disagreed with them. Unity/IL2CPP logic recovery, +whose measurements disagreed with them. A **hardware pass** then added the layer under the +software tooling: **hardware watchpoints** with their honest `TracerPid` ledger, **ETE/TRBE +differential tracing** with its five gates, and **kernel-side injection templates** — every +host-checkable piece measured, every device-bound piece labelled `unverified` until a gate +opens. Unity/IL2CPP logic recovery, React Native/Hermes bytecode internals, and defeating a server-side authority are **not** covered, and the skill is written to say so and stop rather than apply the nearest documented procedure to a target it was not written for. diff --git a/README.zh-CN.md b/README.zh-CN.md index aeb8fd1..844484f 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -62,6 +62,7 @@ - 分辨真实内存 dump 与**抽取壳骨架**，并知道哪条恢复路线适用——包括当 `frida` 本身被拒绝时走 root 侧 dump。这次测量能看见什么、看不见什么，写在 `skills/apk-reverse/references/advanced-unpacking.md`。 - 当逆向的成本高于调用时，**调用例程而不是逆向它**：在宿主上做模拟执行，或把一个活函数通过 Frida RPC 服务化。 - 当一个 native 函数被 OLLVM 碾平成状态机时，去读**指令级执行证据**——包括在真实设备上实测到的 Stalker 反咬的两种方式。 +- 当每一个软件级 tracer 都在修改它所观察的东西时，转向**硬件层**：arm64 硬件观察点（零代码改动，但带诚实的 `TracerPid` 账本）、带五道闸门的 ETE/TRBE 差分追踪，以及在拥有匹配闸门的设备把它们打响之前始终标注为未核实的内核注入模板。 - 认出**用户态 hooking 根本够不到那个检查**的情况（裸 `svc` 系统调用、`init_array` 早期检测）、上下相邻层实际能做什么，以及什么时候继续升级是错误答案。 - 处理**非 REST 的协议**——无 schema 的 protobuf、gRPC、QUIC/HTTP3——以及无视系统信任库的 native 侧证书固定（pinning）。 - **直接在手机上工作**：MT Manager 的编辑/重打包/签名流程及其 APK MCP 接口、LSPosed Manager、设备上的数据检查，与 PC 工具链并用，而不是用它来替代 PC 工具链。 @@ -200,6 +201,24 @@ references/ 按需加载，每个文件一个主题 用户态 hooking 可证明够不到检查：裸 `svc`、init_array 早期 检测、每种 root 方案隐藏了什么、带版本闸门的 内核路线图，以及什么时候停止升级 + hardware-watchpoints.md arm64 上的隐形内存观察点：硬件调试槽位、带着诚实 + `TracerPid` 账本的 ptrace 交付路线、8 字节窗口对齐 + 规则、各个陷阱（槽位按线程生效、信号转发、断点的 + 步越舞步），以及每次命中前都欠着的那次对照运行 + ete-diff-tracing.md 硬件执行流追踪（CoreSight ETE/TRBE）加差分法：五道 + 闸门与它们各自如何静默失败、按线程的 `cs_etm` 采集， + 以及把两次只差一个输入位的运行还原成依赖输入的块集合 + ——另附从源码读出的两个开源平坦化家族的状态机差异， + 以及 xvmp IR 虚拟机的结构 + hardware-sampling-and-crash.md + 无需预知地址的数据流（SPE：每采样带 PC + 数据地址）、 + 栈不可信时的崩溃取证（BRBE 分支记录 / ETE 尾部）， + 以及把运行时地址变成符号与静态偏移 + （`trace_addr2sym.py`，PIE vaddr/文件偏移规则） + kernel-event-injection.md 把驱动当伸进目标里的手：从内核布置、无 tracer 关系的 + 观察点、陷阱处的寄存器改写、按路线（KPM/LKM/eBPF/ + Zygisk）的能力对照表，以及把这一层与另外两层叠加的 + 组合配方 on-device-tooling.md 直接在手机上工作：MT Manager 的编辑/重打包/签名及其 APK MCP、 LSPosed Manager、Termux+frida、设备上数据检查 java2c-and-jni-sinking.md Java2C 与 JNI sinking，最容易与抽取壳混淆的两种加固 @@ -322,6 +341,28 @@ scripts/ 参数化、与路径无关 kernelsu_syscall_mask.py 生成 KernelSU/APatch 系统调用遮蔽脚手架：一个可安装的 用户态模块骨架加上 KPM/LKM/eBPF 内核侧模板，每个 都带自己的版本闸门和明确的未核实标注 + hw_watch.py arm64 上的隐形内存观察点/执行断点：plan（带 8 字节窗口 + 规则的 WCR/BCR 编码器）、emit（自包含的设备端 C 助手 —— + 可干净编译为静态 aarch64 ELF）、gates、run（构建+推送+ + 流式采集命中）与 decode（每条命中的模块+偏移） + trace_diff.py 差分轨迹归约：把一条轨迹归约为 游程+直方图，或把两条 + 轨迹对齐出 首次分歧 / 分歧区段 / 各自独有块 —— 与生产者 + 无关（Stalker 日志或任何每行一个地址的流，含解码后的 cs_etm） + ete_capture.py 先过闸门的 CoreSight ETE 采集：五道闸门（内核年代、 + coresight sysfs、cs_etm PMU、带 CORESIGHT 的 perf、权限）， + perf/OpenCSD 构建配方，以及 `--control` 验证运行必须先 + 通过的按线程记录封装 + kernel_instrument_scaffold.py + 生成内核侧插桩脚手架：KPM/LKM 观察点+寄存器改写模板与一个 + 诚实的用户态载体，各带自己的闸门和由 `verify` 强制执行的 + `NOT COMPILED AND NOT LOADED` 横幅 + trace_addr2sym.py 运行时地址 → 符号 + 静态偏移，零依赖 ELF64 + （PT_LOAD vaddr/文件偏移换算、.dynsym/.symtab、 + strip 后降级）：接受 hw_watch 命中、trace 的 mod+off 键 + 或裸地址列表；用真实交叉编译的 aarch64 .so 验证过 + vm_handler_slice.py 把解释器轨迹去虚拟化：dispatcher 识别、按分发入口切成 + handler 段、签名聚类，以及通过可选 handler→opcode 映射 + 得到的 guest opcode 轨迹 —— 按开源 xvmp IR 虚拟机校准 rasc_build.py 构建并验证 rasc，即 ASC 索引器的 Rust 重实现： --check 检查已有什么、--build 克隆并 cargo、--verify 用 droidasc 校验一个 APK，并在任何类集合差异上失败 @@ -403,7 +444,7 @@ export APKREV_TOOLS=: # POSIX 为处理你自己的应用、你被授权分析的样本，以及 CTF/竞赛沙箱而构建。它不含任何内置的第三方二进制，也不含任何目标特定数据。 -它覆盖什么、刻意不覆盖什么，写在 `SKILL.md` 顶部 **Coverage** 一节。简版：仅 Android（不含 iOS），并且在经过真实打磨的层上深入 —— dex 补丁、重打包、壳与自定义加载器、native 篡改响应，以及 Flutter/Dart AOT。一次扩展迭代加入了第二梯队的已记录路线：重打包受阻时的**模块侧交付**、**抽取壳恢复**及其 VMP 边界、用于「调用而不是阅读」的**模拟执行与实时 RPC**、针对 OLLVM 的**指令级追踪**、REST 之外的**协议逆向**、当用户态 hooking 可证明够不到时用的**内核路线**图，以及**设备上工具**。随后一次**基准迭代**把公开目标放到这些路线之下（`tests/benchmark.md`）：它加入了 **Java2C 判别**（那种会让 agent 去追一个从不存在的解密 DEX 的误诊）、**split APK / App Bundle 处理**、**无 schema 的 protobuf 解码**、一个 **Dex-VMP 差分**测试台，以及**带版本闸门的内核模块模板** —— 并纠正了先前两条与自身测量结果不一致的说法。Unity/IL2CPP 逻辑恢复、React Native/Hermes 字节码内部，以及击败服务端权威**不**在覆盖范围内，skill 的写法就是明说这一点并停下，而不是把最近的一条已记录流程套用到一个它并非为之编写的目标上。 +它覆盖什么、刻意不覆盖什么，写在 `SKILL.md` 顶部 **Coverage** 一节。简版：仅 Android（不含 iOS），并且在经过真实打磨的层上深入 —— dex 补丁、重打包、壳与自定义加载器、native 篡改响应，以及 Flutter/Dart AOT。一次扩展迭代加入了第二梯队的已记录路线：重打包受阻时的**模块侧交付**、**抽取壳恢复**及其 VMP 边界、用于「调用而不是阅读」的**模拟执行与实时 RPC**、针对 OLLVM 的**指令级追踪**、REST 之外的**协议逆向**、当用户态 hooking 可证明够不到时用的**内核路线**图，以及**设备上工具**。随后一次**基准迭代**把公开目标放到这些路线之下（`tests/benchmark.md`）：它加入了 **Java2C 判别**（那种会让 agent 去追一个从不存在的解密 DEX 的误诊）、**split APK / App Bundle 处理**、**无 schema 的 protobuf 解码**、一个 **Dex-VMP 差分**测试台，以及**带版本闸门的内核模块模板** —— 并纠正了先前两条与自身测量结果不一致的说法。随后一次**硬件迭代**加入了软件工具之下的那一层：带诚实 `TracerPid` 账本的**硬件观察点**、带五道闸门的 **ETE/TRBE 差分追踪**，以及**内核侧注入模板** —— 凡是宿主可验证的部分都实测过，凡是设备绑定的部分在闸门打开前一律标注 `unverified`。Unity/IL2CPP 逻辑恢复、React Native/Hermes 字节码内部，以及击败服务端权威**不**在覆盖范围内，skill 的写法就是明说这一点并停下，而不是把最近的一条已记录流程套用到一个它并非为之编写的目标上。 Coverage 一节完整陈述、同时也应当放在这里的四条限定： diff --git a/docs/tool-verification/EXTENSION-ete-diff.md b/docs/tool-verification/EXTENSION-ete-diff.md new file mode 100644 index 0000000..c214257 --- /dev/null +++ b/docs/tool-verification/EXTENSION-ete-diff.md @@ -0,0 +1,75 @@ +# Extension verification — ETE differential tracing (gates, reducer, differ) + +Covers the hardware-layer pass that added `scripts/ete_capture.py`, `scripts/trace_diff.py` and +`references/ete-diff-tracing.md`. Strength labels as this directory's README defines them. + +Reference environment: Linux container host, Python 3.13, adb present with no device. No +coresight-capable hardware exists in this environment at all; the reference device of the +earlier passes (kernel 4.14.186) has the ETE gate closed permanently (`observed` there). + +**Headline: the gate table, the recipe and the capture wrapper are host-measured; the +differential reducer is host-measured on synthetic flattened-dispatcher fixtures. No ETE trace +was captured or decoded anywhere in this pass — the producer side is `unverified` and the +`--control` run exists precisely so the next device says so in one command.** + +--- + +## 1. The differ, measured on a synthetic OLLVM shape — observed + +Two fixture logs built to the flattened-dispatcher rhythm `D R1 D R2 …` — run A taking +`R5/R7` (wrong password), run B taking `R3/R4` (right password), one log in raw +`stalker_trace.js` grammar, one wrapped by `run_probe.py` (`[ts] [ts] TRACE BLK …`) to prove +the parser survives both: + +``` +$ python skills/apk-reverse/scripts/trace_diff.py diff --a a.log --b b.log +run A: 8 events, 5 distinct blocks +run B: 8 events, 5 distinct blocks +shared block executions: 6 divergent regions: 2 +first divergence: after run #5 (A) / #5 (B), 5/5 events in, last common block liba.so+0x1000 +region 1 (replace) + A: liba.so+0x6000 B: liba.so+0x4000 +blocks unique to A ...: 0x6000, 0x8000 blocks unique to B ...: 0x4000, 0x5000 +``` + +What the output pins, and why each number is the one the method needs: + +- **the identical prefix** (`D R1 D R2`, 5 events) is the shared machinery — exactly where + parsing ends and the input-dependent path begins; +- **first divergence** names the dispatcher entry whose state constant differed; +- **unique-to-each-side** lists are the input-dependent block set, which is the deliverable + the reference file's §4 interprets per obfuscation. + +`reduce` on one side prints the histogram with the dispatcher as the wide-margin top +(4 hits vs 1 for every real block) — the same shape `stalker_report.py` documents for Stalker +logs, reached from a different producer. + +## 2. Gate and capture plumbing — observed (to the no-device boundary) + +`gates` prints the five-gate table; `gates --device` with adb present but no device degrades +to five `OPEN` rows and `RESULT=partial` rather than a traceback — measured. `capture --pid 1` +without a device exits through the pull failure with `RESULT=blocked` and exit code 3 +(no-device capability refusal, per the CLI contract) — measured. `recipe` prints the +Termux and cross build paths for perf-with-CORESIGHT — `inferred` from the kernel's +tools/perf CoreSight documentation and the OpenCSD project, not from a build performed here. + +## 3. What this pass deliberately did not do + +- No perf-with-CORESIGHT was built (host or device): the recipe is a documented path, and a + reader who builds it owes the control record (`capture --control`) before any real capture. +- No decoded cs_etm stream was fed to `trace_diff.py`; the `plain` parser accepts the + address-bearing lines `perf script` emits, but that exact pairing is `unverified`. +- No claim about ETE's runtime overhead or its visibility to targets was measured. The + reference states the mechanism (no code modification, no ptrace) and refuses to go further. + +## 4. Facts this pass pinned for the next one + +- The differ is **producer-agnostic by design**: the same command answers Stalker logs today + and cs_etm decode output tomorrow. That is why the diff subcommand ships now, not after the + first ETE capture — it is useful the day a Stalker pair exists. +- The five gates each fail **silently** downstream; the reference's table says which silence + each gate produces. The one-instruction control record is the only gate that exercises the + whole pipe, which is why `capture` refuses to continue when it fails. +- Run design is part of the tool contract, not user care: `trace_diff.py` refuses empty logs + with a message that points at the zero-event diagnostics, because an empty trace is a + capture failure wearing an analysis result's clothes. diff --git a/docs/tool-verification/EXTENSION-hw-watchpoints.md b/docs/tool-verification/EXTENSION-hw-watchpoints.md new file mode 100644 index 0000000..4bee6c0 --- /dev/null +++ b/docs/tool-verification/EXTENSION-hw-watchpoints.md @@ -0,0 +1,69 @@ +# Extension verification — hardware watchpoints (ptrace route, helper, encoders) + +Covers the hardware-layer pass that added `scripts/hw_watch.py` and +`references/hardware-watchpoints.md`. Strength labels as this directory's README defines them. + +Reference environment: Linux container host, Python 3.13, cross `gcc 16.2.0` targeting +`aarch64-linux-gnu`. No Android device was attached during this pass; `adb` exists on PATH but +reports no devices (`adb: no devices/emulators found`, observed). + +**Headline: the encoders, the plan/emit/decode paths and the helper's build are measured on the +host — including a clean static aarch64 compile of the emitted C. No watchpoint was armed on any +device by this pass; the device-side behaviour is `unverified` by this record.** + +--- + +## 1. Encoder acceptance and refusal — observed + +``` +$ python skills/apk-reverse/scripts/hw_watch.py plan --addr 0x1004 --len 4 --access w --json +{"status": "ok", "specs": [{"kind": "watch", "addr": 4100, "len": 4, "access": "w", "vr": 4096, + "cr": 7701, "cr_fields": "BAS=0xf0 LSC=store PAC=EL0 E=1"}]} +``` + +WCR `0x1E15` = BAS `0xf0` (bytes 4–7 of the aligned window) | store | EL0 | enabled — the bit +fields decompose back to the inputs exactly. Refusals measured with exit 2 and a reason: + +- `--addr 0x7b2000123456 --len 8` (addr&7=6, 6+8>8): *"crosses its aligned 8-byte window"* +- `plan --break 0x1002`: *"breakpoint PC 0x1002 is not 4-byte aligned"* + +The canonical values are pinned by unit test (`tests/unit/test_hw_watch.py`): 8-byte rw watch +`WCR=0x1FFD`, address-match EL0 breakpoint `BCR=0x3F05`. + +## 2. The emitted helper compiles as a static aarch64 ELF — observed + +``` +$ python skills/apk-reverse/scripts/hw_watch.py emit --out work/hwwatch +$ gcc -static -O2 -o hwwatch work/hwwatch/hwwatch_helper.c +$ file hwwatch +hwwatch: ELF 64-bit LSB executable, ARM aarch64, version 1 (GNU/Linux), statically linked +``` + +Zero errors, zero warnings after one fix (a `TRAP_HWBKPT` redefinition against glibc's +`signal.h` needed an `#ifndef` guard; bionic defines the same macro). The compiler here is a +cross glibc toolchain, not the NDK — the binary was not run, and the runtime behaviour of the +ptrace loop (seize/interrupt per tid, `PTRACE_O_TRACECLONE` arming, `TRAP_HWBKPT` siginfo +handling, the disable-step-rearm breakpoint dance, group-stop `PTRACE_LISTEN`) remains +**`unverified`** until a device runs the control procedure in the reference's §3. + +## 3. What this pass deliberately did not do + +- No device was attached: no `READY watch_slots=N` probe, no slot-capacity read, no control + run, no hit. Every device-behaviour claim in `hardware-watchpoints.md` carries `unverified`. +- `decode` was exercised on synthetic `HIT` lines only (unit test); a real `/proc//maps` + pull was not available to feed it. +- `run` (build+push+stream) was exercised to its argument-validation and no-adb exits only; + its `exit 3` on missing adb was measured before a device was ever contacted. + +## 4. Facts this pass pinned for the next one + +- `slot_capacity` reads `dbg_info & 0xff` from the `NT_ARM_HW_WATCH`/`NT_ARM_HW_BREAK` + regsets — the ABI gdb uses for the same purpose; on a masked kernel this returns 0 and the + helper prints `READY ... watch_slots=0`, which the reference file treats as a **closed + gate**, not a tuning problem. +- The helper's output grammar (`READY`/`ARM`/`HIT`/`EXIT`/`DONE`) is load-bearing: `decode` + and the reference's control procedure both parse it. Changing the helper means regenerating + both ends. +- The honesty ledger (ptrace route leaves `TracerPid` visible) is stated in three places on + purpose — script header, reference §1, evidence summary — because the tempting wrong claim + ("invisible") is exactly what a reader under pressure wants to believe. diff --git a/docs/tool-verification/EXTENSION-kernel-instrument.md b/docs/tool-verification/EXTENSION-kernel-instrument.md new file mode 100644 index 0000000..aa03c01 --- /dev/null +++ b/docs/tool-verification/EXTENSION-kernel-instrument.md @@ -0,0 +1,74 @@ +# Extension verification — kernel-side instrumentation scaffold (templates, gates, verify) + +Covers the hardware-layer pass that added `scripts/kernel_instrument_scaffold.py` and +`references/kernel-event-injection.md`. Strength labels as this directory's README defines +them. This pass follows `EXTENSION-kernel-weapons.md`'s shape deliberately: a generator whose +host behaviour is measured, kernel-side templates that have never been compiled or loaded +anywhere, and the gate table that says so. + +Reference environment: Linux container host, Python 3.13, no `aarch64-none-elf-gcc` +(`gates` reports NOT FOUND — observed), no APatch/KernelPatch boot image, no device. + +**Headline: `gates`/`generate`/`verify` are measured on the host. Every generated kernel-side +file carries a `NOT COMPILED AND NOT LOADED` banner that `verify` enforces. The +register-rewrite action is labelled experimental in the template itself — it was never run +anywhere, by anyone in this repository.** + +--- + +## 1. Generate + verify round trip — observed + +``` +$ python skills/apk-reverse/scripts/kernel_instrument_scaffold.py generate \ + --out work/kwatch --pid 12345 --watch w:0x7b2000123450:8:w --break b:0x7b20001234c0 --set-reg x0=0x1 +wrote work/kwatch + kpm/src/kwatch.c kpm/Makefile lkm/kwatch.c lkm/Makefile + module.prop README.md post-fs-data.sh config/instrument.json +RESULT=ok +$ python skills/apk-reverse/scripts/kernel_instrument_scaffold.py verify --dir work/kwatch +verified 4 kernel-side file(s), 0 problem(s) +RESULT=ok +``` + +Spec validation reuses the hardware rules and refuses at generation, before any file is +written: a watch crossing its 8-byte window and a misaligned breakpoint PC both exit 2 with +the reason (`observed`, same refusal strings as `hw_watch.py plan` — one rule, one copy in +each tool, both unit-tested). + +The generated KPM template embeds: `kfunc_def(...)` declarations **without** `extern` (the +loader-rejection lesson recorded in `EXTENSION-kernel-weapons.md`, restated in the template's +own header), a callback whose first statements are the cheap filter, and the watch config as +generated constants with *"regenerate rather than hand-edit"* in the config JSON. + +## 2. The honesty devices this pass ships + +- **The primitive map is a capability table, not a wishlist.** Each cell names what the route + can do, and the `no` cells (eBPF cannot create perf breakpoint events; a userspace module + cannot arm anything) are the rows that stop a reader from spending a day on a route that + cannot express the need. +- **The experimental action is fenced three times**: template comment, script header, and the + reference's stop table ("the first run is a measurement"). Rewriting `pt_regs` in a perf + overflow callback is mechanically what a ptrace stop does, and that is exactly as far as + the claim goes. +- **The userspace carrier cannot arm watchpoints and says so in its own README** — the same + structural correction `EXTENSION-kernel-weapons.md` made for syscall masking, applied + before anyone could misread this module the same way. + +## 3. What this pass deliberately did not do + +- Nothing was compiled: no `aarch64-none-elf-gcc` exists here, no KernelPatch image exists on + any reference device, and the LKM route has no matching kernel source. All three kernel + files are `unverified` templates and are labelled so in the files themselves. +- No claim about whether `register_hw_breakpoint` can be resolved from a KPM on any specific + KernelPatch revision is made; the template says to verify symbol names against the + revision's `kpmodule.h`, which is the same instruction the syscall-mask template gives. + +## 4. Facts this pass pinned for the next one + +- The kernel's **perf breakpoint layer owning the debug registers** is a gate in its own + right: if `CONFIG_HW_PERF_EVENTS` is out, both templates close regardless of root scheme. + Probe: `ls /sys/bus/event_source/devices/breakpoint` on the device. +- `verify` exists so the banner survives edits: someone will adapt a template on a real + device, strip the banner, and the next reader must be able to tell an adapted-unverified + file from a measured one. A banner replaced by a real measurement is the intended + outcome, and `kernel-event-injection.md`'s checklist says so. diff --git a/docs/tool-verification/EXTENSION-sampling-crash.md b/docs/tool-verification/EXTENSION-sampling-crash.md new file mode 100644 index 0000000..5c3a9ea --- /dev/null +++ b/docs/tool-verification/EXTENSION-sampling-crash.md @@ -0,0 +1,100 @@ +# Extension verification — sampling, crash forensics, symbolization, de-virtualization + +Covers the second hardware-layer pass: `references/hardware-sampling-and-crash.md`, +`scripts/trace_addr2sym.py`, `scripts/vm_handler_slice.py`, and the source-reading +corrections to the flattening recipes. Strength labels as this directory's README defines +them. + +Reference environment: Linux container host, Python 3.13, cross `gcc 16.2.0` targeting +`aarch64-linux-gnu` (the toolchain that makes the symbolizer's "real ELF" half testable). +Network access was used once, read-only, to fetch the two public obfuscator implementations +the calibration cites (below). No Android device, no coresight/SPE/BRBE hardware. + +**Headline: the address→symbol loop is measured against a real cross-compiled library, the +de-virtualization slicer is measured on synthetic interpreter traces, and the SPE/BRBE +producer side remains documented-and-gated. The flattening-recipe corrections are read from +public source, and that source is named.** + +--- + +## 1. `trace_addr2sym.py` against a real library — observed + +``` +$ gcc -shared -fPIC -O2 -o libtlib.so tlib.c # cross gcc -> aarch64 ELF64 +$ readelf --syms -W libtlib.so +... 00000000000005e0 132 FUNC GLOBAL DEFAULT 9 encrypt_block +... 0000000000000664 4 FUNC GLOBAL DEFAULT 9 decrypt_block +$ python skills/apk-reverse/scripts/trace_addr2sym.py --elf libtlib.so --offs 0x610 + 0x610 -> encrypt_block+0x30 [vaddr 0x610 file 0x610 .text+0x130] +``` + +The resolution agrees with `readelf` exactly (0x610 − 0x5e0 = 0x30, inside the sized symbol); +a point outside any function falls to the nearest-below symbol with its section shown +(`0x700 -> _fini+0x98 [.eh_frame+0x40]`), which is GNU-addr2line-shaped behaviour and is +pinned by unit test together with the PT_LOAD `vaddr≠offset` conversion on a hand-built +minimal ELF. That minimal-ELF fixture also covers the stripped-library fallback +(section+offset, no symbols). + +## 2. `vm_handler_slice.py` on the calibrated shape — observed + +A synthetic interpreter trace in the shape the public IR virtualizer emits +(dispatcher D, handler blocks, one const-materialization segment, an unclosed tail): + +``` +$ python skills/apk-reverse/scripts/vm_handler_slice.py --log vm.log --opcode-map map.json +dispatcher: libvm.so+0x1000 (41.7% of 24 blocks, 10 hits) +segments: 10 total, 7 unique, avg len 2.00 [last segment UNCLOSED: exit path or truncated window] +opcode trajectory (first 20): OP_LOAD OP_LOAD OP_STORE OP_BR OP_LOAD OP_LOAD OP_BR OP_CALL OP_LOAD OP_RET +``` + +What the output pins, and why each part is the part a de-virtualization needs: the +**dispatcher share** (with the diagnostic that separates a real dispatch loop from a spin +loop), the **segment table** (one row per handler chain, first block = handler entry = +the `--opcode-map` key), the **trajectory** (the guest program, one signature per executed +guest opcode), the **OP_CALL materialization segment** appearing as its own signature (the +virtualizer's const dispatch — the second observation point the calibration names), and the +**UNCLOSED tail** flagged rather than silently dropped. The auto-detection rule is +top-of-histogram **plus predecessor diversity** (≥3 distinct predecessors), not a naive +count margin: the first implementation used a 4x margin and refused this very fixture — +in a stack-shaped VM the hottest handler executes on every guest instruction, exactly as +often as the dispatcher. That refusal-to-accept is the load-bearing part of the auto rule. + +## 3. The flattening-recipe corrections — observed source, inferred runtime + +Fetched read-only from the public implementations the user supplied +(github.com/dpkg123/ollvm-aosp-toolchain: branch `ollvm-r614150-clang23-rebase+omp` +[Hikari family] and `ollvm-r614150-clang23+omp` [Pluto family]; commits 63fec41f and +9b7cf75e for the IR virtualizer; 2026-10): + +- **Hikari `Flattening.cpp`**: `switchVar` is an entry-block **alloca** written through a + `switchVarAddr` double pointer (indirect store per transition); case values are + `scramble32`-permuted; EH-bearing functions are skipped. +- **Pluto `Flattening.cpp`**: two allocas (`switchVar`, `switchXor`) with a **rolling XOR** + (`enc^=delta; key^=delta; cond=enc^key` per dispatch); case values fully random; the true + case exists only in the `switchCond` register for one instruction; EH/`callbr`/`indirectbr` + functions skipped. +- **Hikari `AntiDebugging.cpp`**: on Darwin/AArch64 it injects **inline-assembly raw `svc`** + with a randomized register-setup order and a **randomized svc immediate**; elsewhere it + links precompiled anti-debug IR. Consequence recorded in the reference: byte signatures + cannot survive the randomization; semantic site-location (`svc_scan.py`) and hardware + observation do not care. +- **The IR virtualizer**: `vm_exec(code, size, vm_args, num)` stubs, boxed-argument array, + 11 one-byte opcodes + 19 arithmetic sub-ops, const/call materialization dispatch, + switch-to-if lowering. + +These corrected `hardware-watchpoints.md`'s state-variable recipe (the old text assumed a +global and a readable value — wrong for both families as shipped) and added the +family-difference table to `ete-diff-tracing.md` §5. The fetches live under +`tools/_work/refs/` (git-ignored); **runtime behaviour of these passes is `inferred` from +source, never executed here.** + +## 4. What this pass deliberately did not do + +- No SPE capture, no BRBE record read, no `perf -j any` on any device: those routes ship as + documented-and-gated, and the reference's §5 failure modes say exactly how each gate + fails silently. +- No xvmp-hardened binary was compiled or traced: the slicer's calibration is source-read, + and its first run against a real sample is a measurement, not a confirmation. +- The `--opcode-map` a real target needs is a manual labelling task over the segment table; + this pass generated the fixture's map by hand, which is exactly the workflow the script + documents. diff --git a/docs/tool-verification/README.md b/docs/tool-verification/README.md index 9f3cfd3..2c5ba40 100644 --- a/docs/tool-verification/README.md +++ b/docs/tool-verification/README.md @@ -89,6 +89,19 @@ written down. | `EXTENSION-detection-pipeline.md` | Naming the check that fires: three arms on a public MASTG target (control alive, probe arm dead in ≈300 ms via `strstr("frida")`, with the `TracerPid=0` versus 4-frida-mappings asymmetry), `svc_scan.py`'s two decoders agreeing on an identical 214-site set, the layered Java-to-svc descent, and the ptrace-free dump boundary on targets whose dex is deflated inside the APK | | `EXTENSION-dexsql.md` | The SQL query layer adopted as a **third indexer** beside rasc/droidasc/ddc: identical class-definition sets on four archives (zero differences either way), the string xref agreement and the `=` versus `LIKE` match-semantics trap, `decode` against `ddc disasm` line for line, and the persistent cache measured as **not** a speed switch | +## Hardware-layer pass record + +The hardware pass (hardware watchpoints, ETE differential tracing, kernel-side event injection) +follows the same shape: what runs on a host is measured, and everything that needs a device or a +kernel toolchain is an explicitly-labelled template whose gate says where it closes. + +| File | Covers | +|---|---| +| `EXTENSION-hw-watchpoints.md` | The ptrace hardware-watchpoint route: encoders accepted and refused exactly as the alignment rules demand, and the emitted on-device helper **compiled clean as a static aarch64 ELF** — while no watchpoint was armed on any device, and the ledger that says the ptrace route leaves `TracerPid` visible | +| `EXTENSION-ete-diff.md` | The differential half measured on synthetic flattened-dispatcher fixtures (first divergence, unique-per-side sets), the gate/capture plumbing measured to the no-device boundary, and what was deliberately not done: no perf-with-CORESIGHT build, no cs_etm stream, no overhead claim | +| `EXTENSION-kernel-instrument.md` | The kernel-side scaffold's generate/verify round trip, the spec-refusal rules shared with `hw_watch.py`, the capability map whose `no` cells do the gatekeeping, and the three-time-fenced experimental register-rewrite action | +| `EXTENSION-sampling-crash.md` | The second hardware pass: `trace_addr2sym.py` verified against a real cross-compiled `.so` (readelf-cross-checked), the interpreter-trace slicer measured on the calibrated xvmp shape (including the 4x-margin bug it found in itself), and the flattening-recipe corrections read from the two public obfuscator implementations the user supplied | + ## Adding a run Record the tools, versions and exact commands, and label each claim. Keep target-specific facts out: diff --git a/skills/apk-reverse/SKILL.md b/skills/apk-reverse/SKILL.md index 2e52eef..2260e59 100644 --- a/skills/apk-reverse/SKILL.md +++ b/skills/apk-reverse/SKILL.md @@ -208,6 +208,12 @@ unread in this repository. | You only need to **call** the target's own routine (sign, token, encrypt) rather than change the app | `references/emulation-and-rpc.md` — emulate it, or service-ify the live function over Frida RPC | | A native function is a many-thousand-line `switch` state machine, or the decompiler's output is meaningless | `references/native-dbi-and-deobfuscation.md` — OLLVM shapes, a Stalker trace, and how far a trace actually gets you | | `Stalker.follow` installs but no events arrive, or following a hot libc export crashes the process | `references/native-dbi-and-deobfuscation.md` §6 failure modes — this repository measured both | +| You must observe or alter the target with **no code change, no injected thread, and `TracerPid` = 0** — page checksums catch every hook | `hardware-watchpoints.md` — hardware debug slots via ptrace with its honest stealth ledger, then `kernel-event-injection.md` for the kernel-armed route that needs no tracer at all | +| A crash's backtrace cannot be trusted — frames in wrong modules or anonymous regions, flattened/virtualized code, or a smashed stack | `hardware-sampling-and-crash.md` — BRBE branch records / ETE tail as the real control flow, then `scripts/trace_addr2sym.py` to symbolize every PC | +| You hold a list of **runtime addresses** (hits, trace PCs, divergence points) and need symbols / static offsets, not hex | `hardware-sampling-and-crash.md` §3 The crash loop — `scripts/trace_addr2sym.py` with its vaddr/file-offset rules | +| A trace of a virtualized function is just an **interpreter loop repeating** and reads like noise | `scripts/vm_handler_slice.py` — slice at dispatcher entries into handler segments, cluster signatures, recover the guest-opcode trajectory (xvmp-calibrated) | +| You need to know **where two runs of the same operation diverge** (password/license/key), not just what executed | `ete-diff-tracing.md` — ETE hardware-trace gates plus the differential method; `scripts/trace_diff.py` also diffs Stalker logs | +| A generated KPM/LKM template is in front of you, or the kernel route feels like the plan | `kernel-event-injection.md` — the primitive map, the composition recipes, and the gates every kernel-side file must pass before it is a weapon | | The traffic is protobuf/gRPC/QUIC, or a proxy sees TLS but requests still fail on a Flutter app | `references/protocol-reverse.md` — schema-less protobuf, frame capture, and native-side pinning | | Userspace hooks land and the app still dies: the check reads `/proc/self/status` through a raw `svc`, or runs before `JNI_OnLoad` | `references/kernel-and-environment-hardening.md` — what the next layer up and down can actually do, and when to stop | | You must edit, repack, sign or inspect the APK **from the phone itself** | `references/on-device-tooling.md`, `scripts/mt_mcp_probe.py` | diff --git a/skills/apk-reverse/evals/evals.json b/skills/apk-reverse/evals/evals.json index cafca8e..3eff61e 100644 --- a/skills/apk-reverse/evals/evals.json +++ b/skills/apk-reverse/evals/evals.json @@ -51,6 +51,19 @@ "At least one cross-check with an independent decoder is performed or explicitly flagged as not done." ], "_targets_reference": "scripts/dex_strings.py, scripts/dexutil.py, scripts/find_refs.py, references/byte-level-patching.md" + }, + { + "id": 4, + "prompt": "A hardened .so kills the process whenever I attach frida or set any inline hook; it also checksums its own code pages. I need to know which code writes the plaintext key buffer right before encryption. What route do I use?", + "expected_output": "The answer routes to the hardware layer with its honesty ledger, not to a harder hook. It names the hardware watchpoint as the observation that modifies no code, states plainly that the ptrace delivery leaves TracerPid visible (and that a target reading /proc over raw svc needs the kernel-armed route instead), requires the control run before any hit is trusted, and refuses to present the kernel template as a working weapon.", + "_assertions": [ + "Names hardware watchpoints (debug registers) as the route that survives page checksums because no code byte changes.", + "States the TracerPid trade-off of the ptrace route explicitly rather than calling it invisible.", + "Requires a control run (a deliberately touched watched address producing one HIT) before trusting hits.", + "Does not recommend building a kernel module as the first move, and labels kernel-side templates unverified.", + "Routes to references/hardware-watchpoints.md and scripts/hw_watch.py rather than re-deriving the mechanism." + ], + "_targets_reference": "references/hardware-watchpoints.md, references/kernel-event-injection.md" } ] -} +} \ No newline at end of file diff --git a/skills/apk-reverse/evidence/capability-matrix.json b/skills/apk-reverse/evidence/capability-matrix.json index c644608..d55d215 100644 --- a/skills/apk-reverse/evidence/capability-matrix.json +++ b/skills/apk-reverse/evidence/capability-matrix.json @@ -416,6 +416,83 @@ "evidence_not_shipped": ["docs/tool-verification/README.md (repository root, not shipped)"], "limits": "Determining who owns a gate is covered; breaking an authorization the server performs is not. There is no client-side patch that reaches it, and the honest deliverable is the residual: which authority decides, and why the artifact cannot change it." }, + { + "capability": "hardware-watchpoints-ptrace", + "group": "native-dynamic", + "status": "partial", + "route_measured": false, + "strength": "unverified", + "benchmark_rows": [], + "evidence": ["references/hardware-watchpoints.md", "scripts/hw_watch.py"], + "evidence_not_shipped": ["docs/tool-verification/EXTENSION-hw-watchpoints.md (repository root, not shipped)"], + "limits": "Encoders, refusals and the emitted helper's static aarch64 build are measured on the host; arming a watchpoint on a device, the slot-capacity probe and the control run were never exercised here. The ptrace route leaves TracerPid visible by design -- the ledger in the reference is the contract." + }, + { + "capability": "trace-differential-reduction", + "group": "native-dynamic", + "status": "partial", + "route_measured": true, + "strength": "observed", + "benchmark_rows": [], + "evidence": ["references/ete-diff-tracing.md", "scripts/trace_diff.py"], + "evidence_not_shipped": ["docs/tool-verification/EXTENSION-ete-diff.md (repository root, not shipped)"], + "limits": "The reducer/differ is host-tested on synthetic flattened-dispatcher fixtures in both raw and run_probe-wrapped Stalker grammars. No producer (Stalker pair or cs_etm decode) was ever run through it end to end on a real target." + }, + { + "capability": "ete-hardware-tracing", + "group": "native-dynamic", + "status": "blocked", + "route_measured": false, + "strength": "unverified", + "benchmark_rows": [], + "evidence": ["references/ete-diff-tracing.md", "scripts/ete_capture.py"], + "evidence_not_shipped": ["docs/tool-verification/EXTENSION-ete-diff.md (repository root, not shipped)"], + "limits": "Gates, recipe and capture wrapper are host-measured to the no-device boundary. No coresight capture or decode ever ran here; the reference device's kernel (4.14.186) closes the gate permanently. The --control record is the one-command check that upgrades or refutes the route on a real device." + }, + { + "capability": "kernel-instrument-injection", + "group": "kernel", + "status": "blocked", + "route_measured": false, + "strength": "unverified", + "benchmark_rows": [], + "evidence": ["references/kernel-event-injection.md", "scripts/kernel_instrument_scaffold.py"], + "evidence_not_shipped": ["docs/tool-verification/EXTENSION-kernel-instrument.md (repository root, not shipped)"], + "limits": "Generate/verify and the shared spec-refusal rules are measured on the host. No kernel-side file was ever compiled or loaded; the register-rewrite action is labelled experimental in the template itself. Same gate family as the syscall-mask scaffold: a userspace carrier arms nothing." + }, + { + "capability": "spe-brbe-crash-forensics", + "group": "native-dynamic", + "status": "partial", + "route_measured": false, + "strength": "unverified", + "benchmark_rows": [], + "evidence": ["references/hardware-sampling-and-crash.md"], + "evidence_not_shipped": ["docs/tool-verification/EXTENSION-sampling-crash.md (repository root, not shipped)"], + "limits": "SPE data-flow sampling and BRBE branch records are documented-and-gated: no sample or record was ever captured on any device this repository used. The crash loop's ordering (tombstone -> branch records -> symbolize) is the deliverable." + }, + { + "capability": "address-symbolization", + "group": "native-static", + "status": "ok", + "route_measured": true, + "strength": "observed", + "benchmark_rows": [], + "evidence": ["references/hardware-sampling-and-crash.md", "scripts/trace_addr2sym.py"], + "evidence_not_shipped": ["docs/tool-verification/EXTENSION-sampling-crash.md (repository root, not shipped)"], + "limits": "The zero-dependency ELF64 walker is host-verified against a real cross-compiled aarch64 .so (symbols cross-checked with readelf) and a hand-built minimal ELF. Runtime-address inputs (hits, traces) were exercised on fixtures only, never on a live device trace." + }, + { + "capability": "interpreter-trace-devirtualization", + "group": "native-dynamic", + "status": "partial", + "route_measured": true, + "strength": "observed", + "benchmark_rows": [], + "evidence": ["references/ete-diff-tracing.md", "scripts/vm_handler_slice.py"], + "evidence_not_shipped": ["docs/tool-verification/EXTENSION-sampling-crash.md (repository root, not shipped)"], + "limits": "Dispatcher detection, segment slicing, open-tail handling and the opcode-map trajectory are host-tested on synthetic interpreter traces. Calibration facts (vm_exec shape, opcode table, boxed args, const-materialization dispatch) are read from a public IR virtualizer's source; no hardened binary was ever traced by this repository." + }, { "capability": "dex-sql-query-layer-dexsql", "group": "dex", diff --git a/skills/apk-reverse/evidence/known-limitations.md b/skills/apk-reverse/evidence/known-limitations.md index f042ef2..305b8f5 100644 --- a/skills/apk-reverse/evidence/known-limitations.md +++ b/skills/apk-reverse/evidence/known-limitations.md @@ -36,6 +36,13 @@ mechanism's truth. It means nobody has yet put a command and its output behind i - **The runtime routes need a real device.** Hooking, dumping, install-then-launch verification and the module route all require a rooted device; several of them additionally require a framework the device must have active. +- **The hardware layer is a host-verified toolkit, not a device-verified one.** The watchpoint + encoders and the emitted helper's build (static aarch64, cross gcc) are measured, the differential + reducer is fixture-tested, the ELF symbolizer is verified against a real cross-compiled `.so`, and + the kernel templates generate and verify — but **no watchpoint was armed, no ETE/SPE trace + captured, no BRBE record read, and no kernel file compiled or loaded on any device by this + repository**. Every device-side step carries its control procedure in the reference file, and a + control that has not run means the result is `unverified`, not working-as-intended. - **The evidence record and the benchmark matrix do not ship.** Their paths appear in older documents as if they were openable files. They resolve only in the repository they were written in. diff --git a/skills/apk-reverse/references/coverage-and-limits.md b/skills/apk-reverse/references/coverage-and-limits.md index 97be066..35ada80 100644 --- a/skills/apk-reverse/references/coverage-and-limits.md +++ b/skills/apk-reverse/references/coverage-and-limits.md @@ -104,6 +104,25 @@ its own strength note at the top. - **What to do when userspace hooking provably cannot reach the check** — raw `svc` syscalls, `init_array`-early detection, and the kernel-route map with its version gate (`kernel-and-environment-hardening.md`). +- **Hardware watchpoints (ptrace route)** — observing who reads/writes one location with zero + code modification, its honest `TracerPid` ledger, and the control run every hit is owed. + Encoders and the on-device helper's build are host-measured; arming on a device is not + (`hardware-watchpoints.md`, `scripts/hw_watch.py`). +- **ETE differential tracing** — the five gates and how each fails silently, per-thread `cs_etm` + capture, and the one-bit-different pair design that turns two traces into the input-dependent + block set. The reducer/differ is host-measured; the producer side needs a coresight device this + repository never had (`ete-diff-tracing.md`, `scripts/ete_capture.py`, `scripts/trace_diff.py`). +- **SPE/BRBE sampling and crash forensics** — data flow without a known address, branch records + when a crash's stack cannot be trusted, and the address→symbol loop. The ELF symbolizer is + host-verified; no SPE sample or BRBE record was ever captured here + (`hardware-sampling-and-crash.md`, `scripts/trace_addr2sym.py`). +- **Interpreter-trace de-virtualization** — dispatcher slicing, handler-signature clustering and + the guest-opcode trajectory, host-tested on synthetic interpreter traces and calibrated + against a public IR virtualizer's source; no hardened binary was ever traced + (`scripts/vm_handler_slice.py`). +- **Kernel-side event injection** — invisible watchpoints and register rewrites from a driver, as + generated templates with their gates; nothing kernel-side was ever compiled or loaded here + (`kernel-event-injection.md`, `scripts/kernel_instrument_scaffold.py`). - **Working from the phone itself** — MT Manager, its APK MCP surface, LSPosed Manager, on-device data inspection (`on-device-tooling.md`, `scripts/mt_mcp_probe.py`). diff --git a/skills/apk-reverse/references/coverage/route-inventory.md b/skills/apk-reverse/references/coverage/route-inventory.md index 8738952..1b492e1 100644 --- a/skills/apk-reverse/references/coverage/route-inventory.md +++ b/skills/apk-reverse/references/coverage/route-inventory.md @@ -43,6 +43,11 @@ route was not measured on this repository's evidence**. Treat them as leads with | Split APK / App Bundle sets | reading a set, merge versus unified re-signing, the install refusal each mistake produces — `split-apk.md` | | Dex-VMP differential analysis | the known-plaintext route, what it can and cannot automate — `vmp-differential-analysis.md` | | Kernel-module templates | a scaffold **and** its version gate; a userspace module cannot change a syscall return value — `kernelsu_syscall_mask.py` | +| Hardware watchpoints (ptrace) | who reads/writes one address with zero code change, the encoders host-tested and the helper compiled — never armed on a device here — `hardware-watchpoints.md` | +| ETE differential tracing | two one-bit-different runs into the input-dependent block set; gates probed by script, producer unverified — `ete-diff-tracing.md` | +| SPE/BRBE sampling and crash forensics | data flow without a known address, branch records past an untrustworthy stack, addresses→symbols — `hardware-sampling-and-crash.md` | +| Interpreter-trace de-virtualization | handler slicing + guest-opcode trajectory, calibrated on an open IR-VM's source — `vm_handler_slice.py` | +| Kernel-side event injection | watchpoints and register rewrites from a driver, as gated templates — `kernel-event-injection.md` | | Working from the phone itself | MT Manager edit/repack/sign and its APK MCP, LSPosed Manager, Termux — `on-device-tooling.md` | | Publishing without publishing the target | the leak scanner, the do-not-anonymize list, and the graded precedent library — `desensitization-and-leak-scans.md`, `scripts/scan_leaks.py`, `precedents/` | @@ -58,7 +63,11 @@ dependency yourself. - **Device work needs `adb` plus a device the app does not object to**; dynamic work additionally needs a `frida-server` matching the host version. - **A kernel-side answer needs a module and a kernel that accepts it** — the version gate is in - `kernel-and-environment-hardening.md`. + `kernel-and-environment-hardening.md`; the instrumentation templates add their own + (the kernel's perf breakpoint layer must exist — `kernel-event-injection.md`). +- **The ETE route needs coresight hardware this skill cannot conjure** — an ETE/TRBE-capable core, + a kernel with the drivers, and a perf built against OpenCSD; without them the route is closed, + not expensive (`ete-diff-tracing.md`). ## Not covered — say so rather than improvise diff --git a/skills/apk-reverse/references/ete-diff-tracing.md b/skills/apk-reverse/references/ete-diff-tracing.md new file mode 100644 index 0000000..fc043dc --- /dev/null +++ b/skills/apk-reverse/references/ete-diff-tracing.md @@ -0,0 +1,199 @@ +# ETE differential tracing — hardware execution streams, two runs, one answer + +Load this when the question is **where two runs of the same code diverge** — password +accepted versus rejected, license valid versus invalid, key A versus key B — and every +software tracer either modifies the code it traces (Stalker: the DBI cache), crashes under +the load (the measured `load average 34.11` case), or delivers nothing at all (the measured +zero-event follow in `references/native-dbi-and-deobfuscation.md` §6 Failure modes). An +ARMv8.4-era core's **ETE** streams executed-branch records off-core **in hardware**, and +**TRBE** collects them self-hosted; the target's own image, its timing profile and its +`TracerPid` are untouched. This file owns the gates, the capture and the differential +method; `scripts/ete_capture.py` runs the gates and the capture, and `scripts/trace_diff.py` +is the differ — which is producer-agnostic and works on Stalker logs too. + +**Claim strength.** The gates, the recipe and the capture wrapper run on the host — +`observed`. The differential reducer is host-tested on synthetic flattened-dispatcher +fixtures — `observed` for the tool, `inferred` for the method. **No ETE trace has been +captured or decoded by this repository**: no reference device here has coresight compiled in +(reference kernel 4.14.186 — the ETE gate is closed there, `observed`). The command shapes +follow the kernel's `tools/perf` CoreSight documentation and the OpenCSD project — treat +them as the documented upstream interface until your own control record decodes. + +## 1. The mechanism, in the size that matters + +| Piece | What it is | Why you care | +|---|---|---| +| **ETE** (Embedded Trace Extension) | the ARMv8.4 trace unit; emits executed-branch/instruction packets without instrumenting anything | nothing in the process image changes; page checksums stay clean | +| **TRBE** (Trace Buffer Extension) | per-CPU self-hosted trace buffer | per-thread tracing with no external debugger and no shared sink | +| `cs_etm` PMU | the Linux perf interface to ETE/ETM | `perf record -e cs_etm//u --per-thread` is the whole capture | +| OpenCSD | the decode library perf links against | a stock perf **records** an aux buffer it cannot **decode** — the classic silent failure | +| `scripts/trace_diff.py` | reduce + diff two decoded streams | the differ is the reusable half; it also eats Stalker `BLK` logs | + +The old `etm`-only world (no TRBE) still traces, but through a **shared** sink +(`tmc_etrN`) that cannot do `--per-thread`, is frequently reserved by the vendor, and +mixes CPUs — the differential method still applies but run design gets harder. Prefer +devices with `trbeN` in `/sys/bus/coresight/devices`. + +## 2. The five gates — probe before you design anything + +Every closed gate fails **silently** later: an old kernel reports no PMU; a kernel without +TRBE cannot per-thread; a perf without OpenCSD decodes nothing; SELinux can deny +`perf_event_open` even to root. One command each, then believe the table: + +``` +python skills/apk-reverse/scripts/ete_capture.py gates --device +``` + +| Gate | Expected | Closed means | +|---|---|---| +| kernel era | ≥ 5.10 | the ETE/TRBE drivers are not in your kernel at all | +| coresight devices | `ete` **and** `trbe` in sysfs | `ete` absent: no trace unit; `trbe` absent: shared sink only | +| `cs_etm` PMU | `/sys/bus/event_source/devices/cs_etm` exists | perf cannot speak to coresight | +| perf with CORESIGHT | `perf list` shows `cs_etm` | you can record but not decode | +| permission | a control record of `true` decodes | the pipe is closed at SELinux/paranoid level | + +A closed gate is a **route decision** — record it in one line per +`references/kernel-and-environment-hardening.md` §6 What the reference environment chose +(and why it generalises), and do not re-derive it mid-task. The reference environment's own +line: kernel 4.14.186 → ETE closed, permanently, on that device. + +## 3. Capture + +`ete_capture.py recipe` prints the full build path for a perf-with-CORESIGHT (on-device +Termux or cross). The capture itself, per-thread on TRBE: + +``` +python skills/apk-reverse/scripts/ete_capture.py capture --pid --seconds 10 --out a.data --control +``` + +`--control` is not decoration: it records a **one-instruction process** (`true`) first and +refuses to continue if that does not decode. A capture of the real target that "looks fine" +is worth nothing until the control has produced packets — this is the same discipline as the +Stalker zero-event lesson, paid for once already in this repository. + +Decode to an address stream (device or host, wherever your perf lives): + +``` +perf script -i a.data > a.log +python skills/apk-reverse/scripts/trace_diff.py reduce --log a.log +``` + +Volume rule from `native-dbi-and-deobfuscation.md` §3 transfers: one module, one bounded +window. ETE traces the *whole thread*, so capture around one operation (launch to first +frame, one login attempt, one encrypt call), not a session. + +## 4. The differential method + +**Design the pair before touching the device.** The method answers only the question the +pair was built to ask: + +1. **One changed bit of input, everything else identical.** Wrong password *character*, + not wrong password *length* (length changes parsing). Key `0x0000...` vs `0x8000...`. + License whose checksum byte differs. Two runs, same build, same screen state, same + window boundaries. +2. **Reduce both streams to module-relative keys.** Raw pointers move with ASLR; + `liba.so+0x9f6c0` is comparable across runs, `0x7b20f29f6c0` is not. + `trace_diff.py` keeps `mod+0xoff` keys verbatim for exactly this reason. +3. **Diff and read three things** — `trace_diff.py diff --a a.log --b b.log` prints them: + - **identical prefix span** — the shared machinery before your input bit reached + control flow; it names where parsing/handling ends and the decision begins; + - **first divergence** — the branch that first consumed your input difference. In a + flattened dispatcher this is where the state constant differs; + - **blocks unique to each side, ranked** — the input-dependent set. For "valid vs + invalid", the B-unique list *is* what the valid path does that the invalid one + never reaches. +4. **Interpret per obfuscation:** + - **OLLVM flattening**: dispatcher `D` matches on both sides (it is machinery); the + divergent `R` blocks between `D`s are the real blocks the two inputs select + between. Combined with the state-variable watchpoint in + `references/hardware-watchpoints.md` §4, the diff gives you the state constants to + complete the CFG recovery of `native-dbi-and-deobfuscation.md` §5 without a solver. + - **VMP / interpreter loops**: the handler loop matches (it is the interpreter); + unique blocks are guest-opcode handlers. The *sequence* difference between two + inputs is a private-opcode trace — feed it to the differential mapping machinery in + `references/vmp-differential-analysis.md` as ground truth instead of guessing. + - **Whitebox / key-derived logic**: the unique-to-each-side sets across several key + pairs approximate the key-dependent branch set; a branch that never appears in + either side of any pair is not key-dependent — stop reversing it. +5. **Repeat once with the inputs swapped** if the result decides a patch. A divergence that + moves when the inputs move is real; one that moves when it should not is capture noise. + +**What the tool does not enforce** (its header says so too): same build, same address space, +same window, one-bit difference. A diff across two builds is noise with a confident +rendering. + +## 5. What the obfuscator actually built — read from two open implementations + +The differential reads better when you know what the two families actually emit. The facts +below are read from the public AOSP-toolchain forks (dpkg123/ollvm-aosp-toolchain, branches +`ollvm-r614150-clang23-rebase+omp` — the Hikari family — and `ollvm-r614150-clang23+omp` — +the Pluto family; xvmp commits 63fec41f/9b7cf75e; fetched 2026-10): **source text observed, +runtime behaviour inferred from it**. + +| | Hikari-family flattening | Pluto-family flattening | +|---|---|---| +| state variable | `switchVar` **alloca in the frame** + a `switchVarAddr` double pointer; every state store goes through an indirect load of that pointer | `switchVar` + `switchXor` **two allocas, rolling XOR**: `enc^=delta; key^=delta; cond=enc^key` on every dispatch | +| case values | `scramble32` permuted (not 0,1,2…) | fully random per build; the true case exists only in the `switchCond` register for one instruction | +| what a memory watchpoint sees | the permuted case value — decodable against the static switch table | **noise**: the stored enc and key never contain the case | +| what a trace sees | identical in both: the block sequence — which is why the trace is the robust instrument | +| skipped functions | EH-bearing (landingpad) | EH, `callbr`, `indirectbr` | + +Consequences for this file's method: the **divergent-blocks reading (§4) is family-proof** — +it never decodes the state variable. The **watchpoint-on-state recipe** is family-sensitive: +Hikari-shaped targets can be read at the (frame-resident) slot via the pointer; Pluto-shaped +targets need a breakpoint one instruction after the `switchCond` XOR to read the register +(`references/hardware-watchpoints.md` §4 carries the corrected recipe). And the AntiDebugging +pass of the Hikari family injects **raw `svc` inline asm with randomized register-setup +order and a randomized svc immediate** (source-read): byte signatures cannot survive that +randomization, but ETE's branch stream and `svc_scan.py`'s semantic site-location do not +care — locate the check by what it *is*, never by its bytes. + +**xvmp (IR-level virtualizer, same repo)**: each virtualized function becomes a stub calling +`vm_exec(code, size, vm_args, num)` — bytecode bytes plus a boxed-argument array in the +module; 11 one-byte opcodes with inline-const/slot operands and 19 arithmetic sub-ops; +pointer constants materialize through a generated const/call dispatch at first use. For the +tracing routes this means: the interpreter is one dominant loop (the §4 handler reading), +the **guest inputs are observable at the `vm_args` array before interpretation starts** (a +watchpoint there sees arguments and any key material, plaintext, whatever the bytecode +carries), and each constant's bits pass through the materialization dispatch as a returned +i64 — a second, cheap observation point. `scripts/vm_handler_slice.py` consumes the trace +side and documents the handler->opcode map you can build against a *known* table like this +one. + +## 6. Failure modes + +- **`perf record` succeeds, `perf script` prints nothing address-like** — perf without + OpenCSD decode, or an aux buffer that never wrapped (window too short). The control run + separates the two in one command. +- **`gates --device` shows ete but no trbe** — shared-sink world: drop `--per-thread`, + trace `-C ` with `cs_etm/@tmc_etr0/u`, and pin the target to that CPU + (`taskset`-style, or accept the mixture). Expect much more post-filtering. +- **Aux overflow / gaps** — decode reports non-contiguous ranges: the window was too hot. + Shrink it, or accept that the diff sees only a sampled path and say so in the conclusion. +- **The two runs are not comparable** — different pids with different module bases and raw + keys: everything diverges from address zero. Re-run through `reduce`, check that the + top-of-histogram block is shared before diffing; if the top blocks differ, the pair is + not a pair. +- **ETE closes on the only device you have** — the honest answer, not a challenge; the + fallback ladder for the *question* is Stalker (measured boundaries apply) or the + watchpoint route (single addresses, no paths), never "trace harder". + +## 7. Hand-offs + +- Divergence found, now you need the *values* at that site → hardware watchpoint or + breakpoint (`references/hardware-watchpoints.md`), or QBDI per-instruction replay + (`references/native-dbi-and-deobfuscation.md` §4 Stalker, QBDI, or emulation). +- Divergence found, and you must *change* the decision with no code change → + `references/kernel-event-injection.md`. +- The path is known and you only need to call the routine → `references/emulation-and-rpc.md`. + +## Checklist + +- [ ] `gates --device`: all five closed, or the closed ones recorded as route decisions +- [ ] `--control` record of `true` decodes before any real capture +- [ ] Pair design written down: same build, same window, one changed input bit +- [ ] Both logs reduced; top-of-histogram block shared before diffing +- [ ] Divergences read as: prefix span / first divergence / unique-per-side +- [ ] Input-swap repeat when the diff decides a patch +- [ ] Labels: gates+reducer `observed` (host), ETE capture/decode `unverified` here — + the control run on your device is the only thing that upgrades it diff --git a/skills/apk-reverse/references/evidence-summary.md b/skills/apk-reverse/references/evidence-summary.md index d33f35d..88b09a1 100644 --- a/skills/apk-reverse/references/evidence-summary.md +++ b/skills/apk-reverse/references/evidence-summary.md @@ -55,6 +55,12 @@ next** — and points at the one section that says "the full record is elsewhere | Packer and custom-loader identification | Never exercised end to end; the target that was measured has no packer, so the tooling was recorded as not applicable | partial | unverified | `references/packers.md`, `references/code-virtualization-and-custom-linkers.md` | | Module delivery instead of a repack | The scaffold builds end to end, but the delivery route has no public-target measurement and needs a device with the framework active | partial | inferred | `references/lsposed-and-modules.md`, `scripts/lsposed_scaffold.py` | | Kernel-side syscall answer forging | Blocked on the kernel side: the generator is measured, no kernel artefact was compiled or loaded, and a userspace module cannot change a return value | blocked | unverified | `references/kernel-and-environment-hardening.md`, `scripts/kernelsu_syscall_mask.py` | +| Hardware memory watchpoints (ptrace) | The encoders and refusals are host-tested and the emitted helper compiles as a static aarch64 ELF; **no watchpoint was ever armed on a device**, and the route leaves `TracerPid` visible by design | partial | unverified | `references/hardware-watchpoints.md`, `scripts/hw_watch.py` | +| Differential trace reduction | First divergence, divergent regions and per-side unique blocks measured on synthetic flattened-dispatcher fixtures, in both Stalker and wrapped-log grammars; no real-target producer (Stalker pair or cs_etm decode) has fed it yet | partial | observed | `references/ete-diff-tracing.md`, `scripts/trace_diff.py` | +| ETE hardware tracing | Gates and capture wrapper host-measured to the no-device boundary; no coresight capture or decode ever ran here, and `--control` exists to say so on a real device in one command | blocked | unverified | `references/ete-diff-tracing.md`, `scripts/ete_capture.py` | +| SPE/BRBE sampling and crash forensics | The address→symbol loop is host-verified on a real cross-compiled `.so`; SPE/BRBE mechanisms are documented-and-gated, never captured here | partial | unverified | `references/hardware-sampling-and-crash.md`, `scripts/trace_addr2sym.py` | +| Interpreter-trace de-virtualization | Dispatcher slicing, signature clustering and the opcode trajectory measured on synthetic interpreter traces; the calibration facts come from a public IR-VM's source, and no hardened binary was traced | partial | observed | `references/ete-diff-tracing.md` §5, `scripts/vm_handler_slice.py` | +| Kernel-side instrumentation/injection | The scaffold's generate/verify is measured and every kernel file ships unbuilt with its gate; the register-rewrite action is explicitly experimental and never ran anywhere | blocked | unverified | `references/kernel-event-injection.md`, `scripts/kernel_instrument_scaffold.py` | | `svc` site scanning | Two independent decoders agreed on an identical 214-site set; a byte scan also matches data, so neighbour context decides | ok | observed | `references/kernel-and-environment-hardening.md`, `scripts/svc_scan.py` | | On-device tooling (MT Manager MCP) | The service-down path is measured; the connected path needs the service started by hand | partial | observed | `references/on-device-tooling.md`, `scripts/mt_mcp_probe.py` | | Client-side ads and server-issued UI config | Measured on a real mid-size target's ad chain, with a repacked build's on-screen change confirmed | ok | observed | `references/ad-removal.md`, `references/server-config-and-updates.md` | diff --git a/skills/apk-reverse/references/hardware-sampling-and-crash.md b/skills/apk-reverse/references/hardware-sampling-and-crash.md new file mode 100644 index 0000000..b38fead --- /dev/null +++ b/skills/apk-reverse/references/hardware-sampling-and-crash.md @@ -0,0 +1,140 @@ +# Hardware sampling and crash forensics — SPE data flow, BRBE branch records, and turning addresses into symbols + +Load this when the question is **data flow rather than control flow** ("which code touches this +buffer" without knowing the buffer's address yet — that is SPE's job, not a watchpoint's), when +**a crash's stack unwind cannot be trusted** (obfuscated code, frame-pointer omission, stacks +smashed by the fault itself — that is BRBE's job), or when you hold a list of runtime addresses +that must become **symbols and static offsets** before they mean anything (that is +`scripts/trace_addr2sym.py`'s job, and it closes the loop for every other tool here). + +**Claim strength.** The ELF walker behind `trace_addr2sym.py` is host-tested against a real +cross-compiled aarch64 shared library and a hand-built minimal ELF — `observed`. SPE/BRBE +mechanisms and their kernel interfaces follow the ARM ARM and the kernel/perf documentation — +`inferred`; **no SPE sample or BRBE record was ever captured on a device by this repository** +(no reference device carries either feature) — the producer side is `unverified` and every +route below starts with its gate probe. + +## 1. SPE — the watchpoint that does not need to know the address + +The Statistical Profiling Extension (FEAT_SPE, ARMv8.2+, present from Cortex-A75/A55-class +cores onward — far older hardware than ETE) samples instruction execution at a programmable +interval and records, per sample: the **PC**, the **data virtual address** the instruction +touched (for loads/stores), latency, and microarchitectural events (cache/TLB misses, branch +mispredict). Two properties make it a reverse-engineering instrument rather than a profiler: + +1. **Data address + PC in one record.** That answers "which code reads/writes key-shaped + buffers" *without a priori address knowledge* — the exact gap a watchpoint has (a + watchpoint is precise and cheap but must be told where to look; SPE is sampled and noisy + but asks nothing in advance). The workflow: capture SPE around the operation, group + samples by data address, find addresses whose accessors are concentrated in the hardened + `.so`, then graduate those addresses to a hardware watchpoint for precision. +2. **It runs whether or not anyone cooperates.** No symbols, no relocations, no hooks; the + aux buffer fills from user space with `perf record -e arm_spe_0// --per-thread` + (`arm_spe` PMU; kernel `CONFIG_ARM_SPE_PMU`; probe + `ls /sys/bus/event_source/devices/`). Decode with the same perf-with-OpenCSD-class build + as ETE (`scripts/ete_capture.py recipe` builds both), or import the capture into + Perfetto, which renders SPE records natively. + +Gates and limits: SPE is **sampling** — a buffer touched twice may appear zero times; treat +absence as weak evidence only (the same discipline as trace absence in +`references/native-dbi-and-deobfuscation.md` §7 Reporting what a trace proves). The sampling +interval trades coverage for overhead; and the data-address field is truncated/tagged on some +implementations — verify on one known access first. + +## 2. BRBE — the last N branches before the crash + +The Branch Record Buffer Extension (FEAT_BRBE, ARMv9.2+, Cortex-X3/A715-class cores and +newer) is hardware that **continuously records the last N taken branches** (from→to pairs, +32–64 records) into a buffer the kernel can read. It is the ARM equivalent of Intel's last +branch record, and it exists precisely for the question a crash asks: *what was the control +flow that led here?* — answered by hardware even when the stack is gone, the frame pointers +were optimized out, or the code was flattened so the unwinder wanders. + +Two ways to get at it, both gated: + +| Route | What | Gate | +|---|---|---| +| **perf branch stacks** | `perf record -b`/`-j any`; on arm64 kernels with BRBE support (6.1-era merges; `inferred`) the branch-stack samples are hardware-fed by BRBE | kernel with BRBE perf support; probe `perf record -j any -o /dev/null -- true` | +| **crash-time read** | your own tracer (ptrace stop at the fatal signal, or a kernel-side module) reads the BRBE records before the process is reaped | kernel exposure of the records (`unverified` — the registers are EL1-accessed; a KPM-side reader belongs to `references/kernel-event-injection.md`) | + +Use it when: the tombstone's backtrace is the thing you do not trust (`scripts/native_crash.py` +prints the stack split by owner — when the frames sit in wrong modules or anonymous regions, +that is the signal); the crash is inside flattened/virtualized code whose stack is decorative; +or the death is a deliberate fault (`references/native-tamper-and-suicide.md`) and the branch +records name the site that jumped to it. On hardware without BRBE, the honest fallback is the +ETE aux buffer: stop the process at the crash (never let it die), and read the tail of the +trace — same answer, more plumbing, gates in `references/ete-diff-tracing.md` §2 The five +gates. + +## 3. The crash loop: three sources, one order of asking + +1. **Tombstone/logcat first** — `scripts/native_crash.py`: signal, fault address, registers, + frames split app-vs-system, and the arranged-fault flag. Cheap, always available. +2. **If the stack is not trustworthy** — branch records (BRBE if the gate opens; ETE tail + otherwise). This is the *real* control flow, and it survives obfuscation by construction. +3. **Then symbolize everything** — every PC from 1 or 2 through + `scripts/trace_addr2sym.py`, with the maps pulled **in the same process lifetime**: + +``` +python skills/apk-reverse/scripts/trace_addr2sym.py --hits hwwatch_hits.log --maps maps.txt --elf liba.so +python skills/apk-reverse/scripts/trace_addr2sym.py --elf liba.so --addrs 0x7b20001234c0 --base 0x7b2000000000 +python skills/apk-reverse/scripts/trace_addr2sym.py --trace vm_trace.log --mod libvm.so --elf libvm.so +``` + +The offset-space rules that decide whether the number you paste into IDA is right (vaddr vs +file offset, `p_vaddr != p_offset` payloads) are stated once, in the tool's header — read them +before distrusting a symbol. + +## 4. What this adds to the deobfuscation routes + +- **OLLVM**: SPE finds the arithmetic that touches key material without a trace + (`references/native-dbi-and-deobfuscation.md` §1's table, data-flow half); BRBE/ETE-tail + recover the real path into a flattened function that crashes — the state machine's taken + states, read from hardware, no unwinder. +- **VMP/xvmp-class**: SPE answers "which handler sequence touches the argument slots" when + the bytecode is opaque (`vm_exec`'s boxed arguments live in a plain array — + `scripts/vm_handler_slice.py`'s header records where); a crash inside the interpreter + keeps its branch records, so the fatal guest opcode is identifiable from the last + handler segment. +- **Anti-debug/anti-hook inserts**: a deliberate `svc`-based check (the Hikari-family + AntiDebugging pass injects inline-assembly syscalls with randomized immediate fields on + Darwin targets, and links precompiled anti-debug IR elsewhere — source-read, `observed` + text / `inferred` behaviour) is invisible to byte-signature scans precisely because the + fields are random. Hardware-level observation does not care: ETE shows the branch into + and back out of the check; a breakpoint at the *semantic* site (the `svc` itself, + located by `scripts/svc_scan.py`) beats a signature by construction. + +## 5. Failure modes + +- **`arm_spe_0` absent from `/sys/bus/event_source/devices/`** — kernel without + `CONFIG_ARM_SPE_PMU` or a core without FEAT_SPE (pre-ARMv8.2): closed gate, one line, + move on; the watchpoint route still exists if you can name the address. +- **`perf record -j any` on arm64 returns samples with empty branch stacks** — the kernel + took the software path (no BRBE backend); the records are call-chain-ish, not + hardware branch pairs. Check `perf report`'s branch-stack content on a known-hot control + flow before trusting a negative. +- **SPE shows nothing at an address you know is touched** — sampling missed it. Lower the + interval (more samples, more overhead), or graduate to a watchpoint for that one address. +- **Symbolization disagrees with the disassembler** — the offset space was wrong, not the + tool: re-read the header's vaddr/file-offset rules, check `--offs-are-file`, and confirm + the module's load base from the same process lifetime. A runtime-materialized library + (lib_map flags those) has a base that never appears in any APK. + +## 6. Hand-offs + +- Full-path capture, divergence diffs → `references/ete-diff-tracing.md`. +- Single-address precision, zero code modification → `references/hardware-watchpoints.md`. +- Reading BRBE from a kernel module, crash-time record reads → + `references/kernel-event-injection.md`. +- Triage of the crash itself (signal, fault, arranged-fault) → + `references/native-tamper-and-suicide.md`, `scripts/native_crash.py`. + +## Checklist + +- [ ] Gate probed before design: `arm_spe_0` present? BRBE branch stacks real or software? +- [ ] SPE: one known access verified visible before absence is interpreted as anything +- [ ] Crash loop in order: tombstone → branch records → symbolize; maps from the same + process lifetime +- [ ] Offsets pasted into a disassembler cross-checked against the tool's offset-space rules +- [ ] Labels: addr2sym host-`observed`; SPE/BRBE producer `unverified` here — your gate + probe is the first real measurement diff --git a/skills/apk-reverse/references/hardware-watchpoints.md b/skills/apk-reverse/references/hardware-watchpoints.md new file mode 100644 index 0000000..ce2eec9 --- /dev/null +++ b/skills/apk-reverse/references/hardware-watchpoints.md @@ -0,0 +1,172 @@ +# Hardware watchpoints — observing memory without touching the code + +Load this when you need to know **who writes or reads a memory location**, and every +in-process tool you have modifies the thing it observes: an inline hook is caught by the +library's own page checksum, a Stalker follow rewrites the code it traces, and the target +dies before your hook exists. An arm64 hardware watchpoint compares every load/store +address **in silicon** and traps on match: no code byte changes, no thread appears, no +mapping is added. This file owns the mechanism, its honest stealth ledger, the traps, and +the procedure; `scripts/hw_watch.py` is the tool; `references/ete-diff-tracing.md` owns +full-path tracing, and `references/kernel-event-injection.md` owns the ptrace-free delivery. + +**Claim strength.** The WCR/BCR encoders and the emitted helper are host-tested, and the +helper **compiles clean as a static aarch64 ELF** (cross gcc 16.2, `-static`) — `observed` +on the host. **No watchpoint has been armed on a device by this repository's passes** — the +ptrace dance follows the kernel ABI (arm64 `NT_ARM_HW_WATCH`/`NT_ARM_HW_BREAK` regsets, +`TRAP_HWBKPT` siginfo) and is `unverified` until the control run below passes on your +device. Slots, capacities and per-thread semantics are ARM DDI 0487 / kernel +`arch/arm64/kernel/hw_breakpoint.c` facts — `inferred` from documented interfaces. + +## 1. The stealth ledger — read it before choosing this route + +| delivery | code modified | foreign thread/mapping | `TracerPid` | survives page checksums | per-hit cost | +|---|---|---|---|---|---| +| inline hook (Frida `Interceptor`) | yes | no | no | **no — caught by construction** | ~none | +| Stalker follow | yes (DBI copy) | yes (agent + code cache) | no | no | large constant | +| **ptrace hardware watchpoint** (this file) | **no** | **no** | **yes — visible** | **yes** | near-zero | +| kernel-armed hardware watchpoint (`kernel-event-injection.md`) | no | no | **no** | yes | near-zero | + +Two conclusions follow, and both are instructions: a watchpoint is the answer to +*self-checksumming* code (there is nothing in the image to find), and it is **not** an +answer to a target that reads `TracerPid` over a raw `svc` — for that, the kernel route +exists, with its own gate. Never present the ptrace route as invisible; it is invisible to +the process image, not to `/proc`. + +## 2. The mechanism, in the size that matters + +Each arm64 core carries a small bank of debug comparators — typically **6 breakpoints +(instruction addresses) and 4 watchpoints (data addresses)**; the kernel reports the true +counts and `scripts/hw_watch.py plan`'s device-side counterpart reads them +(`dbg_info & 0xff` from the `NT_ARM_HW_WATCH` regset). Three hardware rules decide what is +expressible: + +1. **A watchpoint covers at most 8 bytes, inside one aligned 8-byte window** (byte-select + field `BAS`). A watch at `0x...4` with length 8 crosses the window and needs two slots. + `hw_watch.py plan` refuses it locally instead of letting the device. +2. **Execution breakpoints need a 4-byte-aligned PC** (A64 instruction width), and matching + uses the whole address. +3. **Slots are per-thread.** A process with N threads needs the watchpoint armed N times, + and a thread created later runs unarmed until your tracer catches its first stop — which + is why the emitted helper seizes with `PTRACE_O_TRACECLONE` and arms every new thread at + birth. + +The two canonical encodings (unit-tested; everything else is a variation): + +``` +watch 8 bytes, load+store, EL0 : WCR = 0x1FFD (BAS=0xff, LSC=both, PAC=EL0, E=1) +break, address match, EL0 : BCR = 0x3F05 (BAS all-ones, PMC=EL0, BT=unlinked, E=1) +``` + +Delivery is the standard ptrace regset ABI — `PTRACE_SEIZE` + `PTRACE_INTERRUPT` every +thread, `PTRACE_SETREGSET` with `NT_ARM_HW_WATCH`/`NT_ARM_HW_BREAK`, `PTRACE_CONT`, then +`waitpid` for a `SIGTRAP` whose `si_code == TRAP_HWBKPT (4)` and whose `si_addr` is the +data address that matched. On hit you read `NT_PRSTATUS` for the register file. To *continue* +from an execution breakpoint you must disable the slot, `PTRACE_SINGLESTEP`, re-enable — +without that dance the trap re-fires on the same PC forever. + +## 3. Procedure + +**1. Plan on the host** — validate alignment and see the exact encodings: + +``` +python skills/apk-reverse/scripts/hw_watch.py plan --addr 0x7b2000123456 --len 8 --access w +python skills/apk-reverse/scripts/hw_watch.py plan --break 0x7b20001234c0 +``` + +**2. Emit the helper** (self-contained C; NDK clang on the host, or Termux clang on the +device — build commands are in the generated README): + +``` +python skills/apk-reverse/scripts/hw_watch.py emit --out work/hwwatch +``` + +**3. Gates, then the control run — a watchpoint that has never fired is not armed.** Pick a +global you can touch on demand (a flag the app exposes, or a location you write through +`/proc//mem` from the host as root) and confirm one `HIT` arrives with a sensible PC +**before** you spend the window on the real address. The measured Stalker lesson transfers +exactly: zero events from a correctly-installed instrument is a real failure mode +(`references/native-dbi-and-deobfuscation.md` §6 Failure modes), and the only defence is a +producer you have already seen fire. + +``` +python skills/apk-reverse/scripts/hw_watch.py run --pkg --addr 0x... --len 8 --access rw --seconds 20 +python skills/apk-reverse/scripts/hw_watch.py decode --hits hwwatch_hits.log --maps maps.txt +``` + +`decode` turns each hit's PC into `module+offset` given a `/proc//maps` file (pull it +in the same process lifetime — bases move with ASLR) or a `lib_map.py` JSON. + +**4. Read the hits the way a trace taught you to**: a hit's PC sequence is caller evidence; +run it through `scripts/trace_diff.py reduce` if you collected several, and correlate with +`lib_map.py` for which library actually owns the address. + +## 4. Where a watchpoint beats a hook — the recipes + +- **The key that never appears in a readable form.** OLLVM-substituted and VM-wrapped crypto + still materialises plaintext in a buffer before use. Watch the buffer you identified + statically (the JNI argument location, the `memcpy` destination) with `--access w`: the + PC that fills it is the decryptor, obfuscation irrelevant — control flow does not matter + to a comparator. +- **The state variable of a flattened dispatcher.** Family-sensitive, and the difference is + real (read from two open implementations — Hikari vs Pluto flattening — in + `references/ete-diff-tracing.md` §5 What the obfuscator actually built — read from two + open implementations): in the Hikari shape the state is a frame-resident slot written + through an indirect pointer — watch the slot and each `HIT` PC is a real block's tail, + with the written (permuted) state value readable afterwards. In the Pluto shape the state + is stored **rolling-XOR encoded**: memory only ever holds noise, so the recipe is a + breakpoint one instruction after the dispatch decode (`switchCond = enc ^ key`) and the + condition register is read at the trap, not from memory. If you cannot tell which family + you have, the block-sequence routes (ETE diff, `scripts/vm_handler_slice.py`) never need + the decoded value at all. That decoding is half of the CFG recovery in + `references/native-dbi-and-deobfuscation.md` §5. +- **The self-modifying / self-decrypting region.** Watch the code page itself with + `--access x`-adjacent breakpoints on entry and `--access w` on the page: the PC that + writes executable memory is the unpacker, whatever the section headers claim. +- **"Does this check even read my patch?"** Watch the patched location: zero reads during a + run that enforces the old behaviour is direct evidence the check reads something else — + cheaper than a full trace and answerable in one window. + +A stack slot cannot be watched this way if the frame moves between runs; watch the *caller's* +stable global or the buffer the function receives instead. The 8-byte window limit means +large structs need either the one field that matters or several slots. + +## 5. Failure modes and traps + +- **`READY ... watch_slots=0`** — the kernel masked hardware debug (some vendor kernels + reserve the slots for their own telemetry, or the SoC reports none). That is a closed + gate; record it and move to `references/kernel-event-injection.md` or the ETE route. +- **Only the arming thread traps.** The watchpoint is per-thread: if you armed one tid and + the work happens on another, you get silence, not an error. The helper arms every thread, + but check the `ARM tid=` lines against `ls /proc//task` before trusting a quiet run. +- **A second tracer wins or you break one.** Only one tracer is allowed; if frida is + attached, `PTRACE_SEIZE` fails with `EPERM`, and vice versa. Detach everything else first. +- **Swallowed signals.** A naive loop that resumes with `PTRACE_CONT(tid, 0, 0)` drops the + target's own signals (its crash handlers, its SIGALRMs) and the app misbehaves in ways you + will blame on the target. The helper forwards non-debug `SIGTRAP`s and other signals, and + uses `PTRACE_LISTEN` for group stops; if you write your own loop, this is the part to get + right, not the encodings. +- **Root, and only root.** `ptrace` of an app process from the shell needs `su`; the helper + checks and says so. +- **Detach cleanly or leave the target traced forever**: slots cleared, every tid detached. + The helper does this on every exit path including the caps; a `kill -9` of the helper + leaves `PTRACE_O_EXITKILL` to do the damage — do not do that. + +## 6. Hand-offs + +- Full execution paths, not single addresses → `references/ete-diff-tracing.md`. +- `TracerPid` must stay 0 → `references/kernel-event-injection.md` (same hardware, armed + from the kernel, no tracer relationship). +- You only need to *call* the routine you located → `references/emulation-and-rpc.md`. +- The watchpoint found the site and you must neutralise a terminate path → the rules in + `references/native-tamper-and-suicide.md` apply unchanged. + +## Checklist + +- [ ] `plan` accepted the address locally (no window crossing, PC aligned) +- [ ] `emit` + build succeeded; helper runs and prints `READY` with slots > 0 +- [ ] **Control run passed**: one deliberate touch produced one `HIT` with a sane PC +- [ ] maps pulled from the same process lifetime; `decode` resolves module+offset +- [ ] Every `ARM tid=` line accounted for against `/proc//task` +- [ ] No other tracer (frida, lldb) attached during the window +- [ ] Strength labels on every conclusion: encoders `observed` (host), device behaviour + `unverified` until your control run, kernel-armed route `unverified` until compiled diff --git a/skills/apk-reverse/references/kernel-and-environment-hardening.md b/skills/apk-reverse/references/kernel-and-environment-hardening.md index 9265ef6..d87fc2b 100644 --- a/skills/apk-reverse/references/kernel-and-environment-hardening.md +++ b/skills/apk-reverse/references/kernel-and-environment-hardening.md @@ -17,7 +17,10 @@ Division of labour, so the two files never duplicate: **This repository does not do kernel development.** Everything in the kernel section is a map of externally documented mechanisms with sources, labelled `inferred` — none of it was executed here. That is deliberate: the purpose is to stop you from spending a day rediscovering the version -gate () or the conflict (), not to teach kernel hacking. +gate () or the conflict (), not to teach kernel hacking. When the goal is not *hiding from a +detector* but *instrumenting or altering the target from the kernel* — invisible watchpoints, +register rewrites at a trap — that is a different activity with its own primitive map and recipes: +`references/kernel-event-injection.md`. The two files share the §4a gate table and the stop rules. **Extension pass, read the next paragraph before relying on it.** This file now has a companion generator — `scripts/kernelsu_syscall_mask.py` — which emits a loadable userspace module skeleton diff --git a/skills/apk-reverse/references/kernel-event-injection.md b/skills/apk-reverse/references/kernel-event-injection.md new file mode 100644 index 0000000..4c48eb7 --- /dev/null +++ b/skills/apk-reverse/references/kernel-event-injection.md @@ -0,0 +1,127 @@ +# Kernel-side event injection — the driver as your hands inside the target + +Load this when the observation or the change you need is **impossible to deliver from the +target's own process** — `TracerPid` must stay 0, the code must not change by one byte, the +decision must flip at a PC you choose, and it must all happen without any thread or mapping +the target could enumerate. A kernel-side driver can do this because the arm64 debug +registers can be programmed **on the target's behalf** through the kernel's +perf/breakpoint layer, and the overflow callback runs holding the interrupted context's +`pt_regs`. This file owns the primitive map and the composition recipes; +`scripts/kernel_instrument_scaffold.py` generates the templates. + +**Division of labour, so nothing gets said twice.** `references/kernel-and-environment-hardening.md` +owns *hiding from detection* — which root scheme, which module, when escalation is wrong; +its §4a gate table applies to this file unchanged. `references/hardware-watchpoints.md` +owns the *ptrace* delivery of the same hardware, including its honest `TracerPid` column. +This file is the third leg: **instrumentation and injection from the kernel**, when the +other two deliveries are unacceptable. + +**Claim strength.** `gates`/`generate`/`verify` run on the host — `observed` for the +generator. **No kernel-side file this scaffold emits has been compiled or loaded anywhere, +by anyone in this repository's passes** — every generated file says so in its own header, +and the register-rewrite action is explicitly labelled experimental. The API surface +(`register_user_hw_breakpoint` / `perf_event_create_kernel_counter`, +`PERF_TYPE_BREAKPOINT` with `bp_type`/`bp_addr`/`bp_len`) is the kernel's documented +GPL-exported interface — +`inferred` as a mechanism, `unverified` as a route until a device with the matching gate +compiles and loads it. + +## 1. The primitive map — what each route can actually do + +| primitive | KPM (KernelPatch) | out-of-tree LKM | eBPF | Zygisk module | +|---|---|---|---|---| +| hardware watchpoint on the target task, hit logged to dmesg | yes (via perf bp layer, symbols resolved at load) | yes (`register_user_hw_breakpoint`, GPL-exported) | **no** — no perf breakpoint creation from BPF | no | +| rewrite a register at the trap (flip a branch decision, no code change) | template action, **experimental** | template action, **experimental** | no | no | +| read/write the target's memory from outside it | yes | yes (`access_process_vm`) | read via helpers, write rarely | in-process only (it *is* the process) | +| force a syscall return value | yes — see `kernelsu_syscall_mask.py` | yes (syscall table / inline hooks) | observe; rewrite only for `ALLOW_ERROR_INJECTION` targets | no (wrong layer) | +| deliver a signal at a chosen moment | yes | yes | yes | in-process | +| visible to the target (`TracerPid`, foreign threads, code deltas) | **none** | **none** | **none** | in-process, pre-init_array | + +Read the table as instructions: the **only** route that observes *and* alters with zero +in-process footprint is a kernel module; eBPF is an observer; Zygisk is early *inside* the +process — the right answer for init_array-early checks +(`references/kernel-and-environment-hardening.md` §4 Kernel-level interception — the map, +and the version gate), not for invisible watchpoints. + +## 2. The recipes — "wish-mode", assembled from the other two files + +These are compositions, not new mechanisms. Each names its deliverable and its fallback, +because a kernel route is never the first thing to try — it is what you escalate to when +the cheaper route's ledger (`references/hardware-watchpoints.md` §1 The stealth ledger) +says the target will see you. + +**Recipe 1 — "give me the plaintext key, whatever wraps the cipher."** +Watch the buffer the crypto fills, kernel-armed so the RASP never notices, and read the +buffer at the hit. Steps: locate the buffer statically (`svc_scan.py`, decompiler) → +`kernel_instrument_scaffold.py generate --pid --watch w::8:w` → load on a +device with the KPM/LKM gate closed → `dmesg` gives the filling PC → at the next hit read +the buffer (`/proc//mem` from root outside the process is enough; no injection needed). +*Fallback:* ptrace watchpoint when `TracerPid` is not read by the target — cheaper, same hit. + +**Recipe 2 — "make the check pass without changing one byte."** +The differential (`references/ete-diff-tracing.md` §4) names the branch that first consumes +the valid/invalid difference. Instead of patching that branch (which the page checksum +watches), arm an execution breakpoint at its PC from the kernel and rewrite the condition +register at the trap: the decision flips, the image stays byte-identical. +`generate --break b: --set-reg x0=0x1`. *This is the experimental action* — the +register rewrite in a perf overflow callback was never run by this repository; validate on +a device you can reboot, and prefer patching the code when the checksum threat model does +not actually include self-verification of that page. + +**Recipe 3 — "trace it, but leave TracerPid alone."** +ETE capture needs no ptrace at all (it is perf + aux buffers), and kernel watchpoints need +none either — so the entire observation half of this toolkit composes into a zero-ptrace +session: `ete_capture.py capture` for paths, kernel watchpoints for addresses, dmesg as the +event bus. What remains visible is the perf events themselves (a determined kernel-state +reader can enumerate them) — say so rather than claiming true invisibility. + +## 3. The scaffold and its gates + +``` +python skills/apk-reverse/scripts/kernel_instrument_scaffold.py gates +python skills/apk-reverse/scripts/kernel_instrument_scaffold.py generate \ + --out work/kwatch --pid --watch w:0x7b2000123450:8:w --break b:0x7b20001234c0 --set-reg x0=0x1 +python skills/apk-reverse/scripts/kernel_instrument_scaffold.py verify --dir work/kwatch +``` + +The gates are the §4a table's: **KPM** needs a KernelPatch-patched boot image plus +`aarch64-none-elf-gcc`; **LKM** needs kernel source matching the exact vermagic; the +**userspace module** is an honest carrier that can arm nothing. One gate is new and belongs +to this file: the kernel's **perf breakpoint layer must own the debug registers** +(`CONFIG_HW_PERF_EVENTS`); if it is compiled out, `register_user_hw_breakpoint` does +not exist and both templates close — check +`ls /sys/bus/event_source/devices/breakpoint` on the +device. Design notes carried in the generated files, all load-bearing: the callback runs +on **every hit, system-wide** — the generated template prints **unfiltered** on entry, so +adding a cheap first-statement filter is a required change before you rely on any hit +(the same lesson the syscall-mask template learned publicly); symbols in a KPM are +declared without `extern` or the loader rejects the module; and a slot that fails to arm +is logged, never silently skipped. + +## 4. When to stop — the same stop rules, extended one row + +Every rule of `references/kernel-and-environment-hardening.md` §7 When to stop escalating — +do not counter-escalate a userspace check with kernel development +applies verbatim, plus this file's own: + +| Situation | Do this | Not this | +|---|---|---| +| The check sees `TracerPid` but not perf/kernel state | kernel watchpoint + ETE, zero ptrace | patching the boot image to hide one field | +| The decision site is known and the page is not self-verified | byte patch (`references/native-and-so.md`) | kernel register rewrite | +| You want a kernel route on a 4.14/Magisk device | record the closed gate, use ptrace watchpoint or Zygisk | "just build a quick module" | +| The register rewrite did not take effect | report it — experimental means the first run is a measurement | a second, larger rewrite from the same unverified template | + +## Checklist + +- [ ] Cheaper route's ledger consulted first (hardware-watchpoints §1); kernel chosen + because the target's detection actually demands it, not because it is stronger +- [ ] `gates` run; KPM/LKM/perf-bp gates answered for **this** device, closed ones recorded +- [ ] Watch specs passed `plan`-style validation at generation (window, alignment) +- [ ] Control: one deliberate touch on a watched global produces one dmesg line before the + real window — an unverified template owes this before any hit is trusted +- [ ] Callback filter first, no sleeping, `printk` only (plus the explicitly-labelled + experimental register action, only on a device you can reboot) +- [ ] Every kernel file still carries its `NOT COMPILED AND NOT LOADED` banner after your + edits (`verify --dir`), or the banner is replaced by your own measurement +- [ ] Deliverable sentence re-read: a kernel-route result is a **rooted-device result**, + labelled a fallback unless the task asked for exactly that diff --git a/skills/apk-reverse/references/native-dbi-and-deobfuscation.md b/skills/apk-reverse/references/native-dbi-and-deobfuscation.md index a1219a4..f5f2442 100644 --- a/skills/apk-reverse/references/native-dbi-and-deobfuscation.md +++ b/skills/apk-reverse/references/native-dbi-and-deobfuscation.md @@ -53,6 +53,7 @@ script header in `scripts/stalker_trace.js`. | Question | Observation | Tool | |---|---|---| | Which blocks executed, and how often? | Block-level trace with execution counts | `scripts/stalker_trace.js` + `scripts/stalker_report.py` | +| Where did two runs of the same operation diverge? | Differential reduction of two block streams — the input-dependent set | `scripts/trace_diff.py` (works on Stalker logs and decoded cs_etm alike; the hardware producer and its gates are `references/ete-diff-tracing.md`) | | Which functions inside the module call each other? | Call edges from the same trace | same pair (`call` events) | | Which imported function does this stub reach? | Relocation table, not trace | `scripts/elf_plt.py` | | Where did the process die, and did it look arranged? | Crash record, signal, fault address, backtrace split by owner | `scripts/native_crash.py` | @@ -221,7 +222,10 @@ zero-event trace survived the treatment arm unchanged. Treat those as two proble **arm64 also raises the floor.** PAC/BTI-bearing code gives a translator more ways to mis-handle a block than armv7 did, which is one more reason a trace that works on an emulator is not evidence about a device. Treat any arm64 result as needing a control run (a followed thread executing a known -loop, nonzero `BLK` lines) before it is interpreted. +loop, nonzero `BLK` lines) before it is interpreted. When the device itself carries ETE/TRBE +(`references/ete-diff-tracing.md` — probe with `scripts/ete_capture.py gates --device`), the +hardware trace replaces Stalker for this question with no code modification and no DBI cache; keep +this file's Stalker route for the devices without it. **Emulation is a legitimate alternative, not a consolation prize.** When the target is a pure computation inside a `.so` and its environment can be faked, a Unidbg/Unicorn trace can be both diff --git a/skills/apk-reverse/references/routing.md b/skills/apk-reverse/references/routing.md index 9fb017e..e17dbdb 100644 --- a/skills/apk-reverse/references/routing.md +++ b/skills/apk-reverse/references/routing.md @@ -67,6 +67,12 @@ reference file is not named here, or when a script is not named here. | You only need to **call** the target's own routine (sign, token, encrypt) rather than change the app | `references/emulation-and-rpc.md` — emulate it, or service-ify the live function over Frida RPC | | A native function is a many-thousand-line `switch` state machine, or the decompiler's output is meaningless | `references/native-dbi-and-deobfuscation.md` — OLLVM shapes, a Stalker trace, and how far a trace actually gets you | | `Stalker.follow` installs but no events arrive, or following a hot libc export crashes the process | `references/native-dbi-and-deobfuscation.md` §6 failure modes — this repository measured both | +| You must observe or alter the target with **no code change, no injected thread, and `TracerPid` = 0** — page checksums catch every hook | `hardware-watchpoints.md` — hardware debug slots via ptrace with its honest stealth ledger, then `kernel-event-injection.md` for the kernel-armed route that needs no tracer at all | +| A crash's backtrace cannot be trusted — frames in wrong modules or anonymous regions, flattened/virtualized code, or a smashed stack | `hardware-sampling-and-crash.md` — BRBE branch records / ETE tail as the real control flow, then `scripts/trace_addr2sym.py` to symbolize every PC | +| You hold a list of **runtime addresses** (hits, trace PCs, divergence points) and need symbols / static offsets, not hex | `hardware-sampling-and-crash.md` §3 The crash loop — `scripts/trace_addr2sym.py` with its vaddr/file-offset rules | +| A trace of a virtualized function is just an **interpreter loop repeating** and reads like noise | `scripts/vm_handler_slice.py` — slice at dispatcher entries into handler segments, cluster signatures, recover the guest-opcode trajectory (xvmp-calibrated) | +| You need to know **where two runs of the same operation diverge** (password/license/key), not just what executed | `ete-diff-tracing.md` — ETE hardware-trace gates plus the differential method; `scripts/trace_diff.py` also diffs Stalker logs | +| A generated KPM/LKM template is in front of you, or the kernel route feels like the plan | `kernel-event-injection.md` — the primitive map, the composition recipes, and the gates every kernel-side file must pass before it is a weapon | | The traffic is protobuf/gRPC/QUIC, or a proxy sees TLS but requests still fail on a Flutter app | `references/protocol-reverse.md` — schema-less protobuf, frame capture, and native-side pinning | | Userspace hooks land and the app still dies: the check reads `/proc/self/status` through a raw `svc`, or runs before `JNI_OnLoad` | `references/kernel-and-environment-hardening.md` — what the next layer up and down can actually do, and when to stop | | You must edit, repack, sign or inspect the APK **from the phone itself** | `references/on-device-tooling.md`, `scripts/mt_mcp_probe.py` | @@ -124,6 +130,10 @@ _54 row(s) below the header._ | `references/lsposed-and-modules.md` | The client-side logic is reachable but **a rebuilt APK is refused**: delivering a system-level hook module instead, its gradle-free build chain, scope configuration and verification, and the layer a Java module cannot reach | | `references/emulation-and-rpc.md` | You need to **call** a routine rather than change the app — a signing routine, a token, a cipher: emulated execution (Unidbg/Unicorn) with its environment-filling cost, versus service-ifying the live function over Frida RPC | | `references/native-dbi-and-deobfuscation.md` | A native function is an OLLVM state machine, or you need instruction-level execution evidence: Stalker traces, the trace-to-CFG route, the Stalker/QBDI/emulation decision, and the **measured** zero-event and crash boundaries | +| `references/hardware-watchpoints.md` | You must observe who reads/writes one memory location with **no code modification** — hardware watchpoints/breakpoints via the arm64 debug registers (ptrace route, its honest `TracerPid` ledger, the alignment rules, the control run you owe any hit) | +| `references/hardware-sampling-and-crash.md` | The question is **data flow without a known address** (SPE: PC + data address per sample), a **crash whose stack is not trustworthy** (BRBE/ETE-tail branch records), or a list of runtime addresses to symbolize (`trace_addr2sym.py`, PIE vaddr/file-offset rules) | +| `references/ete-diff-tracing.md` | You need **where two runs diverge** (password/license/key) with hardware tracing: the ETE/TRBE gates, per-thread `cs_etm` capture, and the differential method that turns two traces into the input-dependent block set | +| `references/kernel-event-injection.md` | The observation or change must come from a **driver** — invisible watchpoints, register rewrites at a trap, cross-memory writes, zero ptrace: the primitive map, the composition recipes, and the gates the scaffold's templates must pass | | `references/protocol-reverse.md` | The traffic is protobuf without a schema, gRPC, or QUIC/HTTP3; or a proxy sees TLS while the app still fails — schema recovery, frame capture, and native-side certificate pinning (Flutter/BoringSSL) with its boundaries | | `references/kernel-and-environment-hardening.md` | Userspace hooking provably cannot reach the check — raw `svc` syscalls, `init_array`-early detection: what each layer can still do, the kernel-route map with its version gate, and when escalating is wrong | | `references/on-device-tooling.md` | Working **from the phone itself**: MT Manager edit/repack/sign and its built-in APK MCP, LSPosed Manager, Termux+frida, and on-device data inspection | @@ -185,6 +195,12 @@ _43 row(s) below the header._ | `scripts/protobuf_decode_raw.py` | **Schema-free protobuf decode**: hex / file / stdin to a JSON tree, every length-delimited field as a candidate set with ties labelled; `--reencode --check` for a byte-exact round trip | | `scripts/vmp_diff_harness.py` | Differential hardening for Dex-VMP: build a labelled opcode-coverage fixture (**218 of 224 opcodes, measured**), derive a candidate private-opcode map from an original/hardened dex pair, verify it in a closed loop, render smali | | `scripts/kernelsu_syscall_mask.py` | Generate a KernelSU/APatch syscall-masking scaffold: an installable userspace module plus KPM/LKM/eBPF kernel-side templates, each with its version gate and an explicit unverified label. **A userspace module cannot change a syscall return value** | +| `scripts/hw_watch.py` | **Invisible memory watchpoints / execution breakpoints** on arm64: plan (WCR/BCR encoders with the 8-byte-window rules, host-tested), emit (self-contained on-device C helper, compiles clean as static aarch64), gates, run (build+push+stream hits), decode (module+offset per hit). ptrace route: `TracerPid` stays visible — the ledger is in the reference | +| `scripts/trace_diff.py` | **Differential trace reduction**: reduce one trace to runs+histogram, or diff two traces into first-divergence / divergent regions / blocks-unique-to-each-side. Producer-agnostic (Stalker `BLK` logs or any address-per-line stream, incl. decoded cs_etm) | +| `scripts/ete_capture.py` | **CoreSight ETE capture with gates first**: the five gates (kernel era, coresight sysfs, cs_etm PMU, perf-with-CORESIGHT, permission), the perf/OpenCSD build recipe, and the per-thread `cs_etm//u` record wrapper with the `--control` validation run | +| `scripts/kernel_instrument_scaffold.py` | Generate the **kernel-side instrumentation scaffold**: KPM/LKM watchpoint+register-rewrite templates plus an honest userspace carrier, each with its gate and a `NOT COMPILED AND NOT LOADED` banner that `verify` enforces | +| `scripts/trace_addr2sym.py` | **Runtime addresses -> symbols + static offsets**, zero-dependency ELF64: PT_LOAD vaddr/file-off conversion, `.dynsym`/`.symtab`, stripped fallback; accepts hw_watch hits, stalker/trace `mod+off` keys, or raw address lists -- host-verified against a real cross-compiled aarch64 `.so` | +| `scripts/vm_handler_slice.py` | **De-virtualize an interpreter trace**: dispatcher detection (top + predecessor diversity), segment slicing at dispatch entries, signature clustering, and the guest-**opcode trajectory** via an optional handler->opcode map -- calibrated against the open xvmp IR virtualizer (`vm_exec`, slot machine, 11 opcodes) | | `scripts/capabilities.py` | The capability registry `doctor.py` reads: per capability the requires-closure, what is present, what is missing, an install hint and an estimated cost — so "can this machine do X" is computed rather than guessed | | `scripts/device_shell.py` | Shared device-side command construction: POSIX single-quote escaping for `su -c`, argv arrays rather than string concatenation, and Android identifier validation (`--pkg`, components) so a malformed value is rejected here instead of being parsed by the device shell | | `scripts/rasc_build.py` | Build and verify **rasc**, the Rust re-implementation of ASC: `--check` what is present, `--build` clone + cargo (no release asset exists), `--verify ` compares its class-definition set against the Python `droidasc` and fails on any difference | diff --git a/skills/apk-reverse/scripts/ete_capture.py b/skills/apk-reverse/scripts/ete_capture.py new file mode 100644 index 0000000..061d3ba --- /dev/null +++ b/skills/apk-reverse/scripts/ete_capture.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""ete_capture.py -- arm64 hardware execution tracing (CoreSight ETE) capture, with gates first. + +What this is +------------ +Modern arm64 cores (ARMv8.4+ era: Cortex-A78/X1/A710/X2 and later, all ARMv9 big +cores) carry **ETE**, a trace unit that streams executed-branch/instruction +records off-core in hardware, and **TRBE**, a per-CPU buffer that collects them +*self-hosted* -- no debugger, no JTAG, no code modification, no DBI code cache. +The Linux perf subsystem exposes it as the `cs_etm` PMU: a per-thread trace +recorded with `perf record -e cs_etm//u --per-thread`, decoded on the host or +device by perf built against OpenCSD. `references/ete-diff-tracing.md` owns the +method (and the pairing with `scripts/trace_diff.py`); this script owns the +gates, the recipe and the capture. + +Why the gates come first +------------------------ +The ETE route is closed on most devices people actually own, and every closed +gate fails *silently* if you skip it: an old kernel reports no cs_etm PMU; a +kernel with coresight but no TRBE cannot do per-thread buffers; a perf without +OpenCSD records an aux area it cannot decode; SELinux can deny perf_event_open +even to root. Probing costs one command per gate. Guessing costs an afternoon. + + gate expected check + kernel era >= 5.10 for ETE/TRBE drivers uname -r + coresight devices eteN / trbeN in sysfs ls /sys/bus/coresight/devices + cs_etm PMU dir exists ls /sys/bus/event_source/devices/cs_etm + perf with CORESIGHT perf lists cs_etm perf list | grep cs_etm + permission a control record decodes perf record -e cs_etm//u --per-thread -- true + +Measured vs unverified +---------------------- +`gates` and `recipe` run on this host (gates probes adb and prints the +device-side plan; `capture` requires a device and exits 3 without one -- +contract-tested in tests/cli). **No ETE trace has been captured or decoded by +this repository's passes**: no reference device here has coresight compiled in +(reference kernel 4.14.186). The command shapes follow the kernel's own +tools/perf CoreSight documentation and the OpenCSD project docs -- treat them +as the documented upstream interface, and validate with the control record +before trusting a real capture. + +Examples + python ete_capture.py gates + python ete_capture.py gates --device # also runs the device-side probes + python ete_capture.py recipe + python ete_capture.py capture --pid 12345 --seconds 10 --out perf.data + python ete_capture.py capture --pid 12345 --control # validation run first +""" +import argparse +import json +import re +import shlex +import shutil +import subprocess +import sys + +GATES = [ + {"gate": "kernel era", "expect": ">= 5.10 (ETE/TRBE driver support in-tree)", + "check": "uname -r", "why": "the ETE source and TRBE sink drivers landed with the " + "5.10-era coresight updates; older kernels expose no ETE"}, + {"gate": "coresight devices", "expect": "ete and trbe under /sys/bus/coresight/devices", + "check": "ls /sys/bus/coresight/devices", "why": "eteN = the trace unit, trbeN = the " + "self-hosted buffer; etmN/tmc_etrN without " + "trbeN means shared-sink tracing only"}, + {"gate": "cs_etm PMU", "expect": "/sys/bus/event_source/devices/cs_etm exists", + "check": "ls /sys/bus/event_source/devices/cs_etm", "why": "perf speaks to coresight " + "through this PMU"}, + {"gate": "perf with CORESIGHT", "expect": "perf lists cs_etm, and `perf report -D` decodes " + "packets (OpenCSD linked in)", + "check": "perf list | grep cs_etm", "why": "a stock perf records an aux buffer it cannot " + "decode; decode needs perf built with CORESIGHT=1"}, + {"gate": "permission", "expect": "a control record of `true` produces decodable output", + "check": "perf record -e cs_etm//u --per-thread -o ctl.data -- true", + "why": "SELinux can deny perf_event_open even to root; prove the whole pipe on a " + "one-instruction process before the target"}, +] + +RECIPE = """Building perf with CoreSight decode (pick one): + +A. On-device (Termux, no cross toolchain): + pkg install clang make cmake git python + git clone --depth 1 https://github.com/Linaro/OpenCSD.git && cd OpenCSD + cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j + # then the kernel's tools/perf against it: + git clone --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git + cd linux/tools/perf && make CORESIGHT=1 \\ + EXTLIBS=-l$(pwd)/../../OpenCSD/build/lib/libopencsd \\ + O=~/perf-build WERROR=0 + ~/perf-build/perf record -e cs_etm//u --per-thread -p + +B. Cross (host aarch64 toolchain): same two sources, same CORESIGHT=1, with + CROSS_COMPILE= and the OpenCSD built for aarch64 first. Static-link + the result so it drops onto /data/local/tmp as one file. + +C. Sanity checks once built (run these in order, read each before continuing): + perf list | grep cs_etm # nothing -> no PMU visibility + perf record -e cs_etm//u --per-thread -o ctl.data -- true + perf report -i ctl.data -D | head # CoreSight packets -> decode works + The control record is not optional: it is the only thing that separates a + closed gate from a capture that looks fine and decodes to nothing. + +Capture shape (TRBE per-thread): + perf record -e cs_etm//u --per-thread -p -o perf.data -- sleep +Shared sink (no TRBE; system-wide-ish, cannot --per-thread): + perf record -e cs_etm/@tmc_etr0/u -C -o perf.data -- sleep +Decode to an address stream for scripts/trace_diff.py: + perf script -i perf.data # synthesized instruction/branch events + # then feed the address-bearing lines straight to: + python scripts/trace_diff.py diff --a a.log --b b.log +""" + + +def adb_base(serial): + """Locate adb and the serial prefix; (path, prefix) or (None, []).""" + cmd = shutil.which("adb") + if not cmd: + return None, None + return cmd, (["-s", serial] if serial else []) + + +def adb_shell(adb, prefix, cmd, timeout=15): + """Run one adb shell command string; return its result.""" + return subprocess.run([adb] + prefix + ["shell", cmd], capture_output=True, + text=True, timeout=timeout) + + +def probe_device(args): + """Run every device-side gate probe; return (rows, ok_count, total).""" + adb, prefix = adb_base(args.serial) + if not adb: + return None, 0, len(GATES) + rows = [] + r = adb_shell(adb, prefix, "uname -r") + kern = (r.stdout or "").strip() + kern_ok = bool(re.match(r"^(\d+)\.(\d+)", kern)) and _kern_at_least(kern, 5, 10) + rows.append({"gate": "kernel era", "probe": "uname -r", "value": kern or "?", + "ok": kern_ok}) + r = adb_shell(adb, prefix, "ls /sys/bus/coresight/devices 2>/dev/null") + dev = (r.stdout or "").strip() + has_ete = "ete" in dev + has_trbe = "trbe" in dev + rows.append({"gate": "coresight devices", "probe": "ls /sys/bus/coresight/devices", + "value": (dev or "")[:200], "ok": has_ete and has_trbe, + "note": "ete=%s trbe=%s (trbe absent -> shared-sink only)" + % (has_ete, has_trbe)}) + r = adb_shell(adb, prefix, "ls /sys/bus/event_source/devices/cs_etm 2>/dev/null") + rows.append({"gate": "cs_etm PMU", "probe": "ls .../cs_etm", + "value": "present" if (r.stdout or "").strip() else "absent", + "ok": r.returncode == 0 and bool((r.stdout or "").strip())}) + r = adb_shell(adb, prefix, "su -c 'perf list' 2>/dev/null | grep -c cs_etm") + n = (r.stdout or "").strip().splitlines()[-1] if (r.stdout or "").strip() else "0" + rows.append({"gate": "perf with CORESIGHT", "probe": "perf list | grep cs_etm", + "value": "%s cs_etm entries" % n, "ok": n not in ("", "0")}) + rows.append({"gate": "permission", "probe": "control record (see recipe)", + "value": "not probed by gates; run capture --control", + "ok": False}) + ok = sum(1 for row in rows if row["ok"]) + return rows, ok, len(GATES) + + +def _kern_at_least(kern, maj, mnr): + """True when the kernel version is at least major.minor.""" + m = re.match(r"^(\d+)\.(\d+)", kern) + if not m: + return False + a, b = int(m.group(1)), int(m.group(2)) + return (a, b) >= (maj, mnr) + + +def cmd_gates(args): + """gates subcommand: host probes plus the device-side gates.""" + adb = shutil.which("adb") + if args.json: + doc = {"status": "ok", "gates": GATES, "adb": bool(adb)} + if args.device and adb: + rows, ok, total = probe_device(args) + doc["device_rows"] = rows + doc["device_ok"] = ok + doc["device_total"] = total + doc["status"] = "ok" if ok == total else "partial" + print(json.dumps(doc, indent=1)) + return 0 + print("== ETE capture gates ==") + for g in GATES: + print("\n %s\n expect: %s\n check : %s\n why : %s" + % (g["gate"], g["expect"], g["check"], g["why"])) + print("\nhost: adb %s" % ("present" if adb else "ABSENT -- capture cannot run here")) + device_verdict = "ok" + if args.device and adb: + rows, ok, total = probe_device(args) + print("\ndevice probes (%d/%d closed):" % (ok, total)) + for row in rows: + mark = "ok " if row["ok"] else "OPEN" + print(" [%s] %-22s %s" % (mark, row["gate"], row["value"])) + if ok < total: + device_verdict = "partial" + print("\nAn OPEN gate is a route decision, not a failure -- record it per") + print("references/kernel-and-environment-hardening.md and do not retry mid-task.") + elif args.device: + print("\n--device asked for device probes but adb is absent") + print("\nRESULT=%s" % ("ok" if (adb and device_verdict == "ok") else "partial")) + return 0 if adb else 0 + + +def cmd_recipe(args): + """recipe subcommand: build perf with CoreSight/OpenCSD.""" + print(RECIPE) + print("RESULT=ok") + return 0 + + +def control_record_command(out_name): + """The one-instruction control record: `perf record` of `true` alone. + + The whole command travels inside one `su -c` string, so quoting is + two-layered: quote the filename inside the inner command, then quote the + complete inner command for `su -c` (a quote spliced between the literal + single quotes would run in the outer device shell). + """ + inner = ("cd /data/local/tmp && perf record -e cs_etm//u --per-thread " + "-o %s -- true" % shlex.quote(out_name)) + return "su -c %s 2>&1" % shlex.quote(inner) + + +def control_decode_command(out_name): + """The control's decode gate: `perf script` on the control recording.""" + inner = "cd /data/local/tmp && perf script -i %s" % shlex.quote(out_name) + return "su -c %s 2>&1" % shlex.quote(inner) + + +def perf_record_command(args): + """The capture command line handed to `adb shell`. + + Two rules that are easy to get wrong and load-bearing here: `-o` must + stay BEFORE the `--` workload separator (perf hands everything after + `--` to the target command, so a trailing `-o` never reaches perf), and + every interpolated value is shell-quoted because the whole command + travels inside one `su -c '...'` string -- an unquoted value containing + a quote would run in the outer device shell. `--cmd` is left unquoted + on purpose: the workload argument list is meant to be word-split by the + device shell. + """ + q = shlex.quote + target = "-p %s" % q(str(args.pid)) if args.pid else "-- %s" % args.cmd + remote = "/data/local/tmp/%s" % (args.out.split("/")[-1] or "perf.data") + inner = ("cd /data/local/tmp && timeout %d perf record -e %s --per-thread " + "-o %s %s" % (args.seconds, q(args.event), q(remote), target)) + return "su -c %s 2>&1" % q(inner) + + +def cmd_capture(args): + """capture subcommand: control record, record, pull.""" + if args.seconds < 1: + # `timeout 0` disables the timer (or kills instantly, by flavour): + # either way the --pid route has no capture window and the CLI would + # wait on a record that never ends. Refused before any device check: + # argument validity must not depend on adb being present. + print("RESULT=refused") + print("--seconds must be a positive capture window, got %d" % args.seconds, + file=sys.stderr) + return 2 + adb = shutil.which("adb") + if not adb: + print("RESULT=blocked") + print("no adb on PATH: capture needs a rooted device with the ETE gates closed " + "(run `gates --device` first)", file=sys.stderr) + return 3 + prefix = ["-s", args.serial] if args.serial else [] + if args.control: + out = args.out.replace(".data", "") + ".ctl.data" if args.out.endswith(".data") \ + else args.out + ".ctl.data" + ctl = out.split("/")[-1] + r = adb_shell(adb, prefix, control_record_command(ctl), timeout=60) + print(r.stdout or "") + if (r.returncode != 0 + or ("cs_etm" in (r.stdout or "") and "failed" in (r.stdout or "").lower())): + print("RESULT=blocked") + print("control record failed: the ETE pipe is not usable on this device as-is") + return 3 + # A record exiting 0 only proves perf was not refused -- the documented + # --control guarantee (references/ete-diff-tracing.md: "the control has + # produced packets") is that the record DECODES. Hold the capture to + # the same bar the reference checklist applies to a human. + dec = adb_shell(adb, prefix, control_decode_command(ctl), timeout=60) + print(dec.stdout or "") + dec_lines = [ln for ln in (dec.stdout or "").splitlines() + if ln.strip() and not ln.startswith("#")] + if dec.returncode != 0 or not dec_lines: + print("RESULT=blocked") + print("control record did not decode (perf script exit %d, %d event lines): " + "a capture taken now would be indistinguishable from an empty one, " + "and an empty trace is a capture failure, not a result" + % (dec.returncode, len(dec_lines)), file=sys.stderr) + return 3 + print("control record done: %s (decoded %d event lines)" % (out, len(dec_lines))) + if not args.pid and not args.cmd: + print("RESULT=refused") + print("capture needs --pid or --cmd", file=sys.stderr) + return 2 + perf_cmd = perf_record_command(args) + remote = "/data/local/tmp/%s" % (args.out.split("/")[-1] or "perf.data") + # a perf.data left by an earlier run would let the pull below succeed on + # stale data; remove it first (best effort -- the exit-status check below + # is what fails the run when the pipe is unusable) + adb_shell(adb, prefix, + "su -c %s 2>&1" % shlex.quote("rm -f %s" % shlex.quote(remote)), + timeout=15) + print("running: %s" % perf_cmd) + proc = subprocess.Popen([adb] + prefix + ["shell", perf_cmd], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + for line in proc.stdout: + print(line.rstrip()) + rc = proc.wait() + if rc not in (0, 124): # 124 is `timeout`'s end-of-window kill: the normal end + print("RESULT=blocked") + print("capture did not complete: adb shell exited %d (su denied, perf refused, " + "or the workload never started)" % rc, file=sys.stderr) + return 3 + local = args.out + pull = subprocess.run([adb] + prefix + ["pull", remote, local], + capture_output=True, text=True) + if pull.returncode != 0: + print("RESULT=blocked") + print("pull failed: %s" % (pull.stderr or "").strip()) + return 3 + print("pulled %s" % local) + print("decode next: perf script -i %s (feed the address lines to trace_diff.py)" + % local) + print("RESULT=ok") + return 0 + + +def main(): + """CLI entry: subcommand dispatch.""" + ap = argparse.ArgumentParser( + description="CoreSight ETE hardware trace capture on arm64 Android: gates, " + "perf-with-CORESIGHT recipe, per-thread record wrapper.", + epilog="references/ete-diff-tracing.md owns the method and the differential " + "that turns two captures into an answer.") + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("gates", help="the five gates, why each exists, and how to probe it") + p.add_argument("--device", action="store_true", + help="also run the device-side probes over adb (needs a device)") + p.add_argument("--serial", help="adb device serial") + p.add_argument("--json", action="store_true") + p.set_defaults(fn=cmd_gates) + + p = sub.add_parser("recipe", help="print how to build perf with CoreSIGHT/OpenCSD decode") + p.set_defaults(fn=cmd_recipe) + + p = sub.add_parser("capture", help="record a per-thread cs_etm trace on the device") + p.add_argument("--pid", help="trace this pid (--per-thread, TRBE path)") + p.add_argument("--cmd", help="spawn and trace this command instead of --pid") + p.add_argument("--seconds", type=int, default=10, help="record window (default 10)") + p.add_argument("--event", default="cs_etm//u", + help="perf event spec (default cs_etm//u; shared sink: cs_etm/@tmc_etr0/u)") + p.add_argument("--out", default="perf.data", help="local output file after pull") + p.add_argument("--control", action="store_true", + help="run the one-instruction control record first and refuse on failure") + p.add_argument("--serial", help="adb device serial") + p.set_defaults(fn=cmd_capture) + + args = ap.parse_args() + try: + return args.fn(args) + except BrokenPipeError: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/apk-reverse/scripts/hw_watch.py b/skills/apk-reverse/scripts/hw_watch.py new file mode 100644 index 0000000..6c0048b --- /dev/null +++ b/skills/apk-reverse/scripts/hw_watch.py @@ -0,0 +1,977 @@ +#!/usr/bin/env python3 +"""hw_watch.py -- invisible memory watchpoints and execution breakpoints on arm64 Android. + +What this is, and the honest stealth ledger +------------------------------------------- +An arm64 core carries a small number of *hardware* debug slots (typically 6 +breakpoints, 4 watchpoints). A watchpoint compares the effective data address of +every load/store the core executes -- in silicon, before the instruction +retires -- and traps to the kernel when it matches. Nothing about the target's +code or memory changes: no inline patch, no trampoline, no injected thread, no +foreign mapping, no DBI code cache. A library that checksums its own pages, +scans for hooks, or counts threads finds nothing, because there is nothing to +find in its process image. + +The one thing a ptrace-based watchpoint *does* leave is ``TracerPid`` in +``/proc//status``: the tracer relationship is a kernel fact, and this tool +does not hide it. The ledger, in full: + +| delivery | code modified | foreign thread/mapping | TracerPid | survives page checksums | +|---------------------|---------------|------------------------|-----------|-------------------------| +| inline hook | yes | no | no | NO -- caught by design | +| Frida Stalker | yes (copy) | yes (agent + cache) | no | no | +| ptrace hw watchpoint| NO | no | YES | yes | +| kernel hw watchpoint| NO | no | no | yes (kernel-event-injection.md) | + +Limits that decide whether the tool fits the question: 8-byte maximum per +watchpoint, byte-select inside one aligned 8-byte window only, per-thread slots +(every thread must be armed, new threads arrive unarmed until caught), and the +process must be ptraceable at all (root; nobody else may already be tracing it). + +What runs where +--------------- +``plan`` pure host: encode DBGWCR/DBGBCR values for an address and check the + alignment rules bite before anything touches a device. +``emit`` pure host: write the self-contained on-device helper (C, one file, + no dependency beyond bionic). +``gates`` host probe: is adb here, is an arm64 compiler here -- plus the exact + one-liners to run on the device to close the device-side gates. +``run`` device: build (NDK clang by default, Termux clang on request), push, + run the helper as root, stream hits to a log. +``decode`` pure host: turn a hit log into module+offset, given a maps file. + +Measured vs unverified +---------------------- +The encoders and the plan/emit/decode paths are host-tested +(tests/unit/test_hw_watch.py), and the emitted helper **compiles clean as a +static aarch64 ELF** on a cross toolchain (observed: gcc 16.2, `-static`, zero +errors). **No watchpoint has been armed on a device by this repository's +passes** -- the ptrace dance, the slot capacity probe and the clone-catch path +follow the kernel ABI (arch/arm64 ptrace + hw_breakpoint) and must be validated +on your device with the control procedure in references/hardware-watchpoints.md +before any hit is trusted. + +Examples + python hw_watch.py plan --addr 0x7b2000123456 --len 16 --access w + python hw_watch.py plan --break 0x7b20001234c0 + python hw_watch.py emit --out work/hwwatch + python hw_watch.py gates + python hw_watch.py run --pkg --addr 0x7b2000123456 --len 8 --access rw \\ + --seconds 20 --out hits.log + python hw_watch.py decode --hits hits.log --maps maps.txt +""" +import argparse +import json +import os +import shutil +import subprocess +import sys + +# --------------------------------------------------------------------------- +# Kernel ABI constants (arm64 Linux). These are stable uapi numbers, not +# toolchain facts: include/uapi/linux/ptrace.h, include/uapi/linux/elf.h, +# arch/arm64/include/uapi/asm/ptrace.h. +# --------------------------------------------------------------------------- +PTRACE_CONT = 7 +PTRACE_SINGLESTEP = 9 +PTRACE_ATTACH = 16 +PTRACE_DETACH = 17 +PTRACE_GETSIGINFO = 0x4202 +PTRACE_SETSIGINFO = 0x4203 +PTRACE_GETREGSET = 0x4204 +PTRACE_SETREGSET = 0x4205 +PTRACE_SEIZE = 0x4206 +PTRACE_INTERRUPT = 0x4207 +PTRACE_LISTEN = 0x4208 +PTRACE_O_EXITKILL = 0x01000000 +PTRACE_O_TRACECLONE = 0x00000008 + +NT_PRSTATUS = 1 +NT_ARM_HW_BREAK = 0x402 +NT_ARM_HW_WATCH = 0x403 + +TRAP_HWBKPT = 4 # siginfo si_code for a hardware break/watchpoint hit +USER_PT_REGS_BYTES = 272 # 31 x-regs + sp + pc + pstate, 8 bytes each +MAX_SLOTS = 16 # struct user_hwdebug_state.dbg_regs[] size + +# DBGWCR_EL1 field map (ARM DDI 0487): E[0] enable; PAC[2:1] privilege access +# control (0b10 = EL0/unprivileged only -- the value a userspace watchpoint +# wants); LSC[4:3] load/store control (01 load, 10 store, 11 both); BAS[12:5] +# byte-address-select inside the 8-byte window named by DBGWVR. +WCR_E = 1 +WCR_PAC_EL0 = 0b10 << 1 +WCR_LSC = {"r": 0b01 << 3, "w": 0b10 << 3, "rw": 0b11 << 3} +WCR_BAS_SHIFT = 5 + +# DBGBCR_EL1 for an unlinked address-matching A64 execution breakpoint: +# BAS[13:8] = 0b111111 (whole-address match), PMC[2:1] = 0b10 (EL0), BT = 0. +BCR_ADDR_MATCH_EL0 = (0x3F << 8) | WCR_PAC_EL0 | WCR_E + +# arm64 user_pt_regs offsets (bytes) -- used by the helper and by decode. +REG_OFFSETS = {("x%d" % i): i * 8 for i in range(31)} +REG_OFFSETS.update({"sp": 31 * 8, "pc": 32 * 8, "pstate": 33 * 8}) + + +class PlanError(ValueError): + """An address/length/access combination the hardware cannot express.""" + + +def encode_watch(addr, length, access): + """Return (wvr, wcr) for a watchpoint, or raise PlanError with the reason. + + The hardware rule that bites: BAS selects bytes inside ONE aligned 8-byte + window. A watch that starts at 0x...4 with length 8 crosses the window and + cannot be expressed in one slot (move the address or use two). + """ + if length not in range(1, 9): + raise PlanError("watch length must be 1..8 bytes, got %d" % length) + if access not in WCR_LSC: + raise PlanError("access must be one of r/w/rw, got %r" % access) + off = addr & 7 + if off + length > 8: + raise PlanError( + "watch [0x%x + %d] crosses its aligned 8-byte window " + "(addr&7=%d): shift the address or split across two slots" + % (addr, length, off)) + bas = (((1 << length) - 1) << off) << WCR_BAS_SHIFT + wcr = bas | WCR_LSC[access] | WCR_PAC_EL0 | WCR_E + return addr & ~0x7, wcr + + +def encode_break(addr): + """Return (bvr, bcr) for an execution breakpoint at a 4-byte-aligned PC.""" + if addr & 3: + raise PlanError( + "breakpoint PC 0x%x is not 4-byte aligned -- A64 instructions are" + % addr) + return addr, BCR_ADDR_MATCH_EL0 + + +def canonical_examples(): + """The two encodings everything else is a variation of (unit-tested).""" + return encode_watch(0, 8, "rw")[1], BCR_ADDR_MATCH_EL0 + + +# --------------------------------------------------------------------------- +# The on-device helper. One C file, no dependency beyond bionic; every ptrace +# call goes through syscall() so no header availability can change behaviour. +# +# STATUS: the syntax is host-checked (gcc -fsyntax-only where a compiler +# exists) and the ABI numbers are the kernel's uapi values, but this helper +# has not been run on a device by this repository's passes. Validate with the +# control procedure in references/hardware-watchpoints.md first. +# --------------------------------------------------------------------------- +HWWATCH_C = r'''/* hwwatch_helper.c -- arm64 hardware watchpoint/breakpoint driver. + * + * Generated by skills/apk-reverse/scripts/hw_watch.py -- do not edit by hand. + * TEMPLATE NOT YET DEVICE-TESTED BY THIS REPOSITORY. Build it, run the control + * procedure (references/hardware-watchpoints.md) before trusting a hit, and + * remember: ptrace makes TracerPid non-zero in /proc//status. + * + * Usage (as root): + * hwwatch w::: [w:... ...] [b: ...] + * [--max-hits N] [--seconds S] + * Output grammar (one logical line per event): + * READY pid= watch_slots= break_slots= + * ARM tid= watch= break= + * HIT tid= kind=watch|break pc=0x.. addr=0x.. sp=0x.. lr=0x.. x0=0x.. .. x7=0x.. + * EXIT tid= code= + * DONE hits= reason=max-hits|seconds|exit|clean-detach|error + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* --- kernel ABI numbers (arm64); kept local so headers cannot disagree --- */ +#define PTRACE_CONT 7 +#define PTRACE_SINGLESTEP 9 +#define PTRACE_DETACH 17 +#define PTRACE_GETSIGINFO 0x4202 +#define PTRACE_GETREGSET 0x4204 +#define PTRACE_SETREGSET 0x4205 +#define PTRACE_SEIZE 0x4206 +#define PTRACE_INTERRUPT 0x4207 +#define PTRACE_LISTEN 0x4208 +#define PTRACE_O_EXITKILL 0x01000000 +#define PTRACE_O_TRACECLONE 0x00000008 +#define NT_PRSTATUS 1 +#define NT_ARM_HW_BREAK 0x402 +#define NT_ARM_HW_WATCH 0x403 +#ifndef TRAP_HWBKPT +#define TRAP_HWBKPT 4 +#endif +#define PTRACE_EVENT_CLONE 3 +#define PTRACE_EVENT_STOP 128 + +/* Slot layout is the kernel ABI (arch/arm64/include/uapi/asm/ptrace.h, + * struct user_hwdebug_state): per slot a 64-bit ADDRESS register first, then a + * 32-bit CONTROL register, then 4 bytes of pad -- 16 bytes per slot. Swapping + * the two arms every slot with the control bits as the address. */ +struct hwslot { uint64_t wvr; uint32_t wcr; uint32_t pad; }; +struct hwstate { uint32_t dbg_info; uint32_t pad; struct hwslot slot[16]; }; +struct uregs { uint64_t x[31]; uint64_t sp, pc, pstate; }; + +static long pt(long req, long pid, long addr, long data) { + return syscall(SYS_ptrace, req, pid, (void *)addr, (void *)data); +} + +/* ---- configuration filled from argv ---- */ +struct spec { int is_break; uint64_t addr; int len; char access[3]; }; +static struct spec specs[16]; +static int nspec = 0, max_hits = 0; +static long run_seconds = 0; + +#define MAXTIDS 256 +static int tids[MAXTIDS]; /* known threads of the target */ +static int ntids = 0; +static int target_pid = 0; +static long hits = 0; + +static void say(const char *fmt, ...) { + va_list ap; va_start(ap, fmt); + vfprintf(stdout, fmt, ap); + fputc('\n', stdout); + fflush(stdout); + va_end(ap); +} + +static int known(int tid) { + for (int i = 0; i < ntids; i++) if (tids[i] == tid) return 1; + return 0; +} + +static void remember(int tid) { + if (tid <= 0 || known(tid) || ntids >= MAXTIDS) return; + tids[ntids++] = tid; +} + +static int get_regset(int tid, int nt, void *buf, size_t len) { + struct iovec iov = { buf, len }; + return (int)pt(PTRACE_GETREGSET, tid, nt, (long)&iov); +} +static int set_regset(int tid, int nt, const void *buf, size_t len) { + struct iovec iov = { (void *)buf, len }; + return (int)pt(PTRACE_SETREGSET, tid, nt, (long)&iov); +} + +/* Read the slot capacity the kernel reports (dbg_info & 0xff). */ +static int slot_capacity(int tid, int nt) { + struct hwstate st; + memset(&st, 0, sizeof st); + if (get_regset(tid, nt, &st, sizeof st) != 0) return -1; + return (int)(st.dbg_info & 0xff); +} + +static int wcap = 0, bcap = 0; /* slot capacities, probed once */ + +/* Arm every watch/break spec on one stopped thread. Watchpoints and + * breakpoints live in separate tables on arm64, so slot indexes of the two + * kinds may overlap. */ +static int arm_thread(int tid) { + struct hwstate w, b; + int nw = 0, nb = 0; + memset(&w, 0, sizeof w); memset(&b, 0, sizeof b); + for (int i = 0; i < nspec; i++) { + const struct spec *s = &specs[i]; + if (s->is_break) { + if (nb >= bcap) { say("ARMFAIL tid=%d slot=break%d capacity=%d", tid, nb, bcap); return -1; } + b.slot[nb].wcr = (0x3Fu << 8) | (0x2u << 1) | 1u; /* BAS all-ones, PMC=EL0, E */ + b.slot[nb].wvr = s->addr; + nb++; + } else { + if (nw >= wcap) { say("ARMFAIL tid=%d slot=watch%d capacity=%d", tid, nw, wcap); return -1; } + unsigned bas = (((1u << s->len) - 1u) << (s->addr & 7)) << 5; + unsigned lsc = s->access[0] == 'r' && s->access[1] == 'w' ? 3u + : (s->access[0] == 'r' ? 1u : 2u); + w.slot[nw].wcr = bas | (lsc << 3) | (0x2u << 1) | 1u; + w.slot[nw].wvr = s->addr & ~0x7ull; + nw++; + } + } + if (nw && set_regset(tid, NT_ARM_HW_WATCH, &w, 8 + 16 * nw) != 0) return -1; + if (nb && set_regset(tid, NT_ARM_HW_BREAK, &b, 8 + 16 * nb) != 0) return -1; + say("ARM tid=%d watch=%d break=%d", tid, nw, nb); + return 0; +} + +static int clear_thread(int tid) { + struct hwstate z; + memset(&z, 0, sizeof z); + set_regset(tid, NT_ARM_HW_WATCH, &z, 8 + 16 * 1); + set_regset(tid, NT_ARM_HW_BREAK, &z, 8 + 16 * 1); + return 0; +} + +/* Undo everything: clear slots, detach every seized thread. Every error + * path after the first successful PTRACE_SEIZE must go through here: + * PTRACE_O_EXITKILL means an exit without detach SIGKILLs the target -- + * an analysis failure must not terminate the app under analysis. */ +static void release_all(void) { + for (int i = 0; i < ntids; i++) clear_thread(tids[i]); + for (int i = 0; i < ntids; i++) pt(PTRACE_DETACH, tids[i], 0, 0); +} + +/* Enumerate /proc//task, seizing and interrupting every thread. */ +static int seize_all(void) { + char path[64]; + snprintf(path, sizeof path, "/proc/%d/task", target_pid); + DIR *d = opendir(path); + if (!d) { say("DONE hits=%ld reason=error detail=no-such-process", hits); return -1; } + struct dirent *e; + while ((e = readdir(d))) { + if (e->d_name[0] == '.') continue; + int tid = atoi(e->d_name); + if (!tid || known(tid)) continue; + if (pt(PTRACE_SEIZE, tid, 0, PTRACE_O_EXITKILL | PTRACE_O_TRACECLONE) != 0) { + if (errno == EPERM) + say("FATAL seize tid=%d EPERM: not root, or already traced", tid); + else + say("FATAL seize tid=%d errno=%d", tid, errno); + closedir(d); + /* partial seize: threads seized before this one must not be + * EXITKILLed when we exit -- detach them now */ + for (int i = 0; i < ntids; i++) pt(PTRACE_DETACH, tids[i], 0, 0); + return -1; + } + remember(tid); + } + closedir(d); + for (int i = 0; i < ntids; i++) + pt(PTRACE_INTERRUPT, tids[i], 0, 0); + /* wait for each to report stopped; other tids' stops stay queued and are + * reaped by the main loop */ + for (int i = 0; i < ntids; i++) { + int st; + pid_t r; + do { r = waitpid(tids[i], &st, __WALL); } while (r < 0 && errno == EINTR); + (void)r; + } + return 0; +} + +static void print_hit(int tid, int is_break, struct uregs *r, uint64_t waddr) { + printf("HIT tid=%d kind=%s pc=0x%llx addr=0x%llx sp=0x%llx lr=0x%llx " + "x0=0x%llx x1=0x%llx x2=0x%llx x3=0x%llx x4=0x%llx x5=0x%llx " + "x6=0x%llx x7=0x%llx\n", + tid, is_break ? "break" : "watch", + (unsigned long long)r->pc, (unsigned long long)waddr, + (unsigned long long)r->sp, (unsigned long long)r->x[30], + (unsigned long long)r->x[0], (unsigned long long)r->x[1], + (unsigned long long)r->x[2], (unsigned long long)r->x[3], + (unsigned long long)r->x[4], (unsigned long long)r->x[5], + (unsigned long long)r->x[6], (unsigned long long)r->x[7]); + fflush(stdout); +} + +/* Step over an execution breakpoint: disable the slot, single-step, re-arm. + * Without this dance the re-enabled trap fires on the same PC forever. */ +static int step_over(int tid, uint64_t pc) { + struct hwstate b; + int nb = 0; + memset(&b, 0, sizeof b); + for (int i = 0; i < nspec; i++) + if (specs[i].is_break && specs[i].addr != pc) { + b.slot[nb].wcr = (0x3Fu << 8) | (0x2u << 1) | 1u; + b.slot[nb].wvr = specs[i].addr; + nb++; + } + set_regset(tid, NT_ARM_HW_BREAK, &b, 8 + 16 * 16); + pt(PTRACE_SINGLESTEP, tid, 0, 0); + int st; + waitpid(tid, &st, __WALL); + return arm_thread(tid); +} + +int main(int argc, char **argv) { + if (argc < 3) { fprintf(stderr, "usage: hwwatch w:0xADDR:LEN:rw [b:0xADDR] ... [--max-hits N] [--seconds S]\n"); return 2; } + target_pid = atoi(argv[1]); + if (target_pid <= 0) return 2; + for (int i = 2; i < argc; i++) { + if (!strncmp(argv[i], "--max-hits", 10)) { max_hits = atoi(argv[++i]); continue; } + if (!strncmp(argv[i], "--seconds", 9)) { run_seconds = atol(argv[++i]); continue; } + struct spec s; + memset(&s, 0, sizeof s); + if (argv[i][0] == 'b' && argv[i][1] == ':') { + s.is_break = 1; + s.addr = strtoull(argv[i] + 2, NULL, 0); + if (s.addr & 3) { fprintf(stderr, "break PC must be 4-byte aligned\n"); return 2; } + } else if (argv[i][0] == 'w' && argv[i][1] == ':') { + char *p = argv[i] + 2, *q; + s.addr = strtoull(p, &q, 0); + if (*q != ':') return 2; + s.len = (int)strtol(q + 1, &q, 0); + if (*q != ':') return 2; + snprintf(s.access, sizeof s.access, "%s", q + 1); + unsigned off = s.addr & 7; + if (s.len < 1 || s.len > 8 || off + s.len > 8) { + fprintf(stderr, "watch [0x%llx+%d] crosses its 8-byte window\n", + (unsigned long long)s.addr, s.len); + return 2; + } + } else return 2; + if (nspec < 16) specs[nspec++] = s; + } + if (geteuid() != 0) { + fprintf(stderr, "FATAL not root: ptrace of an app process requires root\n"); + return 3; + } + + if (seize_all() != 0) return 1; /* seize_all detached its own partials */ + wcap = slot_capacity(tids[0], NT_ARM_HW_WATCH); + bcap = slot_capacity(tids[0], NT_ARM_HW_BREAK); + if (wcap < 0 || bcap < 0) { + say("FATAL GETREGSET NT_ARM_HW_* errno=%d -- not arm64, or hw debug masked", errno); + release_all(); + return 1; + } + say("READY pid=%d watch_slots=%d break_slots=%d", target_pid, wcap, bcap); + for (int i = 0; i < ntids; i++) + if (arm_thread(tids[i]) != 0) { release_all(); return 1; } + for (int i = 0; i < ntids; i++) pt(PTRACE_CONT, tids[i], 0, 0); + + const char *reason = "exit"; + long deadline = run_seconds ? time(NULL) + run_seconds : 0; + for (;;) { + if (max_hits && hits >= max_hits) { reason = "max-hits"; break; } + if (deadline && time(NULL) >= deadline) { reason = "seconds"; break; } + int st; + pid_t tid = waitpid(-1, &st, __WALL | WNOHANG); + if (tid == 0) { struct timespec ts = {0, 10 * 1000 * 1000}; nanosleep(&ts, NULL); continue; } + if (tid < 0) { + if (errno == EINTR) continue; + reason = "error"; break; + } + if (WIFEXITED(st) || WIFSIGNALED(st)) { + say("EXIT tid=%d code=%d", tid, WIFEXITED(st) ? WEXITSTATUS(st) : -1); + if (tid == target_pid) { reason = "exit"; break; } + continue; + } + if (!WIFSTOPPED(st)) continue; + int sig = WSTOPSIG(st); + unsigned event = (unsigned)st >> 16; + + /* a thread we have never seen: with TRACECLONE this is a new child -- + * arm it before it runs a single instruction if we can */ + if (!known(tid) && event == PTRACE_EVENT_STOP && sig == SIGSTOP) { + remember(tid); + arm_thread(tid); + pt(PTRACE_CONT, tid, 0, 0); + continue; + } + if (sig == SIGTRAP) { + if (event != 0) { + /* ptrace event stop (clone/exec/...): consumed by us, and it + * is not a signal the target generated -- restart clean */ + pt(PTRACE_CONT, tid, 0, 0); + continue; + } + siginfo_t si; + memset(&si, 0, sizeof si); + if (pt(PTRACE_GETSIGINFO, tid, 0, (long)&si) != 0) { + pt(PTRACE_CONT, tid, 0, 0); + continue; + } + if (si.si_code == TRAP_HWBKPT) { + struct uregs r; + memset(&r, 0, sizeof r); + get_regset(tid, NT_PRSTATUS, &r, sizeof r); + int is_break = 0; + for (int i = 0; i < nspec; i++) + if (specs[i].is_break && specs[i].addr == r.pc) is_break = 1; + print_hit(tid, is_break, &r, (uint64_t)(uintptr_t)si.si_addr); + hits++; + if (is_break) { step_over(tid, r.pc); pt(PTRACE_CONT, tid, 0, 0); continue; } + pt(PTRACE_CONT, tid, 0, 0); + continue; + } + /* signal-delivery stop that is not our hardware trap: a genuine + * target SIGTRAP (raise/tgkill/int3). Forward it, or the target's + * handler never runs and the app misbehaves in ways this tool + * would then be blamed for. */ + pt(PTRACE_CONT, tid, 0, sig); + continue; + } + if ((sig == SIGSTOP || sig == SIGTSTP || sig == SIGTTIN || sig == SIGTTOU) + && event == PTRACE_EVENT_STOP) { + pt(PTRACE_LISTEN, tid, 0, 0); + continue; + } + pt(PTRACE_CONT, tid, 0, sig); /* the target's own signal: forward */ + } + + /* clean detach: stop everyone, clear slots, let go */ + for (int i = 0; i < ntids; i++) { + pt(PTRACE_INTERRUPT, tids[i], 0, 0); + int st2; + waitpid(tids[i], &st2, __WALL); + } + release_all(); + say("DONE hits=%ld reason=%s", hits, reason); + return 0; +} +''' + +HELPER_README = """# hwwatch helper (generated) + +Built from `hw_watch.py emit`. Two ways to build it: + +NDK (host build, static): + + /toolchains/llvm/prebuilt//bin/aarch64-linux-android24-clang \\ + -O2 -static -o hwwatch hwwatch_helper.c + +Termux (on-device build): + + pkg install clang + clang -O2 -static -o hwwatch hwwatch_helper.c + +Run as root, one spec per slot: + + su -c '/data/local/tmp/hwwatch w:0x7b2000123456:8:rw b:0x7b20001234c0 --seconds 20' + +Checks before trusting any hit (references/hardware-watchpoints.md): + 1. READY reports watch_slots/break_slots > 0 (0 means the kernel masked hw debug). + 2. Run the control: watch a global you can touch on demand, confirm a HIT arrives. + 3. TracerPid will be non-zero while this runs -- that is the ptrace route's + one visible artifact, and the kernel route exists for when that matters. + +This helper is a template: its encodings are host-unit-tested, its device +behaviour was not measured by this repository. +""" + + +# --------------------------------------------------------------------------- +# subcommands +# --------------------------------------------------------------------------- +def parse_int(text): + """Parse 0x/0o/decimal text the way the CLI accepts addresses.""" + return int(text, 0) + + +def cmd_plan(args): + """plan subcommand: encode specs locally, refuse impossible ones.""" + rows = [] + err = None + try: + if args.addr is not None: + wvr, wcr = encode_watch(parse_int(args.addr), args.len, args.access) + rows.append(("watch", parse_int(args.addr), args.len, args.access, wvr, wcr)) + if args.break_addr is not None: + bvr, bcr = encode_break(parse_int(args.break_addr)) + rows.append(("break", parse_int(args.break_addr), 4, "x", bvr, bcr)) + if not rows: + err = "nothing to plan: give --addr or --break" + except PlanError as exc: + err = str(exc) + if err: + if args.json: + print(json.dumps({"status": "refused", "reason": err})) + else: + print("refused: %s" % err) + return 2 + if args.json: + print(json.dumps({ + "status": "ok", + "specs": [{"kind": k, "addr": a, "len": n, "access": ac, + "vr": v, "cr": c, + "cr_fields": explain_cr(k, c)} for (k, a, n, ac, v, c) in rows], + })) + return 0 + specs = [] + for (k, a, n, ac, v, c) in rows: + print("%s addr=0x%x len=%s access=%s" % (k, a, n, ac)) + print(" %-4s = 0x%x" % ("VR", v)) + print(" %-4s = 0x%x %s" % ("CR", c, explain_cr(k, c))) + specs.append("w:0x%x:%d:%s" % (a, n, ac) if k == "watch" else "b:0x%x" % a) + print("\nspec for the device: %s" % " ".join(specs)) + print("RESULT=ok") + return 0 + + +def explain_cr(kind, cr): + """Decode a WCR/BCR value into its ARM DDI 0487 field meanings.""" + if kind == "watch": + bas = (cr >> WCR_BAS_SHIFT) & 0xFF + lsc = (cr >> 3) & 0b11 + mode = {1: "load", 2: "store", 3: "load+store"}[lsc] + return "BAS=0x%02x LSC=%s PAC=EL0 E=1" % (bas, mode) + return "BAS=all-ones PMC=EL0 BT=unlinked-addr E=1" + + +def cmd_emit(args): + """emit subcommand: write the on-device C helper and its README.""" + out = os.path.abspath(args.out) + os.makedirs(out, exist_ok=True) + c_path = os.path.join(out, "hwwatch_helper.c") + with open(c_path, "w", encoding="utf-8") as fh: + fh.write(HWWATCH_C) + with open(os.path.join(out, "README.md"), "w", encoding="utf-8") as fh: + fh.write(HELPER_README) + print("wrote %s" % c_path) + print("wrote %s" % os.path.join(out, "README.md")) + print("RESULT=ok") + return 0 + + +def find_ndk_clang(): + """Best-effort NDK clang discovery; env ANDROID_NDK_HOME wins.""" + env = os.environ.get("ANDROID_NDK_HOME") or os.environ.get("ANDROID_NDK") + roots = [] + if env: + roots.append(env) + for base in (os.path.expanduser("~/Android/Sdk/ndk"), + os.path.expanduser("~/Android/sdk/ndk"), + "/opt/android-sdk/ndk"): + if os.path.isdir(base): + roots.extend(os.path.join(base, n) for n in sorted(os.listdir(base), reverse=True)) + for root in roots: + for pre in ("toolchains/llvm/prebuilt",): + pre_dir = os.path.join(root, pre) + if not os.path.isdir(pre_dir): + continue + host = os.listdir(pre_dir)[0] + clang = os.path.join(pre_dir, host, "bin", "aarch64-linux-android24-clang") + if os.path.isfile(clang): + return clang + clang2 = os.path.join(pre_dir, host, "bin", "clang") + if os.path.isfile(clang2): + return clang2 + " --target=aarch64-linux-android24" + return None + + +def cmd_gates(args): + """gates subcommand: host probes plus the device-side gates.""" + rows = [] + adb = shutil.which("adb") + rows.append(("adb on PATH", "yes" if adb else "NO -- device orchestration (run) unavailable here")) + clang = find_ndk_clang() + rows.append(("NDK arm64 clang", + clang if clang else "NO -- pass --clang, or build on device with Termux clang")) + cc = shutil.which("gcc") or shutil.which("cc") or shutil.which("clang") + rows.append(("host C syntax check", cc if cc else "NO -- emitted C goes unchecked on this host")) + if args.json: + print(json.dumps({"status": "ok" if adb else "partial", + "gates": [{"gate": g, "value": v} for g, v in rows]}, + ensure_ascii=False)) + return 0 if adb else 0 + print("== hw_watch gates (host) ==") + for g, v in rows: + print(" %-18s %s" % (g + ":", v)) + print("\ndevice-side gates -- run these via scripts/devsh.py and read them back:") + print(" cat /proc//status | grep TracerPid # must be 0: nobody else is tracing") + print(" ls /proc//task # thread count to arm") + print(" su -c 'hwwatch w:0x..:8:rw --max-hits 1' # READY must report slots>0") + print("RESULT=%s" % ("ok" if adb else "partial")) + return 0 if adb else 0 + + +def adb_cmd(adb, serial, *argv, **kw): + """Run one adb command; return its CompletedProcess unchecked.""" + cmd = [adb] + if serial: + cmd += ["-s", serial] + return subprocess.run(cmd + list(argv), capture_output=True, text=kw.pop("text", True), **kw) + + +def cmd_run(args): + """run subcommand: build, push and run the helper; stream hits.""" + adb = shutil.which("adb") + if not adb: + print("RESULT=blocked") + print("no adb on PATH: the run path needs a rooted device (see gates)", file=sys.stderr) + return 3 + # 1. locate / build the helper + helper_dir = os.path.abspath(args.helper_dir or os.path.join(".", "hwwatch_work")) + src = os.path.join(helper_dir, "hwwatch_helper.c") + if not os.path.isfile(src): + os.makedirs(helper_dir, exist_ok=True) + with open(src, "w", encoding="utf-8") as fh: + fh.write(HWWATCH_C) + if args.helper: + local_bin = args.helper + else: + clang_argv = (args.clang or find_ndk_clang()).split() if (args.clang or find_ndk_clang()) else None + if args.on_device_build: + local_bin = None # built on the device below + elif not clang_argv: + print("RESULT=blocked") + print("no arm64 compiler: pass --clang, --on-device-build, or --helper", + file=sys.stderr) + return 3 + else: + local_bin = os.path.join(helper_dir, "hwwatch") + built = subprocess.run(clang_argv + ["-O2", "-static", "-o", local_bin, src], + capture_output=True, text=True) + if built.returncode != 0: + print("RESULT=blocked") + print("helper build failed:\n%s" % built.stderr, file=sys.stderr) + return 3 + # 2. resolve the pid + if args.pid: + pid = args.pid + elif args.pkg: + out = adb_cmd(adb, args.serial, "shell", "pidof %s" % args.pkg) + pid = (out.stdout or "").strip().split() + if not pid: + print("RESULT=blocked") + print("no running process for %s (launch it first)" % args.pkg, file=sys.stderr) + return 3 + pid = pid[0] + else: + print("RESULT=blocked") + print("need --pid or --pkg", file=sys.stderr) + return 2 + # 3. build the spec list and validate locally, before touching the device + try: + specs = [] + if args.addr is not None: + encode_watch(parse_int(args.addr), args.len, args.access) + specs.append("w:%s:%d:%s" % (args.addr, args.len, args.access)) + for b in (args.break_addr or []): + encode_break(parse_int(b)) + specs.append("b:%s" % b) + if not specs: + raise PlanError("nothing to watch: give --addr or --break") + except PlanError as exc: + print("RESULT=refused") + print("refused: %s" % exc, file=sys.stderr) + return 2 + remote = "/data/local/tmp/hwwatch" + + def fail(step, detail): + """Report RESULT=blocked with the step's output; return exit 3.""" + print("RESULT=blocked") + print("%s failed: %s" % (step, (detail or "").strip()[:300]), file=sys.stderr) + return 3 + + if local_bin: + r = adb_cmd(adb, args.serial, "push", local_bin, remote) + if r.returncode != 0: + return fail("push", r.stderr or r.stdout) + else: + r = adb_cmd(adb, args.serial, "push", src, remote + ".c") + if r.returncode != 0: + return fail("push", r.stderr or r.stdout) + build = adb_cmd(adb, args.serial, "shell", + "su -c 'clang -O2 -static -o %s %s.c'" % (remote, remote)) + if build.returncode != 0: + return fail("on-device build (Termux clang)", build.stdout or build.stderr) + r = adb_cmd(adb, args.serial, "shell", "chmod 755 %s" % remote) + if r.returncode != 0: + return fail("chmod", r.stderr or r.stdout) + argv_remote = [remote, str(pid)] + specs + if args.max_hits: + argv_remote += ["--max-hits", str(args.max_hits)] + if args.seconds: + argv_remote += ["--seconds", str(args.seconds)] + quoted = " ".join(argv_remote) + print("running: su -c '%s'" % quoted) + proc = subprocess.Popen( + [adb] + (["-s", args.serial] if args.serial else []) + ["shell", "su -c '%s'" % quoted], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + out_path = args.out or "hwwatch_hits.log" + hits = 0 + with open(out_path, "w", encoding="utf-8") as log: + for line in proc.stdout: + log.write(line) + print(line.rstrip()) + if line.startswith("HIT"): + hits += 1 + rc = proc.wait() + if rc != 0: + # a denied su, a failed push/build above, or a helper FATAL exits + # nonzero: reporting `ok-no-hits` here would call a dead capture a + # quiet one -- exactly the false negative a control run exists to catch + print("RESULT=blocked") + print("capture did not complete: adb shell exited %d -- read the log above " + "for the helper's FATAL/READY lines" % rc, file=sys.stderr) + return 3 + print("\nhits=%d log=%s" % (hits, out_path)) + print("decode with: python hw_watch.py decode --hits %s --maps " % out_path) + print("RESULT=ok" if hits else "RESULT=ok-no-hits") + return 0 + + +HIT_KEYS = ("pc", "addr", "sp", "lr", "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7") + + +def parse_hits(path): + """Parse a hit log into dicts; tolerant of interleaved ARM/READY/DONE lines.""" + hits = [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + line = line.strip() + if not line.startswith("HIT"): + continue + row = {} + for tok in line.split(): + if "=" in tok: + k, v = tok.split("=", 1) + if k in ("tid", "kind") or k in HIT_KEYS: + row[k] = v + if row.get("pc"): + hits.append(row) + return hits + + +def parse_maps(path): + """Accept /proc//maps text or lib_map.py JSON; return [(start,end,path)].""" + ranges = [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + first = fh.read(4096) + fh.seek(0) + if first.lstrip().startswith("{"): + doc = json.load(fh) + rows = doc if isinstance(doc, list) else doc.get("rows", []) + for row in rows: + base = row.get("base") + path = row.get("path", "?") + if isinstance(base, int): + # one range per library, base..base+8MB heuristic is wrong; + # lib_map carries per-path base only -- flag it as such. + ranges.append((base, base + 0x1000, path + " (base only, no size in json)")) + return ranges + for line in fh: + parts = line.split(None, 5) + if len(parts) >= 5 and "-" in parts[0]: + start, _, end = parts[0].partition("-") + try: + path = parts[5].strip() if len(parts) > 5 else "" + ranges.append((int(start, 16), int(end, 16), path)) + except ValueError: + continue + return ranges + + +def module_bases(ranges): + """Per-path module base = that module's LOWEST mapping start. + + A library maps as several ranges (r--p, r-xp, rw-p); a PC hit in the + r-xp range must still report its offset from the module base -- the + number IDA/Ghidra expects -- not from the current mapping's start, which + resets the offset at every mapping boundary.""" + bases = {} + for start, _end, path in ranges: + if not path: + continue + if path not in bases or start < bases[path]: + bases[path] = start + return bases + + +def resolve(addr, ranges, bases=None): + """Name a runtime address as path+offset, offset taken from the module base.""" + if bases is None: + bases = module_bases(ranges) + for start, end, path in ranges: + if start <= addr < end: + base = bases.get(path, start) # anon ranges keep their own start + return "%s+0x%x" % (path or "anon", addr - base) + return None + + +def cmd_decode(args): + """decode subcommand: hit log plus maps -> module+offset rows.""" + hits = parse_hits(args.hits) + ranges = parse_maps(args.maps) if args.maps else [] + bases = module_bases(ranges) + rows = [] + unresolved = 0 + for h in hits: + pc = parse_int(h["pc"]) + site = resolve(pc, ranges, bases) if ranges else None + if site is None: + unresolved += 1 + rows.append({"tid": h.get("tid"), "kind": h.get("kind"), "pc": h["pc"], + "site": site or "unresolved", + "watch_addr": h.get("addr")}) + if args.json: + print(json.dumps({"status": "ok", "hits": len(rows), + "unresolved_pc": unresolved, "rows": rows})) + return 0 + print("hits=%d unresolved_pc=%d%s" % (len(rows), unresolved, + "" if ranges else " (no --maps: PCs stay raw)")) + for r in rows: + print(" tid=%s kind=%-5s pc=%s -> %s" % (r["tid"], r["kind"], r["pc"], r["site"])) + if r["kind"] == "watch": + print(" watched addr=%s" % r["watch_addr"]) + print("RESULT=ok") + return 0 + + +def main(): + """CLI entry: subcommand dispatch.""" + ap = argparse.ArgumentParser( + description="Invisible memory watchpoints / execution breakpoints on arm64 Android " + "via hardware debug registers (plan, emit, gates, run, decode).", + epilog="references/hardware-watchpoints.md owns the procedure, its stealth ledger, " + "and the control run you owe any hit before you trust it.") + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("plan", help="encode WCR/BCR for an address and validate alignment rules") + p.add_argument("--addr", help="watchpoint data address (0x..)") + p.add_argument("--len", type=int, default=8, choices=range(1, 9), + help="watch length in bytes, 1..8 (default 8)") + p.add_argument("--access", default="rw", choices=["r", "w", "rw"], + help="trap on read / write / both (default rw)") + p.add_argument("--break", dest="break_addr", metavar="ADDR", + help="execution breakpoint PC (4-byte aligned) instead of / in addition to a watch") + p.add_argument("--json", action="store_true") + p.set_defaults(fn=cmd_plan) + + p = sub.add_parser("emit", help="write the self-contained on-device C helper") + p.add_argument("--out", required=True, help="directory to write into") + p.set_defaults(fn=cmd_emit) + + p = sub.add_parser("gates", help="host probe: adb, arm64 compiler; plus device-side checks") + p.add_argument("--json", action="store_true") + p.set_defaults(fn=cmd_gates) + + p = sub.add_parser("run", help="build, push, run the helper on a rooted device, stream hits") + p.add_argument("--pid", help="target pid (or --pkg)") + p.add_argument("--pkg", help="resolve pid by package name") + p.add_argument("--addr", help="watchpoint data address") + p.add_argument("--len", type=int, default=8, choices=range(1, 9)) + p.add_argument("--access", default="rw", choices=["r", "w", "rw"]) + p.add_argument("--break", dest="break_addr", metavar="ADDR", action="append", + help="execution breakpoint PC (repeatable)") + p.add_argument("--seconds", type=int, default=30, help="capture window (default 30)") + p.add_argument("--max-hits", type=int, default=0, help="stop after N hits (0 = no cap)") + p.add_argument("--out", default="hwwatch_hits.log", help="local hit log") + p.add_argument("--serial", help="adb device serial") + p.add_argument("--clang", help="explicit arm64 compiler command (quoted)") + p.add_argument("--on-device-build", action="store_true", + help="build the helper with Termux clang on the device") + p.add_argument("--helper", help="path to an already-built helper binary") + p.add_argument("--helper-dir", help="where the generated source lives (default ./hwwatch_work)") + p.set_defaults(fn=cmd_run) + + p = sub.add_parser("decode", help="module+offset for each HIT in a log") + p.add_argument("--hits", required=True, help="hit log from run/") + p.add_argument("--maps", help="/proc//maps text or lib_map.py JSON") + p.add_argument("--json", action="store_true") + p.set_defaults(fn=cmd_decode) + + args = ap.parse_args() + try: + return args.fn(args) + except BrokenPipeError: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/apk-reverse/scripts/kernel_instrument_scaffold.py b/skills/apk-reverse/scripts/kernel_instrument_scaffold.py new file mode 100644 index 0000000..3df4c4b --- /dev/null +++ b/skills/apk-reverse/scripts/kernel_instrument_scaffold.py @@ -0,0 +1,611 @@ +#!/usr/bin/env python3 +"""kernel_instrument_scaffold.py -- generate a kernel-side instrumentation scaffold: +invisible watchpoints and event injection from a driver. + +What "event injection" means here, and what this generates +--------------------------------------------------------- +The escalation this scaffold exists for: you want to observe or alter a target +process with **no ptrace** (so `TracerPid` stays 0), **no injected thread**, +**no code modification** (so page checksums stay clean) and **no DBI cache**. +Only a kernel-side driver can do all four at once, because the hardware debug +registers can be programmed on the target's behalf from the kernel, and the +overflow callback runs with the interrupted user context's `pt_regs` in hand: + + watch(addr,len,access) -> a hardware watchpoint armed on the target task + through the kernel perf/breakpoint API; the + callback logs pc + data address (dmesg) + break(pc) + set-reg -> an execution breakpoint whose callback rewrites a + register in the interrupted frame: flip a branch + decision without touching one byte of code + read/write memory -> `access_process_vm` on the target from the driver + (patch data, dump buffers), no process_vm_* syscalls + +Three artifacts, one per route, each with its gate (all from +`references/kernel-and-environment-hardening.md` §4a): + + kpm KernelPatch/APatch KPM -- needs a KernelPatch-patched boot image + plus a bare-metal ARM64 toolchain (aarch64-none-elf-gcc, NOT NDK) + lkm out-of-tree module -- needs kernel source matching the device's + exact vermagic, and on >= 5.7 a way to resolve unexported symbols + userspace a Magisk/KernelSU-format module skeleton -- an honest carrier for + the config and the install path; it CANNOT itself arm a watchpoint + on another process, and says so in its own README + +Measured vs unverified +---------------------- +`gates`, `generate` and `verify` run on the host +(docs/tool-verification/EXTENSION-kernel-instrument.md). **No kernel-side file +has been compiled or loaded anywhere, by anyone in this repository's passes.** +Every generated kernel file says so in its own header. The register-rewrite +action is the most experimental part of the design: rewriting user pt_regs in a +perf overflow callback is mechanically what a ptrace stop does, but this +repository never ran it, and on some paths the callback runs with interrupts +disabled where you must not sleep. Treat the template as a starting point that +removes the blank page, never as a weapon that has been fired. + +Examples + python kernel_instrument_scaffold.py gates + python kernel_instrument_scaffold.py generate --out work/kwatch --pid 12345 \\ + --watch 0x7b2000123456:8:w + python kernel_instrument_scaffold.py generate --out work/kwatch --pid 12345 \\ + --break 0x7b20001234c0 --set-reg x0=0x1 + python kernel_instrument_scaffold.py verify --dir work/kwatch +""" +import argparse +import json +import os +import re +import shutil +import sys + +UNVERIFIED = ( + "TEMPLATE, NOT COMPILED AND NOT LOADED. This file was generated by " + "skills/apk-reverse/scripts/kernel_instrument_scaffold.py and has never " + "been built or loaded on any device by this repository. Verify every " + "symbol name, signature and export against the kernel/KPM headers you " + "build against before trusting it." +) + +GATES = [ + {"route": "KernelPatch / APatch KPM", + "gate": "KernelPatch-patched boot image + aarch64-none-elf-gcc", + "why": "KPMs load through kpimg injected into the kernel image; building " + "one needs a bare-metal ARM64 compiler, not the NDK.", + "check": "ls /data/adb/ | grep -i apatch ; which aarch64-none-elf-gcc"}, + {"route": "Out-of-tree LKM", + "gate": "kernel source matching exact vermagic", + "why": "register_hw_breakpoint/perf_event_create_kernel_counter are " + "GPL-exported, but find_task_by_vpid and friends are not exported " + "on every kernel, and a vermagic mismatch is refused at load time.", + "check": "uname -r ; ls /proc/sys/kernel/ ; find matching kernel source"}, + {"route": "userspace carrier module", + "gate": "any root (Magisk/KernelSU)", + "why": "carries config and documents the route; cannot itself program " + "another process's debug registers.", + "check": "ls /data/adb/modules"}, + {"route": "perf hw-breakpoint slots", + "gate": "the kernel reserves debug slots for perf (CONFIG_HW_PERF_EVENTS)", + "why": "the kernel's perf subsystem owns the arm64 debug registers; if " + "perf's breakpoint layer is not compiled in, register_hw_breakpoint " + "does not exist and this whole route closes.", + "check": "grep -c hw_breakpoint /proc/kallsyms (root) ; ls /sys/bus/event_source/devices/breakpoint"}, +] + + +def parse_spec_watch(tok): + """w:0xADDR:LEN:ACCESS -> dict; validated with hw_watch-style rules.""" + parts = tok.split(":") + if len(parts) != 4 or parts[0] != "w": + raise ValueError("watch spec must be w:0xADDR:LEN:ACCESS, got %r" % tok) + addr = int(parts[1], 0) + length = int(parts[2], 0) + access = parts[3] + if length not in range(1, 9): + raise ValueError("watch length must be 1..8") + if access not in ("r", "w", "rw"): + raise ValueError("watch access must be r/w/rw") + if (addr & 7) + length > 8: + raise ValueError("watch 0x%x+%d crosses its aligned 8-byte window" + % (addr, length)) + return {"addr": "0x%x" % addr, "len": length, "access": access} + + +def parse_spec_break(tok, set_reg=None): + """b:0xADDR[:reg=value] -> dict.""" + parts = tok.split(":") + if not parts or parts[0] != "b" or len(parts) < 2: + raise ValueError("break spec must be b:0xADDR, got %r" % tok) + addr = int(parts[1], 0) + if addr & 3: + raise ValueError("break PC 0x%x is not 4-byte aligned" % addr) + row = {"pc": "0x%x" % addr} + if set_reg: + m = re.match(r"^(x\d+)=(0x[0-9a-fA-F]+|\d+)$", set_reg) + if not m or int(m.group(1)[1:]) > 30: + # x31 is sp's slot in user_pt_regs and the array ends at x30: an + # index past 30 writes sp, pc, pstate, or off the frame. sp/pc are + # refused too: the template has no rewrite path for them, and a + # generate that "succeeds" while silently dropping the action is + # the misleading kind of success + raise ValueError("--set-reg must be x0..x30= (sp/pc have no " + "rewrite path in the template), got %r" % set_reg) + row["set_reg"] = m.group(1) + row["set_value"] = m.group(2) + return row + + +# --------------------------------------------------------------------------- +# templates +# --------------------------------------------------------------------------- +KPM_C = r'''// SPDX-License-Identifier: GPL-2.0 +/* + * kwatch.c -- invisible hardware watchpoints + register rewrite at trap, + * as a KernelPatch / APatch KPM. + * + * {unverified} + * + * Design notes that are load-bearing: + * - The arm64 debug registers are owned by the kernel perf/breakpoint + * layer. A KPM does not poke DBGWVR/DBGWCR directly (it cannot, from + * kernel C code safely); it asks perf to create a breakpoint event on the + * target task, and perf programs the slots at context switch. + * - The overflow callback receives the interrupted context's pt_regs. That + * is where the "event injection" half lives: rewriting regs->regs[n] + * changes what the userspace instruction stream sees after the trap. + * This is mechanically what a ptrace stop does -- but it was never run + * here, and on some interrupt contexts you must not sleep or allocate. + * LOGGING IS SAFE; REGISTER REWRITING IS EXPERIMENTAL. + * - TracerPid stays 0: no ptrace relationship is ever created. + */ +#include +#include +#include +#include +#include + +KPM_NAME("{id}"); +KPM_VERSION("{version}"); +KPM_LICENSE("GPL"); +KPM_AUTHOR("apk-reverse"); +KPM_DESCRIPTION("config-driven invisible watchpoints on one task"); + +/* kfunc declarations must NOT carry `extern` (see kernelsu_syscall_mask.py's + * KPM notes): tentative definitions allocate .bss slots the loader fills. */ +int kfunc_def(find_task_by_vpid)(int vnr); +int kfunc_def(perf_event_create_kernel_counter)( + struct perf_event_attr *attr, int cpu, void *task, + void *overflow_handler, void *context); +int kfunc_def(perf_event_release_kernel)(void *event); + +static struct perf_event *events[{max_events}]; +static int nevents = 0; + +struct wp_cfg {{ + u64 addr; int len; int bp_type; const char *label; +}}; + +/* PERF_TYPE_BREAKPOINT = 5; bp_type: 1=read 2=write 4=execute (uapi) */ +static struct wp_cfg cfg[] = {{ +{cfg_rows} +}}; + +static void wp_hit(struct perf_event *event, void *data, struct pt_regs *regs) +{{ + /* Runs on every hit, in interrupt context on the hitting CPU, for EVERY + * event registered below -- watch and break alike. The generated body is + * an UNFILTERED printk: add a cheap first-statement filter (event/addr) + * before treating any hit as signal on a hot path. */ + printk(KERN_INFO "KWATCH hit pc=0x%llx x0=0x%llx x1=0x%llx\n", + (unsigned long long)regs->pc, + (unsigned long long)regs->regs[0], + (unsigned long long)regs->regs[1]); +{inject_rows} +}} + +static int kwatch_init(void) {{ + struct task_struct *task = (void *)(unsigned long)kfunc_find_task_by_vpid({pid}); + if (!task) {{ + printk(KERN_ERR "KWATCH: no task pid={pid}\n"); + return -1; + }} + for (int i = 0; i < sizeof(cfg)/sizeof(cfg[0]); i++) {{ + struct perf_event_attr attr; + memset(&attr, 0, sizeof(attr)); + attr.type = 5 /* PERF_TYPE_BREAKPOINT */; + attr.sample_period = 1; /* hw_breakpoint_init's firing line: a period + * of 0 is a non-sampling event and wp_hit + * never runs -- armed but silent */ + attr.bp_type = cfg[i].bp_type; + attr.bp_addr = cfg[i].addr; + attr.bp_len = cfg[i].len; + attr.size = sizeof(attr); + attr.disabled = 0; + attr.inherit = 0; + events[nevents] = (void *)(unsigned long) + kfunc_perf_event_create_kernel_counter( + &attr, -1 /* any cpu */, task, wp_hit, NULL); + if (IS_ERR_OR_NULL(events[nevents])) {{ + printk(KERN_ERR "KWATCH: slot %d failed (no slots left? perf bp layer off?)\n", i); + events[nevents] = NULL; + }} else {{ + nevents++; + }} + }} + printk(KERN_INFO "KWATCH armed %d events on pid={pid}\n", nevents); + return 0; +}} + +static void kwatch_exit(void) {{ + for (int i = 0; i < nevents; i++) + if (events[i]) + kfunc_perf_event_release_kernel(events[i]); + printk(KERN_INFO "KWATCH released %d events\n", nevents); +}} + +KPM_INIT(kwatch_init); +KPM_EXIT(kwatch_exit); +''' + +LKM_C = r'''// SPDX-License-Identifier: GPL-2.0 +/* + * kwatch.c -- invisible hardware watchpoints on one task, as an out-of-tree LKM. + * + * {unverified} + * + * Gates that decide whether this file is even buildable: kernel source + * matching the device's exact vermagic; and on >= 5.7 the unexported-symbol + * problem (find_task_by_vpid is not exported on every tree -- resolve it or + * take the pid from a syscall parameter instead). + * Build: make -C M=$PWD modules + */ +#include +#include +#include +#include +#include +#include + +struct wp_cfg {{ + u64 addr; int len; int bp_type; +}}; +static struct wp_cfg cfg[] = {{ +{cfg_rows} +}}; +static struct perf_event *events[{max_events}]; +static int nevents = 0; + +static void wp_hit(struct perf_event *event, struct perf_sample_data *data, + struct pt_regs *regs) +{{ + printk(KERN_INFO "KWATCH hit pc=0x%llx x0=0x%llx x1=0x%llx\n", + (unsigned long long)regs->pc, + (unsigned long long)regs->regs[0], + (unsigned long long)regs->regs[1]); +{inject_rows} +}} + +static int __init kwatch_init(void) +{{ + struct task_struct *task; + rcu_read_lock(); + task = pid_task(find_vpid({pid}), PIDTYPE_PID); + if (task) + /* pin the task past the RCU section: registration below can sleep, + * and an unpinned pointer can be freed after a task exit in between */ + get_task_struct(task); + rcu_read_unlock(); + if (!task) {{ + pr_err("KWATCH: no task pid={pid}\n"); + return -ESRCH; + }} + for (int i = 0; i < ARRAY_SIZE(cfg); i++) {{ + struct perf_event_attr attr; + /* sets size/type/pinned/sample_period=1: a zero period is a + * non-sampling event whose overflow callback (wp_hit) never runs */ + hw_breakpoint_init(&attr); + attr.bp_type = cfg[i].bp_type; + attr.bp_addr = cfg[i].addr; + attr.bp_len = cfg[i].len; + /* the user-task API is (attr, handler, context, task): the per-cpu + * `register_wide_hw_breakpoint` family cannot arm on one target */ + events[nevents] = register_user_hw_breakpoint( + &attr, wp_hit, NULL, task); + if (IS_ERR(events[nevents])) {{ + pr_err("KWATCH: slot %d failed\n", i); + events[nevents] = NULL; + }} else {{ + nevents++; + }} + }} + pr_info("KWATCH armed %d events on pid={pid}\n", nevents); + put_task_struct(task); + return 0; +}} +module_init(kwatch_init); + +static void __exit kwatch_exit(void) +{{ + int i; + for (i = 0; i < nevents; i++) + if (events[i]) + unregister_hw_breakpoint(events[i]); + pr_info("KWATCH released %d events\n", nevents); +}} +module_exit(kwatch_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("config-driven invisible watchpoints on one task"); +''' + +BP_TYPE = {"r": "1 /* read */", "w": "2 /* write */", "rw": "3 /* rw */", "x": "4 /* execute */"} + +INJECT_LOG = " /* action: log only (safe in any context) */" +INJECT_SET = (" /* action: set-reg -- EXPERIMENTAL, never run by this repository.\n" + " * Rewriting the interrupted frame's register is what a ptrace\n" + " * stop does mechanically; doing it from a perf overflow callback\n" + " * was not tested anywhere here. The rewrite is guarded by its\n" + " * breakpoint's own event so a watchpoint hit in this same callback\n" + " * cannot run it. */\n" + " if (event && (event->attr.bp_type & 4 /* HW_BREAKPOINT_X */)\n" + " && event->attr.bp_addr == {addr}) {{\n" + " regs->regs[{idx}] = {value};\n" + " }}") + + +def render_cfg_rows(watches, breaks, for_kpm): + """Spec dicts -> wp_cfg C rows for the kernel template.""" + rows = [] + for w in watches: + bp = BP_TYPE[w["access"]] + rows.append(' { .addr = %s, .len = %d, .bp_type = %s }, /* watch %s:%s */' + % (w["addr"], w["len"], bp, w["addr"], w["access"])) + for b in breaks: + # wp_cfg declares addr/len/bp_type and has no `pc` member; an execution + # breakpoint is .addr with bp_type HW_BREAKPOINT_X (4) and the machine + # word (8 on arm64) as its length + rows.append(' { .addr = %s, .len = 8, .bp_type = 4 /* HW_BREAKPOINT_X */ },' + ' /* execute breakpoint */' + % b["pc"]) + return "\n".join(rows) + + +def render_inject_rows(breaks): + """Break actions -> guarded callback body rows.""" + parts = [INJECT_LOG] + for b in breaks: + if b.get("set_reg"): + reg = b["set_reg"] + idx = reg[1:] if reg.startswith("x") else None + if idx is None: + parts.append(" /* set-reg on %s not expressible in pt_regs by index; " + "extend the template */" % reg) + continue + parts.append(INJECT_SET.format(addr=b["pc"], idx=idx, value=b["set_value"])) + return "\n".join(parts) + + +KPM_MAKEFILE = '''# {unverified} +all: {id}.kpm +{id}.kpm: src/{id}.c +\taarch64-none-elf-gcc -O2 -fno-stack-protector -fPIC -c src/{id}.c -o {id}.o +\t# link per the KernelPatch module docs of your revision into a relocatable ELF +clean: +\trm -f *.o *.kpm +''' + +LKM_MAKEFILE = '''# {unverified} +obj-m += {id}.o +all: +\tmake -C $KSRC M=$$PWD modules +''' + +MODULE_PROP = """id={id} +name={name} +version={version} +versionCode=1 +author=apk-reverse +description={description} +""" + +USERSPACE_README = """# {name} (userspace carrier) + +This module exists to carry `config/instrument.json` and the build/load notes +for the kernel-side artifact. **It cannot arm a watchpoint on another process** +and it does not try: nothing in module.prop/post-fs-data.sh/service.sh ever +runs in the kernel's path. The kernel half (kpm/ or lkm/) is the part that +programs the target's debug registers; it is a template that has never been +compiled or loaded by this repository. + +Layout: + config/instrument.json the watch/break spec + pid (edit, then regenerate + the kernel file rather than hand-editing it) + kpm/, lkm/ generated kernel-side templates + Makefiles + post-fs-data.sh prints the config path into the log, nothing else + +Discipline (references/kernel-event-injection.md): a template is `unverified` +until a device with the matching gate compiles and loads it; TracerPid stays 0 +only because no ptrace relationship exists -- verify that claim on the device +before relying on it. +""" + + +def cmd_gates(args): + """gates subcommand: host probes plus the device-side gates.""" + clang = shutil.which("aarch64-none-elf-gcc") + if args.json: + print(json.dumps({"status": "ok", + "gates": GATES, + "aarch64_none_elf_gcc": bool(clang)}, indent=1)) + return 0 + print("== kernel-side instrumentation routes and their gates ==\n") + for g in GATES: + print(" %-32s %s" % (g["route"] + ":", g["gate"])) + print(" why: %s" % g["why"]) + print(" check: %s\n" % g["check"]) + print("== host probe ==") + print(" aarch64-none-elf-gcc: %s" % ("FOUND" if clang else "NOT FOUND -- KPM route cannot be built from this host")) + print("\nRESULT=ok") + return 0 + + +def cmd_generate(args): + """generate subcommand: write the scaffold under --out.""" + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", args.id or ""): + # --id becomes file names, Makefile targets and module.prop entries; + # a path separator or leading dot would write outside --out + print("RESULT=refused") + print("--id must be a module id ([A-Za-z0-9][A-Za-z0-9._-]*): no path " + "separators, no leading dot, got %r" % args.id, file=sys.stderr) + return 2 + watches = [parse_spec_watch(w) for w in (args.watch or [])] + breaks = [parse_spec_break(b, args.set_reg) for b in (args.break_ or [])] + if not watches and not breaks: + print("RESULT=refused") + print("nothing to instrument: give --watch and/or --break", file=sys.stderr) + return 2 + if not args.pid: + print("RESULT=refused") + print("the kernel side needs a target --pid", file=sys.stderr) + return 2 + out = os.path.abspath(args.out) + os.makedirs(out, exist_ok=True) + targets = [t.strip() for t in args.targets.split(",") if t.strip()] + cfg_rows = render_cfg_rows(watches, breaks, for_kpm=True) + inject_rows = render_inject_rows(breaks) + n_events = max(1, len(watches) + len(breaks)) + written = [] + + def put(rel, body): + """Write one generated file under the root; record it.""" + path = os.path.join(out, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fh: + fh.write(body) + written.append(rel) + + if "kpm" in targets: + put("kpm/src/%s.c" % args.id, KPM_C.format( + id=args.id, version=args.version, unverified=UNVERIFIED, + cfg_rows=cfg_rows, inject_rows=inject_rows, pid=args.pid, + max_events=n_events)) + put("kpm/Makefile", KPM_MAKEFILE.format(id=args.id, unverified=UNVERIFIED)) + if "lkm" in targets: + put("lkm/%s.c" % args.id, LKM_C.format( + unverified=UNVERIFIED, cfg_rows=cfg_rows, + inject_rows=inject_rows, pid=args.pid, max_events=n_events)) + put("lkm/Makefile", LKM_MAKEFILE.format(id=args.id, unverified=UNVERIFIED)) + if "userspace" in targets: + put("module.prop", MODULE_PROP.format( + id=args.id, name=args.name, version=args.version, + description=args.description)) + put("README.md", USERSPACE_README.format(name=args.name)) + put("post-fs-data.sh", "#!/system/bin/sh\n" + "# carrier only -- see README.md; cannot arm watchpoints from here\n" + "MODDIR=${0%/*}\n" + "echo \"$(date) config at $MODDIR/config/instrument.json\" " + ">> /data/adb/" + args.id + ".log\n") + put("config/instrument.json", json.dumps({ + "pid": args.pid, + "watches": watches, + "breaks": breaks, + "note": "regenerate the kernel-side file rather than hand-editing it", + }, indent=1)) + print("wrote %s" % out) + for rel in written: + print(" %s" % rel) + print("RESULT=ok") + return 0 + + +def cmd_verify(args): + """verify subcommand: consistency checks over a generated dir.""" + problems = [] + root = os.path.abspath(args.dir) + expected = ["config/instrument.json"] + for rel in expected: + if not os.path.isfile(os.path.join(root, rel)): + problems.append("missing %s" % rel) + kernel_files = [] + for sub in ("kpm", "lkm"): + d = os.path.join(root, sub) + if not os.path.isdir(d): + continue + for base, _dirs, files in os.walk(d): + for fn in files: + if fn.endswith(".c") or fn == "Makefile": + kernel_files.append(os.path.relpath(os.path.join(base, fn), root)) + for rel in kernel_files: + with open(os.path.join(root, rel), "r", encoding="utf-8") as fh: + body = fh.read() + if "NOT COMPILED AND NOT LOADED" not in body: + problems.append("%s lost its unverified banner" % rel) + cfg_path = os.path.join(root, "config", "instrument.json") + if os.path.isfile(cfg_path): + try: + with open(cfg_path, "r", encoding="utf-8") as fh: + cfg = json.load(fh) + if not cfg.get("pid"): + problems.append("config carries no pid") + except ValueError as exc: + problems.append("config/instrument.json is not JSON: %s" % exc) + if args.json: + print(json.dumps({"status": "ok" if not problems else "problems", + "problems": problems, "kernel_files": kernel_files})) + else: + for p in problems: + print(" problem: %s" % p) + print("verified %d kernel-side file(s), %d problem(s)" + % (len(kernel_files), len(problems))) + print("RESULT=%s" % ("ok" if not problems else "problems")) + return 0 if not problems else 1 + + +def main(): + """CLI entry: subcommand dispatch.""" + ap = argparse.ArgumentParser( + description="Generate the kernel-side invisible-instrumentation scaffold: " + "watchpoint/breakpoint templates (KPM/LKM) + userspace carrier, " + "each with its gate and an explicit unverified label.", + epilog="references/kernel-event-injection.md owns the primitive map and the " + "composition recipes; kernel-and-environment-hardening.md owns the " + "hiding-vs-instrumentation split and the stop rules.") + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("gates", help="print the route/gate table and probe this host") + p.add_argument("--json", action="store_true") + p.set_defaults(fn=cmd_gates) + + p = sub.add_parser("generate", help="write the scaffold") + p.add_argument("--out", required=True, help="directory to write into") + p.add_argument("--pid", type=int, help="target pid (kernel side needs it)") + p.add_argument("--watch", action="append", metavar="w:0xADDR:LEN:ACCESS", + help="hardware watchpoint spec (repeatable)") + p.add_argument("--break", dest="break_", action="append", metavar="b:0xADDR", + help="execution breakpoint PC (repeatable)") + p.add_argument("--set-reg", metavar="REG=VALUE", + help="rewrite this register at the breakpoint trap (e.g. x0=0x1) -- " + "the injection action; experimental, see the script header") + p.add_argument("--id", default="kwatch", help="module id / artifact name") + p.add_argument("--name", default="Kernel Watch", help="human-readable module name") + p.add_argument("--version", default="0.1.0") + p.add_argument("--description", default="config-driven invisible watchpoints + " + "event injection templates") + p.add_argument("--targets", default="kpm,lkm,userspace", + help="comma list from kpm,lkm,userspace") + p.set_defaults(fn=cmd_generate) + + p = sub.add_parser("verify", help="read-only consistency checks over a generated dir") + p.add_argument("--dir", required=True) + p.add_argument("--json", action="store_true") + p.set_defaults(fn=cmd_verify) + + args = ap.parse_args() + try: + return args.fn(args) + except ValueError as exc: + print("RESULT=refused") + print(str(exc), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/apk-reverse/scripts/trace_addr2sym.py b/skills/apk-reverse/scripts/trace_addr2sym.py new file mode 100644 index 0000000..0c0eb2a --- /dev/null +++ b/skills/apk-reverse/scripts/trace_addr2sym.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +"""trace_addr2sym.py -- turn a stream of runtime addresses into symbols and static offsets. + +What question this answers +-------------------------- +Every hardware-layer tool in this skill ends with a list of addresses: `hw_watch.py` hits, +ETE/SPE block streams, `trace_diff.py` divergences, BRBE branch records. An address is not a +finding until it names **which library, which function, which static offset** -- the last part +being what you paste into IDA/Ghidra, where ASLR never existed. This script closes that loop +with a zero-dependency ELF64 reader (no pyelftools, no readelf): PT_LOAD segments for +runtime-address -> vaddr -> file-offset conversion, `.dynsym` first then `.symtab`, and a +stripped-library fallback that still gives section + static offsets. + +PIE and the two offset spaces, stated once +------------------------------------------ +- **runtime address** = what the trace/hit carries (ASLR-affected) +- **vaddr** = runtime - load_bias, where load_bias is derived from the module's + lowest mapping aligned against its PT_LOAD (bias = map_start - (map_foff - + p_offset) - p_vaddr): it equals the mapping base exactly when the lowest + PT_LOAD has p_vaddr 0, the normal PIE case (`lib_map.py`/maps give you this) +- **file offset** = what the disassembler wants; vaddr -> file goes through PT_LOAD again, + and it is **not** always the identity: p_vaddr != p_offset happens in hand-linked and + self-decrypting payloads, and silently wrong offsets are exactly the failure this tool + exists to prevent. +- `mod+0xoff` keys (stalker logs, trace_diff output) are runtime - base == vaddr under the + same assumption; use `--trace --mod --elf ` for that input shape. + +Measured vs unverified: the ELF walker is host-tested against a real cross-compiled +aarch64 shared library (gcc, `.dynsym` + `.symtab`) and against a hand-built minimal ELF +(tests/unit/test_trace_addr2sym.py) -- `observed`. Nothing here touches a device. + +Examples + python trace_addr2sym.py --hits hwwatch_hits.log --maps maps.txt --elf liba.so + python trace_addr2sym.py --trace traceA.log --mod liba.so --elf liba.so + python trace_addr2sym.py --elf liba.so --addrs 0x7b20001234c0 --base 0x7b2000000000 + python trace_addr2sym.py --elf liba.so --offs 0x1234c0 # module offsets directly +""" +import argparse +import bisect +import json +import os +import struct +import sys + +PT_LOAD = 1 +SHT_SYMTAB = 2 +SHT_DYNSYM = 11 +STT_FUNC = 2 +STT_OBJECT = 1 + + +class ElfError(ValueError): + pass + + +class Elf64: + """Zero-dependency ELF64 little-endian reader: segments, sections, symbols.""" + + def __init__(self, path): + """Read the file; build segments, sections and the symbol table.""" + with open(path, "rb") as fh: + self.data = fh.read() + if self.data[:4] != b"\x7fELF": + raise ElfError("%s is not an ELF" % path) + if self.data[4] != 2 or self.data[5] != 1: + raise ElfError("%s is not ELF64 little-endian" % path) + (self.e_type, self.e_machine) = struct.unpack_from(" off else "" + + def _read_symbols(self): + """Flatten .symtab/.dynsym into symbol rows.""" + out = [] + for sec in self.sections: + if sec["type"] not in (SHT_SYMTAB, SHT_DYNSYM) or not sec["entsize"]: + continue + strtab = None + if 0 < sec["link"] < len(self.sections): + strtab = self.sections[sec["link"]]["offset"] + count = sec["size"] // sec["entsize"] if sec["entsize"] else 0 + for i in range(count): + off = sec["offset"] + i * sec["entsize"] + (st_name, st_info) = struct.unpack_from(" file offset through a PT_LOAD, or None.""" + for p_vaddr, end, p_offset in self.segments: + if p_vaddr <= vaddr < end: + return p_offset + (vaddr - p_vaddr) + return None + + def file_to_vaddr(self, foff): + """file offset -> vaddr through a PT_LOAD, or None.""" + for p_vaddr, end, p_offset in self.segments: + if p_offset <= foff < p_offset + (end - p_vaddr): + return p_vaddr + (foff - p_offset) + return None + + def section_of(self, vaddr): + """(section name, offset inside it) for a vaddr.""" + for s in self.sections: + # section addr is not stored in this minimal reader; fall back by file offset + foff = self.vaddr_to_file(vaddr) + if foff is not None and s["offset"] <= foff < s["offset"] + s["size"] and s["name"]: + return s["name"], foff - s["offset"] + return None, None + + def resolve(self, vaddr): + """-> (symbol, sym_off) or (None, None). Nearest symbol at-or-below + vaddr, addr2line-style: a PC between two sized symbols reports the + lower one, because a trace PC in a gap (alignment padding, cold tails, + stripped locals) still owes you its neighbourhood. Callers that must + not attribute foreign PCs should gate on contains_vaddr() first.""" + if not self.funcs: + return None, None + i = bisect.bisect_right(self.symbol_addresses, vaddr) - 1 + if i < 0: + return None, None + value, _size, name = self.funcs[i] + return name, vaddr - value + + +def load_bias_from_maps(maps_path, module_hint, elf): + """Load bias for the module matched by full pathname or basename ONLY. + + A substring match would also admit `liba.so`'s neighbours + (`liba.so.v2`, `xliba.so`) and take their mapping. And the bias is not + simply the lowest mapping start -- that holds only when the lowest + PT_LOAD has p_vaddr 0. Align the mapping's file offset with a PT_LOAD + segment instead: bias = map_start - (map_foff - p_offset) - p_vaddr. + """ + best = None # (lowest mapping start, its file offset) + with open(maps_path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + parts = line.split(None, 5) + if len(parts) >= 6: + path = parts[5].strip() + if module_hint == path or module_hint == os.path.basename(path): + start = int(parts[0].split("-")[0], 16) + foff = int(parts[2], 16) + if best is None or start < best[0]: + best = (start, foff) + if best is None: + return None + start, foff = best + for p_vaddr, end, p_offset in elf.segments: + if p_offset <= foff < p_offset + (end - p_vaddr): + return start - (foff - p_offset) - p_vaddr + return None # maps and ELF disagree: the file offset is not one of its segments + + +def read_hw_hits(path): + """hw_watch.py HIT log -> ('hits', pc) rows.""" + rows = [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + if line.startswith("HIT"): + for tok in line.split(): + if tok.startswith("pc="): + rows.append(("hits", tok[3:])) + return rows + + +def read_addr_lines(path): + """One address per line -> ('addrs', tok) rows.""" + rows = [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + toks = line.split() + if not toks: + continue + tok = None + for t in reversed(toks): + s = t[2:] if t.lower().startswith("0x") else t + try: + int(s, 16) + tok = t + break + except ValueError: + continue + if tok: + rows.append(("addrs", tok)) + return rows + + +def read_off_lines(path): + """One module offset per line -> ('offs', tok) rows.""" + rows = [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + for tok in line.split(): + s = tok[2:] if tok.lower().startswith("0x") else tok + try: + int(s, 16) + except ValueError: + continue + rows.append(("offs", tok)) + break + return rows + + +def read_modoff_trace(path, mod): + """mod+0xoff keys from a stalker/trace log -- only keys for `mod`.""" + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import trace_diff + rows = [] + try: + blocks = trace_diff.read_blocks(path) + except Exception as exc: # noqa: BLE001 -- surfaced as a usage error below + raise ElfError(str(exc)) + for key in blocks: + if "+" in key: + name, _, off = key.rpartition("+") + if name == mod or os.path.basename(name) == mod: + rows.append(("offs", off)) + return rows + + +def describe(elf, vaddr): + """Name one vaddr, refusing PCs that are outside this ELF's span: pinning + another module's PC on this ELF's last symbol is the misleading output, + not a helpful guess.""" + if not elf.contains_vaddr(vaddr): + return (" vaddr 0x%x (outside every PT_LOAD span of this ELF " + "-- another module's PC, or a wrong base?)" % vaddr) + sym, sym_off = elf.resolve(vaddr) + foff = elf.vaddr_to_file(vaddr) + sec, sec_off = elf.section_of(vaddr) + if sym: + return "%s+0x%x [vaddr 0x%x file 0x%s%s]" % ( + sym, sym_off, vaddr, + ("%x" % foff) if foff is not None else "?", + (" %s+0x%x" % (sec, sec_off)) if sec else "") + if sec: + return " %s+0x%x [vaddr 0x%x file 0x%s]" % ( + sec, sec_off, vaddr, ("%x" % foff) if foff is not None else "?") + return " vaddr 0x%x (outside every PT_LOAD -- wrong base?)" % vaddr + + +def cmd_main(args): + """Resolve input rows against the ELF; text or --json output.""" + rows = [] + if args.hits: + rows = read_hw_hits(args.hits) + elif args.trace: + rows = read_modoff_trace(args.trace, args.mod) + elif args.addrs: + rows = [("addrs", a) for a in args.addrs] + elif args.offs: + rows = [("offs", a) for a in args.offs] + elif args.addr_file: + rows = read_addr_lines(args.addr_file) + elif args.off_file: + rows = read_off_lines(args.off_file) + if not rows: + print("RESULT=refused") + print("no addresses: give --hits/--trace/--addrs/--offs/--addr-file/--off-file", + file=sys.stderr) + return 2 + try: + elf = Elf64(args.elf) + except (OSError, ElfError) as exc: + print("RESULT=refused") + print(str(exc), file=sys.stderr) + return 2 + base = args.base + if base is not None: + base = int(base, 0) + if base is None and args.maps: + hint = args.mod or os.path.basename(args.elf) + base = load_bias_from_maps(args.maps, hint, elf) + if base is None: + print("RESULT=refused") + print("no load bias for %r in %s (no exact mapping matched, or its file " + "offset does not align with %s's PT_LOADs)" + % (hint, args.maps, os.path.basename(args.elf)), file=sys.stderr) + return 2 + if args.trace and base is None: + base = 0 # mod+off keys already are module-relative: treat them as vaddr + out = [] + for kind, tok in rows: + val = int(tok, 16) if not tok.lower().startswith("0x") else int(tok, 16) + if kind in ("offs",): + vaddr = val if not args.offs_are_file else elf.file_to_vaddr(val) + if vaddr is None: + out.append({"input": tok, "error": "file offset outside PT_LOAD"}) + continue + else: + if base is None: + print("RESULT=refused") + print("absolute addresses need --base or --maps (ASLR: the trace address " + "means nothing without the module's load base)", file=sys.stderr) + return 2 + vaddr = val - base + out.append({"input": tok, "vaddr": vaddr, "where": describe(elf, vaddr)}) + if args.json: + # a pure JSON document on stdout -- the status field carries what + # RESULT= would say, and appending the marker would break every + # json.loads(stdout) consumer (the convention check_commands.py set) + print(json.dumps({"status": "ok", "elf": args.elf, + "base": base, "resolved": out}, indent=1)) + else: + print("elf=%s base=%s%s" % (args.elf, hex(base) if base is not None else "?", + " (from --offs: offsets treated as %s)" + % ("file offsets" if args.offs_are_file else "vaddrs") + if args.offs else "")) + for row in out: + if "error" in row: + print(" %-16s ERROR %s" % (row["input"], row["error"])) + else: + print(" %-18s -> %s" % (row["input"], row["where"])) + print("RESULT=ok") + return 0 + + +def main(): + """CLI entry: subcommand dispatch.""" + ap = argparse.ArgumentParser( + description="Runtime addresses -> symbol + static offset, zero-dependency ELF64: " + "PT_LOAD conversion, .dynsym/.symtab, stripped fallback.", + epilog="references/hardware-sampling-and-crash.md owns the crash/address loop this " + "closes; the offset-space rules are in this header because a silently wrong " + "offset is the failure mode.") + ap.add_argument("--elf", required=True, help="the .so/.o to resolve against") + ap.add_argument("--hits", help="hw_watch.py HIT log (absolute PCs)") + ap.add_argument("--trace", help="stalker/trace log; use keys for --mod only") + ap.add_argument("--addrs", nargs="*", help="absolute runtime addresses (0x..)") + ap.add_argument("--offs", nargs="*", help="module offsets (vaddr space, or --offs-are-file)") + ap.add_argument("--addr-file", help="file of one address per line") + ap.add_argument("--off-file", help="file of one module offset per line") + ap.add_argument("--mod", help="module name for --trace keys / --maps matching") + ap.add_argument("--maps", help="/proc//maps to find the load base from") + ap.add_argument("--base", help="explicit load base (overrides --maps)") + ap.add_argument("--offs-are-file", action="store_true", + help="--offs carries file offsets, not vaddrs") + ap.add_argument("--json", action="store_true") + args = ap.parse_args() + try: + return cmd_main(args) + except BrokenPipeError: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/apk-reverse/scripts/trace_diff.py b/skills/apk-reverse/scripts/trace_diff.py new file mode 100644 index 0000000..90f7657 --- /dev/null +++ b/skills/apk-reverse/scripts/trace_diff.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +"""trace_diff.py -- differential execution-trace reduction: two runs, one answer. + +What question this answers +-------------------------- +Not "what executed" but "**where did run A stop behaving like run B**". You run +the same operation twice with exactly one input changed -- the password's last +character, license valid versus invalid, key 0x00... versus key 0xff... -- and +diff the two execution traces. Every common block is machinery; every divergent +block is a decision that *your one-bit input difference* reached. Against OLLVM +control-flow flattening the divergences name the dispatcher states actually +taken; against a VM/VMP interpreter they name the handler sequence for the two +inputs; against whitebox-style crypto they name the key-dependent branch set. +This is the "differential" half of `references/ete-diff-tracing.md`, and it is +producer-agnostic by design. + +What it accepts (auto-detected, or forced with --format) +------------------------------------------------------- +stalker `stalker_trace.js` logs: `BLK +0x` block events + (BB/CALL/DONE lines ignored), including logs wrapped by + `run_probe.py` as `[host ts] [dev ts] TRACE BLK ...`. +plain any text where each line carries an address -- the last + whitespace token that parses as hex (`0x7f...`, `7f...`, and + `mod+0xoff` keys kept verbatim so modules stay comparable). + Decoded CoreSight/ETE `perf script` output fits this shape, as do + hand-scrapped disassembly walks. + +What it produces +---------------- +reduce the run-length form of one trace: ordered `(block, count)` runs plus + the execution histogram -- the same reductions `stalker_report.py` + prints, in a form the differ consumes. +diff an alignment of the two run-length sequences (difflib on the block + keys): identical prefix/suffix lengths, **first divergence**, each + divergent region with per-side counts, and the blocks unique to each + side ranked by count. The unique-to-B list is where "what does the + valid path do that the invalid path does not" is answered. + +Discipline the tool does not enforce for you +-------------------------------------------- +Same build and same address space (module+offset keys, never raw pointers, +unless ASLR is off); same input length; same capture window; and a *pair* whose +only difference is the input bit you chose. A diff across two builds or two +code versions is noise, and the tool cannot tell -- garbage in, confident +nonsense out. + +Measured status: the parsers and the reducer/differ are host-tested on +synthetic flattened-dispatcher fixtures (tests/unit/test_trace_diff.py). No +ETE-produced trace has been fed to it by this repository's passes -- see +references/ete-diff-tracing.md for the producer side and its device gates. + +Examples + python trace_diff.py reduce --log traceA.log + python trace_diff.py diff --a traceA.log --b traceB.log --top 15 + python trace_diff.py diff --a wrong.log --b right.log --json +""" +import argparse +import difflib +import json +import re +import sys +from collections import Counter + +# A block key is a hex address: `0x`-prefixed (unambiguous), or a bare run +# of at least 8 hex digits that is NOT pure decimal -- an all-digit run is +# indistinguishable from a date, a PID range or a count, and a log holding +# only those must stay refused (an empty trace is a capture failure, not a +# result). Shorter bare runs would swallow PIDs and timestamp fragments. +HEX_TOK = re.compile(r"^(?:0x[0-9a-fA-F]+|(?=[0-9a-fA-F]*[a-fA-F])[0-9a-fA-F]{8,})$") +MOD_OFF = re.compile(r"^[A-Za-z0-9_.+-]+\+0x[0-9a-fA-F]+$") +BLK_LINE = re.compile(r"(?:^|\s)BLK\s+\d+\s+(\S+)(?:\s|$)") + + +class TraceError(ValueError): + pass + + +# --------------------------------------------------------------------------- +# parsing +# --------------------------------------------------------------------------- +def parse_line_stalker(line): + """One BLK line -> its block key, or None.""" + m = BLK_LINE.search(line) + return m.group(1) if m else None + + +def parse_line_plain(line): + """One plain trace line -> its block key, or None.""" + toks = line.split() + for tok in reversed(toks): + if MOD_OFF.match(tok): + return tok + if HEX_TOK.match(tok): + return tok.lower() + return None + + +PARSERS = {"stalker": parse_line_stalker, "plain": parse_line_plain} + + +def read_blocks(path, fmt=None): + """Ordered block keys from a trace log. Empty input raises: a diff against + nothing is a plumbing failure, not an analysis result (the zero-event + lesson from native-dbi-and-deobfuscation.md applies here too).""" + blocks = [] + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + if fmt: + key = PARSERS[fmt](line) + else: + key = parse_line_stalker(line) or parse_line_plain(line) + if key: + blocks.append(key) + if not blocks: + raise TraceError( + "no block keys parsed from %s -- an empty trace is a capture " + "failure, not a result; see the zero-event diagnostics in " + "references/native-dbi-and-deobfuscation.md" % path) + return blocks + + +def run_length(blocks): + """[(key, run_len)] preserving order -- D D D R1 becomes (D,3),(R1,1).""" + runs = [] + for b in blocks: + if runs and runs[-1][0] == b: + runs[-1][1] += 1 + else: + runs.append([b, 1]) + return [(k, n) for k, n in runs] + + +# --------------------------------------------------------------------------- +# reduction report +# --------------------------------------------------------------------------- +def reduce_trace(blocks): + """Histogram and run-length summary of one block list.""" + hist = Counter(blocks) + return { + "events": len(blocks), + "distinct_blocks": len(hist), + "runs": len(run_length(blocks)), + "top": hist.most_common(), + "sequence": run_length(blocks), + } + + +# --------------------------------------------------------------------------- +# differential +# --------------------------------------------------------------------------- +def diff_traces(a_blocks, b_blocks): + """Structural diff: first divergence, regions, uniques.""" + a_runs, b_runs = run_length(a_blocks), run_length(b_blocks) + a_keys = [k for k, _ in a_runs] + b_keys = [k for k, _ in b_runs] + sm = difflib.SequenceMatcher(a=a_keys, b=b_keys, autojunk=False) + + identical = sum(n for i, j, n in sm.get_matching_blocks()) + regions = [] + first_divergence = None + a_hist, b_hist = Counter(a_blocks), Counter(b_blocks) + + for tag, i1, i2, j1, j2 in sm.get_opcodes(): + if tag == "equal": + continue + if first_divergence is None: + first_divergence = { + "run_index_a": i1, "run_index_b": j1, + "event_index_a": sum(n for _, n in a_runs[:i1]), + "event_index_b": sum(n for _, n in b_runs[:j1]), + "last_common_block": a_runs[i1 - 1][0] if i1 > 0 else None, + } + regions.append({ + "tag": tag, + "a_runs": a_runs[i1:i2], + "b_runs": b_runs[j1:j2], + }) + + only_a = sorted(((k, v) for k, v in (a_hist - b_hist).items()), + key=lambda kv: -kv[1]) + only_b = sorted(((k, v) for k, v in (b_hist - a_hist).items()), + key=lambda kv: -kv[1]) + common = a_hist & b_hist + + return { + "a_events": len(a_blocks), "b_events": len(b_blocks), + "a_distinct": len(a_hist), "b_distinct": len(b_hist), + "shared_blocks": sum(common.values()), + "identical_run_span": identical, + "divergent_regions": len(regions), + "first_divergence": first_divergence, + "regions": regions, + "unique_to_a": only_a, + "unique_to_b": only_b, + } + + +def render_diff(d, top=10, context=3): + """Human-readable rendering of a diff_traces document.""" + out = [] + out.append("run A: %d events, %d distinct blocks" % (d["a_events"], d["a_distinct"])) + out.append("run B: %d events, %d distinct blocks" % (d["b_events"], d["b_distinct"])) + out.append("shared block executions: %d divergent regions: %d" + % (d["shared_blocks"], d["divergent_regions"])) + fd = d["first_divergence"] + if fd is None: + out.append("\nNO DIVERGENCE: the two runs executed the same block sequence.") + out.append("Either the input difference never reached control flow (pure data),") + out.append("or the two logs came from the same run. Check the producer.") + else: + out.append("") + out.append("first divergence: after run #%d (A) / #%d (B), " + "%d/%d events in, last common block %s" + % (fd["run_index_a"], fd["run_index_b"], + fd["event_index_a"], fd["event_index_b"], + fd["last_common_block"] or "")) + for idx, r in enumerate(d["regions"][:top], 1): + out.append("") + out.append("region %d (%s)" % (idx, r["tag"])) + out.append(" A: " + _fmt_runs(r["a_runs"])) + out.append(" B: " + _fmt_runs(r["b_runs"])) + if d["unique_to_a"] or d["unique_to_b"]: + out.append("") + out.append("blocks unique to A (input-dependent, A-side), top %d:" % top) + for k, n in d["unique_to_a"][:top]: + out.append(" %6d %s" % (n, k)) + out.append("blocks unique to B (input-dependent, B-side), top %d:" % top) + for k, n in d["unique_to_b"][:top]: + out.append(" %6d %s" % (n, k)) + return "\n".join(out) + + +def _fmt_runs(runs): + """Format a run list for the region renderer.""" + if not runs: + return "" + return " -> ".join("%s%s" % (k, ("*%d" % n) if n > 1 else "") for k, n in runs[:12]) + \ + (" ... (%d runs)" % len(runs) if len(runs) > 12 else "") + + +# --------------------------------------------------------------------------- +# subcommands +# --------------------------------------------------------------------------- +def cmd_reduce(args): + """reduce subcommand.""" + try: + blocks = read_blocks(args.log, args.format) + except (OSError, TraceError) as exc: + print("RESULT=refused") + print(str(exc), file=sys.stderr) + return 2 + rep = reduce_trace(blocks) + if args.json: + print(json.dumps({"status": "ok", "log": args.log, + "events": rep["events"], + "distinct_blocks": rep["distinct_blocks"], + "runs": rep["runs"], + "top": rep["top"][:args.top]}, indent=1)) + return 0 + print("%s: %d events, %d distinct blocks, %d runs" + % (args.log, rep["events"], rep["distinct_blocks"], rep["runs"])) + print("\ntop %d blocks (dispatcher candidate = the wide-margin top):" % args.top) + for k, n in rep["top"][:args.top]: + print(" %6d %s" % (n, k)) + print("RESULT=ok") + return 0 + + +def cmd_diff(args): + """diff subcommand.""" + try: + a = read_blocks(args.a, args.format) + b = read_blocks(args.b, args.format) + except (OSError, TraceError) as exc: + print("RESULT=refused") + print(str(exc), file=sys.stderr) + return 2 + d = diff_traces(a, b) + if args.json: + doc = dict(d) + doc["status"] = "ok" + doc["regions"] = d["regions"][:args.top] + doc["unique_to_a"] = d["unique_to_a"][:args.top] + doc["unique_to_b"] = d["unique_to_b"][:args.top] + print(json.dumps(doc, indent=1)) + return 0 + print(render_diff(d, top=args.top)) + print("RESULT=ok") + return 0 + + +def main(): + """CLI entry: subcommand dispatch.""" + ap = argparse.ArgumentParser( + description="Differential reduction of two execution traces (stalker or plain " + "address logs): first divergence, divergent regions, unique blocks.", + epilog="references/ete-diff-tracing.md owns the method: same build, same " + "window, one changed input bit -- and what the divergences mean " + "against OLLVM flattening and VMP handler loops.") + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("reduce", help="run-length + histogram of one trace") + p.add_argument("--log", required=True, help="trace log (stalker or plain)") + p.add_argument("--format", choices=sorted(PARSERS), help="force a parser (default: auto)") + p.add_argument("--top", type=int, default=20) + p.add_argument("--json", action="store_true") + p.set_defaults(fn=cmd_reduce) + + p = sub.add_parser("diff", help="align two traces and report the divergences") + p.add_argument("--a", required=True, help="trace log A (e.g. the failing/invalid run)") + p.add_argument("--b", required=True, help="trace log B (e.g. the passing/valid run)") + p.add_argument("--format", choices=sorted(PARSERS), help="force a parser (default: auto)") + p.add_argument("--top", type=int, default=10, help="regions / unique blocks to show") + p.add_argument("--json", action="store_true") + p.set_defaults(fn=cmd_diff) + + args = ap.parse_args() + try: + return args.fn(args) + except BrokenPipeError: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/apk-reverse/scripts/vm_handler_slice.py b/skills/apk-reverse/scripts/vm_handler_slice.py new file mode 100644 index 0000000..55c529d --- /dev/null +++ b/skills/apk-reverse/scripts/vm_handler_slice.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""vm_handler_slice.py -- slice a VM/VMP interpreter trace into handler segments +and recover the guest-opcode trajectory. + +What question this answers +-------------------------- +A trace of a virtualized function is a trace of the interpreter: one dispatch loop, +executed once per guest instruction, with the guest's work hidden inside per-opcode +handler code. Reading that trace block-by-block tells you nothing -- but **slicing it at +dispatcher entries** turns it back into a program: + + D h_a h_b D h_c D h_a h_b D ... (D = dispatcher, h_* = handler blocks) + -> segment "h_a h_b" (count 2), segment "h_c" (count 1) + -> trajectory: S(ab) S(c) S(ab) ... == the guest bytecode, one signature per opcode + +With a handler->opcode map (see below), the trajectory becomes the guest program's +opcode sequence: the disassembly of the VM you cannot read statically. + +Calibration source -- an open IR-level VM, in the shape this tool expects +------------------------------------------------------------------------ +This tool's segment model is calibrated against a public LLVM-IR virtualizer +(dpkg123/ollvm-aosp-toolchain, commits 63fec41f and 9b7cf75e, 2026-09 -- the `xvmp` +generic pass; source facts below are read from that patch, `inferred` for runtime +behaviour, `observed` for the source text): + +- each virtualized function becomes a stub calling ``vm_exec(code, size, vm_args, num)``: + the bytecode array plus a boxed-argument array live in the module, so the *inputs* + (arguments, and any key material passed in) are observable at the ``vm_args`` array + with a hardware watchpoint, before interpretation starts; +- the bytecode is a little-endian stream of one-byte opcodes with two operand spaces + (``VT_CONST`` inline constants, ``VT_SLOT`` slot references): OP_ALLOCA 0x01, + OP_LOAD 0x02, OP_STORE 0x03, OP_GEP 0x05, OP_CMP 0x06, OP_CAST 0x07, OP_BR 0x08, + OP_CALL 0x09, OP_RET 0x0A, OP_ARITH 0x10 (19 arithmetic sub-ops), OP_PARAMMAP 0x11; +- pointer constants are NOT inlined: they materialize through a generated dispatch + (HandleCallTool's const/call chain, `K_Addr/K_IntBits/K_FPBits`) at first use -- + a small set of extra blocks in the trace that is *not* the main interpreter loop, + and a high-value observation point (the returned i64 is the constant's bits); +- an interpreter is a loop, so the dispatcher dominates the block histogram, and each + guest opcode contributes a stable, repeating block subsequence -- which is exactly + the signature clustering below. + +Because that implementation's opcode table is public, `--opcode-map` can name every +segment; for a closed VMP the same slicing still yields the trajectory, and the map +becomes a manual labelling task over the (usually few dozen) unique segments. + +How to build --opcode-map +------------------------- +Run once, read the segment table (each segment's first block = handler entry), and map +handler-entry addresses to opcodes. For the open implementation above, match the entry +block to the code generated for each `case OP_*` by disassembling the `.so` around the +dispatch switch. Map file format: ``{"": "OP_LOAD", ...}``. + +Inputs and behaviour +-------------------- +- `--log` accepts the same grammars as `trace_diff.py` (stalker `BLK` lines, wrapped or + not, and any address-per-line stream), by importing its parsers. +- `--dispatcher` picks the slice boundary: `auto` (the histogram's wide-margin top -- + the same dispatcher-identification rule as `references/native-dbi-and-deobfuscation.md` + §5 From a trace to a deobfuscated function (inferred)) or an explicit block key. +- Segments are the block runs between dispatcher hits; identical runs get one signature + (sha1[:10]). Blocks before the first / after the last dispatcher hit are reported as + `preamble`/`tail`, never silently merged. +- Diagnostics guard the two ways this goes wrong: a dispatcher that is really a spin + loop (every segment length 1, unique-segment ratio ~1.0), and a trace that is not a + VM at all (no dominant block; native-dbi-and-deobfuscation.md's "trace of a VM" note). + +Measured status: host-tested on synthetic interpreter traces in the calibrated shape +(tests/unit/test_vm_handler_slice.py). No real xvmp-hardened binary was ever traced by +this repository -- the calibration facts are from the public source, and the first run +against a real sample is a measurement, not a confirmation. + +Examples + python vm_handler_slice.py --log vm_trace.log + python vm_handler_slice.py --log vm_trace.log --dispatcher liba.so+0x9f6c0 --top 15 + python vm_handler_slice.py --log vm_trace.log --opcode-map map.json +""" +import argparse +import hashlib +import json +import sys +from collections import Counter + +sys.path.insert(0, __file__.rsplit("/", 1)[0]) +import trace_diff # noqa: E402 (sibling import is the repo's convention: dexutil et al) + + +def pick_dispatcher(blocks): + """Dispatcher identification, same rules as native-dbi-and-deobfuscation.md's + trace-to-CFG step: the histogram's top, WITH many distinct predecessors (every + handler tail jumps back to it). The margin alone is not a test: in a stack-shaped + VM the hottest handler can execute on every guest instruction, exactly as often + as the dispatcher, so a 4x rule refuses real interpreter traces.""" + hist = Counter(blocks) + if not hist: + raise trace_diff.TraceError("empty trace") + top = hist.most_common() + cand = top[0][0] + preds = {blocks[i - 1] for i in range(1, len(blocks)) if blocks[i] == cand} + if len(top) > 1 and len(preds) < 3: + # not the only block, and entered from fewer than three predecessors: + # a plain hot loop (or a two-cycle tie), not a dispatch loop. A pure + # frequency margin cannot rescue it -- in a stack-shaped VM the hottest + # handler runs exactly as often as the dispatcher, so frequency is the + # one signal this rule refuses to trust + return None + return cand + + +def slice_segments(blocks, dispatcher): + """-> (segments, preamble, tail): segments = list of (run, closed). + The run after the LAST dispatcher hit is an open segment -- never closed by a + following dispatch (exit path, or a truncated capture window) -- so it joins + the trajectory but carries closed=False and is flagged in the output.""" + idxs = [i for i, b in enumerate(blocks) if b == dispatcher] + if not idxs: + raise trace_diff.TraceError("dispatcher block never executed") + preamble = blocks[:idxs[0]] + tail = blocks[idxs[-1] + 1:] + segments = [] + for a, b in zip(idxs, idxs[1:]): + run = blocks[a + 1:b] + if run: # a dispatcher hit followed immediately by another + segments.append((run, True)) # contributes nothing (miss/no-op path) + if tail: + segments.append((tail, False)) + return segments, preamble, tail + + +def signature(run): + """A segment's run list -> its comparison signature.""" + return hashlib.sha1("\x00".join(run).encode("utf-8")).hexdigest()[:10] + + +def cluster(segments): + """-> ordered trajectory of signatures + per-signature stats.""" + stats = {} + trajectory = [] + for i, (run, closed) in enumerate(segments): + sig = signature(run) + trajectory.append(sig) + if sig not in stats: + stats[sig] = {"sig": sig, "count": 0, "first": i, + "blocks": run, "len": len(run), "open_tail": False} + stats[sig]["count"] += 1 + if not closed: + stats[sig]["open_tail"] = True + return trajectory, stats + + +def diagnostics(blocks, dispatcher, segments, stats): + """Sanity notes (spin loops, degenerate shapes).""" + n = len(blocks) + dcount = blocks.count(dispatcher) + avg_len = sum(len(s) for s in segments) / len(segments) if segments else 0 + unique_ratio = len(stats) / len(segments) if segments else 1.0 + notes = [] + if avg_len <= 1.05: + notes.append("every segment is a single block: the 'dispatcher' is probably a spin " + "loop or the trace granularity is wrong, not an interpreter") + if unique_ratio > 0.9 and len(segments) > 8: + notes.append("almost every segment is unique: not a repeating opcode-handler " + "structure -- either wrong dispatcher or not a VM trace") + if dcount * 1.0 / n < 0.15: + notes.append("dispatcher is under 15%% of blocks (%d/%d): the loop does not " + "dominate; re-check with trace_diff.py reduce's histogram" + % (dcount, n)) + return {"blocks": n, "dispatcher_hits": dcount, + "dispatcher_share": round(dcount * 100.0 / n, 1), + "segments": len(segments), "unique_segments": len(stats), + "avg_segment_len": round(avg_len, 2), "notes": notes} + + +def apply_map(trajectory, stats, map_path): + """Map handler keys to opcode labels; report unmapped blocks.""" + with open(map_path, "r", encoding="utf-8") as fh: + table = json.load(fh) + named, unmapped = [], [] + for sig in trajectory: + run = stats[sig]["blocks"] + key = run[0] + if key in table: + named.append(table[key]) + else: + named.append("?") + unmapped.append(key) + return named, unmapped + + +def cmd_main(args): + """Resolve input rows against the ELF; text or --json output.""" + try: + blocks = trace_diff.read_blocks(args.log, args.format) + except (OSError, trace_diff.TraceError) as exc: + print("RESULT=refused") + print(str(exc), file=sys.stderr) + return 2 + dispatcher = args.dispatcher or pick_dispatcher(blocks) + if dispatcher is None: + print("RESULT=refused") + print("no wide-margin top block: this trace does not look like an interpreter " + "loop (see trace_diff.py reduce, and the VM-trace note in " + "references/native-dbi-and-deobfuscation.md)", file=sys.stderr) + return 2 + try: + segments, preamble, tail = slice_segments(blocks, dispatcher) + except trace_diff.TraceError as exc: + print("RESULT=refused") + print(str(exc), file=sys.stderr) + return 2 + trajectory, stats = cluster(segments) + diag = diagnostics(blocks, dispatcher, segments, stats) + named = unmapped = None + if args.opcode_map: + named, unmapped = apply_map(trajectory, stats, args.opcode_map) + + if args.json: + doc = {"status": "ok", "dispatcher": dispatcher, "diagnostics": diag, + "preamble": preamble, "tail": tail, + "segments": sorted(stats.values(), key=lambda s: -s["count"]), + "trajectory": trajectory} + if named is not None: + doc["opcode_trajectory"] = named + doc["unmapped_handler_blocks"] = sorted(set(unmapped)) + print(json.dumps(doc, indent=1)) + return 0 + + print("dispatcher: %s (%s%% of %d blocks, %d hits)" + % (dispatcher, diag["dispatcher_share"], diag["blocks"], + diag["dispatcher_hits"])) + print("segments: %d total, %d unique, avg len %.2f%s%s" + % (diag["segments"], diag["unique_segments"], diag["avg_segment_len"], + (" preamble=%d blocks" % len(preamble)) if preamble else "", + " [last segment UNCLOSED: exit path or truncated window]" + if any(s.get("open_tail") for s in stats.values()) else "")) + for note in diag["notes"]: + print(" note: %s" % note) + print("\nsegment table (top %d by count):" % args.top) + for s in sorted(stats.values(), key=lambda s: (-s["count"], s["first"]))[:args.top]: + print(" %5dx sig=%s len=%d %s" % (s["count"], s["sig"], s["len"], + " -> ".join(s["blocks"][:8]) + + (" ..." if s["len"] > 8 else ""))) + print("\ntrajectory (first %d): %s" % (args.top, " ".join(trajectory[:args.top]))) + if named is not None: + print("opcode trajectory (first %d): %s" % (args.top, " ".join(named[:args.top]))) + if unmapped: + print("unmapped handler blocks: %d distinct -- extend --opcode-map" + % len(set(unmapped))) + print("\nRESULT=ok") + return 0 + + +def main(): + """CLI entry: subcommand dispatch.""" + ap = argparse.ArgumentParser( + description="Slice a VM interpreter trace into handler segments: dispatcher " + "detection, segment signatures, trajectory, optional handler->opcode map.", + epilog="Calibrated against the public xvmp IR virtualizer (vm_exec + slot machine); " + "references/ete-diff-tracing.md owns the capture, this tool owns the " + "de-virtualization half of the reading.") + ap.add_argument("--log", required=True, help="interpreter trace (stalker or plain)") + ap.add_argument("--format", choices=sorted(trace_diff.PARSERS), + help="force a parser (default: auto, as trace_diff.py)") + ap.add_argument("--dispatcher", help="explicit dispatcher block key (default: auto)") + ap.add_argument("--opcode-map", help="JSON {handler-first-block: opcode-name}") + ap.add_argument("--top", type=int, default=20) + ap.add_argument("--json", action="store_true") + args = ap.parse_args() + try: + return cmd_main(args) + except BrokenPipeError: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_ete_capture.py b/tests/unit/test_ete_capture.py new file mode 100644 index 0000000..b6eca76 --- /dev/null +++ b/tests/unit/test_ete_capture.py @@ -0,0 +1,113 @@ +"""Unit tests for ete_capture.py: the command construction the review broke. + +The capture path needs a device, so what is host-testable is exactly what the +review found wrong before: where `-o` sits relative to the `--` workload +separator, how values are quoted on their way into one `su -c '...'` string, +and that the control record is held to a decode check, not just an exit code +(the decode gate itself is asserted by construction: the control command and +the decode command are what `capture --control` runs, in that order). +""" +import shlex + +from conftest import load_script + +ec = load_script("ete_capture.py") + + +class Args: + """The argparse namespace shape cmd_capture/perf_record_command read.""" + + def __init__(self, pid=None, cmd=None, seconds=10, event="cs_etm//u", + out="perf.data", control=False, serial=None): + """The argparse namespace shape cmd_capture reads.""" + self.pid = pid + self.cmd = cmd + self.seconds = seconds + self.event = event + self.out = out + self.control = control + self.serial = serial + + +def split_device_shell(perf_cmd): + """What the device shell sees: strip the trailing 2>&1 and split su args.""" + assert perf_cmd.endswith(" 2>&1"), perf_cmd + return shlex.split(perf_cmd[:-len(" 2>&1")]) + + +def test_output_option_precedes_the_workload_separator(): + """`perf record` hands everything after `--` to the target command, so a + trailing `-o` is argv for the workload and perf writes to its default + path (the pull then fetches a stale file).""" + cmd = ec.perf_record_command(Args(cmd="app --flag")) + toks = split_device_shell(cmd) + assert toks[:2] == ["su", "-c"] + inner = toks[2] + sep = inner.index(" -- ") # the workload separator, not --per-thread + out_at = inner.index("-o ") + assert out_at < sep, "-o must come before the workload separator" + assert inner[sep:].startswith(" -- app --flag") + + +def test_pid_mode_quotes_the_pid_and_keeps_o_before_target(): + """Pid mode quotes the pid and keeps o before target.""" + cmd = ec.perf_record_command(Args(pid="1234")) + inner = " ".join(split_device_shell(cmd)[2:]) + assert "-p 1234" in inner + assert inner.index("-o ") < inner.index("-p 1234") + + +def test_values_with_quotes_cannot_break_out_of_su_c(): + """Every interpolated value travels inside one su -c string: an unquoted + single quote would close it and run the rest in the outer device shell.""" + evil = "x'; rm -rf /data/local/tmp/*; '" + cmd = ec.perf_record_command(Args(cmd='sh -c "%s"' % evil, + event="cs_etm/@tmc_etr0/u")) + toks = split_device_shell(cmd) + assert toks[:2] == ["su", "-c"] + # the su argument is ONE token: the device shell never sees the payload + # as separate commands, whatever it contains + assert len(toks) == 3 + inner = toks[2] + assert "perf record" in inner and "-o /data/local/tmp/perf.data" in inner + + +def test_out_filename_and_event_are_quoted(): + """Out filename and event are quoted.""" + cmd = ec.perf_record_command(Args(pid="7", event="my event", + out="dir/my perf.data")) + toks = split_device_shell(cmd) + inner = toks[2] + assert "perf record -e 'my event'" in inner + assert "-o '/data/local/tmp/my perf.data'" in inner + + +def test_control_commands_exist_and_decode_runs_the_recorded_file(): + """The --control guarantee is that the record DECODES: the decode command + must point perf script at the control recording's file.""" + rec = ec.control_record_command("a.ctl.data") + dec = ec.control_decode_command("a.ctl.data") + assert "perf record" in rec and "-- true" in rec and "-o a.ctl.data" in rec + assert "perf script" in dec and "-i a.ctl.data" in dec + + +def test_control_commands_quote_across_both_device_shell_layers(): + """A spaced output name travels inside one su -c string: quote it inside + the inner command, then quote the whole command for su -c -- a splice of + literal single quotes would run in the outer device shell.""" + for cmd in (ec.control_record_command("my ctl.data"), + ec.control_decode_command("my ctl.data")): + assert cmd.endswith(" 2>&1") + toks = shlex.split(cmd[:-len(" 2>&1")]) + assert toks[:2] == ["su", "-c"] and len(toks) == 3 # one su argument + assert "my ctl.data" in toks[2] + + +def test_seconds_must_be_a_positive_window(): + """`timeout 0` disables the timer (or kills instantly): the CLI would + wait on a record that never ends. Refused before any adb check.""" + from conftest import run_script + for bad in ("0", "-5"): + r = run_script("ete_capture.py", ["capture", "--pid", "1", "--seconds", bad]) + assert r.returncode == 2, (bad, r.output) + assert "--seconds" in r.output and "RESULT=refused" in r.output diff --git a/tests/unit/test_hw_watch.py b/tests/unit/test_hw_watch.py new file mode 100644 index 0000000..8902d32 --- /dev/null +++ b/tests/unit/test_hw_watch.py @@ -0,0 +1,251 @@ +"""Unit tests for hw_watch.py: encoders, refusals, emit, hit decoding. + +What is asserted here is the **host-measured half** of the watchpoint route: the +WCR/BCR bit encodings against the ARM DDI 0487 field map, the alignment rules +that refuse impossible specs locally instead of on the device, the emitted +helper's shape, and the hit-log decoder. Device behaviour (seize/arm/trap) is +deliberately not simulated -- see docs/tool-verification/EXTENSION-hw-watchpoints.md. +""" +import json +import shutil +import subprocess + +import pytest + +from conftest import load_script, run_script + +hw = load_script("hw_watch.py") + + +# --------------------------------------------------------------------------- +# encoders -- the two canonical values plus the variations +# --------------------------------------------------------------------------- +class TestWatchEncoders: + def test_canonical_8byte_rw(self): + # BAS=0xff, LSC=both(0b11), PAC=EL0(0b10), E=1 + """Canonical 8byte rw.""" + assert hw.encode_watch(0x7b2000000000, 8, "rw") == (0x7b2000000000, 0x1FFD) + + def test_load_only_and_store_only(self): + """Load only and store only.""" + wvr, wcr_r = hw.encode_watch(0x1000, 8, "r") + wvr, wcr_w = hw.encode_watch(0x1000, 8, "w") + assert wcr_r == 0x1FED # LSC=01 (load) + assert wcr_w == 0x1FF5 # LSC=10 (store) + + def test_unaligned_start_shifts_bas(self): + # bytes 4..7 of the window: BAS = 0xf0 << 5 = 0x1E00 + """Unaligned start shifts bas.""" + wvr, wcr = hw.encode_watch(0x1004, 4, "w") + assert wvr == 0x1000 # WVR canonicalised to the aligned window + assert wcr & 0x1FE0 == 0x1E00 + assert wcr == 0x1E15 # | store | PAC | E + + def test_len_1_at_offset_7(self): + """Len 1 at offset 7.""" + wvr, wcr = hw.encode_watch(0x1007, 1, "rw") + assert wcr & 0x1FE0 == (0x80 << 5) + + def test_window_crossing_refused(self): + """Window crossing refused.""" + with pytest.raises(hw.PlanError, match="crosses"): + hw.encode_watch(0x1006, 8, "rw") + with pytest.raises(hw.PlanError, match="crosses"): + hw.encode_watch(0x1004, 8, "r") + + def test_length_bounds(self): + """Length bounds.""" + with pytest.raises(hw.PlanError): + hw.encode_watch(0x1000, 0, "rw") + with pytest.raises(hw.PlanError): + hw.encode_watch(0x1000, 9, "rw") + + def test_access_vocabulary(self): + """Access vocabulary.""" + with pytest.raises(hw.PlanError): + hw.encode_watch(0x1000, 8, "x") + + +class TestBreakEncoders: + def test_canonical_address_match(self): + # BAS all-ones (0x3F << 8), PMC=EL0 (0b10 << 1), BT=0, E=1 + """Canonical address match.""" + assert hw.encode_break(0x7b20001234c0) == (0x7b20001234c0, 0x3F05) + + def test_misaligned_pc_refused(self): + """Misaligned pc refused.""" + with pytest.raises(hw.PlanError, match="4-byte aligned"): + hw.encode_break(0x1002) + with pytest.raises(hw.PlanError): + hw.encode_break(0x1001) + + +def test_kernel_abi_constants_are_the_uapi_values(): + """Pinned because a drifted constant compiles fine and arms nothing.""" + assert hw.PTRACE_GETREGSET == 0x4204 + assert hw.PTRACE_SETREGSET == 0x4205 + assert hw.PTRACE_SEIZE == 0x4206 + assert hw.NT_ARM_HW_BREAK == 0x402 + assert hw.NT_ARM_HW_WATCH == 0x403 + assert hw.TRAP_HWBKPT == 4 + assert hw.USER_PT_REGS_BYTES == 272 + assert hw.REG_OFFSETS["pc"] == 256 and hw.REG_OFFSETS["sp"] == 248 + + +# --------------------------------------------------------------------------- +# emit -- the helper's shape and buildability +# --------------------------------------------------------------------------- +def test_emit_writes_helper_and_readme(tmp_path): + """Emit writes helper and readme.""" + out = tmp_path / "hww" + r = run_script("hw_watch.py", ["emit", "--out", str(out)]) + assert r.returncode == 0, r.output + c = (out / "hwwatch_helper.c").read_text(encoding="utf-8") + readme = (out / "README.md").read_text(encoding="utf-8") + # the ABI surface the encoders promised + for marker in ("NT_ARM_HW_WATCH", "NT_ARM_HW_BREAK", "TRAP_HWBKPT", + "PTRACE_SEIZE", "PTRACE_O_TRACECLONE", "PTRACE_SINGLESTEP", + "PTRACE_LISTEN", "0x3Fu << 8"): + assert marker in c, marker + # the honesty devices + assert "TracerPid" in c + assert "TEMPLATE" in c + assert "aarch64-linux-android24-clang" in readme + + +def test_emitted_helper_compiles_when_a_cross_toolchain_exists(tmp_path): + """When gcc targets aarch64 here (it does on this CI image), the emitted + helper must build as a static ELF. Skipped where no compiler is present: + the compile is evidence, not a portability promise.""" + cc = shutil.which("gcc") + if cc is None: + pytest.skip("no C compiler on this host") + probe = subprocess.run([cc, "-dumpmachine"], capture_output=True, text=True) + if "aarch64" not in (probe.stdout or ""): + pytest.skip("host gcc does not target aarch64 (%r)" % probe.stdout.strip()) + out = tmp_path / "hww" + run_script("hw_watch.py", ["emit", "--out", str(out)]) + exe = tmp_path / "hwwatch" + built = subprocess.run([cc, "-static", "-O2", "-o", str(exe), + str(out / "hwwatch_helper.c")], + capture_output=True, text=True) + assert built.returncode == 0, built.stderr + assert exe.stat().st_size > 10000 + + +# --------------------------------------------------------------------------- +# decode -- hits in, module+offset out +# --------------------------------------------------------------------------- +HIT_LOG = """READY pid=1234 watch_slots=4 break_slots=6 +ARM tid=1234 watch=1 break=0 +ARM tid=1240 watch=1 break=0 +HIT tid=1240 kind=watch pc=0x7b20001234c0 addr=0x7b2000feedd0 sp=0x7fc000123000 lr=0x7b2000120000 x0=0x1 x1=0x2 x2=0x3 x3=0x4 x4=0x5 x5=0x6 x6=0x7 x7=0x8 +ARMFAIL tid=1250 slot=watch1 capacity=0 +HIT tid=1234 kind=break pc=0x7b2000129000 addr=0x7b2000129000 sp=0x7fc000124000 lr=0x0 x0=0x0 x1=0x0 x2=0x0 x3=0x0 x4=0x0 x5=0x0 x6=0x0 x7=0x0 +DONE hits=2 reason=seconds +""" + +MAPS = """7b2000000000-7b2000200000 r--p 00000000 fd:01 100 /data/app/liba.so +7b2000200000-7b2000400000 r-xp 00200000 fd:01 100 /data/app/liba.so +7b2000400000-7b2000420000 rw-p 00400000 fd:01 100 /data/app/liba.so +""" + + +def test_parse_hits_tolerates_interleaved_lines(tmp_path): + """Parse hits tolerates interleaved lines.""" + log = tmp_path / "hits.log" + log.write_text(HIT_LOG, encoding="utf-8") + rows = hw.parse_hits(str(log)) + assert len(rows) == 2 + assert rows[0]["kind"] == "watch" and rows[0]["pc"] == "0x7b20001234c0" + assert rows[1]["kind"] == "break" + + +def test_decode_resolves_module_offset(tmp_path): + """Decode resolves module offset.""" + log = tmp_path / "hits.log" + log.write_text(HIT_LOG, encoding="utf-8") + maps = tmp_path / "maps.txt" + maps.write_text(MAPS, encoding="utf-8") + r = run_script("hw_watch.py", ["decode", "--hits", str(log), "--maps", str(maps)]) + assert r.returncode == 0, r.output + assert "+0x1234c0" in r.output # pc lands in the first r--p range + assert "unresolved_pc=0" in r.output + rj = run_script("hw_watch.py", ["decode", "--hits", str(log), "--maps", str(maps), + "--json"]) + doc = json.loads(rj.stdout) # --json stdout is one pure document + assert doc["status"] == "ok" and doc["hits"] == 2 + assert doc["rows"][0]["site"].endswith("liba.so+0x1234c0") + + +def test_decode_offsets_come_from_the_module_base_not_the_mapping(tmp_path): + """A PC in a library's r-xp mapping (its second mapping) must report the + offset from the MODULE base -- the number IDA/Ghidra expects -- not from + that mapping's own start, which resets the offset at each boundary.""" + log = tmp_path / "hits.log" + log.write_text("HIT tid=1 kind=break pc=0x7b2000212345 addr=0x7b2000212345 " + "sp=0x0 lr=0x0 x0=0x0\n", encoding="utf-8") + maps = tmp_path / "maps.txt" + maps.write_text(MAPS, encoding="utf-8") + r = run_script("hw_watch.py", ["decode", "--hits", str(log), "--maps", str(maps)]) + assert r.returncode == 0, r.output + assert "liba.so+0x212345" in r.output # not +0x12345 (mapping-relative) + + +def test_decode_without_maps_leaves_pcs_raw(tmp_path): + """Decode without maps leaves pcs raw.""" + log = tmp_path / "hits.log" + log.write_text(HIT_LOG, encoding="utf-8") + r = run_script("hw_watch.py", ["decode", "--hits", str(log)]) + assert r.returncode == 0 + assert "unresolved" in r.output and "no --maps" in r.output + + +def test_plan_json_and_refusal_exit_codes(): + """Plan json and refusal exit codes.""" + r = run_script("hw_watch.py", ["plan", "--addr", "0x1000", "--len", "8", + "--access", "rw", "--json"]) + assert r.returncode == 0 + doc = json.loads(r.stdout) + assert doc["specs"][0]["cr"] == 0x1FFD + r = run_script("hw_watch.py", ["plan", "--addr", "0x1004", "--len", "8"]) + assert r.returncode == 2 + assert "crosses" in r.output + + +# --------------------------------------------------------------------------- +# the emitted helper: kernel-ABI regressions pinned in the C source +# --------------------------------------------------------------------------- +def emitted_helper(tmp_path): + """Emit the on-device helper; return its C source.""" + out = tmp_path / "hww" + run_script("hw_watch.py", ["emit", "--out", str(out)]) + return (out / "hwwatch_helper.c").read_text(encoding="utf-8") + + +def test_helper_slot_layout_matches_the_kernel_abi(tmp_path): + """user_hwdebug_state carries the 64-bit address register FIRST and the + 32-bit control register SECOND (arch/arm64 uapi); swapping them arms + every slot with the ctrl bits as the address.""" + c = emitted_helper(tmp_path) + assert "struct hwslot { uint64_t wvr; uint32_t wcr; uint32_t pad; };" in c + + +def test_helper_error_paths_release_seized_threads(tmp_path): + """PTRACE_O_EXITKILL turns an exit-without-detach into a SIGKILL of the + target: every post-seize failure must route through release_all().""" + c = emitted_helper(tmp_path) + assert "static void release_all(void)" in c + # capacity probe failure and arming failure both detach instead of return + assert c.count("release_all();") >= 3 # cap fail, arm fail, clean exit + # partial seize failure detaches the threads seized so far + assert "EXITKILLed when we exit" in c + + +def test_helper_forwards_genuine_sigtrap(tmp_path): + """Only our TRAP_HWBKPT hits and ptrace event stops are consumed; a + target-generated SIGTRAP (raise/tgkill/int3) must be forwarded or the + app's handler never runs.""" + c = emitted_helper(tmp_path) + assert "if (event != 0) {" in c # event stops: consumed + assert "genuine" in c and "pt(PTRACE_CONT, tid, 0, sig);" in c diff --git a/tests/unit/test_kernel_instrument_scaffold.py b/tests/unit/test_kernel_instrument_scaffold.py new file mode 100644 index 0000000..823806a --- /dev/null +++ b/tests/unit/test_kernel_instrument_scaffold.py @@ -0,0 +1,189 @@ +"""Unit tests for kernel_instrument_scaffold.py: spec validation, generate, +verify, and the honesty devices (the unverified banner, the carrier that +cannot arm anything, the experimental action fence).""" +import json + +import pytest + +from conftest import load_script, run_script + +ks = load_script("kernel_instrument_scaffold.py") + + +class TestSpecValidation: + def test_watch_spec_roundtrip(self): + """Watch spec roundtrip.""" + row = ks.parse_spec_watch("w:0x7b2000123450:8:w") + assert row == {"addr": "0x7b2000123450", "len": 8, "access": "w"} + + def test_window_rule_shared_with_hw_watch(self): + """Window rule shared with hw watch.""" + with pytest.raises(ValueError, match="crosses"): + ks.parse_spec_watch("w:0x7b2000123456:8:w") + + def test_break_alignment(self): + """Break alignment.""" + assert ks.parse_spec_break("b:0x1234c0")["pc"] == "0x1234c0" + with pytest.raises(ValueError, match="4-byte aligned"): + ks.parse_spec_break("b:0x1234c2") + + def test_set_reg_shape(self): + """Set reg shape.""" + row = ks.parse_spec_break("b:0x1234c0", set_reg="x0=0x1") + assert row["set_reg"] == "x0" and row["set_value"] == "0x1" + with pytest.raises(ValueError): + ks.parse_spec_break("b:0x1234c0", set_reg="x0") + + def test_set_reg_rejects_registers_the_template_cannot_write(self): + """x31 is sp's slot in user_pt_regs and the array ends at x30: an + index past 30 writes sp/pc/pstate or off the frame.""" + for bad in ("x31=0x1", "x99=0x1", "ax5=0x1", "x0", "y0=0x1"): + with pytest.raises(ValueError): + ks.parse_spec_break("b:0x1234c0", set_reg=bad) + for good in ("x0=0x1", "x30=0x2", "x30=5"): + ks.parse_spec_break("b:0x1234c0", set_reg=good) + + +def generate(tmp_path, extra=()): + """Generate the standard scaffold; return its output directory.""" + out = tmp_path / "kwatch" + argv = ["generate", "--out", str(out), "--pid", "12345", + "--watch", "w:0x7b2000123450:8:w", "--break", "b:0x7b20001234c0", + "--set-reg", "x0=0x1"] + list(extra) + r = run_script("kernel_instrument_scaffold.py", argv) + assert r.returncode == 0, r.output + return out + + +class TestGenerate: + def test_layout_and_config(self, tmp_path): + """Layout and config.""" + out = generate(tmp_path) + expected = ["config/instrument.json", "kpm/src/kwatch.c", "kpm/Makefile", + "lkm/kwatch.c", "lkm/Makefile", "module.prop", "README.md", + "post-fs-data.sh"] + for rel in expected: + assert (out / rel).is_file(), rel + cfg = json.loads((out / "config" / "instrument.json").read_text(encoding="utf-8")) + assert cfg["pid"] == 12345 + assert cfg["watches"][0]["addr"] == "0x7b2000123450" + assert cfg["breaks"][0]["set_reg"] == "x0" + + def test_kernel_files_carry_the_banner_and_the_kpm_lessons(self, tmp_path): + """Kernel files carry the banner and the kpm lessons.""" + out = generate(tmp_path) + kpm = (out / "kpm" / "src" / "kwatch.c").read_text(encoding="utf-8") + lkm = (out / "lkm" / "kwatch.c").read_text(encoding="utf-8") + for body in (kpm, lkm): + assert "NOT COMPILED AND NOT LOADED" in body + assert "EXPERIMENTAL" in body # the set-reg fence + assert "kfunc_def" in kpm + # the banner text mentions the generator, so a stripped banner is traceable + assert "kernel_instrument_scaffold.py" in kpm + # the user-task breakpoint API: four arguments, task last (the per-cpu + # register_wide_* family cannot arm on one target task) + assert "register_user_hw_breakpoint" in lkm + assert "&attr, wp_hit, NULL, task)" in lkm + # the task pointer is pinned past the RCU section and released after + # registration: registration can sleep, an unpinned task can be freed + assert "get_task_struct(task)" in lkm and "put_task_struct(task)" in lkm + # the execution breakpoint row initializes fields wp_cfg declares + # (.addr/.len/.bp_type) -- there is no .pc member to initialize + for body in (kpm, lkm): + assert ".pc = " not in body + assert ".addr = 0x7b20001234c0, .len = 8, .bp_type = 4" in kpm + + def test_breakpoint_rows_initialize_declared_fields(self, tmp_path): + """Breakpoint rows initialize declared fields.""" + out = generate(tmp_path) + for body in ((out / "kpm" / "src" / "kwatch.c").read_text(encoding="utf-8"), + (out / "lkm" / "kwatch.c").read_text(encoding="utf-8")): + assert ".pc = " not in body # no such member + assert ".addr = 0x7b20001234c0, .len = 8, .bp_type = 4" in body + + def test_events_carry_a_sampling_period(self, tmp_path): + """A zero sample_period is a non-sampling event: perf arms it, the + overflow callback (wp_hit) never runs, and the module reports armed + events that stay silent forever.""" + out = generate(tmp_path) + kpm = (out / "kpm" / "src" / "kwatch.c").read_text(encoding="utf-8") + lkm = (out / "lkm" / "kwatch.c").read_text(encoding="utf-8") + assert "attr.sample_period = 1;" in kpm # hw_breakpoint_init, spelled out + assert "hw_breakpoint_init(&attr);" in lkm + + def test_set_reg_rewrite_is_gated_to_its_breakpoint_event(self, tmp_path): + """wp_hit fires for every registered event (watch and break alike), so + the rewrite must check it landed on its own breakpoint.""" + out = generate(tmp_path) + for body in ((out / "kpm" / "src" / "kwatch.c").read_text(encoding="utf-8"), + (out / "lkm" / "kwatch.c").read_text(encoding="utf-8")): + assert "event->attr.bp_addr == 0x7b20001234c0" in body + assert body.count("regs->regs[0] = 0x1;") == 1 + # the write sits inside the guard, not at callback top level + guarded = body.split("regs->regs[0] = 0x1;")[0] + assert "event->attr.bp_type & 4" in guarded + + def test_id_is_restricted_to_a_module_identifier(self, tmp_path): + """--id becomes paths, Makefile targets and module.prop entries: a + path separator or leading dot would write outside --out.""" + for bad in ("../evil", "a/b", ".hidden", "", "a b"): + argv = ["generate", "--out", str(tmp_path / "x"), "--pid", "1", + "--watch", "w:0x1000:8:w", "--id", bad] + r = run_script("kernel_instrument_scaffold.py", argv) + assert r.returncode == 2, bad + argv = ["generate", "--out", str(tmp_path / "ok"), "--pid", "1", + "--watch", "w:0x1000:8:w", "--id", "kwatch.v2-x"] + r = run_script("kernel_instrument_scaffold.py", argv) + assert r.returncode == 0, r.output + + def test_carrier_says_what_it_cannot_do(self, tmp_path): + """Carrier says what it cannot do.""" + out = generate(tmp_path) + readme = (out / "README.md").read_text(encoding="utf-8") + assert "cannot arm a watchpoint" in readme + + def test_no_specs_is_refused(self, tmp_path): + """No specs is refused.""" + r = run_script("kernel_instrument_scaffold.py", + ["generate", "--out", str(tmp_path / "x"), "--pid", "1"]) + assert r.returncode == 2 + + +class TestVerify: + def test_clean_generation_verifies(self, tmp_path): + """Clean generation verifies.""" + out = generate(tmp_path) + r = run_script("kernel_instrument_scaffold.py", ["verify", "--dir", str(out)]) + assert r.returncode == 0, r.output + assert "RESULT=ok" in r.output + + def test_stripped_banner_is_a_problem(self, tmp_path): + """Stripped banner is a problem.""" + out = generate(tmp_path) + target = out / "kpm" / "src" / "kwatch.c" + body = target.read_text(encoding="utf-8").replace( + "NOT COMPILED AND NOT LOADED", "COMPILED SOMEWHERE, TRUST ME") + target.write_text(body, encoding="utf-8") + r = run_script("kernel_instrument_scaffold.py", ["verify", "--dir", str(out)]) + assert r.returncode == 1 + assert "banner" in r.output + + def test_missing_config_is_a_problem(self, tmp_path): + """Missing config is a problem.""" + out = generate(tmp_path) + (out / "config" / "instrument.json").unlink() + r = run_script("kernel_instrument_scaffold.py", ["verify", "--dir", str(out)]) + assert r.returncode == 1 + assert "missing" in r.output + + +class TestGates: + def test_gate_table_includes_the_perf_breakpoint_layer(self): + """Gate table includes the perf breakpoint layer.""" + routes = {g["route"] for g in ks.GATES} + assert "perf hw-breakpoint slots" in routes + assert "KernelPatch / APatch KPM" in routes + r = run_script("kernel_instrument_scaffold.py", ["gates", "--json"]) + assert r.returncode == 0 + doc = json.loads(r.stdout) # --json stdout is one pure document + assert doc["status"] == "ok" and len(doc["gates"]) == len(ks.GATES) diff --git a/tests/unit/test_trace_addr2sym.py b/tests/unit/test_trace_addr2sym.py new file mode 100644 index 0000000..170bdbc --- /dev/null +++ b/tests/unit/test_trace_addr2sym.py @@ -0,0 +1,260 @@ +"""Unit tests for trace_addr2sym.py: the zero-dependency ELF64 walker, both offset +spaces, and the input adapters. + +Two evidence layers, matching docs/tool-verification/EXTENSION-sampling-crash.md: +a hand-built minimal ELF (always runs, covers vaddr!=offset and the stripped +fallback) and a real cross-compiled aarch64 .so (runs where the toolchain exists, +cross-checked against readelf). +""" +import json +import shutil +import struct +import subprocess + +import pytest + +from conftest import load_script, run_script + +tas = load_script("trace_addr2sym.py") + + +# --------------------------------------------------------------------------- +# a hand-built minimal ELF64: one PT_LOAD with vaddr != offset, one dynsym +# --------------------------------------------------------------------------- +def build_minimal_elf(path, vaddr_delta=0x1000, with_gap=False): + """Two-symbol dynsym; one PT_LOAD, or two with an unmapped gap. + + Symbols: sym_a @ 0x2000 (size 0x100), sym_b @ 0x2400 (size 0x10). + The load segment maps file [0x0,0x4000) at vaddr 0x1000..0x5000, so + file offset F <-> vaddr F + vaddr_delta for the segment, while the symbols + live at vaddr 0x2000/0x2400 regardless (they sit inside the segment). + + with_gap=True shrinks segment 1 to vaddr 0x1000..0x3000 and adds a second + PT_LOAD at vaddr 0x4000..0x5000: the span [0x3000,0x4000) is loaded by + NEITHER segment, the shape real shared objects have between PT_LOADs. + """ + assert vaddr_delta == 0x1000 + data = bytearray(b"\x00" * 0x4000) + # ELF header + data[0:4] = b"\x7fELF" + struct.pack_into(" file: the segment adds 0x1000 going file->vaddr + assert elf.vaddr_to_file(0x2000) == 0x1000 # 0x2000 - 0x1000 + assert elf.file_to_vaddr(0x1000) == 0x2000 + sym, off = elf.resolve(0x2042) + assert (sym, off) == ("sym_a", 0x42) + sym, off = elf.resolve(0x2405) + assert (sym, off) == ("sym_b", 5) + # past every symbol but still inside the load segment: nearest-below + # fallback (addr2line-style, gaps included) -- a PC OUTSIDE the segment is + # refused by describe(), not folded onto the last symbol (see below) + sym, off = elf.resolve(0x3f00) + assert (sym, off) == ("sym_b", 0x1b00) + # the span gate: a PC from another module must not be attributed here + assert elf.contains_vaddr(0x3f00) and not elf.contains_vaddr(0xdead0000) + assert "outside-elf" in tas.describe(elf, 0xdead0000) + + +def test_offs_are_file_flag_and_vaddr_space(tmp_path): + """file offset input and vaddr input must not be conflated.""" + p = tmp_path / "min.so" + build_minimal_elf(p) + r1 = run_script("trace_addr2sym.py", ["--elf", str(p), "--offs", "0x2042"]) + assert r1.returncode == 0 and "sym_a+0x42" in r1.output + r2 = run_script("trace_addr2sym.py", ["--elf", str(p), "--offs", "0x2042", + "--offs-are-file"]) + assert r2.returncode == 0 + # file 0x2042 -> vaddr 0x3042: past sym_a's 0x100-size, nearest-below is sym_b + assert "sym_b+0xc42" in r2.output + + +def test_absolute_addrs_need_a_base(tmp_path): + """Absolute addrs need a base.""" + p = tmp_path / "min.so" + build_minimal_elf(p) + r = run_script("trace_addr2sym.py", ["--elf", str(p), "--addrs", "0x7b2000002042"]) + assert r.returncode == 2 + assert "load base" in r.output # ASLR: refuse rather than guess + r = run_script("trace_addr2sym.py", ["--elf", str(p), "--addrs", "0x7b2000002042", + "--base", "0x7b2000000000"]) + assert r.returncode == 0 and "sym_a+0x42" in r.output + + +# --------------------------------------------------------------------------- +# real cross-compiled library (skipped where no aarch64 gcc exists) +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module") +def real_so(tmp_path_factory): + """Real so.""" + cc = shutil.which("gcc") + if cc is None: + pytest.skip("no C compiler on this host") + probe = subprocess.run([cc, "-dumpmachine"], capture_output=True, text=True) + if "aarch64" not in (probe.stdout or ""): + pytest.skip("host gcc does not target aarch64 (%r)" % probe.stdout.strip()) + d = tmp_path_factory.mktemp("so") + src = d / "tlib.c" + src.write_text("int encrypt_block(unsigned char *b,int n){for(int i=0;i 7 and parts[7] == "encrypt_block" and parts[3] == "FUNC": + addr = int(parts[1], 16) + break + assert addr is not None + r = run_script("trace_addr2sym.py", ["--elf", str(real_so), "--offs", + "0x%x" % (addr + 0x30), "--json"]) + assert r.returncode == 0 + doc = json.loads(r.stdout) # --json stdout is a pure JSON document + assert doc["resolved"][0]["where"].startswith("encrypt_block+0x30") + + +def test_real_so_stripped_fallback(real_so, tmp_path): + """Real so stripped fallback.""" + stripped = tmp_path / "stripped.so" + shutil.copy(real_so, stripped) + if shutil.which("strip"): + subprocess.run(["strip", str(stripped)], capture_output=True) + r = run_script("trace_addr2sym.py", ["--elf", str(stripped), "--offs", "0x610"]) + assert r.returncode == 0 + assert ".text+" in r.output # section+offset fallback, no crash + + +def test_hits_and_trace_adapters(tmp_path): + """Hits and trace adapters.""" + p = tmp_path / "min.so" + build_minimal_elf(p) + hits = tmp_path / "hits.log" + hits.write_text("HIT tid=1 kind=watch pc=0x7b2000002042 addr=0x7b200000feedd0 " + "sp=0x0 lr=0x0 x0=0x0 x1=0x0 x2=0x0 x3=0x0 x4=0x0 x5=0x0 x6=0x0 x7=0x0\n", + encoding="utf-8") + maps = tmp_path / "maps.txt" + # physically correct for this ELF: the segment (p_offset 0, p_vaddr 0x1000) + # is mapped at 0x7b2000001000, so the load bias is 0x7b2000001000 - 0x1000 + maps.write_text("7b2000001000-7b2000005000 r--p 00000000 fd:01 1 " + "/data/app/min.so\n", encoding="utf-8") + r = run_script("trace_addr2sym.py", ["--hits", str(hits), "--maps", str(maps), + "--elf", str(p)]) + assert r.returncode == 0, r.output + assert "sym_a+0x42" in r.output + # a neighbour whose name merely CONTAINS the hint must not be matched + maps.write_text("7b3000001000-7b3000005000 r--p 00000000 fd:01 1 " + "/data/app/min.so.v2\n", encoding="utf-8") + r = run_script("trace_addr2sym.py", ["--hits", str(hits), "--maps", str(maps), + "--elf", str(p)]) + assert r.returncode == 2 and "no load bias" in r.output + trace = tmp_path / "t.log" + trace.write_text("BLK 0 min.so+0x2042\nBLK 1 xmin.so+0x2405\n", encoding="utf-8") + r = run_script("trace_addr2sym.py", ["--trace", str(trace), "--mod", "min.so", + "--elf", str(p)]) + assert r.returncode == 0 + assert "sym_a+0x42" in r.output and "sym_b+0x5" not in r.output + + +def test_load_bias_subtracts_segment_vaddr(tmp_path): + """p_vaddr != 0: the bias is map_start - (foff - p_offset) - p_vaddr, + not the mapping start.""" + p = tmp_path / "min.so" + build_minimal_elf(p) + elf = tas.Elf64(str(p)) + maps = tmp_path / "maps.txt" + maps.write_text("7b2000001000-7b2000005000 r--p 00000000 fd:01 1 /data/app/min.so\n", + encoding="utf-8") + assert tas.load_bias_from_maps(str(maps), "min.so", elf) == 0x7b2000000000 + # full-path hint matches too; a mere-substring neighbour does not + maps.write_text("7b2000001000-7b2000005000 r--p 00000000 fd:01 1 /data/app/min.so\n" + "7b3000000000-7b3000004000 r--p 00000000 fd:01 1 /data/app/min.so.v2\n", + encoding="utf-8") + assert tas.load_bias_from_maps(str(maps), "min.so", elf) == 0x7b2000000000 + assert tas.load_bias_from_maps(str(maps), "/data/app/min.so", elf) == 0x7b2000000000 + + +def test_pcs_in_the_gap_between_segments_are_refused(tmp_path): + """Real shared objects leave unmapped gaps between PT_LOADs: a PC in a + gap is outside every segment and must be refused, not named after the + nearest symbol below it.""" + p = tmp_path / "gap.so" + build_minimal_elf(p, with_gap=True) + elf = tas.Elf64(str(p)) + assert elf.contains_vaddr(0x2000) # segment 1 + assert elf.contains_vaddr(0x4500) # segment 2 + assert not elf.contains_vaddr(0x3800) # the gap + assert "outside-elf" in tas.describe(elf, 0x3800) + + +def test_json_stdout_is_pure(tmp_path): + """--json mode must be parseable as one document (no trailing RESULT=).""" + p = tmp_path / "min.so" + build_minimal_elf(p) + r = run_script("trace_addr2sym.py", ["--elf", str(p), "--offs", "0x2042", "--json"]) + assert r.returncode == 0 + doc = json.loads(r.stdout) + assert doc["status"] == "ok" and "RESULT" not in r.stdout diff --git a/tests/unit/test_trace_diff.py b/tests/unit/test_trace_diff.py new file mode 100644 index 0000000..70e5079 --- /dev/null +++ b/tests/unit/test_trace_diff.py @@ -0,0 +1,183 @@ +"""Unit tests for trace_diff.py: parsers, run-length, and the differential. + +The fixtures are synthetic flattened-dispatcher traces (D R1 D R2 ...), because +that is the shape the differ exists to answer: same machinery, different +input-dependent blocks. One fixture is in raw stalker_trace.js grammar, the +other wrapped by run_probe.py, so the parser's tolerance is under test too. +""" +import json + +import pytest + +from conftest import load_script, run_script + +td = load_script("trace_diff.py") + +DISP = {"D": 0x1000, "R1": 0x2000, "R2": 0x3000, "R3": 0x4000, + "R4": 0x5000, "R5": 0x6000, "R7": 0x8000} + + +def stalker_log(seq, wrap=False): + """Render a synthetic stalker log from a block sequence.""" + lines = [] + for i, k in enumerate(seq.split()): + body = "BLK %d liba.so+0x%x" % (i, DISP[k]) + lines.append(("[123.4] [56.7] TRACE %s" % body) if wrap else body) + lines.append("DONE reason=stop-rpc blocks=8 blk=8 calls=0 truncated=0") + return "\n".join(lines) + "\n" + + +RUN_A = "D R1 D R2 D R5 D R7" # wrong password: rejects after two extra blocks +RUN_B = "D R1 D R2 D R3 D R4" # right password: accepts + + +@pytest.fixture +def pair(tmp_path): + """Two synthetic trace logs: unwrapped and run_probe-wrapped.""" + a = tmp_path / "a.log" + b = tmp_path / "b.log" + a.write_text(stalker_log(RUN_A), encoding="utf-8") + b.write_text(stalker_log(RUN_B, wrap=True), encoding="utf-8") + return a, b + + +class TestParsers: + def test_stalker_unwrapped_and_wrapped(self, pair): + """Stalker unwrapped and wrapped.""" + a, b = pair + blocks = td.read_blocks(str(a)) + assert blocks == ["liba.so+0x%x" % DISP[k] for k in RUN_A.split()] + + def test_wrapped_log_parses_identically(self, pair): + """Wrapped log parses identically.""" + a, b = pair + assert td.parse_line_stalker("[h] [d] TRACE BLK 7 liba.so+0x8000") == "liba.so+0x8000" + + def test_plain_format_keeps_module_keys_and_hex(self, tmp_path): + """Plain format keeps module keys and hex.""" + p = tmp_path / "plain.log" + p.write_text("0x7f0000001000\n7f0000002000 x\nlibb.so+0x9f6c0\nnoise line\n", + encoding="utf-8") + blocks = td.read_blocks(str(p), "plain") + # keys are kept verbatim (lowercased), so 0x-less and 0x-full forms stay distinct + assert blocks == ["0x7f0000001000", "7f0000002000", "libb.so+0x9f6c0"] + + def test_plain_format_refuses_ambiguous_decimal_tokens(self, tmp_path): + """Bare runs under 8 hex digits are PIDs, counts and timestamp + fragments; an all-digit 8+ run is indistinguishable from a date -- + taking either as a block key adds noise, and a log with no real + addresses must stay refused (an empty trace is a capture failure, + not a result).""" + p = tmp_path / "pids.log" + p.write_text("app 1234 567\nsample count 99\ndead\n", encoding="utf-8") + with pytest.raises(td.TraceError, match="capture"): + td.read_blocks(str(p), "plain") + d = tmp_path / "date.log" + d.write_text("20261015\n", encoding="utf-8") + with pytest.raises(td.TraceError, match="capture"): + td.read_blocks(str(d), "plain") + p2 = tmp_path / "mixed.log" + p2.write_text("app 1234\n7f0000002000 x\n", encoding="utf-8") + assert td.read_blocks(str(p2), "plain") == ["7f0000002000"] + + def test_empty_log_is_a_capture_failure_not_a_result(self, tmp_path): + """Empty log is a capture failure not a result.""" + p = tmp_path / "empty.log" + p.write_text("READY ...\nDONE reason=timeout blocks=0 blk=0 calls=0\n", + encoding="utf-8") + with pytest.raises(td.TraceError, match="capture"): + td.read_blocks(str(p)) + + def test_auto_detection_prefers_stalker_then_plain(self, tmp_path): + """Auto detection prefers stalker then plain.""" + p = tmp_path / "mixed.log" + p.write_text("BLK 1 liba.so+0x1000\n0x7f0000002000\n", encoding="utf-8") + assert td.read_blocks(str(p)) == ["liba.so+0x1000", "0x7f0000002000"] + + +class TestReduce: + def test_histogram_names_the_dispatcher(self, pair): + """Histogram names the dispatcher.""" + a, _ = pair + rep = td.reduce_trace(td.read_blocks(str(a))) + assert rep["events"] == 8 + assert rep["top"][0] == ("liba.so+0x1000", 4) # D: the wide-margin top + assert rep["runs"] == 8 + + def test_run_length_collapses_repeats(self): + """Run length collapses repeats.""" + assert td.run_length(["D", "D", "D", "R1", "R1"]) == \ + [("D", 3), ("R1", 2)] + + +class TestDiff: + def test_first_divergence_after_the_shared_prefix(self, pair): + """First divergence after the shared prefix.""" + a, b = pair + d = td.diff_traces(td.read_blocks(str(a)), td.read_blocks(str(b))) + fd = d["first_divergence"] + assert fd is not None + # shared prefix D R1 D R2 = 5 events on both sides + assert fd["event_index_a"] == 5 and fd["event_index_b"] == 5 + assert fd["last_common_block"] == "liba.so+0x1000" # the dispatcher + # shared span = prefix D,R1,D,R2,D (5 runs) + trailing D (1) = 6 runs + assert d["identical_run_span"] == 6 + + def test_unique_sets_are_the_input_dependent_blocks(self, pair): + """Unique sets are the input dependent blocks.""" + a, b = pair + d = td.diff_traces(td.read_blocks(str(a)), td.read_blocks(str(b))) + only_a = {k for k, _ in d["unique_to_a"]} + only_b = {k for k, _ in d["unique_to_b"]} + assert only_a == {"liba.so+0x6000", "liba.so+0x8000"} # R5, R7 + assert only_b == {"liba.so+0x4000", "liba.so+0x5000"} # R3, R4 + + def test_identical_traces_report_no_divergence(self, pair): + """Identical traces report no divergence.""" + a, _ = pair + blocks = td.read_blocks(str(a)) + d = td.diff_traces(blocks, blocks) + assert d["first_divergence"] is None + assert d["divergent_regions"] == 0 + assert "NO DIVERGENCE" in td.render_diff(d) + + def test_render_names_every_part(self, pair): + """Render names every part.""" + a, b = pair + d = td.diff_traces(td.read_blocks(str(a)), td.read_blocks(str(b))) + text = td.render_diff(d, top=5) + for marker in ("first divergence", "region 1", "unique to A", "unique to B"): + assert marker in text + + +class TestCli: + def test_reduce_cli(self, pair): + """Reduce cli.""" + a, _ = pair + r = run_script("trace_diff.py", ["reduce", "--log", str(a), "--top", "3"]) + assert r.returncode == 0 and "RESULT=ok" in r.output + assert "liba.so+0x1000" in r.output + + def test_diff_cli_json(self, pair): + """Diff cli json.""" + a, b = pair + r = run_script("trace_diff.py", ["diff", "--a", str(a), "--b", str(b), "--json"]) + assert r.returncode == 0, r.output + doc = json.loads(r.stdout) # --json stdout is one pure document + assert doc["status"] == "ok" + assert doc["first_divergence"]["event_index_a"] == 5 + assert {k for k, _ in doc["unique_to_b"]} == \ + {"liba.so+0x4000", "liba.so+0x5000"} + + def test_missing_file_is_usage_error(self): + """Missing file is usage error.""" + r = run_script("trace_diff.py", ["diff", "--a", "nope.log", "--b", "nada.log"]) + assert r.returncode == 2 + + def test_empty_file_refused_with_the_lesson(self, tmp_path): + """Empty file refused with the lesson.""" + p = tmp_path / "e.log" + p.write_text("DONE reason=timeout blocks=0 blk=0\n", encoding="utf-8") + r = run_script("trace_diff.py", ["reduce", "--log", str(p)]) + assert r.returncode == 2 + assert "capture failure" in r.output or "zero-event" in r.output diff --git a/tests/unit/test_vm_handler_slice.py b/tests/unit/test_vm_handler_slice.py new file mode 100644 index 0000000..1665103 --- /dev/null +++ b/tests/unit/test_vm_handler_slice.py @@ -0,0 +1,167 @@ +"""Unit tests for vm_handler_slice.py: dispatcher detection, slicing, clustering, +the open tail, and the opcode-map trajectory. + +The fixture is a synthetic interpreter trace in the calibrated shape (dispatcher +loop + handler chains + one const-materialization segment + unclosed tail), per +docs/tool-verification/EXTENSION-sampling-crash.md. +""" +import json + +import pytest + +from conftest import load_script, run_script + +vs = load_script("vm_handler_slice.py") + +H = {"D": 0x1000, "load": 0x2000, "arith": 0x3000, "store": 0x4000, + "br": 0x5000, "ret": 0x6000, "mat": 0x7000, "spin": 0x8000} + +RUN = "D load D load arith D store D br D load D load arith store D br D mat D load arith D ret" + +MAP = {"libvm.so+0x2000": "OP_LOAD", "libvm.so+0x3000": "OP_ARITH", + "libvm.so+0x4000": "OP_STORE", "libvm.so+0x5000": "OP_BR", + "libvm.so+0x6000": "OP_RET", "libvm.so+0x7000": "OP_CALL"} + + +@pytest.fixture +def vm_log(tmp_path): + """Vm log.""" + p = tmp_path / "vm.log" + p.write_text("\n".join("BLK %d libvm.so+0x%x" % (i, H[k]) + for i, k in enumerate(RUN.split())) + "\n", + encoding="utf-8") + return p + + +class TestPickDispatcher: + def test_top_plus_predecessor_diversity(self): + """Top plus predecessor diversity.""" + blocks = ["libvm.so+0x%x" % H[k] for k in RUN.split()] + assert vs.pick_dispatcher(blocks) == "libvm.so+0x1000" + + def test_hottest_handler_may_tie_with_dispatcher(self): + """A stack-shaped VM runs OP_LOAD on every guest instruction: D and the + hottest handler occur equally often, and when that handler is a single + block the pair is a genuine 2-cycle (each has one predecessor -- the + other). That is ambiguity, not a dispatcher: refuse and demand an + explicit --dispatcher.""" + seq = "D load D load D load D load D load D load" + blocks = ["libvm.so+0x%x" % H[k] for k in seq.split()] + assert vs.pick_dispatcher(blocks) is None + + def test_tie_with_multiblock_handler_resolves(self): + """Same tie, but the hot handler is two blocks: its tail block is a + predecessor of D that the handler's own head never is, and other + handlers join -- predecessor diversity decides for D.""" + seq = ("D load arith D load arith D load arith " + "D store D load arith D br D load arith") + blocks = ["libvm.so+0x%x" % H[k] for k in seq.split()] + assert vs.pick_dispatcher(blocks) == "libvm.so+0x1000" + + def test_no_loop_refused(self): + """No loop refused.""" + blocks = ["a.so+0x1", "a.so+0x2", "a.so+0x3"] + assert vs.pick_dispatcher(blocks) is None + + def test_frequency_margin_cannot_substitute_for_predecessor_diversity(self): + """A block 3x hotter than the runner-up but entered from one + predecessor is a plain hot loop, not a dispatch loop: the docstring's + rule is top + many distinct predecessors, so no frequency fallback + may rescue it.""" + blocks = ["hot+0x1"] + ["foo+0x2", "hot+0x1"] * 4 + ["bar+0x3", "hot+0x1"] + assert vs.pick_dispatcher(blocks) is None + + def test_single_entry_histogram_is_accepted(self): + """Only one distinct block: nothing to be diverse against.""" + assert vs.pick_dispatcher(["a+0x1"] * 8) == "a+0x1" + + +class TestSlicing: + def test_segments_preamble_tail(self): + """Segments preamble tail.""" + blocks = ["libvm.so+0x%x" % H[k] for k in RUN.split()] + segments, preamble, tail = vs.slice_segments(blocks, "libvm.so+0x1000") + runs = [r for r, _closed in segments] + assert runs[0] == ["libvm.so+0x2000"] # load + assert runs[1] == ["libvm.so+0x2000", "libvm.so+0x3000"] # load arith + assert ["libvm.so+0x7000"] in runs # const materialization + assert segments[-1] == (["libvm.so+0x6000"], False) # ret: unclosed + assert preamble == [] and tail == ["libvm.so+0x6000"] + + def test_preamble_kept_separate(self): + """Preamble kept separate.""" + blocks = ["entry+0x1", "D", "h+0x2", "D", "h+0x3"] + segments, preamble, tail = vs.slice_segments(blocks, "D") + assert preamble == ["entry+0x1"] + assert [r for r, _ in segments] == [["h+0x2"], ["h+0x3"]] + + +class TestCluster: + def test_signatures_and_counts(self): + """Signatures and counts.""" + blocks = ["libvm.so+0x%x" % H[k] for k in RUN.split()] + segments, _, _ = vs.slice_segments(blocks, "libvm.so+0x1000") + trajectory, stats = vs.cluster(segments) + assert len(trajectory) == len(segments) + # the load-only segment occurs twice (positions 0 and 4) + load_sig = vs.signature(["libvm.so+0x2000"]) + assert stats[load_sig]["count"] == 2 + # every trajectory entry is a known signature + assert set(trajectory) <= set(stats) + + def test_spin_loop_diagnostic(self): + """Spin loop diagnostic.""" + blocks = ["a+0x1"] * 20 + segments, _, _ = vs.slice_segments(blocks, "a+0x1") + segments = [(["b+0x2"], True)] * 20 if not segments else segments + trajectory, stats = vs.cluster(segments) + diag = vs.diagnostics(blocks, "a+0x1", [r for r, _ in segments], stats) + assert any("spin" in n for n in diag["notes"]) + + +class TestCli: + def test_full_run_with_opcode_map(self, vm_log, tmp_path): + """Full run with opcode map.""" + mp = tmp_path / "map.json" + mp.write_text(json.dumps(MAP), encoding="utf-8") + r = run_script("vm_handler_slice.py", ["--log", str(vm_log), + "--opcode-map", str(mp)]) + assert r.returncode == 0, r.output + assert "dispatcher: libvm.so+0x1000" in r.output + assert "UNCLOSED" in r.output + assert "OP_CALL" in r.output # const materialization named + assert r.output.rstrip().endswith("RESULT=ok") + + def test_json_shape(self, vm_log, tmp_path): + """Json shape.""" + mp = tmp_path / "map.json" + mp.write_text(json.dumps(MAP), encoding="utf-8") + r = run_script("vm_handler_slice.py", ["--log", str(vm_log), + "--opcode-map", str(mp), "--json"]) + doc = json.loads(r.stdout) # --json stdout is one pure document + assert doc["status"] == "ok" + assert doc["opcode_trajectory"][-1] == "OP_RET" + assert all("sig" in s for s in doc["segments"]) + + def test_explicit_dispatcher(self, vm_log): + """Explicit dispatcher.""" + r = run_script("vm_handler_slice.py", ["--log", str(vm_log), + "--dispatcher", "libvm.so+0x1000"]) + assert r.returncode == 0 and "segments: 10 total" in r.output + + def test_non_vm_trace_refused(self, tmp_path): + """Non vm trace refused.""" + p = tmp_path / "flat.log" + p.write_text("BLK 0 a+0x1\nBLK 1 a+0x2\nBLK 2 a+0x3\n", encoding="utf-8") + r = run_script("vm_handler_slice.py", ["--log", str(p)]) + assert r.returncode == 2 + assert "interpreter" in r.output or "dispatcher" in r.output + + def test_unmapped_handler_blocks_reported(self, vm_log, tmp_path): + """Unmapped handler blocks reported.""" + mp = tmp_path / "map.json" + mp.write_text(json.dumps({"libvm.so+0x2000": "OP_LOAD"}), encoding="utf-8") + r = run_script("vm_handler_slice.py", ["--log", str(vm_log), + "--opcode-map", str(mp)]) + assert r.returncode == 0 + assert "unmapped handler blocks" in r.output