irisandClaude Fable 5.1
86cf4e8088
Neither existing action can produce "hold stationary for LONG_PRESS, then move without lifting, then release": tap has no hold at all, and swipe X1 Y1 X2 Y2 MS interpolates motion across its whole duration starting at t=0, so a long swipe with a short first segment is still continuous motion throughout, never a hold followed by a drag. holddrag X1 Y1 X2 Y2 HOLD_MS MOVE_MS extends the same MotionEvent/injectInputEvent mechanism swipe already uses: DOWN, sleep HOLD_MS, then MOVE at the same fixed 10ms cadence swipe uses over MOVE_MS, then UP -- one continuous touch. Additive; existing commands unchanged. Verified against a real device (this repo has no unit test harness, so verification is driving a device, matching its existing posture): `ui-trace record --do "holddrag 300 1850 300 2050 600 300"` against ai-app-2's iris transcript screen produced a real long-press-then-drag selection (confirmed by the app's own logcat and a screenshot showing the resulting highlighted selection) that neither tap nor swipe could reach. javac --release 17 -Xlint:all -Werror clean. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
409 lines
18 KiB
Python
Executable File
409 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Record what an Android screen does, as text, and ask questions of it later.
|
|
|
|
Why this exists. `adb shell uiautomator dump` is the usual way to read a
|
|
screen's layout, and it costs about two seconds a call -- so an animation, a
|
|
scroll or a settling layout happens entirely in the gap between two samples,
|
|
and the only thing it can answer is what the screen looked like once it had
|
|
stopped. This drives a small resident recorder on the device instead (see
|
|
UiTrace.java beside this script), which connects to the accessibility service
|
|
once and then samples at whatever rate is asked for; 60Hz is comfortable.
|
|
|
|
The other half is that a screen has a few hundred nodes and a question is
|
|
almost always about two or three of them. So recording and reading are separate
|
|
steps: `record` saves everything, and `show` selects. A trace can be
|
|
re-interrogated with a different selector without touching the device again,
|
|
which matters because the interesting question is usually the one you think of
|
|
after seeing the first answer.
|
|
|
|
ui-trace record -d 3000 --do "tap 'Session settings'" -o /tmp/t.txt
|
|
ui-trace elements /tmp/t.txt # what is on screen, to pick from
|
|
ui-trace show /tmp/t.txt # what moved (the default question)
|
|
ui-trace show /tmp/t.txt -m 'Called|ask' # a timeline for those
|
|
|
|
Press things by **name**, not by coordinate. `tap 'Save'` resolves the label
|
|
against the tree at the moment of the gesture, so it survives anything that
|
|
moves the control and fails loudly when the control is genuinely not there.
|
|
`tap X Y` still works and is the exception: a coordinate is a position measured
|
|
once by hand, and the first thing that moves the control makes the tap land on
|
|
whatever now sits there -- which reads exactly like a result.
|
|
|
|
Nothing here is specific to any app.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
SHARE = Path.home() / ".local/share/ui-trace"
|
|
DEVICE_JAR = "/data/local/tmp/uitrace.jar"
|
|
|
|
|
|
def find_adb():
|
|
"""The adb to drive the device with, without assuming anything about PATH.
|
|
|
|
The one installed beside this script comes first, and that is the point
|
|
rather than a convenience: it is the wrapper that fills in `-s` from the
|
|
checkout you are standing in, and without it a machine with two emulators
|
|
attached answers `failed to get feature set: more than one device`. PATH
|
|
order cannot be relied on for that -- the SDK's own platform-tools sits
|
|
ahead of ~/.local/bin in this machine's ambient PATH, and only a project's
|
|
android-env.sh puts it back.
|
|
"""
|
|
sibling = Path(__file__).resolve().parent / "adb"
|
|
if sibling.is_file() and os.access(sibling, os.X_OK):
|
|
return str(sibling)
|
|
for candidate in [
|
|
os.environ.get("ADB"),
|
|
"adb",
|
|
os.environ.get("ANDROID_HOME", "") + "/platform-tools/adb",
|
|
os.environ.get("ANDROID_SDK_ROOT", "") + "/platform-tools/adb",
|
|
str(Path.home() / "Android/Sdk/platform-tools/adb"),
|
|
]:
|
|
if not candidate or candidate.startswith("/platform-tools"):
|
|
continue
|
|
which = subprocess.run(["sh", "-c", f"command -v {candidate!r}"],
|
|
capture_output=True, text=True)
|
|
if which.returncode == 0:
|
|
return which.stdout.strip()
|
|
sys.exit("ui-trace: no adb found; set ANDROID_HOME or ADB")
|
|
|
|
|
|
def ensure_jar():
|
|
"""Build the device recorder if it is missing or older than its source."""
|
|
jar, source = SHARE / "uitrace.jar", SHARE / "UiTrace.java"
|
|
if not source.exists():
|
|
sys.exit(f"ui-trace: recorder source missing at {source}")
|
|
if not jar.exists() or jar.stat().st_mtime < source.stat().st_mtime:
|
|
print("ui-trace: building the device recorder...", file=sys.stderr)
|
|
subprocess.run([str(SHARE / "build.sh")], check=True,
|
|
stdout=subprocess.DEVNULL)
|
|
return jar
|
|
|
|
|
|
# --- recording -------------------------------------------------------------
|
|
|
|
def record(args):
|
|
jar = ensure_jar()
|
|
adb = find_adb()
|
|
target = ["-s", args.serial] if args.serial else []
|
|
pushed = subprocess.run([adb, *target, "push", str(jar), DEVICE_JAR],
|
|
capture_output=True, text=True)
|
|
if pushed.returncode != 0:
|
|
# Two emulators running is the usual cause, and adb says so plainly; there is
|
|
# nothing to add to its own message except which flag fixes it.
|
|
sys.exit(f"ui-trace: {pushed.stderr.strip() or 'adb push failed'}"
|
|
f"{'' if target else ' (use -s SERIAL to pick a device)'}")
|
|
remote = f"CLASSPATH={DEVICE_JAR} app_process /system/bin UiTrace"
|
|
remote += f" --duration {args.duration} --interval {args.interval}"
|
|
for step in args.do:
|
|
remote += f" --do {step!r}"
|
|
out = Path(args.out) if args.out else Path(
|
|
tempfile.mkstemp(prefix="ui-trace-", suffix=".txt")[1])
|
|
with out.open("w") as handle:
|
|
result = subprocess.run([adb, *target, "shell", remote], stdout=handle,
|
|
stderr=subprocess.STDOUT)
|
|
text = out.read_text()
|
|
if result.returncode != 0 or "# error" in text:
|
|
# The error lines rather than the head of the file. An action that failed -- a
|
|
# `tap` whose label is not on screen -- reports itself part way through a trace
|
|
# that may be thousands of frames long, so printing the beginning showed a
|
|
# perfectly ordinary first frame and said nothing about what went wrong.
|
|
problems = [line for line in text.splitlines() if line.startswith(("# error", "# at"))]
|
|
print("\n".join(problems[:40]) if problems else text[:2000], file=sys.stderr)
|
|
sys.exit("ui-trace: the device recorder failed")
|
|
frames, _ = parse(out)
|
|
print(f"ui-trace: {len(frames)} frames over {frames[-1][0] if frames else 0}ms "
|
|
f"-> {out}", file=sys.stderr)
|
|
return out
|
|
|
|
|
|
# --- reading ---------------------------------------------------------------
|
|
|
|
class Node:
|
|
__slots__ = ("depth", "box", "flags", "cls", "rid", "text", "desc", "label")
|
|
|
|
def __init__(self, parts):
|
|
self.depth = int(parts[1])
|
|
self.box = tuple(int(v) for v in parts[2:6])
|
|
self.flags, self.cls, self.rid, self.text, self.desc = parts[6:11]
|
|
self.label = (self.text or self.desc or self.rid.split("/")[-1]
|
|
or self.cls or "?")
|
|
|
|
def haystack(self):
|
|
return "\t".join((self.label, self.rid, self.cls, self.text, self.desc))
|
|
|
|
|
|
def parse(path):
|
|
"""A trace file into (frames, actions). A frame is (t_ms, [Node]).
|
|
|
|
A missing or unreadable trace exits with one line rather than a stack trace: the
|
|
caller is usually reading this in a terminal or a context window, and a traceback
|
|
says nothing a sentence cannot.
|
|
"""
|
|
frames, actions, current = [], [], None
|
|
try:
|
|
text = Path(path).read_text()
|
|
except OSError as problem:
|
|
sys.exit(f"ui-trace: cannot read {path}: {problem.strerror}")
|
|
for line in text.splitlines():
|
|
if line.startswith("F "):
|
|
_, _, at = line.split(" ", 2)
|
|
current = (int(at), [])
|
|
frames.append(current)
|
|
elif line.startswith("N\t"):
|
|
parts = line.split("\t")
|
|
# A label may itself be empty, so pad rather than trusting the count.
|
|
parts += [""] * (11 - len(parts))
|
|
if current:
|
|
current[1].append(Node(parts))
|
|
elif line.startswith("A "):
|
|
at, what = line[2:].split("\t", 1)
|
|
actions.append((int(at), what))
|
|
return frames, actions
|
|
|
|
|
|
def keyed(nodes, pattern):
|
|
"""Selected nodes by a stable key: the label, plus an index when repeated."""
|
|
seen, out = {}, {}
|
|
for node in nodes:
|
|
if pattern and not pattern.search(node.haystack()):
|
|
continue
|
|
count = seen.get(node.label, 0)
|
|
seen[node.label] = count + 1
|
|
out[node.label if count == 0 else f"{node.label} #{count + 1}"] = node
|
|
return out
|
|
|
|
|
|
def sizes(frames, pattern):
|
|
"""{key: set of (width, height)} -- see [summarise] for why this is tracked."""
|
|
seen = {}
|
|
for _, nodes in frames:
|
|
for key, node in keyed(nodes, pattern).items():
|
|
box = node.box
|
|
seen.setdefault(key, set()).add((box[2] - box[0], box[3] - box[1]))
|
|
return seen
|
|
|
|
|
|
def series(frames, pattern, field):
|
|
"""{key: [(t, value or None)]} over every frame, for the chosen field."""
|
|
pick = {"top": lambda b: b[1], "bottom": lambda b: b[3],
|
|
"left": lambda b: b[0], "right": lambda b: b[2],
|
|
"box": lambda b: b}[field]
|
|
tracks, order = {}, []
|
|
for index, (at, nodes) in enumerate(frames):
|
|
present = keyed(nodes, pattern)
|
|
for key, node in present.items():
|
|
if key not in tracks:
|
|
# Backfilled as absent, so every track is one value per frame and
|
|
# a row can be read across by index. An element that appears
|
|
# part-way through is the common case, not the exotic one.
|
|
tracks[key] = [(frames[i][0], None) for i in range(index)]
|
|
order.append(key)
|
|
tracks[key].append((at, pick(node.box)))
|
|
for key in tracks:
|
|
if key not in present:
|
|
tracks[key].append((at, None))
|
|
return order, tracks
|
|
|
|
|
|
def render(value):
|
|
if value is None:
|
|
return "-"
|
|
if isinstance(value, tuple):
|
|
return "{},{}..{},{}".format(*value)
|
|
return str(value)
|
|
|
|
|
|
def cap(rows, limit):
|
|
"""The first [limit] rows, saying how many were left out.
|
|
|
|
Every listing here shortens the same way, because the caller is usually an
|
|
agent with a context window and a screen has hundreds of nodes.
|
|
"""
|
|
if limit <= 0 or len(rows) <= limit:
|
|
return rows
|
|
print(f" (showing {limit} of {len(rows)}; --limit 0 for all)")
|
|
return rows[:limit]
|
|
|
|
|
|
def cmd_elements(args):
|
|
frames, _ = parse(args.trace)
|
|
counts = {}
|
|
for _, nodes in frames:
|
|
for node in nodes:
|
|
# A node whose only name is its class name names nothing -- a screen has
|
|
# hundreds of those and listing them is the noise this tool exists to cut.
|
|
if not args.all and not (node.text or node.desc or node.rid):
|
|
continue
|
|
entry = counts.setdefault(node.label, [0, node.cls, node.rid, node.flags])
|
|
entry[0] += 1
|
|
ranked = sorted(counts.items(), key=lambda kv: -kv[1][0])
|
|
print(f"{len(counts)} named labels over {len(frames)} frames"
|
|
+ ("" if args.all else " (--all also lists unnamed structure)"))
|
|
for label, (count, cls, rid, flags) in cap(ranked, args.limit):
|
|
mark = f" [{flags}]" if flags != "-" else ""
|
|
print(f" {count:5d}x {label[:44]:<44} {cls}{mark}"
|
|
+ (f" {rid.split('/')[-1]}" if rid else ""))
|
|
|
|
|
|
def cmd_show(args):
|
|
frames, actions = parse(args.trace)
|
|
if not frames:
|
|
sys.exit("ui-trace: no frames in that trace")
|
|
pattern = re.compile(args.match) if args.match else None
|
|
order, tracks = series(frames, pattern, args.field)
|
|
if not pattern:
|
|
summarise(order, tracks, sizes(frames, pattern), actions, args)
|
|
return
|
|
timeline(frames, order, tracks, actions, args)
|
|
|
|
|
|
def summarise(order, tracks, shapes, actions, args):
|
|
"""What moved -- the question a trace is usually recorded to answer.
|
|
|
|
Everything that held still is dropped, because a screen is mostly things
|
|
that held still and reading them is the cost this tool exists to avoid.
|
|
"""
|
|
for at, what in actions:
|
|
print(f" action t={at}ms {what}")
|
|
moved = []
|
|
for key in order:
|
|
points = [(t, v) for t, v in tracks[key] if v is not None]
|
|
if not points:
|
|
continue
|
|
values = [v for _, v in points]
|
|
if len(set(values)) > 1:
|
|
moved.append((key, points))
|
|
total = len(order)
|
|
print(f"{len(moved)} of {total} elements changed {args.field} during the trace")
|
|
for key, points in cap(moved, args.limit):
|
|
values = [v for _, v in points]
|
|
changes = [(t, v) for i, (t, v) in enumerate(points)
|
|
if i == 0 or v != points[i - 1][1]]
|
|
span = f"t={changes[1][0]}..{changes[-1][0]}ms" if len(changes) > 1 else ""
|
|
net = values[-1] - values[0] if not isinstance(values[0], tuple) else None
|
|
arrow = f"{render(values[0])} -> {render(values[-1])}"
|
|
delta = f" ({net:+d})" if net is not None else ""
|
|
# A label is only a name, and two different nodes can share one -- a text
|
|
# and the padded control wrapping it both read as their text. When the
|
|
# size changes too, the two readings are probably not the same node, and
|
|
# the move is partly the difference between them rather than travel.
|
|
shape = shapes.get(key, set())
|
|
note = f" [size changed {len(shape)}x -- may be different nodes]" if len(shape) > 1 else ""
|
|
print(f" {key[:40]:<40} {arrow}{delta} {len(changes) - 1} steps {span}{note}")
|
|
if not moved:
|
|
print(" (nothing moved)")
|
|
|
|
|
|
def timeline(frames, order, tracks, actions, args):
|
|
"""A row per change, not a row per frame.
|
|
|
|
Consecutive frames that say the same thing are one row with a time range:
|
|
at 60Hz a settled second is sixty identical rows, and the reader wants the
|
|
moment it stopped being settled.
|
|
"""
|
|
# "Nothing matched" and "matched but never moved" are different answers, and
|
|
# an empty table says neither of them.
|
|
if not order:
|
|
print(f" nothing matched /{args.match}/ in any of {len(frames)} frames."
|
|
f" Try `ui-trace elements` to see what is there.")
|
|
return
|
|
if len(order) > args.max_columns:
|
|
print(f" {len(order)} elements match, which is too many to read across."
|
|
f" Narrowing to the first {args.max_columns}; use a tighter -m,"
|
|
f" or --max-columns to widen.")
|
|
order = order[:args.max_columns]
|
|
width = max(12, max((len(k) for k in order), default=12))
|
|
width = min(width, 24)
|
|
columns = [k[:width] for k in order]
|
|
print(" " + "t (ms)".rjust(13) + " | " + " | ".join(c.ljust(width) for c in columns))
|
|
print(" " + "-" * 13 + "-+-" + "-+-".join("-" * width for _ in columns))
|
|
pending = list(actions)
|
|
previous, start, last = None, None, None
|
|
rows = [[t for t, _ in tracks[order[0]]]] if order else [[]]
|
|
|
|
def flush():
|
|
if previous is None:
|
|
return
|
|
label = f"{start}" if start == last else f"{start}-{last}"
|
|
print(" " + label.rjust(13) + " | "
|
|
+ " | ".join(render(v).ljust(width) for v in previous))
|
|
|
|
rows_out = 0
|
|
for index, (at, _) in enumerate(frames):
|
|
values = tuple(tracks[k][index][1] for k in order)
|
|
while pending and pending[0][0] <= at:
|
|
flush()
|
|
previous = None
|
|
print(f" >>> {pending[0][1]} @ t={pending[0][0]}ms")
|
|
pending.pop(0)
|
|
if values != previous:
|
|
flush()
|
|
rows_out += 1
|
|
if args.limit > 0 and rows_out > args.limit:
|
|
print(f" ... more changes after t={at}ms (--limit 0 for all)")
|
|
return
|
|
previous, start = values, at
|
|
last = at
|
|
flush()
|
|
for at, what in pending:
|
|
print(f" >>> {what} @ t={at}ms")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
prog="ui-trace", description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
subs = parser.add_subparsers(dest="command", required=True)
|
|
|
|
rec = subs.add_parser("record", help="record a trace from a device")
|
|
rec.add_argument("-s", "--serial", help="adb device serial")
|
|
rec.add_argument("-d", "--duration", type=int, default=3000, help="ms to record")
|
|
rec.add_argument("-i", "--interval", type=int, default=16,
|
|
help="ms between samples (0 = as fast as possible)")
|
|
rec.add_argument("--do", action="append", default=[], metavar="ACTION",
|
|
help="'wait MS' | 'tap X Y' | 'swipe X1 Y1 X2 Y2 [MS]' | "
|
|
"'holddrag X1 Y1 X2 Y2 HOLD_MS MOVE_MS', repeatable")
|
|
rec.add_argument("-o", "--out", help="where to save the trace")
|
|
rec.add_argument("-m", "--match", help="show a timeline for these afterwards")
|
|
rec.add_argument("--field", default="top",
|
|
choices=["top", "bottom", "left", "right", "box"])
|
|
rec.add_argument("--max-columns", type=int, default=6)
|
|
rec.add_argument("--limit", type=int, default=10, help="max rows printed; 0 for all")
|
|
|
|
for name, help_text in [("show", "what moved, or a timeline with -m"),
|
|
("elements", "every label in a trace, to pick from")]:
|
|
sub = subs.add_parser(name, help=help_text)
|
|
sub.add_argument("trace")
|
|
if name == "show":
|
|
sub.add_argument("-m", "--match", help="regex over label, id, class, text")
|
|
sub.add_argument("--field", default="top",
|
|
choices=["top", "bottom", "left", "right", "box"])
|
|
sub.add_argument("--max-columns", type=int, default=6)
|
|
sub.add_argument("--limit", type=int, default=10,
|
|
help="max rows printed; 0 for all")
|
|
else:
|
|
sub.add_argument("--limit", type=int, default=20,
|
|
help="max rows printed; 0 for all")
|
|
sub.add_argument("--all", action="store_true",
|
|
help="include nodes with no name of their own")
|
|
|
|
args = parser.parse_args()
|
|
if args.command == "record":
|
|
out = record(args)
|
|
args.trace = out
|
|
cmd_show(args)
|
|
elif args.command == "show":
|
|
cmd_show(args)
|
|
else:
|
|
cmd_elements(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|