diff --git a/README.md b/README.md index 7b57d71..73633ac 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,13 @@ measured, which reads exactly like a result rather than like a failure. Asked for by Iris on 2026-09-03, after ai-app's two benchmark scripts pressed a header button at `tap 723 205` and that button moved. +`hold 'helper 3'` is the same resolution with a press held past the +platform's 500ms long-press threshold, for a row that is held to select it or +anything else that distinguishes the two. Neither other action reaches it: +`tap`'s press is 60ms, and `holddrag` takes coordinates and always drags +afterwards — so the one gesture "hold a list row" needs was the one that could +only be written as a pair of numbers. + A label that is not on screen ends the recording with `# error action failed` and a non-zero exit, so a run that could not press what it meant to press produces no numbers at all. diff --git a/bin/ui-trace b/bin/ui-trace index 63a6cef..61cb109 100755 --- a/bin/ui-trace +++ b/bin/ui-trace @@ -21,7 +21,9 @@ after seeing the first answer. ui-trace show /tmp/t.txt # what moved (the default question) ui-trace show /tmp/t.txt -m 'Called|ask' # a timeline for those -Press things by **name**, not by coordinate. `tap 'Save'` resolves the label +Press things by **name**, not by coordinate. `tap 'Save'` and `hold 'Save'` +(a press held past the platform's long-press threshold, for a list row that is +held to select it) resolve the label against the tree at the moment of the gesture, so it survives anything that moves the control and fails loudly when the control is genuinely not there. `tap X Y` still works and is the exception: a coordinate is a position measured @@ -367,7 +369,8 @@ def main(): rec.add_argument("-i", "--interval", type=int, default=16, help="ms between samples (0 = as fast as possible)") rec.add_argument("--do", action="append", default=[], metavar="ACTION", - help="'wait MS' | 'tap X Y' | 'swipe X1 Y1 X2 Y2 [MS]' | " + help="'wait MS' | 'tap X Y' | \"tap 'Label'\" | " + "\"hold 'Label'\" | 'swipe X1 Y1 X2 Y2 [MS]' | " "'holddrag X1 Y1 X2 Y2 HOLD_MS MOVE_MS', repeatable") rec.add_argument("-o", "--out", help="where to save the trace") rec.add_argument("-m", "--match", help="show a timeline for these afterwards") diff --git a/share/ui-trace/UiTrace.java b/share/ui-trace/UiTrace.java index c08544d..d21f2d3 100644 --- a/share/ui-trace/UiTrace.java +++ b/share/ui-trace/UiTrace.java @@ -33,6 +33,8 @@ public final class UiTrace { private static final int MAX_DEPTH = 60; /** How long `tap LABEL` keeps looking before it decides the label is not there. */ private static final long FIND_TIMEOUT_MS = 3000; + /** Comfortably past the platform's 500ms long-press threshold, for `hold`. */ + private static final long HOLD_MS = 800; private final UiAutomation automation; private final PrintWriter out; @@ -218,6 +220,7 @@ public final class UiTrace { tapLabel(join(step)); } } + case "hold" -> holdLabel(join(step)); case "swipe" -> swipe(Integer.parseInt(step[1]), Integer.parseInt(step[2]), Integer.parseInt(step[3]), Integer.parseInt(step[4]), step.length > 5 ? Long.parseLong(step[5]) : 300); @@ -253,21 +256,48 @@ public final class UiTrace { return label; } + /** Presses whatever is currently labelled `label`, wherever it is. */ + private void tapLabel(String label) { + Rect box = locate(label, "tap"); + tap(box.centerX(), box.centerY()); + } + /** - * Presses whatever is currently labelled `label`, wherever it is. + * Presses whatever is labelled `label` and holds it past the platform's long-press + * threshold, then lets go without moving. * - * This is the action to reach for, and `tap X Y` is the exception. A coordinate is a position - * measured once by hand: anything that moves the thing being pressed -- a control added to the - * row, a font size, a density, a different device -- makes the tap land on whatever now sits - * there, and the script then reports a number that was never measured, which reads exactly - * like a result. A name is what the control already carries for assistive technology, so it - * survives all of that and fails loudly when it genuinely is not there. + * `tap` cannot reach this: its press is 60ms, so nothing that distinguishes a press from a + * long press ever sees one. `holddrag` can hold, but it takes coordinates and always drags + * afterwards -- so the one gesture a list's "hold a row to select it" needs was the one + * gesture that could only be written as a pair of numbers, which is exactly what a script + * driving a UI must not contain. + */ + private void holdLabel(String label) { + Rect box = locate(label, "hold"); + int x = box.centerX(), y = box.centerY(); + long down = SystemClock.uptimeMillis(); + send(MotionEvent.ACTION_DOWN, down, x, y); + SystemClock.sleep(HOLD_MS); + send(MotionEvent.ACTION_UP, down, x, y); + } + + /** + * Where whatever is currently labelled `label` is on screen, for the `verb` about to press + * it. + * + * Pressing by name is the action to reach for, and a coordinate is the exception. A + * coordinate is a position measured once by hand: anything that moves the thing being + * pressed -- a control added to the row, a font size, a density, a different device -- makes + * the press land on whatever now sits there, and the script then reports a number that was + * never measured, which reads exactly like a result. A name is what the control already + * carries for assistive technology, so it survives all of that and fails loudly when it + * genuinely is not there. * * The box is read from the tree at the moment of the gesture rather than from an earlier * frame, because the screen the gesture lands on is the only one that decides where the mark * is. */ - private void tapLabel(String label) { + private Rect locate(String label, String verb) { // Looked for repeatedly for a moment rather than once. Two things make a single look // wrong: the accessibility connection has no window at all for the first frames after it // is made, so an action at the very start of a trace found nothing to search; and a @@ -299,11 +329,12 @@ public final class UiTrace { throw new IllegalStateException("\"" + label + "\" has no size on screen"); } synchronized (out) { - out.println("# tap \"" + label + "\" at " + box.centerX() + "," + box.centerY() - + " in " + box.left + "," + box.top + "," + box.right + "," + box.bottom); + out.println("# " + verb + " \"" + label + "\" at " + box.centerX() + "," + + box.centerY() + " in " + box.left + "," + box.top + "," + box.right + "," + + box.bottom); out.flush(); } - tap(box.centerX(), box.centerY()); + return box; } /** Every visible node whose text or description is exactly `label`, in tree order. */