Add a hold-by-name gesture to ui-trace
A row held to select it was the one gesture that could only be written as a pair of coordinates: `tap`'s press is 60ms so nothing sees a long press, and `holddrag` takes numbers and always drags afterwards. `hold 'helper 3'` resolves the label the way `tap` does -- the resolution is now `locate`, shared by both -- and holds the press past the platform's 500ms threshold. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
86cf4e8088
commit
c9be26c6d1
3 files changed
+54
-13
No files matched your search
@@ -175,6 +175,13 @@ measured, which reads exactly like a result rather than like a failure. Asked
|
|||||||
for by Iris on 2026-09-03, after ai-app's two benchmark scripts pressed a
|
for by Iris on 2026-09-03, after ai-app's two benchmark scripts pressed a
|
||||||
header button at `tap 723 205` and that button moved.
|
header button at `tap 723 205` and that button moved.
|
||||||
|
|
||||||
|
`hold 'helper 3'` is the same resolution with a press held past the
|
||||||
|
platform's 500ms long-press threshold, for a row that is held to select it or
|
||||||
|
anything else that distinguishes the two. Neither other action reaches it:
|
||||||
|
`tap`'s press is 60ms, and `holddrag` takes coordinates and always drags
|
||||||
|
afterwards — so the one gesture "hold a list row" needs was the one that could
|
||||||
|
only be written as a pair of numbers.
|
||||||
|
|
||||||
A label that is not on screen ends the recording with `# error action failed`
|
A label that is not on screen ends the recording with `# error action failed`
|
||||||
and a non-zero exit, so a run that could not press what it meant to press
|
and a non-zero exit, so a run that could not press what it meant to press
|
||||||
produces no numbers at all.
|
produces no numbers at all.
|
||||||
|
|||||||
+5
-2
@@ -21,7 +21,9 @@ after seeing the first answer.
|
|||||||
ui-trace show /tmp/t.txt # what moved (the default question)
|
ui-trace show /tmp/t.txt # what moved (the default question)
|
||||||
ui-trace show /tmp/t.txt -m 'Called|ask' # a timeline for those
|
ui-trace show /tmp/t.txt -m 'Called|ask' # a timeline for those
|
||||||
|
|
||||||
Press things by **name**, not by coordinate. `tap 'Save'` resolves the label
|
Press things by **name**, not by coordinate. `tap 'Save'` and `hold 'Save'`
|
||||||
|
(a press held past the platform's long-press threshold, for a list row that is
|
||||||
|
held to select it) resolve the label
|
||||||
against the tree at the moment of the gesture, so it survives anything that
|
against the tree at the moment of the gesture, so it survives anything that
|
||||||
moves the control and fails loudly when the control is genuinely not there.
|
moves the control and fails loudly when the control is genuinely not there.
|
||||||
`tap X Y` still works and is the exception: a coordinate is a position measured
|
`tap X Y` still works and is the exception: a coordinate is a position measured
|
||||||
@@ -367,7 +369,8 @@ def main():
|
|||||||
rec.add_argument("-i", "--interval", type=int, default=16,
|
rec.add_argument("-i", "--interval", type=int, default=16,
|
||||||
help="ms between samples (0 = as fast as possible)")
|
help="ms between samples (0 = as fast as possible)")
|
||||||
rec.add_argument("--do", action="append", default=[], metavar="ACTION",
|
rec.add_argument("--do", action="append", default=[], metavar="ACTION",
|
||||||
help="'wait MS' | 'tap X Y' | 'swipe X1 Y1 X2 Y2 [MS]' | "
|
help="'wait MS' | 'tap X Y' | \"tap 'Label'\" | "
|
||||||
|
"\"hold 'Label'\" | 'swipe X1 Y1 X2 Y2 [MS]' | "
|
||||||
"'holddrag X1 Y1 X2 Y2 HOLD_MS MOVE_MS', repeatable")
|
"'holddrag X1 Y1 X2 Y2 HOLD_MS MOVE_MS', repeatable")
|
||||||
rec.add_argument("-o", "--out", help="where to save the trace")
|
rec.add_argument("-o", "--out", help="where to save the trace")
|
||||||
rec.add_argument("-m", "--match", help="show a timeline for these afterwards")
|
rec.add_argument("-m", "--match", help="show a timeline for these afterwards")
|
||||||
|
|||||||
+42
-11
@@ -33,6 +33,8 @@ public final class UiTrace {
|
|||||||
private static final int MAX_DEPTH = 60;
|
private static final int MAX_DEPTH = 60;
|
||||||
/** How long `tap LABEL` keeps looking before it decides the label is not there. */
|
/** How long `tap LABEL` keeps looking before it decides the label is not there. */
|
||||||
private static final long FIND_TIMEOUT_MS = 3000;
|
private static final long FIND_TIMEOUT_MS = 3000;
|
||||||
|
/** Comfortably past the platform's 500ms long-press threshold, for `hold`. */
|
||||||
|
private static final long HOLD_MS = 800;
|
||||||
|
|
||||||
private final UiAutomation automation;
|
private final UiAutomation automation;
|
||||||
private final PrintWriter out;
|
private final PrintWriter out;
|
||||||
@@ -218,6 +220,7 @@ public final class UiTrace {
|
|||||||
tapLabel(join(step));
|
tapLabel(join(step));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
case "hold" -> holdLabel(join(step));
|
||||||
case "swipe" -> swipe(Integer.parseInt(step[1]), Integer.parseInt(step[2]),
|
case "swipe" -> swipe(Integer.parseInt(step[1]), Integer.parseInt(step[2]),
|
||||||
Integer.parseInt(step[3]), Integer.parseInt(step[4]),
|
Integer.parseInt(step[3]), Integer.parseInt(step[4]),
|
||||||
step.length > 5 ? Long.parseLong(step[5]) : 300);
|
step.length > 5 ? Long.parseLong(step[5]) : 300);
|
||||||
@@ -253,21 +256,48 @@ public final class UiTrace {
|
|||||||
return label;
|
return label;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Presses whatever is currently labelled `label`, wherever it is. */
|
||||||
|
private void tapLabel(String label) {
|
||||||
|
Rect box = locate(label, "tap");
|
||||||
|
tap(box.centerX(), box.centerY());
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Presses whatever is currently labelled `label`, wherever it is.
|
* Presses whatever is labelled `label` and holds it past the platform's long-press
|
||||||
|
* threshold, then lets go without moving.
|
||||||
*
|
*
|
||||||
* This is the action to reach for, and `tap X Y` is the exception. A coordinate is a position
|
* `tap` cannot reach this: its press is 60ms, so nothing that distinguishes a press from a
|
||||||
* measured once by hand: anything that moves the thing being pressed -- a control added to the
|
* long press ever sees one. `holddrag` can hold, but it takes coordinates and always drags
|
||||||
* row, a font size, a density, a different device -- makes the tap land on whatever now sits
|
* afterwards -- so the one gesture a list's "hold a row to select it" needs was the one
|
||||||
* there, and the script then reports a number that was never measured, which reads exactly
|
* gesture that could only be written as a pair of numbers, which is exactly what a script
|
||||||
* like a result. A name is what the control already carries for assistive technology, so it
|
* driving a UI must not contain.
|
||||||
* survives all of that and fails loudly when it genuinely is not there.
|
*/
|
||||||
|
private void holdLabel(String label) {
|
||||||
|
Rect box = locate(label, "hold");
|
||||||
|
int x = box.centerX(), y = box.centerY();
|
||||||
|
long down = SystemClock.uptimeMillis();
|
||||||
|
send(MotionEvent.ACTION_DOWN, down, x, y);
|
||||||
|
SystemClock.sleep(HOLD_MS);
|
||||||
|
send(MotionEvent.ACTION_UP, down, x, y);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where whatever is currently labelled `label` is on screen, for the `verb` about to press
|
||||||
|
* it.
|
||||||
|
*
|
||||||
|
* Pressing by name is the action to reach for, and a coordinate is the exception. A
|
||||||
|
* coordinate is a position measured once by hand: anything that moves the thing being
|
||||||
|
* pressed -- a control added to the row, a font size, a density, a different device -- makes
|
||||||
|
* the press land on whatever now sits there, and the script then reports a number that was
|
||||||
|
* never measured, which reads exactly like a result. A name is what the control already
|
||||||
|
* carries for assistive technology, so it survives all of that and fails loudly when it
|
||||||
|
* genuinely is not there.
|
||||||
*
|
*
|
||||||
* The box is read from the tree at the moment of the gesture rather than from an earlier
|
* The box is read from the tree at the moment of the gesture rather than from an earlier
|
||||||
* frame, because the screen the gesture lands on is the only one that decides where the mark
|
* frame, because the screen the gesture lands on is the only one that decides where the mark
|
||||||
* is.
|
* is.
|
||||||
*/
|
*/
|
||||||
private void tapLabel(String label) {
|
private Rect locate(String label, String verb) {
|
||||||
// Looked for repeatedly for a moment rather than once. Two things make a single look
|
// Looked for repeatedly for a moment rather than once. Two things make a single look
|
||||||
// wrong: the accessibility connection has no window at all for the first frames after it
|
// wrong: the accessibility connection has no window at all for the first frames after it
|
||||||
// is made, so an action at the very start of a trace found nothing to search; and a
|
// is made, so an action at the very start of a trace found nothing to search; and a
|
||||||
@@ -299,11 +329,12 @@ public final class UiTrace {
|
|||||||
throw new IllegalStateException("\"" + label + "\" has no size on screen");
|
throw new IllegalStateException("\"" + label + "\" has no size on screen");
|
||||||
}
|
}
|
||||||
synchronized (out) {
|
synchronized (out) {
|
||||||
out.println("# tap \"" + label + "\" at " + box.centerX() + "," + box.centerY()
|
out.println("# " + verb + " \"" + label + "\" at " + box.centerX() + ","
|
||||||
+ " in " + box.left + "," + box.top + "," + box.right + "," + box.bottom);
|
+ box.centerY() + " in " + box.left + "," + box.top + "," + box.right + ","
|
||||||
|
+ box.bottom);
|
||||||
out.flush();
|
out.flush();
|
||||||
}
|
}
|
||||||
tap(box.centerX(), box.centerY());
|
return box;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Every visible node whose text or description is exactly `label`, in tree order. */
|
/** Every visible node whose text or description is exactly `label`, in tree order. */
|
||||||
|
|||||||
Reference in new issue
Block a user