// The on-device half of ui-trace: samples the accessibility tree in a loop and // prints one text frame per sample. // // This exists because `uiautomator dump` costs about two seconds per call -- // almost all of it starting a JVM and connecting to the accessibility service -- // which is half a hertz, so an animation happens entirely between two samples. // Connecting once and then sampling in a loop is the whole trick; everything // else here is bookkeeping around it. // // Runs under app_process as the shell user, the same way /system/bin/uiautomator // does. Three pieces of the connection are hidden API and are reached by // reflection: android.app.UiAutomationConnection, the (Looper, connection) // constructor of UiAutomation, and its connect() method. There is no public way // for a shell process to obtain a UiAutomation -- the public entry points all // require an Instrumentation and therefore an app -- and this is the same path // the platform's own uiautomator command takes. import android.accessibilityservice.AccessibilityServiceInfo; import android.app.UiAutomation; import android.graphics.Rect; import android.os.HandlerThread; import android.os.Looper; import android.os.SystemClock; import android.view.InputDevice; import android.view.MotionEvent; import android.view.accessibility.AccessibilityNodeInfo; import java.io.PrintWriter; import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.List; public final class UiTrace { private static final int MAX_DEPTH = 60; /** How long `tap LABEL` keeps looking before it decides the label is not there. */ private static final long FIND_TIMEOUT_MS = 3000; private final UiAutomation automation; private final PrintWriter out; private final long origin = SystemClock.uptimeMillis(); private UiTrace(UiAutomation automation, PrintWriter out) { this.automation = automation; this.out = out; } // prepareMainLooper() is deprecated because an app never needs it -- the // Android environment makes the main looper. This is not an app: app_process // starts a bare VM, so there is no main looper and no supported way to ask // for one. The framework needs it anyway (see below), so the deprecated call // is the only route, and it is scoped to this method alone. @SuppressWarnings("deprecation") public static void main(String[] args) { // The accessibility client builds a Handler on the main looper the first // time a callback arrives, and a bare app_process has no main looper, so // connecting from here died with an NPE inside the framework and the // shell reported only "Killed". So the main thread prepares the looper // and then does nothing but run it; the trace itself is a worker. Looper.prepareMainLooper(); Thread worker = new Thread(() -> record(args), "ui-trace"); worker.start(); Looper.loop(); } private static void record(String[] args) { PrintWriter out = new PrintWriter(new java.io.BufferedWriter(new java.io.OutputStreamWriter(System.out), 1 << 16)); try { long duration = 3000; long interval = 0; List script = new ArrayList<>(); for (int i = 0; i < args.length; i++) { switch (args[i]) { case "--duration" -> duration = Long.parseLong(args[++i]); case "--interval" -> interval = Long.parseLong(args[++i]); case "--do" -> script.add(args[++i].trim().split("\\s+")); default -> throw new IllegalArgumentException("unknown option: " + args[i]); } } UiTrace trace = new UiTrace(connect(), out); trace.run(duration, interval, script); out.println("# done"); } catch (Throwable problem) { out.println("# error " + problem); for (StackTraceElement frame : problem.getStackTrace()) out.println("# at " + frame); out.flush(); System.exit(1); } out.flush(); // The accessibility connection keeps non-daemon threads alive, so a // returning main() would hang here rather than ending the trace. System.exit(0); } private static UiAutomation connect() throws Exception { HandlerThread thread = new HandlerThread("ui-trace"); thread.start(); Object connection = Class.forName("android.app.UiAutomationConnection") .getDeclaredConstructor().newInstance(); Constructor ctor = UiAutomation.class.getDeclaredConstructor( Looper.class, Class.forName("android.app.IUiAutomationConnection")); ctor.setAccessible(true); UiAutomation automation = (UiAutomation) ctor.newInstance(thread.getLooper(), connection); UiAutomation.class.getDeclaredMethod("connect").invoke(automation); // Without these two the tree is the one a screen reader would use: // decorative and off-stage nodes are dropped and view ids are withheld, // and both are things a layout question is usually about. AccessibilityServiceInfo info = automation.getServiceInfo(); info.flags |= AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS | AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS; automation.setServiceInfo(info); return automation; } private void run(long duration, long interval, List script) { // The actions run beside the sampling rather than between samples: an // action that blocked the loop would leave a hole in the recording at // exactly the moment being recorded. Thread actor = new Thread(() -> { for (String[] step : script) { try { perform(step); } catch (Exception problem) { // "# error", not a gentler word: the reader of this trace is usually a // benchmark script, and an action that did not happen must fail the run // rather than leave numbers that look measured. ui-trace's own `record` // exits on this line. synchronized (out) { out.println("# error action failed: " + problem); } } } }, "ui-trace-actions"); actor.setDaemon(true); actor.start(); long end = origin + duration; int frame = 0; while (SystemClock.uptimeMillis() < end) { long at = SystemClock.uptimeMillis() - origin; AccessibilityNodeInfo root = automation.getRootInActiveWindow(); synchronized (out) { out.println("F " + frame + " " + at); if (root == null) out.println("# no active window"); else emit(root, 0); out.flush(); } frame++; if (interval > 0) { long slack = interval - (SystemClock.uptimeMillis() - origin - at); if (slack > 0) SystemClock.sleep(slack); } } // Waited for rather than left behind. The actions run beside the sampling, so a script // whose last step outlasts the recording used to have its outcome thrown away by the // exit -- including the "# error" from a `tap` that never found its label, which is the // one line the caller most needs. A run that could not press what it meant to press must // not end quietly. try { actor.join(FIND_TIMEOUT_MS + 1000); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } if (actor.isAlive()) { synchronized (out) { out.println("# error the recording ended before its actions did -- raise -d"); out.flush(); } } } private void emit(AccessibilityNodeInfo node, int depth) { if (node == null || depth > MAX_DEPTH) return; Rect box = new Rect(); node.getBoundsInScreen(box); // Tab-separated with the free text last, because a label may contain // anything at all including the separator of a friendlier format. StringBuilder line = new StringBuilder("N\t"); line.append(depth).append('\t') .append(box.left).append('\t').append(box.top).append('\t') .append(box.right).append('\t').append(box.bottom).append('\t') .append(flags(node)).append('\t') .append(shortName(node.getClassName())).append('\t') .append(clean(node.getViewIdResourceName())).append('\t') .append(clean(node.getText())).append('\t') .append(clean(node.getContentDescription())); out.println(line); for (int i = 0; i < node.getChildCount(); i++) emit(node.getChild(i), depth + 1); } private static String flags(AccessibilityNodeInfo node) { StringBuilder marks = new StringBuilder(); if (node.isClickable()) marks.append('c'); if (node.isScrollable()) marks.append('s'); if (node.isFocused()) marks.append('f'); if (!node.isEnabled()) marks.append('d'); if (!node.isVisibleToUser()) marks.append('h'); return marks.length() == 0 ? "-" : marks.toString(); } private static String shortName(CharSequence name) { String text = clean(name); int dot = text.lastIndexOf('.'); return dot < 0 ? text : text.substring(dot + 1); } private static String clean(CharSequence value) { if (value == null) return ""; return value.toString().replace('\t', ' ').replace('\n', '⏎').replace('\r', ' '); } private void perform(String[] step) throws Exception { long at = SystemClock.uptimeMillis() - origin; synchronized (out) { out.println("A " + at + "\t" + String.join(" ", step)); out.flush(); } switch (step[0]) { case "wait" -> SystemClock.sleep(Long.parseLong(step[1])); case "tap" -> { if (step.length == 3 && isNumber(step[1]) && isNumber(step[2])) { tap(Integer.parseInt(step[1]), Integer.parseInt(step[2])); } else { tapLabel(join(step)); } } case "swipe" -> swipe(Integer.parseInt(step[1]), Integer.parseInt(step[2]), Integer.parseInt(step[3]), Integer.parseInt(step[4]), step.length > 5 ? Long.parseLong(step[5]) : 300); case "holddrag" -> holdDrag(Integer.parseInt(step[1]), Integer.parseInt(step[2]), Integer.parseInt(step[3]), Integer.parseInt(step[4]), Long.parseLong(step[5]), Long.parseLong(step[6])); default -> throw new IllegalArgumentException("unknown action: " + step[0]); } } private static boolean isNumber(String value) { if (value.isEmpty()) return false; for (int i = 0; i < value.length(); i++) { if (!Character.isDigit(value.charAt(i))) return false; } return true; } /** Everything after the verb, as one label, with any quotes the shell left on it removed. */ private static String join(String[] step) { StringBuilder joined = new StringBuilder(); for (int i = 1; i < step.length; i++) { if (i > 1) joined.append(' '); joined.append(step[i]); } String label = joined.toString(); if (label.length() >= 2) { char first = label.charAt(0), last = label.charAt(label.length() - 1); if ((first == '\'' || first == '"') && first == last) { label = label.substring(1, label.length() - 1); } } return label; } /** * Presses whatever is currently labelled `label`, wherever it is. * * This is the action to reach for, and `tap X Y` is the exception. A coordinate is a position * measured once by hand: anything that moves the thing being pressed -- a control added to the * row, a font size, a density, a different device -- makes the tap land on whatever now sits * there, and the script then reports a number that was never measured, which reads exactly * like a result. A name is what the control already carries for assistive technology, so it * survives all of that and fails loudly when it genuinely is not there. * * The box is read from the tree at the moment of the gesture rather than from an earlier * frame, because the screen the gesture lands on is the only one that decides where the mark * is. */ private void tapLabel(String label) { // Looked for repeatedly for a moment rather than once. Two things make a single look // wrong: the accessibility connection has no window at all for the first frames after it // is made, so an action at the very start of a trace found nothing to search; and a // control revealed by the step before this one -- a dialog's button, a row that has just // loaded -- arrives a frame or two later. The wait is bounded and the failure is still // loud, which is the whole point: what must not happen is pressing the wrong thing. List found = new ArrayList<>(); long giveUp = SystemClock.uptimeMillis() + FIND_TIMEOUT_MS; while (true) { AccessibilityNodeInfo root = automation.getRootInActiveWindow(); if (root != null) collect(root, label, 0, found); if (!found.isEmpty() || SystemClock.uptimeMillis() >= giveUp) break; SystemClock.sleep(50); } if (found.isEmpty()) { throw new IllegalArgumentException("nothing on screen is labelled \"" + label + "\" (looked for " + FIND_TIMEOUT_MS + "ms)"); } // The clickable one where there is a choice: a label and the padded control wrapping it // both carry the text, and the control is the thing a finger is meant to find. Its centre // is also where a ripple is drawn, so this presses what the reader would have pressed. AccessibilityNodeInfo target = found.get(0); for (AccessibilityNodeInfo candidate : found) { if (candidate.isClickable()) { target = candidate; break; } } Rect box = new Rect(); target.getBoundsInScreen(box); if (box.width() <= 0 || box.height() <= 0) { throw new IllegalStateException("\"" + label + "\" has no size on screen"); } synchronized (out) { out.println("# tap \"" + label + "\" at " + box.centerX() + "," + box.centerY() + " in " + box.left + "," + box.top + "," + box.right + "," + box.bottom); out.flush(); } tap(box.centerX(), box.centerY()); } /** Every visible node whose text or description is exactly `label`, in tree order. */ private static void collect(AccessibilityNodeInfo node, String label, int depth, List found) { if (node == null || depth > MAX_DEPTH) return; if (node.isVisibleToUser() && (label.equals(clean(node.getText())) || label.equals(clean(node.getContentDescription())))) { found.add(node); } for (int i = 0; i < node.getChildCount(); i++) collect(node.getChild(i), label, depth + 1, found); } private void send(int action, long down, int x, int y) { MotionEvent event = MotionEvent.obtain(down, SystemClock.uptimeMillis(), action, x, y, 0); event.setSource(InputDevice.SOURCE_TOUCHSCREEN); automation.injectInputEvent(event, true); event.recycle(); } private void tap(int x, int y) { long down = SystemClock.uptimeMillis(); send(MotionEvent.ACTION_DOWN, down, x, y); SystemClock.sleep(60); send(MotionEvent.ACTION_UP, down, x, y); } // Moves are emitted on a fixed 10ms cadence so the velocity tracker behind a // fling sees a real gesture. `input swipe` interpolates over far fewer // points, which is why it so often scrolls without ever flinging. private void swipe(int x1, int y1, int x2, int y2, long millis) { long down = SystemClock.uptimeMillis(); send(MotionEvent.ACTION_DOWN, down, x1, y1); int steps = (int) Math.max(2, millis / 10); for (int i = 1; i <= steps; i++) { float part = (float) i / steps; long due = down + (long) (millis * part); long slack = due - SystemClock.uptimeMillis(); if (slack > 0) SystemClock.sleep(slack); send(MotionEvent.ACTION_MOVE, down, Math.round(x1 + (x2 - x1) * part), Math.round(y1 + (y2 - y1) * part)); } send(MotionEvent.ACTION_UP, down, x2, y2); } /** * A press held stationary for `holdMs`, then moved to `(x2, y2)` over `moveMs`, then * released -- one continuous touch, never lifted between the hold and the move. * * Neither existing action can produce this. `tap` has no hold at all, and `swipe` * interpolates motion across its *whole* duration starting at t=0, so a long `swipe` with a * short first segment is still continuous motion throughout, never a hold followed by a * drag. That gap matters for anything that decides pan-vs-select the way Android itself * does -- a stationary press held past `LONG_PRESS` (500ms) starts a selection, which * further drag then extends -- because neither of the other two actions can reach the * "starts a selection" branch at all. * * The move phase reuses `swipe`'s own fixed 10ms cadence for the same reason: a velocity * tracker needs a real sequence of points, not two. */ private void holdDrag(int x1, int y1, int x2, int y2, long holdMs, long moveMs) { long down = SystemClock.uptimeMillis(); send(MotionEvent.ACTION_DOWN, down, x1, y1); SystemClock.sleep(holdMs); int steps = (int) Math.max(2, moveMs / 10); long moveStart = SystemClock.uptimeMillis(); for (int i = 1; i <= steps; i++) { float part = (float) i / steps; long due = moveStart + (long) (moveMs * part); long slack = due - SystemClock.uptimeMillis(); if (slack > 0) SystemClock.sleep(slack); send(MotionEvent.ACTION_MOVE, down, Math.round(x1 + (x2 - x1) * part), Math.round(y1 + (y2 - y1) * part)); } send(MotionEvent.ACTION_UP, down, x2, y2); } }