// The on-device half of ui-trace: samples the accessibility tree in a loop and // prints one text frame per sample. // // This exists because `uiautomator dump` costs about two seconds per call -- // almost all of it starting a JVM and connecting to the accessibility service -- // which is half a hertz, so an animation happens entirely between two samples. // Connecting once and then sampling in a loop is the whole trick; everything // else here is bookkeeping around it. // // Runs under app_process as the shell user, the same way /system/bin/uiautomator // does. Three pieces of the connection are hidden API and are reached by // reflection: android.app.UiAutomationConnection, the (Looper, connection) // constructor of UiAutomation, and its connect() method. There is no public way // for a shell process to obtain a UiAutomation -- the public entry points all // require an Instrumentation and therefore an app -- and this is the same path // the platform's own uiautomator command takes. import android.accessibilityservice.AccessibilityServiceInfo; import android.app.UiAutomation; import android.graphics.Rect; import android.os.HandlerThread; import android.os.Looper; import android.os.SystemClock; import android.view.InputDevice; import android.view.MotionEvent; import android.view.accessibility.AccessibilityNodeInfo; import java.io.PrintWriter; import java.lang.reflect.Constructor; import java.util.ArrayList; import java.util.List; public final class UiTrace { private static final int MAX_DEPTH = 60; private final UiAutomation automation; private final PrintWriter out; private final long origin = SystemClock.uptimeMillis(); private UiTrace(UiAutomation automation, PrintWriter out) { this.automation = automation; this.out = out; } // prepareMainLooper() is deprecated because an app never needs it -- the // Android environment makes the main looper. This is not an app: app_process // starts a bare VM, so there is no main looper and no supported way to ask // for one. The framework needs it anyway (see below), so the deprecated call // is the only route, and it is scoped to this method alone. @SuppressWarnings("deprecation") public static void main(String[] args) { // The accessibility client builds a Handler on the main looper the first // time a callback arrives, and a bare app_process has no main looper, so // connecting from here died with an NPE inside the framework and the // shell reported only "Killed". So the main thread prepares the looper // and then does nothing but run it; the trace itself is a worker. Looper.prepareMainLooper(); Thread worker = new Thread(() -> record(args), "ui-trace"); worker.start(); Looper.loop(); } private static void record(String[] args) { PrintWriter out = new PrintWriter(new java.io.BufferedWriter(new java.io.OutputStreamWriter(System.out), 1 << 16)); try { long duration = 3000; long interval = 0; List script = new ArrayList<>(); for (int i = 0; i < args.length; i++) { switch (args[i]) { case "--duration" -> duration = Long.parseLong(args[++i]); case "--interval" -> interval = Long.parseLong(args[++i]); case "--do" -> script.add(args[++i].trim().split("\\s+")); default -> throw new IllegalArgumentException("unknown option: " + args[i]); } } UiTrace trace = new UiTrace(connect(), out); trace.run(duration, interval, script); out.println("# done"); } catch (Throwable problem) { out.println("# error " + problem); for (StackTraceElement frame : problem.getStackTrace()) out.println("# at " + frame); out.flush(); System.exit(1); } out.flush(); // The accessibility connection keeps non-daemon threads alive, so a // returning main() would hang here rather than ending the trace. System.exit(0); } private static UiAutomation connect() throws Exception { HandlerThread thread = new HandlerThread("ui-trace"); thread.start(); Object connection = Class.forName("android.app.UiAutomationConnection") .getDeclaredConstructor().newInstance(); Constructor ctor = UiAutomation.class.getDeclaredConstructor( Looper.class, Class.forName("android.app.IUiAutomationConnection")); ctor.setAccessible(true); UiAutomation automation = (UiAutomation) ctor.newInstance(thread.getLooper(), connection); UiAutomation.class.getDeclaredMethod("connect").invoke(automation); // Without these two the tree is the one a screen reader would use: // decorative and off-stage nodes are dropped and view ids are withheld, // and both are things a layout question is usually about. AccessibilityServiceInfo info = automation.getServiceInfo(); info.flags |= AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS | AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS; automation.setServiceInfo(info); return automation; } private void run(long duration, long interval, List script) { // The actions run beside the sampling rather than between samples: an // action that blocked the loop would leave a hole in the recording at // exactly the moment being recorded. Thread actor = new Thread(() -> { for (String[] step : script) { try { perform(step); } catch (Exception problem) { synchronized (out) { out.println("# action failed: " + problem); } } } }, "ui-trace-actions"); actor.setDaemon(true); actor.start(); long end = origin + duration; int frame = 0; while (SystemClock.uptimeMillis() < end) { long at = SystemClock.uptimeMillis() - origin; AccessibilityNodeInfo root = automation.getRootInActiveWindow(); synchronized (out) { out.println("F " + frame + " " + at); if (root == null) out.println("# no active window"); else emit(root, 0); out.flush(); } frame++; if (interval > 0) { long slack = interval - (SystemClock.uptimeMillis() - origin - at); if (slack > 0) SystemClock.sleep(slack); } } } private void emit(AccessibilityNodeInfo node, int depth) { if (node == null || depth > MAX_DEPTH) return; Rect box = new Rect(); node.getBoundsInScreen(box); // Tab-separated with the free text last, because a label may contain // anything at all including the separator of a friendlier format. StringBuilder line = new StringBuilder("N\t"); line.append(depth).append('\t') .append(box.left).append('\t').append(box.top).append('\t') .append(box.right).append('\t').append(box.bottom).append('\t') .append(flags(node)).append('\t') .append(shortName(node.getClassName())).append('\t') .append(clean(node.getViewIdResourceName())).append('\t') .append(clean(node.getText())).append('\t') .append(clean(node.getContentDescription())); out.println(line); for (int i = 0; i < node.getChildCount(); i++) emit(node.getChild(i), depth + 1); } private static String flags(AccessibilityNodeInfo node) { StringBuilder marks = new StringBuilder(); if (node.isClickable()) marks.append('c'); if (node.isScrollable()) marks.append('s'); if (node.isFocused()) marks.append('f'); if (!node.isEnabled()) marks.append('d'); if (!node.isVisibleToUser()) marks.append('h'); return marks.length() == 0 ? "-" : marks.toString(); } private static String shortName(CharSequence name) { String text = clean(name); int dot = text.lastIndexOf('.'); return dot < 0 ? text : text.substring(dot + 1); } private static String clean(CharSequence value) { if (value == null) return ""; return value.toString().replace('\t', ' ').replace('\n', '⏎').replace('\r', ' '); } private void perform(String[] step) throws Exception { long at = SystemClock.uptimeMillis() - origin; synchronized (out) { out.println("A " + at + "\t" + String.join(" ", step)); out.flush(); } switch (step[0]) { case "wait" -> SystemClock.sleep(Long.parseLong(step[1])); case "tap" -> tap(Integer.parseInt(step[1]), Integer.parseInt(step[2])); case "swipe" -> swipe(Integer.parseInt(step[1]), Integer.parseInt(step[2]), Integer.parseInt(step[3]), Integer.parseInt(step[4]), step.length > 5 ? Long.parseLong(step[5]) : 300); default -> throw new IllegalArgumentException("unknown action: " + step[0]); } } private void send(int action, long down, int x, int y) { MotionEvent event = MotionEvent.obtain(down, SystemClock.uptimeMillis(), action, x, y, 0); event.setSource(InputDevice.SOURCE_TOUCHSCREEN); automation.injectInputEvent(event, true); event.recycle(); } private void tap(int x, int y) { long down = SystemClock.uptimeMillis(); send(MotionEvent.ACTION_DOWN, down, x, y); SystemClock.sleep(60); send(MotionEvent.ACTION_UP, down, x, y); } // Moves are emitted on a fixed 10ms cadence so the velocity tracker behind a // fling sees a real gesture. `input swipe` interpolates over far fewer // points, which is why it so often scrolls without ever flinging. private void swipe(int x1, int y1, int x2, int y2, long millis) { long down = SystemClock.uptimeMillis(); send(MotionEvent.ACTION_DOWN, down, x1, y1); int steps = (int) Math.max(2, millis / 10); for (int i = 1; i <= steps; i++) { float part = (float) i / steps; long due = down + (long) (millis * part); long slack = due - SystemClock.uptimeMillis(); if (slack > 0) SystemClock.sleep(slack); send(MotionEvent.ACTION_MOVE, down, Math.round(x1 + (x2 - x1) * part), Math.round(y1 + (y2 - y1) * part)); } send(MotionEvent.ACTION_UP, down, x2, y2); } }