emulator-tools/share/ui-trace/UiTrace.java

// The on-device half of ui-trace: samples the accessibility tree in a loop and
// prints one text frame per sample.
//
// This exists because `uiautomator dump` costs about two seconds per call --
// almost all of it starting a JVM and connecting to the accessibility service --
// which is half a hertz, so an animation happens entirely between two samples.
// Connecting once and then sampling in a loop is the whole trick; everything
// else here is bookkeeping around it.
//
// Runs under app_process as the shell user, the same way /system/bin/uiautomator
// does. Three pieces of the connection are hidden API and are reached by
// reflection: android.app.UiAutomationConnection, the (Looper, connection)
// constructor of UiAutomation, and its connect() method. There is no public way
// for a shell process to obtain a UiAutomation -- the public entry points all
// require an Instrumentation and therefore an app -- and this is the same path
// the platform's own uiautomator command takes.

import android.accessibilityservice.AccessibilityServiceInfo;
import android.app.UiAutomation;
import android.graphics.Rect;
import android.os.HandlerThread;
import android.os.Looper;
import android.os.SystemClock;
import android.view.InputDevice;
import android.view.MotionEvent;
import android.view.accessibility.AccessibilityNodeInfo;
import java.io.PrintWriter;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
import java.util.List;

public final class UiTrace {
    private static final int MAX_DEPTH = 60;
    /** How long `tap LABEL` keeps looking before it decides the label is not there. */
    private static final long FIND_TIMEOUT_MS = 3000;

    private final UiAutomation automation;
    private final PrintWriter out;
    private final long origin = SystemClock.uptimeMillis();

    private UiTrace(UiAutomation automation, PrintWriter out) {
        this.automation = automation;
        this.out = out;
    }

    // prepareMainLooper() is deprecated because an app never needs it -- the
    // Android environment makes the main looper. This is not an app: app_process
    // starts a bare VM, so there is no main looper and no supported way to ask
    // for one. The framework needs it anyway (see below), so the deprecated call
    // is the only route, and it is scoped to this method alone.
    @SuppressWarnings("deprecation")
    public static void main(String[] args) {
        // The accessibility client builds a Handler on the main looper the first
        // time a callback arrives, and a bare app_process has no main looper, so
        // connecting from here died with an NPE inside the framework and the
        // shell reported only "Killed". So the main thread prepares the looper
        // and then does nothing but run it; the trace itself is a worker.
        Looper.prepareMainLooper();
        Thread worker = new Thread(() -> record(args), "ui-trace");
        worker.start();
        Looper.loop();
    }

    private static void record(String[] args) {
        PrintWriter out = new PrintWriter(new java.io.BufferedWriter(new java.io.OutputStreamWriter(System.out), 1 << 16));
        try {
            long duration = 3000;
            long interval = 0;
            List<String[]> script = new ArrayList<>();
            for (int i = 0; i < args.length; i++) {
                switch (args[i]) {
                    case "--duration" -> duration = Long.parseLong(args[++i]);
                    case "--interval" -> interval = Long.parseLong(args[++i]);
                    case "--do" -> script.add(args[++i].trim().split("\\s+"));
                    default -> throw new IllegalArgumentException("unknown option: " + args[i]);
                }
            }
            UiTrace trace = new UiTrace(connect(), out);
            trace.run(duration, interval, script);
            out.println("# done");
        } catch (Throwable problem) {
            out.println("# error " + problem);
            for (StackTraceElement frame : problem.getStackTrace()) out.println("# at " + frame);
            out.flush();
            System.exit(1);
        }
        out.flush();
        // The accessibility connection keeps non-daemon threads alive, so a
        // returning main() would hang here rather than ending the trace.
        System.exit(0);
    }

    private static UiAutomation connect() throws Exception {
        HandlerThread thread = new HandlerThread("ui-trace");
        thread.start();
        Object connection = Class.forName("android.app.UiAutomationConnection")
                .getDeclaredConstructor().newInstance();
        Constructor<?> ctor = UiAutomation.class.getDeclaredConstructor(
                Looper.class, Class.forName("android.app.IUiAutomationConnection"));
        ctor.setAccessible(true);
        UiAutomation automation = (UiAutomation) ctor.newInstance(thread.getLooper(), connection);
        UiAutomation.class.getDeclaredMethod("connect").invoke(automation);
        // Without these two the tree is the one a screen reader would use:
        // decorative and off-stage nodes are dropped and view ids are withheld,
        // and both are things a layout question is usually about.
        AccessibilityServiceInfo info = automation.getServiceInfo();
        info.flags |= AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS
                | AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS;
        automation.setServiceInfo(info);
        return automation;
    }

    private void run(long duration, long interval, List<String[]> script) {
        // The actions run beside the sampling rather than between samples: an
        // action that blocked the loop would leave a hole in the recording at
        // exactly the moment being recorded.
        Thread actor = new Thread(() -> {
            for (String[] step : script) {
                try {
                    perform(step);
                } catch (Exception problem) {
                    // "# error", not a gentler word: the reader of this trace is usually a
                    // benchmark script, and an action that did not happen must fail the run
                    // rather than leave numbers that look measured. ui-trace's own `record`
                    // exits on this line.
                    synchronized (out) { out.println("# error action failed: " + problem); }
                }
            }
        }, "ui-trace-actions");
        actor.setDaemon(true);
        actor.start();

        long end = origin + duration;
        int frame = 0;
        while (SystemClock.uptimeMillis() < end) {
            long at = SystemClock.uptimeMillis() - origin;
            AccessibilityNodeInfo root = automation.getRootInActiveWindow();
            synchronized (out) {
                out.println("F " + frame + " " + at);
                if (root == null) out.println("# no active window");
                else emit(root, 0);
                out.flush();
            }
            frame++;
            if (interval > 0) {
                long slack = interval - (SystemClock.uptimeMillis() - origin - at);
                if (slack > 0) SystemClock.sleep(slack);
            }
        }

        // Waited for rather than left behind. The actions run beside the sampling, so a script
        // whose last step outlasts the recording used to have its outcome thrown away by the
        // exit -- including the "# error" from a `tap` that never found its label, which is the
        // one line the caller most needs. A run that could not press what it meant to press must
        // not end quietly.
        try {
            actor.join(FIND_TIMEOUT_MS + 1000);
        } catch (InterruptedException interrupted) {
            Thread.currentThread().interrupt();
        }
        if (actor.isAlive()) {
            synchronized (out) {
                out.println("# error the recording ended before its actions did -- raise -d");
                out.flush();
            }
        }
    }

    private void emit(AccessibilityNodeInfo node, int depth) {
        if (node == null || depth > MAX_DEPTH) return;
        Rect box = new Rect();
        node.getBoundsInScreen(box);
        // Tab-separated with the free text last, because a label may contain
        // anything at all including the separator of a friendlier format.
        StringBuilder line = new StringBuilder("N\t");
        line.append(depth).append('\t')
            .append(box.left).append('\t').append(box.top).append('\t')
            .append(box.right).append('\t').append(box.bottom).append('\t')
            .append(flags(node)).append('\t')
            .append(shortName(node.getClassName())).append('\t')
            .append(clean(node.getViewIdResourceName())).append('\t')
            .append(clean(node.getText())).append('\t')
            .append(clean(node.getContentDescription()));
        out.println(line);
        for (int i = 0; i < node.getChildCount(); i++) emit(node.getChild(i), depth + 1);
    }

    private static String flags(AccessibilityNodeInfo node) {
        StringBuilder marks = new StringBuilder();
        if (node.isClickable()) marks.append('c');
        if (node.isScrollable()) marks.append('s');
        if (node.isFocused()) marks.append('f');
        if (!node.isEnabled()) marks.append('d');
        if (!node.isVisibleToUser()) marks.append('h');
        return marks.length() == 0 ? "-" : marks.toString();
    }

    private static String shortName(CharSequence name) {
        String text = clean(name);
        int dot = text.lastIndexOf('.');
        return dot < 0 ? text : text.substring(dot + 1);
    }

    private static String clean(CharSequence value) {
        if (value == null) return "";
        return value.toString().replace('\t', ' ').replace('\n', '⏎').replace('\r', ' ');
    }

    private void perform(String[] step) throws Exception {
        long at = SystemClock.uptimeMillis() - origin;
        synchronized (out) { out.println("A " + at + "\t" + String.join(" ", step)); out.flush(); }
        switch (step[0]) {
            case "wait" -> SystemClock.sleep(Long.parseLong(step[1]));
            case "tap" -> {
                if (step.length == 3 && isNumber(step[1]) && isNumber(step[2])) {
                    tap(Integer.parseInt(step[1]), Integer.parseInt(step[2]));
                } else {
                    tapLabel(join(step));
                }
            }
            case "swipe" -> swipe(Integer.parseInt(step[1]), Integer.parseInt(step[2]),
                    Integer.parseInt(step[3]), Integer.parseInt(step[4]),
                    step.length > 5 ? Long.parseLong(step[5]) : 300);
            case "holddrag" -> holdDrag(Integer.parseInt(step[1]), Integer.parseInt(step[2]),
                    Integer.parseInt(step[3]), Integer.parseInt(step[4]),
                    Long.parseLong(step[5]), Long.parseLong(step[6]));
            default -> throw new IllegalArgumentException("unknown action: " + step[0]);
        }
    }

    private static boolean isNumber(String value) {
        if (value.isEmpty()) return false;
        for (int i = 0; i < value.length(); i++) {
            if (!Character.isDigit(value.charAt(i))) return false;
        }
        return true;
    }

    /** Everything after the verb, as one label, with any quotes the shell left on it removed. */
    private static String join(String[] step) {
        StringBuilder joined = new StringBuilder();
        for (int i = 1; i < step.length; i++) {
            if (i > 1) joined.append(' ');
            joined.append(step[i]);
        }
        String label = joined.toString();
        if (label.length() >= 2) {
            char first = label.charAt(0), last = label.charAt(label.length() - 1);
            if ((first == '\'' || first == '"') && first == last) {
                label = label.substring(1, label.length() - 1);
            }
        }
        return label;
    }

    /**
     * Presses whatever is currently labelled `label`, wherever it is.
     *
     * This is the action to reach for, and `tap X Y` is the exception. A coordinate is a position
     * measured once by hand: anything that moves the thing being pressed -- a control added to the
     * row, a font size, a density, a different device -- makes the tap land on whatever now sits
     * there, and the script then reports a number that was never measured, which reads exactly
     * like a result. A name is what the control already carries for assistive technology, so it
     * survives all of that and fails loudly when it genuinely is not there.
     *
     * The box is read from the tree at the moment of the gesture rather than from an earlier
     * frame, because the screen the gesture lands on is the only one that decides where the mark
     * is.
     */
    private void tapLabel(String label) {
        // Looked for repeatedly for a moment rather than once. Two things make a single look
        // wrong: the accessibility connection has no window at all for the first frames after it
        // is made, so an action at the very start of a trace found nothing to search; and a
        // control revealed by the step before this one -- a dialog's button, a row that has just
        // loaded -- arrives a frame or two later. The wait is bounded and the failure is still
        // loud, which is the whole point: what must not happen is pressing the wrong thing.
        List<AccessibilityNodeInfo> found = new ArrayList<>();
        long giveUp = SystemClock.uptimeMillis() + FIND_TIMEOUT_MS;
        while (true) {
            AccessibilityNodeInfo root = automation.getRootInActiveWindow();
            if (root != null) collect(root, label, 0, found);
            if (!found.isEmpty() || SystemClock.uptimeMillis() >= giveUp) break;
            SystemClock.sleep(50);
        }
        if (found.isEmpty()) {
            throw new IllegalArgumentException("nothing on screen is labelled \"" + label
                    + "\" (looked for " + FIND_TIMEOUT_MS + "ms)");
        }
        // The clickable one where there is a choice: a label and the padded control wrapping it
        // both carry the text, and the control is the thing a finger is meant to find. Its centre
        // is also where a ripple is drawn, so this presses what the reader would have pressed.
        AccessibilityNodeInfo target = found.get(0);
        for (AccessibilityNodeInfo candidate : found) {
            if (candidate.isClickable()) { target = candidate; break; }
        }
        Rect box = new Rect();
        target.getBoundsInScreen(box);
        if (box.width() <= 0 || box.height() <= 0) {
            throw new IllegalStateException("\"" + label + "\" has no size on screen");
        }
        synchronized (out) {
            out.println("# tap \"" + label + "\" at " + box.centerX() + "," + box.centerY()
                    + " in " + box.left + "," + box.top + "," + box.right + "," + box.bottom);
            out.flush();
        }
        tap(box.centerX(), box.centerY());
    }

    /** Every visible node whose text or description is exactly `label`, in tree order. */
    private static void collect(AccessibilityNodeInfo node, String label, int depth,
                                List<AccessibilityNodeInfo> found) {
        if (node == null || depth > MAX_DEPTH) return;
        if (node.isVisibleToUser()
                && (label.equals(clean(node.getText())) || label.equals(clean(node.getContentDescription())))) {
            found.add(node);
        }
        for (int i = 0; i < node.getChildCount(); i++) collect(node.getChild(i), label, depth + 1, found);
    }

    private void send(int action, long down, int x, int y) {
        MotionEvent event = MotionEvent.obtain(down, SystemClock.uptimeMillis(), action, x, y, 0);
        event.setSource(InputDevice.SOURCE_TOUCHSCREEN);
        automation.injectInputEvent(event, true);
        event.recycle();
    }

    private void tap(int x, int y) {
        long down = SystemClock.uptimeMillis();
        send(MotionEvent.ACTION_DOWN, down, x, y);
        SystemClock.sleep(60);
        send(MotionEvent.ACTION_UP, down, x, y);
    }

    // Moves are emitted on a fixed 10ms cadence so the velocity tracker behind a
    // fling sees a real gesture. `input swipe` interpolates over far fewer
    // points, which is why it so often scrolls without ever flinging.
    private void swipe(int x1, int y1, int x2, int y2, long millis) {
        long down = SystemClock.uptimeMillis();
        send(MotionEvent.ACTION_DOWN, down, x1, y1);
        int steps = (int) Math.max(2, millis / 10);
        for (int i = 1; i <= steps; i++) {
            float part = (float) i / steps;
            long due = down + (long) (millis * part);
            long slack = due - SystemClock.uptimeMillis();
            if (slack > 0) SystemClock.sleep(slack);
            send(MotionEvent.ACTION_MOVE, down, Math.round(x1 + (x2 - x1) * part),
                    Math.round(y1 + (y2 - y1) * part));
        }
        send(MotionEvent.ACTION_UP, down, x2, y2);
    }

    /**
     * A press held stationary for `holdMs`, then moved to `(x2, y2)` over `moveMs`, then
     * released -- one continuous touch, never lifted between the hold and the move.
     *
     * Neither existing action can produce this. `tap` has no hold at all, and `swipe`
     * interpolates motion across its *whole* duration starting at t=0, so a long `swipe` with a
     * short first segment is still continuous motion throughout, never a hold followed by a
     * drag. That gap matters for anything that decides pan-vs-select the way Android itself
     * does -- a stationary press held past `LONG_PRESS` (500ms) starts a selection, which
     * further drag then extends -- because neither of the other two actions can reach the
     * "starts a selection" branch at all.
     *
     * The move phase reuses `swipe`'s own fixed 10ms cadence for the same reason: a velocity
     * tracker needs a real sequence of points, not two.
     */
    private void holdDrag(int x1, int y1, int x2, int y2, long holdMs, long moveMs) {
        long down = SystemClock.uptimeMillis();
        send(MotionEvent.ACTION_DOWN, down, x1, y1);
        SystemClock.sleep(holdMs);
        int steps = (int) Math.max(2, moveMs / 10);
        long moveStart = SystemClock.uptimeMillis();
        for (int i = 1; i <= steps; i++) {
            float part = (float) i / steps;
            long due = moveStart + (long) (moveMs * part);
            long slack = due - SystemClock.uptimeMillis();
            if (slack > 0) SystemClock.sleep(slack);
            send(MotionEvent.ACTION_MOVE, down, Math.round(x1 + (x2 - x1) * part),
                    Math.round(y1 + (y2 - y1) * part));
        }
        send(MotionEvent.ACTION_UP, down, x2, y2);
    }
}