diff --git a/AGENTS.md b/AGENTS.md index 68887fb..75d0192 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -369,7 +369,7 @@ first if a remote spawn ever mangles an argument. conversation somebody may still be in. **A transcript never goes in this repository**: they hold whatever was said, read and written in that session, and `~/repos` is shared with the host besides. -- **`app/ui-sandbox.sh` is the rig for anything that lists or deletes +- **`app/ui-sandbox.sh` is the rig for driving the UI against invented sessions.** It starts a second `ai-server` with its own `$HOME`, config and data directory, holding eight invented Claude Code transcripts and a `claude` that is two lines of shell. That isolation is the point: the @@ -378,13 +378,31 @@ first if a remote spawn ever mangles an argument. server deletes somebody's conversation and exercising *import* starts a real `--resume` on the owner's account. Neither is a price worth paying to look at a list. It shares the real TLS certificates, because the - installed APK pins that CA, so run it while the ordinary server is down. + installed APK pins that CA. + Its port and root are derived from the checkout's name, so two checkouts' + sandboxes (and the emulators enrolled against them) cannot reach each + other, and its token is generated once into + `~/.config/ai-app/sandbox-token` and carried across restarts along with + any tokens the server's own enrolment flow appended -- so the emulator app + is enrolled **once** (the start banner prints the command) and stays + enrolled. It also carries the driving verbs every UI investigation needs, + so none of this is re-derived per session: + `./ui-sandbox.sh spawn [title]` (an echo session, prints its id), + `./ui-sandbox.sh send SID text|@file`, and + `./ui-sandbox.sh api /path [curl args]` for everything else. It passes `--delay` by default for the reason the next entry gives, and `AI_SANDBOX_BIG_MB` puts one large transcript among the small ones -- `AI_SANDBOX_SPAWN_DELAY` makes the fake CLI slow to start. Both exist because operations that finish in milliseconds have states on the way that nothing can observe, and an unobservable state is one where broken and working look identical. +- **`app/transcript-bench.sh` is the standard scroll measurement.** It + opens the first session (or `-k` keeps the current screen), scrolls a + fixed gesture loop, and prints the app's render report -- the same one + the in-app copy button produces, whose `on screen:` line names what the + viewport was actually holding. Compare two runs of it with the same + gestures; the emulator's absolute frame times transfer nothing, the + report's accounting does. - **`ai-server --delay MS` holds every response back.** Over the tunnel a phone's requests take tens to hundreds of milliseconds, and several faults live entirely in what the app does *while* one is outstanding. On diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt index 859c98c..ad100b8 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt @@ -297,6 +297,16 @@ fun parseSeqEvent(json: String): SeqEvent { * compaction that finished without saying how much it recovered, or a clear nobody has run a turn * since. */ +/** + * Whether [state] is one the session is doing work in -- the states a turn is still open under. + * + * One predicate because two readers have to agree on the list: the session screen's working + * indicator, and the fold's decision that the newest reply is finished + * ([TranscriptItem.AssistantMsg.settled]). Two copies would drift the first time the server grows a + * state, and the drift would be a reply that never splits or one split mid-stream. + */ +fun sessionWorking(state: String): Boolean = state == "running" || state == "compacting" + fun contextAfter(current: Long?, event: SessionEvent): Long? = when (event) { // Falls back to what we had, so a turn the dialect reported no usage for is stale by a diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionListScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionListScreen.kt index f24bfa8..832b087 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionListScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionListScreen.kt @@ -352,7 +352,7 @@ fun StatusText(status: String) { else -> status to MaterialTheme.colorScheme.onSurfaceVariant } Row(verticalAlignment = Alignment.CenterVertically) { - if (status == "running" || status == "compacting") { + if (sessionWorking(status)) { // The same colour as the word beside it: the two are one signal, and a spinner in // the theme's accent says the state is something other than what the label says. CircularProgressIndicator( diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt index f1f9fce..0673bbc 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt @@ -319,7 +319,7 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () // them. From the server rather than from this screen, so a rename sent from the settings // screen -- or from another device -- is drawn waiting here too. var waitingCommands by remember { mutableStateOf(listOf>()) } - val running = status == "running" || status == "compacting" + val running = sessionWorking(status) var moreHistory by remember { mutableStateOf(true) } var loadingHistory by remember { mutableStateOf(false) } var ready by remember { mutableStateOf(false) } diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptItems.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptItems.kt index 1428b02..a04bfd5 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptItems.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptItems.kt @@ -45,7 +45,27 @@ sealed class TranscriptItem { val images: List = emptyList(), ) : TranscriptItem() - data class AssistantMsg(override val seq: Long, val text: String) : TranscriptItem() + data class AssistantMsg( + override val seq: Long, + val text: String, + /** + * Whether this reply is finished: the session has stopped working since its last delta. + * + * What it buys is the split. [transcriptUnits] keeps the newest reply whole because a + * streaming reply's text changes per delta and splitting a changing text is a parse per + * delta -- but "newest" outlives the turn, so a session that ends on a long reply was + * drawing it as one item indefinitely, with every node of it alive. Measured on a Pixel 9 + * Pro XL: one 34,996px reply on screen put the frame's draw phase at 13.8ms, 79% of it the + * framework's own bookkeeping, which grows with alive nodes. + * + * Folded from the status event that ended the turn, rather than read off the screen's + * status, because rows only change through the held-events gate: the split changes the + * newest row's list identity, and doing that from a status flip while somebody is reading + * inside that reply would step the list under them. An event has to wait for the reader to + * be at the newest end; a screen state does not. + */ + val settled: Boolean = false, + ) : TranscriptItem() data class ToolRun( override val seq: Long, @@ -362,7 +382,8 @@ fun foldEvent(items: List, entry: SeqEvent): List, entry: SeqEvent): List items is SessionEvent.Settings -> items - is SessionEvent.Status -> items + is SessionEvent.Status -> settleReply(items, event.state) is SessionEvent.Error -> items + TranscriptItem.ErrorMsg(entry.seq, event.message) is SessionEvent.Image -> // Under the call that produced it when there is one, and a row of @@ -479,6 +500,20 @@ fun foldEvent(items: List, entry: SeqEvent): List items } +/** + * A status saying the session stopped working is the moment its newest reply is finished. + * + * See [TranscriptItem.AssistantMsg.settled] for what the mark buys and why it is made here in the + * fold. Status changes are transcript events with seqs of their own, so a replayed session settles + * its replies the same way a live one does. + */ +private fun settleReply(items: List, state: String): List { + if (sessionWorking(state)) return items + val last = items.lastOrNull() as? TranscriptItem.AssistantMsg ?: return items + if (last.settled) return items + return items.dropLast(1) + last.copy(settled = true) +} + private fun updateTool( items: List, id: String, diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptUnits.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptUnits.kt index 6ccac94..f922032 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptUnits.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptUnits.kt @@ -126,10 +126,12 @@ sealed class TranscriptUnit { * Every settled reply is cut into its blocks ([markdownBlocks], via the caches on [replies] so a * message is only ever split once), and so is an *opened* peer message -- [openNotes] is which ones * those are, which is why the flatten needs it. A shut one is a single heading and cannot be worth - * splitting. The reply still arriving -- the last row -- stays whole: its text changes with every - * delta, and splitting it here would parse the whole message per delta on whichever thread is - * composing. [AssistantMessage]'s own streaming path already parses deltas off the main thread and - * gives the live message a layer per block. + * splitting. The reply still arriving -- the newest row, until the status event that ends its turn + * marks it [TranscriptItem.AssistantMsg.settled] -- stays whole: its text changes with every delta, + * and splitting it here would parse the whole message per delta on whichever thread is composing. + * [AssistantMessage]'s own streaming path already parses deltas off the main thread and gives the + * live message a layer per block. Once settled it splits like every other reply, which is what + * bounds the newest row's cost after a session ends on a long one. * * Runs per fold, so it must stay proportional to what is loaded with no parsing in it on the warm * path: [ParsedReplies.partsOf] and [ParsedReplies.blocksOf] are lookups for any text [warm] has @@ -163,7 +165,9 @@ fun transcriptUnits( ) } } - } else if (item is TranscriptItem.AssistantMsg && index != rows.lastIndex) { + } else if ( + item is TranscriptItem.AssistantMsg && (item.settled || index != rows.lastIndex) + ) { var ordinal = 0 fun gap() = if (ordinal == 0) rowGap else BLOCK_SPACING replies.partsOf(item.text).forEach { part -> diff --git a/app/transcript-bench.sh b/app/transcript-bench.sh new file mode 100755 index 0000000..3f2ea78 --- /dev/null +++ b/app/transcript-bench.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Scrolls the transcript and prints the app's own render report -- the +# standard measurement for "is the transcript smooth", so a perf session +# starts from a number instead of re-inventing this loop in /tmp. +# +# ./transcript-bench.sh restart the app, open the first +# session, scroll, report +# ./transcript-bench.sh -k keep whatever screen is open now +# ./transcript-bench.sh -n 4 swipe cycles (default 6; one cycle is +# two swipes back and two forward) +# +# What it prints is the report the in-app copy button produces, plus any +# duplicate-key warnings the flatten logged. Absolute frame times from the +# emulator are worthless (it renders in software); what transfers is the +# report's own accounting -- what was on screen, and where the draw phase +# went. Compare two runs of this with the same gestures, not one run +# against a phone. +# +# Coordinates are the ai-app AVD's 1080x2400 screen: the report button in +# the session top bar, and the first row of the session list. +set -eu +cd "$(dirname "$0")" +. ./android-env.sh >/dev/null 2>&1 + +keep="" +cycles=6 +while getopts kn: flag; do + case $flag in + k) keep=1 ;; + n) cycles=$OPTARG ;; + *) exit 2 ;; + esac +done + +if [ -z "$keep" ]; then + adb shell am force-stop com.example.aiapp + adb shell am start -n com.example.aiapp/.MainActivity >/dev/null + sleep 5 + ui-trace record -d 3000 --do 'tap 500 545' -o /tmp/bench-open.txt >/dev/null 2>&1 + sleep 3 +fi + +# First tap resets the report's "work since last copied" window, so the +# numbers cover exactly the swipes between the two taps. +ui-trace record -d 2000 --do 'tap 723 205' -o /tmp/bench-reset.txt >/dev/null 2>&1 +adb logcat -c + +DO="" +i=0 +while [ "$i" -lt "$cycles" ]; do + DO="$DO --do 'swipe 540 700 540 1600 200' --do 'wait 500'" + DO="$DO --do 'swipe 540 700 540 1600 200' --do 'wait 500'" + DO="$DO --do 'swipe 540 1600 540 700 200' --do 'wait 500'" + DO="$DO --do 'swipe 540 1600 540 700 200' --do 'wait 500'" + i=$((i + 1)) +done +eval ui-trace record -d $((cycles * 4400 + 2000)) $DO -o /tmp/bench-scroll.txt >/dev/null 2>&1 + +ui-trace record -d 2000 --do 'tap 723 205' -o /tmp/bench-report.txt >/dev/null 2>&1 +sleep 1 +adb logcat -d -s ai-app:I | sed -n '/ai-app render report/,$p' | sed 's/^.*ai-app : //' +adb logcat -d 2>/dev/null | grep -i "duplicate unit key" | head -3 || true diff --git a/app/ui-sandbox.sh b/app/ui-sandbox.sh index e229976..d307a6e 100755 --- a/app/ui-sandbox.sh +++ b/app/ui-sandbox.sh @@ -30,19 +30,56 @@ # nobody can see. set -eu -ROOT=${AI_SANDBOX_ROOT:-${XDG_RUNTIME_DIR:-/tmp}/ai-app-sandbox} -TOKEN=${AI_SANDBOX_TOKEN:-sandbox} -PORT=${AI_SANDBOX_PORT:-8443} +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +SERVER_DIR=$SCRIPT_DIR/../server +# The checkout's name, because several checkouts of this repo run sessions +# at once and each has its own emulator: the root and the port both carry +# it, so one checkout's sandbox (and the phone enrolled against it) can +# never reach another's. Same rule as the per-checkout AVDs. +CHECKOUT=$(basename "$(dirname "$SCRIPT_DIR")") + +ROOT=${AI_SANDBOX_ROOT:-${XDG_RUNTIME_DIR:-/tmp}/ai-app-sandbox-$CHECKOUT} +# Derived, not chosen: stable for this checkout across sessions (so an +# enrolled emulator app keeps working), different between checkouts, and +# away from 8443 where real dev servers get started. +if [ -n "${AI_SANDBOX_PORT:-}" ]; then + PORT=$AI_SANDBOX_PORT +elif [ -f "$ROOT/port" ]; then + # Whatever the running (or last) server was actually started on, so the + # driving verbs below reach it even when it was started with an override. + PORT=$(cat "$ROOT/port") +else + PORT=$((8500 + $(printf %s "$CHECKOUT" | cksum | cut -d' ' -f1) % 80)) +fi DELAY=${AI_SANDBOX_DELAY:-1200} SPAWN_DELAY=${AI_SANDBOX_SPAWN_DELAY:-0} BIG_MB=${AI_SANDBOX_BIG_MB:-40} CERTS=${AI_SANDBOX_CERTS:-${XDG_CONFIG_HOME:-$HOME/.config}/ai-app/certs} -SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) -SERVER_DIR=$SCRIPT_DIR/../server +# Generated once and kept outside the repo (it is a credential, however +# small the stakes), so the emulator app enrolled against the sandbox stays +# enrolled across restarts and VM reboots instead of every session +# re-deriving why the server says "invalid bearer token". URL-safe +# characters only, so the enrolment deep link needs no encoding. +TOKEN_FILE=${XDG_CONFIG_HOME:-$HOME/.config}/ai-app/sandbox-token +if [ -z "${AI_SANDBOX_TOKEN:-}" ] && [ ! -f "$TOKEN_FILE" ]; then + mkdir -p "$(dirname "$TOKEN_FILE")" + (umask 077 && head -c 24 /dev/urandom | base64 | tr '+/' '-_' >"$TOKEN_FILE") +fi +TOKEN=${AI_SANDBOX_TOKEN:-$(cat "$TOKEN_FILE")} +hash=$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1) + PIDFILE=$ROOT/server.pid LOG=$ROOT/server.log +# An authenticated request to the running sandbox, so nothing driving it +# has to re-derive the port and token: `./ui-sandbox.sh api /sessions`. +api() { + api_path=$1 + shift + curl -sk "https://127.0.0.1:$PORT$api_path" -H "Authorization: Bearer $TOKEN" "$@" +} + # By pid rather than by pattern: a `pkill -f` for something as generic as # "ai-server" also matches the shell running this script, which kills the # script mid-flight and leaves the restart never having happened. @@ -56,12 +93,65 @@ stop_server() { rm -f "$PIDFILE" } -if [ "${1:-start}" = stop ]; then +case "${1:-start}" in +stop) stop_server exit 0 -fi + ;; +# The driving verbs live here rather than in each session's /tmp scripts, +# because every UI investigation needs the same three: a session to point +# the phone at, a message in it (often a large one, hence @file), and an +# arbitrary authenticated request for everything else. +api) # ./ui-sandbox.sh api /path [curl args...] + shift + api "$@" + echo + exit 0 + ;; +spawn) # ./ui-sandbox.sh spawn [title] -- an echo session; prints its id + api /sessions -X POST -H 'content-type: application/json' \ + -d "{\"setup\":\"local\",\"provider\":\"echo\",\"title\":\"${2:-test}\"}" | + python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])' + exit 0 + ;; +send) # ./ui-sandbox.sh send SID text... (or: send SID @file) + sid=$2 + shift 2 + python3 -c 'import json, sys +arg = sys.argv[1] +text = open(arg[1:]).read() if arg.startswith("@") else " ".join(sys.argv[1:]) +print(json.dumps({"text": text}))' "$@" >"$ROOT/send.json" + api "/sessions/$sid/message" -X POST -H 'content-type: application/json' \ + --data-binary "@$ROOT/send.json" + echo + exit 0 + ;; +start) ;; +*) + echo "ui-sandbox.sh: unknown command '$1' (start, stop, api, spawn, send)" >&2 + exit 2 + ;; +esac stop_server +# Tokens the server's own enrolment flow appended to the old config are the +# phones enrolled against this sandbox; a restart regenerates the fixtures +# but must not orphan those, or the app greets the next session with +# "server rejected this device's token" and an afternoon of why. +salvaged="" +if [ -f "$ROOT/config.ron" ]; then + salvaged=$(awk ' + /^tokens: \[/ { in_tokens = 1; next } + in_tokens && /^\],/ { exit } + in_tokens { + entry = entry $0 "\n" + if ($0 ~ /\),/) { + if (entry !~ h) entries = entries entry + entry = "" + } + } + END { printf "%s", entries }' h="$hash" "$ROOT/config.ron") +fi rm -rf "$ROOT/home" "$ROOT/sessions" "$ROOT/config.ron" PROJECTS=$ROOT/home/.claude/projects/-home-bob-repos-sandbox mkdir -p "$PROJECTS" "$ROOT/sessions" @@ -130,14 +220,13 @@ awk -v mb="$BIG_MB" 'BEGIN { print "{\"type\":\"assistant\",\"message\":{\"role\":\"assistant\",\"usage\":{\"input_tokens\":180000,\"output_tokens\":900}}}" }' > "$big" -hash=$(printf '%s' "$TOKEN" | sha256sum | cut -d' ' -f1) cat >"$ROOT/config.ron" </dev/null || true echo "$pid" >"$PIDFILE" +echo "$PORT" >"$ROOT/port" # Waited for rather than assumed: the enrolment below fails silently against # a server that has not bound yet, and the app then shows a network error @@ -191,12 +281,20 @@ while [ "$tries" -lt 50 ]; do sleep 0.2 done +# Percent-encoded because the app URL-decodes the deep link's query: a +# token with '+' in it enrols as one with a space, and nothing reports it. +enc=$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$TOKEN") cat <