From 9477cd288aaa3fa7a0f6063f36536e49762997a4 Mon Sep 17 00:00:00 2001 From: iris <2+iris@noreply.localhost> Date: Mon, 31 Aug 2026 14:49:44 -0400 Subject: [PATCH 1/3] Stop the keyboard re-laying-out the whole screen, and float the composer Opening the keyboard was late on 82% of frames on the Pixel while the transcript itself cost 0.25ms of each -- the cost was everywhere else. imePadding() sat on the activity's root box, so every frame of the IME animation resized the entire tree: measured on the emulator (with the new app-root timers) as a full re-measure (4.1ms), re-place (2.5ms) and re-record (1.0ms) of everything, ~34 frames per open, while the newly added recomposition counters read zero -- pure layout traversal, no recomposition to fix. So the keyboard now touches only what actually moves. The session screen's composer (status row, suggestions, attachments, text field, buttons) is a bottom-aligned overlay on its own layer, translated by the IME inset read inside the graphicsLayer block -- a keyboard frame invalidates layer properties only. The transcript box reserves the overlay's measured height plus imePadding, and that modifier is the whole of what the keyboard re-measures: the box's own size never changes, so the header and everything above it are untouched. The overlay is opaque for the one frame between it growing and the reservation catching up. imePadding moved off the activity root onto AppRoot's other screens, which keep the old arrangement -- none of them has a keyboard open over anything that scrolls at 120Hz. Same five-open protocol on the emulator, before and after: the app root is now measured zero times (was 168), per-frame app work 7.6ms -> 1.9ms (transcript measure 1.2 + place 0.5 + record 0.2), draw-phase p90 9.6ms -> 5.0ms, waited p50 2.6ms -> 0.4ms. What is left per frame is the transcript's own one-box remeasure, whose children skip measurement because their width is unchanged. Verified the states the overlay could have broken: keyboard over a long and a two-message conversation (content hangs from the composer in both), a three-line draft growing the composer upward with the reserve following, slash suggestions stacking above the field, and the closed state identical to before. The app-root timers and the recomposition counters stay in: they are the difference between this report saying "draw is high" and saying where. Co-Authored-By: Claude Fable 5 --- .../main/kotlin/com/example/aiapp/AppRoot.kt | 84 +- .../kotlin/com/example/aiapp/MainActivity.kt | 45 +- .../kotlin/com/example/aiapp/SessionScreen.kt | 1123 +++++++++-------- .../com/example/aiapp/SessionUsageBar.kt | 1 + 4 files changed, 699 insertions(+), 554 deletions(-) diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/AppRoot.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/AppRoot.kt index 8121c1f..f2b4e09 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/AppRoot.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/AppRoot.kt @@ -1,6 +1,8 @@ package com.example.aiapp import androidx.activity.compose.BackHandler +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.imePadding import androidx.compose.foundation.layout.padding import androidx.compose.material3.AlertDialog import androidx.compose.material3.MaterialTheme @@ -93,14 +95,16 @@ fun AppRoot(settingsVersion: Int, openRequest: SessionOpenRequest?) { if (current == null) { // Not enrolled yet: settings is the only usable screen. The QR // path lands in MainActivity and recomposes from the top. - SettingsScreen( - existing = null, - onSaved = { saved -> - settings = saved - screen = Screen.Main - }, - onBack = null, - ) + Box(Modifier.imePadding()) { + SettingsScreen( + existing = null, + onSaved = { saved -> + settings = saved + screen = Screen.Main + }, + onBack = null, + ) + } return } @@ -148,19 +152,25 @@ fun AppRoot(settingsVersion: Int, openRequest: SessionOpenRequest?) { ) } + // Every screen but the session takes the keyboard as bottom padding here. The session + // screen deliberately does not: resizing a whole screen on every frame of the keyboard + // animation is the cost that made it lag, so it moves only its composer and transcript -- + // see the layout note in SessionScreen. when (val here = screen) { is Screen.Main -> - MainScreen( - settings = current, - reloadToken = reloadToken, - onOpen = { screen = Screen.Session(it) }, - onSpawn = { screen = Screen.Spawn }, - onImported = { imported -> - reloadToken++ - screen = Screen.Session(imported) - }, - onSettings = { screen = Screen.Settings }, - ) + Box(Modifier.imePadding()) { + MainScreen( + settings = current, + reloadToken = reloadToken, + onOpen = { screen = Screen.Session(it) }, + onSpawn = { screen = Screen.Spawn }, + onImported = { imported -> + reloadToken++ + screen = Screen.Session(imported) + }, + onSettings = { screen = Screen.Settings }, + ) + } is Screen.Session -> // Keyed on the id, because a different session is a different screen rather than this // one showing other rows. SessionScreen remembers a transcript, an open event stream, a @@ -172,23 +182,27 @@ fun AppRoot(settingsVersion: Int, openRequest: SessionOpenRequest?) { SessionScreen(settings = current, summary = here.summary, onBack = goToMain) } is Screen.Spawn -> - SpawnScreen( - settings = current, - onSpawned = { spawned -> - reloadToken++ - screen = Screen.Session(spawned) - }, - onBack = goToMain, - ) + Box(Modifier.imePadding()) { + SpawnScreen( + settings = current, + onSpawned = { spawned -> + reloadToken++ + screen = Screen.Session(spawned) + }, + onBack = goToMain, + ) + } is Screen.Settings -> - SettingsScreen( - existing = current, - onSaved = { saved -> - settings = saved - goToMain() - }, - onBack = goToMain, - ) + Box(Modifier.imePadding()) { + SettingsScreen( + existing = current, + onSaved = { saved -> + settings = saved + goToMain() + }, + onBack = goToMain, + ) + } } // Last, so it draws over the screen above rather than under it: these are stacked in the Box diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/MainActivity.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/MainActivity.kt index fc86104..a0e0bd1 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/MainActivity.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/MainActivity.kt @@ -11,7 +11,6 @@ import androidx.activity.enableEdgeToEdge import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.foundation.layout.imePadding import androidx.compose.foundation.layout.navigationBarsPadding import androidx.compose.foundation.layout.statusBarsPadding import androidx.compose.material3.MaterialTheme @@ -21,7 +20,9 @@ import androidx.compose.runtime.mutableIntStateOf import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.drawWithContent import androidx.compose.ui.graphics.luminance +import androidx.compose.ui.layout.layout import androidx.core.view.WindowCompat class MainActivity : ComponentActivity() { @@ -91,14 +92,52 @@ class MainActivity : ComponentActivity() { Surface(modifier = Modifier.fillMaxSize()) { Box( modifier = - Modifier.fillMaxSize() + // Timed like the transcript times itself, and for the same reason: + // the frame's draw phase is where Compose's measurement lands, and + // a report saying "draw is high" cannot otherwise say whether the + // cost is the transcript or the chrome around it. The keyboard is + // the case that made it matter -- every frame of the IME animation + // relays out and re-records this whole box. + Modifier.layout { measurable, constraints -> + val started = System.nanoTime() + val placeable = measurable.measure(constraints) + DebugStats.record( + "measure: the app root", + System.nanoTime() - started, + ) + layout(placeable.width, placeable.height) { + val placing = System.nanoTime() + placeable.place(0, 0) + DebugStats.record( + "place: the app root", + System.nanoTime() - placing, + ) + } + } + .drawWithContent { + val started = System.nanoTime() + drawContent() + DebugStats.record( + "record: the app root", + System.nanoTime() - started, + ) + } + .fillMaxSize() .statusBarsPadding() // The gesture strip at the bottom of most // phones. Without it the send row sits under // the swipe area, where a tap is as likely to // navigate away as to press a button. + // + // No imePadding here, deliberately: applied at the root it + // resizes this whole box on every frame of the keyboard + // animation, which re-measures, re-places and re-records every + // screen's entire tree per frame -- measured above as most of + // the frame budget. Each screen takes the keyboard itself + // (AppRoot wraps the ordinary ones; the session screen moves + // only its composer and transcript), so the per-frame cost is + // scoped to what actually moves. .navigationBarsPadding() - .imePadding() ) { AppRoot(settingsVersion, openRequest) } diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt index 893ecaf..895398f 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt @@ -8,15 +8,20 @@ import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.PickVisualMediaRequest import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.Image +import androidx.compose.foundation.background import androidx.compose.foundation.gestures.awaitEachGesture import androidx.compose.foundation.gestures.awaitFirstDown import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.WindowInsets import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.ime +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.navigationBars import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width @@ -42,6 +47,7 @@ import androidx.compose.runtime.Immutable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableIntStateOf import androidx.compose.runtime.mutableLongStateOf import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember @@ -53,10 +59,12 @@ import androidx.compose.runtime.snapshots.Snapshot import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.drawWithContent +import androidx.compose.ui.graphics.graphicsLayer import androidx.compose.ui.input.pointer.PointerEventPass import androidx.compose.ui.input.pointer.pointerInput import androidx.compose.ui.layout.onSizeChanged import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.semantics.contentDescription import androidx.compose.ui.semantics.semantics import androidx.compose.ui.text.style.TextOverflow @@ -624,6 +632,7 @@ private suspend fun warm(replies: ParsedReplies, rows: List) { @Composable fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () -> Unit) { + DebugStats.count("session screen recomposed") val scope = rememberCoroutineScope() val topEdgeHeld = remember { TopEdgeHold() } var items by remember { mutableStateOf(listOf()) } @@ -1356,562 +1365,643 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () var usageOpen by remember { mutableStateOf(false) } var settingsOpen by remember { mutableStateOf(false) } - Column(Modifier.fillMaxSize()) { - Row( - verticalAlignment = Alignment.CenterVertically, - modifier = Modifier.fillMaxWidth().padding(horizontal = 16.dp), - ) { - GlyphButton(BACK_GLYPH, "Back", onBack) - // A ring's worth, which is what the arrow already keeps on its other three sides -- - // the pair of glyph buttons at the far end of this row get theirs from each other. - Spacer(Modifier.width(GLYPH_BUTTON_MARGIN)) - Column(Modifier.weight(1f)) { - Text(title, style = MaterialTheme.typography.titleMedium) - // Machine first, then what runs on it -- the same order and the same wording - // everywhere this pair appears, so it reads as one fact rather than as two - // sentences with different grammar. The "on" that used to sit in the middle - // made it a phrase, which only works in one order and stops working the moment - // the pair is shown anywhere else. + // The composer floats over the bottom of the screen instead of sitting under the transcript + // in one column, and the keyboard moves it by a layer translation rather than by relayout. + // With everything in one column under a root imePadding, every frame of the keyboard + // animation re-measured, re-placed and re-recorded the entire screen -- measured on the + // emulator at ~7.6ms of main-thread work per frame across ~34 frames per open, and on the + // Pixel as 82% late frames while the transcript itself cost 0.25ms. Scoped this way, a + // keyboard frame costs one layer transform for the composer and one re-measure of the + // transcript box, whose children skip measurement (width unchanged) and whose rows are + // already layers. + var composerHeight by remember { mutableIntStateOf(0) } + val imeInsets = WindowInsets.ime + val navInsets = WindowInsets.navigationBars + Box(Modifier.fillMaxSize()) { + Column(Modifier.fillMaxSize()) { + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.fillMaxWidth().padding(horizontal = 16.dp), + ) { + GlyphButton(BACK_GLYPH, "Back", onBack) + // A ring's worth, which is what the arrow already keeps on its other three sides -- + // the pair of glyph buttons at the far end of this row get theirs from each other. + Spacer(Modifier.width(GLYPH_BUTTON_MARGIN)) + Column(Modifier.weight(1f)) { + Text(title, style = MaterialTheme.typography.titleMedium) + // Machine first, then what runs on it -- the same order and the same wording + // everywhere this pair appears, so it reads as one fact rather than as two + // sentences with different grammar. The "on" that used to sit in the middle + // made it a phrase, which only works in one order and stops working the moment + // the pair is shown anywhere else. + // + // No model. The picker in the footer already shows what this session is set to, + // and showing it twice means two things to keep in step -- they disagreed for a + // moment on every model change, since one follows the request and the other the + // session's own answer. + Text( + "${summary.setupName} · ${summary.provider}", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + // Beside the provider it reports on, which is the line directly to its left. // - // No model. The picker in the footer already shows what this session is set to, - // and showing it twice means two things to keep in step -- they disagreed for a - // moment on every model change, since one follows the request and the other the - // session's own answer. + // Its real home is this provider's settings, which do not exist yet; until they do, + // the session is the only place the provider is already named, so it is the only + // place the button can sit without inventing a scope for itself. What it shows is + // the paid service's own numbers, so a session on a provider with no such service + // gets an honest "unavailable" rather than a hidden button -- a control that comes + // and goes makes its absence the signal, and absence cannot say why. + // Coloured by the worst window behind it, so the row says whether the limits are + // worth opening before anybody opens them. Blue at every ordinary level and only + // yellow or red near a limit -- and the theme's plain control colour whenever there + // is no measurement, since blue is the low end of the scale here and would read as + // "checked, and fine" about a machine nobody could reach. + Row { + // Left of the numbers about the *conversation*, because it is the same kind of + // thing about the *app*: what this session is costing to draw. It copies rather + // than opens, because what it produces is for somewhere else -- a message to + // whoever is looking at the code -- and a screenful of timings read on the + // phone + // is a screenful nobody can act on. + GlyphButton( + SPEED_GLYPH, + "Copy render timings", + onClick = { + val report = + debugReport( + device = + "device: ${Build.MODEL} (${Build.MANUFACTURER})," + + " Android ${Build.VERSION.RELEASE}", + transcript = + listOf( + " ${items.size} events, ${rows.size} rows," + + " ${units.size} units loaded", + " viewport" + + " ${listState.layoutInfo.viewportSize.height}px," + + " ${listState.layoutInfo.visibleItemsInfo.size}" + + " units visible", + " ${expandedTools.size} tool calls and" + + " ${expandedGroups.size} groups open", + ), + frames = frames.lines(context.refreshHz()), + accounting = + frames.drawPhase().let { (nanos, count) -> + drawAccounting(nanos, count) + }, + crash = lastCrash(context), + ) + context.copyToClipboard("ai-app render report", report) + // Also to the log, so a session driving the app over adb can read the + // same report the button copies. The clipboard is not reachable from a + // shell, and a counter nobody can check from here is a counter that + // only + // gets checked by asking Iris to press a button and paste. + Log.i("ai-app", report) + // Only once it is somewhere it can be read from, so a copy that never + // happened does not throw the stack away with it. + clearCrash(context) + // Emptied by the copy, so pressing it twice measures two separate + // stretches + // of scrolling rather than one and then the same one again. + frames.reset() + DebugStats.reset() + Toast.makeText(context, "Copied render report", Toast.LENGTH_SHORT) + .show() + }, + ) + GlyphButton( + USAGE_GLYPH, + "Usage", + { usageOpen = true }, + colour = usageGlyphColour(usage), + ) + // What it opens is about this session, so it sits at the end of the session's + // own row. The name is the whole of what it holds today, which is why it is a + // cog + // and not a word: there will be more, and a bar of words has nowhere to put it. + GlyphButton(SETTINGS_GLYPH, "Session settings", { settingsOpen = true }) + } + } + + // Under the header, above everything the session itself says: it is a fact about the + // machine rather than a turn in the conversation, and it is the number that decides + // whether to keep going -- which was a screen away from where that gets decided. + SessionUsageBar(usage) + + (streamError ?: actionError)?.let { message -> Text( - "${summary.setupName} · ${summary.provider}", + message, + color = MaterialTheme.colorScheme.error, style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp), ) } - // Beside the provider it reports on, which is the line directly to its left. + + // The transcript, reversed: item zero is the newest message and sits at the bottom, so + // the first frame of a session is already the right one, and following new content is + // where the list is rather than a correction it makes; see [TranscriptList]. // - // Its real home is this provider's settings, which do not exist yet; until they do, - // the session is the only place the provider is already named, so it is the only - // place the button can sit without inventing a scope for itself. What it shows is - // the paid service's own numbers, so a session on a provider with no such service - // gets an honest "unavailable" rather than a hidden button -- a control that comes - // and goes makes its absence the signal, and absence cannot say why. - // Coloured by the worst window behind it, so the row says whether the limits are - // worth opening before anybody opens them. Blue at every ordinary level and only - // yellow or red near a limit -- and the theme's plain control colour whenever there - // is no measurement, since blue is the low end of the scale here and would read as - // "checked, and fine" about a machine nobody could reach. - Row { - // Left of the numbers about the *conversation*, because it is the same kind of - // thing about the *app*: what this session is costing to draw. It copies rather - // than opens, because what it produces is for somewhere else -- a message to - // whoever is looking at the code -- and a screenful of timings read on the phone - // is a screenful nobody can act on. - GlyphButton( - SPEED_GLYPH, - "Copy render timings", - onClick = { - val report = - debugReport( - device = - "device: ${Build.MODEL} (${Build.MANUFACTURER})," + - " Android ${Build.VERSION.RELEASE}", - transcript = - listOf( - " ${items.size} events, ${rows.size} rows," + - " ${units.size} units loaded", - " viewport" + - " ${listState.layoutInfo.viewportSize.height}px," + - " ${listState.layoutInfo.visibleItemsInfo.size}" + - " units visible", - " ${expandedTools.size} tool calls and" + - " ${expandedGroups.size} groups open", - ), - frames = frames.lines(context.refreshHz()), - accounting = - frames.drawPhase().let { (nanos, count) -> - drawAccounting(nanos, count) - }, - crash = lastCrash(context), - ) - context.copyToClipboard("ai-app render report", report) - // Also to the log, so a session driving the app over adb can read the - // same report the button copies. The clipboard is not reachable from a - // shell, and a counter nobody can check from here is a counter that only - // gets checked by asking Iris to press a button and paste. - Log.i("ai-app", report) - // Only once it is somewhere it can be read from, so a copy that never - // happened does not throw the stack away with it. - clearCrash(context) - // Emptied by the copy, so pressing it twice measures two separate stretches - // of scrolling rather than one and then the same one again. - frames.reset() - DebugStats.reset() - Toast.makeText(context, "Copied render report", Toast.LENGTH_SHORT).show() - }, - ) - GlyphButton( - USAGE_GLYPH, - "Usage", - { usageOpen = true }, - colour = usageGlyphColour(usage), - ) - // What it opens is about this session, so it sits at the end of the session's - // own row. The name is the whole of what it holds today, which is why it is a cog - // and not a word: there will be more, and a bar of words has nowhere to put it. - GlyphButton(SETTINGS_GLYPH, "Session settings", { settingsOpen = true }) - } - } - - // Under the header, above everything the session itself says: it is a fact about the - // machine rather than a turn in the conversation, and it is the number that decides - // whether to keep going -- which was a screen away from where that gets decided. - SessionUsageBar(usage) - - (streamError ?: actionError)?.let { message -> - Text( - message, - color = MaterialTheme.colorScheme.error, - style = MaterialTheme.typography.bodySmall, - modifier = Modifier.padding(horizontal = 16.dp, vertical = 4.dp), - ) - } - - // The transcript, reversed: item zero is the newest message and sits at the bottom, so - // the first frame of a session is already the right one, and following new content is - // where the list is rather than a correction it makes; see [TranscriptList]. - // - // Drawn only once there is nothing left to put back. Held out of the drawing rather - // than out of the composition, so the restore's scroll is applied against a list that - // is fully built, and there is no frame in which the transcript is somewhere other - // than where it was left. - val settled = !restoring - Box(Modifier.weight(1f).fillMaxWidth()) { - Box(Modifier.fillMaxSize()) { - TranscriptList( - units = units, - state = listState, - moreHistory = moreHistory, - modifier = - Modifier.fillMaxSize().drawWithContent { if (settled) drawContent() }, - below = { - // The last thing in the transcript, because that is where they are in the - // session's reading of events: after everything it has taken in, and not - // yet taken in themselves. What the session is *doing* about them is a - // line below, in [SessionStatusRow]. - if (queued.isNotEmpty() || waitingCommands.isNotEmpty()) { - // The gap the arrangement no longer provides: this item sits flush - // against the newest message otherwise. - Column( - Modifier.padding(top = TRANSCRIPT_SPACING), - horizontalAlignment = Alignment.End, - ) { - waitingCommands.forEach { (_, text) -> - CommandBubble(text, waiting = true) - } - queued.forEach { waiting -> - UserBubble( - settings = settings, - sessionId = summary.id, - text = waiting.text, - images = waiting.images, - pending = true, - ) + // Drawn only once there is nothing left to put back. Held out of the drawing rather + // than out of the composition, so the restore's scroll is applied against a list that + // is fully built, and there is no frame in which the transcript is somewhere other + // than where it was left. + val settled = !restoring + Box( + Modifier.weight(1f) + .fillMaxWidth() + // The room the floating composer needs, measured off it below -- reserving it + // here is what lets the composer be an overlay without covering the newest + // message -- and then the keyboard's, per frame of its animation. This modifier + // is the whole of what the keyboard re-measures: the box's own size never + // changes, so nothing above it is touched. + .padding(bottom = with(LocalDensity.current) { composerHeight.toDp() }) + .imePadding() + ) { + Box(Modifier.fillMaxSize()) { + TranscriptList( + units = units, + state = listState, + moreHistory = moreHistory, + modifier = + Modifier.fillMaxSize().drawWithContent { if (settled) drawContent() }, + below = { + // The last thing in the transcript, because that is where they are in + // the + // session's reading of events: after everything it has taken in, and + // not + // yet taken in themselves. What the session is *doing* about them is a + // line below, in [SessionStatusRow]. + if (queued.isNotEmpty() || waitingCommands.isNotEmpty()) { + // The gap the arrangement no longer provides: this item sits flush + // against the newest message otherwise. + Column( + Modifier.padding(top = TRANSCRIPT_SPACING), + horizontalAlignment = Alignment.End, + ) { + waitingCommands.forEach { (_, text) -> + CommandBubble(text, waiting = true) + } + queued.forEach { waiting -> + UserBubble( + settings = settings, + sessionId = summary.id, + text = waiting.text, + images = waiting.images, + pending = true, + ) + } } } - } - }, - ) { unit -> - when (unit) { - is TranscriptUnit.Block -> MarkdownText(unit.text, replies) - is TranscriptUnit.Memory -> MemoryNote(unit.part, replies) - is TranscriptUnit.Whole -> { - val row = unit.row - Box( - Modifier.holdTopEdge(row.key, topEdgeHeld) { grew -> - // A *request*, not a raw scroll delta: this runs inside - // the measure pass that discovered the new height, and a - // raw delta forces a synchronous remeasure from within - // measure, which is fatal - // ("performMeasureAndLayout called during measure"). - // The request is applied by the same frame's next - // remeasure, so the correction still lands before - // anything is drawn. Reads unobserved, or this row's - // measure would inherit the scroll position as a - // dependency and remeasure on every frame of every - // fling. - Snapshot.withoutReadObservation { - listState.requestScrollToItem( - listState.firstVisibleItemIndex, - (listState.firstVisibleItemScrollOffset + grew) - .coerceAtLeast(0), - ) - } - } - // Which half of this row the touch landed in, for - // [toggleAnchored]. - // On the initial pass and consuming nothing, so every control - // inside - // still gets the gesture exactly as it would have; only visible - // rows - // have one, which is what makes a detector per row affordable. - .pointerInput(row.key) { - awaitEachGesture { - val down = - awaitFirstDown( - requireUnconsumed = false, - pass = PointerEventPass.Initial, + }, + ) { unit -> + when (unit) { + is TranscriptUnit.Block -> MarkdownText(unit.text, replies) + is TranscriptUnit.Memory -> MemoryNote(unit.part, replies) + is TranscriptUnit.Whole -> { + val row = unit.row + Box( + Modifier.holdTopEdge(row.key, topEdgeHeld) { grew -> + // A *request*, not a raw scroll delta: this runs inside + // the measure pass that discovered the new height, and + // a + // raw delta forces a synchronous remeasure from within + // measure, which is fatal + // ("performMeasureAndLayout called during measure"). + // The request is applied by the same frame's next + // remeasure, so the correction still lands before + // anything is drawn. Reads unobserved, or this row's + // measure would inherit the scroll position as a + // dependency and remeasure on every frame of every + // fling. + Snapshot.withoutReadObservation { + listState.requestScrollToItem( + listState.firstVisibleItemIndex, + (listState.firstVisibleItemScrollOffset + grew) + .coerceAtLeast(0), ) - lastTouch.key = row.key - lastTouch.high = down.position.y < size.height / 2f + } } - } - ) { - when (row) { - is TranscriptRow.Tools -> - ToolGroup( - group = row, - expanded = row.id in expandedGroups, - onToggle = { - toggleAnchored(row) { - expandedGroups = - if (row.id in expandedGroups) - expandedGroups - row.id - else expandedGroups + row.id - } - }, - isToolExpanded = { it in expandedTools }, - // Anchored on the group, not the call: opening one call - // makes - // the whole group taller, and the heading the reader is - // under - // is the group's. - onToolToggle = { id -> - toggleAnchored(row) { - expandedTools = - if (id in expandedTools) expandedTools - id - else expandedTools + id - } - }, - onAnswer = { questionId, answers -> - act { - answerQuestion( - settings, - summary.id, - questionId, - answers, + // Which half of this row the touch landed in, for + // [toggleAnchored]. + // On the initial pass and consuming nothing, so every + // control + // inside + // still gets the gesture exactly as it would have; only + // visible + // rows + // have one, which is what makes a detector per row + // affordable. + .pointerInput(row.key) { + awaitEachGesture { + val down = + awaitFirstDown( + requireUnconsumed = false, + pass = PointerEventPass.Initial, ) - } - }, - image = { ref -> - SessionImage(settings, summary.id, ref) - }, - ) - is TranscriptRow.Single -> - when (val item = row.item) { - is TranscriptItem.UserMsg -> - UserBubble( - settings = settings, - sessionId = summary.id, - text = item.text, - images = item.images, - ) - is TranscriptItem.AssistantMsg -> - // A whole assistant row is only ever the reply - // still - // arriving -- every settled reply is flattened into - // block units instead; see [transcriptUnits]. Live - // is - // what earns its blocks a layer each while deltas - // land. - AssistantMessage(item.text, replies, live = true) - is TranscriptItem.ToolRun -> - ToolCard( - tool = item, - expanded = item.id in expandedTools, - onToggle = { - toggleAnchored(row) { - expandedTools = - if (item.id in expandedTools) - expandedTools - item.id - else expandedTools + item.id - } - }, - onAnswer = { questionId, answers -> - act { - answerQuestion( - settings, - summary.id, - questionId, - answers, - ) - } - }, - image = { ref -> - SessionImage(settings, summary.id, ref) - }, - ) - is TranscriptItem.QuestionCard -> - QuestionRow(item) { answers -> + lastTouch.key = row.key + lastTouch.high = down.position.y < size.height / 2f + } + } + ) { + when (row) { + is TranscriptRow.Tools -> + ToolGroup( + group = row, + expanded = row.id in expandedGroups, + onToggle = { + toggleAnchored(row) { + expandedGroups = + if (row.id in expandedGroups) + expandedGroups - row.id + else expandedGroups + row.id + } + }, + isToolExpanded = { it in expandedTools }, + // Anchored on the group, not the call: opening one + // call + // makes + // the whole group taller, and the heading the + // reader is + // under + // is the group's. + onToolToggle = { id -> + toggleAnchored(row) { + expandedTools = + if (id in expandedTools) + expandedTools - id + else expandedTools + id + } + }, + onAnswer = { questionId, answers -> act { answerQuestion( settings, summary.id, - item.id, + questionId, answers, ) } - } - is TranscriptItem.ErrorMsg -> - Text( - item.message, - color = MaterialTheme.colorScheme.error, - style = MaterialTheme.typography.bodyMedium, - ) - is TranscriptItem.ImageItem -> - SessionImage(settings, summary.id, item.ref) - is TranscriptItem.Note -> - Text( - item.text, - style = MaterialTheme.typography.bodySmall, - color = - MaterialTheme.colorScheme.onSurfaceVariant, - ) - is TranscriptItem.CommandRow -> CommandBubble(item.text) - is TranscriptItem.ClearedNote -> ClearedRow() - is TranscriptItem.CompactedNote -> CompactedRow(item) - is TranscriptItem.PeerNote -> - PeerMessageRow( - item = item, - expanded = item.seq in expandedNotes, - replies = replies, - onToggle = { - toggleAnchored(row) { - expandedNotes = - if (item.seq in expandedNotes) - expandedNotes - item.seq - else expandedNotes + item.seq + }, + image = { ref -> + SessionImage(settings, summary.id, ref) + }, + ) + is TranscriptRow.Single -> + when (val item = row.item) { + is TranscriptItem.UserMsg -> + UserBubble( + settings = settings, + sessionId = summary.id, + text = item.text, + images = item.images, + ) + is TranscriptItem.AssistantMsg -> + // A whole assistant row is only ever the reply + // still + // arriving -- every settled reply is flattened + // into + // block units instead; see [transcriptUnits]. + // Live + // is + // what earns its blocks a layer each while + // deltas + // land. + AssistantMessage( + item.text, + replies, + live = true, + ) + is TranscriptItem.ToolRun -> + ToolCard( + tool = item, + expanded = item.id in expandedTools, + onToggle = { + toggleAnchored(row) { + expandedTools = + if (item.id in expandedTools) + expandedTools - item.id + else expandedTools + item.id + } + }, + onAnswer = { questionId, answers -> + act { + answerQuestion( + settings, + summary.id, + questionId, + answers, + ) + } + }, + image = { ref -> + SessionImage(settings, summary.id, ref) + }, + ) + is TranscriptItem.QuestionCard -> + QuestionRow(item) { answers -> + act { + answerQuestion( + settings, + summary.id, + item.id, + answers, + ) } - }, - ) - } + } + is TranscriptItem.ErrorMsg -> + Text( + item.message, + color = MaterialTheme.colorScheme.error, + style = MaterialTheme.typography.bodyMedium, + ) + is TranscriptItem.ImageItem -> + SessionImage(settings, summary.id, item.ref) + is TranscriptItem.Note -> + Text( + item.text, + style = MaterialTheme.typography.bodySmall, + color = + MaterialTheme.colorScheme + .onSurfaceVariant, + ) + is TranscriptItem.CommandRow -> + CommandBubble(item.text) + is TranscriptItem.ClearedNote -> ClearedRow() + is TranscriptItem.CompactedNote -> + CompactedRow(item) + is TranscriptItem.PeerNote -> + PeerMessageRow( + item = item, + expanded = item.seq in expandedNotes, + replies = replies, + onToggle = { + toggleAnchored(row) { + expandedNotes = + if (item.seq in expandedNotes) + expandedNotes - item.seq + else expandedNotes + item.seq + } + }, + ) + } + } } } } } } - } - // Still finding out what this conversation is: the newest page has not arrived, or - // it has and the list is being put back where reading stopped. Both draw no rows at - // all, and a blank page is what this screen otherwise means by "there is nothing - // here" -- so the state that does not know needs its own appearance rather than - // sharing one with the empty answer. - // - // In the middle of the transcript rather than at either end, because it is not - // reporting on the newest message or the oldest; it is standing in for all of them. - // `settled` and not `restoring` alone, so the spinner covers the whole wait: fetching - // the history a saved position needs, and then the frames between those rows arriving - // and the layout that measures them putting the position back. They are the two halves - // of the same wait and the transcript is not drawn for either. - if (!ready || !settled) { - CircularProgressIndicator(Modifier.align(Alignment.Center).size(LOADING_SPINNER)) - } - - // Only while the newest message is off-screen. Reading back - // through a conversation is a place to be, not a state to be - // rescued from, so this waits to be wanted. - // - // Down, and the same chevron a tool group collapses with: the - // list is built upside down internally, but nobody reading it - // knows that -- on screen the newest message is at the bottom, - // which is where this goes. The name is carried in the - // description, since an arrow alone says nothing to a screen - // reader and nothing to whoever finds this in six months. - if (!atNewest) { - Surface( - // Instantly. An animated scroll travels the whole transcript, so the - // further back somebody has read the longer this takes -- the one press - // whose cost grows with how much there is to skip, which is backwards. - // - // Arriving there is all this has to do now. The newest end is where the - // content hangs from, so being at it is the whole of following it, and there - // is no separate flag to set -- which is what this press used to forget, - // landing the reader at the bottom with new messages not bringing the view - // with them. - onClick = { scope.launch { listState.scrollToItem(0) } }, - shape = CircleShape, - color = MaterialTheme.colorScheme.surfaceContainerHigh, - modifier = - Modifier.align(Alignment.BottomCenter).padding(bottom = 12.dp).semantics { - contentDescription = "Jump to latest" - }, - ) { - Chevron( - pointingUp = false, - colour = MaterialTheme.colorScheme.onSurface, - modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), + // Still finding out what this conversation is: the newest page has not arrived, or + // it has and the list is being put back where reading stopped. Both draw no rows at + // all, and a blank page is what this screen otherwise means by "there is nothing + // here" -- so the state that does not know needs its own appearance rather than + // sharing one with the empty answer. + // + // In the middle of the transcript rather than at either end, because it is not + // reporting on the newest message or the oldest; it is standing in for all of them. + // `settled` and not `restoring` alone, so the spinner covers the whole wait: + // fetching + // the history a saved position needs, and then the frames between those rows + // arriving + // and the layout that measures them putting the position back. They are the two + // halves + // of the same wait and the transcript is not drawn for either. + if (!ready || !settled) { + CircularProgressIndicator( + Modifier.align(Alignment.Center).size(LOADING_SPINNER) ) } + + // Only while the newest message is off-screen. Reading back + // through a conversation is a place to be, not a state to be + // rescued from, so this waits to be wanted. + // + // Down, and the same chevron a tool group collapses with: the + // list is built upside down internally, but nobody reading it + // knows that -- on screen the newest message is at the bottom, + // which is where this goes. The name is carried in the + // description, since an arrow alone says nothing to a screen + // reader and nothing to whoever finds this in six months. + if (!atNewest) { + Surface( + // Instantly. An animated scroll travels the whole transcript, so the + // further back somebody has read the longer this takes -- the one press + // whose cost grows with how much there is to skip, which is backwards. + // + // Arriving there is all this has to do now. The newest end is where the + // content hangs from, so being at it is the whole of following it, and + // there + // is no separate flag to set -- which is what this press used to forget, + // landing the reader at the bottom with new messages not bringing the view + // with them. + onClick = { scope.launch { listState.scrollToItem(0) } }, + shape = CircleShape, + color = MaterialTheme.colorScheme.surfaceContainerHigh, + modifier = + Modifier.align(Alignment.BottomCenter) + .padding(bottom = 12.dp) + .semantics { contentDescription = "Jump to latest" }, + ) { + Chevron( + pointingUp = false, + colour = MaterialTheme.colorScheme.onSurface, + modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), + ) + } + } } } - pendingModel?.let { chosen -> - ModelSwitchWarning( - from = modelLabel(model), - to = modelLabel(chosen), - onDismiss = { pendingModel = null }, - onConfirm = { - pendingModel = null - act { setSessionModel(settings, summary.id, chosen) } - }, - ) - } - - SessionStatusRow( - status = status, - compactingFor = compactingFor, - contextTokens = contextTokens, - ) - - // Between the transcript and the box: above what is being typed, so the list does not - // cover the thing the command is about, and below everything that explains it. - CommandSuggestions( - commands = suggestedCommands(input), - onPick = { command -> input = command.typed() }, - ) - - // Always enabled -- a send while the session is running becomes a - // steering message injected at the next tool boundary, which is - // the point of the whole app. - // - // The field gets a row of its own, above the buttons: sharing one - // put the full width behind three controls, so the thing being - // typed into was the narrowest thing on the row. - Column(Modifier.fillMaxWidth().padding(8.dp)) { - // Directly above the box they will be sent from, so what is attached is visible - // rather than counted: the "+2" on the button below said how many and never which. - PendingAttachments( - settings = settings, - sessionId = summary.id, - refs = pendingAttachments, - onRemove = { pendingAttachments = pendingAttachments - it }, - ) - OutlinedTextField( - value = input, - onValueChange = { - input = it - saveDraft(context, summary.id, it) - }, - modifier = Modifier.fillMaxWidth(), - // No longer "(+image)": the images are on screen above this, and a placeholder - // saying so said it in words beside the thing itself. - placeholder = { Text("Message") }, - maxLines = 4, - ) - Row( - verticalAlignment = Alignment.CenterVertically, - modifier = Modifier.fillMaxWidth(), - ) { - TextButton( - onClick = { - pickImage.launch( - PickVisualMediaRequest( - ActivityResultContracts.PickVisualMedia.ImageOnly - ) - ) - } - ) { - // Just "+" now. The count was standing in for showing them. - Text("+") + // Everything from here down floats: bottom-aligned over the transcript, moved up with + // the keyboard by a translation on its own layer. The translation is read inside the + // graphicsLayer block, so a keyboard frame invalidates layer properties only -- no + // measure, no recomposition, no re-recording of anything. Its height is reported to the + // transcript box above, which reserves that much room; the opaque background covers the + // one frame between this growing (a suggestion row, a second draft line) and that + // reservation catching up. + Column( + Modifier.align(Alignment.BottomCenter) + .fillMaxWidth() + .onSizeChanged { composerHeight = it.height } + .graphicsLayer { + translationY = + -(imeInsets.getBottom(this) - navInsets.getBottom(this)) + .coerceAtLeast(0) + .toFloat() } - // The settings share what is left after the actions have - // taken what they need. A Row hands out intrinsic widths in - // order and clips whatever runs past the edge, so with - // these laid out first the arrival of Stop pushed Send off - // the screen entirely -- the app's central control, gone at - // exactly the moment the app is most in use. + .background(MaterialTheme.colorScheme.background) + ) { + pendingModel?.let { chosen -> + ModelSwitchWarning( + from = modelLabel(model), + to = modelLabel(chosen), + onDismiss = { pendingModel = null }, + onConfirm = { + pendingModel = null + act { setSessionModel(settings, summary.id, chosen) } + }, + ) + } + + SessionStatusRow( + status = status, + compactingFor = compactingFor, + contextTokens = contextTokens, + ) + + // Between the transcript and the box: above what is being typed, so the list does not + // cover the thing the command is about, and below everything that explains it. + CommandSuggestions( + commands = suggestedCommands(input), + onPick = { command -> input = command.typed() }, + ) + + // Always enabled -- a send while the session is running becomes a + // steering message injected at the next tool boundary, which is + // the point of the whole app. + // + // The field gets a row of its own, above the buttons: sharing one + // put the full width behind three controls, so the thing being + // typed into was the narrowest thing on the row. + Column(Modifier.fillMaxWidth().padding(8.dp)) { + // Directly above the box they will be sent from, so what is attached is visible + // rather than counted: the "+2" on the button below said how many and never which. + PendingAttachments( + settings = settings, + sessionId = summary.id, + refs = pendingAttachments, + onRemove = { pendingAttachments = pendingAttachments - it }, + ) + OutlinedTextField( + value = input, + onValueChange = { + input = it + saveDraft(context, summary.id, it) + }, + modifier = Modifier.fillMaxWidth(), + // No longer "(+image)": the images are on screen above this, and a placeholder + // saying so said it in words beside the thing itself. + placeholder = { Text("Message") }, + maxLines = 4, + ) Row( verticalAlignment = Alignment.CenterVertically, - modifier = Modifier.weight(1f), + modifier = Modifier.fillMaxWidth(), ) { - if (offeredModels.isNotEmpty()) { + TextButton( + onClick = { + pickImage.launch( + PickVisualMediaRequest( + ActivityResultContracts.PickVisualMedia.ImageOnly + ) + ) + } + ) { + // Just "+" now. The count was standing in for showing them. + Text("+") + } + // The settings share what is left after the actions have + // taken what they need. A Row hands out intrinsic widths in + // order and clips whatever runs past the edge, so with + // these laid out first the arrival of Stop pushed Send off + // the screen entirely -- the app's central control, gone at + // exactly the moment the app is most in use. + Row( + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.weight(1f), + ) { + if (offeredModels.isNotEmpty()) { + PickerButton( + current = modelLabel(model), + // What the machine offers, plus the state a session is in when it + // has chosen none of them. The button has always been able to say + // "default"; until this the list could not, so leaving it was a + // one-way trip. + options = listOf(DEFAULT_MODEL) + offeredModels, + // Not set here. The button follows what the session reports it + // is set to, which arrives a moment later and is sometimes a + // different answer -- a name the CLI resolved, or no change at all + // on a provider whose model is fixed when it starts. + // Asked about first, unless there is nothing to lose by it -- + // see [ModelSwitchWarning]. + onPick = { chosen -> + if ( + modelLabel(chosen) == modelLabel(model) || items.isEmpty() + ) { + act { setSessionModel(settings, summary.id, chosen) } + } else { + pendingModel = chosen + } + }, + ) + } PickerButton( - current = modelLabel(model), - // What the machine offers, plus the state a session is in when it - // has chosen none of them. The button has always been able to say - // "default"; until this the list could not, so leaving it was a - // one-way trip. - options = listOf(DEFAULT_MODEL) + offeredModels, - // Not set here. The button follows what the session reports it - // is set to, which arrives a moment later and is sometimes a - // different answer -- a name the CLI resolved, or no change at all - // on a provider whose model is fixed when it starts. - // Asked about first, unless there is nothing to lose by it -- - // see [ModelSwitchWarning]. + current = permissionMode, + options = PERMISSION_MODES, onPick = { chosen -> - if (modelLabel(chosen) == modelLabel(model) || items.isEmpty()) { - act { setSessionModel(settings, summary.id, chosen) } - } else { - pendingModel = chosen - } + act { setSessionPermissionMode(settings, summary.id, chosen) } }, ) } - PickerButton( - current = permissionMode, - options = PERMISSION_MODES, - onPick = { chosen -> - act { setSessionPermissionMode(settings, summary.id, chosen) } - }, - ) - } - // The same filled shape as the button beside it, not an outlined one: these are - // two things you can do about the session, and weighting one of them as secondary - // said they were a primary action and its qualifier. What separates them is the - // colour and the mark, which is what they mean. - // - // Always here, rather than arriving with the turn as it used to. A control that - // comes and goes makes its own presence the signal, and its absence could not say - // whether there was nothing to do; a button that is always in the same place also - // cannot push Send off the end of the row by turning up. - val process = - when { - running -> ProcessAction.Pause - status == "exited" -> ProcessAction.Start - else -> ProcessAction.Stop - } - Button( - onClick = { - processInFlight = true - act(onDone = { processInFlight = false }) { - process.perform(settings, summary.id) + // The same filled shape as the button beside it, not an outlined one: these are + // two things you can do about the session, and weighting one of them as + // secondary + // said they were a primary action and its qualifier. What separates them is the + // colour and the mark, which is what they mean. + // + // Always here, rather than arriving with the turn as it used to. A control that + // comes and goes makes its own presence the signal, and its absence could not + // say + // whether there was nothing to do; a button that is always in the same place + // also + // cannot push Send off the end of the row by turning up. + val process = + when { + running -> ProcessAction.Pause + status == "exited" -> ProcessAction.Start + else -> ProcessAction.Stop } - }, - enabled = !processInFlight, - colors = actionButtonColors(process.colour()), - ) { - Glyph( - process.glyph, - colour = LocalContentColor.current, - modifier = Modifier.semantics { contentDescription = process.label }, - ) - } - Spacer(Modifier.width(8.dp)) - // The paper plane, with a clock on it while a turn is in flight: sending then - // queues the message for the next tool boundary rather than starting a turn of - // its own, and the two have to be told apart at a glance. The label says the same - // thing to a screen reader, which has nothing else to read. - // - // Disabled while there is nothing to send, rather than pressable and silent: - // `send` has always returned early on an empty composer, so the button promised - // something it would not do, and the only feedback was the ripple. Disabled and - // not hidden, for the reason the button beside it is always here. - Button( - onClick = { send() }, - enabled = input.isNotBlank() || pendingAttachments.isNotEmpty(), - colors = actionButtonColors(if (running) queueColor else sendColor), - ) { - Glyph( - if (running) QUEUE_GLYPH else SEND_GLYPH, - colour = LocalContentColor.current, - modifier = Modifier.semantics { contentDescription = sendLabel(running) }, - ) + Button( + onClick = { + processInFlight = true + act(onDone = { processInFlight = false }) { + process.perform(settings, summary.id) + } + }, + enabled = !processInFlight, + colors = actionButtonColors(process.colour()), + ) { + Glyph( + process.glyph, + colour = LocalContentColor.current, + modifier = Modifier.semantics { contentDescription = process.label }, + ) + } + Spacer(Modifier.width(8.dp)) + // The paper plane, with a clock on it while a turn is in flight: sending then + // queues the message for the next tool boundary rather than starting a turn of + // its own, and the two have to be told apart at a glance. The label says the + // same + // thing to a screen reader, which has nothing else to read. + // + // Disabled while there is nothing to send, rather than pressable and silent: + // `send` has always returned early on an empty composer, so the button promised + // something it would not do, and the only feedback was the ripple. Disabled and + // not hidden, for the reason the button beside it is always here. + Button( + onClick = { send() }, + enabled = input.isNotBlank() || pendingAttachments.isNotEmpty(), + colors = actionButtonColors(if (running) queueColor else sendColor), + ) { + Glyph( + if (running) QUEUE_GLYPH else SEND_GLYPH, + colour = LocalContentColor.current, + modifier = + Modifier.semantics { contentDescription = sendLabel(running) }, + ) + } } } } @@ -2097,6 +2187,7 @@ private fun SessionStatusRow( contextTokens: Long?, modifier: Modifier = Modifier, ) { + DebugStats.count("status row recomposed") Row( verticalAlignment = Alignment.CenterVertically, modifier = modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 4.dp), diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionUsageBar.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionUsageBar.kt index 1cea984..ada615f 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionUsageBar.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionUsageBar.kt @@ -116,6 +116,7 @@ fun usageGlyphColour(usage: SessionUsage): Color = */ @Composable fun SessionUsageBar(usage: SessionUsage, modifier: Modifier = Modifier) { + DebugStats.count("usage bar recomposed") // The countdown moves even when the numbers do not, so it is driven by a clock of its own // rather than recomputed at draw time: a percentage that comes back unchanged is an equal // value, Compose skips the recomposition, and a "left" that only ticked when the quota From a63ae8347ffb332c2aa2f82c47ddc47ab9884ddc Mon Sep 17 00:00:00 2001 From: iris <2+iris@noreply.localhost> Date: Mon, 31 Aug 2026 15:04:04 -0400 Subject: [PATCH 2/3] Load history before the reader arrives, and estimate room from real sizes The first loading boundary sat barely off-screen: the opening page is deliberately small (sized for time-to-first-frame), so one short scroll met a spinner and a round trip. And the prefetch that should have hidden every later boundary estimated room-ahead from the average size of the units currently visible -- the worst possible sample, since two tall blocks filling a viewport multiply out over dozens of unseen one-line rows and report screens of room when the end is one swipe away. Iris's report showed exactly that: viewport 676px, two units visible. Three changes, one per cause: - The opening page is followed by a full page fetched in the background as soon as the screen is up -- the same reasoning that keeps the opening page off the critical path puts the first real page right behind it. A restore skips this; it just paged as deep as its anchor needed. - Room ahead is now added up from the real size of every unit the list has ever laid out, recorded by key as units pass through the viewport, with the running average standing in for units never seen. The walk early-outs at the cushion, so a frame's check stops after a few units in the common case. - The cushion is six screens (was three) and a page is 400 events (was 800). The cushion grew because firing early costs a page nobody may read while firing late is a spinner under a finger for a tunnel round trip; the page shrank because pages now load before anybody waits on them, so a page only has to stay above the fold floor -- the median run of deltas folding into one reply is ~400 events, and below that a page can add no visible room at all. Smaller pages cross the tunnel faster and land in smaller frame spikes, which is what makes the boundary hard to catch. Verified on the emulator against the biggest transcript on the machine at --delay 300: opening a session loads the first background page unscrolled, twenty-five hard flings paged the rest in with zero frames in which the spinner was on screen, and at --delay 2000 -- outrunning the chain deliberately -- the boundary shows the spinner and then heals with the paragraph being read holding its position. Co-Authored-By: Claude Fable 5 --- .../kotlin/com/example/aiapp/SessionScreen.kt | 122 +++++++++++------- 1 file changed, 76 insertions(+), 46 deletions(-) diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt index 895398f..48755b5 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt @@ -95,41 +95,46 @@ private const val RECONNECT_DELAY_MS = 1500L private val LOADING_SPINNER = 48.dp /** - * How much history to keep loaded past the oldest row on screen, counted in screenfuls. + * How close, in screenfuls of estimated scroll, the reader may come to the end of loaded history + * before the next page is fetched. * - * Both the point at which history starts loading and how much of it a load has to produce before it - * stops. Multiplied by the viewport to give a number of *pixels* of scroll, which is the distance - * the question is actually about: how far the reader can keep going before they run out. A row is + * Multiplied by the viewport to give a number of *pixels* of scroll, which is the distance the + * question is actually about: how far the reader can keep going before they run out. A row is * anything from one line to a page, so a count of rows is that distance only by accident. Eight - * rows was the number, and on a tool-heavy transcript eight rows is less than one screen: the + * rows was the number once, and on a tool-heavy transcript eight rows is less than one screen: the * reader reached the end of what was loaded on *every* swipe and waited a round trip standing - * there, which is a list running out of transcript rather than a slow frame. Counting screenfuls of - * rows fixed the size of the mistake without fixing its kind; pixels are the unit itself. + * there, which is a list running out of transcript rather than a slow frame. * - * Three, so a fling lands on rows that are already there and the page after them is on its way. The - * cost of being generous is a page fetched that nobody reads; the cost of being mean is a list that - * stops under a finger, and those are not the same size. - * - * Counted at the server rather than inferred from the screen, which is the only measurement here - * that does not depend on how the emulator renders: against a 24,000-event transcript at `--delay - * 120`, ten swipes asked for **ten** pages before this and **three** after. + * Six, because the two ways to be wrong are not the same size: firing early costs a page fetched + * that nobody reads, firing late is a spinner under somebody's finger for a whole round trip over + * the tunnel -- and the distance a hard fling covers while that fetch is in flight is several + * screens on its own. The estimate this multiplies is built from measured unit sizes, so a bigger + * cushion no longer amplifies a bad guess the way it would have when the guess came from whatever + * happened to be on screen. */ -private const val HISTORY_SCREENS = 3 +private const val HISTORY_SCREENS = 6 /** - * How many events a backwards page asks for, which is ten times what the opening page takes. + * How many events a backwards page asks for, which is five times what the opening page takes. * - * Because an event is not a row, and the ratio is nothing like one to one. Measured on a real - * transcript (2,426 events, 2026-08-30): the whole conversation is *seven* assistant messages, and - * the median run of consecutive text deltas that fold into one of them is four hundred. A page of - * eighty is therefore a fifth of a single row, and reaching a screenful of fresh rows took about - * thirty sequential round trips inside one collect -- a stutter on loopback, and four or five - * seconds of a list that will not move over the tunnel, which reads as history having run out. + * The floor is that an event is not a row, and the ratio is nothing like one to one. Measured on a + * real transcript (2,426 events, 2026-08-30): the whole conversation is *seven* assistant messages, + * and the median run of consecutive text deltas that fold into one of them is four hundred. A page + * of eighty is therefore a fifth of a single row, and reaching a screenful of fresh rows took about + * thirty sequential round trips inside one collect. Below this number a page can add no visible + * room at all, and the fetch chain degenerates into those round trips again. + * + * At the floor rather than above it, because pages are fetched in the background before the reader + * arrives -- the cushion decides how deep loading runs, and a page that was not enough is followed + * by another without anybody waiting on either. What a *smaller* page buys is hiding: it crosses + * the tunnel in half the time and lands in a smaller frame spike, so the case where the reader + * outruns an in-flight fetch is rarer and cheaper. This was 800 when the reader was the one + * standing at the boundary and each round trip had to be amortized as far as it would go. * * The opening page stays small: it is the one on the critical path of showing the screen at all, * and it only has to fill a viewport. */ -private const val HISTORY_PAGE = 800 +private const val HISTORY_PAGE = 400 /** * The most events one request of a restore may ask for. @@ -1078,6 +1083,22 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () // state the screen can draw, and a permanently blank one is not. restoring = false ready = true + // The opening page is sized for time-to-first-frame, not for reading: it fills a + // viewport or two, so the first "still loading" boundary sat barely off-screen and the + // first upward scroll met it and waited a round trip. The same reasoning that keeps the + // opening page off the critical path puts the first full page right behind it, while + // the screen is already up. A restore skips this: it has just paged as deep as the + // anchor needed. + if (savedAnchor == null && moreHistory && !loadingHistory) { + loadingHistory = true + try { + loadOlderPage() + } catch (_: ApiException) { + // The next scroll asks again. + } finally { + loadingHistory = false + } + } } // Only while the screen is actually on screen. Android stops the @@ -1214,37 +1235,46 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () ) } } - // Reaching the far end of what is loaded fetches the page before it. + // Reaching within a few screens of the far end of what is loaded fetches the page before + // it. // // The question is pixels of scroll -- how far can the reader keep going before they run out // -- and a lazy list cannot answer it exactly, because it has never measured the items it - // has not composed. So the room ahead is *estimated*: the units past the last visible one, - // at the typical size of the units that are on screen. A unit is at most a block of a reply, - // which is what makes the estimate usable where a count of rows was not -- a row is anything - // from one line to twenty-five screens, a block is roughly a paragraph. Being wrong is - // cheap and one-sided in effect: too low fetches a page early, too high is corrected a few - // frames later as the real sizes scroll in, and the spinner item stands at the edge for - // whatever slips through. + // has not composed. So the room ahead is added up from the real size of every unit the + // list *has* laid out, kept by key as units pass through the viewport, with the running + // average standing in for the ones it has never seen. It used to be the average of the + // units currently on screen, and the units on screen are the worst possible sample: two + // tall blocks fill a viewport, multiply out over dozens of unseen one-line rows, and + // report screens of room when the end is one swipe away -- so the reader met the spinner + // at every boundary, which is exactly what the cushion exists to prevent. // // There is no correction beside this one. Following the newest message is not an effect: // the list is reversed, so an arriving message extends the end the viewport is pinned to, // and a page of history lands past every visible index and moves nothing. + val unitSizes = remember(summary.id) { HashMap() } LaunchedEffect(listState, moreHistory) { - snapshotFlow { - val info = listState.layoutInfo - val visible = info.visibleItemsInfo - if (visible.isEmpty()) null - else - Triple( - info.totalItemsCount - 1 - visible.last().index, - visible.sumOf { it.size } / visible.size, - info.viewportSize.height, - ) - } - .collect { measured -> - val (ahead, typical, viewport) = measured ?: return@collect - if (restoring || !moreHistory || loadingHistory || viewport == 0) return@collect - if (ahead.toLong() * typical >= viewport.toLong() * HISTORY_SCREENS) return@collect + snapshotFlow { listState.layoutInfo } + .collect { info -> + val visible = info.visibleItemsInfo + if (visible.isEmpty()) return@collect + // Before the guards below, so sizes keep accumulating while a page is in + // flight and the next estimate starts better informed. + visible.forEach { unitSizes[it.key] = it.size } + if (restoring || !moreHistory || loadingHistory) return@collect + val viewport = info.viewportSize.height + if (viewport == 0) return@collect + val loaded = currentUnits + val average = unitSizes.values.sum() / unitSizes.size + // From the last visible lazy index: item zero is the "below" slot, so lazy + // index equals units index plus one -- starting the walk at `last().index` + // begins one unit past the last visible one, and a visible spinner makes the + // range empty, which is room of zero. + var room = 0L + val cushion = viewport.toLong() * HISTORY_SCREENS + for (index in visible.last().index until loaded.size) { + room += unitSizes[loaded[index].key] ?: average + if (room >= cushion) return@collect + } loadingHistory = true try { // One page, and then this fires again if it was not enough -- the estimate From d07325401d13af15d1011d4abe2721f30e325162 Mon Sep 17 00:00:00 2001 From: iris <2+iris@noreply.localhost> Date: Mon, 31 Aug 2026 15:28:15 -0400 Subject: [PATCH 3/3] Clean up the app: one fix per rule already written down somewhere else A pass over every Kotlin file, fixing each place a rule the codebase had already learned was applied to only one member of its set: - Notifications: checkSelfPermission(POST_NOTIFICATIONS) on Android 12 and below answers "denied" for a permission that does not exist there, so every notification on API 24-32 was silently dropped. Version-guarded; before 13 the app-level switch is the whole answer. - The permission-mode list was written three times and had drifted: the import screen was missing "plan". One list in Api.kt now. - The import screen's delete refetched the whole list through a loading spinner -- the exact fault the session list's delete already fixed and documented. It now removes only the deleted row. - The import screen truncated paths at a hardcoded 40 characters; it now uses StartEllipsis against the row's real width, like the models screen. - warm() bypassed the partsOf cache built for it, re-scanning every loaded message per page, and warmed a multi-block memory note under keys no row looks up. It now mirrors transcriptUnits through the same caches. - The transcript's data model (TranscriptItem, foldEvent, joinPages, warm) moved out of SessionScreen.kt into TranscriptItems.kt: pure folding with no screen in it, changing for unrelated reasons in the same file. - One image fetch/decode/failed block was written twice; it is rememberSessionBitmap in SessionImage.kt now. - Lint is fully clean: android.media.ExifInterface replaced with the androidx one (the framework copy lacks the hostile-image parsing fixes, and these images arrive from outside the phone), highlights bumped to 1.1.0, and the notification fix above closed InlinedApi. - Dead weight out: an unused act() onFailure parameter, and five orphaned or misattached doc comments (UserBubble carried SessionImage's doc). Verified: ktfmt, compileDebugKotlin, lintDebug (0 issues), server suite (93 passed), clippy and rustfmt clean; exercised on the emulator against a real imported transcript -- paging, tool groups, block rendering, no crashes. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 7 +- app/androidApp/build.gradle.kts | 1 + .../src/main/kotlin/com/example/aiapp/Api.kt | 10 + .../kotlin/com/example/aiapp/Attachments.kt | 2 +- .../kotlin/com/example/aiapp/EventStream.kt | 12 +- .../kotlin/com/example/aiapp/ImportScreen.kt | 27 +- .../kotlin/com/example/aiapp/MemoryNote.kt | 17 +- .../kotlin/com/example/aiapp/Notifications.kt | 10 +- .../com/example/aiapp/PendingAttachments.kt | 22 +- .../com/example/aiapp/ResetCountdown.kt | 10 +- .../kotlin/com/example/aiapp/SessionImage.kt | 44 +- .../com/example/aiapp/SessionListScreen.kt | 3 - .../kotlin/com/example/aiapp/SessionScreen.kt | 446 +----------------- .../kotlin/com/example/aiapp/SpawnScreen.kt | 7 - .../main/kotlin/com/example/aiapp/ToolRows.kt | 5 +- .../com/example/aiapp/TranscriptItems.kt | 439 +++++++++++++++++ app/gradle/libs.versions.toml | 10 +- 17 files changed, 547 insertions(+), 525 deletions(-) create mode 100644 app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptItems.kt diff --git a/AGENTS.md b/AGENTS.md index 8f069f4..fe3aee8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -205,9 +205,10 @@ first if a remote spawn ever mangles an argument. `./run-android.sh` to build, install, and launch on the emulator. - **Android Lint is not optional and is not run by a build.** It found a crash that had been shipping: `java.time` on a minSdk-24 app with - desugaring off. It is clean now apart from Compose 1.11.1 having a 1.12.0 - available; keep it that way, and suppress with `tools:ignore` plus a - written reason rather than by lowering the bar. + desugaring off — and later a permission check that silently dropped every + notification on Android 12 and below. It is fully clean as of 2026-08-31; + keep it that way, and suppress with `tools:ignore` plus a written reason + rather than by lowering the bar. - **The APK pins the CA of the machine that builds it**, read at build time from `$XDG_CONFIG_HOME/ai-app/certs/ca.pem` (`AI_APP_CA` overrides) and generated into a constant. So the server must have started once on that diff --git a/app/androidApp/build.gradle.kts b/app/androidApp/build.gradle.kts index 2310db2..236042d 100644 --- a/app/androidApp/build.gradle.kts +++ b/app/androidApp/build.gradle.kts @@ -146,4 +146,5 @@ dependencies { implementation(libs.zxing.embedded) implementation(libs.markdown.renderer) implementation(libs.highlights) + implementation(libs.androidx.exifinterface) } diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt index af14e79..b83f787 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt @@ -636,6 +636,16 @@ fun setSessionModel(settings: ServerSettings, sessionId: String, model: String) ) {} } +/** + * The permission modes the Claude CLI accepts, in the order they give up asking. "manual" asks for + * everything (each ask arrives on the phone as a question card); the others are the CLI's own + * escalating levels of autonomy. + * + * One list for every screen that offers them -- spawn, import, and the session's own picker -- + * because three copies had already drifted: the import screen was missing "plan". + */ +val PERMISSION_MODES = listOf("manual", "acceptEdits", "auto", "bypassPermissions", "plan") + /** Switches how much a running session asks before acting, also in place. */ fun setSessionPermissionMode(settings: ServerSettings, sessionId: String, mode: String) { requestFromServer( diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt index ad70d83..556aad5 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt @@ -4,8 +4,8 @@ import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory import android.graphics.Matrix -import android.media.ExifInterface import android.net.Uri +import androidx.exifinterface.media.ExifInterface import java.io.ByteArrayOutputStream import kotlin.math.max diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/EventStream.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/EventStream.kt index 006149f..20749d3 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/EventStream.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/EventStream.kt @@ -4,6 +4,12 @@ import java.io.IOException import java.net.HttpURLConnection import java.net.URL +/** + * The frame name the server uses to say a cursor was too far behind to continue from. Must match + * `send_backlog` in the backend's routes.rs. + */ +private const val RESET_EVENT = "reset" + /** * The SSE half of the API: one long-lived GET per open session screen, replaying the transcript * after a cursor and then following it live. @@ -13,12 +19,6 @@ import java.net.URL * of throwing, so a deliberate close doesn't surface as a connection error. The caller owns * reconnecting (with the last seq it saw as the new cursor) -- see SessionScreen. */ -/** - * The frame name the server uses to say a cursor was too far behind to continue from. Must match - * `send_backlog` in the backend's routes.rs. - */ -private const val RESET_EVENT = "reset" - class EventStream(private val settings: ServerSettings, private val sessionId: String) { @Volatile private var connection: HttpURLConnection? = null @Volatile private var closed = false diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/ImportScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/ImportScreen.kt index 839c4bd..b310f1e 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/ImportScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/ImportScreen.kt @@ -26,6 +26,7 @@ import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch @@ -128,7 +129,7 @@ fun ImportScreen(settings: ServerSettings, reloadToken: Int, onImported: (Sessio } else { ChipGroup( label = "Permissions", - options = listOf("manual", "acceptEdits", "auto", "bypassPermissions"), + options = PERMISSION_MODES, selected = permissionMode, onSelect = { permissionMode = it }, ) @@ -199,7 +200,17 @@ fun ImportScreen(settings: ServerSettings, reloadToken: Int, onImported: (Sessio withContext(Dispatchers.IO) { deleteImportable(settings, setup.id, session.id) } - loadSessions(setup) + // Only this row, and only what changed -- the same rule as the + // session list's delete. Refetching instead put every other row + // back through a loading spinner to report a change that was + // never in doubt. + val loaded = sessions + if (loaded is LoadState.Loaded) { + sessions = + LoadState.Loaded( + loaded.value.filterNot { it.id == session.id } + ) + } } catch (err: Exception) { failure = err.message ?: "Couldn't delete that session" } @@ -273,16 +284,17 @@ private fun ImportableList( // it is one long value with no natural break, where the // lines below it are short enough to wrap readably. // Cut at the head, because a path is identified by its - // tail and these all share a long prefix. + // tail and these all share a long prefix. By the row's + // real width rather than a character count, which was + // one guess for every font size and screen. session.cwd .takeIf { it.isNotEmpty() } ?.let { cwd -> Text( - if (cwd.length > PATH_CHARS) - "…" + cwd.takeLast(PATH_CHARS) - else cwd, + cwd, style = MaterialTheme.typography.bodySmall, maxLines = 1, + overflow = TextOverflow.StartEllipsis, color = MaterialTheme.colorScheme.onSurfaceVariant, ) @@ -333,9 +345,6 @@ private fun humanSize(bytes: Long): String? = else -> "$bytes B" } -/** How much of a path a row shows before cutting its front off. */ -private const val PATH_CHARS = 40 - /** What this session is: the measurements, in the order they are worth knowing. */ private fun statsOf(session: Importable, importing: String?): String = listOfNotNull( diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/MemoryNote.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/MemoryNote.kt index a3edd1c..be47a74 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/MemoryNote.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/MemoryNote.kt @@ -54,26 +54,19 @@ fun AssistantMessage( * * A reply carrying no notes is drawn from the message as it arrived rather than from the trimmed * prose part made while looking for them -- inspecting a message must not change it. That belongs - * here rather than at the two places that need the answer, because [markdownIn] has to name the - * same strings this draws: a string warmed under a key no row ever looks up is a miss that nothing - * reports, and the row pays the parse in the frame it appears, which is the cost being removed. + * here rather than at the places that need the answer, because [warm] has to name the same strings + * the rows draw: a string warmed under a key no row ever looks up is a miss that nothing reports, + * and the row pays the parse in the frame it appears, which is the cost being removed. * * Public because [transcriptUnits] flattens settled replies into the same parts; go through - * [ParsedReplies.partsOf] on any path that runs per fold, so the scan happens once per message. + * [ParsedReplies.partsOf] on any path that runs per fold or per page, so the scan happens once per + * message. */ fun messageParts(text: String): List { val parts = splitMemoryNotes(text) return if (parts.singleOrNull() is MessagePart.Prose) listOf(MessagePart.Prose(text)) else parts } -/** - * Every string a reply will be drawn from, for [ParsedReplies.warm] to make ready. - * - * A string warmed under a key no row ever looks up is a miss that nothing reports, so this has to - * name what the rows actually draw rather than what the message contains. - */ -fun markdownIn(text: String): List = messageParts(text).map { it.text } - @Composable fun MemoryNote(note: MessagePart.Remembered, replies: ParsedReplies) { Card(Modifier.fillMaxWidth()) { diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Notifications.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Notifications.kt index 656ffa6..9970668 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Notifications.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Notifications.kt @@ -151,9 +151,15 @@ class NotificationService : Service() { // refused, and notifications switched off for the app in Android's own settings. Neither // is reported anywhere -- the person said no, and saying it back to them through the // channel they closed is not available anyway. + // + // The permission only exists from Android 13. Asking an older version about it gets + // "denied" for a name it does not know, which read as the person having said no -- so + // every notification on Android 12 and below was silently dropped. Before 13 the + // switch in Android's own settings, checked below, is the whole of the answer. val allowed = - ContextCompat.checkSelfPermission(this, Manifest.permission.POST_NOTIFICATIONS) == - PackageManager.PERMISSION_GRANTED + Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU || + ContextCompat.checkSelfPermission(this, Manifest.permission.POST_NOTIFICATIONS) == + PackageManager.PERMISSION_GRANTED if (!allowed || !manager.areNotificationsEnabled()) { return } diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt index fce2c8f..efebf56 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt @@ -1,6 +1,5 @@ package com.example.aiapp -import android.graphics.BitmapFactory import androidx.compose.foundation.Image import androidx.compose.foundation.background import androidx.compose.foundation.border @@ -17,23 +16,14 @@ import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable -import androidx.compose.runtime.LaunchedEffect -import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf -import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip -import androidx.compose.ui.graphics.ImageBitmap -import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.semantics.contentDescription import androidx.compose.ui.semantics.semantics import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext /** * What is about to be sent, directly above the box it will be sent from. @@ -78,17 +68,7 @@ private fun PendingThumbnail( ref: String, onRemove: () -> Unit, ) { - var bitmap by remember(ref) { mutableStateOf(null) } - var failed by remember(ref) { mutableStateOf(false) } - LaunchedEffect(ref) { - try { - val bytes = withContext(Dispatchers.IO) { fetchSessionFile(settings, sessionId, ref) } - bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)?.asImageBitmap() - failed = bitmap == null - } catch (_: ApiException) { - failed = true - } - } + val (bitmap, failed) = rememberSessionBitmap(settings, sessionId, ref) val shape = RoundedCornerShape(8.dp) Box( Modifier.size(THUMBNAIL) diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/ResetCountdown.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/ResetCountdown.kt index aea19cf..b6252e6 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/ResetCountdown.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/ResetCountdown.kt @@ -3,13 +3,9 @@ package com.example.aiapp import java.time.Duration import java.time.OffsetDateTime -/** - * How long is left in a usage window. - * - * Shared by the session bar and the usage screen: the arithmetic is the same in both and only the - * sentence around it differs, so this returns the span on its own and leaves the wording to the - * caller. - */ +// How long is left in a usage window. Shared by the session bar and the usage screen: the +// arithmetic is the same in both and only the sentence around it differs, so everything here +// returns the span or the state on its own and leaves the wording to the caller. /** "1d 4h", "3h 12m", "12m" -- the span alone, with no leading or trailing words. */ fun formatSpan(until: Duration): String = diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionImage.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionImage.kt index b81caee..e44b2d5 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionImage.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionImage.kt @@ -36,6 +36,38 @@ import androidx.compose.ui.window.DialogProperties import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +/** + * One image from the session's files route: the bitmap once it arrives, and whether it never will. + * + * [failed] exists because the two empty states differ in kind -- still coming and never coming -- + * and a reader can act on the second; each caller supplies its own words for them. + */ +data class SessionBitmap(val bitmap: ImageBitmap?, val failed: Boolean) + +/** + * Fetches (authenticated, pinned) and decodes one transcript image, remembered per ref so scrolling + * does not refetch. + * + * Shared by the transcript's images and the composer's pending attachments, because the fetch, the + * decode and the two-state answer are one block of logic that had been written twice. + */ +@Composable +fun rememberSessionBitmap(settings: ServerSettings, sessionId: String, ref: String): SessionBitmap { + var state by remember(ref) { mutableStateOf(SessionBitmap(null, failed = false)) } + LaunchedEffect(ref) { + state = + try { + val bytes = + withContext(Dispatchers.IO) { fetchSessionFile(settings, sessionId, ref) } + val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)?.asImageBitmap() + SessionBitmap(decoded, failed = decoded == null) + } catch (_: ApiException) { + SessionBitmap(null, failed = true) + } + } + return state +} + /** * An image in the transcript: a fixed-height thumbnail that opens full screen. * @@ -50,18 +82,8 @@ import kotlinx.coroutines.withContext */ @Composable fun SessionImage(settings: ServerSettings, sessionId: String, ref: String) { - var bitmap by remember(ref) { mutableStateOf(null) } - var failed by remember(ref) { mutableStateOf(false) } + val (bitmap, failed) = rememberSessionBitmap(settings, sessionId, ref) var full by remember(ref) { mutableStateOf(false) } - LaunchedEffect(ref) { - try { - val bytes = withContext(Dispatchers.IO) { fetchSessionFile(settings, sessionId, ref) } - bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)?.asImageBitmap() - failed = bitmap == null - } catch (_: ApiException) { - failed = true - } - } val height = thumbnailHeight() val heightPx = with(LocalDensity.current) { height.roundToPx() } Box(Modifier.fillMaxWidth().height(height), contentAlignment = Alignment.CenterStart) { diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionListScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionListScreen.kt index 87c8c72..4a230f3 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionListScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionListScreen.kt @@ -35,9 +35,6 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext -// Status colors, keyed by the wire strings in Events.kt. Light theme only, -// as in dev-updater. - /** * The sessions tab: sessions awaiting an answer sort to the top, which is the "your turn" inbox. * diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt index 48755b5..e4cc8fa 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt @@ -7,7 +7,6 @@ import android.widget.Toast import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.PickVisualMediaRequest import androidx.activity.result.contract.ActivityResultContracts -import androidx.compose.foundation.Image import androidx.compose.foundation.background import androidx.compose.foundation.gestures.awaitEachGesture import androidx.compose.foundation.gestures.awaitFirstDown @@ -43,7 +42,6 @@ import androidx.compose.material3.Text import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect -import androidx.compose.runtime.Immutable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue @@ -210,430 +208,9 @@ private fun Modifier.holdTopEdge(key: Any, held: TopEdgeHold, hold: (Int) -> Uni } } -/** - * What the transcript renders: the event stream folded into displayable rows (see [foldEvent]). The - * stream is the only data source -- opening this screen replays from seq 0, and a reconnect resumes - * from the last seq seen, so there is no separate history fetch to drift from it. - */ -@Immutable -sealed class TranscriptItem { - /** - * The transcript sequence number this row started at, and its identity on screen. - * - * The list is drawn newest-first, so every new message is an insertion at index 0 and every - * page of history is an insertion at the far end. Without an identity that survives both, the - * list is addressed by position: whatever somebody had scrolled to keeps its index while the - * content underneath it slides, which reads as the view scrolling on its own. - * - * A seq is the right identity because it is what the transcript itself is ordered by, it never - * changes, and it is already carried by every event. A row built from several events -- a - * streaming message, a tool call and its result -- keeps the seq of the first, so it holds - * still while the rest of it arrives. - */ - abstract val seq: Long - - data class UserMsg( - override val seq: Long, - val text: String, - /** Refs of what was attached, drawn inside the bubble. */ - val images: List = emptyList(), - ) : TranscriptItem() - - data class AssistantMsg(override val seq: Long, val text: String) : TranscriptItem() - - data class ToolRun( - override val seq: Long, - val id: String, - /** - * The run of adjacent calls this one belongs to, named once when the call is folded in and - * never recomputed. - * - * Carried rather than derived because a run can gain members at *either* end -- a new call - * arriving beside it, or a page of history arriving in front of it -- so no function of its - * current members is stable. It is the first call's id at the moment the run started, which - * is a name rather than a description: [joinPages] hands it to older calls that turn out to - * belong to the same run, instead of renaming the run they joined. - */ - val runId: String, - val tool: String, - val input: String, - val output: String, - val done: Boolean, - /** - * The questions this call is waiting on, in the order they were asked. - * - * On the call's own row rather than beside it: an ask used to arrive as a second card - * repeating the input verbatim, so the reader saw the same command twice and had to work - * out that it was one event. The backend says which call a question is about, so this is a - * fact rather than a match on the input. - * - * A list because AskUserQuestion asks up to four at once, and they are one decision to make - * -- a permission is the case of exactly one, not a different shape. - */ - val asks: List = emptyList(), - /** - * Images this call's result carried, drawn under it. - * - * Beside it they had to be paired by position, and position is the thing a page boundary - * breaks -- a screenshot loaded on one page and its call on the next read as unrelated. - */ - val images: List = emptyList(), - ) : TranscriptItem() - - data class QuestionCard( - override val seq: Long, - val id: String, - val prompt: String, - /** A few words naming what this is about, when the asker offered one. */ - val header: String?, - val options: List, - /** Whether several options may be chosen at once. */ - val multiSelect: Boolean, - /** What was chosen, once something was; empty until then. */ - val answers: List, - ) : TranscriptItem() - - data class ErrorMsg(override val seq: Long, val message: String) : TranscriptItem() - - /** An image by server-side ref, fetched from the session's files route. */ - data class ImageItem(override val seq: Long, val ref: String) : TranscriptItem() - - /** - * A message another agent sent this session. - * - * Its own row rather than a [UserMsg]: see [PeerMessageRow] for why the voice matters. - */ - data class PeerNote(override val seq: Long, val from: String, val text: String) : - TranscriptItem() - - /** - * A command the session ran on itself -- `/compact`, `/rename`. - * - * Kept in the transcript rather than only shown while it waits, because it explains what - * follows: a conversation that suddenly has half the context, or a session with a new name. - */ - data class CommandRow(override val seq: Long, val text: String) : TranscriptItem() - - /** Placeholder row for events this build can't render (newer kinds). */ - data class Note(override val seq: Long, val text: String) : TranscriptItem() - - /** - * A clear that happened: everything above it left the session's context and stayed on screen. - * - * Carries only its position, because that is all it means. - */ - data class ClearedNote(override val seq: Long) : TranscriptItem() - - /** - * A compaction that happened, and what it recovered. - * - * In the transcript rather than only in the status line, because the status is gone the moment - * it finishes and this is the part worth keeping: it is the explanation for a gap in the - * conversation, and for a minute or two in which the session was busy with nothing to show. - * - * The wire also says what triggered it, and this deliberately does not carry that: the row says - * the two sizes and nothing else (see [compactionSummary]), so keeping the trigger here would - * be a field nothing can read. - */ - data class CompactedNote( - override val seq: Long, - val preTokens: Long?, - val postTokens: Long?, - ) : TranscriptItem() -} - -/** - * The run a call joins: the one it lands next to, or a new one named after itself. - * - * Only ever consulted when the call is first folded in. That is what makes the name stable -- a run - * keeps whatever it was called when it started, however many calls arrive at either end of it - * afterwards. - * - * A question to the reader is in a run of its own, which is what puts it on the transcript as a row - * rather than inside a collapsed "Called 6 tools" card. Two things follow from being alone: it is - * always visible, since a run of one is drawn as itself rather than as a group; and the calls - * around it fall into a group before it and a group after it, so where the reader was asked - * something is legible in the shape of the transcript without opening anything. It ends the run - * before it as well as starting a fresh one after -- the moment somebody was asked is a boundary in - * the work, not a gap in the middle of one run. - */ -private fun runIdFor(items: List, id: String, tool: String): String { - val previous = items.lastOrNull() as? TranscriptItem.ToolRun ?: return id - if (tool == ASK_USER_QUESTION || previous.tool == ASK_USER_QUESTION) return id - return previous.runId -} - -/** - * Puts a page of older items in front of the ones already loaded, healing whatever the page - * boundary cut in two. - * - * Two things straddle a boundary: a tool call separated from its result, and a message separated - * from the rest of itself. Both were one thing before the transcript was cut into pages, and both - * have to be one thing again -- a reply drawn as two messages is the same defect as a call drawn - * twice, arriving from the same cause. - * - * A boundary lands wherever it lands, and roughly half the time that is between a call and its - * result. The newer page then holds a `ToolEnd` whose start it never saw, which [foldEvent] draws - * as a row of its own -- correctly, because a call that renders as nothing is indistinguishable - * from one that never happened. When the older page arrives it brings the real `ToolStart`, and - * concatenating the two lists left *both*: the same call twice, once as a proper card and once as a - * nameless placeholder. Visible as a run of four calls reporting "Called 5 tools", and worse than - * the miscount -- the extra row is at the join, so it also moves everything the reader was looking - * at. - * - * Merged by the call's own id rather than by position, because position is exactly what a page - * boundary destroys. The older row wins on what a start knows (the tool's name, its input) and the - * newer on what an end knows (the output, and whether it finished), which is the only way round - * that loses nothing. - */ -fun joinPages(earlier: List, later: List): List { - val (older, newer) = healSplitMessage(earlier, later) - val startedEarlier = - older.filterIsInstance().mapTo(mutableSetOf()) { it.id } - if (startedEarlier.isEmpty()) return older + newer - val endedLater = - newer - .filterIsInstance() - .associateBy { it.id } - .filterKeys { it in startedEarlier } - if (endedLater.isEmpty()) return older + newer - val healed = older.map { row -> - val half = (row as? TranscriptItem.ToolRun)?.let { endedLater[it.id] } - if (row is TranscriptItem.ToolRun && half != null) { - row.copy( - output = half.output, - done = half.done, - // Kept from both halves: a question or an image can be attached to either, - // depending on which side of the boundary its event fell. - asks = row.asks + half.asks, - images = row.images + half.images, - ) - } else { - row - } - } - val kept = newer.filterNot { it is TranscriptItem.ToolRun && it.id in endedLater } - return adoptRun(healed, kept) + kept -} - -/** - * Rejoins a message the page boundary cut, and hands back the two pages to concatenate. - * - * [foldEvent] never leaves two assistant messages next to each other inside one page -- deltas - * accumulate into the message before them -- so two meeting at a join are always the two halves of - * one reply, and leaving them apart drew a single answer as two, with a paragraph break through the - * middle of a sentence. - * - * The newer half keeps its identity, for the reason [adoptRun] gives: it is the row already on - * screen, and renaming that is how the list loses its anchor. It grows by what the older half - * brings, which is safe here and nowhere else -- the join is at the oldest end of what is loaded, - * so the growth extends off the top of the screen, away from the row the list anchors to. - */ -private fun healSplitMessage( - earlier: List, - later: List, -): Pair, List> { - val head = earlier.lastOrNull() - val tail = later.firstOrNull() - if (head !is TranscriptItem.AssistantMsg || tail !is TranscriptItem.AssistantMsg) { - return earlier to later - } - return earlier.dropLast(1) to (listOf(tail.copy(text = head.text + tail.text)) + later.drop(1)) -} - -/** - * Hands the older calls at the join the name of the run they are joining. - * - * The two pages were folded separately, so a run split by the boundary came back as two runs with - * two names. Naming the joined run after the *older* half would be the obvious way round and is the - * wrong one: the newer half is the part already on screen, and renaming it is renaming the row the - * reader is looking at, which is how a list loses its anchor and steps under them. So the arriving - * calls take the name of the ones already there, and nothing visible changes identity. - */ -private fun adoptRun( - earlier: List, - later: List, -): List { - val first = later.firstOrNull() as? TranscriptItem.ToolRun ?: return earlier - // A question is in a run of its own on both sides of the join, the same as it would be had - // the two pages been folded as one -- see `runIdFor`. Without this the heal would merge a - // group straight through the row the reader was asked something on. - if (first.tool == ASK_USER_QUESTION) return earlier - val joining = first.runId - val tail = earlier.takeLastWhile { - it is TranscriptItem.ToolRun && it.tool != ASK_USER_QUESTION - } - if (tail.isEmpty()) return earlier - return earlier.dropLast(tail.size) + - tail.map { (it as TranscriptItem.ToolRun).copy(runId = joining) } -} - -fun foldEvent(items: List, entry: SeqEvent): List = - when (val event = entry.event) { - is SessionEvent.UserMessage -> - items + TranscriptItem.UserMsg(entry.seq, event.text, event.images) - is SessionEvent.AssistantText -> { - // Deltas accumulate into the message they're streaming, which keeps the seq of the - // first of them: a row whose identity changed with every delta would be a new row on - // every frame, and the list would jump for the whole of a streamed answer. - val last = items.lastOrNull() - if (last is TranscriptItem.AssistantMsg) { - items.dropLast(1) + last.copy(text = last.text + event.delta) - } else { - items + TranscriptItem.AssistantMsg(entry.seq, event.delta) - } - } - is SessionEvent.ToolStart -> - items + - TranscriptItem.ToolRun( - entry.seq, - event.id, - runIdFor(items, event.id, event.tool), - event.tool, - event.input, - "", - done = false, - ) - is SessionEvent.ToolUpdate -> updateTool(items, event.id) { it.copy(output = event.output) } - is SessionEvent.ToolEnd -> - // Created when its start is not here, rather than dropped. A - // fold that only ever *updates* loses the whole call when the - // start fell outside the loaded window, and a tool call that - // renders as nothing is indistinguishable from one that never - // happened. The name is unknown from an end alone; loading the - // page before this one replaces the row with the real thing. - if (items.any { it is TranscriptItem.ToolRun && it.id == event.id }) { - updateTool(items, event.id) { it.copy(output = event.output, done = true) } - } else { - items + - TranscriptItem.ToolRun( - entry.seq, - event.id, - // The name is not known from an end alone, so a call that was an ask - // cannot be recognised as one here; loading the page before this - // replaces the row with the real thing, which is when it splits out. - runIdFor(items, event.id, "tool"), - "tool", - "", - event.output, - done = true, - ) - } - is SessionEvent.Question -> { - val card = - TranscriptItem.QuestionCard( - entry.seq, - event.id, - event.prompt, - event.header, - event.options, - event.multiSelect, - emptyList(), - ) - // A question with no tool behind it -- AskUserQuestion, or an ask - // whose call fell outside the loaded window -- is a card of its - // own, which is what every question was before this. - if ( - event.about != null && - items.any { it is TranscriptItem.ToolRun && it.id == event.about } - ) { - updateTool(items, event.about) { it.copy(asks = it.asks + card) } - } else { - items + card - } - } - is SessionEvent.Answered -> - // Resolved wherever it is drawn: a card of its own, or a tool - // row's ask. Missing the second left an Allow/Deny pair live on - // a question already answered from another device. - items.map { - when { - it is TranscriptItem.QuestionCard && it.id == event.id -> - it.copy(answers = event.answers) - it is TranscriptItem.ToolRun && it.asks.any { ask -> ask.id == event.id } -> - it.copy( - asks = - it.asks.map { ask -> - if (ask.id == event.id) ask.copy(answers = event.answers) - else ask - } - ) - else -> it - } - } - is SessionEvent.PeerMessage -> - items + TranscriptItem.PeerNote(entry.seq, event.from, event.text) - is SessionEvent.CommandSent -> items + TranscriptItem.CommandRow(entry.seq, event.text) - // Screen-level state, not transcript rows -- see SessionScreen. - is SessionEvent.CommandQueued -> items - // No row of its own: a message that is still waiting is drawn as a pending bubble below - // the transcript, and becomes an ordinary one where the session read it. - is SessionEvent.MessageQueued -> items - is SessionEvent.Settings -> items - is SessionEvent.Status -> items - is SessionEvent.Error -> items + TranscriptItem.ErrorMsg(entry.seq, event.message) - is SessionEvent.Image -> - // Under the call that produced it when there is one, and a row of - // its own when there is not -- a person's own attachment belongs - // to no call, and neither does one whose call fell outside the - // loaded window. - if ( - event.about != null && - items.any { it is TranscriptItem.ToolRun && it.id == event.about } - ) { - updateTool(items, event.about) { it.copy(images = it.images + event.ref) } - } else { - items + TranscriptItem.ImageItem(entry.seq, event.ref) - } - is SessionEvent.Cleared -> items + TranscriptItem.ClearedNote(entry.seq) - is SessionEvent.Compacted -> - items + TranscriptItem.CompactedNote(entry.seq, event.preTokens, event.postTokens) - is SessionEvent.Unknown -> items + TranscriptItem.Note(entry.seq, "[${event.type}]") - // Screen-level state, not transcript rows -- see SessionScreen. - is SessionEvent.UsageDelta -> items - } - -private fun updateTool( - items: List, - id: String, - change: (TranscriptItem.ToolRun) -> TranscriptItem.ToolRun, -): List = items.map { - if (it is TranscriptItem.ToolRun && it.id == id) change(it) else it -} - -/** - * Where markdown is parsed ahead of being drawn: two threads, never all of them. - * - * The default dispatcher sizes itself to the machine, which is right for work somebody is waiting - * on and wrong for work nobody is. A page of history is hundreds of parses arriving at once, and - * taking every core for them leaves the thread that draws the frame queueing behind one -- measured - * on a Pixel 9 Pro XL as 21ms of `waited` at the 90th percentile, which is the frame failing to - * *start* rather than taking too long once it had. - */ -@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class) -private val parsingThreads = Dispatchers.Default.limitedParallelism(2) - -/** - * Parses the replies among [rows], off whatever thread is drawing. - * - * Called where a page of transcript is folded rather than where a row is composed, which is the - * whole point: the work happens seconds before the reader reaches the rows it was done for. See - * [ParsedReplies]. - */ -private suspend fun warm(replies: ParsedReplies, rows: List) { - // Including the search for what to parse, which is not the cheap half it looks like: - // [markdownIn] splits every assistant message looking for memory notes, and this is handed - // the *whole* loaded transcript on every page, so the scan grows with the conversation while - // the work it finds stays one page's worth. Off the calling thread it is nobody's frame. - withContext(parsingThreads) { - val texts = - rows - .filterIsInstance() - .flatMap { markdownIn(it.text) } - .flatMap { replies.blocksOf(it) } - if (texts.isNotEmpty()) replies.warm(texts) - } -} +// The transcript's data model -- TranscriptItem, foldEvent, joinPages, warm -- lives in +// TranscriptItems.kt: it is pure event folding with no screen in it, and the two halves changed +// for unrelated reasons while they shared this file. @Composable fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () -> Unit) { @@ -1308,14 +885,13 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () } } - fun act(onFailure: () -> Unit = {}, onDone: () -> Unit = {}, action: () -> Unit) { + fun act(onDone: () -> Unit = {}, action: () -> Unit) { scope.launch { try { withContext(Dispatchers.IO) { action() } actionError = null } catch (e: ApiException) { actionError = e.message - onFailure() } finally { // Whatever happened, including the failure above: a caller that re-enables a // control here must get it back on the path where the request was refused too, @@ -2094,8 +1670,10 @@ private fun ProcessAction.perform(settings: ServerSettings, sessionId: String) = } /** - * An inline transcript image, fetched (authenticated, pinned) from the session's files route. The - * bitmap is remembered per ref, so scrolling doesn't refetch. + * A message the person holding the phone sent, in a bubble at their end of the conversation. + * + * [pending] is one the server has taken and the session has not read yet -- drawn quieter, because + * "said" and "heard" are different claims and the transcript must not merge them. */ @Composable private fun UserBubble( @@ -2145,11 +1723,6 @@ private fun UserBubble( /** A message the server has accepted and the session has not read yet. */ private data class QueuedMessage(val id: String, val text: String, val images: List) -/** - * Collapsed by default: name plus a spinner while running, expandable to the input and output. The - * spinner-while-unfinished is exactly "ToolStart with no matching ToolEnd yet". - */ - /** * Asked before switching model, because switching is not free and the cost is invisible. * @@ -2330,9 +1903,6 @@ private fun QuestionRow( */ private const val ONE_TAP_MS = 250L -/** The modes the CLI accepts, in the order they give up asking. */ -private val PERMISSION_MODES = listOf("manual", "acceptEdits", "auto", "bypassPermissions", "plan") - /** * A control that reads as its own value. * diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SpawnScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SpawnScreen.kt index f64f4e6..699c81f 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SpawnScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SpawnScreen.kt @@ -33,13 +33,6 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch import kotlinx.coroutines.withContext -// Claude Code 2.x permission modes. "manual" asks for everything (each ask -// arrives on the phone as a question card); the others are the CLI's own -// escalating levels of autonomy. -private val PERMISSION_MODES = listOf("manual", "acceptEdits", "auto", "bypassPermissions", "plan") - -/** Runs on the backend machine itself -- the "no host" case. */ - /** * The spawn screen: what to run, where to run it, and the per-kind fields. * diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/ToolRows.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/ToolRows.kt index 69fb472..55b6f57 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/ToolRows.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/ToolRows.kt @@ -33,13 +33,12 @@ import androidx.compose.ui.unit.Dp import androidx.compose.ui.unit.dp /** - * A run of consecutive tool calls, or anything else, in the order they will be drawn. + * One row as the transcript draws it: a run of consecutive tool calls, or anything else. * * Grouping is decided here rather than when events are folded, because it is a display decision: * the transcript's own order is what paging and the event stream depend on, and one screen's idea * of "these belong together" must not reach back into it. - */ -/** + * * Immutable, and said so, because Compose cannot tell. * * A row is a value: it is rebuilt from the transcript rather than edited, and two rows describing diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptItems.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptItems.kt new file mode 100644 index 0000000..54b5b2d --- /dev/null +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/TranscriptItems.kt @@ -0,0 +1,439 @@ +package com.example.aiapp + +import androidx.compose.runtime.Immutable +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** + * What the transcript renders: the event stream folded into displayable rows (see [foldEvent]). The + * stream is the only data source -- opening a session screen replays from seq 0, and a reconnect + * resumes from the last seq seen, so there is no separate history fetch to drift from it. + */ +@Immutable +sealed class TranscriptItem { + /** + * The transcript sequence number this row started at, and its identity on screen. + * + * The list is drawn newest-first, so every new message is an insertion at index 0 and every + * page of history is an insertion at the far end. Without an identity that survives both, the + * list is addressed by position: whatever somebody had scrolled to keeps its index while the + * content underneath it slides, which reads as the view scrolling on its own. + * + * A seq is the right identity because it is what the transcript itself is ordered by, it never + * changes, and it is already carried by every event. A row built from several events -- a + * streaming message, a tool call and its result -- keeps the seq of the first, so it holds + * still while the rest of it arrives. + */ + abstract val seq: Long + + data class UserMsg( + override val seq: Long, + val text: String, + /** Refs of what was attached, drawn inside the bubble. */ + val images: List = emptyList(), + ) : TranscriptItem() + + data class AssistantMsg(override val seq: Long, val text: String) : TranscriptItem() + + data class ToolRun( + override val seq: Long, + val id: String, + /** + * The run of adjacent calls this one belongs to, named once when the call is folded in and + * never recomputed. + * + * Carried rather than derived because a run can gain members at *either* end -- a new call + * arriving beside it, or a page of history arriving in front of it -- so no function of its + * current members is stable. It is the first call's id at the moment the run started, which + * is a name rather than a description: [joinPages] hands it to older calls that turn out to + * belong to the same run, instead of renaming the run they joined. + */ + val runId: String, + val tool: String, + val input: String, + val output: String, + val done: Boolean, + /** + * The questions this call is waiting on, in the order they were asked. + * + * On the call's own row rather than beside it: an ask used to arrive as a second card + * repeating the input verbatim, so the reader saw the same command twice and had to work + * out that it was one event. The backend says which call a question is about, so this is a + * fact rather than a match on the input. + * + * A list because AskUserQuestion asks up to four at once, and they are one decision to make + * -- a permission is the case of exactly one, not a different shape. + */ + val asks: List = emptyList(), + /** + * Images this call's result carried, drawn under it. + * + * Beside it they had to be paired by position, and position is the thing a page boundary + * breaks -- a screenshot loaded on one page and its call on the next read as unrelated. + */ + val images: List = emptyList(), + ) : TranscriptItem() + + data class QuestionCard( + override val seq: Long, + val id: String, + val prompt: String, + /** A few words naming what this is about, when the asker offered one. */ + val header: String?, + val options: List, + /** Whether several options may be chosen at once. */ + val multiSelect: Boolean, + /** What was chosen, once something was; empty until then. */ + val answers: List, + ) : TranscriptItem() + + data class ErrorMsg(override val seq: Long, val message: String) : TranscriptItem() + + /** An image by server-side ref, fetched from the session's files route. */ + data class ImageItem(override val seq: Long, val ref: String) : TranscriptItem() + + /** + * A message another agent sent this session. + * + * Its own row rather than a [UserMsg]: see [PeerMessageRow] for why the voice matters. + */ + data class PeerNote(override val seq: Long, val from: String, val text: String) : + TranscriptItem() + + /** + * A command the session ran on itself -- `/compact`, `/rename`. + * + * Kept in the transcript rather than only shown while it waits, because it explains what + * follows: a conversation that suddenly has half the context, or a session with a new name. + */ + data class CommandRow(override val seq: Long, val text: String) : TranscriptItem() + + /** Placeholder row for events this build can't render (newer kinds). */ + data class Note(override val seq: Long, val text: String) : TranscriptItem() + + /** + * A clear that happened: everything above it left the session's context and stayed on screen. + * + * Carries only its position, because that is all it means. + */ + data class ClearedNote(override val seq: Long) : TranscriptItem() + + /** + * A compaction that happened, and what it recovered. + * + * In the transcript rather than only in the status line, because the status is gone the moment + * it finishes and this is the part worth keeping: it is the explanation for a gap in the + * conversation, and for a minute or two in which the session was busy with nothing to show. + * + * The wire also says what triggered it, and this deliberately does not carry that: the row says + * the two sizes and nothing else (see [compactionSummary]), so keeping the trigger here would + * be a field nothing can read. + */ + data class CompactedNote( + override val seq: Long, + val preTokens: Long?, + val postTokens: Long?, + ) : TranscriptItem() +} + +/** + * The run a call joins: the one it lands next to, or a new one named after itself. + * + * Only ever consulted when the call is first folded in. That is what makes the name stable -- a run + * keeps whatever it was called when it started, however many calls arrive at either end of it + * afterwards. + * + * A question to the reader is in a run of its own, which is what puts it on the transcript as a row + * rather than inside a collapsed "Called 6 tools" card. Two things follow from being alone: it is + * always visible, since a run of one is drawn as itself rather than as a group; and the calls + * around it fall into a group before it and a group after it, so where the reader was asked + * something is legible in the shape of the transcript without opening anything. It ends the run + * before it as well as starting a fresh one after -- the moment somebody was asked is a boundary in + * the work, not a gap in the middle of one run. + */ +private fun runIdFor(items: List, id: String, tool: String): String { + val previous = items.lastOrNull() as? TranscriptItem.ToolRun ?: return id + if (tool == ASK_USER_QUESTION || previous.tool == ASK_USER_QUESTION) return id + return previous.runId +} + +/** + * Puts a page of older items in front of the ones already loaded, healing whatever the page + * boundary cut in two. + * + * Two things straddle a boundary: a tool call separated from its result, and a message separated + * from the rest of itself. Both were one thing before the transcript was cut into pages, and both + * have to be one thing again -- a reply drawn as two messages is the same defect as a call drawn + * twice, arriving from the same cause. + * + * A boundary lands wherever it lands, and roughly half the time that is between a call and its + * result. The newer page then holds a `ToolEnd` whose start it never saw, which [foldEvent] draws + * as a row of its own -- correctly, because a call that renders as nothing is indistinguishable + * from one that never happened. When the older page arrives it brings the real `ToolStart`, and + * concatenating the two lists left *both*: the same call twice, once as a proper card and once as a + * nameless placeholder. Visible as a run of four calls reporting "Called 5 tools", and worse than + * the miscount -- the extra row is at the join, so it also moves everything the reader was looking + * at. + * + * Merged by the call's own id rather than by position, because position is exactly what a page + * boundary destroys. The older row wins on what a start knows (the tool's name, its input) and the + * newer on what an end knows (the output, and whether it finished), which is the only way round + * that loses nothing. + */ +fun joinPages(earlier: List, later: List): List { + val (older, newer) = healSplitMessage(earlier, later) + val startedEarlier = + older.filterIsInstance().mapTo(mutableSetOf()) { it.id } + if (startedEarlier.isEmpty()) return older + newer + val endedLater = + newer + .filterIsInstance() + .associateBy { it.id } + .filterKeys { it in startedEarlier } + if (endedLater.isEmpty()) return older + newer + val healed = older.map { row -> + val half = (row as? TranscriptItem.ToolRun)?.let { endedLater[it.id] } + if (row is TranscriptItem.ToolRun && half != null) { + row.copy( + output = half.output, + done = half.done, + // Kept from both halves: a question or an image can be attached to either, + // depending on which side of the boundary its event fell. + asks = row.asks + half.asks, + images = row.images + half.images, + ) + } else { + row + } + } + val kept = newer.filterNot { it is TranscriptItem.ToolRun && it.id in endedLater } + return adoptRun(healed, kept) + kept +} + +/** + * Rejoins a message the page boundary cut, and hands back the two pages to concatenate. + * + * [foldEvent] never leaves two assistant messages next to each other inside one page -- deltas + * accumulate into the message before them -- so two meeting at a join are always the two halves of + * one reply, and leaving them apart drew a single answer as two, with a paragraph break through the + * middle of a sentence. + * + * The newer half keeps its identity, for the reason [adoptRun] gives: it is the row already on + * screen, and renaming that is how the list loses its anchor. It grows by what the older half + * brings, which is safe here and nowhere else -- the join is at the oldest end of what is loaded, + * so the growth extends off the top of the screen, away from the row the list anchors to. + */ +private fun healSplitMessage( + earlier: List, + later: List, +): Pair, List> { + val head = earlier.lastOrNull() + val tail = later.firstOrNull() + if (head !is TranscriptItem.AssistantMsg || tail !is TranscriptItem.AssistantMsg) { + return earlier to later + } + return earlier.dropLast(1) to (listOf(tail.copy(text = head.text + tail.text)) + later.drop(1)) +} + +/** + * Hands the older calls at the join the name of the run they are joining. + * + * The two pages were folded separately, so a run split by the boundary came back as two runs with + * two names. Naming the joined run after the *older* half would be the obvious way round and is the + * wrong one: the newer half is the part already on screen, and renaming it is renaming the row the + * reader is looking at, which is how a list loses its anchor and steps under them. So the arriving + * calls take the name of the ones already there, and nothing visible changes identity. + */ +private fun adoptRun( + earlier: List, + later: List, +): List { + val first = later.firstOrNull() as? TranscriptItem.ToolRun ?: return earlier + // A question is in a run of its own on both sides of the join, the same as it would be had + // the two pages been folded as one -- see `runIdFor`. Without this the heal would merge a + // group straight through the row the reader was asked something on. + if (first.tool == ASK_USER_QUESTION) return earlier + val joining = first.runId + val tail = earlier.takeLastWhile { + it is TranscriptItem.ToolRun && it.tool != ASK_USER_QUESTION + } + if (tail.isEmpty()) return earlier + return earlier.dropLast(tail.size) + + tail.map { (it as TranscriptItem.ToolRun).copy(runId = joining) } +} + +fun foldEvent(items: List, entry: SeqEvent): List = + when (val event = entry.event) { + is SessionEvent.UserMessage -> + items + TranscriptItem.UserMsg(entry.seq, event.text, event.images) + is SessionEvent.AssistantText -> { + // Deltas accumulate into the message they're streaming, which keeps the seq of the + // first of them: a row whose identity changed with every delta would be a new row on + // every frame, and the list would jump for the whole of a streamed answer. + val last = items.lastOrNull() + if (last is TranscriptItem.AssistantMsg) { + items.dropLast(1) + last.copy(text = last.text + event.delta) + } else { + items + TranscriptItem.AssistantMsg(entry.seq, event.delta) + } + } + is SessionEvent.ToolStart -> + items + + TranscriptItem.ToolRun( + entry.seq, + event.id, + runIdFor(items, event.id, event.tool), + event.tool, + event.input, + "", + done = false, + ) + is SessionEvent.ToolUpdate -> updateTool(items, event.id) { it.copy(output = event.output) } + is SessionEvent.ToolEnd -> + // Created when its start is not here, rather than dropped. A + // fold that only ever *updates* loses the whole call when the + // start fell outside the loaded window, and a tool call that + // renders as nothing is indistinguishable from one that never + // happened. The name is unknown from an end alone; loading the + // page before this one replaces the row with the real thing. + if (items.any { it is TranscriptItem.ToolRun && it.id == event.id }) { + updateTool(items, event.id) { it.copy(output = event.output, done = true) } + } else { + items + + TranscriptItem.ToolRun( + entry.seq, + event.id, + // The name is not known from an end alone, so a call that was an ask + // cannot be recognised as one here; loading the page before this + // replaces the row with the real thing, which is when it splits out. + runIdFor(items, event.id, "tool"), + "tool", + "", + event.output, + done = true, + ) + } + is SessionEvent.Question -> { + val card = + TranscriptItem.QuestionCard( + entry.seq, + event.id, + event.prompt, + event.header, + event.options, + event.multiSelect, + emptyList(), + ) + // A question with no tool behind it -- AskUserQuestion, or an ask + // whose call fell outside the loaded window -- is a card of its + // own, which is what every question was before this. + if ( + event.about != null && + items.any { it is TranscriptItem.ToolRun && it.id == event.about } + ) { + updateTool(items, event.about) { it.copy(asks = it.asks + card) } + } else { + items + card + } + } + is SessionEvent.Answered -> + // Resolved wherever it is drawn: a card of its own, or a tool + // row's ask. Missing the second left an Allow/Deny pair live on + // a question already answered from another device. + items.map { + when { + it is TranscriptItem.QuestionCard && it.id == event.id -> + it.copy(answers = event.answers) + it is TranscriptItem.ToolRun && it.asks.any { ask -> ask.id == event.id } -> + it.copy( + asks = + it.asks.map { ask -> + if (ask.id == event.id) ask.copy(answers = event.answers) + else ask + } + ) + else -> it + } + } + is SessionEvent.PeerMessage -> + items + TranscriptItem.PeerNote(entry.seq, event.from, event.text) + is SessionEvent.CommandSent -> items + TranscriptItem.CommandRow(entry.seq, event.text) + // Screen-level state, not transcript rows -- see SessionScreen. + is SessionEvent.CommandQueued -> items + // No row of its own: a message that is still waiting is drawn as a pending bubble below + // the transcript, and becomes an ordinary one where the session read it. + is SessionEvent.MessageQueued -> items + is SessionEvent.Settings -> items + is SessionEvent.Status -> items + is SessionEvent.Error -> items + TranscriptItem.ErrorMsg(entry.seq, event.message) + is SessionEvent.Image -> + // Under the call that produced it when there is one, and a row of + // its own when there is not -- a person's own attachment belongs + // to no call, and neither does one whose call fell outside the + // loaded window. + if ( + event.about != null && + items.any { it is TranscriptItem.ToolRun && it.id == event.about } + ) { + updateTool(items, event.about) { it.copy(images = it.images + event.ref) } + } else { + items + TranscriptItem.ImageItem(entry.seq, event.ref) + } + is SessionEvent.Cleared -> items + TranscriptItem.ClearedNote(entry.seq) + is SessionEvent.Compacted -> + items + TranscriptItem.CompactedNote(entry.seq, event.preTokens, event.postTokens) + is SessionEvent.Unknown -> items + TranscriptItem.Note(entry.seq, "[${event.type}]") + // Screen-level state, not transcript rows -- see SessionScreen. + is SessionEvent.UsageDelta -> items + } + +private fun updateTool( + items: List, + id: String, + change: (TranscriptItem.ToolRun) -> TranscriptItem.ToolRun, +): List = items.map { + if (it is TranscriptItem.ToolRun && it.id == id) change(it) else it +} + +/** + * Where markdown is parsed ahead of being drawn: two threads, never all of them. + * + * The default dispatcher sizes itself to the machine, which is right for work somebody is waiting + * on and wrong for work nobody is. A page of history is hundreds of parses arriving at once, and + * taking every core for them leaves the thread that draws the frame queueing behind one -- measured + * on a Pixel 9 Pro XL as 21ms of `waited` at the 90th percentile, which is the frame failing to + * *start* rather than taking too long once it had. + */ +@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class) +private val parsingThreads = Dispatchers.Default.limitedParallelism(2) + +/** + * Parses the replies among [rows], off whatever thread is drawing. + * + * Called where a page of transcript is folded rather than where a row is composed, which is the + * whole point: the work happens seconds before the reader reaches the rows it was done for. See + * [ParsedReplies]. + * + * What is warmed mirrors what the rows draw, unit by unit -- prose split into its blocks, a memory + * note whole -- because a string warmed under a key no row ever looks up is a miss that nothing + * reports; see [transcriptUnits], which is the flatten this has to agree with. It reads the same + * [ParsedReplies.partsOf] and [ParsedReplies.blocksOf] caches the flatten does, so a message is + * scanned once however many pages hand it back through here, while the whole loaded transcript + * crosses this on every page. + */ +suspend fun warm(replies: ParsedReplies, rows: List) { + withContext(parsingThreads) { + val texts = + rows + .filterIsInstance() + .flatMap { replies.partsOf(it.text) } + .flatMap { part -> + when (part) { + is MessagePart.Prose -> replies.blocksOf(part.text) + // Drawn as one MarkdownText, so its whole text is the key looked up. + is MessagePart.Remembered -> listOf(part.text) + } + } + if (texts.isNotEmpty()) replies.warm(texts) + } +} diff --git a/app/gradle/libs.versions.toml b/app/gradle/libs.versions.toml index caf8991..cbddc57 100644 --- a/app/gradle/libs.versions.toml +++ b/app/gradle/libs.versions.toml @@ -28,8 +28,13 @@ zxing-embedded = "4.3.0" markdown-renderer = "0.45.0" # Syntax highlighting for a tool call's input. Same reasoning as the markdown # renderer: a language's lexical rules are somebody else's specification. -# Latest stable, checked 2026-08-29 against Maven Central. -highlights = "1.0.0" +# Latest stable, checked 2026-08-31 against Maven Central. +highlights = "1.1.0" +# The support ExifInterface rather than android.media's, which lint warns off: +# the framework one is missing formats and the fixes for parsing hostile +# images, and images here arrive from outside the phone. Latest stable, +# checked 2026-08-31 against Google Maven. +androidx-exifinterface = "1.4.2" # Declared rather than inherited for the same reason as core-ktx: SessionScreen # now calls repeatOnLifecycle/LocalLifecycleOwner directly, to hold the event # stream open only while the screen is on screen. Latest stable, checked @@ -59,6 +64,7 @@ desugar-jdk-libs = { module = "com.android.tools:desugar_jdk_libs", version.ref # theme, so the app's Catppuccin scheme is what it draws with. markdown-renderer = { module = "com.mikepenz:multiplatform-markdown-renderer-m3", version.ref = "markdown-renderer" } highlights = { module = "dev.snipme:highlights", version.ref = "highlights" } +androidx-exifinterface = { module = "androidx.exifinterface:exifinterface", version.ref = "androidx-exifinterface" } # Declared directly rather than through the plugin's `compose.*` accessors, # which are deprecated as of CMP 1.11. compose-runtime = { module = "org.jetbrains.compose.runtime:runtime", version.ref = "compose-multiplatform" }