diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt index d353440..ea99a32 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt @@ -153,6 +153,15 @@ data class SessionSummary( * Every token this session has spent, as the server counts it -- see `SessionEvent.UsageDelta`. */ val totalTokens: Long, + /** + * The longest edge an image should have when it reaches this session, or null where the + * provider has no limit. + * + * Null and "a big number" are different answers, and only the first stays true: a provider that + * does not care about size should not be given a threshold this app invented. Decided by the + * server because that is where a provider's kind is known -- see `uploadPickedImage`. + */ + val maxImageEdge: Int?, val status: String, val lastActivity: Double, ) @@ -170,6 +179,7 @@ private fun parseSession(session: JSONObject) = imported = session.optBoolean("imported", false), notify = session.optBoolean("notify", true), totalTokens = session.optLong("totalTokens", 0), + maxImageEdge = session.optInt("maxImageEdge", 0).takeIf { it > 0 }, status = session.getString("status"), lastActivity = session.getDouble("lastActivity"), ) diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt new file mode 100644 index 0000000..ad70d83 --- /dev/null +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt @@ -0,0 +1,104 @@ +package com.example.aiapp + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import android.media.ExifInterface +import android.net.Uri +import java.io.ByteArrayOutputStream +import kotlin.math.max + +/** + * Getting a picked photo to a session, at a size the session can actually take. + * + * A phone camera produces twelve megapixels and several megabytes. The Claude API resizes anything + * larger than 1568px on its long edge before looking at it and refuses images past a much higher + * bound outright, so a photo sent straight off the camera roll was uploaded whole over the tunnel + * to be either thrown away or rejected -- which is what "sending an image is broken" was. + * + * Shrunk here rather than on the backend, so the bytes that never mattered are never sent: the + * expensive part of this on a phone is the upload, not the decode. What the limit *is* comes from + * the server, per session -- see `DriverKind::max_image_edge` -- because that is where a provider's + * requirements are known, and a phone that carried its own copy of them would be a second place to + * update when one changes. + */ +suspend fun uploadPickedImage( + context: Context, + settings: ServerSettings, + sessionId: String, + uri: Uri, + maxEdge: Int?, +): String { + val (bytes, mime) = readForUpload(context, uri, maxEdge) + return uploadAttachment(settings, sessionId, bytes, mime) +} + +/** + * The bytes to upload and what they are, scaled down only if they need to be. + * + * An image already inside the limit is uploaded exactly as it came, rather than decoded and + * re-encoded to the same size: a round trip through JPEG loses a little every time, and there is + * nothing to gain from it. This is also the path a provider with no limit always takes. + */ +private fun readForUpload(context: Context, uri: Uri, maxEdge: Int?): Pair { + val resolver = context.contentResolver + val mime = resolver.getType(uri) ?: "image/jpeg" + val original = + resolver.openInputStream(uri)?.use { it.readBytes() } + ?: throw ApiException("couldn't read the picked image") + if (maxEdge == null) return original to mime + + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(original, 0, original.size, bounds) + val longest = max(bounds.outWidth, bounds.outHeight) + // outWidth is -1 when the bytes are not an image this device can decode. Sent on untouched: + // this function's job is the size, and refusing something the server might understand is a + // decision it has no business making. + if (longest <= 0 || longest <= maxEdge) return original to mime + + // Powers of two first, which is all the decoder can do, and then the exact scale. Decoding + // the full twelve megapixels only to shrink it is how this runs out of memory on the images + // it most needs to handle. + val decode = + BitmapFactory.Options().apply { + inSampleSize = Integer.highestOneBit(max(1, longest / maxEdge)) + } + val decoded = + BitmapFactory.decodeByteArray(original, 0, original.size, decode) ?: return original to mime + val scale = maxEdge.toFloat() / max(decoded.width, decoded.height) + val matrix = Matrix() + if (scale < 1f) matrix.postScale(scale, scale) + // The camera writes which way up the picture is into EXIF rather than rotating the pixels, and + // re-encoding drops the tag -- so a portrait photo would arrive at the model on its side, with + // nothing anywhere saying so. Applied to the same matrix as the scale, so it costs no second + // copy of the bitmap. + matrix.postRotate(exifRotation(original)) + val scaled = Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, matrix, true) + val out = ByteArrayOutputStream() + // JPEG whatever came in: this is a photograph being made smaller, which is what JPEG is for, + // and a PNG of a resampled photo is several times the size for no visible difference. + scaled.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, out) + return out.toByteArray() to "image/jpeg" +} + +/** How far to turn the picture so it is the way up it was taken. */ +private fun exifRotation(bytes: ByteArray): Float = + try { + when ( + ExifInterface(bytes.inputStream()) + .getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) + ) { + ExifInterface.ORIENTATION_ROTATE_90 -> 90f + ExifInterface.ORIENTATION_ROTATE_180 -> 180f + ExifInterface.ORIENTATION_ROTATE_270 -> 270f + else -> 0f + } + } catch (_: java.io.IOException) { + // No EXIF, or none this can read. Upright is the assumption every + // image without the tag is displayed under anyway. + 0f + } + +/** High enough that resampling is what the reader notices, not the encoder. */ +private const val JPEG_QUALITY = 90 diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt index ee245db..65d0c9c 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt @@ -29,6 +29,15 @@ sealed class SessionEvent { * bubbles and clearing whichever one matched first would leave the wrong one on screen. */ val id: String?, + /** + * What was attached to it, by the ref the files route serves. + * + * On the message rather than beside it: these arrived as separate image events until + * 2026-08-30, which drew somebody's screenshot as a row floating above the bubble that sent + * it, and left this app deciding from adjacency alone which message an image went with -- + * something the sender knew and could simply have said. + */ + val images: List, ) : SessionEvent() /** @@ -41,7 +50,8 @@ sealed class SessionEvent { * Resolved by the [UserMessage] carrying the same id, exactly as [CommandQueued] is resolved by * [CommandSent]. */ - data class MessageQueued(val id: String, val text: String) : SessionEvent() + data class MessageQueued(val id: String, val text: String, val images: List) : + SessionEvent() data class AssistantText(val delta: String) : SessionEvent() @@ -144,6 +154,17 @@ sealed class SessionEvent { data class Unknown(val type: String) : SessionEvent() } +/** + * A JSON array of strings under [name], empty when the field is absent. + * + * Absent is the ordinary case -- most messages carry no attachment, and the server omits the field + * rather than sending an empty list -- so this is the shape every caller wants. + */ +private fun JSONObject.stringList(name: String): List { + val array = optJSONArray(name) ?: return emptyList() + return (0 until array.length()).map { array.getString(it) } +} + fun parseSeqEvent(json: String): SeqEvent { val body = JSONObject(json) val event = @@ -152,9 +173,14 @@ fun parseSeqEvent(json: String): SeqEvent { SessionEvent.UserMessage( body.getString("text"), body.optString("id").ifEmpty { null }, + body.stringList("images"), ) "messageQueued" -> - SessionEvent.MessageQueued(body.getString("id"), body.getString("text")) + SessionEvent.MessageQueued( + body.getString("id"), + body.getString("text"), + body.stringList("images"), + ) "assistantText" -> SessionEvent.AssistantText(body.getString("delta")) "toolStart" -> SessionEvent.ToolStart( diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/NerdIcons.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/NerdIcons.kt index 6a0314c..71f126b 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/NerdIcons.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/NerdIcons.kt @@ -64,6 +64,9 @@ val STOP_GLYPH = glyph(0xF04DB) */ val QUEUE_GLYPH = glyph(0xF1163) +/** `md-close` -- take this off again: an attachment picked and not wanted. */ +val CLOSE_GLYPH = glyph(0xF0156) + /** `md-arrow_left` -- back one level, to whatever this was opened from. */ val BACK_GLYPH = glyph(0xF004D) diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt new file mode 100644 index 0000000..fce2c8f --- /dev/null +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt @@ -0,0 +1,145 @@ +package com.example.aiapp + +import android.graphics.BitmapFactory +import androidx.compose.foundation.Image +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.horizontalScroll +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** + * What is about to be sent, directly above the box it will be sent from. + * + * The count on the "+" button was the whole of what said an image was attached, so the only way to + * find out *which* image was to send it. A control belongs with the thing it acts on, and what + * these are attached to is the message being typed -- which is why they sit here rather than + * anywhere else on the screen. + * + * Scrolls sideways rather than wrapping or shrinking: the row keeps one thumbnail size whatever is + * in it, so four attachments look like four of the same thing rather than four smaller ones. + */ +@Composable +fun PendingAttachments( + settings: ServerSettings, + sessionId: String, + refs: List, + onRemove: (String) -> Unit, + modifier: Modifier = Modifier, +) { + if (refs.isEmpty()) return + Row( + modifier = modifier.horizontalScroll(rememberScrollState()).padding(bottom = 8.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + refs.forEach { ref -> PendingThumbnail(settings, sessionId, ref) { onRemove(ref) } } + } +} + +/** + * One attachment, square, tap to take it back off. + * + * Removal is here because there is nowhere else it could be: an image picked by mistake could + * otherwise only be dealt with by sending it. The whole thumbnail is the target rather than a + * corner cross -- a cross small enough to sit on a 64dp square is smaller than a fingertip -- and + * the label is what says so, since nothing about the picture does. + */ +@Composable +private fun PendingThumbnail( + settings: ServerSettings, + sessionId: String, + ref: String, + onRemove: () -> Unit, +) { + var bitmap by remember(ref) { mutableStateOf(null) } + var failed by remember(ref) { mutableStateOf(false) } + LaunchedEffect(ref) { + try { + val bytes = withContext(Dispatchers.IO) { fetchSessionFile(settings, sessionId, ref) } + bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)?.asImageBitmap() + failed = bitmap == null + } catch (_: ApiException) { + failed = true + } + } + val shape = RoundedCornerShape(8.dp) + Box( + Modifier.size(THUMBNAIL) + .clip(shape) + // An outline as well as a fill. Most of what gets attached here is a screenshot of a + // dark app, and cropped to a square its middle is often near-black -- against this + // background the tile then had no edge at all, and the only thing saying an image was + // attached was the cross drawn on top of nothing. + .border(1.dp, MaterialTheme.colorScheme.outlineVariant, shape) + // Behind the picture as well as under a missing one, so the tile is a tile before + // anything has arrived to fill it. + .background(MaterialTheme.colorScheme.surfaceVariant) + .clickable(onClick = onRemove) + .semantics { contentDescription = "Attached image, tap to remove" }, + contentAlignment = Alignment.Center, + ) { + when (val image = bitmap) { + // The two are told apart for the same reason the transcript's images are: one of them + // is worth waiting for and the other never resolves. + null -> + Text( + if (failed) "!" else "…", + style = MaterialTheme.typography.bodyLarge, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + else -> + Image( + bitmap = image, + contentDescription = null, + contentScale = ContentScale.Crop, + modifier = Modifier.size(THUMBNAIL), + ) + } + // The whole square removes it, and this only says so. A cross small enough to sit in + // the corner of a 64dp thumbnail is smaller than a fingertip, so making it the target + // would be a control drawn at a size nobody can hit. + // + // The disc is sized here and the mark centred inside it, rather than the glyph being + // aligned directly: a glyph's box is wider than the cross it draws, so aligning the box + // to the corner hung the visible mark over the edge and put its backing somewhere the + // eye reads as a second, misplaced square. + Box( + Modifier.align(Alignment.TopEnd) + .padding(2.dp) + .size(20.dp) + .background(MaterialTheme.colorScheme.surface.copy(alpha = 0.75f), CircleShape), + contentAlignment = Alignment.Center, + ) { + Glyph(CLOSE_GLYPH, colour = MaterialTheme.colorScheme.onSurface, size = 12.sp) + } + } +} + +private val THUMBNAIL = 64.dp diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt index 5060c46..2b832f9 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt @@ -99,7 +99,12 @@ sealed class TranscriptItem { */ abstract val seq: Long - data class UserMsg(override val seq: Long, val text: String) : TranscriptItem() + data class UserMsg( + override val seq: Long, + val text: String, + /** Refs of what was attached, drawn inside the bubble. */ + val images: List = emptyList(), + ) : TranscriptItem() data class AssistantMsg(override val seq: Long, val text: String) : TranscriptItem() @@ -311,7 +316,8 @@ private fun adoptRun( fun foldEvent(items: List, entry: SeqEvent): List = when (val event = entry.event) { - is SessionEvent.UserMessage -> items + TranscriptItem.UserMsg(entry.seq, event.text) + is SessionEvent.UserMessage -> + items + TranscriptItem.UserMsg(entry.seq, event.text, event.images) is SessionEvent.AssistantText -> { // Deltas accumulate into the message they're streaming, which keeps the seq of the // first of them: a row whose identity changed with every delta would be a new row on @@ -517,7 +523,7 @@ fun SessionScreen( // them. From the event stream rather than from what this screen sent, so they are still here // after leaving the session or restarting the app -- and so a message sent from another device // is drawn waiting on this one too. - var queued by remember { mutableStateOf(listOf>()) } + var queued by remember { mutableStateOf(listOf()) } // Commands the session has been asked to run and cannot yet, by the id that will resolve // them. From the server rather than from this screen, so a rename sent from the settings // screen -- or from another device -- is drawn waiting here too. @@ -577,10 +583,10 @@ fun SessionScreen( // Waiting, then read. Matched by id: the same message sent twice is two // bubbles, and clearing by text would take away whichever matched first. if (event is SessionEvent.MessageQueued) { - queued = queued + (event.id to event.text) + queued = queued + QueuedMessage(event.id, event.text, event.images) } if (event is SessionEvent.UserMessage) { - queued = queued.filterNot { it.first == event.id } + queued = queued.filterNot { it.id == event.id } } // Waiting, then gone: a command leaves this list when the session takes it, // and the row it becomes is added by `foldEvent` in the same pass. @@ -924,12 +930,16 @@ fun SessionScreen( try { val id = withContext(Dispatchers.IO) { - val bytes = - context.contentResolver.openInputStream(uri)?.use { - it.readBytes() - } ?: throw ApiException("couldn't read the picked image") - val mime = context.contentResolver.getType(uri) ?: "image/jpeg" - uploadAttachment(settings, summary.id, bytes, mime) + // Shrunk to what this session's provider takes before it is + // uploaded, so a twelve-megapixel photo does not cross the tunnel + // to be rejected at the far end -- see `uploadPickedImage`. + uploadPickedImage( + context, + settings, + summary.id, + uri, + summary.maxImageEdge, + ) } pendingAttachments = pendingAttachments + id actionError = null @@ -1042,7 +1052,15 @@ fun SessionScreen( waitingCommands.forEach { (_, text) -> CommandBubble(text, waiting = true) } - queued.forEach { (_, text) -> UserBubble(text, pending = true) } + queued.forEach { waiting -> + UserBubble( + settings = settings, + sessionId = summary.id, + text = waiting.text, + images = waiting.images, + pending = true, + ) + } } } } @@ -1084,7 +1102,13 @@ fun SessionScreen( ) is TranscriptRow.Single -> when (val item = row.item) { - is TranscriptItem.UserMsg -> UserBubble(item.text) + is TranscriptItem.UserMsg -> + UserBubble( + settings = settings, + sessionId = summary.id, + text = item.text, + images = item.images, + ) is TranscriptItem.AssistantMsg -> AssistantMessage(item.text) is TranscriptItem.ToolRun -> ToolCard( @@ -1210,6 +1234,14 @@ fun SessionScreen( // put the full width behind three controls, so the thing being // typed into was the narrowest thing on the row. Column(Modifier.fillMaxWidth().padding(8.dp)) { + // Directly above the box they will be sent from, so what is attached is visible + // rather than counted: the "+2" on the button below said how many and never which. + PendingAttachments( + settings = settings, + sessionId = summary.id, + refs = pendingAttachments, + onRemove = { pendingAttachments = pendingAttachments - it }, + ) OutlinedTextField( value = input, onValueChange = { @@ -1217,9 +1249,9 @@ fun SessionScreen( saveDraft(context, summary.id, it) }, modifier = Modifier.fillMaxWidth(), - placeholder = { - Text(if (pendingAttachments.isEmpty()) "Message" else "Message (+image)") - }, + // No longer "(+image)": the images are on screen above this, and a placeholder + // saying so said it in words beside the thing itself. + placeholder = { Text("Message") }, maxLines = 4, ) Row( @@ -1235,7 +1267,8 @@ fun SessionScreen( ) } ) { - Text(if (pendingAttachments.isEmpty()) "+" else "+${pendingAttachments.size}") + // Just "+" now. The count was standing in for showing them. + Text("+") } // The settings share what is left after the actions have // taken what they need. A Row hands out intrinsic widths in @@ -1323,7 +1356,13 @@ private fun sendLabel(running: Boolean) = if (running) "Queue" else "Send" * bitmap is remembered per ref, so scrolling doesn't refetch. */ @Composable -private fun UserBubble(text: String, pending: Boolean = false) { +private fun UserBubble( + settings: ServerSettings, + sessionId: String, + text: String, + images: List = emptyList(), + pending: Boolean = false, +) { Box(Modifier.fillMaxWidth()) { Card( // A message the session has not read yet is drawn quieter than @@ -1338,17 +1377,31 @@ private fun UserBubble(text: String, pending: Boolean = false) { ), modifier = Modifier.align(Alignment.CenterEnd).padding(start = 48.dp), ) { - Text( - text, - modifier = Modifier.padding(12.dp), - color = - if (pending) MaterialTheme.colorScheme.onSurfaceVariant - else MaterialTheme.colorScheme.onPrimaryContainer, - ) + Column(Modifier.padding(12.dp)) { + // Above the words, in the order they were composed: the picture was attached + // before the sentence about it was typed, and it is what the sentence refers to. + images.forEach { ref -> + SessionImage(settings, sessionId, ref) + Spacer(Modifier.height(4.dp)) + } + // A message can be nothing but an attachment, and an empty line under a picture + // is a bubble with a gap in it for a sentence nobody wrote. + if (text.isNotEmpty()) { + Text( + text, + color = + if (pending) MaterialTheme.colorScheme.onSurfaceVariant + else MaterialTheme.colorScheme.onPrimaryContainer, + ) + } + } } } } +/** A message the server has accepted and the session has not read yet. */ +private data class QueuedMessage(val id: String, val text: String, val images: List) + /** * Collapsed by default: name plus a spinner while running, expandable to the input and output. The * spinner-while-unfinished is exactly "ToolStart with no matching ToolEnd yet". diff --git a/app/androidApp/src/main/res/font/nerd_icons.ttf b/app/androidApp/src/main/res/font/nerd_icons.ttf index 7456825..ef7791a 100644 Binary files a/app/androidApp/src/main/res/font/nerd_icons.ttf and b/app/androidApp/src/main/res/font/nerd_icons.ttf differ diff --git a/app/build-icon-font.sh b/app/build-icon-font.sh index 2d9c18e..873af93 100755 --- a/app/build-icon-font.sh +++ b/app/build-icon-font.sh @@ -33,6 +33,7 @@ GLYPHS=( U+F048A # md-send U+F04DB # md-stop U+F1163 # md-send_clock + U+F0156 # md-close U+F004D # md-arrow_left U+F201 # fa-line_chart -- Font Awesome's, asked for by name ) diff --git a/server/src/config.rs b/server/src/config.rs index 346bc72..6e04974 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -135,6 +135,32 @@ pub enum DriverKind { } impl DriverKind { + /// The longest edge, in pixels, an image should have when it reaches + /// this kind of session -- `None` where nothing here has a limit worth + /// enforcing. + /// + /// Reported to the phone rather than applied here, so the bytes are made + /// small before they cross the tunnel instead of after: a modern phone + /// photo is several megabytes and twelve megapixels, and every one of + /// those bytes was being uploaded over WireGuard only to be rejected at + /// the other end. What decides the number is the provider, which is why + /// it lives beside the kind rather than in the app -- a phone that knew + /// each provider's limits would be a second place to update when one + /// changes. + /// + /// 1568 for the Claude CLI because that is the longest edge the API + /// itself resizes to; anything larger is charged the same and spends the + /// upload for nothing, and far larger is refused outright, which is what + /// "sending an image is broken" turned out to be. The others take images + /// through no path that cares, so they get no limit rather than a made-up + /// one. + pub fn max_image_edge(self) -> Option { + match self { + DriverKind::ClaudeCli => Some(1568), + DriverKind::Echo | DriverKind::LlamaCpp => None, + } + } + /// Whether the conversation exists outside this app, so that deleting /// the session here does not end it. /// diff --git a/server/src/session/claude.rs b/server/src/session/claude.rs index 488ff3f..ed57340 100644 --- a/server/src/session/claude.rs +++ b/server/src/session/claude.rs @@ -139,7 +139,7 @@ struct Queue { /// Written, not yet announced, oldest first, each with the id of the /// `MessageQueued` that told the phone it was waiting -- so the /// announcement can name which bubble it resolves. - awaiting: VecDeque<(String, String)>, + awaiting: VecDeque<(String, String, Vec)>, /// The process is gone, so nothing can be taken up any more. /// /// Needed because every other way out of a turn is an `Idle` this @@ -159,7 +159,7 @@ impl Queue { fn close(&mut self, sink: &EventSink, why: &str) { self.closed = true; self.running = false; - let lost: Vec = self.awaiting.drain(..).map(|(_, text)| text).collect(); + let lost: Vec = self.awaiting.drain(..).map(|(_, text, _)| text).collect(); if lost.is_empty() { return; } @@ -523,9 +523,11 @@ impl Driver for ClaudeDriver { // session or restarting the app drew nothing pending while a // message was still in the queue. let id = super::random_hex(); - queue.awaiting.push_back((id.clone(), text.clone())); + queue + .awaiting + .push_back((id.clone(), text.clone(), images.clone())); drop(queue); - let _ = self.sink.send(Event::MessageQueued { id, text }); + let _ = self.sink.send(Event::MessageQueued { id, text, images }); self.send_line(line); return; } @@ -534,7 +536,11 @@ impl Driver for ClaudeDriver { // Nothing is in flight, so there is nothing to wait for: this // message *is* the turn about to start, and it never had a // `MessageQueued` to resolve. - let _ = self.sink.send(Event::MessageTaken { id: None, text }); + let _ = self.sink.send(Event::MessageTaken { + id: None, + text, + images, + }); let _ = self.sink.send(Event::Status { state: SessionStatus::Running, }); @@ -945,13 +951,17 @@ fn proves_a_turn(event: &Event) -> bool { /// [`translate_line`], and the pair is the whole of the rule -- a steer /// announced anywhere else lands above output that predates it. fn announce_steers(queue: &Arc>, sink: &EventSink) -> bool { - let taken: Vec<(String, String)> = { + let taken: Vec<(String, String, Vec)> = { let mut queue = queue.lock().unwrap(); queue.awaiting.drain(..).collect() }; - for (id, text) in taken { + for (id, text, images) in taken { if sink - .send(Event::MessageTaken { id: Some(id), text }) + .send(Event::MessageTaken { + id: Some(id), + text, + images, + }) .is_err() { return false; @@ -1151,11 +1161,11 @@ mod tests { ); // Typed after the first delta, with the answer still arriving. let events = events_with_interjection(&lines, 1, |queue| { - queue - .lock() - .unwrap() - .awaiting - .push_back(("q1".into(), "do the other one instead".into())) + queue.lock().unwrap().awaiting.push_back(( + "q1".into(), + "do the other one instead".into(), + Vec::new(), + )) }); let at = |find: fn(&Event) -> bool| { @@ -1212,7 +1222,7 @@ mod tests { .lock() .unwrap() .awaiting - .push_back(("q2".into(), "never mind".into())) + .push_back(("q2".into(), "never mind".into(), Vec::new())) }); let taken = events @@ -1375,8 +1385,12 @@ mod tests { running: true, ..Queue::default() }; - queue.awaiting.push_back(("q1".into(), "first".into())); - queue.awaiting.push_back(("q2".into(), "second".into())); + queue + .awaiting + .push_back(("q1".into(), "first".into(), Vec::new())); + queue + .awaiting + .push_back(("q2".into(), "second".into(), Vec::new())); queue.close(&sink, "the session ended"); // Named rather than counted, because these never reached the diff --git a/server/src/session/driver.rs b/server/src/session/driver.rs index e7695b4..bb69ef9 100644 --- a/server/src/session/driver.rs +++ b/server/src/session/driver.rs @@ -81,6 +81,16 @@ pub enum Event { #[serde(default, skip_serializing_if = "Option::is_none")] id: Option, text: String, + /// What was attached to it, by the ref the files route serves. + /// + /// On the message rather than beside it. These used to be their own + /// `Image` events emitted just before, which drew a person's + /// screenshot as a row of its own floating above the bubble that + /// sent it -- and left the phone to decide, from nothing but + /// adjacency, which message an image belonged to. Belonging is not + /// something to infer when the sender knew. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + images: Vec, }, /// A message accepted from the phone that the session cannot read yet. /// @@ -98,6 +108,11 @@ pub enum Event { MessageQueued { id: String, text: String, + /// Carried for the same reason [`Event::UserMessage`] carries it, + /// and it matters more here: a waiting message is on screen for as + /// long as the turn runs, so its attachment has nowhere else to be. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + images: Vec, }, /// A driver has taken one of the user's messages and started reading /// it. The manager turns this into the `UserMessage` above, so it @@ -114,6 +129,9 @@ pub enum Event { /// waited. Carried through onto the `UserMessage`. id: Option, text: String, + /// Carried through onto the `UserMessage` with everything else. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + images: Vec, }, /// Streaming assistant text; the phone renders the concatenation as /// markdown. diff --git a/server/src/session/echo.rs b/server/src/session/echo.rs index 1923fb1..cd0fa80 100644 --- a/server/src/session/echo.rs +++ b/server/src/session/echo.rs @@ -85,7 +85,7 @@ pub struct EchoDriver { busy: Arc, /// Held messages with the id of the `MessageQueued` each one announced, /// so the announcement can say which waiting bubble it resolves. - queued: Arc>>, + queued: Arc>>, /// Where `/mixed` writes the images it references, which is the same /// directory the files route serves them from. session_dir: PathBuf, @@ -214,7 +214,7 @@ impl EchoDriver { /// transcript, and a command is not -- the manager has already /// recorded that one was sent, and saying so twice drew the same /// line in both colours. - fn handle(&self, text: String, _images: Vec, announce: bool) { + fn handle(&self, text: String, images: Vec, announce: bool) { let sink = self.sink.clone(); // Mid-turn messages are held rather than answered, the way a real @@ -227,9 +227,12 @@ impl EchoDriver { // an echo session has to produce the same events or the states // it exists to exercise are not the app's real ones. let id = super::random_hex(); - self.queued.lock().unwrap().push((id.clone(), text.clone())); + self.queued + .lock() + .unwrap() + .push((id.clone(), text.clone(), images.clone())); if announce { - self.emit(Event::MessageQueued { id, text }); + self.emit(Event::MessageQueued { id, text, images }); } return; } @@ -245,6 +248,7 @@ impl EchoDriver { self.emit(Event::MessageTaken { id: None, text: text.clone(), + images: images.clone(), }); } self.emit(Event::PeerMessage { @@ -266,7 +270,11 @@ impl EchoDriver { // this is the typed path onto it. if text.trim() == "/compact" { if announce { - self.emit(Event::MessageTaken { id: None, text }); + self.emit(Event::MessageTaken { + id: None, + text, + images, + }); } self.compact(); return; @@ -274,7 +282,11 @@ impl EchoDriver { if text.trim() == "/ask" { if announce { - self.emit(Event::MessageTaken { id: None, text }); + self.emit(Event::MessageTaken { + id: None, + text, + images, + }); } self.ask_user_question(); return; @@ -358,6 +370,7 @@ impl EchoDriver { send(Event::MessageTaken { id: None, text: text.clone(), + images: images.clone(), }); } send(Event::Status { @@ -607,13 +620,20 @@ async fn write_beat(sink: &EventSink, session_dir: &Path, beat: usize) { tokio::time::sleep(Duration::from_millis(120)).await; } +/// A message written during a turn and waiting for it to end: the id of the +/// `MessageQueued` that announced it, what it said, and what was attached to +/// it. All three, because all three are what the `MessageTaken` at the other +/// end owes -- named rather than written out at each of the four places that +/// mention it. +type Held = (String, String, Vec); + /// Ending a turn is also when anything held during it is taken up -- the /// moment a real CLI would have injected it. One place, because a turn has /// several ways to end (a reply, an interrupt, a compaction) and every one /// of them owes the same answer. -fn finish_turn(sink: &EventSink, queued: &Mutex>, busy: &AtomicBool) { +fn finish_turn(sink: &EventSink, queued: &Mutex>, busy: &AtomicBool) { let held = std::mem::take(&mut *queued.lock().unwrap()); - for (id, text) in held { + for (id, text, images) in held { // Announced before it is answered, in that order: a phone showing // the message as pending needs the signal that it has been read, // and the answer is meaningless above a message still drawn as @@ -621,6 +641,7 @@ fn finish_turn(sink: &EventSink, queued: &Mutex>, busy: &A let _ = sink.send(Event::MessageTaken { id: Some(id), text: text.clone(), + images, }); let _ = sink.send(Event::AssistantText { delta: format!("\n(taken from the queue) You said: {text}"), diff --git a/server/src/session/import.rs b/server/src/session/import.rs index 64194c3..e49bdc8 100644 --- a/server/src/session/import.rs +++ b/server/src/session/import.rs @@ -540,7 +540,15 @@ fn push_user(events: &mut Vec, content: &Value, session_dir: &std::path:: if !text.trim().is_empty() { // Replayed from the CLI's own file: it was read long ago, so // there is no waiting bubble for it to resolve. - events.push(Event::UserMessage { id: None, text }); + // The images in this record are saved and referenced separately just + // above, because a replayed message's pictures came out of somebody + // else's file rather than out of this app's composer -- there is no + // upload here whose refs could ride on the message. + events.push(Event::UserMessage { + id: None, + text, + images: Vec::new(), + }); } } diff --git a/server/src/session/llama.rs b/server/src/session/llama.rs index 39c9dda..ad10d91 100644 --- a/server/src/session/llama.rs +++ b/server/src/session/llama.rs @@ -347,6 +347,9 @@ impl Driver for LlamaDriver { let _ = sink.send(Event::MessageTaken { id: None, text: text.clone(), + // Never any: this driver refuses images above, and saying + // so is what the refusal above is for. + images: Vec::new(), }); let _ = sink.send(Event::Status { state: SessionStatus::Running, @@ -636,6 +639,7 @@ mod tests { Event::UserMessage { id: None, text: "hello".into(), + images: Vec::new(), }, Event::AssistantText { delta: "hi ".into(), @@ -649,6 +653,7 @@ mod tests { Event::UserMessage { id: None, text: "again".into(), + images: Vec::new(), }, Event::AssistantText { delta: "yes".into(), @@ -680,6 +685,7 @@ mod tests { Event::UserMessage { id: None, text: "count".into(), + images: Vec::new(), }, Event::AssistantText { delta: "one two".into(), @@ -705,6 +711,7 @@ mod tests { Event::UserMessage { id: None, text: "hello".into(), + images: Vec::new(), }, Event::Error { message: "something went wrong".into(), @@ -732,6 +739,7 @@ mod tests { Event::UserMessage { id: None, text: "the long expensive conversation".into(), + images: Vec::new(), }, Event::AssistantText { delta: "at length".into(), @@ -740,6 +748,7 @@ mod tests { Event::UserMessage { id: None, text: "a fresh start".into(), + images: Vec::new(), }, Event::AssistantText { delta: "cheaply".into(), @@ -759,16 +768,19 @@ mod tests { Event::UserMessage { id: None, text: "one".into(), + images: Vec::new(), }, Event::Cleared, Event::UserMessage { id: None, text: "two".into(), + images: Vec::new(), }, Event::Cleared, Event::UserMessage { id: None, text: "three".into(), + images: Vec::new(), }, ]); let messages = conversation(&path); diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 98b9217..478fe8d 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -146,6 +146,13 @@ pub struct SessionInfo { /// Every token this session has spent, so a phone showing a total does /// not have to add up a transcript it only holds part of. pub total_tokens: u64, + /// The longest edge an image should have by the time it gets here, or + /// absent where this provider has no limit -- see + /// [`DriverKind::max_image_edge`]. Absent rather than a large number, + /// because "no limit" and "a limit that happens to be big" are different + /// answers and only one of them stays true. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_image_edge: Option, /// Whether this session announces itself -- reported for the same /// reason `permission_mode` is: a switch that guesses its own position /// is how you turn something off while believing you are reading it. @@ -290,17 +297,11 @@ impl LiveSession { /// turn it waits, and writing it down on the way past would put it /// above output that happened before the session ever saw it. pub fn send_message(&self, text: String, images: Vec) { - // Attachments are the exception, recorded on the way past: they - // are uploaded whether or not the message waits, and the phone - // fetches them by the same ref the files route serves. So a queued - // message's picture appears a little before its text. - for image in &images { - let _ = self.sink.send(Event::Image { - image: image.clone(), - // A person's own attachment belongs to no tool call. - about: None, - }); - } + // The attachments ride *on* the message rather than as `Image` + // events emitted just before it. They used to be the latter, which + // drew a person's screenshot as a row floating above the bubble + // that sent it, and left the phone inferring from adjacency which + // message an image went with -- a thing the sender already knew. self.driver.send_user_message(text, images); } @@ -370,7 +371,13 @@ impl LiveSession { /// `setup_name` is passed in rather than stored: only the manager /// holds the config, and the label can change under a running session. - fn info(&self, setup_name: &str, imported: bool, keeps_own_transcript: bool) -> SessionInfo { + /// `kind` rather than the facts derived from it: two of this row's + /// fields are answers about the provider's *kind*, and passing them + /// separately meant every caller deriving each one and a third arriving + /// as a third parameter. `None` where the provider has been edited away, + /// which is a session that cannot run -- so both answers are the + /// cautious one rather than a guess. + fn info(&self, setup_name: &str, imported: bool, kind: Option) -> SessionInfo { SessionInfo { id: self.meta.id.clone(), provider: self.meta.provider.clone(), @@ -381,8 +388,9 @@ impl LiveSession { permission_mode: self.shared.permission_mode.lock().unwrap().clone(), total_tokens: *self.shared.total_tokens.lock().unwrap(), notify: *self.shared.notify.lock().unwrap(), + max_image_edge: kind.and_then(DriverKind::max_image_edge), imported, - keeps_own_transcript, + keeps_own_transcript: kind.is_some_and(DriverKind::keeps_own_transcript), cwd: self.meta.cwd.clone(), status: *self.shared.status.lock().unwrap(), last_activity: *self.shared.last_activity.lock().unwrap(), @@ -706,7 +714,7 @@ impl SessionManager { Some(session) => session.info( label_of(&inner.config, &meta.setup), import::read_cursor(&self.data_dir.join(&meta.id)).is_some(), - keeps_own_transcript(&inner.config, &meta.setup, &meta.provider), + kind_of(&inner.config, &meta.setup, &meta.provider), ), None => SessionInfo { id: meta.id.clone(), @@ -717,6 +725,8 @@ impl SessionManager { model: meta.model.clone(), permission_mode: meta.permission_mode.clone(), total_tokens: 0, + max_image_edge: kind_of(&inner.config, &meta.setup, &meta.provider) + .and_then(DriverKind::max_image_edge), notify: meta.notify, imported: import::read_cursor(&self.data_dir.join(&meta.id)).is_some(), keeps_own_transcript: keeps_own_transcript( @@ -853,7 +863,7 @@ impl SessionManager { let info = session.info( &setup.name, import::read_cursor(&self.data_dir.join(&id)).is_some(), - provider.kind.keeps_own_transcript(), + Some(provider.kind), ); inner.live.insert(id, session); Ok(info) @@ -1057,10 +1067,17 @@ fn label_of<'a>(config: &'a Config, id: &'a str) -> &'a str { /// "this can be brought back" on no evidence is the answer that loses /// somebody's conversation. fn keeps_own_transcript(config: &Config, setup: &str, provider: &str) -> bool { + kind_of(config, setup, provider).is_some_and(DriverKind::keeps_own_transcript) +} + +/// What a session's provider is, for the questions answered by its *kind* +/// rather than by its name. `None` for a provider that has been edited away, +/// which is a session that cannot run at all. +fn kind_of(config: &Config, setup: &str, provider: &str) -> Option { config .setup(setup) .and_then(|setup| setup.providers.iter().find(|it| it.name == provider)) - .is_some_and(|provider| provider.kind.keeps_own_transcript()) + .map(|provider| provider.kind) } /// Names for a failure message: what there is, so the reader can see what @@ -1363,7 +1380,7 @@ async fn pump( // message here rather than being carried alongside it. One rule // for where a user's message sits: where the session read it. let event = match event { - Event::MessageTaken { id, text } => Event::UserMessage { id, text }, + Event::MessageTaken { id, text, images } => Event::UserMessage { id, text, images }, // The running total is the pump's to keep, for the reason the // field gives: a driver knows what its own turn cost and // nothing else does. Added here rather than at each driver so @@ -1606,7 +1623,7 @@ mod tests { assert_eq!(first.session_id, info.id); // The title travels with it, because the phone may have no screen // open to look one up on. - assert_eq!(first.title, session.info("m", false, false).title); + assert_eq!(first.title, session.info("m", false, None).title); manager.set_session_notify(&info.id, false).expect("off"); // Subscribed before the message, or the turn can finish in the gap diff --git a/server/src/session/transcript.rs b/server/src/session/transcript.rs index 7e022ad..07b56d7 100644 --- a/server/src/session/transcript.rs +++ b/server/src/session/transcript.rs @@ -358,6 +358,7 @@ mod tests { Event::UserMessage { id: None, text: "hi".into(), + images: Vec::new(), }, text("hello"), Event::ToolStart {