iris/ai-app
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of git.arirex.me:iris/ai-app

Browse Source
This commit is contained in:
iris committed 2026-08-30 00:58:04 -04:00
2 parents ad791a0d84 da65c1571f
commit 03a8d7d3d3
16 files changed
+529 -70
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt
+10
View File
@@ -153,6 +153,15 @@ data class SessionSummary(
* Every token this session has spent, as the server counts it -- see `SessionEvent.UsageDelta`.
*/
val totalTokens: Long,
/**
* The longest edge an image should have when it reaches this session, or null where the
* provider has no limit.
*
* Null and "a big number" are different answers, and only the first stays true: a provider that
* does not care about size should not be given a threshold this app invented. Decided by the
* server because that is where a provider's kind is known -- see `uploadPickedImage`.
*/
val maxImageEdge: Int?,
val status: String,
val lastActivity: Double,
)
@@ -170,6 +179,7 @@ private fun parseSession(session: JSONObject) =
imported = session.optBoolean("imported", false),
notify = session.optBoolean("notify", true),
totalTokens = session.optLong("totalTokens", 0),
maxImageEdge = session.optInt("maxImageEdge", 0).takeIf { it > 0 },
status = session.getString("status"),
lastActivity = session.getDouble("lastActivity"),
)
app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt
+104
View File
@@ -0,0 +1,104 @@
package com.example.aiapp
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.Matrix
import android.media.ExifInterface
import android.net.Uri
import java.io.ByteArrayOutputStream
import kotlin.math.max
/**
* Getting a picked photo to a session, at a size the session can actually take.
*
* A phone camera produces twelve megapixels and several megabytes. The Claude API resizes anything
* larger than 1568px on its long edge before looking at it and refuses images past a much higher
* bound outright, so a photo sent straight off the camera roll was uploaded whole over the tunnel
* to be either thrown away or rejected -- which is what "sending an image is broken" was.
*
* Shrunk here rather than on the backend, so the bytes that never mattered are never sent: the
* expensive part of this on a phone is the upload, not the decode. What the limit *is* comes from
* the server, per session -- see `DriverKind::max_image_edge` -- because that is where a provider's
* requirements are known, and a phone that carried its own copy of them would be a second place to
* update when one changes.
*/
suspend fun uploadPickedImage(
context: Context,
settings: ServerSettings,
sessionId: String,
uri: Uri,
maxEdge: Int?,
): String {
val (bytes, mime) = readForUpload(context, uri, maxEdge)
return uploadAttachment(settings, sessionId, bytes, mime)
}
/**
* The bytes to upload and what they are, scaled down only if they need to be.
*
* An image already inside the limit is uploaded exactly as it came, rather than decoded and
* re-encoded to the same size: a round trip through JPEG loses a little every time, and there is
* nothing to gain from it. This is also the path a provider with no limit always takes.
*/
private fun readForUpload(context: Context, uri: Uri, maxEdge: Int?): Pair<ByteArray, String> {
val resolver = context.contentResolver
val mime = resolver.getType(uri) ?: "image/jpeg"
val original =
resolver.openInputStream(uri)?.use { it.readBytes() }
?: throw ApiException("couldn't read the picked image")
if (maxEdge == null) return original to mime
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeByteArray(original, 0, original.size, bounds)
val longest = max(bounds.outWidth, bounds.outHeight)
// outWidth is -1 when the bytes are not an image this device can decode. Sent on untouched:
// this function's job is the size, and refusing something the server might understand is a
// decision it has no business making.
if (longest <= 0 || longest <= maxEdge) return original to mime
// Powers of two first, which is all the decoder can do, and then the exact scale. Decoding
// the full twelve megapixels only to shrink it is how this runs out of memory on the images
// it most needs to handle.
val decode =
BitmapFactory.Options().apply {
inSampleSize = Integer.highestOneBit(max(1, longest / maxEdge))
}
val decoded =
BitmapFactory.decodeByteArray(original, 0, original.size, decode) ?: return original to mime
val scale = maxEdge.toFloat() / max(decoded.width, decoded.height)
val matrix = Matrix()
if (scale < 1f) matrix.postScale(scale, scale)
// The camera writes which way up the picture is into EXIF rather than rotating the pixels, and
// re-encoding drops the tag -- so a portrait photo would arrive at the model on its side, with
// nothing anywhere saying so. Applied to the same matrix as the scale, so it costs no second
// copy of the bitmap.
matrix.postRotate(exifRotation(original))
val scaled = Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, matrix, true)
val out = ByteArrayOutputStream()
// JPEG whatever came in: this is a photograph being made smaller, which is what JPEG is for,
// and a PNG of a resampled photo is several times the size for no visible difference.
scaled.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, out)
return out.toByteArray() to "image/jpeg"
}
/** How far to turn the picture so it is the way up it was taken. */
private fun exifRotation(bytes: ByteArray): Float =
try {
when (
ExifInterface(bytes.inputStream())
.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
) {
ExifInterface.ORIENTATION_ROTATE_90 -> 90f
ExifInterface.ORIENTATION_ROTATE_180 -> 180f
ExifInterface.ORIENTATION_ROTATE_270 -> 270f
else -> 0f
}
} catch (_: java.io.IOException) {
// No EXIF, or none this can read. Upright is the assumption every
// image without the tag is displayed under anyway.
0f
}
/** High enough that resampling is what the reader notices, not the encoder. */
private const val JPEG_QUALITY = 90
app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt
+28 -2
View File
@@ -29,6 +29,15 @@ sealed class SessionEvent {
* bubbles and clearing whichever one matched first would leave the wrong one on screen.
*/
val id: String?,
/**
* What was attached to it, by the ref the files route serves.
*
* On the message rather than beside it: these arrived as separate image events until
* 2026-08-30, which drew somebody's screenshot as a row floating above the bubble that sent
* it, and left this app deciding from adjacency alone which message an image went with --
* something the sender knew and could simply have said.
*/
val images: List<String>,
) : SessionEvent()
/**
@@ -41,7 +50,8 @@ sealed class SessionEvent {
* Resolved by the [UserMessage] carrying the same id, exactly as [CommandQueued] is resolved by
* [CommandSent].
*/
data class MessageQueued(val id: String, val text: String) : SessionEvent()
data class MessageQueued(val id: String, val text: String, val images: List<String>) :
SessionEvent()
data class AssistantText(val delta: String) : SessionEvent()
@@ -144,6 +154,17 @@ sealed class SessionEvent {
data class Unknown(val type: String) : SessionEvent()
}
/**
* A JSON array of strings under [name], empty when the field is absent.
*
* Absent is the ordinary case -- most messages carry no attachment, and the server omits the field
* rather than sending an empty list -- so this is the shape every caller wants.
*/
private fun JSONObject.stringList(name: String): List<String> {
val array = optJSONArray(name) ?: return emptyList()
return (0 until array.length()).map { array.getString(it) }
}
fun parseSeqEvent(json: String): SeqEvent {
val body = JSONObject(json)
val event =
@@ -152,9 +173,14 @@ fun parseSeqEvent(json: String): SeqEvent {
SessionEvent.UserMessage(
body.getString("text"),
body.optString("id").ifEmpty { null },
body.stringList("images"),
)
"messageQueued" ->
SessionEvent.MessageQueued(body.getString("id"), body.getString("text"))
SessionEvent.MessageQueued(
body.getString("id"),
body.getString("text"),
body.stringList("images"),
)
"assistantText" -> SessionEvent.AssistantText(body.getString("delta"))
"toolStart" ->
SessionEvent.ToolStart(
app/androidApp/src/main/kotlin/com/example/aiapp/NerdIcons.kt
+3
View File
@@ -64,6 +64,9 @@ val STOP_GLYPH = glyph(0xF04DB)
*/
val QUEUE_GLYPH = glyph(0xF1163)
/** `md-close` -- take this off again: an attachment picked and not wanted. */
val CLOSE_GLYPH = glyph(0xF0156)
/** `md-arrow_left` -- back one level, to whatever this was opened from. */
val BACK_GLYPH = glyph(0xF004D)
app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt
+145
View File
@@ -0,0 +1,145 @@
package com.example.aiapp
import android.graphics.BitmapFactory
import androidx.compose.foundation.Image
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.clickable
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.semantics.contentDescription
import androidx.compose.ui.semantics.semantics
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
/**
* What is about to be sent, directly above the box it will be sent from.
*
* The count on the "+" button was the whole of what said an image was attached, so the only way to
* find out *which* image was to send it. A control belongs with the thing it acts on, and what
* these are attached to is the message being typed -- which is why they sit here rather than
* anywhere else on the screen.
*
* Scrolls sideways rather than wrapping or shrinking: the row keeps one thumbnail size whatever is
* in it, so four attachments look like four of the same thing rather than four smaller ones.
*/
@Composable
fun PendingAttachments(
settings: ServerSettings,
sessionId: String,
refs: List<String>,
onRemove: (String) -> Unit,
modifier: Modifier = Modifier,
) {
if (refs.isEmpty()) return
Row(
modifier = modifier.horizontalScroll(rememberScrollState()).padding(bottom = 8.dp),
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
refs.forEach { ref -> PendingThumbnail(settings, sessionId, ref) { onRemove(ref) } }
}
}
/**
* One attachment, square, tap to take it back off.
*
* Removal is here because there is nowhere else it could be: an image picked by mistake could
* otherwise only be dealt with by sending it. The whole thumbnail is the target rather than a
* corner cross -- a cross small enough to sit on a 64dp square is smaller than a fingertip -- and
* the label is what says so, since nothing about the picture does.
*/
@Composable
private fun PendingThumbnail(
settings: ServerSettings,
sessionId: String,
ref: String,
onRemove: () -> Unit,
) {
var bitmap by remember(ref) { mutableStateOf<ImageBitmap?>(null) }
var failed by remember(ref) { mutableStateOf(false) }
LaunchedEffect(ref) {
try {
val bytes = withContext(Dispatchers.IO) { fetchSessionFile(settings, sessionId, ref) }
bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)?.asImageBitmap()
failed = bitmap == null
} catch (_: ApiException) {
failed = true
}
}
val shape = RoundedCornerShape(8.dp)
Box(
Modifier.size(THUMBNAIL)
.clip(shape)
// An outline as well as a fill. Most of what gets attached here is a screenshot of a
// dark app, and cropped to a square its middle is often near-black -- against this
// background the tile then had no edge at all, and the only thing saying an image was
// attached was the cross drawn on top of nothing.
.border(1.dp, MaterialTheme.colorScheme.outlineVariant, shape)
// Behind the picture as well as under a missing one, so the tile is a tile before
// anything has arrived to fill it.
.background(MaterialTheme.colorScheme.surfaceVariant)
.clickable(onClick = onRemove)
.semantics { contentDescription = "Attached image, tap to remove" },
contentAlignment = Alignment.Center,
) {
when (val image = bitmap) {
// The two are told apart for the same reason the transcript's images are: one of them
// is worth waiting for and the other never resolves.
null ->
Text(
if (failed) "!" else "…",
style = MaterialTheme.typography.bodyLarge,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
else ->
Image(
bitmap = image,
contentDescription = null,
contentScale = ContentScale.Crop,
modifier = Modifier.size(THUMBNAIL),
)
}
// The whole square removes it, and this only says so. A cross small enough to sit in
// the corner of a 64dp thumbnail is smaller than a fingertip, so making it the target
// would be a control drawn at a size nobody can hit.
//
// The disc is sized here and the mark centred inside it, rather than the glyph being
// aligned directly: a glyph's box is wider than the cross it draws, so aligning the box
// to the corner hung the visible mark over the edge and put its backing somewhere the
// eye reads as a second, misplaced square.
Box(
Modifier.align(Alignment.TopEnd)
.padding(2.dp)
.size(20.dp)
.background(MaterialTheme.colorScheme.surface.copy(alpha = 0.75f), CircleShape),
contentAlignment = Alignment.Center,
) {
Glyph(CLOSE_GLYPH, colour = MaterialTheme.colorScheme.onSurface, size = 12.sp)
}
}
}
private val THUMBNAIL = 64.dp
app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt
+78 -25
View File
@@ -99,7 +99,12 @@ sealed class TranscriptItem {
*/
abstract val seq: Long
data class UserMsg(override val seq: Long, val text: String) : TranscriptItem()
data class UserMsg(
override val seq: Long,
val text: String,
/** Refs of what was attached, drawn inside the bubble. */
val images: List<String> = emptyList(),
) : TranscriptItem()
data class AssistantMsg(override val seq: Long, val text: String) : TranscriptItem()
@@ -311,7 +316,8 @@ private fun adoptRun(
fun foldEvent(items: List<TranscriptItem>, entry: SeqEvent): List<TranscriptItem> =
when (val event = entry.event) {
is SessionEvent.UserMessage -> items + TranscriptItem.UserMsg(entry.seq, event.text)
is SessionEvent.UserMessage ->
items + TranscriptItem.UserMsg(entry.seq, event.text, event.images)
is SessionEvent.AssistantText -> {
// Deltas accumulate into the message they're streaming, which keeps the seq of the
// first of them: a row whose identity changed with every delta would be a new row on
@@ -517,7 +523,7 @@ fun SessionScreen(
// them. From the event stream rather than from what this screen sent, so they are still here
// after leaving the session or restarting the app -- and so a message sent from another device
// is drawn waiting on this one too.
var queued by remember { mutableStateOf(listOf<Pair<String, String>>()) }
var queued by remember { mutableStateOf(listOf<QueuedMessage>()) }
// Commands the session has been asked to run and cannot yet, by the id that will resolve
// them. From the server rather than from this screen, so a rename sent from the settings
// screen -- or from another device -- is drawn waiting here too.
@@ -577,10 +583,10 @@ fun SessionScreen(
// Waiting, then read. Matched by id: the same message sent twice is two
// bubbles, and clearing by text would take away whichever matched first.
if (event is SessionEvent.MessageQueued) {
queued = queued + (event.id to event.text)
queued = queued + QueuedMessage(event.id, event.text, event.images)
}
if (event is SessionEvent.UserMessage) {
queued = queued.filterNot { it.first == event.id }
queued = queued.filterNot { it.id == event.id }
}
// Waiting, then gone: a command leaves this list when the session takes it,
// and the row it becomes is added by `foldEvent` in the same pass.
@@ -924,12 +930,16 @@ fun SessionScreen(
try {
val id =
withContext(Dispatchers.IO) {
val bytes =
context.contentResolver.openInputStream(uri)?.use {
it.readBytes()
} ?: throw ApiException("couldn't read the picked image")
val mime = context.contentResolver.getType(uri) ?: "image/jpeg"
uploadAttachment(settings, summary.id, bytes, mime)
// Shrunk to what this session's provider takes before it is
// uploaded, so a twelve-megapixel photo does not cross the tunnel
// to be rejected at the far end -- see `uploadPickedImage`.
uploadPickedImage(
context,
settings,
summary.id,
uri,
summary.maxImageEdge,
)
}
pendingAttachments = pendingAttachments + id
actionError = null
@@ -1042,7 +1052,15 @@ fun SessionScreen(
waitingCommands.forEach { (_, text) ->
CommandBubble(text, waiting = true)
}
queued.forEach { (_, text) -> UserBubble(text, pending = true) }
queued.forEach { waiting ->
UserBubble(
settings = settings,
sessionId = summary.id,
text = waiting.text,
images = waiting.images,
pending = true,
)
}
}
}
}
@@ -1084,7 +1102,13 @@ fun SessionScreen(
)
is TranscriptRow.Single ->
when (val item = row.item) {
is TranscriptItem.UserMsg -> UserBubble(item.text)
is TranscriptItem.UserMsg ->
UserBubble(
settings = settings,
sessionId = summary.id,
text = item.text,
images = item.images,
)
is TranscriptItem.AssistantMsg -> AssistantMessage(item.text)
is TranscriptItem.ToolRun ->
ToolCard(
@@ -1210,6 +1234,14 @@ fun SessionScreen(
// put the full width behind three controls, so the thing being
// typed into was the narrowest thing on the row.
Column(Modifier.fillMaxWidth().padding(8.dp)) {
// Directly above the box they will be sent from, so what is attached is visible
// rather than counted: the "+2" on the button below said how many and never which.
PendingAttachments(
settings = settings,
sessionId = summary.id,
refs = pendingAttachments,
onRemove = { pendingAttachments = pendingAttachments - it },
)
OutlinedTextField(
value = input,
onValueChange = {
@@ -1217,9 +1249,9 @@ fun SessionScreen(
saveDraft(context, summary.id, it)
},
modifier = Modifier.fillMaxWidth(),
placeholder = {
Text(if (pendingAttachments.isEmpty()) "Message" else "Message (+image)")
},
// No longer "(+image)": the images are on screen above this, and a placeholder
// saying so said it in words beside the thing itself.
placeholder = { Text("Message") },
maxLines = 4,
)
Row(
@@ -1235,7 +1267,8 @@ fun SessionScreen(
)
}
) {
Text(if (pendingAttachments.isEmpty()) "+" else "+${pendingAttachments.size}")
// Just "+" now. The count was standing in for showing them.
Text("+")
}
// The settings share what is left after the actions have
// taken what they need. A Row hands out intrinsic widths in
@@ -1323,7 +1356,13 @@ private fun sendLabel(running: Boolean) = if (running) "Queue" else "Send"
* bitmap is remembered per ref, so scrolling doesn't refetch.
*/
@Composable
private fun UserBubble(text: String, pending: Boolean = false) {
private fun UserBubble(
settings: ServerSettings,
sessionId: String,
text: String,
images: List<String> = emptyList(),
pending: Boolean = false,
) {
Box(Modifier.fillMaxWidth()) {
Card(
// A message the session has not read yet is drawn quieter than
@@ -1338,17 +1377,31 @@ private fun UserBubble(text: String, pending: Boolean = false) {
),
modifier = Modifier.align(Alignment.CenterEnd).padding(start = 48.dp),
) {
Text(
text,
modifier = Modifier.padding(12.dp),
color =
if (pending) MaterialTheme.colorScheme.onSurfaceVariant
else MaterialTheme.colorScheme.onPrimaryContainer,
)
Column(Modifier.padding(12.dp)) {
// Above the words, in the order they were composed: the picture was attached
// before the sentence about it was typed, and it is what the sentence refers to.
images.forEach { ref ->
SessionImage(settings, sessionId, ref)
Spacer(Modifier.height(4.dp))
}
// A message can be nothing but an attachment, and an empty line under a picture
// is a bubble with a gap in it for a sentence nobody wrote.
if (text.isNotEmpty()) {
Text(
text,
color =
if (pending) MaterialTheme.colorScheme.onSurfaceVariant
else MaterialTheme.colorScheme.onPrimaryContainer,
)
}
}
}
}
}
/** A message the server has accepted and the session has not read yet. */
private data class QueuedMessage(val id: String, val text: String, val images: List<String>)
/**
* Collapsed by default: name plus a spinner while running, expandable to the input and output. The
* spinner-while-unfinished is exactly "ToolStart with no matching ToolEnd yet".
app/androidApp/src/main/res/font/nerd_icons.ttf
BIN
View File
Binary file not shown.
app/build-icon-font.sh
+1
View File
@@ -33,6 +33,7 @@ GLYPHS=(
U+F048A # md-send
U+F04DB # md-stop
U+F1163 # md-send_clock
U+F0156 # md-close
U+F004D # md-arrow_left
U+F201 # fa-line_chart -- Font Awesome's, asked for by name
)
server/src/config.rs
+26
View File
@@ -135,6 +135,32 @@ pub enum DriverKind {
}
impl DriverKind {
/// The longest edge, in pixels, an image should have when it reaches
/// this kind of session -- `None` where nothing here has a limit worth
/// enforcing.
///
/// Reported to the phone rather than applied here, so the bytes are made
/// small before they cross the tunnel instead of after: a modern phone
/// photo is several megabytes and twelve megapixels, and every one of
/// those bytes was being uploaded over WireGuard only to be rejected at
/// the other end. What decides the number is the provider, which is why
/// it lives beside the kind rather than in the app -- a phone that knew
/// each provider's limits would be a second place to update when one
/// changes.
///
/// 1568 for the Claude CLI because that is the longest edge the API
/// itself resizes to; anything larger is charged the same and spends the
/// upload for nothing, and far larger is refused outright, which is what
/// "sending an image is broken" turned out to be. The others take images
/// through no path that cares, so they get no limit rather than a made-up
/// one.
pub fn max_image_edge(self) -> Option<u32> {
match self {
DriverKind::ClaudeCli => Some(1568),
DriverKind::Echo | DriverKind::LlamaCpp => None,
}
}
/// Whether the conversation exists outside this app, so that deleting
/// the session here does not end it.
///
server/src/session/claude.rs
+30 -16
View File
@@ -139,7 +139,7 @@ struct Queue {
/// Written, not yet announced, oldest first, each with the id of the
/// `MessageQueued` that told the phone it was waiting -- so the
/// announcement can name which bubble it resolves.
awaiting: VecDeque<(String, String)>,
awaiting: VecDeque<(String, String, Vec<ImageRef>)>,
/// The process is gone, so nothing can be taken up any more.
///
/// Needed because every other way out of a turn is an `Idle` this
@@ -159,7 +159,7 @@ impl Queue {
fn close(&mut self, sink: &EventSink, why: &str) {
self.closed = true;
self.running = false;
let lost: Vec<String> = self.awaiting.drain(..).map(|(_, text)| text).collect();
let lost: Vec<String> = self.awaiting.drain(..).map(|(_, text, _)| text).collect();
if lost.is_empty() {
return;
}
@@ -523,9 +523,11 @@ impl Driver for ClaudeDriver {
// session or restarting the app drew nothing pending while a
// message was still in the queue.
let id = super::random_hex();
queue.awaiting.push_back((id.clone(), text.clone()));
queue
.awaiting
.push_back((id.clone(), text.clone(), images.clone()));
drop(queue);
let _ = self.sink.send(Event::MessageQueued { id, text });
let _ = self.sink.send(Event::MessageQueued { id, text, images });
self.send_line(line);
return;
}
@@ -534,7 +536,11 @@ impl Driver for ClaudeDriver {
// Nothing is in flight, so there is nothing to wait for: this
// message *is* the turn about to start, and it never had a
// `MessageQueued` to resolve.
let _ = self.sink.send(Event::MessageTaken { id: None, text });
let _ = self.sink.send(Event::MessageTaken {
id: None,
text,
images,
});
let _ = self.sink.send(Event::Status {
state: SessionStatus::Running,
});
@@ -945,13 +951,17 @@ fn proves_a_turn(event: &Event) -> bool {
/// [`translate_line`], and the pair is the whole of the rule -- a steer
/// announced anywhere else lands above output that predates it.
fn announce_steers(queue: &Arc<Mutex<Queue>>, sink: &EventSink) -> bool {
let taken: Vec<(String, String)> = {
let taken: Vec<(String, String, Vec<ImageRef>)> = {
let mut queue = queue.lock().unwrap();
queue.awaiting.drain(..).collect()
};
for (id, text) in taken {
for (id, text, images) in taken {
if sink
.send(Event::MessageTaken { id: Some(id), text })
.send(Event::MessageTaken {
id: Some(id),
text,
images,
})
.is_err()
{
return false;
@@ -1151,11 +1161,11 @@ mod tests {
);
// Typed after the first delta, with the answer still arriving.
let events = events_with_interjection(&lines, 1, |queue| {
queue
.lock()
.unwrap()
.awaiting
.push_back(("q1".into(), "do the other one instead".into()))
queue.lock().unwrap().awaiting.push_back((
"q1".into(),
"do the other one instead".into(),
Vec::new(),
))
});
let at = |find: fn(&Event) -> bool| {
@@ -1212,7 +1222,7 @@ mod tests {
.lock()
.unwrap()
.awaiting
.push_back(("q2".into(), "never mind".into()))
.push_back(("q2".into(), "never mind".into(), Vec::new()))
});
let taken = events
@@ -1375,8 +1385,12 @@ mod tests {
running: true,
..Queue::default()
};
queue.awaiting.push_back(("q1".into(), "first".into()));
queue.awaiting.push_back(("q2".into(), "second".into()));
queue
.awaiting
.push_back(("q1".into(), "first".into(), Vec::new()));
queue
.awaiting
.push_back(("q2".into(), "second".into(), Vec::new()));
queue.close(&sink, "the session ended");
// Named rather than counted, because these never reached the
server/src/session/driver.rs
+18
View File
@@ -81,6 +81,16 @@ pub enum Event {
#[serde(default, skip_serializing_if = "Option::is_none")]
id: Option<String>,
text: String,
/// What was attached to it, by the ref the files route serves.
///
/// On the message rather than beside it. These used to be their own
/// `Image` events emitted just before, which drew a person's
/// screenshot as a row of its own floating above the bubble that
/// sent it -- and left the phone to decide, from nothing but
/// adjacency, which message an image belonged to. Belonging is not
/// something to infer when the sender knew.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
images: Vec<ImageRef>,
},
/// A message accepted from the phone that the session cannot read yet.
///
@@ -98,6 +108,11 @@ pub enum Event {
MessageQueued {
id: String,
text: String,
/// Carried for the same reason [`Event::UserMessage`] carries it,
/// and it matters more here: a waiting message is on screen for as
/// long as the turn runs, so its attachment has nowhere else to be.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
images: Vec<ImageRef>,
},
/// A driver has taken one of the user's messages and started reading
/// it. The manager turns this into the `UserMessage` above, so it
@@ -114,6 +129,9 @@ pub enum Event {
/// waited. Carried through onto the `UserMessage`.
id: Option<String>,
text: String,
/// Carried through onto the `UserMessage` with everything else.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
images: Vec<ImageRef>,
},
/// Streaming assistant text; the phone renders the concatenation as
/// markdown.
server/src/session/echo.rs
+29 -8
View File
@@ -85,7 +85,7 @@ pub struct EchoDriver {
busy: Arc<AtomicBool>,
/// Held messages with the id of the `MessageQueued` each one announced,
/// so the announcement can say which waiting bubble it resolves.
queued: Arc<Mutex<Vec<(String, String)>>>,
queued: Arc<Mutex<Vec<Held>>>,
/// Where `/mixed` writes the images it references, which is the same
/// directory the files route serves them from.
session_dir: PathBuf,
@@ -214,7 +214,7 @@ impl EchoDriver {
/// transcript, and a command is not -- the manager has already
/// recorded that one was sent, and saying so twice drew the same
/// line in both colours.
fn handle(&self, text: String, _images: Vec<ImageRef>, announce: bool) {
fn handle(&self, text: String, images: Vec<ImageRef>, announce: bool) {
let sink = self.sink.clone();
// Mid-turn messages are held rather than answered, the way a real
@@ -227,9 +227,12 @@ impl EchoDriver {
// an echo session has to produce the same events or the states
// it exists to exercise are not the app's real ones.
let id = super::random_hex();
self.queued.lock().unwrap().push((id.clone(), text.clone()));
self.queued
.lock()
.unwrap()
.push((id.clone(), text.clone(), images.clone()));
if announce {
self.emit(Event::MessageQueued { id, text });
self.emit(Event::MessageQueued { id, text, images });
}
return;
}
@@ -245,6 +248,7 @@ impl EchoDriver {
self.emit(Event::MessageTaken {
id: None,
text: text.clone(),
images: images.clone(),
});
}
self.emit(Event::PeerMessage {
@@ -266,7 +270,11 @@ impl EchoDriver {
// this is the typed path onto it.
if text.trim() == "/compact" {
if announce {
self.emit(Event::MessageTaken { id: None, text });
self.emit(Event::MessageTaken {
id: None,
text,
images,
});
}
self.compact();
return;
@@ -274,7 +282,11 @@ impl EchoDriver {
if text.trim() == "/ask" {
if announce {
self.emit(Event::MessageTaken { id: None, text });
self.emit(Event::MessageTaken {
id: None,
text,
images,
});
}
self.ask_user_question();
return;
@@ -358,6 +370,7 @@ impl EchoDriver {
send(Event::MessageTaken {
id: None,
text: text.clone(),
images: images.clone(),
});
}
send(Event::Status {
@@ -607,13 +620,20 @@ async fn write_beat(sink: &EventSink, session_dir: &Path, beat: usize) {
tokio::time::sleep(Duration::from_millis(120)).await;
}
/// A message written during a turn and waiting for it to end: the id of the
/// `MessageQueued` that announced it, what it said, and what was attached to
/// it. All three, because all three are what the `MessageTaken` at the other
/// end owes -- named rather than written out at each of the four places that
/// mention it.
type Held = (String, String, Vec<ImageRef>);
/// Ending a turn is also when anything held during it is taken up -- the
/// moment a real CLI would have injected it. One place, because a turn has
/// several ways to end (a reply, an interrupt, a compaction) and every one
/// of them owes the same answer.
fn finish_turn(sink: &EventSink, queued: &Mutex<Vec<(String, String)>>, busy: &AtomicBool) {
fn finish_turn(sink: &EventSink, queued: &Mutex<Vec<Held>>, busy: &AtomicBool) {
let held = std::mem::take(&mut *queued.lock().unwrap());
for (id, text) in held {
for (id, text, images) in held {
// Announced before it is answered, in that order: a phone showing
// the message as pending needs the signal that it has been read,
// and the answer is meaningless above a message still drawn as
@@ -621,6 +641,7 @@ fn finish_turn(sink: &EventSink, queued: &Mutex<Vec<(String, String)>>, busy: &A
let _ = sink.send(Event::MessageTaken {
id: Some(id),
text: text.clone(),
images,
});
let _ = sink.send(Event::AssistantText {
delta: format!("\n(taken from the queue) You said: {text}"),
server/src/session/import.rs
+9 -1
View File
@@ -540,7 +540,15 @@ fn push_user(events: &mut Vec<Event>, content: &Value, session_dir: &std::path::
if !text.trim().is_empty() {
// Replayed from the CLI's own file: it was read long ago, so
// there is no waiting bubble for it to resolve.
events.push(Event::UserMessage { id: None, text });
// The images in this record are saved and referenced separately just
// above, because a replayed message's pictures came out of somebody
// else's file rather than out of this app's composer -- there is no
// upload here whose refs could ride on the message.
events.push(Event::UserMessage {
id: None,
text,
images: Vec::new(),
});
}
}
server/src/session/llama.rs
+12
View File
@@ -347,6 +347,9 @@ impl Driver for LlamaDriver {
let _ = sink.send(Event::MessageTaken {
id: None,
text: text.clone(),
// Never any: this driver refuses images above, and saying
// so is what the refusal above is for.
images: Vec::new(),
});
let _ = sink.send(Event::Status {
state: SessionStatus::Running,
@@ -636,6 +639,7 @@ mod tests {
Event::UserMessage {
id: None,
text: "hello".into(),
images: Vec::new(),
},
Event::AssistantText {
delta: "hi ".into(),
@@ -649,6 +653,7 @@ mod tests {
Event::UserMessage {
id: None,
text: "again".into(),
images: Vec::new(),
},
Event::AssistantText {
delta: "yes".into(),
@@ -680,6 +685,7 @@ mod tests {
Event::UserMessage {
id: None,
text: "count".into(),
images: Vec::new(),
},
Event::AssistantText {
delta: "one two".into(),
@@ -705,6 +711,7 @@ mod tests {
Event::UserMessage {
id: None,
text: "hello".into(),
images: Vec::new(),
},
Event::Error {
message: "something went wrong".into(),
@@ -732,6 +739,7 @@ mod tests {
Event::UserMessage {
id: None,
text: "the long expensive conversation".into(),
images: Vec::new(),
},
Event::AssistantText {
delta: "at length".into(),
@@ -740,6 +748,7 @@ mod tests {
Event::UserMessage {
id: None,
text: "a fresh start".into(),
images: Vec::new(),
},
Event::AssistantText {
delta: "cheaply".into(),
@@ -759,16 +768,19 @@ mod tests {
Event::UserMessage {
id: None,
text: "one".into(),
images: Vec::new(),
},
Event::Cleared,
Event::UserMessage {
id: None,
text: "two".into(),
images: Vec::new(),
},
Event::Cleared,
Event::UserMessage {
id: None,
text: "three".into(),
images: Vec::new(),
},
]);
let messages = conversation(&path);
server/src/session/mod.rs
+35 -18
View File
@@ -146,6 +146,13 @@ pub struct SessionInfo {
/// Every token this session has spent, so a phone showing a total does
/// not have to add up a transcript it only holds part of.
pub total_tokens: u64,
/// The longest edge an image should have by the time it gets here, or
/// absent where this provider has no limit -- see
/// [`DriverKind::max_image_edge`]. Absent rather than a large number,
/// because "no limit" and "a limit that happens to be big" are different
/// answers and only one of them stays true.
#[serde(skip_serializing_if = "Option::is_none")]
pub max_image_edge: Option<u32>,
/// Whether this session announces itself -- reported for the same
/// reason `permission_mode` is: a switch that guesses its own position
/// is how you turn something off while believing you are reading it.
@@ -290,17 +297,11 @@ impl LiveSession {
/// turn it waits, and writing it down on the way past would put it
/// above output that happened before the session ever saw it.
pub fn send_message(&self, text: String, images: Vec<ImageRef>) {
// Attachments are the exception, recorded on the way past: they
// are uploaded whether or not the message waits, and the phone
// fetches them by the same ref the files route serves. So a queued
// message's picture appears a little before its text.
for image in &images {
let _ = self.sink.send(Event::Image {
image: image.clone(),
// A person's own attachment belongs to no tool call.
about: None,
});
}
// The attachments ride *on* the message rather than as `Image`
// events emitted just before it. They used to be the latter, which
// drew a person's screenshot as a row floating above the bubble
// that sent it, and left the phone inferring from adjacency which
// message an image went with -- a thing the sender already knew.
self.driver.send_user_message(text, images);
}
@@ -370,7 +371,13 @@ impl LiveSession {
/// `setup_name` is passed in rather than stored: only the manager
/// holds the config, and the label can change under a running session.
fn info(&self, setup_name: &str, imported: bool, keeps_own_transcript: bool) -> SessionInfo {
/// `kind` rather than the facts derived from it: two of this row's
/// fields are answers about the provider's *kind*, and passing them
/// separately meant every caller deriving each one and a third arriving
/// as a third parameter. `None` where the provider has been edited away,
/// which is a session that cannot run -- so both answers are the
/// cautious one rather than a guess.
fn info(&self, setup_name: &str, imported: bool, kind: Option<DriverKind>) -> SessionInfo {
SessionInfo {
id: self.meta.id.clone(),
provider: self.meta.provider.clone(),
@@ -381,8 +388,9 @@ impl LiveSession {
permission_mode: self.shared.permission_mode.lock().unwrap().clone(),
total_tokens: *self.shared.total_tokens.lock().unwrap(),
notify: *self.shared.notify.lock().unwrap(),
max_image_edge: kind.and_then(DriverKind::max_image_edge),
imported,
keeps_own_transcript,
keeps_own_transcript: kind.is_some_and(DriverKind::keeps_own_transcript),
cwd: self.meta.cwd.clone(),
status: *self.shared.status.lock().unwrap(),
last_activity: *self.shared.last_activity.lock().unwrap(),
@@ -706,7 +714,7 @@ impl SessionManager {
Some(session) => session.info(
label_of(&inner.config, &meta.setup),
import::read_cursor(&self.data_dir.join(&meta.id)).is_some(),
keeps_own_transcript(&inner.config, &meta.setup, &meta.provider),
kind_of(&inner.config, &meta.setup, &meta.provider),
),
None => SessionInfo {
id: meta.id.clone(),
@@ -717,6 +725,8 @@ impl SessionManager {
model: meta.model.clone(),
permission_mode: meta.permission_mode.clone(),
total_tokens: 0,
max_image_edge: kind_of(&inner.config, &meta.setup, &meta.provider)
.and_then(DriverKind::max_image_edge),
notify: meta.notify,
imported: import::read_cursor(&self.data_dir.join(&meta.id)).is_some(),
keeps_own_transcript: keeps_own_transcript(
@@ -853,7 +863,7 @@ impl SessionManager {
let info = session.info(
&setup.name,
import::read_cursor(&self.data_dir.join(&id)).is_some(),
provider.kind.keeps_own_transcript(),
Some(provider.kind),
);
inner.live.insert(id, session);
Ok(info)
@@ -1057,10 +1067,17 @@ fn label_of<'a>(config: &'a Config, id: &'a str) -> &'a str {
/// "this can be brought back" on no evidence is the answer that loses
/// somebody's conversation.
fn keeps_own_transcript(config: &Config, setup: &str, provider: &str) -> bool {
kind_of(config, setup, provider).is_some_and(DriverKind::keeps_own_transcript)
}
/// What a session's provider is, for the questions answered by its *kind*
/// rather than by its name. `None` for a provider that has been edited away,
/// which is a session that cannot run at all.
fn kind_of(config: &Config, setup: &str, provider: &str) -> Option<DriverKind> {
config
.setup(setup)
.and_then(|setup| setup.providers.iter().find(|it| it.name == provider))
.is_some_and(|provider| provider.kind.keeps_own_transcript())
.map(|provider| provider.kind)
}
/// Names for a failure message: what there is, so the reader can see what
@@ -1363,7 +1380,7 @@ async fn pump(
// message here rather than being carried alongside it. One rule
// for where a user's message sits: where the session read it.
let event = match event {
Event::MessageTaken { id, text } => Event::UserMessage { id, text },
Event::MessageTaken { id, text, images } => Event::UserMessage { id, text, images },
// The running total is the pump's to keep, for the reason the
// field gives: a driver knows what its own turn cost and
// nothing else does. Added here rather than at each driver so
@@ -1606,7 +1623,7 @@ mod tests {
assert_eq!(first.session_id, info.id);
// The title travels with it, because the phone may have no screen
// open to look one up on.
assert_eq!(first.title, session.info("m", false, false).title);
assert_eq!(first.title, session.info("m", false, None).title);
manager.set_session_notify(&info.id, false).expect("off");
// Subscribed before the message, or the turn can finish in the gap
server/src/session/transcript.rs
+1
View File
@@ -358,6 +358,7 @@ mod tests {
Event::UserMessage {
id: None,
text: "hi".into(),
images: Vec::new(),
},
text("hello"),
Event::ToolStart {