iris/ai-app
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of git.arirex.me:iris/ai-app

Browse Source
This commit is contained in:
iris committed 2026-08-30 00:58:04 -04:00
2 parents ad791a0d84 da65c1571f
commit 03a8d7d3d3
16 files changed
+529 -70
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt
+10
View File
@@ -153,6 +153,15 @@ data class SessionSummary(
* Every token this session has spent, as the server counts it -- see `SessionEvent.UsageDelta`.
*/
val totalTokens: Long,
/**
* The longest edge an image should have when it reaches this session, or null where the
* provider has no limit.
*
* Null and "a big number" are different answers, and only the first stays true: a provider that
* does not care about size should not be given a threshold this app invented. Decided by the
* server because that is where a provider's kind is known -- see `uploadPickedImage`.
*/
val maxImageEdge: Int?,
val status: String,
val lastActivity: Double,
)
@@ -170,6 +179,7 @@ private fun parseSession(session: JSONObject) =
imported = session.optBoolean("imported", false),
notify = session.optBoolean("notify", true),
totalTokens = session.optLong("totalTokens", 0),
maxImageEdge = session.optInt("maxImageEdge", 0).takeIf { it > 0 },
status = session.getString("status"),
lastActivity = session.getDouble("lastActivity"),
)
app/androidApp/src/main/kotlin/com/example/aiapp/Attachments.kt
+104
View File
@@ -0,0 +1,104 @@
package com.example.aiapp
import android.content.Context
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.graphics.Matrix
import android.media.ExifInterface
import android.net.Uri
import java.io.ByteArrayOutputStream
import kotlin.math.max
/**
* Getting a picked photo to a session, at a size the session can actually take.
*
* A phone camera produces twelve megapixels and several megabytes. The Claude API resizes anything
* larger than 1568px on its long edge before looking at it and refuses images past a much higher
* bound outright, so a photo sent straight off the camera roll was uploaded whole over the tunnel
* to be either thrown away or rejected -- which is what "sending an image is broken" was.
*
* Shrunk here rather than on the backend, so the bytes that never mattered are never sent: the
* expensive part of this on a phone is the upload, not the decode. What the limit *is* comes from
* the server, per session -- see `DriverKind::max_image_edge` -- because that is where a provider's
* requirements are known, and a phone that carried its own copy of them would be a second place to
* update when one changes.
*/
suspend fun uploadPickedImage(
context: Context,
settings: ServerSettings,
sessionId: String,
uri: Uri,
maxEdge: Int?,
): String {
val (bytes, mime) = readForUpload(context, uri, maxEdge)
return uploadAttachment(settings, sessionId, bytes, mime)
}
/**
* The bytes to upload and what they are, scaled down only if they need to be.
*
* An image already inside the limit is uploaded exactly as it came, rather than decoded and
* re-encoded to the same size: a round trip through JPEG loses a little every time, and there is
* nothing to gain from it. This is also the path a provider with no limit always takes.
*/
private fun readForUpload(context: Context, uri: Uri, maxEdge: Int?): Pair<ByteArray, String> {
val resolver = context.contentResolver
val mime = resolver.getType(uri) ?: "image/jpeg"
val original =
resolver.openInputStream(uri)?.use { it.readBytes() }
?: throw ApiException("couldn't read the picked image")
if (maxEdge == null) return original to mime
val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true }
BitmapFactory.decodeByteArray(original, 0, original.size, bounds)
val longest = max(bounds.outWidth, bounds.outHeight)
// outWidth is -1 when the bytes are not an image this device can decode. Sent on untouched:
// this function's job is the size, and refusing something the server might understand is a
// decision it has no business making.
if (longest <= 0 || longest <= maxEdge) return original to mime
// Powers of two first, which is all the decoder can do, and then the exact scale. Decoding
// the full twelve megapixels only to shrink it is how this runs out of memory on the images
// it most needs to handle.
val decode =
BitmapFactory.Options().apply {
inSampleSize = Integer.highestOneBit(max(1, longest / maxEdge))
}
val decoded =
BitmapFactory.decodeByteArray(original, 0, original.size, decode) ?: return original to mime
val scale = maxEdge.toFloat() / max(decoded.width, decoded.height)
val matrix = Matrix()
if (scale < 1f) matrix.postScale(scale, scale)
// The camera writes which way up the picture is into EXIF rather than rotating the pixels, and
// re-encoding drops the tag -- so a portrait photo would arrive at the model on its side, with
// nothing anywhere saying so. Applied to the same matrix as the scale, so it costs no second
// copy of the bitmap.
matrix.postRotate(exifRotation(original))
val scaled = Bitmap.createBitmap(decoded, 0, 0, decoded.width, decoded.height, matrix, true)
val out = ByteArrayOutputStream()
// JPEG whatever came in: this is a photograph being made smaller, which is what JPEG is for,
// and a PNG of a resampled photo is several times the size for no visible difference.
scaled.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, out)
return out.toByteArray() to "image/jpeg"
}
/** How far to turn the picture so it is the way up it was taken. */
private fun exifRotation(bytes: ByteArray): Float =
try {
when (
ExifInterface(bytes.inputStream())
.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL)
) {
ExifInterface.ORIENTATION_ROTATE_90 -> 90f
ExifInterface.ORIENTATION_ROTATE_180 -> 180f
ExifInterface.ORIENTATION_ROTATE_270 -> 270f
else -> 0f
}
} catch (_: java.io.IOException) {
// No EXIF, or none this can read. Upright is the assumption every
// image without the tag is displayed under anyway.
0f
}
/** High enough that resampling is what the reader notices, not the encoder. */
private const val JPEG_QUALITY = 90
app/androidApp/src/main/kotlin/com/example/aiapp/Events.kt
+28 -2
View File
@@ -29,6 +29,15 @@ sealed class SessionEvent {
* bubbles and clearing whichever one matched first would leave the wrong one on screen.
*/
val id: String?,
/**
* What was attached to it, by the ref the files route serves.
*
* On the message rather than beside it: these arrived as separate image events until
* 2026-08-30, which drew somebody's screenshot as a row floating above the bubble that sent
* it, and left this app deciding from adjacency alone which message an image went with --
* something the sender knew and could simply have said.
*/
val images: List<String>,
) : SessionEvent()
/**
@@ -41,7 +50,8 @@ sealed class SessionEvent {
* Resolved by the [UserMessage] carrying the same id, exactly as [CommandQueued] is resolved by
* [CommandSent].
*/
data class MessageQueued(val id: String, val text: String) : SessionEvent()
data class MessageQueued(val id: String, val text: String, val images: List<String>) :
SessionEvent()
data class AssistantText(val delta: String) : SessionEvent()
@@ -144,6 +154,17 @@ sealed class SessionEvent {
data class Unknown(val type: String) : SessionEvent()
}
/**
* A JSON array of strings under [name], empty when the field is absent.
*
* Absent is the ordinary case -- most messages carry no attachment, and the server omits the field
* rather than sending an empty list -- so this is the shape every caller wants.
*/
private fun JSONObject.stringList(name: String): List<String> {
val array = optJSONArray(name) ?: return emptyList()
return (0 until array.length()).map { array.getString(it) }
}
fun parseSeqEvent(json: String): SeqEvent {
val body = JSONObject(json)
val event =
@@ -152,9 +173,14 @@ fun parseSeqEvent(json: String): SeqEvent {
SessionEvent.UserMessage(
body.getString("text"),
body.optString("id").ifEmpty { null },
body.stringList("images"),
)
"messageQueued" ->
SessionEvent.MessageQueued(body.getString("id"), body.getString("text"))
SessionEvent.MessageQueued(
body.getString("id"),
body.getString("text"),
body.stringList("images"),
)
"assistantText" -> SessionEvent.AssistantText(body.getString("delta"))
"toolStart" ->
SessionEvent.ToolStart(
app/androidApp/src/main/kotlin/com/example/aiapp/NerdIcons.kt
+3
View File
@@ -64,6 +64,9 @@ val STOP_GLYPH = glyph(0xF04DB)
*/
val QUEUE_GLYPH = glyph(0xF1163)
/** `md-close` -- take this off again: an attachment picked and not wanted. */
val CLOSE_GLYPH = glyph(0xF0156)
/** `md-arrow_left` -- back one level, to whatever this was opened from. */
val BACK_GLYPH = glyph(0xF004D)
app/androidApp/src/main/kotlin/com/example/aiapp/PendingAttachments.kt
+145
View File
@@ -0,0 +1,145 @@
package com.example.aiapp
import android.graphics.BitmapFactory
import androidx.compose.foundation.Image
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.clickable
import androidx.compose.foundation.horizontalScroll
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.shape.CircleShape
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.ImageBitmap
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.layout.ContentScale
import androidx.compose.ui.semantics.contentDescription
import androidx.compose.ui.semantics.semantics
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
/**
* What is about to be sent, directly above the box it will be sent from.
*
* The count on the "+" button was the whole of what said an image was attached, so the only way to
* find out *which* image was to send it. A control belongs with the thing it acts on, and what
* these are attached to is the message being typed -- which is why they sit here rather than
* anywhere else on the screen.
*
* Scrolls sideways rather than wrapping or shrinking: the row keeps one thumbnail size whatever is
* in it, so four attachments look like four of the same thing rather than four smaller ones.
*/
@Composable
fun PendingAttachments(
settings: ServerSettings,
sessionId: String,
refs: List<String>,
onRemove: (String) -> Unit,
modifier: Modifier = Modifier,
) {
if (refs.isEmpty()) return
Row(
modifier = modifier.horizontalScroll(rememberScrollState()).padding(bottom = 8.dp),
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
refs.forEach { ref -> PendingThumbnail(settings, sessionId, ref) { onRemove(ref) } }
}
}
/**
* One attachment, square, tap to take it back off.
*
* Removal is here because there is nowhere else it could be: an image picked by mistake could
* otherwise only be dealt with by sending it. The whole thumbnail is the target rather than a
* corner cross -- a cross small enough to sit on a 64dp square is smaller than a fingertip -- and
* the label is what says so, since nothing about the picture does.
*/
@Composable
private fun PendingThumbnail(
settings: ServerSettings,
sessionId: String,
ref: String,
onRemove: () -> Unit,
) {
var bitmap by remember(ref) { mutableStateOf<ImageBitmap?>(null) }
var failed by remember(ref) { mutableStateOf(false) }
LaunchedEffect(ref) {
try {
val bytes = withContext(Dispatchers.IO) { fetchSessionFile(settings, sessionId, ref) }
bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)?.asImageBitmap()
failed = bitmap == null
} catch (_: ApiException) {
failed = true
}
}
val shape = RoundedCornerShape(8.dp)
Box(
Modifier.size(THUMBNAIL)
.clip(shape)
// An outline as well as a fill. Most of what gets attached here is a screenshot of a
// dark app, and cropped to a square its middle is often near-black -- against this
// background the tile then had no edge at all, and the only thing saying an image was
// attached was the cross drawn on top of nothing.
.border(1.dp, MaterialTheme.colorScheme.outlineVariant, shape)
// Behind the picture as well as under a missing one, so the tile is a tile before
// anything has arrived to fill it.
.background(MaterialTheme.colorScheme.surfaceVariant)
.clickable(onClick = onRemove)
.semantics { contentDescription = "Attached image, tap to remove" },
contentAlignment = Alignment.Center,
) {
when (val image = bitmap) {
// The two are told apart for the same reason the transcript's images are: one of them
// is worth waiting for and the other never resolves.
null ->
Text(
if (failed) "!" else "…",
style = MaterialTheme.typography.bodyLarge,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
else ->
Image(
bitmap = image,
contentDescription = null,
contentScale = ContentScale.Crop,
modifier = Modifier.size(THUMBNAIL),
)
}
// The whole square removes it, and this only says so. A cross small enough to sit in
// the corner of a 64dp thumbnail is smaller than a fingertip, so making it the target
// would be a control drawn at a size nobody can hit.
//
// The disc is sized here and the mark centred inside it, rather than the glyph being
// aligned directly: a glyph's box is wider than the cross it draws, so aligning the box
// to the corner hung the visible mark over the edge and put its backing somewhere the
// eye reads as a second, misplaced square.
Box(
Modifier.align(Alignment.TopEnd)
.padding(2.dp)
.size(20.dp)
.background(MaterialTheme.colorScheme.surface.copy(alpha = 0.75f), CircleShape),
contentAlignment = Alignment.Center,
) {
Glyph(CLOSE_GLYPH, colour = MaterialTheme.colorScheme.onSurface, size = 12.sp)
}
}
}
private val THUMBNAIL = 64.dp
app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt
+78 -25
View File
@@ -99,7 +99,12 @@ sealed class TranscriptItem {
*/
abstract val seq: Long
data class UserMsg(override val seq: Long, val text: String) : TranscriptItem()
data class UserMsg(
override val seq: Long,
val text: String,
/** Refs of what was attached, drawn inside the bubble. */
val images: List<String> = emptyList(),
) : TranscriptItem()
data class AssistantMsg(override val seq: Long, val text: String) : TranscriptItem()
@@ -311,7 +316,8 @@ private fun adoptRun(
fun foldEvent(items: List<TranscriptItem>, entry: SeqEvent): List<TranscriptItem> =
when (val event = entry.event) {
is SessionEvent.UserMessage -> items + TranscriptItem.UserMsg(entry.seq, event.text)
is SessionEvent.UserMessage ->
items + TranscriptItem.UserMsg(entry.seq, event.text, event.images)
is SessionEvent.AssistantText -> {
// Deltas accumulate into the message they're streaming, which keeps the seq of the
// first of them: a row whose identity changed with every delta would be a new row on
@@ -517,7 +523,7 @@ fun SessionScreen(
// them. From the event stream rather than from what this screen sent, so they are still here
// after leaving the session or restarting the app -- and so a message sent from another device
// is drawn waiting on this one too.
var queued by remember { mutableStateOf(listOf<Pair<String, String>>()) }
var queued by remember { mutableStateOf(listOf<QueuedMessage>()) }
// Commands the session has been asked to run and cannot yet, by the id that will resolve
// them. From the server rather than from this screen, so a rename sent from the settings
// screen -- or from another device -- is drawn waiting here too.
@@ -577,10 +583,10 @@ fun SessionScreen(
// Waiting, then read. Matched by id: the same message sent twice is two
// bubbles, and clearing by text would take away whichever matched first.
if (event is SessionEvent.MessageQueued) {
queued = queued + (event.id to event.text)
queued = queued + QueuedMessage(event.id, event.text, event.images)
}
if (event is SessionEvent.UserMessage) {
queued = queued.filterNot { it.first == event.id }
queued = queued.filterNot { it.id == event.id }
}
// Waiting, then gone: a command leaves this list when the session takes it,
// and the row it becomes is added by `foldEvent` in the same pass.
@@ -924,12 +930,16 @@ fun SessionScreen(
try {
val id =
withContext(Dispatchers.IO) {
val bytes =
context.contentResolver.openInputStream(uri)?.use {
it.readBytes()
} ?: throw ApiException("couldn't read the picked image")
val mime = context.contentResolver.getType(uri) ?: "image/jpeg"
uploadAttachment(settings, summary.id, bytes, mime)
// Shrunk to what this session's provider takes before it is
// uploaded, so a twelve-megapixel photo does not cross the tunnel
// to be rejected at the far end -- see `uploadPickedImage`.
uploadPickedImage(
context,
settings,
summary.id,
uri,
summary.maxImageEdge,
)
}
pendingAttachments = pendingAttachments + id
actionError = null
@@ -1042,7 +1052,15 @@ fun SessionScreen(
waitingCommands.forEach { (_, text) ->
CommandBubble(text, waiting = true)
}
queued.forEach { (_, text) -> UserBubble(text, pending = true) }
queued.forEach { waiting ->
UserBubble(
settings = settings,
sessionId = summary.id,
text = waiting.text,
images = waiting.images,
pending = true,
)
}
}
}
}
@@ -1084,7 +1102,13 @@ fun SessionScreen(
)
is TranscriptRow.Single ->
when (val item = row.item) {
is TranscriptItem.UserMsg -> UserBubble(item.text)
is TranscriptItem.UserMsg ->
UserBubble(
settings = settings,
sessionId = summary.id,
text = item.text,
images = item.images,
)
is TranscriptItem.AssistantMsg -> AssistantMessage(item.text)
is TranscriptItem.ToolRun ->
ToolCard(
@@ -1210,6 +1234,14 @@ fun SessionScreen(
// put the full width behind three controls, so the thing being
// typed into was the narrowest thing on the row.
Column(Modifier.fillMaxWidth().padding(8.dp)) {
// Directly above the box they will be sent from, so what is attached is visible
// rather than counted: the "+2" on the button below said how many and never which.
PendingAttachments(
settings = settings,
sessionId = summary.id,
refs = pendingAttachments,
onRemove = { pendingAttachments = pendingAttachments - it },
)
OutlinedTextField(
value = input,
onValueChange = {
@@ -1217,9 +1249,9 @@ fun SessionScreen(
saveDraft(context, summary.id, it)
},
modifier = Modifier.fillMaxWidth(),
placeholder = {
Text(if (pendingAttachments.isEmpty()) "Message" else "Message (+image)")
},
// No longer "(+image)": the images are on screen above this, and a placeholder
// saying so said it in words beside the thing itself.
placeholder = { Text("Message") },
maxLines = 4,
)
Row(
@@ -1235,7 +1267,8 @@ fun SessionScreen(
)
}
) {
Text(if (pendingAttachments.isEmpty()) "+" else "+${pendingAttachments.size}")
// Just "+" now. The count was standing in for showing them.
Text("+")
}
// The settings share what is left after the actions have
// taken what they need. A Row hands out intrinsic widths in
@@ -1323,7 +1356,13 @@ private fun sendLabel(running: Boolean) = if (running) "Queue" else "Send"
* bitmap is remembered per ref, so scrolling doesn't refetch.
*/
@Composable
private fun UserBubble(text: String, pending: Boolean = false) {
private fun UserBubble(
settings: ServerSettings,
sessionId: String,
text: String,
images: List<String> = emptyList(),
pending: Boolean = false,
) {
Box(Modifier.fillMaxWidth()) {
Card(
// A message the session has not read yet is drawn quieter than
@@ -1338,17 +1377,31 @@ private fun UserBubble(text: String, pending: Boolean = false) {
),
modifier = Modifier.align(Alignment.CenterEnd).padding(start = 48.dp),
) {
Text(
text,
modifier = Modifier.padding(12.dp),
color =
if (pending) MaterialTheme.colorScheme.onSurfaceVariant
else MaterialTheme.colorScheme.onPrimaryContainer,
)
Column(Modifier.padding(12.dp)) {
// Above the words, in the order they were composed: the picture was attached
// before the sentence about it was typed, and it is what the sentence refers to.
images.forEach { ref ->
SessionImage(settings, sessionId, ref)
Spacer(Modifier.height(4.dp))
}
// A message can be nothing but an attachment, and an empty line under a picture
// is a bubble with a gap in it for a sentence nobody wrote.
if (text.isNotEmpty()) {
Text(
text,
color =
if (pending) MaterialTheme.colorScheme.onSurfaceVariant
else MaterialTheme.colorScheme.onPrimaryContainer,
)
}
}
}
}
}
/** A message the server has accepted and the session has not read yet. */
private data class QueuedMessage(val id: String, val text: String, val images: List<String>)
/**
* Collapsed by default: name plus a spinner while running, expandable to the input and output. The
* spinner-while-unfinished is exactly "ToolStart with no matching ToolEnd yet".
app/androidApp/src/main/res/font/nerd_icons.ttf
BIN
View File
Binary file not shown.