Phase 2 complete: images both ways

Inbound: POST /sessions/{id}/attachments stores a picked photo under the
session; message attachmentIds become base64 image blocks in the
stream-json user message (verified live: an uploaded red PNG answered
"Red."). Outbound: image parts in tool results are decoded into the
session's files/ dir and referenced by Image events -- the transcript
stays lean -- and GET /sessions/{id}/files/{ref} serves them (verified
via the Read tool round-tripping the same PNG). The app grows an attach
button (system photo picker, upload-on-pick) and renders Image events
inline with an authenticated pinned fetch. Sent attachments are echoed
into the transcript as Image events so every device shows them.

Attachments and files are addressed under their session (a deviation
from PLAN.md's original bare /attachments -- recorded there) so their
lifecycle is the session directory's: deleting the session is still the
complete path out.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017xn8nHw1tw1R6PtiY1eEtw
This commit is contained in:
irisandClaude Fable 5 committed 2026-08-24 21:40:25 -04:00
1 parent 95d389e2b8
commit f2430671a2
7 files changed
+389 -55

No files matched your search

@@ -30,6 +30,8 @@ fun <T> requestFromServer(
path: String, path: String,
method: String = "GET", method: String = "GET",
jsonBody: String? = null, jsonBody: String? = null,
/** Raw request body as content-type to bytes -- the upload path. */
binaryBody: Pair<String, ByteArray>? = null,
readTimeoutMs: Int = 5000, readTimeoutMs: Int = 5000,
readBody: (HttpURLConnection) -> T, readBody: (HttpURLConnection) -> T,
): T { ): T {
@@ -44,6 +46,10 @@ fun <T> requestFromServer(
connection.doOutput = true connection.doOutput = true
connection.setRequestProperty("Content-Type", "application/json") connection.setRequestProperty("Content-Type", "application/json")
connection.outputStream.use { it.write(jsonBody.encodeToByteArray()) } connection.outputStream.use { it.write(jsonBody.encodeToByteArray()) }
} else if (binaryBody != null) {
connection.doOutput = true
connection.setRequestProperty("Content-Type", binaryBody.first)
connection.outputStream.use { it.write(binaryBody.second) }
} }
if (connection.responseCode !in 200..299) { if (connection.responseCode !in 200..299) {
val detail = connection.errorStream?.bufferedReader()?.readText()?.trim() val detail = connection.errorStream?.bufferedReader()?.readText()?.trim()
@@ -141,15 +147,54 @@ fun spawnSession(
) )
} }
fun sendMessage(settings: ServerSettings, sessionId: String, text: String) { fun sendMessage(
settings: ServerSettings,
sessionId: String,
text: String,
attachmentIds: List<String> = emptyList(),
) {
requestFromServer( requestFromServer(
settings, settings,
"/sessions/$sessionId/message", "/sessions/$sessionId/message",
method = "POST", method = "POST",
jsonBody = JSONObject().put("text", text).toString(), jsonBody = JSONObject()
.put("text", text)
.put("attachmentIds", JSONArray(attachmentIds))
.toString(),
) {} ) {}
} }
/** Uploads one picked image; the returned id goes into [sendMessage]. */
fun uploadAttachment(
settings: ServerSettings,
sessionId: String,
bytes: ByteArray,
mime: String,
): String {
val boundary = "----aiapp-${System.currentTimeMillis()}"
val head = (
"--$boundary\r\n" +
"Content-Disposition: form-data; name=\"file\"; filename=\"image\"\r\n" +
"Content-Type: $mime\r\n\r\n"
).encodeToByteArray()
val tail = "\r\n--$boundary--\r\n".encodeToByteArray()
return requestFromServer(
settings,
"/sessions/$sessionId/attachments",
method = "POST",
binaryBody = "multipart/form-data; boundary=$boundary" to (head + bytes + tail),
readTimeoutMs = 60000,
) { connection ->
JSONObject(connection.inputStream.bufferedReader().readText()).getString("id")
}
}
/** Fetches an image the transcript references (produced or uploaded). */
fun fetchSessionFile(settings: ServerSettings, sessionId: String, name: String): ByteArray =
requestFromServer(settings, "/sessions/$sessionId/files/$name", readTimeoutMs = 30000) {
it.inputStream.readBytes()
}
fun answerQuestion(settings: ServerSettings, sessionId: String, questionId: String, answer: String) { fun answerQuestion(settings: ServerSettings, sessionId: String, questionId: String, answer: String) {
requestFromServer( requestFromServer(
settings, settings,
@@ -33,6 +33,7 @@ import androidx.compose.runtime.rememberCoroutineScope
import androidx.compose.runtime.setValue import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.asImageBitmap
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
@@ -66,7 +67,9 @@ sealed class TranscriptItem {
val answer: String?, val answer: String?,
) : TranscriptItem() ) : TranscriptItem()
data class ErrorMsg(val message: String) : TranscriptItem() data class ErrorMsg(val message: String) : TranscriptItem()
/** Placeholder row for events this build can't render (images, newer kinds). */ /** An image by server-side ref, fetched from the session's files route. */
data class ImageItem(val ref: String) : TranscriptItem()
/** Placeholder row for events this build can't render (newer kinds). */
data class Note(val text: String) : TranscriptItem() data class Note(val text: String) : TranscriptItem()
} }
@@ -94,7 +97,7 @@ fun foldEvent(items: List<TranscriptItem>, event: SessionEvent): List<Transcript
if (it is TranscriptItem.QuestionCard && it.id == event.id) it.copy(answer = event.answer) else it if (it is TranscriptItem.QuestionCard && it.id == event.id) it.copy(answer = event.answer) else it
} }
is SessionEvent.Error -> items + TranscriptItem.ErrorMsg(event.message) is SessionEvent.Error -> items + TranscriptItem.ErrorMsg(event.message)
is SessionEvent.Image -> items + TranscriptItem.Note("[image ${event.ref}]") is SessionEvent.Image -> items + TranscriptItem.ImageItem(event.ref)
is SessionEvent.Unknown -> items + TranscriptItem.Note("[${event.type}]") is SessionEvent.Unknown -> items + TranscriptItem.Note("[${event.type}]")
// Screen-level state, not transcript rows -- see SessionScreen. // Screen-level state, not transcript rows -- see SessionScreen.
is SessionEvent.Status, is SessionEvent.UsageDelta -> items is SessionEvent.Status, is SessionEvent.UsageDelta -> items
@@ -118,6 +121,9 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: ()
var actionError by remember { mutableStateOf<String?>(null) } var actionError by remember { mutableStateOf<String?>(null) }
var input by remember { mutableStateOf("") } var input by remember { mutableStateOf("") }
var expandedTools by remember { mutableStateOf(setOf<String>()) } var expandedTools by remember { mutableStateOf(setOf<String>()) }
// Uploaded-but-not-yet-sent attachment ids; sent with the next message.
var pendingAttachments by remember { mutableStateOf(listOf<String>()) }
val context = androidx.compose.ui.platform.LocalContext.current
// The resume cursor, written from the stream's IO thread. // The resume cursor, written from the stream's IO thread.
val lastSeq = remember { AtomicLong(0) } val lastSeq = remember { AtomicLong(0) }
val activeStream = remember { AtomicReference<EventStream?>(null) } val activeStream = remember { AtomicReference<EventStream?>(null) }
@@ -177,9 +183,35 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: ()
fun send() { fun send() {
val text = input.trim() val text = input.trim()
if (text.isEmpty()) return val attachments = pendingAttachments
if (text.isEmpty() && attachments.isEmpty()) return
input = "" input = ""
act { sendMessage(settings, summary.id, text) } pendingAttachments = emptyList()
act { sendMessage(settings, summary.id, text, attachments) }
}
// The system photo picker; the image uploads as soon as it's chosen,
// so Send only has ids to reference.
val pickImage = androidx.activity.compose.rememberLauncherForActivityResult(
androidx.activity.result.contract.ActivityResultContracts.PickVisualMedia(),
) { uri ->
if (uri != null) {
scope.launch {
try {
val id = withContext(Dispatchers.IO) {
val bytes = context.contentResolver.openInputStream(uri)
?.use { it.readBytes() }
?: throw ApiException("couldn't read the picked image")
val mime = context.contentResolver.getType(uri) ?: "image/jpeg"
uploadAttachment(settings, summary.id, bytes, mime)
}
pendingAttachments = pendingAttachments + id
actionError = null
} catch (e: ApiException) {
actionError = e.message
}
}
}
} }
Column(Modifier.fillMaxSize()) { Column(Modifier.fillMaxSize()) {
@@ -239,6 +271,7 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: ()
color = MaterialTheme.colorScheme.error, color = MaterialTheme.colorScheme.error,
style = MaterialTheme.typography.bodyMedium, style = MaterialTheme.typography.bodyMedium,
) )
is TranscriptItem.ImageItem -> SessionImage(settings, summary.id, item.ref)
is TranscriptItem.Note -> Text( is TranscriptItem.Note -> Text(
item.text, item.text,
style = MaterialTheme.typography.bodySmall, style = MaterialTheme.typography.bodySmall,
@@ -255,11 +288,21 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: ()
verticalAlignment = Alignment.CenterVertically, verticalAlignment = Alignment.CenterVertically,
modifier = Modifier.fillMaxWidth().padding(8.dp), modifier = Modifier.fillMaxWidth().padding(8.dp),
) { ) {
TextButton(onClick = {
pickImage.launch(
androidx.activity.result.PickVisualMediaRequest(
androidx.activity.result.contract.ActivityResultContracts
.PickVisualMedia.ImageOnly,
),
)
}) {
Text(if (pendingAttachments.isEmpty()) "+" else "+${pendingAttachments.size}")
}
OutlinedTextField( OutlinedTextField(
value = input, value = input,
onValueChange = { input = it }, onValueChange = { input = it },
modifier = Modifier.weight(1f), modifier = Modifier.weight(1f),
placeholder = { Text("Message") }, placeholder = { Text(if (pendingAttachments.isEmpty()) "Message" else "Message (+image)") },
maxLines = 4, maxLines = 4,
) )
Spacer(Modifier.width(8.dp)) Spacer(Modifier.width(8.dp))
@@ -274,6 +317,41 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: ()
} }
} }
/**
* An inline transcript image, fetched (authenticated, pinned) from the
* session's files route. The bitmap is remembered per ref, so scrolling
* doesn't refetch.
*/
@Composable
private fun SessionImage(settings: ServerSettings, sessionId: String, ref: String) {
var bitmap by remember(ref) {
mutableStateOf<androidx.compose.ui.graphics.ImageBitmap?>(null)
}
var failed by remember(ref) { mutableStateOf(false) }
LaunchedEffect(ref) {
try {
val bytes = withContext(Dispatchers.IO) { fetchSessionFile(settings, sessionId, ref) }
bitmap = android.graphics.BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
?.asImageBitmap()
failed = bitmap == null
} catch (e: ApiException) {
failed = true
}
}
when (val image = bitmap) {
null -> Text(
if (failed) "[image $ref unavailable]" else "[loading image…]",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
else -> androidx.compose.foundation.Image(
bitmap = image,
contentDescription = "session image",
modifier = Modifier.fillMaxWidth(),
)
}
}
@Composable @Composable
private fun UserBubble(text: String) { private fun UserBubble(text: String) {
Box(Modifier.fillMaxWidth()) { Box(Modifier.fillMaxWidth()) {
+39
View File
@@ -155,6 +155,7 @@ dependencies = [
"matchit", "matchit",
"memchr", "memchr",
"mime", "mime",
"multer",
"percent-encoding", "percent-encoding",
"pin-project-lite", "pin-project-lite",
"serde_core", "serde_core",
@@ -368,6 +369,15 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
[[package]]
name = "encoding_rs"
version = "0.8.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "equivalent" name = "equivalent"
version = "1.0.2" version = "1.0.2"
@@ -714,6 +724,23 @@ dependencies = [
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
[[package]]
name = "multer"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
dependencies = [
"bytes",
"encoding_rs",
"futures-util",
"http",
"httparse",
"memchr",
"mime",
"spin",
"version_check",
]
[[package]] [[package]]
name = "nu-ansi-term" name = "nu-ansi-term"
version = "0.50.3" version = "0.50.3"
@@ -1015,6 +1042,12 @@ dependencies = [
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
[[package]]
name = "spin"
version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.11.1" version = "0.11.1"
@@ -1279,6 +1312,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.11.1+wasi-snapshot-preview1" version = "0.11.1+wasi-snapshot-preview1"
+1 -1
View File
@@ -8,7 +8,7 @@ name = "ai-server"
path = "src/main.rs" path = "src/main.rs"
[dependencies] [dependencies]
axum = { version = "0.8", features = ["json"] } axum = { version = "0.8", features = ["json", "multipart"] }
axum-server = { version = "0.8", features = ["tls-rustls"] } axum-server = { version = "0.8", features = ["tls-rustls"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "sync", "time", "process", "io-util"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "sync", "time", "process", "io-util"] }
tokio-stream = "0.1" tokio-stream = "0.1"
+61 -2
View File
@@ -11,11 +11,13 @@
//! POST /sessions/{id}/interrupt //! POST /sessions/{id}/interrupt
//! POST /sessions/{id}/model {model} //! POST /sessions/{id}/model {model}
//! POST /sessions/{id}/compact //! POST /sessions/{id}/compact
//! POST /sessions/{id}/attachments multipart image upload -> {id}, referenced by /message
//! GET /sessions/{id}/files/{name} images the session produced or was sent
//! DELETE /sessions/{id} kill process, delete transcript + files //! DELETE /sessions/{id} kill process, delete transcript + files
//! ``` //! ```
//! //!
//! Later phases add: `POST /attachments`, `GET /files/{session}/{id}`, //! Later phases add: `GET /usage`, `GET|PUT /hosts` and `/models` -- see
//! `GET /usage`, `GET|PUT /hosts` and `/models` -- see PLAN.md's table. //! PLAN.md's table.
//! //!
//! Everything here works purely in the common event model; nothing may //! Everything here works purely in the common event model; nothing may
//! branch on the session kind (that's what drivers are for). //! branch on the session kind (that's what drivers are for).
@@ -48,6 +50,10 @@ pub fn router(manager: Arc<SessionManager>) -> Router {
.route("/sessions/{id}/interrupt", post(interrupt)) .route("/sessions/{id}/interrupt", post(interrupt))
.route("/sessions/{id}/model", post(set_model)) .route("/sessions/{id}/model", post(set_model))
.route("/sessions/{id}/compact", post(compact)) .route("/sessions/{id}/compact", post(compact))
.route("/sessions/{id}/attachments", post(upload_attachment))
.route("/sessions/{id}/files/{name}", get(serve_file))
// Phone photos overflow axum's 2 MB default body cap.
.layer(axum::extract::DefaultBodyLimit::max(32 * 1024 * 1024))
// An explicit fallback so the auth middleware (layered around the // An explicit fallback so the auth middleware (layered around the
// whole router in main.rs) also covers unknown paths -- a scanner // whole router in main.rs) also covers unknown paths -- a scanner
// gets the same 401 everywhere, never a route map. // gets the same 401 everywhere, never a route map.
@@ -202,6 +208,59 @@ async fn compact(
Ok(StatusCode::NO_CONTENT) Ok(StatusCode::NO_CONTENT)
} }
/// Accepts one image (any multipart field) and stores it under the
/// session; the returned id goes into a later `/message`'s attachmentIds.
async fn upload_attachment(
State(manager): State<Arc<SessionManager>>,
UrlPath(id): UrlPath<String>,
mut multipart: axum::extract::Multipart,
) -> Result<axum::Json<serde_json::Value>, ApiError> {
let session = lookup(&manager, &id)?;
let field = multipart
.next_field()
.await
.map_err(|err| ApiError::BadRequest(format!("bad upload: {err}")))?
.ok_or_else(|| ApiError::BadRequest("no file in the upload".to_string()))?;
let content_type = field.content_type().unwrap_or("image/jpeg").to_string();
let bytes = field
.bytes()
.await
.map_err(|err| ApiError::BadRequest(format!("upload read failed: {err}")))?;
let name = session.save_attachment(&bytes, &content_type).map_err(bad_request)?;
Ok(axum::Json(serde_json::json!({ "id": name })))
}
/// Serves a session's stored images -- both `files/` (produced by tools)
/// and `attachments/` (uploaded from the phone), by the id events and
/// uploads reference.
async fn serve_file(
State(manager): State<Arc<SessionManager>>,
UrlPath((id, name)): UrlPath<(String, String)>,
) -> Result<Response, ApiError> {
// Ids are server-generated hex + extension; anything else (and any
// path separator in particular) is refused, not resolved.
if !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '.') || name.contains("..") {
return Err(ApiError::BadRequest("invalid file id".to_string()));
}
let session = lookup(&manager, &id)?;
let candidates =
[session.dir().join("files").join(&name), session.dir().join("attachments").join(&name)];
let Some(path) = candidates.iter().find(|path| path.is_file()) else {
return Err(ApiError::BadRequest(format!("no file {name} in session {id}")));
};
let bytes = std::fs::read(path).map_err(|err| {
tracing::error!("read {} failed: {err}", path.display());
ApiError::BadRequest("file unreadable".to_string())
})?;
let content_type = match name.rsplit('.').next() {
Some("png") => "image/png",
Some("gif") => "image/gif",
Some("webp") => "image/webp",
_ => "image/jpeg",
};
Ok(([(axum::http::header::CONTENT_TYPE, content_type)], bytes).into_response())
}
#[derive(Deserialize)] #[derive(Deserialize)]
struct EventsQuery { struct EventsQuery {
#[serde(default)] #[serde(default)]
+119 -40
View File
@@ -92,7 +92,7 @@ impl ClaudeDriver {
let stdin = child.stdin.take().expect("piped stdin"); let stdin = child.stdin.take().expect("piped stdin");
let stdout = child.stdout.take().expect("piped stdout"); let stdout = child.stdout.take().expect("piped stdout");
let stderr = child.stderr.take().expect("piped stderr"); let stderr = child.stderr.take().expect("piped stderr");
let state = Arc::new(Mutex::new(Translator::default())); let state = Arc::new(Mutex::new(Translator::new(session_dir.to_path_buf())));
// Writer: everything for the child funnels through one channel so // Writer: everything for the child funnels through one channel so
// driver methods stay sync and writes can't interleave. // driver methods stay sync and writes can't interleave.
@@ -354,15 +354,20 @@ struct PendingRequest {
answers: HashMap<String, String>, answers: HashMap<String, String>,
} }
/// Pure translation state: stream-json lines in, common events out. No /// Translation state: stream-json lines in, common events out. The one
/// I/O, so the whole dialect mapping is unit-testable from recorded lines. /// side effect is saving images a tool result carries into the session
#[derive(Default)] /// dir (they'd bloat the transcript as base64); everything else is pure,
/// so the dialect mapping is unit-testable from recorded lines.
struct Translator { struct Translator {
session_id: Option<String>, session_id: Option<String>,
pending: HashMap<String, PendingRequest>, pending: HashMap<String, PendingRequest>,
session_dir: PathBuf,
} }
impl Translator { impl Translator {
fn new(session_dir: PathBuf) -> Self {
Self { session_id: None, pending: HashMap::new(), session_dir }
}
fn translate(&mut self, message: &Value) -> Vec<Event> { fn translate(&mut self, message: &Value) -> Vec<Event> {
// Events from subagents (Task tool internals) carry a // Events from subagents (Task tool internals) carry a
// parent_tool_use_id; the transcript shows the Task tool's own // parent_tool_use_id; the transcript shows the Task tool's own
@@ -381,7 +386,7 @@ impl Translator {
} }
Some("stream_event") => self.translate_stream_event(&message["event"]), Some("stream_event") => self.translate_stream_event(&message["event"]),
Some("assistant") => self.translate_assistant(&message["message"]), Some("assistant") => self.translate_assistant(&message["message"]),
Some("user") => translate_user(message), Some("user") => self.translate_user(message),
Some("control_request") => self.translate_control_request(message), Some("control_request") => self.translate_control_request(message),
Some("control_response") => { Some("control_response") => {
let response = &message["response"]; let response = &message["response"];
@@ -539,36 +544,77 @@ impl Translator {
} }
} }
/// `user` messages: tool results become ToolEnd (with any images saved impl Translator {
/// out-of-band by the caller -- phase 2b); replayed/synthetic user text is /// `user` messages: tool results become ToolEnd, with any image parts
/// skipped, since the manager already recorded the user's side. /// saved into the session dir and referenced by an Image event (the
fn translate_user(message: &Value) -> Vec<Event> { /// phone fetches them from `/sessions/{id}/files/{ref}`). Replayed and
let Some(content) = message["message"].get("content").and_then(Value::as_array) else { /// synthetic user text is skipped -- the manager already recorded the
return Vec::new(); /// user's side.
}; fn translate_user(&self, message: &Value) -> Vec<Event> {
content let Some(content) = message["message"].get("content").and_then(Value::as_array) else {
.iter() return Vec::new();
.filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_result")) };
.map(|block| { let mut events = Vec::new();
let output = match block.get("content") { for block in content {
Some(Value::String(text)) => text.clone(), if block.get("type").and_then(Value::as_str) != Some("tool_result") {
Some(Value::Array(parts)) => parts continue;
.iter() }
.filter_map(|part| part.get("text").and_then(Value::as_str)) let mut texts = Vec::new();
.collect::<Vec<_>>() match block.get("content") {
.join("\n"), Some(Value::String(text)) => texts.push(text.clone()),
_ => String::new(), Some(Value::Array(parts)) => {
}; for part in parts {
Event::ToolEnd { match part.get("type").and_then(Value::as_str) {
Some("text") => {
if let Some(text) = part.get("text").and_then(Value::as_str) {
texts.push(text.to_string());
}
}
Some("image") => {
if let Some(name) = self.save_image(part) {
events.push(Event::Image { image: name });
}
}
_ => {}
}
}
}
_ => {}
}
events.push(Event::ToolEnd {
id: block id: block
.get("tool_use_id") .get("tool_use_id")
.and_then(Value::as_str) .and_then(Value::as_str)
.unwrap_or_default() .unwrap_or_default()
.to_string(), .to_string(),
output, output: texts.join("\n"),
} });
}) }
.collect() events
}
/// Decodes one base64 image block into `files/` and returns its ref.
fn save_image(&self, part: &Value) -> Option<String> {
let source = part.get("source")?;
let data = source.get("data")?.as_str()?;
use base64::Engine;
let bytes = base64::engine::general_purpose::STANDARD.decode(data).ok()?;
let extension = match source.get("media_type").and_then(Value::as_str) {
Some("image/jpeg") => "jpg",
Some("image/gif") => "gif",
Some("image/webp") => "webp",
_ => "png",
};
let name = format!("{}.{extension}", super::random_hex());
let dir = self.session_dir.join("files");
if let Err(err) =
std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(dir.join(&name), bytes))
{
tracing::error!("couldn't save produced image: {err}");
return None;
}
Some(name)
}
} }
#[cfg(test)] #[cfg(test)]
@@ -584,7 +630,8 @@ mod tests {
#[test] #[test]
fn captures_the_resume_token_from_init() { fn captures_the_resume_token_from_init() {
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines( let events = translate_lines(
&mut translator, &mut translator,
&[r#"{"type":"system","subtype":"init","cwd":"/x","session_id":"5ecf21da-d53f","tools":[],"model":"claude-haiku-4-5-20251001"}"#], &[r#"{"type":"system","subtype":"init","cwd":"/x","session_id":"5ecf21da-d53f","tools":[],"model":"claude-haiku-4-5-20251001"}"#],
@@ -596,7 +643,8 @@ mod tests {
#[test] #[test]
fn streams_text_deltas_and_skips_the_consolidated_copy() { fn streams_text_deltas_and_skips_the_consolidated_copy() {
// Real lines (trimmed) from the 2.1.237 probe. // Real lines (trimmed) from the 2.1.237 probe.
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[ let events = translate_lines(&mut translator, &[
r#"{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Done."}},"session_id":"s","parent_tool_use_id":null}"#, r#"{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Done."}},"session_id":"s","parent_tool_use_id":null}"#,
r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Done."}]},"parent_tool_use_id":null,"session_id":"s"}"#, r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Done."}]},"parent_tool_use_id":null,"session_id":"s"}"#,
@@ -607,7 +655,8 @@ mod tests {
#[test] #[test]
fn tool_use_and_result_become_tool_events() { fn tool_use_and_result_become_tool_events() {
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[ let events = translate_lines(&mut translator, &[
r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"echo probe-ok"}}]},"parent_tool_use_id":null}"#, r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"echo probe-ok"}}]},"parent_tool_use_id":null}"#,
r#"{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01","type":"tool_result","content":"probe-ok","is_error":false}]},"parent_tool_use_id":null}"#, r#"{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01","type":"tool_result","content":"probe-ok","is_error":false}]},"parent_tool_use_id":null}"#,
@@ -624,7 +673,8 @@ mod tests {
#[test] #[test]
fn subagent_events_are_not_duplicated_into_the_transcript() { fn subagent_events_are_not_duplicated_into_the_transcript() {
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[ let events = translate_lines(&mut translator, &[
r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_02","name":"Bash","input":{}}]},"parent_tool_use_id":"toolu_parent"}"#, r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_02","name":"Bash","input":{}}]},"parent_tool_use_id":"toolu_parent"}"#,
]); ]);
@@ -633,7 +683,8 @@ mod tests {
#[test] #[test]
fn a_permission_request_becomes_an_allow_deny_question() { fn a_permission_request_becomes_an_allow_deny_question() {
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[ let events = translate_lines(&mut translator, &[
r#"{"type":"control_request","request_id":"req-1","request":{"subtype":"can_use_tool","tool_name":"Bash","input":{"command":"rm -rf /tmp/x"},"tool_use_id":"toolu_03"}}"#, r#"{"type":"control_request","request_id":"req-1","request":{"subtype":"can_use_tool","tool_name":"Bash","input":{"command":"rm -rf /tmp/x"},"tool_use_id":"toolu_03"}}"#,
]); ]);
@@ -660,7 +711,8 @@ mod tests {
#[test] #[test]
fn denying_a_permission_sends_deny() { fn denying_a_permission_sends_deny() {
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
translate_lines(&mut translator, &[ translate_lines(&mut translator, &[
r#"{"type":"control_request","request_id":"req-2","request":{"subtype":"can_use_tool","tool_name":"Write","input":{"file_path":"/etc/passwd"}}}"#, r#"{"type":"control_request","request_id":"req-2","request":{"subtype":"can_use_tool","tool_name":"Write","input":{"file_path":"/etc/passwd"}}}"#,
]); ]);
@@ -674,7 +726,8 @@ mod tests {
fn ask_user_question_rides_the_same_flow_with_answers_keyed_by_question() { fn ask_user_question_rides_the_same_flow_with_answers_keyed_by_question() {
// The real 2.1.237 shape, verified live: answers go back inside // The real 2.1.237 shape, verified live: answers go back inside
// updatedInput, keyed by the question text. // updatedInput, keyed by the question text.
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[ let events = translate_lines(&mut translator, &[
r#"{"type":"control_request","request_id":"req-3","request":{"subtype":"can_use_tool","tool_name":"AskUserQuestion","input":{"questions":[{"question":"Which color?","header":"Color","options":[{"label":"Red"},{"label":"Blue"}],"multiSelect":false},{"question":"Which size?","header":"Size","options":[{"label":"S"},{"label":"L"}],"multiSelect":false}]},"tool_use_id":"toolu_04","requires_user_interaction":true}}"#, r#"{"type":"control_request","request_id":"req-3","request":{"subtype":"can_use_tool","tool_name":"AskUserQuestion","input":{"questions":[{"question":"Which color?","header":"Color","options":[{"label":"Red"},{"label":"Blue"}],"multiSelect":false},{"question":"Which size?","header":"Size","options":[{"label":"S"},{"label":"L"}],"multiSelect":false}]},"tool_use_id":"toolu_04","requires_user_interaction":true}}"#,
]); ]);
@@ -702,9 +755,33 @@ mod tests {
assert_eq!(updated["questions"][0]["question"], "Which color?"); assert_eq!(updated["questions"][0]["question"], "Which color?");
} }
#[test]
fn images_in_tool_results_are_saved_and_referenced() {
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
// A 1x1 PNG, the smallest real payload worth round-tripping.
let png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
let line = format!(
r#"{{"type":"user","message":{{"role":"user","content":[{{"type":"tool_result","tool_use_id":"toolu_05","content":[{{"type":"text","text":"took a screenshot"}},{{"type":"image","source":{{"type":"base64","media_type":"image/png","data":"{png}"}}}}]}}]}},"parent_tool_use_id":null}}"#
);
let events = translator.translate(&serde_json::from_str(&line).expect("json"));
let Event::Image { image } = &events[0] else {
panic!("expected an image event, got {events:?}");
};
assert!(image.ends_with(".png"));
let saved = dir.path().join("files").join(image);
assert!(saved.is_file(), "image not saved at {}", saved.display());
assert_eq!(events[1], Event::ToolEnd {
id: "toolu_05".to_string(),
output: "took a screenshot".to_string(),
});
}
#[test] #[test]
fn a_turn_result_reports_usage_and_returns_to_idle() { fn a_turn_result_reports_usage_and_returns_to_idle() {
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[ let events = translate_lines(&mut translator, &[
r#"{"type":"result","subtype":"success","is_error":false,"num_turns":2,"stop_reason":"end_turn","session_id":"s","total_cost_usd":0.0149,"usage":{"input_tokens":18,"output_tokens":164}}"#, r#"{"type":"result","subtype":"success","is_error":false,"num_turns":2,"stop_reason":"end_turn","session_id":"s","total_cost_usd":0.0149,"usage":{"input_tokens":18,"output_tokens":164}}"#,
]); ]);
@@ -716,7 +793,8 @@ mod tests {
#[test] #[test]
fn an_error_result_surfaces_the_message() { fn an_error_result_surfaces_the_message() {
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[ let events = translate_lines(&mut translator, &[
r#"{"type":"result","subtype":"error_during_execution","is_error":true,"result":"something broke","usage":{}}"#, r#"{"type":"result","subtype":"error_during_execution","is_error":true,"result":"something broke","usage":{}}"#,
]); ]);
@@ -726,7 +804,8 @@ mod tests {
#[test] #[test]
fn replayed_and_synthetic_user_text_is_skipped() { fn replayed_and_synthetic_user_text_is_skipped() {
let mut translator = Translator::default(); let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[ let events = translate_lines(&mut translator, &[
r#"{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]},"isReplay":true,"parent_tool_use_id":null}"#, r#"{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]},"isReplay":true,"parent_tool_use_id":null}"#,
r#"{"type":"user","message":{"role":"user","content":[{"type":"text","text":"[continue]"}]},"isSynthetic":true,"parent_tool_use_id":null}"#, r#"{"type":"user","message":{"role":"user","content":[{"type":"text","text":"[continue]"}]},"isSynthetic":true,"parent_tool_use_id":null}"#,
+39 -5
View File
@@ -94,6 +94,11 @@ impl LiveSession {
/// driver -- which queues it for injection mid-run rather than at the /// driver -- which queues it for injection mid-run rather than at the
/// end of the turn (the point of the whole app). /// end of the turn (the point of the whole app).
pub fn send_message(&self, text: String, images: Vec<ImageRef>) { pub fn send_message(&self, text: String, images: Vec<ImageRef>) {
// Attachments render in the transcript like any produced image --
// the files route serves uploads by the same ref.
for image in &images {
let _ = self.sink.send(Event::Image { image: image.clone() });
}
let _ = self.sink.send(Event::UserMessage { text: text.clone() }); let _ = self.sink.send(Event::UserMessage { text: text.clone() });
self.driver.send_user_message(text, images); self.driver.send_user_message(text, images);
} }
@@ -122,6 +127,30 @@ impl LiveSession {
&self.transcript_path &self.transcript_path
} }
/// The session's directory (attachments in, produced files out live in
/// `attachments/` and `files/` under it).
pub fn dir(&self) -> &Path {
self.transcript_path.parent().expect("transcript lives in the session dir")
}
/// Stores one uploaded attachment, returning the id `POST /message`
/// references it by. Removed with the session directory on delete --
/// the same path out as everything else in it.
pub fn save_attachment(&self, bytes: &[u8], content_type: &str) -> Result<String> {
let extension = match content_type {
"image/png" => "png",
"image/gif" => "gif",
"image/webp" => "webp",
_ => "jpg",
};
let name = format!("{}.{extension}", random_hex());
let dir = self.dir().join("attachments");
std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?;
std::fs::write(dir.join(&name), bytes)
.with_context(|| format!("write attachment {name}"))?;
Ok(name)
}
fn info(&self) -> SessionInfo { fn info(&self) -> SessionInfo {
SessionInfo { SessionInfo {
id: self.meta.id.clone(), id: self.meta.id.clone(),
@@ -313,13 +342,18 @@ fn default_title(kind: SessionKind) -> String {
} }
/// 8 random bytes, hex -- short enough for a URL, unique enough forever at /// 8 random bytes, hex -- short enough for a URL, unique enough forever at
/// this scale. Still checked against the existing list out of caution. /// this scale.
fn unique_id(config: &Config) -> String { pub fn random_hex() -> String {
use rand::Rng; use rand::Rng;
let mut bytes = [0u8; 8];
rand::rng().fill_bytes(&mut bytes);
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
/// A [`random_hex`] id not already taken -- checked out of caution.
fn unique_id(config: &Config) -> String {
loop { loop {
let mut bytes = [0u8; 8]; let id = random_hex();
rand::rng().fill_bytes(&mut bytes);
let id: String = bytes.iter().map(|b| format!("{b:02x}")).collect();
if !config.sessions.iter().any(|meta| meta.id == id) { if !config.sessions.iter().any(|meta| meta.id == id) {
return id; return id;
} }