diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt index cb2a296..a1d6e50 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Api.kt @@ -30,6 +30,8 @@ fun requestFromServer( path: String, method: String = "GET", jsonBody: String? = null, + /** Raw request body as content-type to bytes -- the upload path. */ + binaryBody: Pair? = null, readTimeoutMs: Int = 5000, readBody: (HttpURLConnection) -> T, ): T { @@ -44,6 +46,10 @@ fun requestFromServer( connection.doOutput = true connection.setRequestProperty("Content-Type", "application/json") connection.outputStream.use { it.write(jsonBody.encodeToByteArray()) } + } else if (binaryBody != null) { + connection.doOutput = true + connection.setRequestProperty("Content-Type", binaryBody.first) + connection.outputStream.use { it.write(binaryBody.second) } } if (connection.responseCode !in 200..299) { val detail = connection.errorStream?.bufferedReader()?.readText()?.trim() @@ -141,15 +147,54 @@ fun spawnSession( ) } -fun sendMessage(settings: ServerSettings, sessionId: String, text: String) { +fun sendMessage( + settings: ServerSettings, + sessionId: String, + text: String, + attachmentIds: List = emptyList(), +) { requestFromServer( settings, "/sessions/$sessionId/message", method = "POST", - jsonBody = JSONObject().put("text", text).toString(), + jsonBody = JSONObject() + .put("text", text) + .put("attachmentIds", JSONArray(attachmentIds)) + .toString(), ) {} } +/** Uploads one picked image; the returned id goes into [sendMessage]. */ +fun uploadAttachment( + settings: ServerSettings, + sessionId: String, + bytes: ByteArray, + mime: String, +): String { + val boundary = "----aiapp-${System.currentTimeMillis()}" + val head = ( + "--$boundary\r\n" + + "Content-Disposition: form-data; name=\"file\"; filename=\"image\"\r\n" + + "Content-Type: $mime\r\n\r\n" + ).encodeToByteArray() + val tail = "\r\n--$boundary--\r\n".encodeToByteArray() + return requestFromServer( + settings, + "/sessions/$sessionId/attachments", + method = "POST", + binaryBody = "multipart/form-data; boundary=$boundary" to (head + bytes + tail), + readTimeoutMs = 60000, + ) { connection -> + JSONObject(connection.inputStream.bufferedReader().readText()).getString("id") + } +} + +/** Fetches an image the transcript references (produced or uploaded). */ +fun fetchSessionFile(settings: ServerSettings, sessionId: String, name: String): ByteArray = + requestFromServer(settings, "/sessions/$sessionId/files/$name", readTimeoutMs = 30000) { + it.inputStream.readBytes() + } + fun answerQuestion(settings: ServerSettings, sessionId: String, questionId: String, answer: String) { requestFromServer( settings, diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt index 1959f97..fbe4d99 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/SessionScreen.kt @@ -33,6 +33,7 @@ import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.unit.dp import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay @@ -66,7 +67,9 @@ sealed class TranscriptItem { val answer: String?, ) : TranscriptItem() data class ErrorMsg(val message: String) : TranscriptItem() - /** Placeholder row for events this build can't render (images, newer kinds). */ + /** An image by server-side ref, fetched from the session's files route. */ + data class ImageItem(val ref: String) : TranscriptItem() + /** Placeholder row for events this build can't render (newer kinds). */ data class Note(val text: String) : TranscriptItem() } @@ -94,7 +97,7 @@ fun foldEvent(items: List, event: SessionEvent): List items + TranscriptItem.ErrorMsg(event.message) - is SessionEvent.Image -> items + TranscriptItem.Note("[image ${event.ref}]") + is SessionEvent.Image -> items + TranscriptItem.ImageItem(event.ref) is SessionEvent.Unknown -> items + TranscriptItem.Note("[${event.type}]") // Screen-level state, not transcript rows -- see SessionScreen. is SessionEvent.Status, is SessionEvent.UsageDelta -> items @@ -118,6 +121,9 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () var actionError by remember { mutableStateOf(null) } var input by remember { mutableStateOf("") } var expandedTools by remember { mutableStateOf(setOf()) } + // Uploaded-but-not-yet-sent attachment ids; sent with the next message. + var pendingAttachments by remember { mutableStateOf(listOf()) } + val context = androidx.compose.ui.platform.LocalContext.current // The resume cursor, written from the stream's IO thread. val lastSeq = remember { AtomicLong(0) } val activeStream = remember { AtomicReference(null) } @@ -177,9 +183,35 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () fun send() { val text = input.trim() - if (text.isEmpty()) return + val attachments = pendingAttachments + if (text.isEmpty() && attachments.isEmpty()) return input = "" - act { sendMessage(settings, summary.id, text) } + pendingAttachments = emptyList() + act { sendMessage(settings, summary.id, text, attachments) } + } + + // The system photo picker; the image uploads as soon as it's chosen, + // so Send only has ids to reference. + val pickImage = androidx.activity.compose.rememberLauncherForActivityResult( + androidx.activity.result.contract.ActivityResultContracts.PickVisualMedia(), + ) { uri -> + if (uri != null) { + scope.launch { + try { + val id = withContext(Dispatchers.IO) { + val bytes = context.contentResolver.openInputStream(uri) + ?.use { it.readBytes() } + ?: throw ApiException("couldn't read the picked image") + val mime = context.contentResolver.getType(uri) ?: "image/jpeg" + uploadAttachment(settings, summary.id, bytes, mime) + } + pendingAttachments = pendingAttachments + id + actionError = null + } catch (e: ApiException) { + actionError = e.message + } + } + } } Column(Modifier.fillMaxSize()) { @@ -239,6 +271,7 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () color = MaterialTheme.colorScheme.error, style = MaterialTheme.typography.bodyMedium, ) + is TranscriptItem.ImageItem -> SessionImage(settings, summary.id, item.ref) is TranscriptItem.Note -> Text( item.text, style = MaterialTheme.typography.bodySmall, @@ -255,11 +288,21 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth().padding(8.dp), ) { + TextButton(onClick = { + pickImage.launch( + androidx.activity.result.PickVisualMediaRequest( + androidx.activity.result.contract.ActivityResultContracts + .PickVisualMedia.ImageOnly, + ), + ) + }) { + Text(if (pendingAttachments.isEmpty()) "+" else "+${pendingAttachments.size}") + } OutlinedTextField( value = input, onValueChange = { input = it }, modifier = Modifier.weight(1f), - placeholder = { Text("Message") }, + placeholder = { Text(if (pendingAttachments.isEmpty()) "Message" else "Message (+image)") }, maxLines = 4, ) Spacer(Modifier.width(8.dp)) @@ -274,6 +317,41 @@ fun SessionScreen(settings: ServerSettings, summary: SessionSummary, onBack: () } } +/** + * An inline transcript image, fetched (authenticated, pinned) from the + * session's files route. The bitmap is remembered per ref, so scrolling + * doesn't refetch. + */ +@Composable +private fun SessionImage(settings: ServerSettings, sessionId: String, ref: String) { + var bitmap by remember(ref) { + mutableStateOf(null) + } + var failed by remember(ref) { mutableStateOf(false) } + LaunchedEffect(ref) { + try { + val bytes = withContext(Dispatchers.IO) { fetchSessionFile(settings, sessionId, ref) } + bitmap = android.graphics.BitmapFactory.decodeByteArray(bytes, 0, bytes.size) + ?.asImageBitmap() + failed = bitmap == null + } catch (e: ApiException) { + failed = true + } + } + when (val image = bitmap) { + null -> Text( + if (failed) "[image $ref unavailable]" else "[loading image…]", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + else -> androidx.compose.foundation.Image( + bitmap = image, + contentDescription = "session image", + modifier = Modifier.fillMaxWidth(), + ) + } +} + @Composable private fun UserBubble(text: String) { Box(Modifier.fillMaxWidth()) { diff --git a/server/Cargo.lock b/server/Cargo.lock index f29853a..0232c89 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -155,6 +155,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "serde_core", @@ -368,6 +369,15 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -714,6 +724,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1015,6 +1042,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "strsim" version = "0.11.1" @@ -1279,6 +1312,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/server/Cargo.toml b/server/Cargo.toml index 99838d3..bb802cf 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -8,7 +8,7 @@ name = "ai-server" path = "src/main.rs" [dependencies] -axum = { version = "0.8", features = ["json"] } +axum = { version = "0.8", features = ["json", "multipart"] } axum-server = { version = "0.8", features = ["tls-rustls"] } tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "sync", "time", "process", "io-util"] } tokio-stream = "0.1" diff --git a/server/src/routes.rs b/server/src/routes.rs index a5b8f58..fa1702a 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -11,11 +11,13 @@ //! POST /sessions/{id}/interrupt //! POST /sessions/{id}/model {model} //! POST /sessions/{id}/compact +//! POST /sessions/{id}/attachments multipart image upload -> {id}, referenced by /message +//! GET /sessions/{id}/files/{name} images the session produced or was sent //! DELETE /sessions/{id} kill process, delete transcript + files //! ``` //! -//! Later phases add: `POST /attachments`, `GET /files/{session}/{id}`, -//! `GET /usage`, `GET|PUT /hosts` and `/models` -- see PLAN.md's table. +//! Later phases add: `GET /usage`, `GET|PUT /hosts` and `/models` -- see +//! PLAN.md's table. //! //! Everything here works purely in the common event model; nothing may //! branch on the session kind (that's what drivers are for). @@ -48,6 +50,10 @@ pub fn router(manager: Arc) -> Router { .route("/sessions/{id}/interrupt", post(interrupt)) .route("/sessions/{id}/model", post(set_model)) .route("/sessions/{id}/compact", post(compact)) + .route("/sessions/{id}/attachments", post(upload_attachment)) + .route("/sessions/{id}/files/{name}", get(serve_file)) + // Phone photos overflow axum's 2 MB default body cap. + .layer(axum::extract::DefaultBodyLimit::max(32 * 1024 * 1024)) // An explicit fallback so the auth middleware (layered around the // whole router in main.rs) also covers unknown paths -- a scanner // gets the same 401 everywhere, never a route map. @@ -202,6 +208,59 @@ async fn compact( Ok(StatusCode::NO_CONTENT) } +/// Accepts one image (any multipart field) and stores it under the +/// session; the returned id goes into a later `/message`'s attachmentIds. +async fn upload_attachment( + State(manager): State>, + UrlPath(id): UrlPath, + mut multipart: axum::extract::Multipart, +) -> Result, ApiError> { + let session = lookup(&manager, &id)?; + let field = multipart + .next_field() + .await + .map_err(|err| ApiError::BadRequest(format!("bad upload: {err}")))? + .ok_or_else(|| ApiError::BadRequest("no file in the upload".to_string()))?; + let content_type = field.content_type().unwrap_or("image/jpeg").to_string(); + let bytes = field + .bytes() + .await + .map_err(|err| ApiError::BadRequest(format!("upload read failed: {err}")))?; + let name = session.save_attachment(&bytes, &content_type).map_err(bad_request)?; + Ok(axum::Json(serde_json::json!({ "id": name }))) +} + +/// Serves a session's stored images -- both `files/` (produced by tools) +/// and `attachments/` (uploaded from the phone), by the id events and +/// uploads reference. +async fn serve_file( + State(manager): State>, + UrlPath((id, name)): UrlPath<(String, String)>, +) -> Result { + // Ids are server-generated hex + extension; anything else (and any + // path separator in particular) is refused, not resolved. + if !name.chars().all(|c| c.is_ascii_alphanumeric() || c == '.') || name.contains("..") { + return Err(ApiError::BadRequest("invalid file id".to_string())); + } + let session = lookup(&manager, &id)?; + let candidates = + [session.dir().join("files").join(&name), session.dir().join("attachments").join(&name)]; + let Some(path) = candidates.iter().find(|path| path.is_file()) else { + return Err(ApiError::BadRequest(format!("no file {name} in session {id}"))); + }; + let bytes = std::fs::read(path).map_err(|err| { + tracing::error!("read {} failed: {err}", path.display()); + ApiError::BadRequest("file unreadable".to_string()) + })?; + let content_type = match name.rsplit('.').next() { + Some("png") => "image/png", + Some("gif") => "image/gif", + Some("webp") => "image/webp", + _ => "image/jpeg", + }; + Ok(([(axum::http::header::CONTENT_TYPE, content_type)], bytes).into_response()) +} + #[derive(Deserialize)] struct EventsQuery { #[serde(default)] diff --git a/server/src/session/claude.rs b/server/src/session/claude.rs index ad42565..21cf3d2 100644 --- a/server/src/session/claude.rs +++ b/server/src/session/claude.rs @@ -92,7 +92,7 @@ impl ClaudeDriver { let stdin = child.stdin.take().expect("piped stdin"); let stdout = child.stdout.take().expect("piped stdout"); let stderr = child.stderr.take().expect("piped stderr"); - let state = Arc::new(Mutex::new(Translator::default())); + let state = Arc::new(Mutex::new(Translator::new(session_dir.to_path_buf()))); // Writer: everything for the child funnels through one channel so // driver methods stay sync and writes can't interleave. @@ -354,15 +354,20 @@ struct PendingRequest { answers: HashMap, } -/// Pure translation state: stream-json lines in, common events out. No -/// I/O, so the whole dialect mapping is unit-testable from recorded lines. -#[derive(Default)] +/// Translation state: stream-json lines in, common events out. The one +/// side effect is saving images a tool result carries into the session +/// dir (they'd bloat the transcript as base64); everything else is pure, +/// so the dialect mapping is unit-testable from recorded lines. struct Translator { session_id: Option, pending: HashMap, + session_dir: PathBuf, } impl Translator { + fn new(session_dir: PathBuf) -> Self { + Self { session_id: None, pending: HashMap::new(), session_dir } + } fn translate(&mut self, message: &Value) -> Vec { // Events from subagents (Task tool internals) carry a // parent_tool_use_id; the transcript shows the Task tool's own @@ -381,7 +386,7 @@ impl Translator { } Some("stream_event") => self.translate_stream_event(&message["event"]), Some("assistant") => self.translate_assistant(&message["message"]), - Some("user") => translate_user(message), + Some("user") => self.translate_user(message), Some("control_request") => self.translate_control_request(message), Some("control_response") => { let response = &message["response"]; @@ -539,36 +544,77 @@ impl Translator { } } -/// `user` messages: tool results become ToolEnd (with any images saved -/// out-of-band by the caller -- phase 2b); replayed/synthetic user text is -/// skipped, since the manager already recorded the user's side. -fn translate_user(message: &Value) -> Vec { - let Some(content) = message["message"].get("content").and_then(Value::as_array) else { - return Vec::new(); - }; - content - .iter() - .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_result")) - .map(|block| { - let output = match block.get("content") { - Some(Value::String(text)) => text.clone(), - Some(Value::Array(parts)) => parts - .iter() - .filter_map(|part| part.get("text").and_then(Value::as_str)) - .collect::>() - .join("\n"), - _ => String::new(), - }; - Event::ToolEnd { +impl Translator { + /// `user` messages: tool results become ToolEnd, with any image parts + /// saved into the session dir and referenced by an Image event (the + /// phone fetches them from `/sessions/{id}/files/{ref}`). Replayed and + /// synthetic user text is skipped -- the manager already recorded the + /// user's side. + fn translate_user(&self, message: &Value) -> Vec { + let Some(content) = message["message"].get("content").and_then(Value::as_array) else { + return Vec::new(); + }; + let mut events = Vec::new(); + for block in content { + if block.get("type").and_then(Value::as_str) != Some("tool_result") { + continue; + } + let mut texts = Vec::new(); + match block.get("content") { + Some(Value::String(text)) => texts.push(text.clone()), + Some(Value::Array(parts)) => { + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("text") => { + if let Some(text) = part.get("text").and_then(Value::as_str) { + texts.push(text.to_string()); + } + } + Some("image") => { + if let Some(name) = self.save_image(part) { + events.push(Event::Image { image: name }); + } + } + _ => {} + } + } + } + _ => {} + } + events.push(Event::ToolEnd { id: block .get("tool_use_id") .and_then(Value::as_str) .unwrap_or_default() .to_string(), - output, - } - }) - .collect() + output: texts.join("\n"), + }); + } + events + } + + /// Decodes one base64 image block into `files/` and returns its ref. + fn save_image(&self, part: &Value) -> Option { + let source = part.get("source")?; + let data = source.get("data")?.as_str()?; + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD.decode(data).ok()?; + let extension = match source.get("media_type").and_then(Value::as_str) { + Some("image/jpeg") => "jpg", + Some("image/gif") => "gif", + Some("image/webp") => "webp", + _ => "png", + }; + let name = format!("{}.{extension}", super::random_hex()); + let dir = self.session_dir.join("files"); + if let Err(err) = + std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(dir.join(&name), bytes)) + { + tracing::error!("couldn't save produced image: {err}"); + return None; + } + Some(name) + } } #[cfg(test)] @@ -584,7 +630,8 @@ mod tests { #[test] fn captures_the_resume_token_from_init() { - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines( &mut translator, &[r#"{"type":"system","subtype":"init","cwd":"/x","session_id":"5ecf21da-d53f","tools":[],"model":"claude-haiku-4-5-20251001"}"#], @@ -596,7 +643,8 @@ mod tests { #[test] fn streams_text_deltas_and_skips_the_consolidated_copy() { // Real lines (trimmed) from the 2.1.237 probe. - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines(&mut translator, &[ r#"{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Done."}},"session_id":"s","parent_tool_use_id":null}"#, r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Done."}]},"parent_tool_use_id":null,"session_id":"s"}"#, @@ -607,7 +655,8 @@ mod tests { #[test] fn tool_use_and_result_become_tool_events() { - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines(&mut translator, &[ r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"echo probe-ok"}}]},"parent_tool_use_id":null}"#, r#"{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01","type":"tool_result","content":"probe-ok","is_error":false}]},"parent_tool_use_id":null}"#, @@ -624,7 +673,8 @@ mod tests { #[test] fn subagent_events_are_not_duplicated_into_the_transcript() { - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines(&mut translator, &[ r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_02","name":"Bash","input":{}}]},"parent_tool_use_id":"toolu_parent"}"#, ]); @@ -633,7 +683,8 @@ mod tests { #[test] fn a_permission_request_becomes_an_allow_deny_question() { - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines(&mut translator, &[ r#"{"type":"control_request","request_id":"req-1","request":{"subtype":"can_use_tool","tool_name":"Bash","input":{"command":"rm -rf /tmp/x"},"tool_use_id":"toolu_03"}}"#, ]); @@ -660,7 +711,8 @@ mod tests { #[test] fn denying_a_permission_sends_deny() { - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); translate_lines(&mut translator, &[ r#"{"type":"control_request","request_id":"req-2","request":{"subtype":"can_use_tool","tool_name":"Write","input":{"file_path":"/etc/passwd"}}}"#, ]); @@ -674,7 +726,8 @@ mod tests { fn ask_user_question_rides_the_same_flow_with_answers_keyed_by_question() { // The real 2.1.237 shape, verified live: answers go back inside // updatedInput, keyed by the question text. - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines(&mut translator, &[ r#"{"type":"control_request","request_id":"req-3","request":{"subtype":"can_use_tool","tool_name":"AskUserQuestion","input":{"questions":[{"question":"Which color?","header":"Color","options":[{"label":"Red"},{"label":"Blue"}],"multiSelect":false},{"question":"Which size?","header":"Size","options":[{"label":"S"},{"label":"L"}],"multiSelect":false}]},"tool_use_id":"toolu_04","requires_user_interaction":true}}"#, ]); @@ -702,9 +755,33 @@ mod tests { assert_eq!(updated["questions"][0]["question"], "Which color?"); } + #[test] + fn images_in_tool_results_are_saved_and_referenced() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); + // A 1x1 PNG, the smallest real payload worth round-tripping. + let png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; + let line = format!( + r#"{{"type":"user","message":{{"role":"user","content":[{{"type":"tool_result","tool_use_id":"toolu_05","content":[{{"type":"text","text":"took a screenshot"}},{{"type":"image","source":{{"type":"base64","media_type":"image/png","data":"{png}"}}}}]}}]}},"parent_tool_use_id":null}}"# + ); + let events = translator.translate(&serde_json::from_str(&line).expect("json")); + + let Event::Image { image } = &events[0] else { + panic!("expected an image event, got {events:?}"); + }; + assert!(image.ends_with(".png")); + let saved = dir.path().join("files").join(image); + assert!(saved.is_file(), "image not saved at {}", saved.display()); + assert_eq!(events[1], Event::ToolEnd { + id: "toolu_05".to_string(), + output: "took a screenshot".to_string(), + }); + } + #[test] fn a_turn_result_reports_usage_and_returns_to_idle() { - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines(&mut translator, &[ r#"{"type":"result","subtype":"success","is_error":false,"num_turns":2,"stop_reason":"end_turn","session_id":"s","total_cost_usd":0.0149,"usage":{"input_tokens":18,"output_tokens":164}}"#, ]); @@ -716,7 +793,8 @@ mod tests { #[test] fn an_error_result_surfaces_the_message() { - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines(&mut translator, &[ r#"{"type":"result","subtype":"error_during_execution","is_error":true,"result":"something broke","usage":{}}"#, ]); @@ -726,7 +804,8 @@ mod tests { #[test] fn replayed_and_synthetic_user_text_is_skipped() { - let mut translator = Translator::default(); + let dir = tempfile::tempdir().expect("tempdir"); + let mut translator = Translator::new(dir.path().to_path_buf()); let events = translate_lines(&mut translator, &[ r#"{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]},"isReplay":true,"parent_tool_use_id":null}"#, r#"{"type":"user","message":{"role":"user","content":[{"type":"text","text":"[continue]"}]},"isSynthetic":true,"parent_tool_use_id":null}"#, diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 29b5fcc..d6572a5 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -94,6 +94,11 @@ impl LiveSession { /// driver -- which queues it for injection mid-run rather than at the /// end of the turn (the point of the whole app). pub fn send_message(&self, text: String, images: Vec) { + // Attachments render in the transcript like any produced image -- + // the files route serves uploads by the same ref. + for image in &images { + let _ = self.sink.send(Event::Image { image: image.clone() }); + } let _ = self.sink.send(Event::UserMessage { text: text.clone() }); self.driver.send_user_message(text, images); } @@ -122,6 +127,30 @@ impl LiveSession { &self.transcript_path } + /// The session's directory (attachments in, produced files out live in + /// `attachments/` and `files/` under it). + pub fn dir(&self) -> &Path { + self.transcript_path.parent().expect("transcript lives in the session dir") + } + + /// Stores one uploaded attachment, returning the id `POST /message` + /// references it by. Removed with the session directory on delete -- + /// the same path out as everything else in it. + pub fn save_attachment(&self, bytes: &[u8], content_type: &str) -> Result { + let extension = match content_type { + "image/png" => "png", + "image/gif" => "gif", + "image/webp" => "webp", + _ => "jpg", + }; + let name = format!("{}.{extension}", random_hex()); + let dir = self.dir().join("attachments"); + std::fs::create_dir_all(&dir).with_context(|| format!("create {}", dir.display()))?; + std::fs::write(dir.join(&name), bytes) + .with_context(|| format!("write attachment {name}"))?; + Ok(name) + } + fn info(&self) -> SessionInfo { SessionInfo { id: self.meta.id.clone(), @@ -313,13 +342,18 @@ fn default_title(kind: SessionKind) -> String { } /// 8 random bytes, hex -- short enough for a URL, unique enough forever at -/// this scale. Still checked against the existing list out of caution. -fn unique_id(config: &Config) -> String { +/// this scale. +pub fn random_hex() -> String { use rand::Rng; + let mut bytes = [0u8; 8]; + rand::rng().fill_bytes(&mut bytes); + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// A [`random_hex`] id not already taken -- checked out of caution. +fn unique_id(config: &Config) -> String { loop { - let mut bytes = [0u8; 8]; - rand::rng().fill_bytes(&mut bytes); - let id: String = bytes.iter().map(|b| format!("{b:02x}")).collect(); + let id = random_hex(); if !config.sessions.iter().any(|meta| meta.id == id) { return id; }