iris/ai-app
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 2 complete: images both ways

Browse Source 
Inbound: POST /sessions/{id}/attachments stores a picked photo under the
session; message attachmentIds become base64 image blocks in the
stream-json user message (verified live: an uploaded red PNG answered
"Red."). Outbound: image parts in tool results are decoded into the
session's files/ dir and referenced by Image events -- the transcript
stays lean -- and GET /sessions/{id}/files/{ref} serves them (verified
via the Read tool round-tripping the same PNG). The app grows an attach
button (system photo picker, upload-on-pick) and renders Image events
inline with an authenticated pinned fetch. Sent attachments are echoed
into the transcript as Image events so every device shows them.

Attachments and files are addressed under their session (a deviation
from PLAN.md's original bare /attachments -- recorded there) so their
lifecycle is the session directory's: deleting the session is still the
complete path out.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017xn8nHw1tw1R6PtiY1eEtw
This commit is contained in:
irisandClaude Fable 5 committed 2026-08-24 21:40:25 -04:00
1 parent 95d389e2b8
commit f2430671a2
7 files changed
+389 -55
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/session/claude.rs
+119 -40
View File
@@ -92,7 +92,7 @@ impl ClaudeDriver {
let stdin = child.stdin.take().expect("piped stdin");
let stdout = child.stdout.take().expect("piped stdout");
let stderr = child.stderr.take().expect("piped stderr");
let state = Arc::new(Mutex::new(Translator::default()));
let state = Arc::new(Mutex::new(Translator::new(session_dir.to_path_buf())));
// Writer: everything for the child funnels through one channel so
// driver methods stay sync and writes can't interleave.
@@ -354,15 +354,20 @@ struct PendingRequest {
answers: HashMap<String, String>,
}
/// Pure translation state: stream-json lines in, common events out. No
/// I/O, so the whole dialect mapping is unit-testable from recorded lines.
#[derive(Default)]
/// Translation state: stream-json lines in, common events out. The one
/// side effect is saving images a tool result carries into the session
/// dir (they'd bloat the transcript as base64); everything else is pure,
/// so the dialect mapping is unit-testable from recorded lines.
struct Translator {
session_id: Option<String>,
pending: HashMap<String, PendingRequest>,
session_dir: PathBuf,
}
impl Translator {
fn new(session_dir: PathBuf) -> Self {
Self { session_id: None, pending: HashMap::new(), session_dir }
}
fn translate(&mut self, message: &Value) -> Vec<Event> {
// Events from subagents (Task tool internals) carry a
// parent_tool_use_id; the transcript shows the Task tool's own
@@ -381,7 +386,7 @@ impl Translator {
}
Some("stream_event") => self.translate_stream_event(&message["event"]),
Some("assistant") => self.translate_assistant(&message["message"]),
Some("user") => translate_user(message),
Some("user") => self.translate_user(message),
Some("control_request") => self.translate_control_request(message),
Some("control_response") => {
let response = &message["response"];
@@ -539,36 +544,77 @@ impl Translator {
}
}
/// `user` messages: tool results become ToolEnd (with any images saved
/// out-of-band by the caller -- phase 2b); replayed/synthetic user text is
/// skipped, since the manager already recorded the user's side.
fn translate_user(message: &Value) -> Vec<Event> {
let Some(content) = message["message"].get("content").and_then(Value::as_array) else {
return Vec::new();
};
content
.iter()
.filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_result"))
.map(|block| {
let output = match block.get("content") {
Some(Value::String(text)) => text.clone(),
Some(Value::Array(parts)) => parts
.iter()
.filter_map(|part| part.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("\n"),
_ => String::new(),
};
Event::ToolEnd {
impl Translator {
/// `user` messages: tool results become ToolEnd, with any image parts
/// saved into the session dir and referenced by an Image event (the
/// phone fetches them from `/sessions/{id}/files/{ref}`). Replayed and
/// synthetic user text is skipped -- the manager already recorded the
/// user's side.
fn translate_user(&self, message: &Value) -> Vec<Event> {
let Some(content) = message["message"].get("content").and_then(Value::as_array) else {
return Vec::new();
};
let mut events = Vec::new();
for block in content {
if block.get("type").and_then(Value::as_str) != Some("tool_result") {
continue;
}
let mut texts = Vec::new();
match block.get("content") {
Some(Value::String(text)) => texts.push(text.clone()),
Some(Value::Array(parts)) => {
for part in parts {
match part.get("type").and_then(Value::as_str) {
Some("text") => {
if let Some(text) = part.get("text").and_then(Value::as_str) {
texts.push(text.to_string());
}
}
Some("image") => {
if let Some(name) = self.save_image(part) {
events.push(Event::Image { image: name });
}
}
_ => {}
}
}
}
_ => {}
}
events.push(Event::ToolEnd {
id: block
.get("tool_use_id")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string(),
output,
}
})
.collect()
output: texts.join("\n"),
});
}
events
}
/// Decodes one base64 image block into `files/` and returns its ref.
fn save_image(&self, part: &Value) -> Option<String> {
let source = part.get("source")?;
let data = source.get("data")?.as_str()?;
use base64::Engine;
let bytes = base64::engine::general_purpose::STANDARD.decode(data).ok()?;
let extension = match source.get("media_type").and_then(Value::as_str) {
Some("image/jpeg") => "jpg",
Some("image/gif") => "gif",
Some("image/webp") => "webp",
_ => "png",
};
let name = format!("{}.{extension}", super::random_hex());
let dir = self.session_dir.join("files");
if let Err(err) =
std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(dir.join(&name), bytes))
{
tracing::error!("couldn't save produced image: {err}");
return None;
}
Some(name)
}
}
#[cfg(test)]
@@ -584,7 +630,8 @@ mod tests {
#[test]
fn captures_the_resume_token_from_init() {
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(
&mut translator,
&[r#"{"type":"system","subtype":"init","cwd":"/x","session_id":"5ecf21da-d53f","tools":[],"model":"claude-haiku-4-5-20251001"}"#],
@@ -596,7 +643,8 @@ mod tests {
#[test]
fn streams_text_deltas_and_skips_the_consolidated_copy() {
// Real lines (trimmed) from the 2.1.237 probe.
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[
r#"{"type":"stream_event","event":{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Done."}},"session_id":"s","parent_tool_use_id":null}"#,
r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"Done."}]},"parent_tool_use_id":null,"session_id":"s"}"#,
@@ -607,7 +655,8 @@ mod tests {
#[test]
fn tool_use_and_result_become_tool_events() {
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[
r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_01","name":"Bash","input":{"command":"echo probe-ok"}}]},"parent_tool_use_id":null}"#,
r#"{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01","type":"tool_result","content":"probe-ok","is_error":false}]},"parent_tool_use_id":null}"#,
@@ -624,7 +673,8 @@ mod tests {
#[test]
fn subagent_events_are_not_duplicated_into_the_transcript() {
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[
r#"{"type":"assistant","message":{"role":"assistant","content":[{"type":"tool_use","id":"toolu_02","name":"Bash","input":{}}]},"parent_tool_use_id":"toolu_parent"}"#,
]);
@@ -633,7 +683,8 @@ mod tests {
#[test]
fn a_permission_request_becomes_an_allow_deny_question() {
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[
r#"{"type":"control_request","request_id":"req-1","request":{"subtype":"can_use_tool","tool_name":"Bash","input":{"command":"rm -rf /tmp/x"},"tool_use_id":"toolu_03"}}"#,
]);
@@ -660,7 +711,8 @@ mod tests {
#[test]
fn denying_a_permission_sends_deny() {
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
translate_lines(&mut translator, &[
r#"{"type":"control_request","request_id":"req-2","request":{"subtype":"can_use_tool","tool_name":"Write","input":{"file_path":"/etc/passwd"}}}"#,
]);
@@ -674,7 +726,8 @@ mod tests {
fn ask_user_question_rides_the_same_flow_with_answers_keyed_by_question() {
// The real 2.1.237 shape, verified live: answers go back inside
// updatedInput, keyed by the question text.
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[
r#"{"type":"control_request","request_id":"req-3","request":{"subtype":"can_use_tool","tool_name":"AskUserQuestion","input":{"questions":[{"question":"Which color?","header":"Color","options":[{"label":"Red"},{"label":"Blue"}],"multiSelect":false},{"question":"Which size?","header":"Size","options":[{"label":"S"},{"label":"L"}],"multiSelect":false}]},"tool_use_id":"toolu_04","requires_user_interaction":true}}"#,
]);
@@ -702,9 +755,33 @@ mod tests {
assert_eq!(updated["questions"][0]["question"], "Which color?");
}
#[test]
fn images_in_tool_results_are_saved_and_referenced() {
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
// A 1x1 PNG, the smallest real payload worth round-tripping.
let png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
let line = format!(
r#"{{"type":"user","message":{{"role":"user","content":[{{"type":"tool_result","tool_use_id":"toolu_05","content":[{{"type":"text","text":"took a screenshot"}},{{"type":"image","source":{{"type":"base64","media_type":"image/png","data":"{png}"}}}}]}}]}},"parent_tool_use_id":null}}"#
);
let events = translator.translate(&serde_json::from_str(&line).expect("json"));
let Event::Image { image } = &events[0] else {
panic!("expected an image event, got {events:?}");
};
assert!(image.ends_with(".png"));
let saved = dir.path().join("files").join(image);
assert!(saved.is_file(), "image not saved at {}", saved.display());
assert_eq!(events[1], Event::ToolEnd {
id: "toolu_05".to_string(),
output: "took a screenshot".to_string(),
});
}
#[test]
fn a_turn_result_reports_usage_and_returns_to_idle() {
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[
r#"{"type":"result","subtype":"success","is_error":false,"num_turns":2,"stop_reason":"end_turn","session_id":"s","total_cost_usd":0.0149,"usage":{"input_tokens":18,"output_tokens":164}}"#,
]);
@@ -716,7 +793,8 @@ mod tests {
#[test]
fn an_error_result_surfaces_the_message() {
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[
r#"{"type":"result","subtype":"error_during_execution","is_error":true,"result":"something broke","usage":{}}"#,
]);
@@ -726,7 +804,8 @@ mod tests {
#[test]
fn replayed_and_synthetic_user_text_is_skipped() {
let mut translator = Translator::default();
let dir = tempfile::tempdir().expect("tempdir");
let mut translator = Translator::new(dir.path().to_path_buf());
let events = translate_lines(&mut translator, &[
r#"{"type":"user","message":{"role":"user","content":[{"type":"text","text":"hi"}]},"isReplay":true,"parent_tool_use_id":null}"#,
r#"{"type":"user","message":{"role":"user","content":[{"type":"text","text":"[continue]"}]},"isSynthetic":true,"parent_tool_use_id":null}"#,