diff --git a/AGENTS.md b/AGENTS.md index e136a76..21d7da2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,6 +41,11 @@ repo is in PLAN.md's "Backend layout" section. `POST /setups`. Only the tail is replayed (`REPLAY_LINES`) because these files reach tens of megabytes and the CLI reads the real one itself; what crosses the tunnel is what a person reads, not what the model is given. + Images in the replayed tail are written into the session's `files/` by + the same function the live translator uses, and referenced -- so a + screenshot looks the same whether it was watched happening or replayed + afterwards, the phone fetches the bytes only when it draws one, and none + of it goes anywhere near the CLI, which reads its own file. An imported session then **keeps itself level with that file**, so work done at a terminal appears without anyone pressing anything. `--resume` appends to the same transcript rather than forking — measured, but diff --git a/server/src/routes.rs b/server/src/routes.rs index 3276a1e..ae8490d 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -478,7 +478,7 @@ async fn spawn_session( whole thing. Close it there first, then import it here." ))); } - let events = crate::session::import::replay(&transport, &chosen.path) + let records = crate::session::import::read_tail(&transport, &chosen.path) .await .map_err(bad_request)?; // The recorded directory can outlive itself; resuming into one @@ -495,7 +495,7 @@ async fn spawn_session( ); chosen.cwd = String::new(); } - Some((chosen, events)) + Some((chosen, records)) } None => None, }; @@ -532,11 +532,11 @@ async fn spawn_session( }; let info = match seed { - Some((chosen, events)) => { + Some((chosen, records)) => { tracing::info!( - "importing Claude Code session {} ({} events replayed)", + "importing Claude Code session {} ({} lines replayed)", chosen.id, - events.len() + records.lines().count() ); manager.spawn_imported( spec, @@ -549,7 +549,7 @@ async fn spawn_session( path: chosen.path, lines: chosen.lines, }, - events, + records, }, ) } diff --git a/server/src/session/claude.rs b/server/src/session/claude.rs index 676d85c..80a2821 100644 --- a/server/src/session/claude.rs +++ b/server/src/session/claude.rs @@ -96,7 +96,7 @@ fn tail_of(kept: &VecDeque) -> String { /// the conversation back up from Claude's own session files. Kept in the /// session directory rather than config.ron so the shared schema stays /// free of per-driver state. -mod translate; +pub(super) mod translate; const RESUME_FILE: &str = "claude-session.json"; diff --git a/server/src/session/claude/translate.rs b/server/src/session/claude/translate.rs index 4e7e315..45cbf88 100644 --- a/server/src/session/claude/translate.rs +++ b/server/src/session/claude/translate.rs @@ -13,7 +13,7 @@ //! without a process. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use serde_json::{Value, json}; @@ -307,7 +307,7 @@ impl Translator { } } Some("image") => { - if let Some(name) = self.save_image(part) { + if let Some(name) = save_image(&self.session_dir, part) { events.push(Event::Image { image: name }); } } @@ -328,33 +328,39 @@ impl Translator { } events } +} - /// Decodes one base64 image block into `files/` and returns its ref. - fn save_image(&self, part: &Value) -> Option { - let source = part.get("source")?; - let data = source.get("data")?.as_str()?; - use base64::Engine; - let bytes = base64::engine::general_purpose::STANDARD - .decode(data) - .ok()?; - // Screenshots are the overwhelming case, and they are PNG; an - // unrecognized type is more likely a dialect change than a JPEG. - let extension = source - .get("media_type") - .and_then(Value::as_str) - .and_then(crate::media::extension_for) - .unwrap_or("png"); - let name = format!("{}.{extension}", super::super::random_hex()); - let dir = self.session_dir.join("files"); - if let Err(err) = wg_app_link::private::create_dir(&dir) - .map_err(std::io::Error::other) - .and_then(|()| std::fs::write(dir.join(&name), bytes)) - { - tracing::error!("couldn't save produced image: {err}"); - return None; - } - Some(name) +/// Decodes one base64 image block into `files/` and returns its ref. +/// +/// A free function rather than a method because the import replay needs +/// exactly this too: a session's history carries the same image blocks as +/// its live output, and a reader who can see a screenshot while it happens +/// should still see it after a restart. Two copies of this would be two +/// naming schemes for one directory. +pub(in crate::session) fn save_image(session_dir: &Path, part: &Value) -> Option { + let source = part.get("source")?; + let data = source.get("data")?.as_str()?; + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(data) + .ok()?; + // Screenshots are the overwhelming case, and they are PNG; an + // unrecognized type is more likely a dialect change than a JPEG. + let extension = source + .get("media_type") + .and_then(Value::as_str) + .and_then(crate::media::extension_for) + .unwrap_or("png"); + let name = format!("{}.{extension}", super::super::random_hex()); + let dir = session_dir.join("files"); + if let Err(err) = wg_app_link::private::create_dir(&dir) + .map_err(std::io::Error::other) + .and_then(|()| std::fs::write(dir.join(&name), bytes)) + { + tracing::error!("couldn't save produced image: {err}"); + return None; } + Some(name) } #[cfg(test)] diff --git a/server/src/session/import.rs b/server/src/session/import.rs index 23c9ade..f1e16a6 100644 --- a/server/src/session/import.rs +++ b/server/src/session/import.rs @@ -317,21 +317,26 @@ pub async fn directory_exists(transport: &Transport, path: &str) -> bool { transport.capture(&launch).await.is_ok() } -/// Reads the tail of one session's file and turns it into events. +/// Reads the tail of one session's file, as the raw JSONL. /// /// `tail` rather than the whole file, and as [`Launch`] arguments rather /// than a shell string, so the path is an argument and never syntax. -pub async fn replay(transport: &Transport, path: &str) -> Result> { +/// +/// Returns text rather than events because turning records into events has +/// a side effect -- writing out the images they carry -- and it needs the +/// session directory to write them into. That directory does not exist +/// until the session is created, which is after this runs, so the +/// conversion happens there instead. See [`events_from`]. +pub async fn read_tail(transport: &Transport, path: &str) -> Result { let launch = Launch::new( "tail", vec!["-n".to_string(), REPLAY_LINES.to_string(), path.to_string()], None, ); - let text = transport + transport .capture(&launch) .await - .with_context(|| format!("reading {path}"))?; - Ok(events_from(&text)) + .with_context(|| format!("reading {path}")) } /// Claude Code's stored JSONL as this project's events. @@ -339,7 +344,15 @@ pub async fn replay(transport: &Transport, path: &str) -> Result> { /// A partial first line is expected and ignored: `tail -n` cuts at a line /// boundary, but the *file* may have been appended to since, and a line /// that does not parse is one this reader has no opinion about. -pub fn events_from(text: &str) -> Vec { +/// +/// `session_dir` is where images found along the way are written, the same +/// place and by the same function the live translator uses -- so a +/// screenshot looks identical whether it was watched as it happened or +/// replayed afterwards. It is only the *reference* that reaches the phone; +/// the bytes are fetched from `/sessions/{id}/files/{ref}` when something +/// actually draws them, and none of this is ever sent back to the CLI, +/// which reads its own session file. +pub fn events_from(text: &str, session_dir: &std::path::Path) -> Vec { let mut events = Vec::new(); for line in text.lines() { let Ok(record) = serde_json::from_str::(line) else { @@ -355,7 +368,7 @@ pub fn events_from(text: &str) -> Vec { continue; }; match record.get("type").and_then(Value::as_str) { - Some("user") => push_user(&mut events, content), + Some("user") => push_user(&mut events, content, session_dir), Some("assistant") => push_assistant(&mut events, content), _ => {} } @@ -363,16 +376,25 @@ pub fn events_from(text: &str) -> Vec { events } -fn push_user(events: &mut Vec, content: &Value) { +fn push_user(events: &mut Vec, content: &Value, session_dir: &std::path::Path) { // A tool result arrives as a user record, because that is how the API // models it -- but it is the other half of a tool call, not something // a person said, and showing it as a message would put the reader's // own words and a command's output in the same voice. if let Value::Array(blocks) = content { for block in blocks { + // A picture the person attached to their own message, rather + // than one a tool produced. Same block shape, one level up. + push_images(events, std::slice::from_ref(block), session_dir); if block.get("type").and_then(Value::as_str) == Some("tool_result") && let Some(id) = block.get("tool_use_id").and_then(Value::as_str) { + // Before the tool's own row, matching the live translator: + // a screenshot belongs to the call that took it, and after + // the result it reads as belonging to whatever came next. + if let Some(Value::Array(parts)) = block.get("content") { + push_images(events, parts, session_dir); + } events.push(Event::ToolEnd { id: id.to_string(), output: text_of(block.get("content").unwrap_or(&Value::Null)), @@ -386,6 +408,17 @@ fn push_user(events: &mut Vec, content: &Value) { } } +/// Saves every image block in `parts` and references each one. +fn push_images(events: &mut Vec, parts: &[Value], session_dir: &std::path::Path) { + for part in parts { + if part.get("type").and_then(Value::as_str) == Some("image") + && let Some(name) = super::claude::translate::save_image(session_dir, part) + { + events.push(Event::Image { image: name }); + } + } +} + fn push_assistant(events: &mut Vec, content: &Value) { let Value::Array(blocks) = content else { return; @@ -501,7 +534,12 @@ pub async fn line_count(transport: &Transport, path: &str) -> Result { /// Events from the lines after `after`, which is a 0-based count of lines /// already accounted for. -pub async fn replay_after(transport: &Transport, path: &str, after: usize) -> Result> { +pub async fn replay_after( + transport: &Transport, + path: &str, + after: usize, + session_dir: &std::path::Path, +) -> Result> { let launch = Launch::new( "tail", vec![format!("-n+{}", after + 1), path.to_string()], @@ -511,5 +549,50 @@ pub async fn replay_after(transport: &Transport, path: &str, after: usize) -> Re .capture(&launch) .await .with_context(|| format!("reading {path} from line {}", after + 1))?; - Ok(events_from(&text)) + Ok(events_from(&text, session_dir)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A 1x1 PNG, base64 -- the smallest thing with a real header. + const PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk\ + YPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + + #[test] + fn replayed_screenshots_are_saved_and_referenced() { + let dir = tempfile::tempdir().expect("tempdir"); + // The shape a screenshot actually has in these files: an image + // part inside a tool result, beside its text. + let line = format!( + r#"{{"type":"user","message":{{"role":"user","content":[{{"type":"tool_result","tool_use_id":"toolu_1","content":[{{"type":"text","text":"took a screenshot"}},{{"type":"image","source":{{"type":"base64","media_type":"image/png","data":"{PNG}"}}}}]}}]}}}}"# + ); + + let events = events_from(&line, dir.path()); + let Some(Event::Image { image }) = events.first() else { + panic!("a replayed screenshot must become an image event: {events:?}"); + }; + assert!(image.ends_with(".png")); + // On disk, where the files route serves it from -- the phone + // fetches it only when something draws it. + assert!(dir.path().join("files").join(image).is_file()); + + // And it comes before the tool row it belongs to, so it does not + // read as belonging to whatever happened next. + assert!( + matches!(events.get(1), Some(Event::ToolEnd { .. })), + "{events:?}" + ); + } + + #[test] + fn a_record_with_no_image_writes_nothing() { + let dir = tempfile::tempdir().expect("tempdir"); + let line = r#"{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"plain output"}]}}"#; + let events = events_from(line, dir.path()); + assert_eq!(events.len(), 1, "{events:?}"); + // No stray directory for a session that never produced one. + assert!(!dir.path().join("files").exists()); + } } diff --git a/server/src/session/mod.rs b/server/src/session/mod.rs index 60fe647..2830de5 100644 --- a/server/src/session/mod.rs +++ b/server/src/session/mod.rs @@ -831,7 +831,7 @@ fn spawn_import_sync( if lines <= cursor.lines { continue; } - match import::replay_after(&transport, &cursor.path, cursor.lines).await { + match import::replay_after(&transport, &cursor.path, cursor.lines, &dir).await { Ok(events) => { tracing::info!( "{} grew by {} lines with nothing from here; replaying {} events", @@ -866,10 +866,14 @@ pub struct Seed { /// Where that session's file is and how much of it has been shown, so /// the session can keep itself up to date afterwards. pub cursor: import::Cursor, - /// Replayed into the transcript so the phone shows the conversation it - /// is joining. The CLI reads the real file itself, so this is what the - /// reader sees rather than what the model is given. - pub events: Vec, + /// The tail of that session's file, as the raw JSONL. + /// + /// Turned into events in `launch`, not before, because doing so writes + /// out the images the records carry and that needs the session + /// directory to write them into -- which does not exist until the + /// session does. The CLI reads the real file itself, so this only ever + /// decides what the *reader* sees. + pub records: String, } fn launch( @@ -890,7 +894,7 @@ fn launch( claude::write_resume_token(&dir, &seed.resume); import::write_cursor(&dir, &seed.cursor); let at = now(); - for event in seed.events { + for event in import::events_from(&seed.records, &dir) { transcript.append(event, at)?; } }