Fix Codex transcript streaming and images

This commit is contained in:
iris committed 2026-09-09 15:14:24 -04:00
1 parent 14dd520719
commit 4dc3e3d784
7 files changed
+422 -53

No files matched your search

+5 -15
View File
@@ -1148,23 +1148,13 @@ pub(in crate::session) fn save_image(session_dir: &Path, part: &Value) -> Option
let bytes = base64::engine::general_purpose::STANDARD
.decode(data)
.ok()?;
// Screenshots are the overwhelming case and they are PNG; an unrecognized
// type is more likely a dialect change than a JPEG.
let extension = source
// Screenshots are the overwhelming case and they are PNG; an unrecognized type is more likely
// a dialect change than a JPEG. `store_image` owns that fallback.
let media_type = source
.get("media_type")
.and_then(Value::as_str)
.and_then(crate::media::extension_for)
.unwrap_or("png");
let name = format!("{}.{extension}", super::super::random_hex());
let dir = session_dir.join("files");
if let Err(err) = wg_app_link::private::create_dir(&dir)
.map_err(std::io::Error::other)
.and_then(|()| std::fs::write(dir.join(&name), bytes))
{
tracing::error!("couldn't save produced image: {err}");
return None;
}
Some(name)
.unwrap_or("image/png");
super::super::driver::store_image(session_dir, media_type, &bytes)
}
#[cfg(test)]
+238 -8
View File
@@ -18,7 +18,9 @@ use serde_json::{Value, json};
use tokio::io::AsyncWriteExt;
use tokio::sync::mpsc;
use super::driver::{AttachmentRef, Driver, Event, EventSink, SessionStatus, Unqueued};
use super::driver::{
AttachmentRef, Driver, Event, EventSink, SessionStatus, Unqueued, store_image,
};
use super::process;
use super::transport::{Launch, Streams, Transport};
use crate::config::{ProviderConfig, SessionConfig};
@@ -459,10 +461,15 @@ fn input_for(inner: &Inner, message: &Waiting) -> Result<Vec<Value>> {
let mut input = Vec::new();
for attachment in &message.attachments {
let path = attachment_path(&inner.session_dir, attachment)?;
if crate::media::media_type_for(attachment).is_some()
&& matches!(inner.transport, Transport::Here)
{
input.push(json!({"type": "localImage", "path": path}));
if let Some(media_type) = crate::media::media_type_for(attachment) {
if matches!(inner.transport, Transport::Here) {
input.push(json!({"type": "localImage", "path": path}));
} else {
// The upload is on the server, not on the machine reached over ssh. Inline image
// input carries those bytes across the app-server connection just as Claude's
// image block does; naming the server path left Codex unable to see it.
input.push(inline_image(&path, media_type)?);
}
} else {
if !text.is_empty() {
text.push_str("\n\n");
@@ -476,6 +483,17 @@ fn input_for(inner: &Inner, message: &Waiting) -> Result<Vec<Value>> {
Ok(input)
}
fn inline_image(path: &Path, media_type: &str) -> Result<Value> {
use base64::Engine;
let bytes =
std::fs::read(path).with_context(|| format!("read attachment {}", path.display()))?;
let data = base64::engine::general_purpose::STANDARD.encode(bytes);
Ok(json!({
"type": "image",
"url": format!("data:{media_type};base64,{data}")
}))
}
fn dispatch_waiting(inner: &Arc<Inner>) {
let Some(thread_id) = read_thread(&inner.session_dir) else {
return;
@@ -560,8 +578,14 @@ async fn follow(inner: Arc<Inner>, mut record: process::Record, mut offset: u64)
let stdout = inner.session_dir.join(STDOUT_LOG);
let stderr = inner.session_dir.join(STDERR_LOG);
let mut translator = Translator::default();
// `offset` is the durable boundary after the last complete record. `read_at` may move beyond
// it while app-server is still writing one record. Image-bearing tool results can be several
// megabytes long, and rereading their incomplete prefix every 50 ms made arrival over ssh
// quadratic in time and allocation.
let mut read_at = offset;
let mut pending = Vec::new();
while inner.reading.load(Ordering::SeqCst) {
let (bytes, _) = match process::read_from(&stdout, offset) {
let (bytes, next) = match process::read_from(&stdout, read_at) {
Ok(read) => read,
Err(err) => {
let _ = inner.sink.send(Event::Error {
@@ -570,12 +594,14 @@ async fn follow(inner: Arc<Inner>, mut record: process::Record, mut offset: u64)
return;
}
};
let complete = bytes
read_at = next;
pending.extend_from_slice(&bytes);
let complete = pending
.iter()
.rposition(|byte| *byte == b'\n')
.map(|at| at + 1)
.unwrap_or(0);
for line in String::from_utf8_lossy(&bytes[..complete]).lines() {
for line in String::from_utf8_lossy(&pending[..complete]).lines() {
let Ok(value) = serde_json::from_str::<Value>(line) else {
tracing::warn!(
"unparseable Codex JSONL line: {}",
@@ -587,6 +613,7 @@ async fn follow(inner: Arc<Inner>, mut record: process::Record, mut offset: u64)
}
if complete > 0 {
offset += complete as u64;
pending.drain(..complete);
record.detail = process::Detail::Stdio {
stdout_read: offset,
};
@@ -667,6 +694,9 @@ fn handle_line(inner: &Arc<Inner>, translator: &mut Translator, line: &Value) {
if item.get("type").and_then(Value::as_str) == Some("userMessage") {
announce_user(inner, item);
}
for event in image_events(inner, item) {
let _ = inner.sink.send(event);
}
}
Some("turn/completed") => {
let mut state = inner.state.lock().unwrap();
@@ -686,6 +716,123 @@ fn handle_line(inner: &Arc<Inner>, translator: &mut Translator, line: &Value) {
}
}
/// Images embedded in a structured tool result, copied into the session before the translator's
/// `ToolEnd` is emitted so they stay attached to that call in transcript order.
fn image_events(inner: &Inner, item: &Value) -> Vec<Event> {
let Some(id) = item.get("id").and_then(Value::as_str) else {
return Vec::new();
};
let mut images = Vec::new();
match item.get("type").and_then(Value::as_str) {
Some("dynamicToolCall") => save_data_images(
&inner.session_dir,
item.get("contentItems").and_then(Value::as_array),
"inputImage",
"imageUrl",
&mut images,
),
Some("mcpToolCall") => {
if let Some(parts) = item.pointer("/result/content").and_then(Value::as_array) {
for part in parts {
if part.get("type").and_then(Value::as_str) != Some("image") {
continue;
}
if let (Some(data), Some(media_type)) = (
part.get("data").and_then(Value::as_str),
part.get("mimeType")
.or_else(|| part.get("mime_type"))
.and_then(Value::as_str),
) && let Some(image) =
save_base64_image(&inner.session_dir, media_type, data)
{
images.push(image);
}
}
}
}
Some("functionCallOutput") => save_data_images(
&inner.session_dir,
item.get("output").and_then(Value::as_array),
"input_image",
"image_url",
&mut images,
),
Some("imageView") => {
if let Some(path) = item.get("path").and_then(Value::as_str)
&& let Some(image) = save_viewed_image(inner, path)
{
images.push(image);
}
}
_ => {}
}
images
.into_iter()
.map(|image| Event::Image {
image,
about: Some(id.to_string()),
})
.collect()
}
fn save_data_images(
session_dir: &Path,
parts: Option<&Vec<Value>>,
image_kind: &str,
url_field: &str,
images: &mut Vec<String>,
) {
for part in parts.into_iter().flatten() {
if part.get("type").and_then(Value::as_str) != Some(image_kind) {
continue;
}
let Some(url) = part.get(url_field).and_then(Value::as_str) else {
continue;
};
let Some((header, data)) = url.split_once(',') else {
continue;
};
let Some(media_type) = header
.strip_prefix("data:")
.and_then(|header| header.strip_suffix(";base64"))
else {
continue;
};
if let Some(image) = save_base64_image(session_dir, media_type, data) {
images.push(image);
}
}
}
fn save_base64_image(session_dir: &Path, media_type: &str, data: &str) -> Option<String> {
use base64::Engine;
let bytes = base64::engine::general_purpose::STANDARD
.decode(data)
.ok()?;
store_image(session_dir, media_type, &bytes)
}
fn save_viewed_image(inner: &Inner, path: &str) -> Option<String> {
let media_type = crate::media::media_type_for(path).unwrap_or("image/png");
let bytes = match &inner.transport {
Transport::Here => std::fs::read(path).map_err(anyhow::Error::from),
Transport::Ssh { .. } => tokio::task::block_in_place(|| {
inner.transport.capture_bytes_blocking(&Launch::new(
"cat",
vec![path.to_string()],
None,
))
}),
};
match bytes {
Ok(bytes) => store_image(&inner.session_dir, media_type, &bytes),
Err(err) => {
tracing::error!("couldn't save image Codex read from {path}: {err:#}");
None
}
}
}
fn handle_response(inner: &Arc<Inner>, line: &Value) {
let Some(id) = line.get("id").and_then(Value::as_str) else {
return;
@@ -707,6 +854,9 @@ fn handle_response(inner: &Arc<Inner>, line: &Value) {
let method = match read_thread(&inner.session_dir) {
Some(thread) => {
params["threadId"] = Value::String(thread);
// This app already owns and pages its common transcript. Hydrating the complete
// Codex history here sends it a second time, including every base64 screenshot.
params["excludeTurns"] = Value::Bool(true);
"thread/resume"
}
None => "thread/start",
@@ -918,6 +1068,42 @@ done
printf '%s\n' "$state"
"#;
const READ_CONTEXT_SCRIPT: &str = r#"
for f in "$HOME"/.codex/sessions/*/*/*/rollout-*-${1}.jsonl; do
[ -f "$f" ] || continue
grep '"type":"token_count"' "$f" | tail -1
exit 0
done
"#;
/// The last measured prompt size from Codex's own rollout, for a session whose common transcript
/// predates context events. This is the same `last.inputTokens` app-server reports live, under the
/// rollout writer's snake-case names.
pub async fn context_of(transport: &Transport, id: &str) -> Option<u64> {
if !valid_thread_id(id) {
return None;
}
let launch = Launch::new(
"sh",
vec![
"-c".to_string(),
READ_CONTEXT_SCRIPT.to_string(),
"sh".to_string(),
id.to_string(),
],
None,
);
let line = transport.capture(&launch).await.ok()?;
context_from_rollout(&line)
}
fn context_from_rollout(line: &str) -> Option<u64> {
serde_json::from_str::<Value>(line)
.ok()?
.pointer("/payload/info/last_token_usage/input_tokens")?
.as_u64()
}
/// Removes the rollout whose suffix is this thread id.
pub async fn delete_transcript(transport: &Transport, id: &str) -> Result<()> {
if !valid_thread_id(id) {
@@ -964,6 +1150,39 @@ mod tests {
assert_eq!(state.waiting[0].id, "q1");
}
#[test]
fn a_structured_image_is_saved_outside_the_transcript() {
let dir = tempfile::tempdir().expect("tempdir");
let parts = vec![json!({
"type": "inputImage",
"imageUrl": "data:image/png;base64,aGVsbG8="
})];
let mut images = Vec::new();
save_data_images(
dir.path(),
Some(&parts),
"inputImage",
"imageUrl",
&mut images,
);
assert_eq!(images.len(), 1);
assert_eq!(
std::fs::read(dir.path().join("files").join(&images[0])).expect("saved image"),
b"hello"
);
}
#[test]
fn an_inline_image_carries_its_bytes_to_a_remote_codex() {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("shot.png");
std::fs::write(&path, b"hello").expect("image");
assert_eq!(
inline_image(&path, "image/png").expect("inline image"),
json!({"type": "image", "url": "data:image/png;base64,aGVsbG8="})
);
}
#[test]
fn transcript_delete_resolves_only_the_named_codex_rollout() {
use std::process::Command;
@@ -985,4 +1204,15 @@ mod tests {
assert!(other.exists());
assert!(!valid_thread_id("../../something"));
}
#[test]
fn context_is_read_from_the_last_codex_model_call() {
assert_eq!(
context_from_rollout(
r#"{"type":"event_msg","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":118866,"cached_input_tokens":118144,"output_tokens":37,"total_tokens":118903}}}}"#
),
Some(118_866)
);
assert_eq!(context_from_rollout(""), None);
}
}
+113 -29
View File
@@ -4,8 +4,6 @@
//! therefore match only the records that have a useful common equivalent and
//! ignore the rest; an added Codex item must not make a live session go deaf.
use std::collections::HashSet;
use serde_json::{Value, json};
use super::super::driver::{Event, SessionStatus};
@@ -15,8 +13,12 @@ pub(super) struct Translator {
pub(super) thread_id: Option<String>,
completed: bool,
limited: bool,
streamed_messages: HashSet<String>,
pending_usage: Option<u64>,
pending_usage: Option<Usage>,
}
struct Usage {
tokens: u64,
context: Option<u64>,
}
impl Translator {
@@ -43,23 +45,23 @@ impl Translator {
self.completed = false;
self.limited = false;
self.pending_usage = None;
self.streamed_messages.clear();
vec![Event::Status {
state: SessionStatus::Running,
}]
}
Some("item.started") | Some("item/started") => start_item(&body["item"]),
Some("item.updated") => update_item(&line["item"]),
Some("item.completed") | Some("item/completed") => {
complete_item(&body["item"], &self.streamed_messages)
}
// The old `codex exec --json` dialect reports only the completed message. App-server
// reports every message through durable delta notifications and its completed copy
// must always be skipped. That rule cannot live in an in-memory set: after a backend
// restart the deltas are behind the persisted log cursor while the completion is not,
// which used to append the whole message again after its already-recorded prefix.
Some("item.completed") => complete_item(&body["item"], true),
Some("item/completed") => complete_item(&body["item"], false),
Some("item/agentMessage/delta") => {
let Some(delta) = body.get("delta").and_then(Value::as_str) else {
return Vec::new();
};
if let Some(id) = body.get("itemId").and_then(Value::as_str) {
self.streamed_messages.insert(id.to_string());
}
vec![Event::AssistantText {
delta: delta.to_string(),
}]
@@ -77,18 +79,24 @@ impl Translator {
}]
}
Some("thread/tokenUsage/updated") => {
self.pending_usage = body
.pointer("/tokenUsage/last/totalTokens")
.and_then(Value::as_u64);
let last = &body["tokenUsage"]["last"];
self.pending_usage =
last.get("totalTokens")
.and_then(Value::as_u64)
.map(|tokens| Usage {
tokens,
// Cached input is a subset of this figure, not an additional count.
context: last.get("inputTokens").and_then(Value::as_u64),
});
Vec::new()
}
Some("turn.completed") | Some("turn/completed") => {
self.completed = true;
let mut events = Vec::new();
if let Some(tokens) = self.pending_usage.take() {
if let Some(usage) = self.pending_usage.take() {
events.push(Event::UsageDelta {
tokens,
context: None,
tokens: usage.tokens,
context: usage.context,
});
} else if let Some(usage) = line.get("usage") {
let input = number(usage, "input_tokens");
@@ -96,9 +104,7 @@ impl Translator {
if input.is_some() || output.is_some() {
events.push(Event::UsageDelta {
tokens: input.unwrap_or(0) + output.unwrap_or(0),
// `exec` reports the sum across every model call in
// a turn, not the final call's context.
context: None,
context: input,
});
}
}
@@ -169,17 +175,13 @@ fn update_item(item: &Value) -> Vec<Event> {
.unwrap_or_default()
}
fn complete_item(item: &Value, streamed_messages: &HashSet<String>) -> Vec<Event> {
fn complete_item(item: &Value, include_agent_message: bool) -> Vec<Event> {
match item.get("type").and_then(Value::as_str) {
Some("agent_message" | "agentMessage") => item
.get("text")
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
.filter(|_| {
item.get("id")
.and_then(Value::as_str)
.is_none_or(|id| !streamed_messages.contains(id))
})
.filter(|_| include_agent_message)
.map(|delta| {
vec![Event::AssistantText {
delta: delta.to_string(),
@@ -217,6 +219,24 @@ fn tool(item: &Value) -> Option<(String, String, Value)> {
.unwrap_or_else(|| "mcp".to_string()),
item.get("arguments").cloned().unwrap_or(Value::Null),
),
"dynamicToolCall" => (
item.get("tool")
.and_then(Value::as_str)
.unwrap_or("tool")
.to_string(),
item.get("arguments").cloned().unwrap_or(Value::Null),
),
"functionCallOutput" => (
item.get("name")
.and_then(Value::as_str)
.unwrap_or("tool")
.to_string(),
Value::Null,
),
"imageView" => (
"view_image".to_string(),
json!({"path": item.get("path").cloned().unwrap_or(Value::Null)}),
),
"web_search" | "webSearch" => (
"web_search".to_string(),
json!({"query": item.get("query").cloned().unwrap_or(Value::Null)}),
@@ -228,6 +248,20 @@ fn tool(item: &Value) -> Option<(String, String, Value)> {
}
fn tool_output(item: &Value) -> String {
match item.get("type").and_then(Value::as_str) {
Some("dynamicToolCall") => {
return content_text(item.get("contentItems"), "inputText");
}
Some("mcpToolCall") => {
if let Some(message) = item.pointer("/error/message").and_then(Value::as_str) {
return message.to_string();
}
return content_text(item.pointer("/result/content"), "text");
}
Some("functionCallOutput") => return content_text(item.get("output"), "input_text"),
Some("imageView") => return String::new(),
_ => {}
}
for key in [
"aggregated_output",
"aggregatedOutput",
@@ -247,6 +281,22 @@ fn tool_output(item: &Value) -> String {
}
}
/// Text from a structured result, deliberately excluding its image data. The driver saves images
/// beside the transcript; serializing a data URL here makes a screenshot a megabytes-long line and
/// still cannot draw it.
fn content_text(value: Option<&Value>, text_kind: &str) -> String {
match value {
Some(Value::String(text)) => text.clone(),
Some(Value::Array(parts)) => parts
.iter()
.filter(|part| part.get("type").and_then(Value::as_str) == Some(text_kind))
.filter_map(|part| part.get("text").and_then(Value::as_str))
.collect::<Vec<_>>()
.join("\n"),
_ => String::new(),
}
}
fn value_text(value: &Value) -> Option<String> {
value
.as_str()
@@ -316,7 +366,7 @@ mod tests {
events[0],
Event::UsageDelta {
tokens: 18,
context: None
context: Some(13)
}
);
assert!(translator.completed());
@@ -380,7 +430,7 @@ mod tests {
assert!(
translator
.translate(&line(
r#"{"method":"thread/tokenUsage/updated","params":{"tokenUsage":{"last":{"totalTokens":42}}}}"#
r#"{"method":"thread/tokenUsage/updated","params":{"tokenUsage":{"last":{"inputTokens":39,"cachedInputTokens":30,"outputTokens":3,"reasoningOutputTokens":1,"totalTokens":42},"total":{"inputTokens":100,"cachedInputTokens":80,"outputTokens":9,"reasoningOutputTokens":2,"totalTokens":109},"modelContextWindow":258400}}}"#
))
.is_empty()
);
@@ -391,7 +441,7 @@ mod tests {
vec![
Event::UsageDelta {
tokens: 42,
context: None
context: Some(39)
},
Event::Status {
state: SessionStatus::Idle
@@ -399,4 +449,38 @@ mod tests {
]
);
}
#[test]
fn structured_tool_results_keep_text_but_not_image_data() {
let mut translator = Translator::default();
let started = translator.translate(&line(
r#"{"method":"item/started","params":{"item":{"id":"tool-1","type":"dynamicToolCall","tool":"view_image","arguments":{"path":"shot.png"},"status":"inProgress"}}}"#,
));
assert!(matches!(&started[0], Event::ToolStart { tool, .. } if tool == "view_image"));
let ended = translator.translate(&line(
r#"{"method":"item/completed","params":{"item":{"id":"tool-1","type":"dynamicToolCall","tool":"view_image","arguments":{},"status":"completed","contentItems":[{"type":"inputText","text":"looked"},{"type":"inputImage","imageUrl":"data:image/png;base64,aGVsbG8="}]}}}"#,
));
assert_eq!(
ended,
vec![Event::ToolEnd {
id: "tool-1".to_string(),
output: "looked".to_string()
}]
);
}
#[test]
fn an_app_server_completion_never_repeats_streamed_text_after_adoption() {
// A newly adopted translator has not seen the deltas already recorded by the previous
// backend. The dialect, rather than process-local memory, decides that this is a copy.
let mut adopted = Translator::default();
assert!(
adopted
.translate(&line(
r#"{"method":"item/completed","params":{"item":{"id":"message-1","type":"agentMessage","text":"the complete message"}}}"#
))
.is_empty()
);
}
}
+21
View File
@@ -15,6 +15,27 @@ use tokio::sync::mpsc;
/// renders them identically.
pub type ImageRef = String;
/// Stores image bytes where the files route serves them and returns their transcript reference.
/// Both CLI dialects produce images in different envelopes; the durable file and naming rule are
/// part of the common event model and must not vary with that envelope.
pub(in crate::session) fn store_image(
session_dir: &std::path::Path,
media_type: &str,
bytes: &[u8],
) -> Option<ImageRef> {
let extension = crate::media::extension_for(media_type).unwrap_or("png");
let name = format!("{}.{extension}", super::random_hex());
let dir = session_dir.join("files");
if let Err(err) = wg_app_link::private::create_dir(&dir)
.map_err(std::io::Error::other)
.and_then(|()| std::fs::write(dir.join(&name), bytes))
{
tracing::error!("couldn't save produced image: {err}");
return None;
}
Some(name)
}
/// The name an upload is stored and served under: an image is
/// `<hex>.<extension>` and is an [`ImageRef`] like any other; any other file
/// keeps its own name after the hex, `<hex>-<name>`, because the name is what
+16
View File
@@ -2416,6 +2416,22 @@ fn launch(
}
});
}
if provider.kind == DriverKind::CodexCli
&& shared.context_tokens.lock().unwrap().is_none()
&& let Some(thread_id) = codex::read_thread(&dir)
{
let transport = Transport::for_setup(setup);
let shared = Arc::clone(&shared);
tokio::spawn(async move {
if let Some(context) = codex::context_of(&transport, &thread_id).await {
// A live turn may have answered while the rollout read was in flight.
let mut held = shared.context_tokens.lock().unwrap();
if held.is_none() {
*held = Some(context);
}
}
});
}
// An imported session shares its transcript file with the CLI, so work
// done at a terminal belongs in this session too and arrives without
+7 -1
View File
@@ -252,6 +252,12 @@ impl Transport {
/// would otherwise need a runtime to ask a machine a question. Both build the
/// invocation the same way.
pub fn capture_blocking(&self, launch: &Launch) -> Result<String> {
Ok(String::from_utf8_lossy(&self.capture_bytes_blocking(launch)?).into_owned())
}
/// The byte-preserving form of [`capture_blocking`](Self::capture_blocking), used when a
/// driver copies a file back from the machine it runs on.
pub fn capture_bytes_blocking(&self, launch: &Launch) -> Result<Vec<u8>> {
let host = match self {
Self::Here => None,
Self::Ssh { ssh, .. } => Some(ssh),
@@ -273,7 +279,7 @@ impl Transport {
stderr
});
}
Ok(String::from_utf8_lossy(&output.stdout).into_owned())
Ok(output.stdout)
}
/// Runs `launch` with `input` on its stdin and reports everything it