iris/ai-app
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prune commentary and stale Rust port notes

Browse Source
This commit is contained in:
iris committed 2026-09-10 00:44:13 -04:00
1 parent 5428cd75c9
commit 25370731d0
193 files changed
+693 -16219
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/src/session/llama.rs
+1 -137
View File
@@ -1,33 +1,3 @@
//! The llama.cpp driver: a `llama-server` process per session, spoken to over
//! its OpenAI-compatible HTTP API and translated into the common event model.
//!
//! Two things make this shaped differently from the Claude driver.
//!
//! **It is spawned but not spoken to over stdio.** The process is started
//! through the same [`Transport`] as any other and then reached over HTTP on a
//! loopback port. That is the second half of what a transport is -- "run this"
//! plus "reach this port" -- and it is what lets a session run on another
//! machine: [`Transport::reserve_port`] hands back a port the server binds
//! *there* and one that reaches it *here*, and the ssh connection carrying the
//! command carries the tunnel between them. The far `llama-server` binds
//! loopback only, so a model is never served to that machine's network.
//!
//! **The model file is the far machine's, not this one's.** A remote setup
//! names its own models directory (`SshConfig::models_dir`, defaulting to where
//! this backend keeps its downloads), and the file is looked for *there* -- so
//! a session naming a model that machine does not have says so, instead of
//! starting a server that will never load one. Downloading to another machine
//! is not built; the model gets there however anything else does.
//!
//! **The server is stateless between requests**, so the whole conversation goes
//! with every one. It is rebuilt from the session's transcript rather than kept
//! in this struct, which is not tidiness: a copy in driver memory is invisible
//! to a second device and gone when this process restarts.
//!
//! That leaves the Claude driver as the odd one out rather than this one -- the
//! CLI's own memory of a conversation is a cache in front of the same
//! transcript. Resolve any inconsistency in this direction.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
@@ -46,7 +16,6 @@ use crate::config::{ProviderConfig, SessionConfig};
/// is generous -- the failure it exists for is a server that will never answer.
const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300);
/// One turn in the conversation this driver keeps on the server's behalf.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct Message {
role: String,
@@ -55,28 +24,16 @@ struct Message {
pub struct LlamaDriver {
sink: EventSink,
/// Where this session's own llama-server answers.
endpoint: String,
/// Where the conversation is read back from, one line per event.
transcript: PathBuf,
/// Sampling settings chosen at spawn, sent with every request.
sampling: serde_json::Map<String, serde_json::Value>,
/// Set by [`Driver::interrupt`]; the streaming loop checks it between
/// chunks and stops, leaving what was generated in the transcript.
cancel: Arc<AtomicBool>,
/// Where this session's process record lives, so [`Driver::stop`] can find
/// the server it has to end.
session_dir: PathBuf,
}
impl LlamaDriver {
/// Takes charge of this session's `llama-server`: the one already loaded if
/// there is one, otherwise a new one.
///
/// One entry point, for the reason `ClaudeDriver::launch` gives, expensive
/// in a different currency: two servers holding the same model is twice the
/// memory, and the second would bind a different port while the phone kept
/// talking to the first.
#[allow(clippy::too_many_arguments)]
pub fn launch(
meta: &SessionConfig,
@@ -86,9 +43,6 @@ impl LlamaDriver {
transcript: &Path,
session_dir: &Path,
sink: EventSink,
// llama.cpp has no notion of a Task call, so this is accepted only
// to keep one shape across every driver's launch -- see
// `SUBAGENTS.md`'s "Server layout".
_subagents: Arc<super::subagent::Subagents>,
) -> Result<Self> {
let model = meta.model.as_deref().context(
@@ -120,17 +74,12 @@ impl LlamaDriver {
));
}
// Where it listens on its own machine, and where that is reached
// from here -- the same number when that machine is this one.
let forward = transport
.reserve_port()
.context("finding a port for llama-server")?;
let mut args: Vec<String> = vec![
"-m".into(),
path.clone(),
// Loopback there, whichever machine there is: what reaches it
// from outside that machine is the ssh tunnel and nothing
// else.
"--host".into(),
"127.0.0.1".into(),
"--port".into(),
@@ -152,10 +101,6 @@ impl LlamaDriver {
let program = provider.program();
let launch = Launch::new(program, args, meta.cwd.as_deref()).reaching(forward);
// Its output goes to files, not pipes. Not only so the process can
// outlive this server: nothing ever read those pipes, so a chatty
// llama-server filled the 64 KB buffer and blocked mid-load with no sign
// of why.
let child = transport.spawn(
&launch,
Streams::Detached {
@@ -203,8 +148,6 @@ impl LlamaDriver {
))
}
/// The driver for a `llama-server` at `endpoint`, however it got there.
///
/// Shared by starting one and adopting one, because everything after "there
/// is a server at this address" is identical -- including waiting for it to
/// answer, which an adopted one still owes: a recorded pid says a process
@@ -217,9 +160,6 @@ impl LlamaDriver {
session_dir: &Path,
sink: EventSink,
) -> Self {
// Loading is slow enough to be worth saying so: the session shows as
// running until the model is in memory, rather than looking ready and
// refusing the first message.
let _ = sink.send(Event::Status {
state: SessionStatus::Running,
});
@@ -273,9 +213,6 @@ impl LlamaDriver {
}
}
/// Where llama-server's own output goes. One file for both streams: it is
/// diagnostics nobody parses, and interleaving them is how it reads in a
/// terminal anyway.
const SERVER_LOG: &str = "llama-server.log";
/// How often a loaded server is checked for still being there. Slower than the
@@ -283,8 +220,6 @@ const SERVER_LOG: &str = "llama-server.log";
/// to notice a server that has gone.
const WATCH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2);
/// An owner-only log opened for appending, so the two streams pointed at
/// it do not overwrite each other and a reattach keeps what came before.
fn log_file(path: &Path) -> Result<std::fs::File> {
use std::os::unix::fs::OpenOptionsExt;
std::fs::OpenOptions::new()
@@ -295,21 +230,12 @@ fn log_file(path: &Path) -> Result<std::fs::File> {
.with_context(|| format!("opening {}", path.display()))
}
/// Reports the server going away, for as long as the session is there to report
/// it to.
///
/// Polled rather than waited on, for the reason the Claude driver gives: after a
/// restart this server is not the process's parent, so liveness has to be a
/// question asked of the record -- and asking it two different ways is how the
/// two answers come to disagree.
fn watch(session_dir: PathBuf, sink: EventSink) {
std::thread::spawn(move || {
loop {
std::thread::sleep(WATCH_INTERVAL);
match process::recorded(&session_dir) {
Some((_, process::Liveness::Alive)) => {}
// Nothing recorded means the session was stopped or deleted
// deliberately, and whoever did that has already said so.
None => return,
Some((_, process::Liveness::Dead)) => {
let _ = sink.send(Event::Error {
@@ -348,8 +274,6 @@ impl Driver for LlamaDriver {
let cancel = Arc::clone(&self.cancel);
cancel.store(false, Ordering::Relaxed);
// Its own thread: the request blocks for as long as the model takes to
// generate, which is the whole point of streaming it.
std::thread::spawn(move || {
// Nothing is ever held back here -- there is no queue to wait in --
// so the message is taken the moment it arrives. Said anyway,
@@ -372,9 +296,6 @@ impl Driver for LlamaDriver {
role: "user".into(),
content: text,
});
// The reply is not stored: the deltas below are the durable record,
// so the next turn reads back exactly what the phone was shown --
// including a partial one that was interrupted.
if let Err(err) = generate(&endpoint, &messages, &sampling, &cancel, &sink) {
let _ = sink.send(Event::Error {
message: format!("{err:#}"),
@@ -386,16 +307,12 @@ impl Driver for LlamaDriver {
});
}
fn answer_question(&self, _id: &str, _answers: &[String]) {
// Nothing here asks questions: this driver has no tools.
}
fn answer_question(&self, _id: &str, _answers: &[String]) {}
fn interrupt(&self) {
self.cancel.store(true, Ordering::Relaxed);
}
// Nothing to forward: this process has no notion of what the conversation
// is called, and the rename has already happened where the name lives.
fn set_title(&self, _title: &str) {}
fn set_permission_mode(&self, _mode: &str) {
@@ -430,15 +347,9 @@ impl Driver for LlamaDriver {
}
fn clear(&self) {
// All of it. `conversation` folds from the last of these, so recording
// the marker *is* the reset -- there is no driver state to keep in step
// with it, which is the same property that makes a second device see the
// same conversation this one does.
let _ = self.sink.send(Event::Cleared);
}
/// Stops generating and leaves the server loaded.
///
/// Worth being deliberate about, because the cost points the other way from
/// the Claude driver's: a `llama-server` holds its whole model in memory, so
/// a leaked one is gigabytes nobody is using. It is left anyway, because the
@@ -458,12 +369,6 @@ impl Driver for LlamaDriver {
}
}
/// The conversation so far, folded out of the transcript.
///
/// Consecutive `AssistantText` deltas are one assistant turn, closed by the next
/// user message -- which is also what makes an interrupted reply come back as
/// the partial text the phone actually saw.
///
/// This must stay a pure function of the transcript and must never re-render
/// earlier turns. llama.cpp caches the prompt prefix, so a growing conversation
/// reprocesses almost nothing -- but only while every turn is byte-identical to
@@ -475,9 +380,6 @@ fn conversation(path: &Path) -> Vec<Message> {
};
let mut messages: Vec<Message> = Vec::new();
let mut pending = String::new();
// Everything before the last clear is still in the transcript and is
// deliberately not in the conversation. Folding from zero would put it back,
// which is the whole of what clearing had to undo.
let events = match events.iter().rposition(|e| e.event == Event::Cleared) {
Some(at) => &events[at + 1..],
None => &events[..],
@@ -509,8 +411,6 @@ fn conversation(path: &Path) -> Vec<Message> {
messages
}
/// Where a model key resolves to on disk, refusing anything that climbs
/// out of the models directory -- the key arrives from a phone.
fn model_path(models_dir: &Path, key: &str) -> Result<PathBuf> {
let mut path = models_dir.to_path_buf();
for part in key.split('/') {
@@ -525,29 +425,16 @@ fn model_path(models_dir: &Path, key: &str) -> Result<PathBuf> {
Ok(path)
}
/// The model file's path **on the machine that will serve it**, confirmed to be
/// there.
///
/// One function rather than a local check and hope for the other case: the same
/// question has to be asked of two filesystems. The remote answer is measured
/// for the reason the local one is -- a missing file otherwise becomes a
/// `llama-server` that starts, fails to load, and reports as a session that
/// never became ready, which reads as the machine being slow.
///
/// One blocking round trip on a remote spawn, which is what the spawn is
/// already paying to start ssh. The alternative is a path built here from a `~`
/// this machine cannot expand.
fn model_on(transport: &Transport, models_dir: &Path, key: &str) -> Result<String> {
let Transport::Ssh { name, .. } = transport else {
return Ok(model_path(models_dir, key)?.to_string_lossy().into_owned());
};
// The same directory the spawn screen listed for this machine, and one
// function for the same reason: a list from one place and a load from
// another is a model that appears and then fails.
let dir = crate::models::dir_on(transport, models_dir);
// Checked here rather than in the script: `..` in a key would walk out of
// the models directory on a machine this server can start processes on,
// and the phone is where the key comes from.
for part in key.split('/') {
if part.is_empty() || part == "." || part == ".." {
bail!("\"{key}\" is not a model key this can resolve");
@@ -580,8 +467,6 @@ fn model_on(transport: &Transport, models_dir: &Path, key: &str) -> Result<Strin
}
}
/// Polls until the server says it is ready, or gives up.
///
/// Watches the process as well as the port, because the two failures need
/// different words and one of them is common: a model that will not load,
/// a port already taken on the far machine, a `llama-server` too old for
@@ -616,8 +501,6 @@ fn wait_until_ready(endpoint: &str, session_dir: &Path) -> Result<()> {
}
}
/// The end of `llama-server`'s own log, for a failure message.
///
/// Its account of what went wrong is the useful half -- "failed to load
/// model", "bind: Address already in use" -- and on a remote session it
/// is the only half, since nobody reading the phone can open a file on
@@ -636,7 +519,6 @@ fn log_tail(session_dir: &Path) -> String {
)
}
/// How much of that log to carry into a message somebody reads on a phone.
const LOG_TAIL_LINES: usize = 6;
/// One streamed completion: posts the conversation, emits each delta as it
@@ -666,16 +548,12 @@ fn generate(
let reader = std::io::BufReader::new(response.body_mut().as_reader());
let mut tokens = 0u64;
// The prompt side only, which is what the model is holding -- the same
// definition the other dialects report, so one word on the phone means one
// thing whichever kind of session it is.
let mut context = None;
for line in std::io::BufRead::lines(reader) {
if cancel.load(Ordering::Relaxed) {
break;
}
let line = line.context("reading the generation stream")?;
// Server-sent events: the payload lines are the ones that matter.
let Some(payload) = line.strip_prefix("data: ") else {
continue;
};
@@ -723,8 +601,6 @@ mod tests {
use super::*;
use crate::session::transcript::Transcript;
/// Writes a transcript the way the pump does, so the fold is tested against
/// the real file format rather than a hand-built vector.
fn transcript_with(events: &[Event]) -> (tempfile::TempDir, PathBuf) {
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.path().join("transcript.jsonl");
@@ -777,10 +653,6 @@ mod tests {
}
#[test]
/// The interrupted case, which decides what a resumed conversation is built
/// from: whatever the phone was shown. The deltas that arrived before the
/// stop are in the transcript, so they are in the prompt -- the model is
/// never told it said something the user did not see.
fn an_interrupted_reply_stays_in_the_conversation() {
let (_dir, path) = transcript_with(&[
Event::UserMessage {
@@ -801,9 +673,6 @@ mod tests {
}
#[test]
/// Events this driver does not produce must not disturb the fold: a
/// transcript can carry errors and status changes from a session that
/// was, say, relaunched.
fn other_events_are_not_part_of_the_conversation() {
let (_dir, path) = transcript_with(&[
Event::Status {
@@ -832,9 +701,6 @@ mod tests {
}
#[test]
/// Clearing decides what the *model* is given, not just what the phone
/// draws. Everything above the marker stays in the transcript and none of it
/// is sent.
fn the_conversation_starts_after_the_last_clear() {
let (_dir, path) = transcript_with(&[
Event::UserMessage {
@@ -862,8 +728,6 @@ mod tests {
}
#[test]
/// The *last* one, so clearing twice does not resurrect what the
/// first clear dropped.
fn only_the_newest_clear_counts() {
let (_dir, path) = transcript_with(&[
Event::UserMessage {