From 7e7910083c0eaac810cc1e751cbfaccba461c33a Mon Sep 17 00:00:00 2001 From: iris-ai <4+iris-ai@noreply.localhost> Date: Mon, 21 Sep 2026 01:37:36 -0400 Subject: [PATCH] Let a machine have more than one llama.cpp A model whose kernels are not upstream needs the fork that has them, and the ordinary models still want the ordinary build. Anything under `~/.local/share/ai-app/llama//` -- `llama-server`, or the `bin/llama-server` a `cmake --install --prefix` leaves -- is now discovered beside the one on PATH and becomes a provider called `llama-cpp-`, with its own router, preset and model settings. That keeps the module's security property rather than bending it: the phone still names no command, because what runs is still decided by what somebody put on the machine. Each probe answer is tagged with what was asked for, since two of these are now the same program under different paths. Flash attention joins the model settings (`flash-attn` in the preset). llama.cpp's `auto` stays the default; the control is for a model whose publisher asks for `on` outright, which Prism ML's ternary Bonsai does. Verified against the fork built into that directory: discovery answers `llama-cpp-prism`, the child server is started with `--flash-attn on`, and Ternary-Bonsai-2-27B PTQ1_0 loads and answers through a session. --- .claude/skills/ai-app-rigs/SKILL.md | 32 ++++++ AGENTS.md | 10 ++ PLAN.md | 29 ++++++ server/src/config.rs | 13 +++ server/src/machines.rs | 151 +++++++++++++++++++++++++--- server/src/session/llama/router.rs | 1 + 6 files changed, 220 insertions(+), 16 deletions(-) diff --git a/.claude/skills/ai-app-rigs/SKILL.md b/.claude/skills/ai-app-rigs/SKILL.md index cd1b57d..f3de233 100644 --- a/.claude/skills/ai-app-rigs/SKILL.md +++ b/.claude/skills/ai-app-rigs/SKILL.md @@ -174,6 +174,38 @@ Two models are downloaded under `~/.local/share/ai-app/models`: projector being kept out of the models a provider offers. Qwen3-0.6B beside it is the other half of that rig -- the model that answers `refused`. +**A second llama.cpp is installed here, and it is the rig for a custom +build.** `~/.local/share/ai-app/llama/prism/` is Prism ML's fork +(`prism` branch, `~/repos/llama.cpp-prism`, Vulkan, `cmake --install +--prefix`), so discovery finds it as a provider called `llama-cpp-prism` +beside the ordinary `llama-cpp`. It is what exercises that mechanism at all, +and it serves `prism-ml/Ternary-Bonsai-2-27B-gguf` -- ternary packings stock +llama.cpp rejects as unknown types. Rebuild it with +`cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON +-DCMAKE_INSTALL_RPATH='$ORIGIN/../lib'`, about six minutes at `-j8`. + +**Which Bonsai packing runs on the GPU is the backend's question, not the +model's.** Measured 2026-09-21 with `llama-bench -p 512 -n 64 -r 2 -fa 1 +-ngl 99` on the free card: + +| packing | backend | pp512 | tg64 | +| --- | --- | ---: | ---: | +| `PTQ1_0`, 5.53 GiB | Vulkan | 519 t/s | 7.5 t/s | +| `PQ2_0`, 7.21 GiB | **CPU**, 8 cores | unfinished after 9 min | -- | + +The fork's Vulkan port covers `PTQ1_0` only -- shaders, a `mul_mat_vec` and +the FWHT included -- so `PQ2_0` has no kernel there and every matmul falls +back to the CPU. That reads exactly like a stuck load: the process sits at +700% CPU for minutes with the card idle. On CUDA and HIP it is the other way +round, since `mmq.cu` guards `PTQ1_0` out of the HIP build (it wants Turing +MMA) and leaves `PQ2_0` in. **ROCm cannot be tested in this VM**: there is no +`/dev/kfd`, because the GPU here is virtio-gpu rather than a passed-through +card. + +7.5 tok/s is the honest speed of that Vulkan kernel, against 42 for the +IQ3_S 27B beside it -- smaller weights, slower decode. Nothing is +misconfigured; the fork's fast kernels are CUDA and Metal. + **Do not test with a 2-bit quant**: the IQ2_XXS of the 0.6B produces fluent nonsense, which reads exactly like a broken driver — `llama-cli` produces the same from the file directly, which is how to tell the two apart in a hurry. diff --git a/AGENTS.md b/AGENTS.md index 8e145c8..c47749c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,16 @@ Module-by-module intent is in PLAN.md's "Backend layout". that model; and the preset is read back before every edit, because a router adopted from an earlier run is serving sections this process has never seen and rewriting without them unloads those. + **A machine can have more than one llama.cpp** (2026-09-21, + `machines.rs`): anything at `~/.local/share/ai-app/llama//llama-server` + or `...//bin/llama-server` is discovered beside the one on PATH and + becomes a provider called `llama-cpp-`, with its own router, preset + and model settings. That is how a model whose kernels are not upstream is + served -- Prism ML's ternary Bonsai is the one here, built from the + `prism` branch into `~/.local/share/ai-app/llama/prism` -- without the + phone ever naming a command, which is the property this module exists for. + Both providers offer the machine's whole models directory, since which + build reads which packing is not answerable from the file. **A llama.cpp session runs on its configured machine** (built 2026-09-04, the last of phase 5): `Transport::reserve_port` returns the port the server binds *there* and the port that reaches it *here*, and diff --git a/PLAN.md b/PLAN.md index 1d2ca01..0777458 100644 --- a/PLAN.md +++ b/PLAN.md @@ -74,6 +74,17 @@ a control that silently did nothing). enrolled token cannot introduce a command. The escape hatch for a binary somewhere unusual is editing `config.ron`, deliberately the one authority the phone does not have. +- **A machine may have more than one llama.cpp** (2026-09-21). Anything under + `~/.local/share/ai-app/llama//` — `llama-server`, or the + `bin/llama-server` a `cmake --install --prefix` leaves — is discovered + beside the one on PATH and becomes a provider called `llama-cpp-`. + This is the same authority as before, not a new one: what runs is still + decided by what somebody put on the machine, and the phone still names no + command. It exists because one build is not enough — a model whose kernels + are not upstream needs the fork that has them, and the ordinary models + should keep being served by the ordinary build. A provider of its own is + what that has to be, because a router process, its preset file and a + model's load settings all hang off the provider. - **Migration code is deleted once the update carrying it is received.** The providers/hosts migration ran on the one host there is and is gone. A file in the old shape now fails to parse, which is correct because no such file @@ -556,6 +567,24 @@ deliberate and easy to undo by accident: server that cannot load it, and it stays in the machine's own model list, which is where a file on a disk is managed. +- **A build that is not the machine's `llama-server` is another provider** + (2026-09-21, the case being Prism ML's ternary Bonsai, whose `PTQ1_0` and + `PQ2_0` packings stock llama.cpp rejects as unknown types). Nothing about + the fork is named anywhere: it is a directory under + `~/.local/share/ai-app/llama/`, found by the same probe that finds the + binaries on PATH, and everything downstream — the router, the preset file, + a model's load settings, a session — already hangs off the provider, so + there is nothing for a second build to special-case. The consequence worth + knowing is that both providers offer the machine's whole models directory, + including the files the other one is the answer for: which build reads + which packing is not something this side can tell from the file, and a + model that will not load says so in the transcript. + - **Flash attention is a model setting** (`flashAttention`, written as the + preset's `flash-attn`). llama.cpp's own `auto` remains the default and is + the right one; the control exists because a publisher can ask for `on` + outright, which is a statement about the file rather than about the + backend that would be deciding. + ### Models (2026-08-28, rebuilt per machine 2026-09-19) - **A download belongs to the model, not to the request.** Keyed by diff --git a/server/src/config.rs b/server/src/config.rs index 3fe6a88..467c1c7 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -550,6 +550,19 @@ pub const LLAMA_MODEL_PARAMS: &[ParamSpec] = &[ kind: ParamKind::Integer, restart: true, }, + ParamSpec { + key: "flashAttention", + label: "Flash attention", + // llama.cpp's `auto` is the right default and stays it. The control + // is here because a model's publisher can ask for `on` outright -- + // Prism ML's ternary Bonsai does -- which is a statement about the + // file that `auto` would be taking from the backend instead. + unset: "llama.cpp's own choice", + kind: ParamKind::Choice { + options: &["auto", "on", "off"], + }, + restart: true, + }, ParamSpec { key: "speculative", label: "Speculative decoding", diff --git a/server/src/machines.rs b/server/src/machines.rs index 8e913f8..92f26e4 100644 --- a/server/src/machines.rs +++ b/server/src/machines.rs @@ -12,7 +12,9 @@ //! //! The cost is that a program somewhere unusual is invisible. The escape hatch //! is editing `config.ron` on the backend, which is exactly the authority the -//! phone is not being given. +//! phone is not being given -- and for the case that keeps arising, a second +//! llama.cpp built to serve a model the ordinary one cannot, there is a +//! directory a build is put in to be found: [`LLAMA_BUILDS`]. use anyhow::{Context, Result}; use serde::Serialize; @@ -38,17 +40,49 @@ const PROBES: &[(&str, &str, DriverKind)] = &[ /// screen, not a restriction -- the field stays free text. const CLAUDE_MODELS: &[&str] = &["fable", "opus", "sonnet", "haiku"]; -/// Asks `transport`'s machine which of [`PROBES`] it has. +/// Where a machine keeps the llama.cpp builds it has besides the one on its +/// PATH: one directory per build, holding either `llama-server` itself or the +/// `bin/llama-server` that `cmake --install` puts there. +/// +/// A model whose kernels are not upstream -- Prism ML's ternary Bonsai +/// packings are the case this was built for -- needs the fork that has them, +/// while the ordinary models still want the ordinary build. So each directory +/// found here is a provider of its own, with its own router process, its own +/// model settings and its own sessions. +/// +/// **A directory rather than a path the phone could type**, which is the whole +/// design of this module: no route accepts a command to run, so putting a +/// build here is a decision somebody makes on the machine itself. The name of +/// the directory is what the provider is called, so it is worth choosing. +const LLAMA_BUILDS: &str = "$HOME/.local/share/ai-app/llama"; + +/// The word a [`LLAMA_BUILDS`] line is tagged with, which is not a program +/// name and so cannot collide with one. +const BUILD: &str = "build"; + +/// Asks `transport`'s machine which of [`PROBES`] it has, and which llama.cpp +/// builds are in [`LLAMA_BUILDS`]. /// /// One round trip rather than one per program: over ssh each would be a separate /// connection and handshake. `command -v` is POSIX and a shell builtin, so it -/// works whatever is installed -- and `|| true` keeps a missing program from -/// ending the loop, since the caller wants the whole answer. +/// works whatever is installed -- and each answer is tagged with what was asked +/// for, since two of these are the same program under different paths. +/// +/// Each test is an `if`'s condition rather than the loop body's last command, +/// so that finding nothing is not the script's exit status -- the caller reads +/// a non-zero exit as a machine it could not reach. And a build has to be a +/// *file*, since a directory carries the executable bit too. pub async fn discover(transport: &Transport) -> Result> { let wanted: Vec<&str> = PROBES.iter().map(|(_, binary, _)| *binary).collect(); let script = format!( - "for p in {}; do command -v \"$p\" || true; done", - wanted.join(" ") + "for p in {wanted}; do \ + if q=$(command -v \"$p\"); then printf '%s\\t%s\\n' \"$p\" \"$q\"; fi; \ + done; \ + d={LLAMA_BUILDS}; \ + for p in \"$d\"/*/llama-server \"$d\"/*/bin/llama-server; do \ + if [ -f \"$p\" ] && [ -x \"$p\" ]; then printf '{BUILD}\\t%s\\n' \"$p\"; fi; \ + done", + wanted = wanted.join(" "), ); let launch = Launch::new("sh", vec!["-c".to_string(), script], None); let found = transport.capture(&launch).await.map_err(explain)?; @@ -60,17 +94,38 @@ pub async fn discover(transport: &Transport) -> Result> { if matches!(transport, Transport::Here) { providers.push(crate::config::Config::echo_provider()); } - for (name, binary, kind) in PROBES { - let path = found - .lines() - .map(str::trim) - .find(|line| line.rsplit('/').next() == Some(*binary)); - let Some(path) = path else { + providers.extend(probed(&found)); + Ok(providers) +} + +/// What the probe script's output says is installed. +/// +/// Separated from the round trip so it can be exercised without a machine. +fn probed(found: &str) -> Vec { + let mut providers: Vec = Vec::new(); + for line in found.lines() { + let Some((key, path)) = line.trim().split_once('\t') else { continue; }; + let (name, kind) = if key == BUILD { + let Some(name) = build_name(path) else { + continue; + }; + (name, DriverKind::LlamaCpp) + } else { + let Some((name, _, kind)) = PROBES.iter().find(|(_, binary, _)| *binary == key) else { + continue; + }; + ((*name).to_string(), *kind) + }; + // A directory holding both shapes is matched by both globs. Two + // providers of one name is a config that silently loses one of them. + if providers.iter().any(|already| already.name == name) { + continue; + } providers.push(ProviderConfig { - name: (*name).to_string(), - kind: *kind, + name, + kind, // The resolved path rather than the bare name: PATH under a // non-interactive ssh session is not the one a person sees when they // log in, so "it is on my PATH" is not enough. @@ -79,7 +134,7 @@ pub async fn discover(transport: &Transport) -> Result> { DriverKind::ClaudeCli => CLAUDE_MODELS.iter().map(|m| (*m).to_string()).collect(), _ => Vec::new(), }, - mcp_servers: mcp_defaults(*kind), + mcp_servers: mcp_defaults(kind), // What a probe cannot know: how this machine's models are loaded // is configured after the fact, and a re-probe keeps it -- see // `SessionManager::update_machine`. @@ -87,7 +142,21 @@ pub async fn discover(transport: &Transport) -> Result> { max_loaded: None, }); } - Ok(providers) + providers +} + +/// What to call the provider for a build found at `path`: the name of its own +/// directory, under the `llama-cpp` the plain one already has. +/// +/// The `bin/` a `cmake --install` produces is not part of the name -- a build +/// installed as a prefix and one that is a single binary in a directory are +/// the same build, and naming them differently would make moving between them +/// lose the model settings kept against the name. +fn build_name(path: &str) -> Option { + let dir = path.rsplit_once('/')?.0; + let dir = dir.strip_suffix("/bin").unwrap_or(dir); + let name = dir.rsplit('/').next()?; + (!name.is_empty()).then(|| format!("llama-cpp-{name}")) } /// One model a picker can offer, and what to call it there. @@ -316,6 +385,56 @@ mod tests { assert_eq!(tidy(" "), None); } + #[test] + fn a_llama_build_is_a_provider_named_for_its_directory() { + let found = "llama-server\t/usr/bin/llama-server\n\ + build\t/home/me/.local/share/ai-app/llama/prism/bin/llama-server\n\ + build\t/home/me/.local/share/ai-app/llama/nightly/llama-server\n"; + let providers = probed(found); + let named: Vec<(&str, &str)> = providers + .iter() + .map(|p| (p.name.as_str(), p.program())) + .collect(); + assert_eq!( + named, + vec![ + ("llama-cpp", "/usr/bin/llama-server"), + ( + "llama-cpp-prism", + "/home/me/.local/share/ai-app/llama/prism/bin/llama-server" + ), + ( + "llama-cpp-nightly", + "/home/me/.local/share/ai-app/llama/nightly/llama-server" + ), + ] + ); + assert!(providers.iter().all(|p| p.kind == DriverKind::LlamaCpp)); + } + + #[test] + fn one_build_matched_twice_is_one_provider() { + let found = "build\t/home/me/.local/share/ai-app/llama/prism/llama-server\n\ + build\t/home/me/.local/share/ai-app/llama/prism/bin/llama-server\n"; + let providers = probed(found); + assert_eq!(providers.len(), 1); + assert_eq!( + providers[0].program(), + "/home/me/.local/share/ai-app/llama/prism/llama-server" + ); + } + + #[test] + fn a_line_this_does_not_understand_is_not_a_provider() { + let found = "warning: something on stderr\n\ + \n\ + ruby\t/usr/bin/ruby\n\ + codex\t/usr/bin/codex\n"; + let providers = probed(found); + assert_eq!(providers.len(), 1); + assert_eq!(providers[0].name, "codex-cli"); + } + #[test] fn codex_models_are_the_selectable_non_hidden_catalog_entries() { let answer = json!({"result": {"data": [ diff --git a/server/src/session/llama/router.rs b/server/src/session/llama/router.rs index c08f5f2..67db18f 100644 --- a/server/src/session/llama/router.rs +++ b/server/src/session/llama/router.rs @@ -675,6 +675,7 @@ fn section(found: &Model, settings: &BTreeMap) -> String { ("contextSize", "ctx-size"), ("gpuLayers", "n-gpu-layers"), ("threads", "threads"), + ("flashAttention", "flash-attn"), // How far ahead the draft head guesses. Not defaulted: 2 measured 7% // faster than llama.cpp's 3 on this machine's GPU, once, which is a // reason to make the knob reachable and not a reason to move it for