Serve a machine's models from one shared llama-server

A llama.cpp session had its own `llama-server`: two sessions on one model held two copies of it in memory, a model change bought a load only that session benefited from, and the process was a session's to end. A machine's models are now served by one `llama-server` in **router mode** -- no `-m`, a preset file naming models and their flags, a child server per model asked for, and each request routed by its `model` field. So one server per model with that model's own settings is what a machine runs, while this backend has one process, one port and one record per machine to keep track of. The record is the mechanism every other driver already uses, so a restart adopts it; a session records the same pid in its own directory as `Detail::Shared`, and `process::signal` refuses to signal one of those -- which is what keeps stopping, deleting or cleaning up after one session from unloading a model every other session is using. Nothing stops a router on its own. That is deliberate (a loaded model is minutes of disk) and it is why the machines tab now has a card per provider that opens its own screen: how each model is loaded, how many stay in memory, Unload, and Stop. How a model is *loaded* therefore belongs to the model on its machine rather than to a session -- context size, GPU layers, threads, slots, speculative decoding -- written into the preset as llama-server's own argument names. Saving them re-reads that file, which unloads the model; that is the change taking effect, and the dialog says so before you save. What stays a session's is everything that rides on a request, including which tools it offers: the router hosts one set for the machine and the choice is a filter applied here, so it costs no reload (2,181 tokens of prompt with all seven, 698 with none). Verified end to end against the scratch backend and the emulator: two sessions sharing one loaded model with one child process, a second session joining it with a 26ms prefill, a backend restart adopting the router and answering with the prompt cache intact, the same over ssh to this VM, a model's settings reaching the running server, Unload, and Stop leaving every session `exited` with no error line.
2026-09-19 17:37:31 -04:00
1 parent 74cda485e5
commit 8c323fc7a9
@@ -403,20 +403,20 @@ private fun parseProvider(provider: JSONObject): Provider {
        models = provider.optJSONArray("models")?.strings().orEmpty(),
        permissionModes = provider.optJSONArray("permissionModes")?.strings().orEmpty(),
        defaultPermissionMode = provider.optString("defaultPermissionMode").ifEmpty { null },
-        params =
-            provider.optJSONArray("params")?.mapObjects { spec ->
-                ParamSpec(
-                    key = spec.getString("key"),
-                    label = spec.getString("label"),
-                    unset = spec.getString("unset"),
-                    kind = spec.getString("kind"),
-                    options = spec.optJSONArray("options")?.strings().orEmpty(),
-                    restart = spec.optBoolean("restart"),
-                )
-            } ?: emptyList(),
+        params = provider.optJSONArray("params")?.mapObjects(::parseParamSpec) ?: emptyList(),
    )
 }

+private fun parseParamSpec(spec: JSONObject) =
+    ParamSpec(
+        key = spec.getString("key"),
+        label = spec.getString("label"),
+        unset = spec.getString("unset"),
+        kind = spec.getString("kind"),
+        options = spec.optJSONArray("options")?.strings().orEmpty(),
+        restart = spec.optBoolean("restart"),
+    )
+
 private fun parseMachine(machine: JSONObject) =
    Machine(
        id = machine.getString("id"),
@@ -1473,6 +1473,146 @@ fun fetchProviderModels(
        }
    }

+/**
+ * One provider on one machine: what it is, every model it offers with the settings that decide how
+ * that model is loaded, and what its shared server is holding.
+ *
+ * [server] is null for a provider that runs no server of its own — a CLI, or echo. That is not the
+ * same as a server that is down, and the screen says so differently.
+ */
+data class ProviderView(
+    val machine: String,
+    val name: String,
+    val kind: String,
+    val command: String?,
+    /** What each of this provider's models takes; empty for one that loads no models. */
+    val modelParams: List<ParamSpec>,
+    val maxLoaded: Int?,
+    val models: List<ProviderModel>,
+    val mcpServers: List<String>,
+    val server: ServerState?,
+)
+
+/**
+ * A model this provider offers, its saved settings, and what the server is doing with it.
+ *
+ * [status] is null where nothing was asked — a provider with no server, or one that is not running
+ * — rather than a guess at "unloaded", which is a fact about the server nobody checked.
+ */
+data class ProviderModel(
+    val id: String,
+    val label: String,
+    val settings: Map<String, String>,
+    val status: String?,
+)
+
+/** The shared server behind a provider: whether it is up, and where the backend reaches it. */
+data class ServerState(val running: Boolean, val port: Int?)
+
+private fun providerPath(machineId: String, provider: String, tail: String = "") =
+    "/machines/${machineId.urlEncoded()}/providers/${provider.urlEncoded()}$tail"
+
+fun fetchProvider(settings: ServerSettings, machineId: String, provider: String): ProviderView =
+    requestFromServer(settings, providerPath(machineId, provider), readTimeoutMs = 40000) {
+        val body = it.jsonObject()
+        ProviderView(
+            machine = body.getString("machine"),
+            name = body.getString("name"),
+            kind = body.getString("kind"),
+            command = body.optString("command").ifEmpty { null },
+            modelParams = body.optJSONArray("modelParams")?.mapObjects(::parseParamSpec).orEmpty(),
+            maxLoaded = if (body.isNull("maxLoaded")) null else body.optInt("maxLoaded"),
+            models =
+                body
+                    .optJSONArray("models")
+                    ?.mapObjects { model ->
+                        ProviderModel(
+                            id = model.getString("id"),
+                            label = model.getString("label"),
+                            settings = model.optJSONObject("settings").stringMap(),
+                            status = model.optString("status").ifEmpty { null },
+                        )
+                    }
+                    .orEmpty(),
+            mcpServers = body.optJSONArray("mcpServers")?.strings().orEmpty(),
+            server =
+                body.optJSONObject("server")?.let { server ->
+                    ServerState(
+                        running = server.optBoolean("running", false),
+                        port = if (server.isNull("port")) null else server.optInt("port"),
+                    )
+                },
+        )
+    }
+
+/** How many models this provider's server keeps loaded at once; null for the default. */
+fun setProviderSettings(
+    settings: ServerSettings,
+    machineId: String,
+    provider: String,
+    maxLoaded: Int?,
+) {
+    requestFromServer(
+        settings,
+        providerPath(machineId, provider, "/settings"),
+        method = "POST",
+        jsonBody =
+            JSONObject().apply { if (maxLoaded != null) put("maxLoaded", maxLoaded) }.toString(),
+    ) {}
+}
+
+/**
+ * How one model is loaded. Whole, like a session's params: what is absent is unset.
+ *
+ * Slow on purpose: the backend writes it where the machine's server reads it, which unloads the
+ * model if it was loaded — that is the change taking effect, not a side effect.
+ */
+fun setModelSettings(
+    settings: ServerSettings,
+    machineId: String,
+    provider: String,
+    model: String,
+    params: Map<String, String>,
+) {
+    requestFromServer(
+        settings,
+        providerPath(machineId, provider, "/model-settings"),
+        method = "POST",
+        jsonBody =
+            JSONObject()
+                .put("model", model)
+                .put("params", JSONObject(params as Map<*, *>))
+                .toString(),
+        readTimeoutMs = 40000,
+    ) {}
+}
+
+/** Ends the machine's shared server and everything it was holding. */
+fun stopProviderServer(settings: ServerSettings, machineId: String, provider: String) {
+    requestFromServer(
+        settings,
+        providerPath(machineId, provider, "/stop"),
+        method = "POST",
+        readTimeoutMs = 40000,
+    ) {}
+}
+
+/** Takes one model out of memory, leaving the server and every other model alone. */
+fun unloadProviderModel(
+    settings: ServerSettings,
+    machineId: String,
+    provider: String,
+    model: String,
+) {
+    requestFromServer(
+        settings,
+        providerPath(machineId, provider, "/unload"),
+        method = "POST",
+        jsonBody = JSONObject().put("model", model).toString(),
+        readTimeoutMs = 40000,
+    ) {}
+}
+
 fun fetchModels(settings: ServerSettings): Models =
    requestFromServer(settings, "/models") { connection ->
        val body = JSONObject(connection.inputStream.bufferedReader().readText())