Serve a machine's models from one shared llama-server
A llama.cpp session had its own `llama-server`: two sessions on one model held two copies of it in memory, a model change bought a load only that session benefited from, and the process was a session's to end. A machine's models are now served by one `llama-server` in **router mode** -- no `-m`, a preset file naming models and their flags, a child server per model asked for, and each request routed by its `model` field. So one server per model with that model's own settings is what a machine runs, while this backend has one process, one port and one record per machine to keep track of. The record is the mechanism every other driver already uses, so a restart adopts it; a session records the same pid in its own directory as `Detail::Shared`, and `process::signal` refuses to signal one of those -- which is what keeps stopping, deleting or cleaning up after one session from unloading a model every other session is using. Nothing stops a router on its own. That is deliberate (a loaded model is minutes of disk) and it is why the machines tab now has a card per provider that opens its own screen: how each model is loaded, how many stay in memory, Unload, and Stop. How a model is *loaded* therefore belongs to the model on its machine rather than to a session -- context size, GPU layers, threads, slots, speculative decoding -- written into the preset as llama-server's own argument names. Saving them re-reads that file, which unloads the model; that is the change taking effect, and the dialog says so before you save. What stays a session's is everything that rides on a request, including which tools it offers: the router hosts one set for the machine and the choice is a filter applied here, so it costs no reload (2,181 tokens of prompt with all seven, 698 with none). Verified end to end against the scratch backend and the emulator: two sessions sharing one loaded model with one child process, a second session joining it with a 26ms prefill, a backend restart adopting the router and answering with the prompt cache intact, the same over ssh to this VM, a model's settings reaching the running server, Unload, and Stop leaving every session `exited` with no error line.
This commit is contained in:
1 parent
74cda485e5
commit
8c323fc7a9
19 files changed
+2601
-511
No files matched your search
@@ -1,5 +1,6 @@
|
||||
package com.example.aiapp
|
||||
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
@@ -10,6 +11,7 @@ import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.lazy.LazyColumn
|
||||
import androidx.compose.material3.AlertDialog
|
||||
import androidx.compose.material3.Card
|
||||
import androidx.compose.material3.CardDefaults
|
||||
import androidx.compose.material3.CircularProgressIndicator
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.OutlinedTextField
|
||||
@@ -24,6 +26,8 @@ import androidx.compose.runtime.rememberCoroutineScope
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.semantics.contentDescription
|
||||
import androidx.compose.ui.semantics.semantics
|
||||
import androidx.compose.ui.unit.dp
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.launch
|
||||
@@ -37,7 +41,12 @@ import kotlinx.coroutines.withContext
|
||||
* which is what keeps the enrolled token from being able to introduce commands.
|
||||
*/
|
||||
@Composable
|
||||
fun MachinesScreen(settings: ServerSettings, reloadToken: Int) {
|
||||
fun MachinesScreen(
|
||||
settings: ServerSettings,
|
||||
reloadToken: Int,
|
||||
/** Opens one provider on one machine -- its settings, and what its server is holding. */
|
||||
onProvider: (String, String) -> Unit,
|
||||
) {
|
||||
val scope = rememberCoroutineScope()
|
||||
var state by remember { mutableStateOf<LoadState<List<Machine>>>(LoadState.Loading) }
|
||||
var adding by remember { mutableStateOf(false) }
|
||||
@@ -108,6 +117,7 @@ fun MachinesScreen(settings: ServerSettings, reloadToken: Int) {
|
||||
},
|
||||
onDelete = { confirmingDelete = machine },
|
||||
onSignIn = { provider -> signingIn = machine to provider },
|
||||
onProvider = { provider -> onProvider(machine.id, provider.name) },
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -214,6 +224,7 @@ private fun MachineCard(
|
||||
onRediscover: () -> Unit,
|
||||
onDelete: () -> Unit,
|
||||
onSignIn: (Provider) -> Unit,
|
||||
onProvider: (Provider) -> Unit,
|
||||
) {
|
||||
Card(Modifier.fillMaxWidth().padding(vertical = 4.dp)) {
|
||||
Column(Modifier.padding(12.dp)) {
|
||||
@@ -233,14 +244,34 @@ private fun MachineCard(
|
||||
)
|
||||
} else {
|
||||
machine.providers.forEach { provider ->
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
modifier = Modifier.fillMaxWidth(),
|
||||
// A card of its own rather than a line of text: a provider is where the
|
||||
// settings that belong to *this machine* live -- how each of its models is
|
||||
// loaded, and the server holding them -- and those had nowhere to be until
|
||||
// one llama-server came to serve every session on a machine.
|
||||
Card(
|
||||
Modifier.fillMaxWidth().padding(vertical = 2.dp).clickable {
|
||||
onProvider(provider)
|
||||
},
|
||||
colors =
|
||||
CardDefaults.cardColors(
|
||||
containerColor = MaterialTheme.colorScheme.surfaceVariant
|
||||
),
|
||||
) {
|
||||
Text(provider.name, style = MaterialTheme.typography.bodySmall)
|
||||
if (provider.kind == "claude_cli") {
|
||||
Row(
|
||||
verticalAlignment = Alignment.CenterVertically,
|
||||
modifier = Modifier.fillMaxWidth().padding(horizontal = 12.dp),
|
||||
) {
|
||||
Text(provider.name, style = MaterialTheme.typography.bodyMedium)
|
||||
Spacer(Modifier.weight(1f))
|
||||
TextButton(onClick = { onSignIn(provider) }) { Text("Sign in") }
|
||||
if (provider.kind == "claude_cli") {
|
||||
TextButton(onClick = { onSignIn(provider) }) { Text("Sign in") }
|
||||
}
|
||||
Chevron(
|
||||
Pointing.Right,
|
||||
Modifier.padding(start = 4.dp).semantics {
|
||||
contentDescription = "Settings for ${provider.name}"
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user