Report the context a session holds, not what it has spent
The number on the status row was a running total of tokens spent, so it could only ever climb: a session compacted from 128k down to 10k, or cleared outright, went on reporting the larger figure, and disagreed with the divider directly above it saying what the compaction had recovered. It now reports what the model is holding -- prompt plus both cache figures -- folded through `driver::context_after`, which is the one rule the pump, the transcript and the phone all use: a turn sets it, a compaction replaces it with what the compaction measured, and a clear leaves it unmeasured. Unmeasured says so in words, because an empty context and one nobody has counted used to look identical. Taken from the turn's last assistant message rather than its `result`: measured against CLI 2.1.237, a two-message turn reported a cache read of 40,211, being 14,259 and 25,952 -- the same conversation counted twice, and no size the model ever held.
This commit is contained in:
1 parent
81c8a57181
commit
5e11b9da80
12 files changed
+442
-130
No files matched your search
@@ -150,9 +150,15 @@ data class SessionSummary(
|
||||
*/
|
||||
val notify: Boolean,
|
||||
/**
|
||||
* Every token this session has spent, as the server counts it -- see `SessionEvent.UsageDelta`.
|
||||
* How much context this session is holding, as the server last measured it -- see
|
||||
* `SessionEvent.UsageDelta`.
|
||||
*
|
||||
* Null where nothing has been measured: a session that has not run a turn, a provider that does
|
||||
* not report usage, or a clear nobody has run a turn since. That is not zero, and the status
|
||||
* row says so in words rather than drawing an empty context for a conversation that may be
|
||||
* nearly full.
|
||||
*/
|
||||
val totalTokens: Long,
|
||||
val contextTokens: Long?,
|
||||
/**
|
||||
* The longest edge an image should have when it reaches this session, or null where the
|
||||
* provider has no limit.
|
||||
@@ -178,7 +184,8 @@ private fun parseSession(session: JSONObject) =
|
||||
permissionMode = session.optString("permissionMode").ifEmpty { null },
|
||||
imported = session.optBoolean("imported", false),
|
||||
notify = session.optBoolean("notify", true),
|
||||
totalTokens = session.optLong("totalTokens", 0),
|
||||
contextTokens =
|
||||
if (session.has("contextTokens")) session.getLong("contextTokens") else null,
|
||||
maxImageEdge = session.optInt("maxImageEdge", 0).takeIf { it > 0 },
|
||||
status = session.getString("status"),
|
||||
lastActivity = session.getDouble("lastActivity"),
|
||||
|
||||
@@ -37,7 +37,14 @@ fun compactionSummary(item: TranscriptItem.CompactedNote): String {
|
||||
}
|
||||
}
|
||||
|
||||
private fun tokens(count: Long): String = "%,d".format(count)
|
||||
/**
|
||||
* A token count as a reader reads one.
|
||||
*
|
||||
* Shared with the status row rather than formatted at each: the divider and the row report the same
|
||||
* quantity about the same moment, and one of them grouping its thousands while the other did not
|
||||
* read as two different measurements.
|
||||
*/
|
||||
fun tokens(count: Long): String = "%,d".format(count)
|
||||
|
||||
/**
|
||||
* What the working indicator says while a compaction is running.
|
||||
|
||||
@@ -113,14 +113,16 @@ sealed class SessionEvent {
|
||||
data class Settings(val model: String?, val permissionMode: String?) : SessionEvent()
|
||||
|
||||
/**
|
||||
* What a turn cost, and what the session has cost in total.
|
||||
* What a turn cost, and how much the model was holding when it ended.
|
||||
*
|
||||
* [total] is the server's running figure, carried on the event so a reader never adds up its
|
||||
* own: a phone opens a session on the newest page of the transcript, so a sum it computed would
|
||||
* be that page's share of the conversation wearing the whole conversation's label. Zero on
|
||||
* entries recorded before the backend sent it.
|
||||
* [context] is prompt plus both cache figures, measured by the backend from the turn's own
|
||||
* usage. Carried on the event rather than summed by the reader, because it is not a sum: a
|
||||
* conversation's context drops at a compaction and a clear, so adding turns up would report a
|
||||
* figure the session stopped being true of. Null where the dialect did not say, and on entries
|
||||
* recorded before the backend sent it -- which leaves the context unmeasured rather than
|
||||
* unchanged.
|
||||
*/
|
||||
data class UsageDelta(val tokens: Long, val total: Long) : SessionEvent()
|
||||
data class UsageDelta(val tokens: Long, val context: Long?) : SessionEvent()
|
||||
|
||||
/**
|
||||
* A compaction that finished, and how much context it recovered.
|
||||
@@ -235,7 +237,10 @@ fun parseSeqEvent(json: String): SeqEvent {
|
||||
permissionMode = body.optString("permissionMode").ifEmpty { null },
|
||||
)
|
||||
"usageDelta" ->
|
||||
SessionEvent.UsageDelta(body.getLong("tokens"), body.optLong("total", 0))
|
||||
SessionEvent.UsageDelta(
|
||||
body.getLong("tokens"),
|
||||
if (body.has("context")) body.getLong("context") else null,
|
||||
)
|
||||
"compacted" ->
|
||||
SessionEvent.Compacted(
|
||||
preTokens = if (body.has("preTokens")) body.getLong("preTokens") else null,
|
||||
@@ -248,3 +253,29 @@ fun parseSeqEvent(json: String): SeqEvent {
|
||||
}
|
||||
return SeqEvent(seq = body.getLong("seq"), ts = body.getDouble("ts"), event = event)
|
||||
}
|
||||
|
||||
/**
|
||||
* The context after [event], given what it was before.
|
||||
*
|
||||
* The same rule the server folds with, because the screen has to keep up between page loads: the
|
||||
* summary it opened with is a measurement from before this stream started, and every event that
|
||||
* moves the figure arrives here.
|
||||
*
|
||||
* The two that lower it are the point. A clear takes the conversation away and a compaction
|
||||
* replaces it with a summary, so a figure measured before either stopped being true at that moment
|
||||
* -- and carrying it forward is how a session that had just been cleared went on reporting the
|
||||
* context it no longer had.
|
||||
*
|
||||
* Null is "we don't know", which is a state each of them can reach: nothing measured yet, a
|
||||
* compaction that finished without saying how much it recovered, or a clear nobody has run a turn
|
||||
* since.
|
||||
*/
|
||||
fun contextAfter(current: Long?, event: SessionEvent): Long? =
|
||||
when (event) {
|
||||
// Falls back to what we had, so a turn the dialect reported no usage for is stale by a
|
||||
// turn -- which every context figure is -- rather than unknown.
|
||||
is SessionEvent.UsageDelta -> event.context ?: current
|
||||
is SessionEvent.Compacted -> event.postTokens
|
||||
is SessionEvent.Cleared -> null
|
||||
else -> current
|
||||
}
|
||||
@@ -457,10 +457,10 @@ fun SessionScreen(
|
||||
val scope = rememberCoroutineScope()
|
||||
var items by remember { mutableStateOf(listOf<TranscriptItem>()) }
|
||||
var status by remember { mutableStateOf(summary.status) }
|
||||
// Seeded from the row this screen was opened from, so a conversation that has spent
|
||||
// something says so before any turn happens here. Zero used to mean "nothing yet" and
|
||||
// "nothing in the page I loaded" at once, and the second one is most of the long sessions.
|
||||
var totalTokens by remember(summary.id) { mutableLongStateOf(summary.totalTokens) }
|
||||
// Seeded from the row this screen was opened from, so a conversation already under way says
|
||||
// how much it is holding before any turn happens here. Null is "nobody has measured it",
|
||||
// which is a different answer from an empty context and is drawn differently.
|
||||
var contextTokens by remember(summary.id) { mutableStateOf(summary.contextTokens) }
|
||||
// When this screen saw the current compaction start, on this device's own clock, and how long
|
||||
// ago that is. See `compactingLabel`: null is the honest answer whenever the start was not
|
||||
// witnessed here, which is what opening a session that is already compacting looks like.
|
||||
@@ -611,14 +611,14 @@ fun SessionScreen(
|
||||
*/
|
||||
fun apply(entry: SeqEvent) {
|
||||
lastSeq.set(entry.seq)
|
||||
// Before the rest, and for every event rather than only the usage ones: a compaction and
|
||||
// a clear move this as much as a turn does, which is the whole reason it is a fold and
|
||||
// not a running total. See `contextAfter`.
|
||||
contextTokens = contextAfter(contextTokens, entry.event)
|
||||
when (val event = entry.event) {
|
||||
// Taken, not accumulated: the server's running total is on the event, and adding
|
||||
// up the deltas this screen happened to receive counted one page of a conversation
|
||||
// and called it the whole. `max` because pages arrive in no guaranteed order and an
|
||||
// older event's total is a smaller true answer, never a correction downwards; it
|
||||
// also leaves the seeded figure alone for transcripts recorded before the backend
|
||||
// sent a total at all.
|
||||
is SessionEvent.UsageDelta -> totalTokens = maxOf(totalTokens, event.total)
|
||||
// Nothing further: what it carries was folded into the context above, and what a
|
||||
// turn cost is not something the transcript draws.
|
||||
is SessionEvent.UsageDelta -> {}
|
||||
else -> {
|
||||
// What the session says it is set to now, which is the only thing that
|
||||
// says it: picking from either menu asks, and the answer comes back here.
|
||||
@@ -1214,7 +1214,11 @@ fun SessionScreen(
|
||||
)
|
||||
}
|
||||
|
||||
SessionStatusRow(status = status, compactingFor = compactingFor, totalTokens = totalTokens)
|
||||
SessionStatusRow(
|
||||
status = status,
|
||||
compactingFor = compactingFor,
|
||||
contextTokens = contextTokens,
|
||||
)
|
||||
|
||||
// Between the transcript and the box: above what is being typed, so the list does not
|
||||
// cover the thing the command is about, and below everything that explains it.
|
||||
@@ -1468,7 +1472,8 @@ private fun SessionStatusRow(
|
||||
status: String,
|
||||
/** Seconds since this device saw the compaction start; null if it did not see it. */
|
||||
compactingFor: Long?,
|
||||
totalTokens: Long,
|
||||
/** Context the session is holding, or null where nothing has measured it. */
|
||||
contextTokens: Long?,
|
||||
modifier: Modifier = Modifier,
|
||||
) {
|
||||
Row(
|
||||
@@ -1538,15 +1543,19 @@ private fun SessionStatusRow(
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
}
|
||||
// Nothing rather than "0 tok" before anything has been spent: a total of zero is a fact
|
||||
// about a conversation that has not started, and it is the one reading nobody needs.
|
||||
if (totalTokens > 0) {
|
||||
Text(
|
||||
"$totalTokens tok",
|
||||
style = MaterialTheme.typography.labelSmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
// How full the session is, which is the number a reader is asking about -- how much room
|
||||
// is left before the next compaction -- rather than what has been spent getting here.
|
||||
//
|
||||
// "unknown" in words, and always drawn. A context nobody has measured is not an empty
|
||||
// one, and the two used to share an appearance: a session that had just been cleared, one
|
||||
// whose provider never reports usage, and one that has not run a turn all showed nothing
|
||||
// at all, which reads as a conversation with room to spare. It is the same reason the
|
||||
// status word beside it names the quiet state instead of leaving the row blank.
|
||||
Text(
|
||||
contextTokens?.let { "context ${tokens(it)}" } ?: "context unknown",
|
||||
style = MaterialTheme.typography.labelSmall,
|
||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user