Colour code with a scanner of our own instead of the library
dev.snipme:highlights 1.1.0 found comments before it knew the language and paired /* with */ by ordinal, so `//` in any URL commented out the rest of its line, every Rust `#[derive(...)]` greyed out as a comment, a `#` inside a Kotlin string swallowed the line, and `x '*/a/*'` in shell produced a span whose end preceded its start -- the one that crashed a card holding `-path '*/.git/*'`. None of that could be post-processed away, because comments won over strings before the language was known. Highlighter.kt is one left-to-right scanner: at each position it is in a line comment, a block comment, a string, or ordinary code, and every span is emitted by advancing an index, so spans cannot overlap, arrive out of order or run backwards. Languages.kt is a `Rules` row per language -- comment tokens, block comment and whether it nests, the string forms, what opens an attribute, and the keyword set -- so a new language is a table entry. The keyword lists came from the library's SyntaxTokens.kt (Apache-2.0, noted at the table) so nothing that is coloured today turns plain, and RON, TOML, fish and JSON are coloured for the first time. HighlighterTest.kt is a new JVM unit test source set -- 24 cases, the library's mistakes kept as regressions, plus a sweep asserting no span escapes the code for any language on unterminated and empty input. AGENTS.md's app line now runs :androidApp:testDebugUnitTest. Measured on the ai-app emulator, debug build, a ~200-line Kotlin fence sent into a sandbox session: before code highlighted: 1, 101.9ms total, 101.9ms mean, 101.9ms worst after code highlighted: 1, 15.0ms total, 15.0ms mean, 15.0ms worst and a second fence in the same run took 13.9ms, so that is the steady cost rather than class loading. stream-bench.sh after the change: code highlighted: 1, 12.1ms total, 12.1ms mean, 12.1ms worst markdown reparsed while streaming: 1329, 2130.7ms total, 1.6ms mean, 8.7ms worst record: one block: 131, 11.4ms total, 0.1ms mean, 0.4ms worst draw phase 1.21ms per frame, the transcript 0.23ms of it transcript-bench.sh after: draw phase 1.10ms per frame, the transcript 0.49ms (place 0.48), worst place 4.3ms -- unchanged within run-to-run noise, as expected, since the scan happens in `warm` and not while drawing. Looked at on the emulator: a URL inside a Kotlin string, a Rust attribute with a lifetime and a raw string, a shell line with globs and `$#`, a RON fence and a TOML fence all colour correctly; a Bash tool card still colours its command; a plain Python fence -- which this change had no reason to touch -- looks as it did; an unknown language stays plain; and a fence is plain while it streams and colours when it freezes. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
irisandClaude Opus 5 committed
1 parent
3d90e0947c
commit
a2b11d516f
13 files changed
+1136
-495
No files matched your search
@@ -0,0 +1,318 @@
|
||||
package com.example.aiapp
|
||||
|
||||
import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.text.AnnotatedString
|
||||
import androidx.compose.ui.text.SpanStyle
|
||||
import androidx.compose.ui.text.buildAnnotatedString
|
||||
|
||||
/** What a span of code is, in the terms the palette has a colour for. */
|
||||
enum class Kind {
|
||||
KEYWORD,
|
||||
STRING,
|
||||
LITERAL,
|
||||
COMMENT,
|
||||
METADATA,
|
||||
PUNCTUATION,
|
||||
MARK,
|
||||
}
|
||||
|
||||
/** A run of [Kind] in the code, as a half-open range. */
|
||||
data class Span(val start: Int, val end: Int, val kind: Kind)
|
||||
|
||||
/**
|
||||
* The colours the highlighter draws with, ours rather than a library's; [catppuccinSyntax] is the
|
||||
* one instance and lives with the rest of the palette.
|
||||
*/
|
||||
data class SyntaxPalette(
|
||||
val keyword: Color,
|
||||
val string: Color,
|
||||
val literal: Color,
|
||||
val comment: Color,
|
||||
val metadata: Color,
|
||||
val punctuation: Color,
|
||||
val mark: Color,
|
||||
) {
|
||||
fun of(kind: Kind): Color =
|
||||
when (kind) {
|
||||
Kind.KEYWORD -> keyword
|
||||
Kind.STRING -> string
|
||||
Kind.LITERAL -> literal
|
||||
Kind.COMMENT -> comment
|
||||
Kind.METADATA -> metadata
|
||||
Kind.PUNCTUATION -> punctuation
|
||||
Kind.MARK -> mark
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* [code] with its keywords, strings and comments coloured, or plain if there is no language for it.
|
||||
*
|
||||
* Shared by a tool call's input ([ToolInputView]) and a reply's fences ([CodeFence]), so the same
|
||||
* code is the same colours wherever it appears.
|
||||
*
|
||||
* Not a composable, and it takes no colour from the theme, because that is what lets [warm] run it
|
||||
* off the drawing thread: the syntax palette is fixed, and a fence with no language is plain text
|
||||
* which needs no colour of its own -- the style the caller draws it with carries that.
|
||||
*
|
||||
* The timing is the number the highlighter is judged by: the library this replaced took **174ms**
|
||||
* on the emulator for a two-hundred-line Kotlin fence, which is why [ParsedReplies.highlighted]
|
||||
* caches the answer rather than a `remember` inside the fence recomputing it on every scroll back.
|
||||
*/
|
||||
fun highlight(code: String, language: Language?): AnnotatedString {
|
||||
if (language == null) return AnnotatedString(code)
|
||||
val spans = DebugStats.timed("code highlighted") { scan(code, rulesOf(language)) }
|
||||
val palette = catppuccinSyntax()
|
||||
return buildAnnotatedString {
|
||||
append(code)
|
||||
spans.forEach { addStyle(SpanStyle(color = palette.of(it.kind)), it.start, it.end) }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* [code] read once, left to right, into the spans that carry a colour.
|
||||
*
|
||||
* One pass with a small state -- in a comment, in a string, or in ordinary code -- rather than a
|
||||
* locator per token kind over the whole text, which is what the library did and is why it found
|
||||
* comments before it knew the language: a `#` inside a shell string, a `//` inside a URL and a
|
||||
* block-comment opener inside a shell glob each commented out the rest of a line that was nothing
|
||||
* of the sort.
|
||||
*
|
||||
* Every span is produced by advancing an index forward, so the result is ordered, non-overlapping
|
||||
* and inside the code by construction. Nothing here throws: an unterminated string or comment runs
|
||||
* to the end of the code, which is also what it looks like while a fence is still being written.
|
||||
*
|
||||
* In ordinary code the order of recognition is comment, string, attribute, number, word, and
|
||||
* finally a single punctuation or mark character. Punctuation and marks are coloured only in
|
||||
* ordinary code, never inside a string or a comment.
|
||||
*/
|
||||
fun scan(code: String, rules: Rules): List<Span> = Scanner(code, rules).run()
|
||||
|
||||
/** Characters coloured as punctuation, and as marks. Both sets are the ones the library used. */
|
||||
private const val PUNCTUATION = ",.:;"
|
||||
private const val MARKS = "()={}<>-+[]|&"
|
||||
|
||||
private class Scanner(private val code: String, private val rules: Rules) {
|
||||
private val spans = ArrayList<Span>()
|
||||
private var at = 0
|
||||
|
||||
fun run(): List<Span> {
|
||||
while (at < code.length) {
|
||||
// Every branch that answers true has advanced `at`, so this terminates.
|
||||
val consumed =
|
||||
blockComment() ||
|
||||
lineComment() ||
|
||||
rawString() ||
|
||||
characterOrLifetime() ||
|
||||
string() ||
|
||||
attribute() ||
|
||||
number() ||
|
||||
word() ||
|
||||
singleCharacter()
|
||||
if (!consumed) at++
|
||||
}
|
||||
return spans
|
||||
}
|
||||
|
||||
private fun emit(start: Int, kind: Kind) {
|
||||
if (at > start) spans.add(Span(start, at, kind))
|
||||
}
|
||||
|
||||
private fun starts(token: String) = code.startsWith(token, at)
|
||||
|
||||
/** Whether a line comment token here opens one; see [Rules.lineCommentsAtWordStart]. */
|
||||
private fun atWordStart() = at == 0 || code[at - 1].isWhitespace() || code[at - 1] in ";|&("
|
||||
|
||||
/** Whether only whitespace stands between the start of this line and here. */
|
||||
private fun atLineStart(): Boolean {
|
||||
var back = at - 1
|
||||
while (back >= 0 && code[back] != '\n') {
|
||||
if (!code[back].isWhitespace()) return false
|
||||
back--
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
private fun toEndOfLine() {
|
||||
while (at < code.length && code[at] != '\n') at++
|
||||
}
|
||||
|
||||
/** From an open bracket through the one that matches it, or to the end if none does. */
|
||||
private fun toMatchingBracket() {
|
||||
var depth = 0
|
||||
while (at < code.length) {
|
||||
when (code[at]) {
|
||||
'[' -> depth++
|
||||
']' -> depth--
|
||||
}
|
||||
at++
|
||||
if (depth == 0) return
|
||||
}
|
||||
}
|
||||
|
||||
private fun blockComment(): Boolean {
|
||||
val comment = rules.blockComment ?: return false
|
||||
if (!starts(comment.open)) return false
|
||||
val start = at
|
||||
at += comment.open.length
|
||||
var depth = 1
|
||||
while (at < code.length && depth > 0) {
|
||||
// The closer is tried first so that a language whose two delimiters are the same
|
||||
// string -- CoffeeScript's `###` -- closes rather than nesting forever.
|
||||
if (starts(comment.close)) {
|
||||
depth--
|
||||
at += comment.close.length
|
||||
} else if (comment.nests && starts(comment.open)) {
|
||||
depth++
|
||||
at += comment.open.length
|
||||
} else {
|
||||
at++
|
||||
}
|
||||
}
|
||||
emit(start, Kind.COMMENT)
|
||||
return true
|
||||
}
|
||||
|
||||
private fun lineComment(): Boolean {
|
||||
if (rules.lineComments.none { starts(it) }) return false
|
||||
if (rules.lineCommentsAtWordStart && !atWordStart()) return false
|
||||
val start = at
|
||||
toEndOfLine()
|
||||
emit(start, Kind.COMMENT)
|
||||
return true
|
||||
}
|
||||
|
||||
/** Rust and RON: `b`? `r` `#`* `"` … `"` `#`*, with no escapes inside. */
|
||||
private fun rawString(): Boolean {
|
||||
if (!rules.rawStrings) return false
|
||||
var ahead = at
|
||||
if (code.getOrNull(ahead) == 'b') ahead++
|
||||
if (code.getOrNull(ahead) != 'r') return false
|
||||
ahead++
|
||||
var hashes = 0
|
||||
while (code.getOrNull(ahead) == '#') {
|
||||
ahead++
|
||||
hashes++
|
||||
}
|
||||
if (code.getOrNull(ahead) != '"') return false
|
||||
val start = at
|
||||
val closer = "\"" + "#".repeat(hashes)
|
||||
val closed = code.indexOf(closer, ahead + 1)
|
||||
at = if (closed < 0) code.length else closed + closer.length
|
||||
emit(start, Kind.STRING)
|
||||
return true
|
||||
}
|
||||
|
||||
/** See [Rules.lifetimes]: an apostrophe that is not a character literal opens nothing. */
|
||||
private fun characterOrLifetime(): Boolean {
|
||||
if (!rules.lifetimes || code[at] != '\'') return false
|
||||
val next = code.getOrNull(at + 1) ?: return false
|
||||
if (next == '\\' || code.getOrNull(at + 2) == '\'') {
|
||||
quoted(Quote("'", "'", escapes = true))
|
||||
} else {
|
||||
at++
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
private fun string(): Boolean {
|
||||
// Longest opener wins, so Kotlin's `"""` is one delimiter rather than an empty string
|
||||
// followed by a quote. A loop rather than filter/maxBy: this runs at every character of
|
||||
// ordinary code, and the pair of lists that would allocate is the whole cost of the scan.
|
||||
var quote: Quote? = null
|
||||
for (candidate in rules.quotes) {
|
||||
if (starts(candidate.open) && candidate.open.length > (quote?.open?.length ?: 0)) {
|
||||
quote = candidate
|
||||
}
|
||||
}
|
||||
quoted(quote ?: return false)
|
||||
return true
|
||||
}
|
||||
|
||||
private fun quoted(quote: Quote) {
|
||||
val start = at
|
||||
at += quote.open.length
|
||||
while (at < code.length) {
|
||||
if (quote.escapes && code[at] == '\\' && at + 1 < code.length) {
|
||||
at += 2
|
||||
continue
|
||||
}
|
||||
if (starts(quote.close)) {
|
||||
at += quote.close.length
|
||||
break
|
||||
}
|
||||
at++
|
||||
}
|
||||
at = at.coerceAtMost(code.length)
|
||||
emit(start, Kind.STRING)
|
||||
}
|
||||
|
||||
private fun attribute(): Boolean {
|
||||
val start = at
|
||||
when (rules.attributes) {
|
||||
Attributes.NONE -> return false
|
||||
Attributes.AT_WORD -> {
|
||||
if (code[at] != '@' || !isWordStart(code.getOrNull(at + 1))) return false
|
||||
at++
|
||||
while (at < code.length && isWordPart(code[at])) at++
|
||||
}
|
||||
Attributes.HASH_BRACKET -> {
|
||||
if (code[at] != '#') return false
|
||||
var ahead = at + 1
|
||||
if (code.getOrNull(ahead) == '!') ahead++
|
||||
if (code.getOrNull(ahead) != '[') return false
|
||||
at = ahead
|
||||
toMatchingBracket()
|
||||
}
|
||||
Attributes.HASH_LINE -> {
|
||||
if (code[at] != '#' || !atLineStart()) return false
|
||||
toEndOfLine()
|
||||
}
|
||||
Attributes.LINE_BRACKET -> {
|
||||
if (code[at] != '[' || !atLineStart()) return false
|
||||
toMatchingBracket()
|
||||
}
|
||||
}
|
||||
emit(start, Kind.METADATA)
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* A number is a run starting with a digit and carrying on through letters, digits, `_` and `.`
|
||||
* -- which covers `0xFF`, `1_000`, `1u32` and `3.14` without a grammar for any of them.
|
||||
*/
|
||||
private fun number(): Boolean {
|
||||
if (!code[at].isDigit()) return false
|
||||
val start = at
|
||||
while (
|
||||
at < code.length && (code[at].isLetterOrDigit() || code[at] == '_' || code[at] == '.')
|
||||
) {
|
||||
at++
|
||||
}
|
||||
emit(start, Kind.LITERAL)
|
||||
return true
|
||||
}
|
||||
|
||||
private fun word(): Boolean {
|
||||
if (!isWordStart(code[at])) return false
|
||||
val start = at
|
||||
while (at < code.length && isWordPart(code[at])) at++
|
||||
if (code.substring(start, at) in rules.keywords) emit(start, Kind.KEYWORD)
|
||||
return true
|
||||
}
|
||||
|
||||
private fun singleCharacter(): Boolean {
|
||||
val kind =
|
||||
when (code[at]) {
|
||||
in PUNCTUATION -> Kind.PUNCTUATION
|
||||
in MARKS -> Kind.MARK
|
||||
else -> return false
|
||||
}
|
||||
at++
|
||||
emit(at - 1, kind)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
private fun isWordStart(c: Char?) = c != null && (c.isLetter() || c == '_')
|
||||
|
||||
private fun isWordPart(c: Char) = c.isLetterOrDigit() || c == '_'
|
||||
Reference in new issue
Block a user