Colour code with a scanner of our own instead of the library
dev.snipme:highlights 1.1.0 found comments before it knew the language and paired /* with */ by ordinal, so `//` in any URL commented out the rest of its line, every Rust `#[derive(...)]` greyed out as a comment, a `#` inside a Kotlin string swallowed the line, and `x '*/a/*'` in shell produced a span whose end preceded its start -- the one that crashed a card holding `-path '*/.git/*'`. None of that could be post-processed away, because comments won over strings before the language was known. Highlighter.kt is one left-to-right scanner: at each position it is in a line comment, a block comment, a string, or ordinary code, and every span is emitted by advancing an index, so spans cannot overlap, arrive out of order or run backwards. Languages.kt is a `Rules` row per language -- comment tokens, block comment and whether it nests, the string forms, what opens an attribute, and the keyword set -- so a new language is a table entry. The keyword lists came from the library's SyntaxTokens.kt (Apache-2.0, noted at the table) so nothing that is coloured today turns plain, and RON, TOML, fish and JSON are coloured for the first time. HighlighterTest.kt is a new JVM unit test source set -- 24 cases, the library's mistakes kept as regressions, plus a sweep asserting no span escapes the code for any language on unterminated and empty input. AGENTS.md's app line now runs :androidApp:testDebugUnitTest. Measured on the ai-app emulator, debug build, a ~200-line Kotlin fence sent into a sandbox session: before code highlighted: 1, 101.9ms total, 101.9ms mean, 101.9ms worst after code highlighted: 1, 15.0ms total, 15.0ms mean, 15.0ms worst and a second fence in the same run took 13.9ms, so that is the steady cost rather than class loading. stream-bench.sh after the change: code highlighted: 1, 12.1ms total, 12.1ms mean, 12.1ms worst markdown reparsed while streaming: 1329, 2130.7ms total, 1.6ms mean, 8.7ms worst record: one block: 131, 11.4ms total, 0.1ms mean, 0.4ms worst draw phase 1.21ms per frame, the transcript 0.23ms of it transcript-bench.sh after: draw phase 1.10ms per frame, the transcript 0.49ms (place 0.48), worst place 4.3ms -- unchanged within run-to-run noise, as expected, since the scan happens in `warm` and not while drawing. Looked at on the emulator: a URL inside a Kotlin string, a Rust attribute with a lifetime and a raw string, a shell line with globs and `$#`, a RON fence and a TOML fence all colour correctly; a Bash tool card still colours its command; a plain Python fence -- which this change had no reason to touch -- looks as it did; an unknown language stays plain; and a fence is plain while it streams and colours when it freezes. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
irisandClaude Opus 5 committed
1 parent
3d90e0947c
commit
a2b11d516f
13 files changed
+1136
-495
No files matched your search
@@ -0,0 +1,240 @@
|
||||
package com.example.aiapp
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
/**
|
||||
* What the scanner colours, asserted as the text under each span rather than as offsets, so a
|
||||
* failure prints the code that was got wrong instead of a pair of numbers.
|
||||
*
|
||||
* Most of these are the mistakes dev.snipme:highlights 1.1.0 made -- the library this scanner
|
||||
* replaced -- measured against it directly before it was removed. They are here rather than in the
|
||||
* `highlights-repro.sh` script they came from because a case that only a script can ask about is a
|
||||
* case nobody asks about.
|
||||
*/
|
||||
class HighlighterTest {
|
||||
/** Every span of [kind] in [code], as the text it covers. */
|
||||
private fun spans(code: String, language: Language, kind: Kind): List<String> =
|
||||
scan(code, rulesOf(language))
|
||||
.filter { it.kind == kind }
|
||||
.map { code.substring(it.start, it.end) }
|
||||
|
||||
private fun assertSpans(
|
||||
code: String,
|
||||
language: Language,
|
||||
kind: Kind,
|
||||
vararg expected: String,
|
||||
) {
|
||||
assertEquals(expected.toList(), spans(code, language, kind), "$kind in: $code")
|
||||
}
|
||||
|
||||
// The four inputs the repro script asked the library about, and what it answered.
|
||||
|
||||
@Test
|
||||
fun `a quoted glob is one string, not a comment`() {
|
||||
// The library answered a span whose end preceded its start here, which crashed the app.
|
||||
assertSpans("x '*/a/*'", Language.SHELL, Kind.STRING, "'*/a/*'")
|
||||
assertSpans("x '*/a/*'", Language.SHELL, Kind.COMMENT)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a find with globs has no comment in it`() {
|
||||
val code = "find . -path '*/.git/*' -prune -o -name '*.kt' -print"
|
||||
assertSpans(code, Language.SHELL, Kind.STRING, "'*/.git/*'", "'*.kt'")
|
||||
assertSpans(code, Language.SHELL, Kind.COMMENT)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a URL does not comment out the rest of a shell line`() {
|
||||
val code = "curl https://example.com/x && echo done"
|
||||
assertSpans(code, Language.SHELL, Kind.COMMENT)
|
||||
assertSpans(code, Language.SHELL, Kind.KEYWORD, "echo")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a URL inside a Kotlin string stays a string`() {
|
||||
val code = "val url = \"https://example.com\"\nfun f() = 1"
|
||||
assertSpans(code, Language.KOTLIN, Kind.COMMENT)
|
||||
assertSpans(code, Language.KOTLIN, Kind.STRING, "\"https://example.com\"")
|
||||
assertSpans(code, Language.KOTLIN, Kind.KEYWORD, "val", "fun")
|
||||
}
|
||||
|
||||
// Attributes, which the library greyed out as comments.
|
||||
|
||||
@Test
|
||||
fun `a Rust attribute is metadata and the struct after it still colours`() {
|
||||
val code = "#[derive(Debug)]\nstruct A { b: u8 }"
|
||||
assertSpans(code, Language.RUST, Kind.METADATA, "#[derive(Debug)]")
|
||||
assertSpans(code, Language.RUST, Kind.COMMENT)
|
||||
assertSpans(code, Language.RUST, Kind.KEYWORD, "struct")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `an inner Rust attribute closes at its own bracket`() {
|
||||
val code = "#![allow(dead_code)]\nfn f() {}"
|
||||
assertSpans(code, Language.RUST, Kind.METADATA, "#![allow(dead_code)]")
|
||||
assertSpans(code, Language.RUST, Kind.KEYWORD, "fn")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a C preprocessor line is metadata rather than a comment`() {
|
||||
val code = "#include <stdio.h>\nint main() { return 0; }"
|
||||
assertSpans(code, Language.C, Kind.METADATA, "#include <stdio.h>")
|
||||
assertSpans(code, Language.C, Kind.COMMENT)
|
||||
assertSpans(code, Language.C, Kind.KEYWORD, "int", "return")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a Kotlin annotation is metadata`() {
|
||||
assertSpans("@Composable fun f() {}", Language.KOTLIN, Kind.METADATA, "@Composable")
|
||||
}
|
||||
|
||||
// Strings whose contents the library read as code.
|
||||
|
||||
@Test
|
||||
fun `a hash inside a Kotlin string is not a comment`() {
|
||||
val code = "val c = \"#FF0000\"\nval d = 1"
|
||||
assertSpans(code, Language.KOTLIN, Kind.COMMENT)
|
||||
assertSpans(code, Language.KOTLIN, Kind.STRING, "\"#FF0000\"")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `an apostrophe inside a Kotlin string does not open one`() {
|
||||
val code = "val a = \"don't\"\nval b = \"x\""
|
||||
assertSpans(code, Language.KOTLIN, Kind.STRING, "\"don't\"", "\"x\"")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a Rust lifetime does not open a string but a character literal does`() {
|
||||
val code = "fn f<'a>(x: &'a str) { let c = 'x'; }"
|
||||
assertSpans(code, Language.RUST, Kind.STRING, "'x'")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `an escaped quote is inside the Rust character literal`() {
|
||||
assertSpans("let c = '\\'';", Language.RUST, Kind.STRING, "'\\''")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a Rust raw string keeps its inner quotes`() {
|
||||
val code = "let s = r#\"a \"quoted\" b\"#;"
|
||||
assertSpans(code, Language.RUST, Kind.STRING, "r#\"a \"quoted\" b\"#")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a Kotlin triple quoted string is one string`() {
|
||||
assertSpans(
|
||||
"val s = \"\"\"a \"b\" c\"\"\"",
|
||||
Language.KOTLIN,
|
||||
Kind.STRING,
|
||||
"\"\"\"a \"b\" c\"\"\"",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a shell single quoted string takes no escapes`() {
|
||||
// `\` is literal inside shell single quotes, so the string ends at the next apostrophe.
|
||||
assertSpans("echo 'a\\' b", Language.SHELL, Kind.STRING, "'a\\'")
|
||||
}
|
||||
|
||||
// Comments.
|
||||
|
||||
@Test
|
||||
fun `Rust and Kotlin nest block comments`() {
|
||||
val code = "/* a /* b */ c */ x"
|
||||
assertSpans(code, Language.RUST, Kind.COMMENT, "/* a /* b */ c */")
|
||||
assertSpans(code, Language.KOTLIN, Kind.COMMENT, "/* a /* b */ c */")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `C ends a block comment at the first close`() {
|
||||
assertSpans("/* a /* b */ c */ x", Language.C, Kind.COMMENT, "/* a /* b */")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a shell comment starts only at a word boundary`() {
|
||||
val code = "\${#x} \$# a#b # real"
|
||||
assertSpans(code, Language.SHELL, Kind.COMMENT, "# real")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a hash anywhere is a Python comment`() {
|
||||
assertSpans("x = 1 # note", Language.PYTHON, Kind.COMMENT, "# note")
|
||||
}
|
||||
|
||||
// TOML, which the library has no rules for at all.
|
||||
|
||||
@Test
|
||||
fun `a TOML table header is metadata and a hash in a value is not a comment`() {
|
||||
val code = "[server]\ncolour = \"#FF0000\"\nport = 8080 # the real one"
|
||||
assertSpans(code, Language.TOML, Kind.METADATA, "[server]")
|
||||
assertSpans(code, Language.TOML, Kind.STRING, "\"#FF0000\"")
|
||||
assertSpans(code, Language.TOML, Kind.COMMENT, "# the real one")
|
||||
assertSpans(code, Language.TOML, Kind.LITERAL, "8080")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a RON attribute and its values colour`() {
|
||||
val code = "#![enable(implicit_some)]\n(count: 3, on: true)"
|
||||
assertSpans(code, Language.RON, Kind.METADATA, "#![enable(implicit_some)]")
|
||||
assertSpans(code, Language.RON, Kind.KEYWORD, "true")
|
||||
assertSpans(code, Language.RON, Kind.LITERAL, "3")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `an unknown fence language is drawn plain`() {
|
||||
assertEquals(null, fenceLanguage("brainfuck"))
|
||||
assertEquals("+[-]", highlight("+[-]", fenceLanguage("brainfuck")).text)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `every alias the fence table knows has rules`() {
|
||||
Language.entries.forEach { rulesOf(it) }
|
||||
}
|
||||
|
||||
/**
|
||||
* The scanner must never throw and must never answer a span the code does not contain: the
|
||||
* library's reversed range is exactly the shape that crashed a card, and a fence still being
|
||||
* written is an unterminated string or comment on every keystroke.
|
||||
*/
|
||||
@Test
|
||||
fun `spans stay inside the code for every language and every nasty input`() {
|
||||
val nasty =
|
||||
listOf(
|
||||
"",
|
||||
"'",
|
||||
"\"",
|
||||
"\"unterminated",
|
||||
"/* unterminated",
|
||||
"###",
|
||||
"#",
|
||||
"#![",
|
||||
"[",
|
||||
"r#\"",
|
||||
"\\",
|
||||
"'''",
|
||||
"\"\"\"",
|
||||
"0x",
|
||||
"1.2.3",
|
||||
"a#b//c/*d*/'e\"f",
|
||||
"\n\n \n",
|
||||
)
|
||||
for (language in Language.entries) {
|
||||
for (code in nasty) {
|
||||
val spans = scan(code, rulesOf(language))
|
||||
spans.forEach {
|
||||
assertTrue(
|
||||
it.start in 0..it.end && it.end <= code.length,
|
||||
"$language answered $it for ${code.replace("\n", "\\n")}",
|
||||
)
|
||||
}
|
||||
assertEquals(
|
||||
spans.sortedBy { it.start },
|
||||
spans,
|
||||
"$language answered spans out of order for ${code.replace("\n", "\\n")}",
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user