iris/ai-app
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Colour markdown, which is the one language that is not tokens

Browse Source 
The token scanner asks what a character is; markdown's meaning is where it
sits, so a `#` opens a heading at the start of a line and is an ordinary
character three words in. `MarkdownSyntax.kt` reads structure a line at a
time and then each line's prose left to right, and `spansOf` is the one
entry point that hides which of the two scanners a language got.

Conservative wherever a guess would be invisible: emphasis needs a closer on
the same line with no space beside either marker, so the `*p = *q` of a C
fragment opens nothing; an underscore may not start or end inside a word;
and an indented code block is left plain, since four spaces after a blank
line and four after a bullet are the same line.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
irisandClaude Opus 5 committed 2026-09-04 02:37:32 -04:00
1 parent a401e6a7e3
commit 68c5180260
8 files changed
+476 -12
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/androidApp/src/main/kotlin/com/example/aiapp/CodeFence.kt
+2
View File
@@ -201,6 +201,8 @@ private val FENCE_LANGUAGES: Map<String, Language> =
"toml" to Language.TOML,
"fish" to Language.FISH,
"json" to Language.JSON,
"markdown" to Language.MARKDOWN,
"md" to Language.MARKDOWN,
)
/**
app/androidApp/src/main/kotlin/com/example/aiapp/FileLines.kt
+1 -2
View File
@@ -73,8 +73,7 @@ private constructor(
DebugStats.timed("file scanned and cut into lines") {
val body = text.removeSuffix("\n")
val lines = body.split('\n')
val rules = language?.let { rulesOf(it) }
val scanned = if (rules == null) emptyList() else scan(body, rules)
val scanned = if (language == null) emptyList() else spansOf(body, language)
FileLines(lines, bucket(lines, scanned), lines.maxOf(::columnsOf))
}
app/androidApp/src/main/kotlin/com/example/aiapp/Highlighter.kt
+1 -1
View File
@@ -60,7 +60,7 @@ data class SyntaxPalette(
*/
fun highlight(code: String, language: Language?): AnnotatedString {
if (language == null) return AnnotatedString(code)
val spans = DebugStats.timed("code highlighted") { scan(code, rulesOf(language)) }
val spans = DebugStats.timed("code highlighted") { spansOf(code, language) }
val palette = catppuccinSyntax()
return buildAnnotatedString {
append(code)
app/androidApp/src/main/kotlin/com/example/aiapp/Languages.kt
+22 -3
View File
@@ -1,11 +1,14 @@
package com.example.aiapp
/**
* A language the highlighter has rules for.
* A language the highlighter can colour.
*
* The names the reader writes after the backticks are aliases onto these; [fenceLanguage] holds
* that table. A word with no entry there is null, and null is drawn plain, because a fence coloured
* by another language's rules looks highlighted and is wrong in a way the reader cannot see.
*
* Nearly all of them are a row of [RULES], read by one shared scanner. [MARKDOWN] is the one that
* is not; see [spansOf].
*/
enum class Language {
C,
@@ -19,6 +22,7 @@ enum class Language {
JAVASCRIPT,
JSON,
KOTLIN,
MARKDOWN,
PERL,
PHP,
PYTHON,
@@ -83,8 +87,23 @@ enum class Attributes {
LINE_BRACKET,
}
/** The rules for [language]. */
fun rulesOf(language: Language): Rules = RULES.getValue(language)
/**
* The spans [language] colours in [code] -- the one way to ask, whatever the language turns out to
* be made of.
*
* Nearly every language here is tokens: keywords, strings and comments, which is a row of [RULES]
* and the one shared scanner in [scan]. Markdown has none of those, and what a character means
* there depends on where on the line it sits, so it brings a scanner of its own ([scanMarkdown]).
* That is the whole extension point -- a new language is a row of rules or an entry in [SCANNERS],
* and no caller learns which one it got.
*/
fun spansOf(code: String, language: Language): List<Span> = SCANNERS.getValue(language)(code)
// Lazy for the same reason [RULES] is, since it reads it.
private val SCANNERS: Map<Language, (String) -> List<Span>> by lazy {
RULES.mapValues { (_, rules) -> { code: String -> scan(code, rules) } } +
mapOf(Language.MARKDOWN to ::scanMarkdown)
}
private val C_STYLE = BlockComment("/*", "*/", nests = false)
private val NESTING = BlockComment("/*", "*/", nests = true)
app/androidApp/src/main/kotlin/com/example/aiapp/MarkdownSyntax.kt
+321
View File
@@ -0,0 +1,321 @@
package com.example.aiapp
/**
* Markdown read into the spans that carry a colour -- a ```markdown fence in a reply, and a `.md`
* file in the viewer.
*
* Its own scanner rather than a row of [Rules] because markdown has neither keywords nor strings:
* what a character means depends on where it sits. A `#` opens a heading at the start of a line and
* is an ordinary character three words in; a `*` opens emphasis only if something closes it on the
* same line. The token scanner cannot ask either question, and answering them with its rules is how
* a highlighter comes to grey out the second half of a paragraph.
*
* Structure is read a line at a time and each line's prose is then read left to right, so every
* decision is made inside one line -- except a fenced block, which is the one piece of state
* carried across them. An unclosed fence therefore colours the rest of the text, which is also what
* it looks like while somebody is still writing it.
*
* What is deliberately *not* recognised: an indented code block. Four spaces after a blank line is
* one, and four spaces after a bullet is a list item's second paragraph, and the two are told apart
* by what came before rather than by the line itself. Colouring the wrong one of those as code is a
* mistake the reader cannot see, so both are left plain, which is the safe answer.
*
* Like [scan], the spans come out ordered, non-overlapping and inside the text by construction:
* every one is emitted by a pass that only moves forward, and nothing here throws.
*/
fun scanMarkdown(code: String): List<Span> = MarkdownScanner(code).run()
/** The characters an unordered list may be bulleted with. */
private const val BULLETS = "-*+"
/** The characters a thematic break, or a setext heading's underline, can be drawn with. */
private const val RULE_MARKERS = "-*_="
/** The characters that can open emphasis, strong emphasis or a strikethrough. */
private const val EMPHASIS = "*_~"
private class MarkdownScanner(private val code: String) {
private val spans = ArrayList<Span>()
fun run(): List<Span> {
var at = 0
// The delimiter run that opened the fenced block we are inside, or null between them.
var fence: String? = null
while (at <= code.length) {
val newline = code.indexOf('\n', at)
val end = if (newline < 0) code.length else newline
val open = fence
if (open != null) {
// The content and the closing line alike: a fence is one block of code, and its
// own delimiters belong to it the way a string's quotes belong to the string.
emit(at, end, Kind.STRING)
if (closesFence(at, end, open)) fence = null
} else {
val opened = opensFence(at, end)
if (opened == null) structure(at, end)
fence = opened
}
if (end == code.length) break
at = end + 1
}
return spans
}
/**
* Spans, coalesced with the one before when they touch and agree.
*
* Worth doing here rather than leaving it to the caller: the line scanner emits per marker and
* per word, so a heading would otherwise arrive as a dozen abutting spans of one colour.
*/
private fun emit(start: Int, end: Int, kind: Kind) {
if (end <= start) return
val last = spans.lastOrNull()
if (last != null && last.kind == kind && last.end == start) {
spans[spans.size - 1] = Span(last.start, end, kind)
} else {
spans.add(Span(start, end, kind))
}
}
/** The first character of the line at or after [start] that is not indentation. */
private fun indented(start: Int, end: Int): Int {
var at = start
while (at < end && (code[at] == ' ' || code[at] == '\t')) at++
return at
}
/** The run of backticks or tildes that could open or close a fence on this line, or null. */
private fun fenceRun(start: Int, end: Int): IntRange? {
val at = indented(start, end)
if (at == end) return null
val marker = code[at]
if (marker != '`' && marker != '~') return null
var run = at
while (run < end && code[run] == marker) run++
return if (run - at >= 3) at until run else null
}
/** Draws an opening fence line and answers its delimiter, or null if this is not one. */
private fun opensFence(start: Int, end: Int): String? {
val run = fenceRun(start, end) ?: return null
emit(run.first, run.last + 1, Kind.STRING)
// The info word is what the fence is a fence *of*, which is metadata about the block
// rather than part of it -- the same reading as a Rust attribute above a struct.
emit(indented(run.last + 1, end), end, Kind.METADATA)
return code.substring(run.first, run.last + 1)
}
/**
* Whether this line closes a fence opened by [open].
*
* The same character, at least as many of them, and nothing else on the line -- so a longer run
* closes a shorter one and a line of backticks with a word after it does not close anything.
*/
private fun closesFence(start: Int, end: Int, open: String): Boolean {
val run = fenceRun(start, end) ?: return false
if (code[run.first] != open[0] || run.last + 1 - run.first < open.length) return false
return indented(run.last + 1, end) == end
}
/** One ordinary line: what its opening characters make it, and then its prose. */
private fun structure(start: Int, end: Int) {
var at = indented(start, end)
// Quote markers come before everything else and can be several deep, and what follows one
// is an ordinary line again -- a heading inside a quote is still a heading.
while (at < end && code[at] == '>') {
at++
emit(at - 1, at, Kind.MARK)
at = indented(at, end)
}
if (at == end) return
if (heading(at, end) || thematicBreak(at, end)) return
inline(bullet(at, end), end)
}
/** `#` to `######` and a space. Without the space it is a word beginning with a hash. */
private fun heading(start: Int, end: Int): Boolean {
var at = start
while (at < end && code[at] == '#') at++
val depth = at - start
if (depth !in 1..6) return false
if (at < end && code[at] != ' ' && code[at] != '\t') return false
emit(start, end, Kind.KEYWORD)
return true
}
/**
* A line made of one repeated rule character and nothing else.
*
* `---`, `***` and `___` are thematic breaks; `===` and `---` are also the underline of a
* setext heading. The two are the same line to look at and mean the same thing to a reader -- a
* rule drawn across the page -- so they get one appearance rather than a lookback to tell them
* apart. One `=` is enough because a setext underline may be a single character; a break needs
* three, which is what keeps a `- ` bullet out of here.
*/
private fun thematicBreak(start: Int, end: Int): Boolean {
val marker = code[start]
if (marker !in RULE_MARKERS) return false
var seen = 0
for (at in start until end) {
val character = code[at]
if (character == marker) seen++ else if (!character.isWhitespace()) return false
}
if (seen < if (marker == '=') 1 else 3) return false
emit(start, end, Kind.MARK)
return true
}
/** Draws a list marker if the line opens with one, and answers where the item's text starts. */
private fun bullet(start: Int, end: Int): Int {
val marker = code[start]
if (marker in BULLETS && spaceOrEnd(start + 1, end)) {
emit(start, start + 1, Kind.MARK)
return indented(start + 1, end)
}
var digits = start
while (digits < end && code[digits].isDigit()) digits++
val delimiter = code.getOrNull(digits)
if (
digits > start && (delimiter == '.' || delimiter == ')') && spaceOrEnd(digits + 1, end)
) {
emit(start, digits + 1, Kind.MARK)
return indented(digits + 1, end)
}
return start
}
private fun spaceOrEnd(at: Int, end: Int) = at >= end || code[at] == ' ' || code[at] == '\t'
/**
* The inline forms, left to right.
*
* Every branch answers a position strictly after [start] of its call, so this terminates
* whether or not the form it was looking at turned out to be one.
*/
private fun inline(start: Int, end: Int) {
var at = start
while (at < end) {
val character = code[at]
at =
when {
// A backslash takes the character after it out of the running entirely, which
// is how `\*` stays an asterisk rather than opening emphasis.
character == '\\' -> at + 2
character == '`' -> codeSpan(at, end)
character == '[' -> link(at, at, end)
character == '!' && code.getOrNull(at + 1) == '[' -> link(at, at + 1, end)
character in EMPHASIS -> emphasis(at, end)
else -> at + 1
}
}
}
/**
* `` `code` ``, closed by a run of exactly as many backticks as opened it.
*
* That count is what lets a span hold a backtick of its own (``` ``a ` b`` ```), and it is why
* the search skips over a shorter or longer run rather than stopping at the first backtick.
*/
private fun codeSpan(start: Int, end: Int): Int {
var open = start
while (open < end && code[open] == '`') open++
val ticks = open - start
var at = open
while (at < end) {
if (code[at] != '`') {
at++
continue
}
var close = at
while (close < end && code[close] == '`') close++
if (close - at == ticks) {
emit(start, close, Kind.STRING)
return close
}
at = close
}
// Nothing closes it on this line, so those were ordinary backticks.
return open
}
/**
* `[text](destination)`, and the same with a leading `!` for an image.
*
* The text is drawn as prose -- it is what the reader reads -- so only the brackets around it
* are marked, and the destination is metadata: the place the link goes rather than anything
* said to the reader. A `[text]` with no destination after it is left plain, because that is
* what a reference link and a bracketed aside look like, and neither is worth guessing at.
*/
private fun link(start: Int, bracket: Int, end: Int): Int {
var depth = 0
var close = bracket
while (close < end) {
when (code[close]) {
'\\' -> close++
'[' -> depth++
']' -> {
depth--
if (depth == 0) break
}
}
close++
}
if (close >= end) return start + 1
val destination = close + 1
if (code.getOrNull(destination) != '(') return start + 1
val paren = code.indexOf(')', destination)
if (paren < 0 || paren >= end) return start + 1
emit(start, bracket + 1, Kind.MARK)
inline(bracket + 1, close)
emit(close, destination, Kind.MARK)
emit(destination, paren + 1, Kind.METADATA)
return paren + 1
}
/**
* `*emph*`, `**strong**`, `_emph_` and `~~struck~~`, drawn markers and all.
*
* Markers and all because that is how the token scanner draws a string: the quotes are part of
* the thing. The two guards are what keep this off code that happens to be in a paragraph --
* the opener must be followed by something to emphasise and the closer preceded by something
* emphasised, so `a * b * c` opens nothing and neither does the `*p = *q` of a C fragment.
* Underscores additionally may not start or end inside a word, or every `snake_case_name` in a
* document would be half emphasised.
*/
private fun emphasis(start: Int, end: Int): Int {
val marker = code[start]
var open = start
while (open < end && code[open] == marker) open++
val length = open - start
if (marker == '~' && length != 2) return open
if (length > 3) return open
if (open == end || code[open].isWhitespace()) return open
if (marker == '_' && start > 0 && isWord(code[start - 1])) return open
var at = open
while (at < end) {
if (code[at] == '\\') {
at += 2
continue
}
if (code[at] != marker) {
at++
continue
}
var close = at
while (close < end && code[close] == marker) close++
val finish = at + length
if (
close - at >= length &&
!code[at - 1].isWhitespace() &&
!(marker == '_' && finish < end && isWord(code[finish]))
) {
emit(start, finish, Kind.LITERAL)
return finish
}
at = close
}
return open
}
}
private fun isWord(character: Char) = character.isLetterOrDigit() || character == '_'
app/androidApp/src/test/kotlin/com/example/aiapp/HighlighterTest.kt
+93 -6
View File
@@ -16,9 +16,7 @@ import kotlin.test.assertTrue
class HighlighterTest {
/** Every span of [kind] in [code], as the text it covers. */
private fun spans(code: String, language: Language, kind: Kind): List<String> =
scan(code, rulesOf(language))
.filter { it.kind == kind }
.map { code.substring(it.start, it.end) }
spansOf(code, language).filter { it.kind == kind }.map { code.substring(it.start, it.end) }
private fun assertSpans(
code: String,
@@ -182,6 +180,86 @@ class HighlighterTest {
assertSpans(code, Language.RON, Kind.LITERAL, "3")
}
// Markdown, which has a scanner of its own: what a character means there is decided by where
// it sits rather than by what it is, so most of these are about the cases where it means
// nothing at all.
@Test
fun `a heading is coloured whole and a hash inside a word is not one`() {
val code = "## Layout\nissue #12 is fixed\n#hashtag"
assertSpans(code, Language.MARKDOWN, Kind.KEYWORD, "## Layout")
}
@Test
fun `seven hashes are not a heading`() {
assertSpans("####### deep", Language.MARKDOWN, Kind.KEYWORD)
}
@Test
fun `a fence carries its language as metadata and its body as one string`() {
val code = "text\n```kotlin\nval x = 1\n```\nmore"
assertSpans(code, Language.MARKDOWN, Kind.METADATA, "kotlin")
assertSpans(code, Language.MARKDOWN, Kind.STRING, "```", "val x = 1", "```")
}
/** The state that crosses a line, so the one worth asking about at both ends. */
@Test
fun `a longer fence is not closed by a shorter one, and a heading inside it is not a heading`() {
val code = "````\n```\n# not a heading\n````\nafter"
assertSpans(code, Language.MARKDOWN, Kind.KEYWORD)
assertSpans(code, Language.MARKDOWN, Kind.STRING, "````", "```", "# not a heading", "````")
}
@Test
fun `an unclosed fence runs to the end rather than throwing`() {
assertSpans("```\nstill going", Language.MARKDOWN, Kind.STRING, "```", "still going")
}
@Test
fun `list markers and quote markers colour without their text`() {
val code = "- one\n2. two\n> quoted"
assertSpans(code, Language.MARKDOWN, Kind.MARK, "-", "2.", ">")
}
@Test
fun `a rule and a setext underline are the same mark`() {
assertSpans("Title\n=====\n\n---", Language.MARKDOWN, Kind.MARK, "=====", "---")
}
@Test
fun `emphasis needs something on both sides of it`() {
assertSpans("**bold** and *thin*", Language.MARKDOWN, Kind.LITERAL, "**bold**", "*thin*")
// The case the guards exist for: a C fragment written in a paragraph.
assertSpans("a * b * c and *p = *q", Language.MARKDOWN, Kind.LITERAL)
}
@Test
fun `an underscore inside a word emphasises nothing`() {
assertSpans("snake_case_name and _real_", Language.MARKDOWN, Kind.LITERAL, "_real_")
}
@Test
fun `a code span holds a backtick when opened with two`() {
assertSpans("``a ` b`` and `c`", Language.MARKDOWN, Kind.STRING, "``a ` b``", "`c`")
}
@Test
fun `an unclosed code span is ordinary text`() {
assertSpans("a ` b", Language.MARKDOWN, Kind.STRING)
}
@Test
fun `a link marks its brackets and colours its destination`() {
val code = "see [the plan](PLAN.md) now"
assertSpans(code, Language.MARKDOWN, Kind.MARK, "[", "]")
assertSpans(code, Language.MARKDOWN, Kind.METADATA, "(PLAN.md)")
}
@Test
fun `a bracket with no destination after it is left plain`() {
assertSpans("an [aside] here", Language.MARKDOWN, Kind.MARK)
}
@Test
fun `an unknown fence language is drawn plain`() {
assertEquals(null, fenceLanguage("brainfuck"))
@@ -189,8 +267,8 @@ class HighlighterTest {
}
@Test
fun `every alias the fence table knows has rules`() {
Language.entries.forEach { rulesOf(it) }
fun `every language the fence table knows has a scanner`() {
Language.entries.forEach { spansOf("x", it) }
}
/**
@@ -218,11 +296,20 @@ class HighlighterTest {
"0x",
"1.2.3",
"a#b//c/*d*/'e\"f",
"```",
"*",
"**",
"~~",
"> ",
"- ",
"1.",
"[x](",
"#######",
"\n\n \n",
)
for (language in Language.entries) {
for (code in nasty) {
val spans = scan(code, rulesOf(language))
val spans = spansOf(code, language)
spans.forEach {
assertTrue(
it.start in 0..it.end && it.end <= code.length,
app/ui-sandbox.sh
+21
View File
@@ -270,6 +270,27 @@ printf 'def main():\n # a comment\n print("hello")\n' >"$FILES/main.py"
printf '#!/bin/sh\n# a comment\necho hello\n' >"$FILES/run.sh"
chmod +x "$FILES/run.sh"
printf '{"a": 1, "b": [true, null]}\n' >"$FILES/data.json"
# Markdown's scanner is line-structured rather than tokens, so the fixture holds one of each
# thing it decides by position: a heading, a fence, a list, a quote, a link and a rule.
cat >"$FILES/notes.md" <<'MARKDOWN'
# Notes
A paragraph with `code`, **bold** and a [link](PLAN.md).
Not emphasis: a * b * c, and snake_case_name.
## A list
- one
- two
> quoted
```rust
fn main() { println!("hello"); }
```
---
MARKDOWN
# Not UTF-8, so it reads as binary rather than as mojibake.
printf '\377\376\000\001binary\n' >"$FILES/picture.bin"
# Over FILE_LIMIT (1 MiB), so the read refuses before anything transfers.