Colour markdown, which is the one language that is not tokens

The token scanner asks what a character is; markdown's meaning is where it
sits, so a `#` opens a heading at the start of a line and is an ordinary
character three words in. `MarkdownSyntax.kt` reads structure a line at a
time and then each line's prose left to right, and `spansOf` is the one
entry point that hides which of the two scanners a language got.

Conservative wherever a guess would be invisible: emphasis needs a closer on
the same line with no space beside either marker, so the `*p = *q` of a C
fragment opens nothing; an underscore may not start or end inside a word;
and an indented code block is left plain, since four spaces after a blank
line and four after a bullet are the same line.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

TRANSCRIPT_RENDERING.md

@@ -239,6 +239,21 @@ and are the reason several tempting simplifications were rejected.
   to the end of the code. `HighlighterTest.kt` is the JVM unit test
   (`./gradlew :androidApp:testDebugUnitTest`); the cases in it are the
   library's mistakes, kept as regressions.
+  **Markdown is the one language that is not a `Rules` row**
+  (`MarkdownSyntax.kt`, added 2026-09-04). It has no keywords and no
+  strings, and what a character means there depends on where on the line it
+  sits -- a `#` opens a heading at the start of a line and is an ordinary
+  character three words in -- which is a question the token scanner cannot
+  ask. So it carries a scanner of its own, structure a line at a time and
+  then the inline forms left to right, and `spansOf` in `Languages.kt` is
+  the one entry point that hides which of the two a language got. It is
+  deliberately conservative where a guess would be invisible: emphasis needs
+  a closer on the same line with no space beside either marker (so the
+  `*p = *q` of a C fragment in a paragraph opens nothing), an underscore may
+  not start or end inside a word (`snake_case_name`), and an indented code
+  block is left plain, because four spaces after a blank line and four
+  spaces after a bullet are the same line and only what came before tells
+  them apart.
   It replaced dev.snipme:highlights 1.1.0 on 2026-09-03, which found
   comments before it knew the language and paired `/*` with `*/` by
   ordinal. That library used one set of delimiters for every language, so

app/androidApp/src/main/kotlin/com/example/aiapp/CodeFence.kt

@@ -201,6 +201,8 @@ private val FENCE_LANGUAGES: Map<String, Language> =
         "toml" to Language.TOML,
         "fish" to Language.FISH,
         "json" to Language.JSON,
+        "markdown" to Language.MARKDOWN,
+        "md" to Language.MARKDOWN,
     )
 
 /**

app/androidApp/src/main/kotlin/com/example/aiapp/FileLines.kt

@@ -73,8 +73,7 @@ private constructor(
             DebugStats.timed("file scanned and cut into lines") {
                 val body = text.removeSuffix("\n")
                 val lines = body.split('\n')
-                val rules = language?.let { rulesOf(it) }
+                val scanned = if (language == null) emptyList() else spansOf(body, language)
-                val scanned = if (rules == null) emptyList() else scan(body, rules)
                 FileLines(lines, bucket(lines, scanned), lines.maxOf(::columnsOf))
             }
 

app/androidApp/src/main/kotlin/com/example/aiapp/Highlighter.kt

@@ -60,7 +60,7 @@ data class SyntaxPalette(
  */
 fun highlight(code: String, language: Language?): AnnotatedString {
     if (language == null) return AnnotatedString(code)
-    val spans = DebugStats.timed("code highlighted") { scan(code, rulesOf(language)) }
+    val spans = DebugStats.timed("code highlighted") { spansOf(code, language) }
     val palette = catppuccinSyntax()
     return buildAnnotatedString {
         append(code)

app/androidApp/src/main/kotlin/com/example/aiapp/Languages.kt

@@ -1,11 +1,14 @@
 package com.example.aiapp
 
 /**
- * A language the highlighter has rules for.
+ * A language the highlighter can colour.
  *
  * The names the reader writes after the backticks are aliases onto these; [fenceLanguage] holds
  * that table. A word with no entry there is null, and null is drawn plain, because a fence coloured
  * by another language's rules looks highlighted and is wrong in a way the reader cannot see.
+ *
+ * Nearly all of them are a row of [RULES], read by one shared scanner. [MARKDOWN] is the one that
+ * is not; see [spansOf].
  */
 enum class Language {
     C,
@@ -19,6 +22,7 @@ enum class Language {
     JAVASCRIPT,
     JSON,
     KOTLIN,
+    MARKDOWN,
     PERL,
     PHP,
     PYTHON,
@@ -83,8 +87,23 @@ enum class Attributes {
     LINE_BRACKET,
 }
 
-/** The rules for [language]. */
+/**
-fun rulesOf(language: Language): Rules = RULES.getValue(language)
+ * The spans [language] colours in [code] -- the one way to ask, whatever the language turns out to
+ * be made of.
+ *
+ * Nearly every language here is tokens: keywords, strings and comments, which is a row of [RULES]
+ * and the one shared scanner in [scan]. Markdown has none of those, and what a character means
+ * there depends on where on the line it sits, so it brings a scanner of its own ([scanMarkdown]).
+ * That is the whole extension point -- a new language is a row of rules or an entry in [SCANNERS],
+ * and no caller learns which one it got.
+ */
+fun spansOf(code: String, language: Language): List<Span> = SCANNERS.getValue(language)(code)
+
+// Lazy for the same reason [RULES] is, since it reads it.
+private val SCANNERS: Map<Language, (String) -> List<Span>> by lazy {
+    RULES.mapValues { (_, rules) -> { code: String -> scan(code, rules) } } +
+        mapOf(Language.MARKDOWN to ::scanMarkdown)
+}
 
 private val C_STYLE = BlockComment("/*", "*/", nests = false)
 private val NESTING = BlockComment("/*", "*/", nests = true)

app/androidApp/src/main/kotlin/com/example/aiapp/MarkdownSyntax.kt

@@ -0,0 +1,321 @@
+package com.example.aiapp
+
+/**
+ * Markdown read into the spans that carry a colour -- a ```markdown fence in a reply, and a `.md`
+ * file in the viewer.
+ *
+ * Its own scanner rather than a row of [Rules] because markdown has neither keywords nor strings:
+ * what a character means depends on where it sits. A `#` opens a heading at the start of a line and
+ * is an ordinary character three words in; a `*` opens emphasis only if something closes it on the
+ * same line. The token scanner cannot ask either question, and answering them with its rules is how
+ * a highlighter comes to grey out the second half of a paragraph.
+ *
+ * Structure is read a line at a time and each line's prose is then read left to right, so every
+ * decision is made inside one line -- except a fenced block, which is the one piece of state
+ * carried across them. An unclosed fence therefore colours the rest of the text, which is also what
+ * it looks like while somebody is still writing it.
+ *
+ * What is deliberately *not* recognised: an indented code block. Four spaces after a blank line is
+ * one, and four spaces after a bullet is a list item's second paragraph, and the two are told apart
+ * by what came before rather than by the line itself. Colouring the wrong one of those as code is a
+ * mistake the reader cannot see, so both are left plain, which is the safe answer.
+ *
+ * Like [scan], the spans come out ordered, non-overlapping and inside the text by construction:
+ * every one is emitted by a pass that only moves forward, and nothing here throws.
+ */
+fun scanMarkdown(code: String): List<Span> = MarkdownScanner(code).run()
+
+/** The characters an unordered list may be bulleted with. */
+private const val BULLETS = "-*+"
+
+/** The characters a thematic break, or a setext heading's underline, can be drawn with. */
+private const val RULE_MARKERS = "-*_="
+
+/** The characters that can open emphasis, strong emphasis or a strikethrough. */
+private const val EMPHASIS = "*_~"
+
+private class MarkdownScanner(private val code: String) {
+    private val spans = ArrayList<Span>()
+
+    fun run(): List<Span> {
+        var at = 0
+        // The delimiter run that opened the fenced block we are inside, or null between them.
+        var fence: String? = null
+        while (at <= code.length) {
+            val newline = code.indexOf('\n', at)
+            val end = if (newline < 0) code.length else newline
+            val open = fence
+            if (open != null) {
+                // The content and the closing line alike: a fence is one block of code, and its
+                // own delimiters belong to it the way a string's quotes belong to the string.
+                emit(at, end, Kind.STRING)
+                if (closesFence(at, end, open)) fence = null
+            } else {
+                val opened = opensFence(at, end)
+                if (opened == null) structure(at, end)
+                fence = opened
+            }
+            if (end == code.length) break
+            at = end + 1
+        }
+        return spans
+    }
+
+    /**
+     * Spans, coalesced with the one before when they touch and agree.
+     *
+     * Worth doing here rather than leaving it to the caller: the line scanner emits per marker and
+     * per word, so a heading would otherwise arrive as a dozen abutting spans of one colour.
+     */
+    private fun emit(start: Int, end: Int, kind: Kind) {
+        if (end <= start) return
+        val last = spans.lastOrNull()
+        if (last != null && last.kind == kind && last.end == start) {
+            spans[spans.size - 1] = Span(last.start, end, kind)
+        } else {
+            spans.add(Span(start, end, kind))
+        }
+    }
+
+    /** The first character of the line at or after [start] that is not indentation. */
+    private fun indented(start: Int, end: Int): Int {
+        var at = start
+        while (at < end && (code[at] == ' ' || code[at] == '\t')) at++
+        return at
+    }
+
+    /** The run of backticks or tildes that could open or close a fence on this line, or null. */
+    private fun fenceRun(start: Int, end: Int): IntRange? {
+        val at = indented(start, end)
+        if (at == end) return null
+        val marker = code[at]
+        if (marker != '`' && marker != '~') return null
+        var run = at
+        while (run < end && code[run] == marker) run++
+        return if (run - at >= 3) at until run else null
+    }
+
+    /** Draws an opening fence line and answers its delimiter, or null if this is not one. */
+    private fun opensFence(start: Int, end: Int): String? {
+        val run = fenceRun(start, end) ?: return null
+        emit(run.first, run.last + 1, Kind.STRING)
+        // The info word is what the fence is a fence *of*, which is metadata about the block
+        // rather than part of it -- the same reading as a Rust attribute above a struct.
+        emit(indented(run.last + 1, end), end, Kind.METADATA)
+        return code.substring(run.first, run.last + 1)
+    }
+
+    /**
+     * Whether this line closes a fence opened by [open].
+     *
+     * The same character, at least as many of them, and nothing else on the line -- so a longer run
+     * closes a shorter one and a line of backticks with a word after it does not close anything.
+     */
+    private fun closesFence(start: Int, end: Int, open: String): Boolean {
+        val run = fenceRun(start, end) ?: return false
+        if (code[run.first] != open[0] || run.last + 1 - run.first < open.length) return false
+        return indented(run.last + 1, end) == end
+    }
+
+    /** One ordinary line: what its opening characters make it, and then its prose. */
+    private fun structure(start: Int, end: Int) {
+        var at = indented(start, end)
+        // Quote markers come before everything else and can be several deep, and what follows one
+        // is an ordinary line again -- a heading inside a quote is still a heading.
+        while (at < end && code[at] == '>') {
+            at++
+            emit(at - 1, at, Kind.MARK)
+            at = indented(at, end)
+        }
+        if (at == end) return
+        if (heading(at, end) || thematicBreak(at, end)) return
+        inline(bullet(at, end), end)
+    }
+
+    /** `#` to `######` and a space. Without the space it is a word beginning with a hash. */
+    private fun heading(start: Int, end: Int): Boolean {
+        var at = start
+        while (at < end && code[at] == '#') at++
+        val depth = at - start
+        if (depth !in 1..6) return false
+        if (at < end && code[at] != ' ' && code[at] != '\t') return false
+        emit(start, end, Kind.KEYWORD)
+        return true
+    }
+
+    /**
+     * A line made of one repeated rule character and nothing else.
+     *
+     * `---`, `***` and `___` are thematic breaks; `===` and `---` are also the underline of a
+     * setext heading. The two are the same line to look at and mean the same thing to a reader -- a
+     * rule drawn across the page -- so they get one appearance rather than a lookback to tell them
+     * apart. One `=` is enough because a setext underline may be a single character; a break needs
+     * three, which is what keeps a `- ` bullet out of here.
+     */
+    private fun thematicBreak(start: Int, end: Int): Boolean {
+        val marker = code[start]
+        if (marker !in RULE_MARKERS) return false
+        var seen = 0
+        for (at in start until end) {
+            val character = code[at]
+            if (character == marker) seen++ else if (!character.isWhitespace()) return false
+        }
+        if (seen < if (marker == '=') 1 else 3) return false
+        emit(start, end, Kind.MARK)
+        return true
+    }
+
+    /** Draws a list marker if the line opens with one, and answers where the item's text starts. */
+    private fun bullet(start: Int, end: Int): Int {
+        val marker = code[start]
+        if (marker in BULLETS && spaceOrEnd(start + 1, end)) {
+            emit(start, start + 1, Kind.MARK)
+            return indented(start + 1, end)
+        }
+        var digits = start
+        while (digits < end && code[digits].isDigit()) digits++
+        val delimiter = code.getOrNull(digits)
+        if (
+            digits > start && (delimiter == '.' || delimiter == ')') && spaceOrEnd(digits + 1, end)
+        ) {
+            emit(start, digits + 1, Kind.MARK)
+            return indented(digits + 1, end)
+        }
+        return start
+    }
+
+    private fun spaceOrEnd(at: Int, end: Int) = at >= end || code[at] == ' ' || code[at] == '\t'
+
+    /**
+     * The inline forms, left to right.
+     *
+     * Every branch answers a position strictly after [start] of its call, so this terminates
+     * whether or not the form it was looking at turned out to be one.
+     */
+    private fun inline(start: Int, end: Int) {
+        var at = start
+        while (at < end) {
+            val character = code[at]
+            at =
+                when {
+                    // A backslash takes the character after it out of the running entirely, which
+                    // is how `\*` stays an asterisk rather than opening emphasis.
+                    character == '\\' -> at + 2
+                    character == '`' -> codeSpan(at, end)
+                    character == '[' -> link(at, at, end)
+                    character == '!' && code.getOrNull(at + 1) == '[' -> link(at, at + 1, end)
+                    character in EMPHASIS -> emphasis(at, end)
+                    else -> at + 1
+                }
+        }
+    }
+
+    /**
+     * `` `code` ``, closed by a run of exactly as many backticks as opened it.
+     *
+     * That count is what lets a span hold a backtick of its own (``` ``a ` b`` ```), and it is why
+     * the search skips over a shorter or longer run rather than stopping at the first backtick.
+     */
+    private fun codeSpan(start: Int, end: Int): Int {
+        var open = start
+        while (open < end && code[open] == '`') open++
+        val ticks = open - start
+        var at = open
+        while (at < end) {
+            if (code[at] != '`') {
+                at++
+                continue
+            }
+            var close = at
+            while (close < end && code[close] == '`') close++
+            if (close - at == ticks) {
+                emit(start, close, Kind.STRING)
+                return close
+            }
+            at = close
+        }
+        // Nothing closes it on this line, so those were ordinary backticks.
+        return open
+    }
+
+    /**
+     * `[text](destination)`, and the same with a leading `!` for an image.
+     *
+     * The text is drawn as prose -- it is what the reader reads -- so only the brackets around it
+     * are marked, and the destination is metadata: the place the link goes rather than anything
+     * said to the reader. A `[text]` with no destination after it is left plain, because that is
+     * what a reference link and a bracketed aside look like, and neither is worth guessing at.
+     */
+    private fun link(start: Int, bracket: Int, end: Int): Int {
+        var depth = 0
+        var close = bracket
+        while (close < end) {
+            when (code[close]) {
+                '\\' -> close++
+                '[' -> depth++
+                ']' -> {
+                    depth--
+                    if (depth == 0) break
+                }
+            }
+            close++
+        }
+        if (close >= end) return start + 1
+        val destination = close + 1
+        if (code.getOrNull(destination) != '(') return start + 1
+        val paren = code.indexOf(')', destination)
+        if (paren < 0 || paren >= end) return start + 1
+        emit(start, bracket + 1, Kind.MARK)
+        inline(bracket + 1, close)
+        emit(close, destination, Kind.MARK)
+        emit(destination, paren + 1, Kind.METADATA)
+        return paren + 1
+    }
+
+    /**
+     * `*emph*`, `**strong**`, `_emph_` and `~~struck~~`, drawn markers and all.
+     *
+     * Markers and all because that is how the token scanner draws a string: the quotes are part of
+     * the thing. The two guards are what keep this off code that happens to be in a paragraph --
+     * the opener must be followed by something to emphasise and the closer preceded by something
+     * emphasised, so `a * b * c` opens nothing and neither does the `*p = *q` of a C fragment.
+     * Underscores additionally may not start or end inside a word, or every `snake_case_name` in a
+     * document would be half emphasised.
+     */
+    private fun emphasis(start: Int, end: Int): Int {
+        val marker = code[start]
+        var open = start
+        while (open < end && code[open] == marker) open++
+        val length = open - start
+        if (marker == '~' && length != 2) return open
+        if (length > 3) return open
+        if (open == end || code[open].isWhitespace()) return open
+        if (marker == '_' && start > 0 && isWord(code[start - 1])) return open
+        var at = open
+        while (at < end) {
+            if (code[at] == '\\') {
+                at += 2
+                continue
+            }
+            if (code[at] != marker) {
+                at++
+                continue
+            }
+            var close = at
+            while (close < end && code[close] == marker) close++
+            val finish = at + length
+            if (
+                close - at >= length &&
+                    !code[at - 1].isWhitespace() &&
+                    !(marker == '_' && finish < end && isWord(code[finish]))
+            ) {
+                emit(start, finish, Kind.LITERAL)
+                return finish
+            }
+            at = close
+        }
+        return open
+    }
+}
+
+private fun isWord(character: Char) = character.isLetterOrDigit() || character == '_'

app/androidApp/src/test/kotlin/com/example/aiapp/HighlighterTest.kt

@@ -16,9 +16,7 @@ import kotlin.test.assertTrue
 class HighlighterTest {
     /** Every span of [kind] in [code], as the text it covers. */
     private fun spans(code: String, language: Language, kind: Kind): List<String> =
-        scan(code, rulesOf(language))
+        spansOf(code, language).filter { it.kind == kind }.map { code.substring(it.start, it.end) }
-            .filter { it.kind == kind }
-            .map { code.substring(it.start, it.end) }
 
     private fun assertSpans(
         code: String,
@@ -182,6 +180,86 @@ class HighlighterTest {
         assertSpans(code, Language.RON, Kind.LITERAL, "3")
     }
 
+    // Markdown, which has a scanner of its own: what a character means there is decided by where
+    // it sits rather than by what it is, so most of these are about the cases where it means
+    // nothing at all.
+
+    @Test
+    fun `a heading is coloured whole and a hash inside a word is not one`() {
+        val code = "## Layout\nissue #12 is fixed\n#hashtag"
+        assertSpans(code, Language.MARKDOWN, Kind.KEYWORD, "## Layout")
+    }
+
+    @Test
+    fun `seven hashes are not a heading`() {
+        assertSpans("####### deep", Language.MARKDOWN, Kind.KEYWORD)
+    }
+
+    @Test
+    fun `a fence carries its language as metadata and its body as one string`() {
+        val code = "text\n```kotlin\nval x = 1\n```\nmore"
+        assertSpans(code, Language.MARKDOWN, Kind.METADATA, "kotlin")
+        assertSpans(code, Language.MARKDOWN, Kind.STRING, "```", "val x = 1", "```")
+    }
+
+    /** The state that crosses a line, so the one worth asking about at both ends. */
+    @Test
+    fun `a longer fence is not closed by a shorter one, and a heading inside it is not a heading`() {
+        val code = "````\n```\n# not a heading\n````\nafter"
+        assertSpans(code, Language.MARKDOWN, Kind.KEYWORD)
+        assertSpans(code, Language.MARKDOWN, Kind.STRING, "````", "```", "# not a heading", "````")
+    }
+
+    @Test
+    fun `an unclosed fence runs to the end rather than throwing`() {
+        assertSpans("```\nstill going", Language.MARKDOWN, Kind.STRING, "```", "still going")
+    }
+
+    @Test
+    fun `list markers and quote markers colour without their text`() {
+        val code = "- one\n2. two\n> quoted"
+        assertSpans(code, Language.MARKDOWN, Kind.MARK, "-", "2.", ">")
+    }
+
+    @Test
+    fun `a rule and a setext underline are the same mark`() {
+        assertSpans("Title\n=====\n\n---", Language.MARKDOWN, Kind.MARK, "=====", "---")
+    }
+
+    @Test
+    fun `emphasis needs something on both sides of it`() {
+        assertSpans("**bold** and *thin*", Language.MARKDOWN, Kind.LITERAL, "**bold**", "*thin*")
+        // The case the guards exist for: a C fragment written in a paragraph.
+        assertSpans("a * b * c and *p = *q", Language.MARKDOWN, Kind.LITERAL)
+    }
+
+    @Test
+    fun `an underscore inside a word emphasises nothing`() {
+        assertSpans("snake_case_name and _real_", Language.MARKDOWN, Kind.LITERAL, "_real_")
+    }
+
+    @Test
+    fun `a code span holds a backtick when opened with two`() {
+        assertSpans("``a ` b`` and `c`", Language.MARKDOWN, Kind.STRING, "``a ` b``", "`c`")
+    }
+
+    @Test
+    fun `an unclosed code span is ordinary text`() {
+        assertSpans("a ` b", Language.MARKDOWN, Kind.STRING)
+    }
+
+    @Test
+    fun `a link marks its brackets and colours its destination`() {
+        val code = "see [the plan](PLAN.md) now"
+        assertSpans(code, Language.MARKDOWN, Kind.MARK, "[", "]")
+        assertSpans(code, Language.MARKDOWN, Kind.METADATA, "(PLAN.md)")
+    }
+
+    @Test
+    fun `a bracket with no destination after it is left plain`() {
+        assertSpans("an [aside] here", Language.MARKDOWN, Kind.MARK)
+    }
+
     @Test
     fun `an unknown fence language is drawn plain`() {
         assertEquals(null, fenceLanguage("brainfuck"))
@@ -189,8 +267,8 @@ class HighlighterTest {
     }
 
     @Test
-    fun `every alias the fence table knows has rules`() {
+    fun `every language the fence table knows has a scanner`() {
-        Language.entries.forEach { rulesOf(it) }
+        Language.entries.forEach { spansOf("x", it) }
     }
 
     /**
@@ -218,11 +296,20 @@ class HighlighterTest {
                 "0x",
                 "1.2.3",
                 "a#b//c/*d*/'e\"f",
+                "```",
+                "*",
+                "**",
+                "~~",
+                "> ",
+                "- ",
+                "1.",
+                "[x](",
+                "#######",
                 "\n\n   \n",
             )
         for (language in Language.entries) {
             for (code in nasty) {
-                val spans = scan(code, rulesOf(language))
+                val spans = spansOf(code, language)
                 spans.forEach {
                     assertTrue(
                         it.start in 0..it.end && it.end <= code.length,

app/ui-sandbox.sh

@@ -270,6 +270,27 @@ printf 'def main():\n    # a comment\n    print("hello")\n' >"$FILES/main.py"
 printf '#!/bin/sh\n# a comment\necho hello\n' >"$FILES/run.sh"
 chmod +x "$FILES/run.sh"
 printf '{"a": 1, "b": [true, null]}\n' >"$FILES/data.json"
+# Markdown's scanner is line-structured rather than tokens, so the fixture holds one of each
+# thing it decides by position: a heading, a fence, a list, a quote, a link and a rule.
+cat >"$FILES/notes.md" <<'MARKDOWN'
+# Notes
+
+A paragraph with `code`, **bold** and a [link](PLAN.md).
+Not emphasis: a * b * c, and snake_case_name.
+
+## A list
+
+- one
+- two
+
+> quoted
+
+```rust
+fn main() { println!("hello"); }
+```
+
+---
+MARKDOWN
 # Not UTF-8, so it reads as binary rather than as mojibake.
 printf '\377\376\000\001binary\n' >"$FILES/picture.bin"
 # Over FILE_LIMIT (1 MiB), so the read refuses before anything transfers.