From 68c518026030fbca18955e8a3939e7085ca45824 Mon Sep 17 00:00:00 2001 From: iris <2+iris@noreply.localhost> Date: Fri, 4 Sep 2026 02:37:32 -0400 Subject: [PATCH] Colour markdown, which is the one language that is not tokens The token scanner asks what a character is; markdown's meaning is where it sits, so a `#` opens a heading at the start of a line and is an ordinary character three words in. `MarkdownSyntax.kt` reads structure a line at a time and then each line's prose left to right, and `spansOf` is the one entry point that hides which of the two scanners a language got. Conservative wherever a guess would be invisible: emphasis needs a closer on the same line with no space beside either marker, so the `*p = *q` of a C fragment opens nothing; an underscore may not start or end inside a word; and an indented code block is left plain, since four spaces after a blank line and four after a bullet are the same line. Co-Authored-By: Claude Opus 5 --- TRANSCRIPT_RENDERING.md | 15 + .../kotlin/com/example/aiapp/CodeFence.kt | 2 + .../kotlin/com/example/aiapp/FileLines.kt | 3 +- .../kotlin/com/example/aiapp/Highlighter.kt | 2 +- .../kotlin/com/example/aiapp/Languages.kt | 25 +- .../com/example/aiapp/MarkdownSyntax.kt | 321 ++++++++++++++++++ .../com/example/aiapp/HighlighterTest.kt | 99 +++++- app/ui-sandbox.sh | 21 ++ 8 files changed, 476 insertions(+), 12 deletions(-) create mode 100644 app/androidApp/src/main/kotlin/com/example/aiapp/MarkdownSyntax.kt diff --git a/TRANSCRIPT_RENDERING.md b/TRANSCRIPT_RENDERING.md index 72ac5a9..68bdccc 100644 --- a/TRANSCRIPT_RENDERING.md +++ b/TRANSCRIPT_RENDERING.md @@ -239,6 +239,21 @@ and are the reason several tempting simplifications were rejected. to the end of the code. `HighlighterTest.kt` is the JVM unit test (`./gradlew :androidApp:testDebugUnitTest`); the cases in it are the library's mistakes, kept as regressions. + **Markdown is the one language that is not a `Rules` row** + (`MarkdownSyntax.kt`, added 2026-09-04). It has no keywords and no + strings, and what a character means there depends on where on the line it + sits -- a `#` opens a heading at the start of a line and is an ordinary + character three words in -- which is a question the token scanner cannot + ask. So it carries a scanner of its own, structure a line at a time and + then the inline forms left to right, and `spansOf` in `Languages.kt` is + the one entry point that hides which of the two a language got. It is + deliberately conservative where a guess would be invisible: emphasis needs + a closer on the same line with no space beside either marker (so the + `*p = *q` of a C fragment in a paragraph opens nothing), an underscore may + not start or end inside a word (`snake_case_name`), and an indented code + block is left plain, because four spaces after a blank line and four + spaces after a bullet are the same line and only what came before tells + them apart. It replaced dev.snipme:highlights 1.1.0 on 2026-09-03, which found comments before it knew the language and paired `/*` with `*/` by ordinal. That library used one set of delimiters for every language, so diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/CodeFence.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/CodeFence.kt index f325c05..5bcf9ad 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/CodeFence.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/CodeFence.kt @@ -201,6 +201,8 @@ private val FENCE_LANGUAGES: Map = "toml" to Language.TOML, "fish" to Language.FISH, "json" to Language.JSON, + "markdown" to Language.MARKDOWN, + "md" to Language.MARKDOWN, ) /** diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/FileLines.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/FileLines.kt index cdff8c3..f5fa96c 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/FileLines.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/FileLines.kt @@ -73,8 +73,7 @@ private constructor( DebugStats.timed("file scanned and cut into lines") { val body = text.removeSuffix("\n") val lines = body.split('\n') - val rules = language?.let { rulesOf(it) } - val scanned = if (rules == null) emptyList() else scan(body, rules) + val scanned = if (language == null) emptyList() else spansOf(body, language) FileLines(lines, bucket(lines, scanned), lines.maxOf(::columnsOf)) } diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Highlighter.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Highlighter.kt index e5122bb..9dc2597 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Highlighter.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Highlighter.kt @@ -60,7 +60,7 @@ data class SyntaxPalette( */ fun highlight(code: String, language: Language?): AnnotatedString { if (language == null) return AnnotatedString(code) - val spans = DebugStats.timed("code highlighted") { scan(code, rulesOf(language)) } + val spans = DebugStats.timed("code highlighted") { spansOf(code, language) } val palette = catppuccinSyntax() return buildAnnotatedString { append(code) diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/Languages.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/Languages.kt index c03c715..2c8d109 100644 --- a/app/androidApp/src/main/kotlin/com/example/aiapp/Languages.kt +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/Languages.kt @@ -1,11 +1,14 @@ package com.example.aiapp /** - * A language the highlighter has rules for. + * A language the highlighter can colour. * * The names the reader writes after the backticks are aliases onto these; [fenceLanguage] holds * that table. A word with no entry there is null, and null is drawn plain, because a fence coloured * by another language's rules looks highlighted and is wrong in a way the reader cannot see. + * + * Nearly all of them are a row of [RULES], read by one shared scanner. [MARKDOWN] is the one that + * is not; see [spansOf]. */ enum class Language { C, @@ -19,6 +22,7 @@ enum class Language { JAVASCRIPT, JSON, KOTLIN, + MARKDOWN, PERL, PHP, PYTHON, @@ -83,8 +87,23 @@ enum class Attributes { LINE_BRACKET, } -/** The rules for [language]. */ -fun rulesOf(language: Language): Rules = RULES.getValue(language) +/** + * The spans [language] colours in [code] -- the one way to ask, whatever the language turns out to + * be made of. + * + * Nearly every language here is tokens: keywords, strings and comments, which is a row of [RULES] + * and the one shared scanner in [scan]. Markdown has none of those, and what a character means + * there depends on where on the line it sits, so it brings a scanner of its own ([scanMarkdown]). + * That is the whole extension point -- a new language is a row of rules or an entry in [SCANNERS], + * and no caller learns which one it got. + */ +fun spansOf(code: String, language: Language): List = SCANNERS.getValue(language)(code) + +// Lazy for the same reason [RULES] is, since it reads it. +private val SCANNERS: Map List> by lazy { + RULES.mapValues { (_, rules) -> { code: String -> scan(code, rules) } } + + mapOf(Language.MARKDOWN to ::scanMarkdown) +} private val C_STYLE = BlockComment("/*", "*/", nests = false) private val NESTING = BlockComment("/*", "*/", nests = true) diff --git a/app/androidApp/src/main/kotlin/com/example/aiapp/MarkdownSyntax.kt b/app/androidApp/src/main/kotlin/com/example/aiapp/MarkdownSyntax.kt new file mode 100644 index 0000000..b1f0032 --- /dev/null +++ b/app/androidApp/src/main/kotlin/com/example/aiapp/MarkdownSyntax.kt @@ -0,0 +1,321 @@ +package com.example.aiapp + +/** + * Markdown read into the spans that carry a colour -- a ```markdown fence in a reply, and a `.md` + * file in the viewer. + * + * Its own scanner rather than a row of [Rules] because markdown has neither keywords nor strings: + * what a character means depends on where it sits. A `#` opens a heading at the start of a line and + * is an ordinary character three words in; a `*` opens emphasis only if something closes it on the + * same line. The token scanner cannot ask either question, and answering them with its rules is how + * a highlighter comes to grey out the second half of a paragraph. + * + * Structure is read a line at a time and each line's prose is then read left to right, so every + * decision is made inside one line -- except a fenced block, which is the one piece of state + * carried across them. An unclosed fence therefore colours the rest of the text, which is also what + * it looks like while somebody is still writing it. + * + * What is deliberately *not* recognised: an indented code block. Four spaces after a blank line is + * one, and four spaces after a bullet is a list item's second paragraph, and the two are told apart + * by what came before rather than by the line itself. Colouring the wrong one of those as code is a + * mistake the reader cannot see, so both are left plain, which is the safe answer. + * + * Like [scan], the spans come out ordered, non-overlapping and inside the text by construction: + * every one is emitted by a pass that only moves forward, and nothing here throws. + */ +fun scanMarkdown(code: String): List = MarkdownScanner(code).run() + +/** The characters an unordered list may be bulleted with. */ +private const val BULLETS = "-*+" + +/** The characters a thematic break, or a setext heading's underline, can be drawn with. */ +private const val RULE_MARKERS = "-*_=" + +/** The characters that can open emphasis, strong emphasis or a strikethrough. */ +private const val EMPHASIS = "*_~" + +private class MarkdownScanner(private val code: String) { + private val spans = ArrayList() + + fun run(): List { + var at = 0 + // The delimiter run that opened the fenced block we are inside, or null between them. + var fence: String? = null + while (at <= code.length) { + val newline = code.indexOf('\n', at) + val end = if (newline < 0) code.length else newline + val open = fence + if (open != null) { + // The content and the closing line alike: a fence is one block of code, and its + // own delimiters belong to it the way a string's quotes belong to the string. + emit(at, end, Kind.STRING) + if (closesFence(at, end, open)) fence = null + } else { + val opened = opensFence(at, end) + if (opened == null) structure(at, end) + fence = opened + } + if (end == code.length) break + at = end + 1 + } + return spans + } + + /** + * Spans, coalesced with the one before when they touch and agree. + * + * Worth doing here rather than leaving it to the caller: the line scanner emits per marker and + * per word, so a heading would otherwise arrive as a dozen abutting spans of one colour. + */ + private fun emit(start: Int, end: Int, kind: Kind) { + if (end <= start) return + val last = spans.lastOrNull() + if (last != null && last.kind == kind && last.end == start) { + spans[spans.size - 1] = Span(last.start, end, kind) + } else { + spans.add(Span(start, end, kind)) + } + } + + /** The first character of the line at or after [start] that is not indentation. */ + private fun indented(start: Int, end: Int): Int { + var at = start + while (at < end && (code[at] == ' ' || code[at] == '\t')) at++ + return at + } + + /** The run of backticks or tildes that could open or close a fence on this line, or null. */ + private fun fenceRun(start: Int, end: Int): IntRange? { + val at = indented(start, end) + if (at == end) return null + val marker = code[at] + if (marker != '`' && marker != '~') return null + var run = at + while (run < end && code[run] == marker) run++ + return if (run - at >= 3) at until run else null + } + + /** Draws an opening fence line and answers its delimiter, or null if this is not one. */ + private fun opensFence(start: Int, end: Int): String? { + val run = fenceRun(start, end) ?: return null + emit(run.first, run.last + 1, Kind.STRING) + // The info word is what the fence is a fence *of*, which is metadata about the block + // rather than part of it -- the same reading as a Rust attribute above a struct. + emit(indented(run.last + 1, end), end, Kind.METADATA) + return code.substring(run.first, run.last + 1) + } + + /** + * Whether this line closes a fence opened by [open]. + * + * The same character, at least as many of them, and nothing else on the line -- so a longer run + * closes a shorter one and a line of backticks with a word after it does not close anything. + */ + private fun closesFence(start: Int, end: Int, open: String): Boolean { + val run = fenceRun(start, end) ?: return false + if (code[run.first] != open[0] || run.last + 1 - run.first < open.length) return false + return indented(run.last + 1, end) == end + } + + /** One ordinary line: what its opening characters make it, and then its prose. */ + private fun structure(start: Int, end: Int) { + var at = indented(start, end) + // Quote markers come before everything else and can be several deep, and what follows one + // is an ordinary line again -- a heading inside a quote is still a heading. + while (at < end && code[at] == '>') { + at++ + emit(at - 1, at, Kind.MARK) + at = indented(at, end) + } + if (at == end) return + if (heading(at, end) || thematicBreak(at, end)) return + inline(bullet(at, end), end) + } + + /** `#` to `######` and a space. Without the space it is a word beginning with a hash. */ + private fun heading(start: Int, end: Int): Boolean { + var at = start + while (at < end && code[at] == '#') at++ + val depth = at - start + if (depth !in 1..6) return false + if (at < end && code[at] != ' ' && code[at] != '\t') return false + emit(start, end, Kind.KEYWORD) + return true + } + + /** + * A line made of one repeated rule character and nothing else. + * + * `---`, `***` and `___` are thematic breaks; `===` and `---` are also the underline of a + * setext heading. The two are the same line to look at and mean the same thing to a reader -- a + * rule drawn across the page -- so they get one appearance rather than a lookback to tell them + * apart. One `=` is enough because a setext underline may be a single character; a break needs + * three, which is what keeps a `- ` bullet out of here. + */ + private fun thematicBreak(start: Int, end: Int): Boolean { + val marker = code[start] + if (marker !in RULE_MARKERS) return false + var seen = 0 + for (at in start until end) { + val character = code[at] + if (character == marker) seen++ else if (!character.isWhitespace()) return false + } + if (seen < if (marker == '=') 1 else 3) return false + emit(start, end, Kind.MARK) + return true + } + + /** Draws a list marker if the line opens with one, and answers where the item's text starts. */ + private fun bullet(start: Int, end: Int): Int { + val marker = code[start] + if (marker in BULLETS && spaceOrEnd(start + 1, end)) { + emit(start, start + 1, Kind.MARK) + return indented(start + 1, end) + } + var digits = start + while (digits < end && code[digits].isDigit()) digits++ + val delimiter = code.getOrNull(digits) + if ( + digits > start && (delimiter == '.' || delimiter == ')') && spaceOrEnd(digits + 1, end) + ) { + emit(start, digits + 1, Kind.MARK) + return indented(digits + 1, end) + } + return start + } + + private fun spaceOrEnd(at: Int, end: Int) = at >= end || code[at] == ' ' || code[at] == '\t' + + /** + * The inline forms, left to right. + * + * Every branch answers a position strictly after [start] of its call, so this terminates + * whether or not the form it was looking at turned out to be one. + */ + private fun inline(start: Int, end: Int) { + var at = start + while (at < end) { + val character = code[at] + at = + when { + // A backslash takes the character after it out of the running entirely, which + // is how `\*` stays an asterisk rather than opening emphasis. + character == '\\' -> at + 2 + character == '`' -> codeSpan(at, end) + character == '[' -> link(at, at, end) + character == '!' && code.getOrNull(at + 1) == '[' -> link(at, at + 1, end) + character in EMPHASIS -> emphasis(at, end) + else -> at + 1 + } + } + } + + /** + * `` `code` ``, closed by a run of exactly as many backticks as opened it. + * + * That count is what lets a span hold a backtick of its own (``` ``a ` b`` ```), and it is why + * the search skips over a shorter or longer run rather than stopping at the first backtick. + */ + private fun codeSpan(start: Int, end: Int): Int { + var open = start + while (open < end && code[open] == '`') open++ + val ticks = open - start + var at = open + while (at < end) { + if (code[at] != '`') { + at++ + continue + } + var close = at + while (close < end && code[close] == '`') close++ + if (close - at == ticks) { + emit(start, close, Kind.STRING) + return close + } + at = close + } + // Nothing closes it on this line, so those were ordinary backticks. + return open + } + + /** + * `[text](destination)`, and the same with a leading `!` for an image. + * + * The text is drawn as prose -- it is what the reader reads -- so only the brackets around it + * are marked, and the destination is metadata: the place the link goes rather than anything + * said to the reader. A `[text]` with no destination after it is left plain, because that is + * what a reference link and a bracketed aside look like, and neither is worth guessing at. + */ + private fun link(start: Int, bracket: Int, end: Int): Int { + var depth = 0 + var close = bracket + while (close < end) { + when (code[close]) { + '\\' -> close++ + '[' -> depth++ + ']' -> { + depth-- + if (depth == 0) break + } + } + close++ + } + if (close >= end) return start + 1 + val destination = close + 1 + if (code.getOrNull(destination) != '(') return start + 1 + val paren = code.indexOf(')', destination) + if (paren < 0 || paren >= end) return start + 1 + emit(start, bracket + 1, Kind.MARK) + inline(bracket + 1, close) + emit(close, destination, Kind.MARK) + emit(destination, paren + 1, Kind.METADATA) + return paren + 1 + } + + /** + * `*emph*`, `**strong**`, `_emph_` and `~~struck~~`, drawn markers and all. + * + * Markers and all because that is how the token scanner draws a string: the quotes are part of + * the thing. The two guards are what keep this off code that happens to be in a paragraph -- + * the opener must be followed by something to emphasise and the closer preceded by something + * emphasised, so `a * b * c` opens nothing and neither does the `*p = *q` of a C fragment. + * Underscores additionally may not start or end inside a word, or every `snake_case_name` in a + * document would be half emphasised. + */ + private fun emphasis(start: Int, end: Int): Int { + val marker = code[start] + var open = start + while (open < end && code[open] == marker) open++ + val length = open - start + if (marker == '~' && length != 2) return open + if (length > 3) return open + if (open == end || code[open].isWhitespace()) return open + if (marker == '_' && start > 0 && isWord(code[start - 1])) return open + var at = open + while (at < end) { + if (code[at] == '\\') { + at += 2 + continue + } + if (code[at] != marker) { + at++ + continue + } + var close = at + while (close < end && code[close] == marker) close++ + val finish = at + length + if ( + close - at >= length && + !code[at - 1].isWhitespace() && + !(marker == '_' && finish < end && isWord(code[finish])) + ) { + emit(start, finish, Kind.LITERAL) + return finish + } + at = close + } + return open + } +} + +private fun isWord(character: Char) = character.isLetterOrDigit() || character == '_' diff --git a/app/androidApp/src/test/kotlin/com/example/aiapp/HighlighterTest.kt b/app/androidApp/src/test/kotlin/com/example/aiapp/HighlighterTest.kt index 4e5a43d..108c633 100644 --- a/app/androidApp/src/test/kotlin/com/example/aiapp/HighlighterTest.kt +++ b/app/androidApp/src/test/kotlin/com/example/aiapp/HighlighterTest.kt @@ -16,9 +16,7 @@ import kotlin.test.assertTrue class HighlighterTest { /** Every span of [kind] in [code], as the text it covers. */ private fun spans(code: String, language: Language, kind: Kind): List = - scan(code, rulesOf(language)) - .filter { it.kind == kind } - .map { code.substring(it.start, it.end) } + spansOf(code, language).filter { it.kind == kind }.map { code.substring(it.start, it.end) } private fun assertSpans( code: String, @@ -182,6 +180,86 @@ class HighlighterTest { assertSpans(code, Language.RON, Kind.LITERAL, "3") } + // Markdown, which has a scanner of its own: what a character means there is decided by where + // it sits rather than by what it is, so most of these are about the cases where it means + // nothing at all. + + @Test + fun `a heading is coloured whole and a hash inside a word is not one`() { + val code = "## Layout\nissue #12 is fixed\n#hashtag" + assertSpans(code, Language.MARKDOWN, Kind.KEYWORD, "## Layout") + } + + @Test + fun `seven hashes are not a heading`() { + assertSpans("####### deep", Language.MARKDOWN, Kind.KEYWORD) + } + + @Test + fun `a fence carries its language as metadata and its body as one string`() { + val code = "text\n```kotlin\nval x = 1\n```\nmore" + assertSpans(code, Language.MARKDOWN, Kind.METADATA, "kotlin") + assertSpans(code, Language.MARKDOWN, Kind.STRING, "```", "val x = 1", "```") + } + + /** The state that crosses a line, so the one worth asking about at both ends. */ + @Test + fun `a longer fence is not closed by a shorter one, and a heading inside it is not a heading`() { + val code = "````\n```\n# not a heading\n````\nafter" + assertSpans(code, Language.MARKDOWN, Kind.KEYWORD) + assertSpans(code, Language.MARKDOWN, Kind.STRING, "````", "```", "# not a heading", "````") + } + + @Test + fun `an unclosed fence runs to the end rather than throwing`() { + assertSpans("```\nstill going", Language.MARKDOWN, Kind.STRING, "```", "still going") + } + + @Test + fun `list markers and quote markers colour without their text`() { + val code = "- one\n2. two\n> quoted" + assertSpans(code, Language.MARKDOWN, Kind.MARK, "-", "2.", ">") + } + + @Test + fun `a rule and a setext underline are the same mark`() { + assertSpans("Title\n=====\n\n---", Language.MARKDOWN, Kind.MARK, "=====", "---") + } + + @Test + fun `emphasis needs something on both sides of it`() { + assertSpans("**bold** and *thin*", Language.MARKDOWN, Kind.LITERAL, "**bold**", "*thin*") + // The case the guards exist for: a C fragment written in a paragraph. + assertSpans("a * b * c and *p = *q", Language.MARKDOWN, Kind.LITERAL) + } + + @Test + fun `an underscore inside a word emphasises nothing`() { + assertSpans("snake_case_name and _real_", Language.MARKDOWN, Kind.LITERAL, "_real_") + } + + @Test + fun `a code span holds a backtick when opened with two`() { + assertSpans("``a ` b`` and `c`", Language.MARKDOWN, Kind.STRING, "``a ` b``", "`c`") + } + + @Test + fun `an unclosed code span is ordinary text`() { + assertSpans("a ` b", Language.MARKDOWN, Kind.STRING) + } + + @Test + fun `a link marks its brackets and colours its destination`() { + val code = "see [the plan](PLAN.md) now" + assertSpans(code, Language.MARKDOWN, Kind.MARK, "[", "]") + assertSpans(code, Language.MARKDOWN, Kind.METADATA, "(PLAN.md)") + } + + @Test + fun `a bracket with no destination after it is left plain`() { + assertSpans("an [aside] here", Language.MARKDOWN, Kind.MARK) + } + @Test fun `an unknown fence language is drawn plain`() { assertEquals(null, fenceLanguage("brainfuck")) @@ -189,8 +267,8 @@ class HighlighterTest { } @Test - fun `every alias the fence table knows has rules`() { - Language.entries.forEach { rulesOf(it) } + fun `every language the fence table knows has a scanner`() { + Language.entries.forEach { spansOf("x", it) } } /** @@ -218,11 +296,20 @@ class HighlighterTest { "0x", "1.2.3", "a#b//c/*d*/'e\"f", + "```", + "*", + "**", + "~~", + "> ", + "- ", + "1.", + "[x](", + "#######", "\n\n \n", ) for (language in Language.entries) { for (code in nasty) { - val spans = scan(code, rulesOf(language)) + val spans = spansOf(code, language) spans.forEach { assertTrue( it.start in 0..it.end && it.end <= code.length, diff --git a/app/ui-sandbox.sh b/app/ui-sandbox.sh index bde9a43..a735549 100755 --- a/app/ui-sandbox.sh +++ b/app/ui-sandbox.sh @@ -270,6 +270,27 @@ printf 'def main():\n # a comment\n print("hello")\n' >"$FILES/main.py" printf '#!/bin/sh\n# a comment\necho hello\n' >"$FILES/run.sh" chmod +x "$FILES/run.sh" printf '{"a": 1, "b": [true, null]}\n' >"$FILES/data.json" +# Markdown's scanner is line-structured rather than tokens, so the fixture holds one of each +# thing it decides by position: a heading, a fence, a list, a quote, a link and a rule. +cat >"$FILES/notes.md" <<'MARKDOWN' +# Notes + +A paragraph with `code`, **bold** and a [link](PLAN.md). +Not emphasis: a * b * c, and snake_case_name. + +## A list + +- one +- two + +> quoted + +```rust +fn main() { println!("hello"); } +``` + +--- +MARKDOWN # Not UTF-8, so it reads as binary rather than as mojibake. printf '\377\376\000\001binary\n' >"$FILES/picture.bin" # Over FILE_LIMIT (1 MiB), so the read refuses before anything transfers.