Colour markdown, which is the one language that is not tokens

The token scanner asks what a character is; markdown's meaning is where it
sits, so a `#` opens a heading at the start of a line and is an ordinary
character three words in. `MarkdownSyntax.kt` reads structure a line at a
time and then each line's prose left to right, and `spansOf` is the one
entry point that hides which of the two scanners a language got.

Conservative wherever a guess would be invisible: emphasis needs a closer on
the same line with no space beside either marker, so the `*p = *q` of a C
fragment opens nothing; an underscore may not start or end inside a word;
and an indented code block is left plain, since four spaces after a blank
line and four after a bullet are the same line.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
irisandClaude Opus 5 committed 2026-09-04 02:37:32 -04:00
1 parent a401e6a7e3
commit 68c5180260
8 files changed
+476 -12

No files matched your search

@@ -16,9 +16,7 @@ import kotlin.test.assertTrue
class HighlighterTest {
/** Every span of [kind] in [code], as the text it covers. */
private fun spans(code: String, language: Language, kind: Kind): List<String> =
scan(code, rulesOf(language))
.filter { it.kind == kind }
.map { code.substring(it.start, it.end) }
spansOf(code, language).filter { it.kind == kind }.map { code.substring(it.start, it.end) }
private fun assertSpans(
code: String,
@@ -182,6 +180,86 @@ class HighlighterTest {
assertSpans(code, Language.RON, Kind.LITERAL, "3")
}
// Markdown, which has a scanner of its own: what a character means there is decided by where
// it sits rather than by what it is, so most of these are about the cases where it means
// nothing at all.
@Test
fun `a heading is coloured whole and a hash inside a word is not one`() {
val code = "## Layout\nissue #12 is fixed\n#hashtag"
assertSpans(code, Language.MARKDOWN, Kind.KEYWORD, "## Layout")
}
@Test
fun `seven hashes are not a heading`() {
assertSpans("####### deep", Language.MARKDOWN, Kind.KEYWORD)
}
@Test
fun `a fence carries its language as metadata and its body as one string`() {
val code = "text\n```kotlin\nval x = 1\n```\nmore"
assertSpans(code, Language.MARKDOWN, Kind.METADATA, "kotlin")
assertSpans(code, Language.MARKDOWN, Kind.STRING, "```", "val x = 1", "```")
}
/** The state that crosses a line, so the one worth asking about at both ends. */
@Test
fun `a longer fence is not closed by a shorter one, and a heading inside it is not a heading`() {
val code = "````\n```\n# not a heading\n````\nafter"
assertSpans(code, Language.MARKDOWN, Kind.KEYWORD)
assertSpans(code, Language.MARKDOWN, Kind.STRING, "````", "```", "# not a heading", "````")
}
@Test
fun `an unclosed fence runs to the end rather than throwing`() {
assertSpans("```\nstill going", Language.MARKDOWN, Kind.STRING, "```", "still going")
}
@Test
fun `list markers and quote markers colour without their text`() {
val code = "- one\n2. two\n> quoted"
assertSpans(code, Language.MARKDOWN, Kind.MARK, "-", "2.", ">")
}
@Test
fun `a rule and a setext underline are the same mark`() {
assertSpans("Title\n=====\n\n---", Language.MARKDOWN, Kind.MARK, "=====", "---")
}
@Test
fun `emphasis needs something on both sides of it`() {
assertSpans("**bold** and *thin*", Language.MARKDOWN, Kind.LITERAL, "**bold**", "*thin*")
// The case the guards exist for: a C fragment written in a paragraph.
assertSpans("a * b * c and *p = *q", Language.MARKDOWN, Kind.LITERAL)
}
@Test
fun `an underscore inside a word emphasises nothing`() {
assertSpans("snake_case_name and _real_", Language.MARKDOWN, Kind.LITERAL, "_real_")
}
@Test
fun `a code span holds a backtick when opened with two`() {
assertSpans("``a ` b`` and `c`", Language.MARKDOWN, Kind.STRING, "``a ` b``", "`c`")
}
@Test
fun `an unclosed code span is ordinary text`() {
assertSpans("a ` b", Language.MARKDOWN, Kind.STRING)
}
@Test
fun `a link marks its brackets and colours its destination`() {
val code = "see [the plan](PLAN.md) now"
assertSpans(code, Language.MARKDOWN, Kind.MARK, "[", "]")
assertSpans(code, Language.MARKDOWN, Kind.METADATA, "(PLAN.md)")
}
@Test
fun `a bracket with no destination after it is left plain`() {
assertSpans("an [aside] here", Language.MARKDOWN, Kind.MARK)
}
@Test
fun `an unknown fence language is drawn plain`() {
assertEquals(null, fenceLanguage("brainfuck"))
@@ -189,8 +267,8 @@ class HighlighterTest {
}
@Test
fun `every alias the fence table knows has rules`() {
Language.entries.forEach { rulesOf(it) }
fun `every language the fence table knows has a scanner`() {
Language.entries.forEach { spansOf("x", it) }
}
/**
@@ -218,11 +296,20 @@ class HighlighterTest {
"0x",
"1.2.3",
"a#b//c/*d*/'e\"f",
"```",
"*",
"**",
"~~",
"> ",
"- ",
"1.",
"[x](",
"#######",
"\n\n \n",
)
for (language in Language.entries) {
for (code in nasty) {
val spans = scan(code, rulesOf(language))
val spans = spansOf(code, language)
spans.forEach {
assertTrue(
it.start in 0..it.end && it.end <= code.length,