package com.example.aiapp /** * Markdown read into the spans that carry a colour -- a ```markdown fence in a reply, and a `.md` * file in the viewer. * * Its own scanner rather than a row of [Rules] because markdown has neither keywords nor strings: * what a character means depends on where it sits. A `#` opens a heading at the start of a line and * is an ordinary character three words in; a `*` opens emphasis only if something closes it on the * same line. The token scanner cannot ask either question. * * Structure is read a line at a time and each line's prose left to right, so every decision is made * inside one line -- except the two that are not. A fenced block is state carried forward, so an * unclosed fence colours the rest of the text, which is what it looks like while somebody is * writing it. A table is found by its delimiter row (`|---|---|`), the only line of one that cannot * be anything else, and its header is the line before that -- the one place here that looks ahead. * * What is deliberately *not* recognised: an indented code block. Four spaces after a blank line is * one, four spaces after a bullet is a list item's second paragraph, and the two are told apart by * what came before. Colouring the wrong one as code is a mistake the reader cannot see. * * Like [scan], the spans come out ordered, non-overlapping and inside the text by construction. */ fun scanMarkdown(code: String): List = MarkdownScanner(code).run() /** The characters an unordered list may be bulleted with. */ private const val BULLETS = "-*+" /** The characters a thematic break, or a setext heading's underline, can be drawn with. */ private const val RULE_MARKERS = "-*_=" /** The characters that can open emphasis, strong emphasis or a strikethrough. */ private const val EMPHASIS = "*_~" /** Characters that end a bare URL wherever they appear, and ones only trimmed off the end. */ private const val URL_STOPS = "<>\"'`|" private const val URL_TRAILING = ".,:;!?" private class MarkdownScanner(private val code: String) { private val spans = ArrayList() fun run(): List { var at = 0 // The delimiter run that opened the fenced block we are inside, or null between them. var fence: String? = null // Whether the row above was part of a table, which is what makes this one a body row. var table = false while (at <= code.length) { val end = lineEnd(at) val open = fence if (open != null) { // The content and the closing line alike: a fence is one block of code, and its own // delimiters belong to it the way a string's quotes belong to the string. emit(at, end, Kind.STRING) if (closesFence(at, end, open)) fence = null } else { val opened = opensFence(at, end) fence = opened if (opened != null) table = false else table = row(at, end, table) } if (end == code.length) break at = end + 1 } return spans } /** The end of the line beginning at [at]: the newline, or the end of the text. */ private fun lineEnd(at: Int): Int { val newline = code.indexOf('\n', at) return if (newline < 0) code.length else newline } /** * One line that is not inside a fence, and whether the table it may be part of is still open. * * A table is recognised by its delimiter row, the only line of one that cannot be anything * else. That row comes *after* the header it belongs to, so the header is found by looking one * line ahead -- the single piece of lookahead here, and cheaper than colouring every `|` in the * document, which would mark the pipes in a shell command written in a paragraph. */ private fun row(start: Int, end: Int, table: Boolean): Boolean { if (tableDelimiter(start, end)) { emit(indented(start, end), end, Kind.MARK) return true } val header = end < code.length && tableDelimiter(end + 1, lineEnd(end + 1)) if ((table || header) && hasPipe(start, end)) { tableRow(start, end) return true } structure(start, end) return false } /** A line of nothing but pipes, dashes, alignment colons and space, with one of each needed. */ private fun tableDelimiter(start: Int, end: Int): Boolean { var dashes = false var pipes = false for (at in indented(start, end) until end) { when (code[at]) { '-' -> dashes = true '|' -> pipes = true ':', ' ', '\t' -> {} else -> return false } } return dashes && pipes } private fun hasPipe(start: Int, end: Int): Boolean { var at = start while (at < end) { if (code[at] == '\\') at += 2 else if (code[at] == '|') return true else at++ } return false } /** A table row: the pipes are the structure, and what is between them is prose. */ private fun tableRow(start: Int, end: Int) { var at = indented(start, end) var cell = at while (at < end) { when (code[at]) { '\\' -> at += 2 '|' -> { inline(cell, at) emit(at, at + 1, Kind.MARK) at++ cell = at } else -> at++ } } inline(cell, end) } /** * Spans, coalesced with the one before when they touch and agree. Worth doing here rather than * leaving it to the caller: the line scanner emits per marker and per word, so a heading would * otherwise arrive as a dozen abutting spans of one colour. */ private fun emit(start: Int, end: Int, kind: Kind) { if (end <= start) return val last = spans.lastOrNull() if (last != null && last.kind == kind && last.end == start) { spans[spans.size - 1] = Span(last.start, end, kind) } else { spans.add(Span(start, end, kind)) } } /** The first character of the line at or after [start] that is not indentation. */ private fun indented(start: Int, end: Int): Int { var at = start while (at < end && (code[at] == ' ' || code[at] == '\t')) at++ return at } /** The run of backticks or tildes that could open or close a fence on this line, or null. */ private fun fenceRun(start: Int, end: Int): IntRange? { val at = indented(start, end) if (at == end) return null val marker = code[at] if (marker != '`' && marker != '~') return null var run = at while (run < end && code[run] == marker) run++ return if (run - at >= 3) at until run else null } /** Draws an opening fence line and answers its delimiter, or null if this is not one. */ private fun opensFence(start: Int, end: Int): String? { val run = fenceRun(start, end) ?: return null emit(run.first, run.last + 1, Kind.STRING) // The info word is what the fence is a fence *of*, which is metadata about the block rather // than part of it. emit(indented(run.last + 1, end), end, Kind.METADATA) return code.substring(run.first, run.last + 1) } /** * Whether this line closes a fence opened by [open]: the same character, at least as many of * them, and nothing else on the line -- so a longer run closes a shorter one and a line of * backticks with a word after it does not close anything. */ private fun closesFence(start: Int, end: Int, open: String): Boolean { val run = fenceRun(start, end) ?: return false if (code[run.first] != open[0] || run.last + 1 - run.first < open.length) return false return indented(run.last + 1, end) == end } /** One ordinary line: what its opening characters make it, and then its prose. */ private fun structure(start: Int, end: Int) { var at = indented(start, end) // Quote markers come before everything else and can be several deep, and what follows one // is an ordinary line again -- a heading inside a quote is still a heading. while (at < end && code[at] == '>') { at++ emit(at - 1, at, Kind.MARK) at = indented(at, end) } if (at == end) return if (heading(at, end) || thematicBreak(at, end)) return inline(bullet(at, end), end) } /** `#` to `######` and a space. Without the space it is a word beginning with a hash. */ private fun heading(start: Int, end: Int): Boolean { var at = start while (at < end && code[at] == '#') at++ val depth = at - start if (depth !in 1..6) return false if (at < end && code[at] != ' ' && code[at] != '\t') return false emit(start, end, Kind.KEYWORD) return true } /** * A line made of one repeated rule character and nothing else. * * `---`, `***` and `___` are thematic breaks; `===` and `---` are also the underline of a * setext heading. The two are the same line to look at and mean the same thing to a reader, so * they get one appearance rather than a lookback. One `=` is enough because a setext underline * may be a single character; a break needs three, which keeps a `- ` bullet out of here. */ private fun thematicBreak(start: Int, end: Int): Boolean { val marker = code[start] if (marker !in RULE_MARKERS) return false var seen = 0 for (at in start until end) { val character = code[at] if (character == marker) seen++ else if (!character.isWhitespace()) return false } if (seen < if (marker == '=') 1 else 3) return false emit(start, end, Kind.MARK) return true } /** Draws a list marker if the line opens with one, and answers where the item's text starts. */ private fun bullet(start: Int, end: Int): Int { val marker = code[start] if (marker in BULLETS && spaceOrEnd(start + 1, end)) { emit(start, start + 1, Kind.MARK) return indented(start + 1, end) } var digits = start while (digits < end && code[digits].isDigit()) digits++ val delimiter = code.getOrNull(digits) if ( digits > start && (delimiter == '.' || delimiter == ')') && spaceOrEnd(digits + 1, end) ) { emit(start, digits + 1, Kind.MARK) return indented(digits + 1, end) } return start } private fun spaceOrEnd(at: Int, end: Int) = at >= end || code[at] == ' ' || code[at] == '\t' /** * The inline forms, left to right. * * Every branch answers a position strictly after [start] of its call, so this terminates * whether or not the form it was looking at turned out to be one. */ private fun inline(start: Int, end: Int) { var at = start while (at < end) { val character = code[at] at = when { // A backslash takes the character after it out of the running entirely, which // is how `\*` stays an asterisk rather than opening emphasis. character == '\\' -> at + 2 character == '`' -> codeSpan(at, end) character == '[' -> link(at, at, end) character == '!' && code.getOrNull(at + 1) == '[' -> link(at, at + 1, end) character == '<' -> autolink(at, end) character in EMPHASIS -> emphasis(at, end) else -> url(at, end) ?: (at + 1) } } } /** * `` `code` ``, closed by a run of exactly as many backticks as opened it. That count is what * lets a span hold a backtick of its own, and why the search skips over a shorter or longer run * rather than stopping at the first backtick. */ private fun codeSpan(start: Int, end: Int): Int { var open = start while (open < end && code[open] == '`') open++ val ticks = open - start var at = open while (at < end) { if (code[at] != '`') { at++ continue } var close = at while (close < end && code[close] == '`') close++ if (close - at == ticks) { emit(start, close, Kind.STRING) return close } at = close } // Nothing closes it on this line, so those were ordinary backticks. return open } /** * `[text](destination)`, and the same with a leading `!` for an image. * * The text is drawn as prose -- it is what the reader reads -- so only the brackets around it * are marked, and the destination is metadata. A `[text]` with no destination after it is left * plain, because that is what a reference link and a bracketed aside look like. */ private fun link(start: Int, bracket: Int, end: Int): Int { var depth = 0 var close = bracket while (close < end) { when (code[close]) { '\\' -> close++ '[' -> depth++ ']' -> { depth-- if (depth == 0) break } } close++ } if (close >= end) return start + 1 val destination = close + 1 if (code.getOrNull(destination) != '(') return start + 1 val paren = code.indexOf(')', destination) if (paren < 0 || paren >= end) return start + 1 emit(start, bracket + 1, Kind.MARK) inline(bracket + 1, close) emit(close, destination, Kind.MARK) emit(destination, paren + 1, Kind.METADATA) return paren + 1 } /** * `` and ``, drawn as the destination they are. * * The angle brackets have to hold no whitespace and something that makes an address of it -- a * scheme's colon or an at sign -- which is what keeps an HTML tag out. */ private fun autolink(start: Int, end: Int): Int { var at = start + 1 var addressed = false while (at < end) { val character = code[at] if (character.isWhitespace() || character == '<') return start + 1 if (character == '>') { if (!addressed) return start + 1 emit(start, at + 1, Kind.METADATA) return at + 1 } if (character == ':' || character == '@') addressed = true at++ } return start + 1 } /** * A bare `scheme://…` written in prose, or null if one does not start here. * * A scheme and `://` rather than a list of them, so `ftp`, `file` and `ssh` need no entry. * * Where it ends is the part worth stating: the sentence's punctuation is not the address, so a * trailing `.` or `,` is given back, and so is a closing bracket unless one opened inside the * URL -- otherwise a link in parentheses loses its `)`. A pipe stops it too, because a URL in a * table cell must not swallow the cell's edge. */ private fun url(start: Int, end: Int): Int? { if (start > 0 && isWord(code[start - 1])) return null var scheme = start while (scheme < end && code[scheme].isLetter()) scheme++ if (scheme == start || !code.startsWith("://", scheme)) return null val body = scheme + 3 var at = body var openers = 0 var closers = 0 while (at < end && !code[at].isWhitespace() && code[at] !in URL_STOPS) { if (code[at] == '(') openers++ else if (code[at] == ')') closers++ at++ } while (at > body) { val last = code[at - 1] if (last in URL_TRAILING) at-- else if (last == ')' && closers > openers) { closers-- at-- } else break } if (at == body) return null emit(start, at, Kind.METADATA) return at } /** * `*emph*`, `**strong**`, `_emph_` and `~~struck~~`, drawn markers and all -- which is how the * token scanner draws a string: the quotes are part of the thing. * * The two guards keep this off code that happens to be in a paragraph: the opener must be * followed by something to emphasise and the closer preceded by something emphasised, so `a * b * * c` opens nothing and neither does the `*p = *q` of a C fragment. Underscores may not start * or end inside a word, or every `snake_case_name` would be half emphasised. */ private fun emphasis(start: Int, end: Int): Int { val marker = code[start] var open = start while (open < end && code[open] == marker) open++ val length = open - start if (marker == '~' && length != 2) return open if (length > 3) return open if (open == end || code[open].isWhitespace()) return open if (marker == '_' && start > 0 && isWord(code[start - 1])) return open var at = open while (at < end) { if (code[at] == '\\') { at += 2 continue } if (code[at] != marker) { at++ continue } var close = at while (close < end && code[close] == marker) close++ val finish = at + length if ( close - at >= length && !code[at - 1].isWhitespace() && !(marker == '_' && finish < end && isWord(code[finish])) ) { emit(start, finish, Kind.LITERAL) return finish } at = close } return open } } private fun isWord(character: Char) = character.isLetterOrDigit() || character == '_'