//! Markdown read into the spans that carry a colour -- a ```markdown fence //! in a reply, and a `.md` file in the viewer. Ported from //! `app/.../MarkdownSyntax.kt`; see that file's doc for why this is its own //! scanner rather than a row of [`super::Rules`] (what a character means //! depends on where it sits, not on what it is) and why an indented code //! block is deliberately not recognised. //! //! Structure is read a line at a time and each line's prose left to right, //! except the two decisions that are not: a fenced block is state carried //! forward, and a table is found by its delimiter row, which comes after //! the header it belongs to (the one place here that looks ahead). use super::{Kind, Span}; /// The characters an unordered list may be bulleted with. const BULLETS: &str = "-*+"; /// The characters a thematic break, or a setext heading's underline, can be /// drawn with. const RULE_MARKERS: &str = "-*_="; /// The characters that can open emphasis, strong emphasis or a strikethrough. const EMPHASIS: &str = "*_~"; /// Characters that end a bare URL wherever they appear, and ones only /// trimmed off the end. const URL_STOPS: &str = "<>\"'`|"; const URL_TRAILING: &str = ".,:;!?"; pub fn scan_markdown(code: &str) -> Vec { MarkdownScanner::new(code).run() } struct MarkdownScanner { code: Vec, spans: Vec, } impl MarkdownScanner { fn new(code: &str) -> Self { Self { code: code.chars().collect(), spans: Vec::new(), } } fn run(mut self) -> Vec { let mut at = 0usize; // The delimiter run that opened the fenced block we are inside, or // None between them. let mut fence: Option> = None; // Whether the row above was part of a table, which is what makes // this one a body row. let mut table = false; loop { let end = self.line_end(at); if let Some(open) = fence.clone() { // The content and the closing line alike: a fence is one // block of code, and its own delimiters belong to it the // way a string's quotes belong to the string. self.emit(at, end, Kind::String); if self.closes_fence(at, end, &open) { fence = None; } } else { let opened = self.opens_fence(at, end); if opened.is_some() { table = false; fence = opened; } else { table = self.row(at, end, table); } } if end == self.code.len() { break; } at = end + 1; } self.spans } /// The end of the line beginning at `at`: the newline, or the end of the text. fn line_end(&self, at: usize) -> usize { self.code[at..] .iter() .position(|&c| c == '\n') .map(|p| at + p) .unwrap_or(self.code.len()) } /// One line that is not inside a fence, and whether the table it may be /// part of is still open. fn row(&mut self, start: usize, end: usize, table: bool) -> bool { if self.table_delimiter(start, end) { let indented = self.indented(start, end); self.emit(indented, end, Kind::Mark); return true; } let header = end < self.code.len() && self.table_delimiter(end + 1, self.line_end(end + 1)); if (table || header) && self.has_pipe(start, end) { self.table_row(start, end); return true; } self.structure(start, end); false } /// A line of nothing but pipes, dashes, alignment colons and space, with /// one of each needed. fn table_delimiter(&self, start: usize, end: usize) -> bool { let mut dashes = false; let mut pipes = false; for at in self.indented(start, end)..end { match self.code[at] { '-' => dashes = true, '|' => pipes = true, ':' | ' ' | '\t' => {} _ => return false, } } dashes && pipes } fn has_pipe(&self, start: usize, end: usize) -> bool { let mut at = start; while at < end { if self.code[at] == '\\' { at += 2; } else if self.code[at] == '|' { return true; } else { at += 1; } } false } /// A table row: the pipes are the structure, and what is between them is prose. fn table_row(&mut self, start: usize, end: usize) { let mut at = self.indented(start, end); let mut cell = at; while at < end { match self.code[at] { '\\' => at += 2, '|' => { self.inline(cell, at); self.emit(at, at + 1, Kind::Mark); at += 1; cell = at; } _ => at += 1, } } self.inline(cell, end); } /// Spans, coalesced with the one before when they touch and agree. fn emit(&mut self, start: usize, end: usize, kind: Kind) { if end <= start { return; } if let Some(last) = self.spans.last_mut() && last.kind == kind && last.end == start { last.end = end; return; } self.spans.push(Span { start, end, kind }); } /// The first character of the line at or after `start` that is not indentation. fn indented(&self, start: usize, end: usize) -> usize { let mut at = start; while at < end && (self.code[at] == ' ' || self.code[at] == '\t') { at += 1; } at } /// The run of backticks or tildes that could open or close a fence on /// this line, or `None`. fn fence_run(&self, start: usize, end: usize) -> Option<(usize, usize)> { let at = self.indented(start, end); if at == end { return None; } let marker = self.code[at]; if marker != '`' && marker != '~' { return None; } let mut run = at; while run < end && self.code[run] == marker { run += 1; } if run - at >= 3 { Some((at, run)) } else { None } } /// Draws an opening fence line and answers its delimiter, or `None` if /// this is not one. fn opens_fence(&mut self, start: usize, end: usize) -> Option> { let (run_start, run_end) = self.fence_run(start, end)?; self.emit(run_start, run_end, Kind::String); // The info word is what the fence is a fence *of*, which is // metadata about the block rather than part of it. let indented = self.indented(run_end, end); self.emit(indented, end, Kind::Metadata); Some(self.code[run_start..run_end].to_vec()) } /// Whether this line closes a fence opened by `open`: the same /// character, at least as many of them, and nothing else on the line. fn closes_fence(&self, start: usize, end: usize, open: &[char]) -> bool { let Some((run_start, run_end)) = self.fence_run(start, end) else { return false; }; if self.code[run_start] != open[0] || run_end - run_start < open.len() { return false; } self.indented(run_end, end) == end } /// One ordinary line: what its opening characters make it, and then its prose. fn structure(&mut self, start: usize, end: usize) { let mut at = start; // Quote markers come before everything else and can be several // deep, and what follows one is an ordinary line again -- a heading // inside a quote is still a heading. while at < end && self.code[at] == '>' { at += 1; self.emit(at - 1, at, Kind::Mark); at = self.indented(at, end); } if at == end { return; } if self.heading(at, end) || self.thematic_break(at, end) { return; } let text_start = self.bullet(at, end); self.inline(text_start, end); } /// `#` to `######` and a space. Without the space it is a word /// beginning with a hash. fn heading(&mut self, start: usize, end: usize) -> bool { let mut at = start; while at < end && self.code[at] == '#' { at += 1; } let depth = at - start; if !(1..=6).contains(&depth) { return false; } if at < end && self.code[at] != ' ' && self.code[at] != '\t' { return false; } self.emit(start, end, Kind::Keyword); true } /// A line made of one repeated rule character and nothing else. fn thematic_break(&mut self, start: usize, end: usize) -> bool { let marker = self.code[start]; if !RULE_MARKERS.contains(marker) { return false; } let mut seen = 0usize; for at in start..end { let c = self.code[at]; if c == marker { seen += 1; } else if !c.is_whitespace() { return false; } } if seen < if marker == '=' { 1 } else { 3 } { return false; } self.emit(start, end, Kind::Mark); true } /// Draws a list marker if the line opens with one, and answers where /// the item's text starts. fn bullet(&mut self, start: usize, end: usize) -> usize { let marker = self.code[start]; if BULLETS.contains(marker) && self.space_or_end(start + 1, end) { self.emit(start, start + 1, Kind::Mark); return self.indented(start + 1, end); } let mut digits = start; while digits < end && self.code[digits].is_ascii_digit() { digits += 1; } let delimiter = self.code.get(digits).copied(); if digits > start && (delimiter == Some('.') || delimiter == Some(')')) && self.space_or_end(digits + 1, end) { self.emit(start, digits + 1, Kind::Mark); return self.indented(digits + 1, end); } start } fn space_or_end(&self, at: usize, end: usize) -> bool { at >= end || self.code[at] == ' ' || self.code[at] == '\t' } /// The inline forms, left to right. Every branch answers a position /// strictly after `start` of its call, so this terminates. fn inline(&mut self, start: usize, end: usize) { let mut at = start; while at < end { let c = self.code[at]; at = if c == '\\' { // A backslash takes the character after it out of the // running entirely, which is how `\*` stays an asterisk // rather than opening emphasis. at + 2 } else if c == '`' { self.code_span(at, end) } else if c == '[' { self.link(at, at, end) } else if c == '!' && self.code.get(at + 1) == Some(&'[') { self.link(at, at + 1, end) } else if c == '<' { self.autolink(at, end) } else if EMPHASIS.contains(c) { self.emphasis(at, end) } else { self.url(at, end).unwrap_or(at + 1) }; } } /// `` `code` ``, closed by a run of exactly as many backticks as opened it. fn code_span(&mut self, start: usize, end: usize) -> usize { let mut open = start; while open < end && self.code[open] == '`' { open += 1; } let ticks = open - start; let mut at = open; while at < end { if self.code[at] != '`' { at += 1; continue; } let mut close = at; while close < end && self.code[close] == '`' { close += 1; } if close - at == ticks { self.emit(start, close, Kind::String); return close; } at = close; } // Nothing closes it on this line, so those were ordinary backticks. open } /// `[text](destination)`, and the same with a leading `!` for an image. fn link(&mut self, start: usize, bracket: usize, end: usize) -> usize { let mut depth = 0i32; let mut close = bracket; while close < end { match self.code[close] { '\\' => close += 1, '[' => depth += 1, ']' => { depth -= 1; if depth == 0 { break; } } _ => {} } close += 1; } if close >= end { return start + 1; } let destination = close + 1; if self.code.get(destination) != Some(&'(') { return start + 1; } let Some(paren_rel) = self.code[destination..].iter().position(|&c| c == ')') else { return start + 1; }; let paren = destination + paren_rel; if paren >= end { return start + 1; } self.emit(start, bracket + 1, Kind::Mark); self.inline(bracket + 1, close); self.emit(close, destination, Kind::Mark); self.emit(destination, paren + 1, Kind::Metadata); paren + 1 } /// `` and ``, drawn as the /// destination they are. fn autolink(&mut self, start: usize, end: usize) -> usize { let mut at = start + 1; let mut addressed = false; while at < end { let c = self.code[at]; if c.is_whitespace() || c == '<' { return start + 1; } if c == '>' { if !addressed { return start + 1; } self.emit(start, at + 1, Kind::Metadata); return at + 1; } if c == ':' || c == '@' { addressed = true; } at += 1; } start + 1 } /// A bare `scheme://...` written in prose, or `None` if one does not /// start here. fn url(&mut self, start: usize, end: usize) -> Option { if start > 0 && is_word(self.code[start - 1]) { return None; } let mut scheme = start; while scheme < end && self.code[scheme].is_alphabetic() { scheme += 1; } if scheme == start || !starts_with(&self.code, scheme, "://") { return None; } let body = scheme + 3; let mut at = body; let mut openers = 0i32; let mut closers = 0i32; while at < end && !self.code[at].is_whitespace() && !URL_STOPS.contains(self.code[at]) { if self.code[at] == '(' { openers += 1; } else if self.code[at] == ')' { closers += 1; } at += 1; } while at > body { let last = self.code[at - 1]; if URL_TRAILING.contains(last) { at -= 1; } else if last == ')' && closers > openers { closers -= 1; at -= 1; } else { break; } } if at == body { return None; } self.emit(start, at, Kind::Metadata); Some(at) } /// `*emph*`, `**strong**`, `_emph_` and `~~struck~~`, drawn markers and /// all. fn emphasis(&mut self, start: usize, end: usize) -> usize { let marker = self.code[start]; let mut open = start; while open < end && self.code[open] == marker { open += 1; } let length = open - start; if marker == '~' && length != 2 { return open; } if length > 3 { return open; } if open == end || self.code[open].is_whitespace() { return open; } if marker == '_' && start > 0 && is_word(self.code[start - 1]) { return open; } let mut at = open; while at < end { if self.code[at] == '\\' { at += 2; continue; } if self.code[at] != marker { at += 1; continue; } let mut close = at; while close < end && self.code[close] == marker { close += 1; } let finish = at + length; if close - at >= length && !self.code[at - 1].is_whitespace() && !(marker == '_' && finish < end && is_word(self.code[finish])) { self.emit(start, finish, Kind::Literal); return finish; } at = close; } open } } fn is_word(c: char) -> bool { c.is_alphanumeric() || c == '_' } fn starts_with(code: &[char], at: usize, token: &str) -> bool { let token: Vec = token.chars().collect(); if at + token.len() > code.len() { return false; } code[at..at + token.len()] == token[..] } #[cfg(test)] mod tests { use super::super::{Kind, Language, span_text, spans_of}; fn spans(code: &str, kind: Kind) -> Vec { let chars: Vec = code.chars().collect(); spans_of(code, Language::Markdown) .into_iter() .filter(|s| s.kind == kind) .map(|s| span_text(&chars, &s)) .collect() } fn assert_spans(code: &str, kind: Kind, expected: &[&str]) { assert_eq!(spans(code, kind), expected.to_vec(), "{kind:?} in: {code}"); } #[test] fn a_heading_is_coloured_whole_and_a_hash_inside_a_word_is_not_one() { let code = "## Layout\nissue #12 is fixed\n#hashtag"; assert_spans(code, Kind::Keyword, &["## Layout"]); } #[test] fn seven_hashes_are_not_a_heading() { assert_spans("####### deep", Kind::Keyword, &[]); } #[test] fn a_fence_carries_its_language_as_metadata_and_its_body_as_one_string() { let code = "text\n```kotlin\nval x = 1\n```\nmore"; assert_spans(code, Kind::Metadata, &["kotlin"]); assert_spans(code, Kind::String, &["```", "val x = 1", "```"]); } #[test] fn a_longer_fence_is_not_closed_by_a_shorter_one_and_a_heading_inside_it_is_not_a_heading() { let code = "````\n```\n# not a heading\n````\nafter"; assert_spans(code, Kind::Keyword, &[]); assert_spans( code, Kind::String, &["````", "```", "# not a heading", "````"], ); } #[test] fn an_unclosed_fence_runs_to_the_end_rather_than_panicking() { assert_spans("```\nstill going", Kind::String, &["```", "still going"]); } #[test] fn list_markers_and_quote_markers_colour_without_their_text() { let code = "- one\n2. two\n> quoted"; assert_spans(code, Kind::Mark, &["-", "2.", ">"]); } #[test] fn a_rule_and_a_setext_underline_are_the_same_mark() { assert_spans("Title\n=====\n\n---", Kind::Mark, &["=====", "---"]); } #[test] fn emphasis_needs_something_on_both_sides_of_it() { assert_spans( "**bold** and *thin*", Kind::Literal, &["**bold**", "*thin*"], ); assert_spans("a * b * c and *p = *q", Kind::Literal, &[]); } #[test] fn an_underscore_inside_a_word_emphasises_nothing() { assert_spans("snake_case_name and _real_", Kind::Literal, &["_real_"]); } #[test] fn a_code_span_holds_a_backtick_when_opened_with_two() { assert_spans("``a ` b`` and `c`", Kind::String, &["``a ` b``", "`c`"]); } #[test] fn an_unclosed_code_span_is_ordinary_text() { assert_spans("a ` b", Kind::String, &[]); } #[test] fn a_link_marks_its_brackets_and_colours_its_destination() { let code = "see [the plan](PLAN.md) now"; assert_spans(code, Kind::Mark, &["[", "]"]); assert_spans(code, Kind::Metadata, &["(PLAN.md)"]); } #[test] fn a_table_is_found_by_its_delimiter_row_and_pipes_elsewhere_are_plain() { let code = "| a | b |\n|---|---|\n| 1 | 2 |\n\nrun a | b in a paragraph"; assert_spans( code, Kind::Mark, &["|", "|", "|", "|---|---|", "|", "|", "|"], ); } #[test] fn a_table_without_outer_pipes_still_colours_and_the_table_ends_with_the_rows() { let code = "a | b\n--- | ---\nnot a row"; assert_spans(code, Kind::Mark, &["|", "--- | ---"]); } #[test] fn an_autolink_colours_and_an_html_tag_does_not() { let code = " and and
and "; assert_spans( code, Kind::Metadata, &["", "", "http://x"], ); } #[test] fn a_bare_url_gives_back_the_sentences_punctuation() { assert_spans( "see https://example.com/a., and ssh://host/x)", Kind::Metadata, &["https://example.com/a", "ssh://host/x"], ); } #[test] fn a_bracket_a_url_opened_itself_stays_in_it() { assert_spans( "https://en.wikipedia.org/wiki/A_(b) here", Kind::Metadata, &["https://en.wikipedia.org/wiki/A_(b)"], ); } #[test] fn a_url_inside_a_link_destination_is_not_coloured_twice() { assert_spans( "[x](https://example.com)", Kind::Metadata, &["(https://example.com)"], ); } #[test] fn a_bracket_with_no_destination_after_it_is_left_plain() { assert_spans("an [aside] here", Kind::Mark, &[]); } }