use super::{Kind, Span}; const BULLETS: &str = "-*+"; const RULE_MARKERS: &str = "-*_="; const EMPHASIS: &str = "*_~"; const URL_STOPS: &str = "<>\"'`|"; const URL_TRAILING: &str = ".,:;!?"; pub fn scan_markdown(code: &str) -> Vec { MarkdownScanner::new(code).run() } struct MarkdownScanner { code: Vec, spans: Vec, } impl MarkdownScanner { fn new(code: &str) -> Self { Self { code: code.chars().collect(), spans: Vec::new(), } } fn run(mut self) -> Vec { let mut at = 0usize; // The delimiter run that opened the fenced block we are inside, or // None between them. let mut fence: Option> = None; let mut table = false; loop { let end = self.line_end(at); if let Some(open) = fence.clone() { self.emit(at, end, Kind::String); if self.closes_fence(at, end, &open) { fence = None; } } else { let opened = self.opens_fence(at, end); if opened.is_some() { table = false; fence = opened; } else { table = self.row(at, end, table); } } if end == self.code.len() { break; } at = end + 1; } self.spans } fn line_end(&self, at: usize) -> usize { self.code[at..] .iter() .position(|&c| c == '\n') .map(|p| at + p) .unwrap_or(self.code.len()) } fn row(&mut self, start: usize, end: usize, table: bool) -> bool { if self.table_delimiter(start, end) { let indented = self.indented(start, end); self.emit(indented, end, Kind::Mark); return true; } let header = end < self.code.len() && self.table_delimiter(end + 1, self.line_end(end + 1)); if (table || header) && self.has_pipe(start, end) { self.table_row(start, end); return true; } self.structure(start, end); false } fn table_delimiter(&self, start: usize, end: usize) -> bool { let mut dashes = false; let mut pipes = false; for c in &self.code[self.indented(start, end)..end] { match c { '-' => dashes = true, '|' => pipes = true, ':' | ' ' | '\t' => {} _ => return false, } } dashes && pipes } fn has_pipe(&self, start: usize, end: usize) -> bool { let mut at = start; while at < end { if self.code[at] == '\\' { at += 2; } else if self.code[at] == '|' { return true; } else { at += 1; } } false } fn table_row(&mut self, start: usize, end: usize) { let mut at = self.indented(start, end); let mut cell = at; while at < end { match self.code[at] { '\\' => at += 2, '|' => { self.inline(cell, at); self.emit(at, at + 1, Kind::Mark); at += 1; cell = at; } _ => at += 1, } } self.inline(cell, end); } fn emit(&mut self, start: usize, end: usize, kind: Kind) { if end <= start { return; } if let Some(last) = self.spans.last_mut() && last.kind == kind && last.end == start { last.end = end; return; } self.spans.push(Span { start, end, kind }); } fn indented(&self, start: usize, end: usize) -> usize { let mut at = start; while at < end && (self.code[at] == ' ' || self.code[at] == '\t') { at += 1; } at } fn fence_run(&self, start: usize, end: usize) -> Option<(usize, usize)> { let at = self.indented(start, end); if at == end { return None; } let marker = self.code[at]; if marker != '`' && marker != '~' { return None; } let mut run = at; while run < end && self.code[run] == marker { run += 1; } if run - at >= 3 { Some((at, run)) } else { None } } fn opens_fence(&mut self, start: usize, end: usize) -> Option> { let (run_start, run_end) = self.fence_run(start, end)?; self.emit(run_start, run_end, Kind::String); let indented = self.indented(run_end, end); self.emit(indented, end, Kind::Metadata); Some(self.code[run_start..run_end].to_vec()) } fn closes_fence(&self, start: usize, end: usize, open: &[char]) -> bool { let Some((run_start, run_end)) = self.fence_run(start, end) else { return false; }; if self.code[run_start] != open[0] || run_end - run_start < open.len() { return false; } self.indented(run_end, end) == end } fn structure(&mut self, start: usize, end: usize) { let mut at = start; while at < end && self.code[at] == '>' { at += 1; self.emit(at - 1, at, Kind::Mark); at = self.indented(at, end); } if at == end { return; } if self.heading(at, end) || self.thematic_break(at, end) { return; } let text_start = self.bullet(at, end); self.inline(text_start, end); } fn heading(&mut self, start: usize, end: usize) -> bool { let mut at = start; while at < end && self.code[at] == '#' { at += 1; } let depth = at - start; if !(1..=6).contains(&depth) { return false; } if at < end && self.code[at] != ' ' && self.code[at] != '\t' { return false; } self.emit(start, end, Kind::Keyword); true } fn thematic_break(&mut self, start: usize, end: usize) -> bool { let marker = self.code[start]; if !RULE_MARKERS.contains(marker) { return false; } let mut seen = 0usize; for &c in &self.code[start..end] { if c == marker { seen += 1; } else if !c.is_whitespace() { return false; } } if seen < if marker == '=' { 1 } else { 3 } { return false; } self.emit(start, end, Kind::Mark); true } fn bullet(&mut self, start: usize, end: usize) -> usize { let marker = self.code[start]; if BULLETS.contains(marker) && self.space_or_end(start + 1, end) { self.emit(start, start + 1, Kind::Mark); return self.indented(start + 1, end); } let mut digits = start; while digits < end && self.code[digits].is_ascii_digit() { digits += 1; } let delimiter = self.code.get(digits).copied(); if digits > start && (delimiter == Some('.') || delimiter == Some(')')) && self.space_or_end(digits + 1, end) { self.emit(start, digits + 1, Kind::Mark); return self.indented(digits + 1, end); } start } fn space_or_end(&self, at: usize, end: usize) -> bool { at >= end || self.code[at] == ' ' || self.code[at] == '\t' } fn inline(&mut self, start: usize, end: usize) { let mut at = start; while at < end { let c = self.code[at]; at = if c == '\\' { at + 2 } else if c == '`' { self.code_span(at, end) } else if c == '[' { self.link(at, at, end) } else if c == '!' && self.code.get(at + 1) == Some(&'[') { self.link(at, at + 1, end) } else if c == '<' { self.autolink(at, end) } else if EMPHASIS.contains(c) { self.emphasis(at, end) } else { self.url(at, end).unwrap_or(at + 1) }; } } fn code_span(&mut self, start: usize, end: usize) -> usize { let mut open = start; while open < end && self.code[open] == '`' { open += 1; } let ticks = open - start; let mut at = open; while at < end { if self.code[at] != '`' { at += 1; continue; } let mut close = at; while close < end && self.code[close] == '`' { close += 1; } if close - at == ticks { self.emit(start, close, Kind::String); return close; } at = close; } open } fn link(&mut self, start: usize, bracket: usize, end: usize) -> usize { let mut depth = 0i32; let mut close = bracket; while close < end { match self.code[close] { '\\' => close += 1, '[' => depth += 1, ']' => { depth -= 1; if depth == 0 { break; } } _ => {} } close += 1; } if close >= end { return start + 1; } let destination = close + 1; if self.code.get(destination) != Some(&'(') { return start + 1; } let Some(paren_rel) = self.code[destination..].iter().position(|&c| c == ')') else { return start + 1; }; let paren = destination + paren_rel; if paren >= end { return start + 1; } self.emit(start, bracket + 1, Kind::Mark); self.inline(bracket + 1, close); self.emit(close, destination, Kind::Mark); self.emit(destination, paren + 1, Kind::Metadata); paren + 1 } fn autolink(&mut self, start: usize, end: usize) -> usize { let mut at = start + 1; let mut addressed = false; while at < end { let c = self.code[at]; if c.is_whitespace() || c == '<' { return start + 1; } if c == '>' { if !addressed { return start + 1; } self.emit(start, at + 1, Kind::Metadata); return at + 1; } if c == ':' || c == '@' { addressed = true; } at += 1; } start + 1 } fn url(&mut self, start: usize, end: usize) -> Option { if start > 0 && is_word(self.code[start - 1]) { return None; } let mut scheme = start; while scheme < end && self.code[scheme].is_alphabetic() { scheme += 1; } if scheme == start || !starts_with(&self.code, scheme, "://") { return None; } let body = scheme + 3; let mut at = body; let mut openers = 0i32; let mut closers = 0i32; while at < end && !self.code[at].is_whitespace() && !URL_STOPS.contains(self.code[at]) { if self.code[at] == '(' { openers += 1; } else if self.code[at] == ')' { closers += 1; } at += 1; } while at > body { let last = self.code[at - 1]; if URL_TRAILING.contains(last) { at -= 1; } else if last == ')' && closers > openers { closers -= 1; at -= 1; } else { break; } } if at == body { return None; } self.emit(start, at, Kind::Metadata); Some(at) } fn emphasis(&mut self, start: usize, end: usize) -> usize { let marker = self.code[start]; let mut open = start; while open < end && self.code[open] == marker { open += 1; } let length = open - start; if marker == '~' && length != 2 { return open; } if length > 3 { return open; } if open == end || self.code[open].is_whitespace() { return open; } if marker == '_' && start > 0 && is_word(self.code[start - 1]) { return open; } let mut at = open; while at < end { if self.code[at] == '\\' { at += 2; continue; } if self.code[at] != marker { at += 1; continue; } let mut close = at; while close < end && self.code[close] == marker { close += 1; } let finish = at + length; if close - at >= length && !self.code[at - 1].is_whitespace() && !(marker == '_' && finish < end && is_word(self.code[finish])) { self.emit(start, finish, Kind::Literal); return finish; } at = close; } open } } fn is_word(c: char) -> bool { c.is_alphanumeric() || c == '_' } fn starts_with(code: &[char], at: usize, token: &str) -> bool { let token: Vec = token.chars().collect(); if at + token.len() > code.len() { return false; } code[at..at + token.len()] == token[..] } #[cfg(test)] mod tests { use super::super::{Kind, Language, span_text, spans_of}; fn spans(code: &str, kind: Kind) -> Vec { let chars: Vec = code.chars().collect(); spans_of(code, Language::Markdown) .into_iter() .filter(|s| s.kind == kind) .map(|s| span_text(&chars, &s)) .collect() } fn assert_spans(code: &str, kind: Kind, expected: &[&str]) { assert_eq!(spans(code, kind), expected.to_vec(), "{kind:?} in: {code}"); } #[test] fn a_heading_is_coloured_whole_and_a_hash_inside_a_word_is_not_one() { let code = "## Layout\nissue #12 is fixed\n#hashtag"; assert_spans(code, Kind::Keyword, &["## Layout"]); } #[test] fn seven_hashes_are_not_a_heading() { assert_spans("####### deep", Kind::Keyword, &[]); } #[test] fn a_fence_carries_its_language_as_metadata_and_its_body_as_one_string() { let code = "text\n```kotlin\nval x = 1\n```\nmore"; assert_spans(code, Kind::Metadata, &["kotlin"]); assert_spans(code, Kind::String, &["```", "val x = 1", "```"]); } #[test] fn a_longer_fence_is_not_closed_by_a_shorter_one_and_a_heading_inside_it_is_not_a_heading() { let code = "````\n```\n# not a heading\n````\nafter"; assert_spans(code, Kind::Keyword, &[]); assert_spans( code, Kind::String, &["````", "```", "# not a heading", "````"], ); } #[test] fn an_unclosed_fence_runs_to_the_end_rather_than_panicking() { assert_spans("```\nstill going", Kind::String, &["```", "still going"]); } #[test] fn list_markers_and_quote_markers_colour_without_their_text() { let code = "- one\n2. two\n> quoted"; assert_spans(code, Kind::Mark, &["-", "2.", ">"]); } #[test] fn a_rule_and_a_setext_underline_are_the_same_mark() { assert_spans("Title\n=====\n\n---", Kind::Mark, &["=====", "---"]); } #[test] fn emphasis_needs_something_on_both_sides_of_it() { assert_spans( "**bold** and *thin*", Kind::Literal, &["**bold**", "*thin*"], ); assert_spans("a * b * c and *p = *q", Kind::Literal, &[]); } #[test] fn an_underscore_inside_a_word_emphasises_nothing() { assert_spans("snake_case_name and _real_", Kind::Literal, &["_real_"]); } #[test] fn a_code_span_holds_a_backtick_when_opened_with_two() { assert_spans("``a ` b`` and `c`", Kind::String, &["``a ` b``", "`c`"]); } #[test] fn an_unclosed_code_span_is_ordinary_text() { assert_spans("a ` b", Kind::String, &[]); } #[test] fn a_link_marks_its_brackets_and_colours_its_destination() { let code = "see [the plan](PLAN.md) now"; assert_spans(code, Kind::Mark, &["[", "]"]); assert_spans(code, Kind::Metadata, &["(PLAN.md)"]); } #[test] fn a_table_is_found_by_its_delimiter_row_and_pipes_elsewhere_are_plain() { let code = "| a | b |\n|---|---|\n| 1 | 2 |\n\nrun a | b in a paragraph"; assert_spans( code, Kind::Mark, &["|", "|", "|", "|---|---|", "|", "|", "|"], ); } #[test] fn a_table_without_outer_pipes_still_colours_and_the_table_ends_with_the_rows() { let code = "a | b\n--- | ---\nnot a row"; assert_spans(code, Kind::Mark, &["|", "--- | ---"]); } #[test] fn an_autolink_colours_and_an_html_tag_does_not() { let code = " and and
and "; assert_spans( code, Kind::Metadata, &["", "", "http://x"], ); } #[test] fn a_bare_url_gives_back_the_sentences_punctuation() { assert_spans( "see https://example.com/a., and ssh://host/x)", Kind::Metadata, &["https://example.com/a", "ssh://host/x"], ); } #[test] fn a_bracket_a_url_opened_itself_stays_in_it() { assert_spans( "https://en.wikipedia.org/wiki/A_(b) here", Kind::Metadata, &["https://en.wikipedia.org/wiki/A_(b)"], ); } #[test] fn a_url_inside_a_link_destination_is_not_coloured_twice() { assert_spans( "[x](https://example.com)", Kind::Metadata, &["(https://example.com)"], ); } #[test] fn a_bracket_with_no_destination_after_it_is_left_plain() { assert_spans("an [aside] here", Kind::Mark, &[]); } }