619 lines
18 KiB
Rust
619 lines
18 KiB
Rust
use super::{Kind, Span};
|
|
|
|
const BULLETS: &str = "-*+";
|
|
const RULE_MARKERS: &str = "-*_=";
|
|
const EMPHASIS: &str = "*_~";
|
|
const URL_STOPS: &str = "<>\"'`|";
|
|
const URL_TRAILING: &str = ".,:;!?";
|
|
|
|
pub fn scan_markdown(code: &str) -> Vec<Span> {
|
|
MarkdownScanner::new(code).run()
|
|
}
|
|
|
|
struct MarkdownScanner {
|
|
code: Vec<char>,
|
|
spans: Vec<Span>,
|
|
}
|
|
|
|
impl MarkdownScanner {
|
|
fn new(code: &str) -> Self {
|
|
Self {
|
|
code: code.chars().collect(),
|
|
spans: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn run(mut self) -> Vec<Span> {
|
|
let mut at = 0usize;
|
|
// The delimiter run that opened the fenced block we are inside, or
|
|
// None between them.
|
|
let mut fence: Option<Vec<char>> = None;
|
|
let mut table = false;
|
|
loop {
|
|
let end = self.line_end(at);
|
|
if let Some(open) = fence.clone() {
|
|
self.emit(at, end, Kind::String);
|
|
if self.closes_fence(at, end, &open) {
|
|
fence = None;
|
|
}
|
|
} else {
|
|
let opened = self.opens_fence(at, end);
|
|
if opened.is_some() {
|
|
table = false;
|
|
fence = opened;
|
|
} else {
|
|
table = self.row(at, end, table);
|
|
}
|
|
}
|
|
if end == self.code.len() {
|
|
break;
|
|
}
|
|
at = end + 1;
|
|
}
|
|
self.spans
|
|
}
|
|
|
|
fn line_end(&self, at: usize) -> usize {
|
|
self.code[at..]
|
|
.iter()
|
|
.position(|&c| c == '\n')
|
|
.map(|p| at + p)
|
|
.unwrap_or(self.code.len())
|
|
}
|
|
|
|
fn row(&mut self, start: usize, end: usize, table: bool) -> bool {
|
|
if self.table_delimiter(start, end) {
|
|
let indented = self.indented(start, end);
|
|
self.emit(indented, end, Kind::Mark);
|
|
return true;
|
|
}
|
|
let header = end < self.code.len() && self.table_delimiter(end + 1, self.line_end(end + 1));
|
|
if (table || header) && self.has_pipe(start, end) {
|
|
self.table_row(start, end);
|
|
return true;
|
|
}
|
|
self.structure(start, end);
|
|
false
|
|
}
|
|
|
|
fn table_delimiter(&self, start: usize, end: usize) -> bool {
|
|
let mut dashes = false;
|
|
let mut pipes = false;
|
|
for c in &self.code[self.indented(start, end)..end] {
|
|
match c {
|
|
'-' => dashes = true,
|
|
'|' => pipes = true,
|
|
':' | ' ' | '\t' => {}
|
|
_ => return false,
|
|
}
|
|
}
|
|
dashes && pipes
|
|
}
|
|
|
|
fn has_pipe(&self, start: usize, end: usize) -> bool {
|
|
let mut at = start;
|
|
while at < end {
|
|
if self.code[at] == '\\' {
|
|
at += 2;
|
|
} else if self.code[at] == '|' {
|
|
return true;
|
|
} else {
|
|
at += 1;
|
|
}
|
|
}
|
|
false
|
|
}
|
|
|
|
fn table_row(&mut self, start: usize, end: usize) {
|
|
let mut at = self.indented(start, end);
|
|
let mut cell = at;
|
|
while at < end {
|
|
match self.code[at] {
|
|
'\\' => at += 2,
|
|
'|' => {
|
|
self.inline(cell, at);
|
|
self.emit(at, at + 1, Kind::Mark);
|
|
at += 1;
|
|
cell = at;
|
|
}
|
|
_ => at += 1,
|
|
}
|
|
}
|
|
self.inline(cell, end);
|
|
}
|
|
|
|
fn emit(&mut self, start: usize, end: usize, kind: Kind) {
|
|
if end <= start {
|
|
return;
|
|
}
|
|
if let Some(last) = self.spans.last_mut()
|
|
&& last.kind == kind
|
|
&& last.end == start
|
|
{
|
|
last.end = end;
|
|
return;
|
|
}
|
|
self.spans.push(Span { start, end, kind });
|
|
}
|
|
|
|
fn indented(&self, start: usize, end: usize) -> usize {
|
|
let mut at = start;
|
|
while at < end && (self.code[at] == ' ' || self.code[at] == '\t') {
|
|
at += 1;
|
|
}
|
|
at
|
|
}
|
|
|
|
fn fence_run(&self, start: usize, end: usize) -> Option<(usize, usize)> {
|
|
let at = self.indented(start, end);
|
|
if at == end {
|
|
return None;
|
|
}
|
|
let marker = self.code[at];
|
|
if marker != '`' && marker != '~' {
|
|
return None;
|
|
}
|
|
let mut run = at;
|
|
while run < end && self.code[run] == marker {
|
|
run += 1;
|
|
}
|
|
if run - at >= 3 { Some((at, run)) } else { None }
|
|
}
|
|
|
|
fn opens_fence(&mut self, start: usize, end: usize) -> Option<Vec<char>> {
|
|
let (run_start, run_end) = self.fence_run(start, end)?;
|
|
self.emit(run_start, run_end, Kind::String);
|
|
let indented = self.indented(run_end, end);
|
|
self.emit(indented, end, Kind::Metadata);
|
|
Some(self.code[run_start..run_end].to_vec())
|
|
}
|
|
|
|
fn closes_fence(&self, start: usize, end: usize, open: &[char]) -> bool {
|
|
let Some((run_start, run_end)) = self.fence_run(start, end) else {
|
|
return false;
|
|
};
|
|
if self.code[run_start] != open[0] || run_end - run_start < open.len() {
|
|
return false;
|
|
}
|
|
self.indented(run_end, end) == end
|
|
}
|
|
|
|
fn structure(&mut self, start: usize, end: usize) {
|
|
let mut at = start;
|
|
while at < end && self.code[at] == '>' {
|
|
at += 1;
|
|
self.emit(at - 1, at, Kind::Mark);
|
|
at = self.indented(at, end);
|
|
}
|
|
if at == end {
|
|
return;
|
|
}
|
|
if self.heading(at, end) || self.thematic_break(at, end) {
|
|
return;
|
|
}
|
|
let text_start = self.bullet(at, end);
|
|
self.inline(text_start, end);
|
|
}
|
|
|
|
fn heading(&mut self, start: usize, end: usize) -> bool {
|
|
let mut at = start;
|
|
while at < end && self.code[at] == '#' {
|
|
at += 1;
|
|
}
|
|
let depth = at - start;
|
|
if !(1..=6).contains(&depth) {
|
|
return false;
|
|
}
|
|
if at < end && self.code[at] != ' ' && self.code[at] != '\t' {
|
|
return false;
|
|
}
|
|
self.emit(start, end, Kind::Keyword);
|
|
true
|
|
}
|
|
|
|
fn thematic_break(&mut self, start: usize, end: usize) -> bool {
|
|
let marker = self.code[start];
|
|
if !RULE_MARKERS.contains(marker) {
|
|
return false;
|
|
}
|
|
let mut seen = 0usize;
|
|
for &c in &self.code[start..end] {
|
|
if c == marker {
|
|
seen += 1;
|
|
} else if !c.is_whitespace() {
|
|
return false;
|
|
}
|
|
}
|
|
if seen < if marker == '=' { 1 } else { 3 } {
|
|
return false;
|
|
}
|
|
self.emit(start, end, Kind::Mark);
|
|
true
|
|
}
|
|
|
|
fn bullet(&mut self, start: usize, end: usize) -> usize {
|
|
let marker = self.code[start];
|
|
if BULLETS.contains(marker) && self.space_or_end(start + 1, end) {
|
|
self.emit(start, start + 1, Kind::Mark);
|
|
return self.indented(start + 1, end);
|
|
}
|
|
let mut digits = start;
|
|
while digits < end && self.code[digits].is_ascii_digit() {
|
|
digits += 1;
|
|
}
|
|
let delimiter = self.code.get(digits).copied();
|
|
if digits > start
|
|
&& (delimiter == Some('.') || delimiter == Some(')'))
|
|
&& self.space_or_end(digits + 1, end)
|
|
{
|
|
self.emit(start, digits + 1, Kind::Mark);
|
|
return self.indented(digits + 1, end);
|
|
}
|
|
start
|
|
}
|
|
|
|
fn space_or_end(&self, at: usize, end: usize) -> bool {
|
|
at >= end || self.code[at] == ' ' || self.code[at] == '\t'
|
|
}
|
|
|
|
fn inline(&mut self, start: usize, end: usize) {
|
|
let mut at = start;
|
|
while at < end {
|
|
let c = self.code[at];
|
|
at = if c == '\\' {
|
|
at + 2
|
|
} else if c == '`' {
|
|
self.code_span(at, end)
|
|
} else if c == '[' {
|
|
self.link(at, at, end)
|
|
} else if c == '!' && self.code.get(at + 1) == Some(&'[') {
|
|
self.link(at, at + 1, end)
|
|
} else if c == '<' {
|
|
self.autolink(at, end)
|
|
} else if EMPHASIS.contains(c) {
|
|
self.emphasis(at, end)
|
|
} else {
|
|
self.url(at, end).unwrap_or(at + 1)
|
|
};
|
|
}
|
|
}
|
|
|
|
fn code_span(&mut self, start: usize, end: usize) -> usize {
|
|
let mut open = start;
|
|
while open < end && self.code[open] == '`' {
|
|
open += 1;
|
|
}
|
|
let ticks = open - start;
|
|
let mut at = open;
|
|
while at < end {
|
|
if self.code[at] != '`' {
|
|
at += 1;
|
|
continue;
|
|
}
|
|
let mut close = at;
|
|
while close < end && self.code[close] == '`' {
|
|
close += 1;
|
|
}
|
|
if close - at == ticks {
|
|
self.emit(start, close, Kind::String);
|
|
return close;
|
|
}
|
|
at = close;
|
|
}
|
|
open
|
|
}
|
|
|
|
fn link(&mut self, start: usize, bracket: usize, end: usize) -> usize {
|
|
let mut depth = 0i32;
|
|
let mut close = bracket;
|
|
while close < end {
|
|
match self.code[close] {
|
|
'\\' => close += 1,
|
|
'[' => depth += 1,
|
|
']' => {
|
|
depth -= 1;
|
|
if depth == 0 {
|
|
break;
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
close += 1;
|
|
}
|
|
if close >= end {
|
|
return start + 1;
|
|
}
|
|
let destination = close + 1;
|
|
if self.code.get(destination) != Some(&'(') {
|
|
return start + 1;
|
|
}
|
|
let Some(paren_rel) = self.code[destination..].iter().position(|&c| c == ')') else {
|
|
return start + 1;
|
|
};
|
|
let paren = destination + paren_rel;
|
|
if paren >= end {
|
|
return start + 1;
|
|
}
|
|
self.emit(start, bracket + 1, Kind::Mark);
|
|
self.inline(bracket + 1, close);
|
|
self.emit(close, destination, Kind::Mark);
|
|
self.emit(destination, paren + 1, Kind::Metadata);
|
|
paren + 1
|
|
}
|
|
|
|
fn autolink(&mut self, start: usize, end: usize) -> usize {
|
|
let mut at = start + 1;
|
|
let mut addressed = false;
|
|
while at < end {
|
|
let c = self.code[at];
|
|
if c.is_whitespace() || c == '<' {
|
|
return start + 1;
|
|
}
|
|
if c == '>' {
|
|
if !addressed {
|
|
return start + 1;
|
|
}
|
|
self.emit(start, at + 1, Kind::Metadata);
|
|
return at + 1;
|
|
}
|
|
if c == ':' || c == '@' {
|
|
addressed = true;
|
|
}
|
|
at += 1;
|
|
}
|
|
start + 1
|
|
}
|
|
|
|
fn url(&mut self, start: usize, end: usize) -> Option<usize> {
|
|
if start > 0 && is_word(self.code[start - 1]) {
|
|
return None;
|
|
}
|
|
let mut scheme = start;
|
|
while scheme < end && self.code[scheme].is_alphabetic() {
|
|
scheme += 1;
|
|
}
|
|
if scheme == start || !starts_with(&self.code, scheme, "://") {
|
|
return None;
|
|
}
|
|
let body = scheme + 3;
|
|
let mut at = body;
|
|
let mut openers = 0i32;
|
|
let mut closers = 0i32;
|
|
while at < end && !self.code[at].is_whitespace() && !URL_STOPS.contains(self.code[at]) {
|
|
if self.code[at] == '(' {
|
|
openers += 1;
|
|
} else if self.code[at] == ')' {
|
|
closers += 1;
|
|
}
|
|
at += 1;
|
|
}
|
|
while at > body {
|
|
let last = self.code[at - 1];
|
|
if URL_TRAILING.contains(last) {
|
|
at -= 1;
|
|
} else if last == ')' && closers > openers {
|
|
closers -= 1;
|
|
at -= 1;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
if at == body {
|
|
return None;
|
|
}
|
|
self.emit(start, at, Kind::Metadata);
|
|
Some(at)
|
|
}
|
|
|
|
fn emphasis(&mut self, start: usize, end: usize) -> usize {
|
|
let marker = self.code[start];
|
|
let mut open = start;
|
|
while open < end && self.code[open] == marker {
|
|
open += 1;
|
|
}
|
|
let length = open - start;
|
|
if marker == '~' && length != 2 {
|
|
return open;
|
|
}
|
|
if length > 3 {
|
|
return open;
|
|
}
|
|
if open == end || self.code[open].is_whitespace() {
|
|
return open;
|
|
}
|
|
if marker == '_' && start > 0 && is_word(self.code[start - 1]) {
|
|
return open;
|
|
}
|
|
let mut at = open;
|
|
while at < end {
|
|
if self.code[at] == '\\' {
|
|
at += 2;
|
|
continue;
|
|
}
|
|
if self.code[at] != marker {
|
|
at += 1;
|
|
continue;
|
|
}
|
|
let mut close = at;
|
|
while close < end && self.code[close] == marker {
|
|
close += 1;
|
|
}
|
|
let finish = at + length;
|
|
if close - at >= length
|
|
&& !self.code[at - 1].is_whitespace()
|
|
&& !(marker == '_' && finish < end && is_word(self.code[finish]))
|
|
{
|
|
self.emit(start, finish, Kind::Literal);
|
|
return finish;
|
|
}
|
|
at = close;
|
|
}
|
|
open
|
|
}
|
|
}
|
|
|
|
fn is_word(c: char) -> bool {
|
|
c.is_alphanumeric() || c == '_'
|
|
}
|
|
|
|
fn starts_with(code: &[char], at: usize, token: &str) -> bool {
|
|
let token: Vec<char> = token.chars().collect();
|
|
if at + token.len() > code.len() {
|
|
return false;
|
|
}
|
|
code[at..at + token.len()] == token[..]
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::super::{Kind, Language, span_text, spans_of};
|
|
|
|
fn spans(code: &str, kind: Kind) -> Vec<String> {
|
|
let chars: Vec<char> = code.chars().collect();
|
|
spans_of(code, Language::Markdown)
|
|
.into_iter()
|
|
.filter(|s| s.kind == kind)
|
|
.map(|s| span_text(&chars, &s))
|
|
.collect()
|
|
}
|
|
|
|
fn assert_spans(code: &str, kind: Kind, expected: &[&str]) {
|
|
assert_eq!(spans(code, kind), expected.to_vec(), "{kind:?} in: {code}");
|
|
}
|
|
|
|
#[test]
|
|
fn a_heading_is_coloured_whole_and_a_hash_inside_a_word_is_not_one() {
|
|
let code = "## Layout\nissue #12 is fixed\n#hashtag";
|
|
assert_spans(code, Kind::Keyword, &["## Layout"]);
|
|
}
|
|
|
|
#[test]
|
|
fn seven_hashes_are_not_a_heading() {
|
|
assert_spans("####### deep", Kind::Keyword, &[]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_fence_carries_its_language_as_metadata_and_its_body_as_one_string() {
|
|
let code = "text\n```kotlin\nval x = 1\n```\nmore";
|
|
assert_spans(code, Kind::Metadata, &["kotlin"]);
|
|
assert_spans(code, Kind::String, &["```", "val x = 1", "```"]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_longer_fence_is_not_closed_by_a_shorter_one_and_a_heading_inside_it_is_not_a_heading() {
|
|
let code = "````\n```\n# not a heading\n````\nafter";
|
|
assert_spans(code, Kind::Keyword, &[]);
|
|
assert_spans(
|
|
code,
|
|
Kind::String,
|
|
&["````", "```", "# not a heading", "````"],
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn an_unclosed_fence_runs_to_the_end_rather_than_panicking() {
|
|
assert_spans("```\nstill going", Kind::String, &["```", "still going"]);
|
|
}
|
|
|
|
#[test]
|
|
fn list_markers_and_quote_markers_colour_without_their_text() {
|
|
let code = "- one\n2. two\n> quoted";
|
|
assert_spans(code, Kind::Mark, &["-", "2.", ">"]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_rule_and_a_setext_underline_are_the_same_mark() {
|
|
assert_spans("Title\n=====\n\n---", Kind::Mark, &["=====", "---"]);
|
|
}
|
|
|
|
#[test]
|
|
fn emphasis_needs_something_on_both_sides_of_it() {
|
|
assert_spans(
|
|
"**bold** and *thin*",
|
|
Kind::Literal,
|
|
&["**bold**", "*thin*"],
|
|
);
|
|
assert_spans("a * b * c and *p = *q", Kind::Literal, &[]);
|
|
}
|
|
|
|
#[test]
|
|
fn an_underscore_inside_a_word_emphasises_nothing() {
|
|
assert_spans("snake_case_name and _real_", Kind::Literal, &["_real_"]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_code_span_holds_a_backtick_when_opened_with_two() {
|
|
assert_spans("``a ` b`` and `c`", Kind::String, &["``a ` b``", "`c`"]);
|
|
}
|
|
|
|
#[test]
|
|
fn an_unclosed_code_span_is_ordinary_text() {
|
|
assert_spans("a ` b", Kind::String, &[]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_link_marks_its_brackets_and_colours_its_destination() {
|
|
let code = "see [the plan](PLAN.md) now";
|
|
assert_spans(code, Kind::Mark, &["[", "]"]);
|
|
assert_spans(code, Kind::Metadata, &["(PLAN.md)"]);
|
|
}
|
|
|
|
#[test]
|
|
fn a_table_is_found_by_its_delimiter_row_and_pipes_elsewhere_are_plain() {
|
|
let code = "| a | b |\n|---|---|\n| 1 | 2 |\n\nrun a | b in a paragraph";
|
|
assert_spans(
|
|
code,
|
|
Kind::Mark,
|
|
&["|", "|", "|", "|---|---|", "|", "|", "|"],
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_table_without_outer_pipes_still_colours_and_the_table_ends_with_the_rows() {
|
|
let code = "a | b\n--- | ---\nnot a row";
|
|
assert_spans(code, Kind::Mark, &["|", "--- | ---"]);
|
|
}
|
|
|
|
#[test]
|
|
fn an_autolink_colours_and_an_html_tag_does_not() {
|
|
let code = "<https://example.com> and <a@b.com> and <div> and <img src=\"http://x\">";
|
|
assert_spans(
|
|
code,
|
|
Kind::Metadata,
|
|
&["<https://example.com>", "<a@b.com>", "http://x"],
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_bare_url_gives_back_the_sentences_punctuation() {
|
|
assert_spans(
|
|
"see https://example.com/a., and ssh://host/x)",
|
|
Kind::Metadata,
|
|
&["https://example.com/a", "ssh://host/x"],
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_bracket_a_url_opened_itself_stays_in_it() {
|
|
assert_spans(
|
|
"https://en.wikipedia.org/wiki/A_(b) here",
|
|
Kind::Metadata,
|
|
&["https://en.wikipedia.org/wiki/A_(b)"],
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_url_inside_a_link_destination_is_not_coloured_twice() {
|
|
assert_spans(
|
|
"[x](https://example.com)",
|
|
Kind::Metadata,
|
|
&["(https://example.com)"],
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_bracket_with_no_destination_after_it_is_left_plain() {
|
|
assert_spans("an [aside] here", Kind::Mark, &[]);
|
|
}
|
|
}
|