client-core: port the ANSI parser, syntax highlighter and markdown scanner
Ports app/.../Ansi.kt, Highlighter.kt, Languages.kt and MarkdownSyntax.kt to client-core, module for module, with every HighlighterTest and AnsiTest case ported alongside (49 tests total). ansi.rs replaces Compose's AnnotatedString/SpanStyle with a plain StyledText/Style pair so the crate stays free of any UI framework, per RUST.md. cargo test (49 passed), clippy --all-targets and fmt clean. Co-Authored-By: Claude Sonnet <noreply@anthropic.com>
This commit is contained in:
irisandClaude Sonnet committed
1 parent
62dd6b7912
commit
762c1290a1
7 files changed
+3495
No files matched your search
@@ -0,0 +1,534 @@
|
||||
//! What a tool printed, with its terminal styling applied and everything
|
||||
//! else taken out. Ported from `app/.../Ansi.kt`, module for module: the
|
||||
//! Kotlin version builds a Compose `AnnotatedString`, which does not exist
|
||||
//! here, so a [`StyledText`] of plain text plus non-overlapping
|
||||
//! `(Range, Style)` spans stands in for it -- a future UI layer maps
|
||||
//! [`Style`] onto whatever it draws with.
|
||||
//!
|
||||
//! Bash output arrives exactly as the program wrote it, escape sequences
|
||||
//! included, and drawn verbatim those are line noise in the middle of the
|
||||
//! thing being read. Stripping them all would be the other half-answer --
|
||||
//! colour is often the whole of what a diff or a test run is saying.
|
||||
//!
|
||||
//! So the sequences that decide how text *looks* become spans, and every
|
||||
//! other one is dropped rather than shown: the rest move a cursor around a
|
||||
//! grid this is not, and "go to column 40" has no meaning in a scrolling
|
||||
//! document.
|
||||
//!
|
||||
//! A carriage return is honoured the way a terminal honours it: what was
|
||||
//! written since the last line break is thrown away and the line starts
|
||||
//! again. That is what makes a progress bar show its final state rather
|
||||
//! than every state it passed through.
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
/// An RGB colour, the same shape wherever this crate names one -- no alpha,
|
||||
/// because the one place that needs partial transparency (dimming) says so
|
||||
/// with a separate flag rather than baking it into the colour.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Rgb {
|
||||
pub r: u8,
|
||||
pub g: u8,
|
||||
pub b: u8,
|
||||
}
|
||||
|
||||
impl Rgb {
|
||||
pub const fn new(r: u8, g: u8, b: u8) -> Self {
|
||||
Self { r, g, b }
|
||||
}
|
||||
}
|
||||
|
||||
/// The sixteen colours a terminal program names, and the two it assumes.
|
||||
///
|
||||
/// Its own palette rather than the syntax one: a program that prints in red
|
||||
/// has chosen red, where a highlighter's colours are this app's reading of
|
||||
/// somebody else's code.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AnsiPalette {
|
||||
/// Indexes 0-7, then 8-15 bright, in the terminal's own order.
|
||||
pub colours: [Rgb; 16],
|
||||
/// What uncoloured text is, needed only where a style has to state a colour.
|
||||
pub foreground: Rgb,
|
||||
/// What the text sits on, needed for reverse video.
|
||||
pub background: Rgb,
|
||||
}
|
||||
|
||||
/// One span's worth of styling. `None` fields mean "unspecified", the same
|
||||
/// meaning `Color.Unspecified` and a null `FontWeight` carried in the Kotlin.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Default)]
|
||||
pub struct Style {
|
||||
pub color: Option<Rgb>,
|
||||
/// How much of `color`'s alpha survives, 0.0-1.0; `None` is opaque.
|
||||
pub alpha: Option<f32>,
|
||||
pub background: Option<Rgb>,
|
||||
pub bold: bool,
|
||||
pub italic: bool,
|
||||
pub underline: bool,
|
||||
pub strikethrough: bool,
|
||||
}
|
||||
|
||||
/// Plain text plus the non-overlapping, ordered spans that style parts of it
|
||||
/// -- this crate's stand-in for Compose's `AnnotatedString`.
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct StyledText {
|
||||
pub text: String,
|
||||
pub spans: Vec<(Range<usize>, Style)>,
|
||||
}
|
||||
|
||||
impl StyledText {
|
||||
fn plain(text: String) -> Self {
|
||||
Self {
|
||||
text,
|
||||
spans: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ESC: char = '\u{1B}';
|
||||
const BELL: char = '\u{7}';
|
||||
|
||||
/// [text] with its terminal styling applied and everything else taken out;
|
||||
/// see the module doc.
|
||||
pub fn ansi_styled(text: &str, palette: &AnsiPalette) -> StyledText {
|
||||
// The common case by a long way -- nothing to do, and nothing allocated
|
||||
// to find that out.
|
||||
if !text.contains(ESC) && !text.contains('\r') {
|
||||
return StyledText::plain(text.to_string());
|
||||
}
|
||||
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
let mut runs: Vec<(String, Option<Style>)> = Vec::new();
|
||||
let mut sgr = Sgr::PLAIN;
|
||||
let mut at = 0usize;
|
||||
let mut plain = String::new();
|
||||
|
||||
let flush = |plain: &mut String, sgr: Sgr, runs: &mut Vec<(String, Option<Style>)>| {
|
||||
if !plain.is_empty() {
|
||||
runs.push((std::mem::take(plain), sgr.span(palette)));
|
||||
}
|
||||
};
|
||||
|
||||
while at < chars.len() {
|
||||
let c = chars[at];
|
||||
if c == ESC {
|
||||
flush(&mut plain, sgr, &mut runs);
|
||||
at = skip_escape(&chars, at, |params, final_byte| {
|
||||
if final_byte == 'm' {
|
||||
sgr = sgr.apply(params, palette);
|
||||
}
|
||||
});
|
||||
} else if c == '\r' && chars.get(at + 1) != Some(&'\n') {
|
||||
// A bare carriage return rewrites the line. One before a newline
|
||||
// is the other half of a Windows line ending: it rewrites
|
||||
// nothing, and it is dropped rather than kept, since that pair
|
||||
// is one line break.
|
||||
flush(&mut plain, sgr, &mut runs);
|
||||
drop_line(&mut runs);
|
||||
at += 1;
|
||||
} else if c == '\r' {
|
||||
at += 1;
|
||||
} else if c >= ' ' || c == '\n' || c == '\t' {
|
||||
// Everything printable, plus the two control characters that are
|
||||
// layout rather than terminal commands. A stray bell or
|
||||
// backspace goes for the same reason a cursor move does.
|
||||
plain.push(c);
|
||||
at += 1;
|
||||
} else {
|
||||
at += 1;
|
||||
}
|
||||
}
|
||||
flush(&mut plain, sgr, &mut runs);
|
||||
|
||||
let mut out = String::new();
|
||||
let mut spans = Vec::new();
|
||||
for (run_text, style) in runs {
|
||||
let start = out.len();
|
||||
out.push_str(&run_text);
|
||||
if let Some(style) = style {
|
||||
spans.push((start..out.len(), style));
|
||||
}
|
||||
}
|
||||
StyledText { text: out, spans }
|
||||
}
|
||||
|
||||
/// Throws away everything written since the last line break, as a carriage
|
||||
/// return does.
|
||||
fn drop_line(runs: &mut Vec<(String, Option<Style>)>) {
|
||||
while let Some((text, style)) = runs.pop() {
|
||||
if let Some(break_at) = text.rfind('\n') {
|
||||
runs.push((text[..=break_at].to_string(), style));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The bytes that end a CSI sequence.
|
||||
fn is_csi_final(c: char) -> bool {
|
||||
('@'..='~').contains(&c)
|
||||
}
|
||||
|
||||
/// Steps over the escape sequence starting at `at`, reporting a CSI's
|
||||
/// parameters and final byte. One reader for every kind, because the point
|
||||
/// is to *leave* them all behind: a sequence this did not recognise would
|
||||
/// otherwise have its body printed as ordinary text. Three shapes -- the CSI
|
||||
/// (`ESC [ ... letter`), the string escapes which run to a terminator, and
|
||||
/// the two-character ones.
|
||||
fn skip_escape(chars: &[char], at: usize, mut on_csi: impl FnMut(&str, char)) -> usize {
|
||||
let Some(&next) = chars.get(at + 1) else {
|
||||
return at + 1;
|
||||
};
|
||||
match next {
|
||||
'[' => {
|
||||
let mut end = at + 2;
|
||||
while end < chars.len() && !is_csi_final(chars[end]) {
|
||||
end += 1;
|
||||
}
|
||||
if end >= chars.len() {
|
||||
// Cut off mid-sequence, which is what a stream that has not
|
||||
// finished arriving looks like: drop the fragment rather
|
||||
// than printing it, and the whole sequence arrives with the
|
||||
// next delta.
|
||||
chars.len()
|
||||
} else {
|
||||
let params: String = chars[at + 2..end].iter().collect();
|
||||
on_csi(¶ms, chars[end]);
|
||||
end + 1
|
||||
}
|
||||
}
|
||||
']' | 'P' | 'X' | '^' | '_' => {
|
||||
// Runs to a string terminator: `ESC \`, or the bell that xterm
|
||||
// allows after an OSC.
|
||||
let mut end = at + 2;
|
||||
while end < chars.len() {
|
||||
if chars[end] == BELL {
|
||||
return end + 1;
|
||||
}
|
||||
if chars[end] == ESC && chars.get(end + 1) == Some(&'\\') {
|
||||
return end + 2;
|
||||
}
|
||||
end += 1;
|
||||
}
|
||||
chars.len()
|
||||
}
|
||||
_ => at + 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything an SGR sequence can turn on, as the terminal tracks it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
struct Sgr {
|
||||
fg: Option<Rgb>,
|
||||
bg: Option<Rgb>,
|
||||
bold: bool,
|
||||
dim: bool,
|
||||
italic: bool,
|
||||
underline: bool,
|
||||
strike: bool,
|
||||
reverse: bool,
|
||||
}
|
||||
|
||||
/// How much of its colour dim text keeps: enough to read, little enough to recede.
|
||||
const DIM_ALPHA: f32 = 0.65;
|
||||
|
||||
impl Sgr {
|
||||
const PLAIN: Sgr = Sgr {
|
||||
fg: None,
|
||||
bg: None,
|
||||
bold: false,
|
||||
dim: false,
|
||||
italic: false,
|
||||
underline: false,
|
||||
strike: false,
|
||||
reverse: false,
|
||||
};
|
||||
|
||||
/// `None` while nothing is set, so unstyled output costs no spans at all.
|
||||
fn span(&self, palette: &AnsiPalette) -> Option<Style> {
|
||||
if *self == Sgr::PLAIN {
|
||||
return None;
|
||||
}
|
||||
let front = if self.reverse {
|
||||
Some(self.bg.unwrap_or(palette.background))
|
||||
} else {
|
||||
self.fg
|
||||
};
|
||||
let back = if self.reverse {
|
||||
Some(self.fg.unwrap_or(palette.foreground))
|
||||
} else {
|
||||
self.bg
|
||||
};
|
||||
// Dim has to have a colour to dim, so where none was named it dims
|
||||
// the ordinary one.
|
||||
let stated = front.or(if self.dim {
|
||||
Some(palette.foreground)
|
||||
} else {
|
||||
None
|
||||
});
|
||||
Some(Style {
|
||||
color: stated,
|
||||
alpha: if self.dim { Some(DIM_ALPHA) } else { None },
|
||||
background: back,
|
||||
bold: self.bold,
|
||||
italic: self.italic,
|
||||
underline: self.underline,
|
||||
strikethrough: self.strike,
|
||||
})
|
||||
}
|
||||
|
||||
/// This state with `params` applied -- one `ESC[...m`, which carries any
|
||||
/// number of them.
|
||||
///
|
||||
/// A code this does not model is ignored rather than reset from: the
|
||||
/// program meant something by it, and starting again would also drop
|
||||
/// the codes beside it that are understood.
|
||||
fn apply(&self, params: &str, palette: &AnsiPalette) -> Sgr {
|
||||
// `ESC[m` means `ESC[0m`, and an empty parameter inside a list is a
|
||||
// zero too.
|
||||
let codes: Vec<i64> = params
|
||||
.split(';')
|
||||
.map(|p| p.trim().parse::<i64>().unwrap_or(0))
|
||||
.collect();
|
||||
let mut state = *self;
|
||||
let mut at = 0usize;
|
||||
while at < codes.len() {
|
||||
let code = codes[at];
|
||||
state = match code {
|
||||
0 => Sgr::PLAIN,
|
||||
1 => Sgr {
|
||||
bold: true,
|
||||
..state
|
||||
},
|
||||
2 => Sgr { dim: true, ..state },
|
||||
3 => Sgr {
|
||||
italic: true,
|
||||
..state
|
||||
},
|
||||
4 => Sgr {
|
||||
underline: true,
|
||||
..state
|
||||
},
|
||||
7 => Sgr {
|
||||
reverse: true,
|
||||
..state
|
||||
},
|
||||
9 => Sgr {
|
||||
strike: true,
|
||||
..state
|
||||
},
|
||||
21 | 22 => Sgr {
|
||||
bold: false,
|
||||
dim: false,
|
||||
..state
|
||||
},
|
||||
23 => Sgr {
|
||||
italic: false,
|
||||
..state
|
||||
},
|
||||
24 => Sgr {
|
||||
underline: false,
|
||||
..state
|
||||
},
|
||||
27 => Sgr {
|
||||
reverse: false,
|
||||
..state
|
||||
},
|
||||
29 => Sgr {
|
||||
strike: false,
|
||||
..state
|
||||
},
|
||||
30..=37 => Sgr {
|
||||
fg: Some(palette.colours[(code - 30) as usize]),
|
||||
..state
|
||||
},
|
||||
90..=97 => Sgr {
|
||||
fg: Some(palette.colours[(code - 90 + 8) as usize]),
|
||||
..state
|
||||
},
|
||||
40..=47 => Sgr {
|
||||
bg: Some(palette.colours[(code - 40) as usize]),
|
||||
..state
|
||||
},
|
||||
100..=107 => Sgr {
|
||||
bg: Some(palette.colours[(code - 100 + 8) as usize]),
|
||||
..state
|
||||
},
|
||||
39 => Sgr { fg: None, ..state },
|
||||
49 => Sgr { bg: None, ..state },
|
||||
38 | 48 => {
|
||||
let (colour, last) = extended_colour(&codes, at, palette);
|
||||
at = last;
|
||||
if code == 38 {
|
||||
Sgr {
|
||||
fg: colour,
|
||||
..state
|
||||
}
|
||||
} else {
|
||||
Sgr {
|
||||
bg: colour,
|
||||
..state
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => state,
|
||||
};
|
||||
at += 1;
|
||||
}
|
||||
state
|
||||
}
|
||||
}
|
||||
|
||||
/// The colour named by a `38`/`48` at `at`, and the index of that colour's
|
||||
/// last parameter.
|
||||
///
|
||||
/// Two forms: `5;n` for the 256-colour table and `2;r;g;b` for a literal
|
||||
/// one. The first sixteen of that table are the palette's own, so a program
|
||||
/// asking for "colour 1" through either spelling gets the same red.
|
||||
fn extended_colour(codes: &[i64], at: usize, palette: &AnsiPalette) -> (Option<Rgb>, usize) {
|
||||
match codes.get(at + 1) {
|
||||
Some(&5) => match codes.get(at + 2) {
|
||||
None => (None, at + 1),
|
||||
Some(&n) => (Some(indexed_colour(n, palette)), at + 2),
|
||||
},
|
||||
Some(&2) => {
|
||||
let r = codes.get(at + 2);
|
||||
let g = codes.get(at + 3);
|
||||
let b = codes.get(at + 4);
|
||||
match (r, g, b) {
|
||||
(Some(&r), Some(&g), Some(&b)) => (
|
||||
Some(Rgb::new(
|
||||
r.clamp(0, 255) as u8,
|
||||
g.clamp(0, 255) as u8,
|
||||
b.clamp(0, 255) as u8,
|
||||
)),
|
||||
at + 4,
|
||||
),
|
||||
_ => (None, at + 1),
|
||||
}
|
||||
}
|
||||
_ => (None, at + 1),
|
||||
}
|
||||
}
|
||||
|
||||
/// The six levels of each channel in the 256-colour cube, as xterm defines them.
|
||||
const CUBE: [u8; 6] = [0, 95, 135, 175, 215, 255];
|
||||
|
||||
/// One of the 256 colours: the palette's sixteen, then a 6x6x6 cube, then a
|
||||
/// grey ramp.
|
||||
fn indexed_colour(n: i64, palette: &AnsiPalette) -> Rgb {
|
||||
if n < 0 {
|
||||
palette.foreground
|
||||
} else if n < 16 {
|
||||
palette.colours[n as usize]
|
||||
} else if n < 232 {
|
||||
let i = (n - 16) as usize;
|
||||
Rgb::new(CUBE[i / 36], CUBE[i / 6 % 6], CUBE[i % 6])
|
||||
} else if n < 256 {
|
||||
let grey = (8 + (n - 232) * 10) as u8;
|
||||
Rgb::new(grey, grey, grey)
|
||||
} else {
|
||||
palette.foreground
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A palette matching the Kotlin test's: `colours[i] = Rgb(i, 0, 0)`,
|
||||
/// white foreground, black background.
|
||||
fn palette() -> AnsiPalette {
|
||||
let mut colours = [Rgb::new(0, 0, 0); 16];
|
||||
for (i, c) in colours.iter_mut().enumerate() {
|
||||
*c = Rgb::new(i as u8, 0, 0);
|
||||
}
|
||||
AnsiPalette {
|
||||
colours,
|
||||
foreground: Rgb::new(255, 255, 255),
|
||||
background: Rgb::new(0, 0, 0),
|
||||
}
|
||||
}
|
||||
|
||||
fn styled(text: &str) -> StyledText {
|
||||
ansi_styled(text, &palette())
|
||||
}
|
||||
|
||||
/// The style covering the first character of `word`, or `None` where
|
||||
/// nothing styles it.
|
||||
fn style_over(text: &str, word: &str) -> Option<Style> {
|
||||
let out = styled(text);
|
||||
let at = out
|
||||
.text
|
||||
.find(word)
|
||||
.unwrap_or_else(|| panic!("no {word:?} in {}", out.text));
|
||||
out.spans
|
||||
.iter()
|
||||
.find(|(range, _)| range.contains(&at))
|
||||
.map(|(_, style)| *style)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_colour_becomes_a_span_and_the_sequence_itself_disappears() {
|
||||
let text = format!("plain {ESC}[31mred{ESC}[0m plain");
|
||||
assert_eq!(styled(&text).text, "plain red plain");
|
||||
assert_eq!(
|
||||
style_over(&text, "red").unwrap().color,
|
||||
Some(Rgb::new(1, 0, 0))
|
||||
);
|
||||
assert!(style_over(&text, "plain").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bright_background_and_256_colour_forms_all_reach_the_same_table() {
|
||||
assert_eq!(
|
||||
style_over(&format!("{ESC}[91mx"), "x").unwrap().color,
|
||||
Some(Rgb::new(9, 0, 0))
|
||||
);
|
||||
assert_eq!(
|
||||
style_over(&format!("{ESC}[44mx"), "x").unwrap().background,
|
||||
Some(Rgb::new(4, 0, 0))
|
||||
);
|
||||
assert_eq!(
|
||||
style_over(&format!("{ESC}[38;5;1mx"), "x").unwrap().color,
|
||||
Some(Rgb::new(1, 0, 0))
|
||||
);
|
||||
assert_eq!(
|
||||
style_over(&format!("{ESC}[38;5;16mx"), "x").unwrap().color,
|
||||
Some(Rgb::new(0, 0, 0))
|
||||
);
|
||||
assert_eq!(
|
||||
style_over(&format!("{ESC}[38;5;231mx"), "x").unwrap().color,
|
||||
Some(Rgb::new(255, 255, 255))
|
||||
);
|
||||
assert_eq!(
|
||||
style_over(&format!("{ESC}[38;2;10;20;30mx"), "x")
|
||||
.unwrap()
|
||||
.color,
|
||||
Some(Rgb::new(10, 20, 30))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn everything_that_is_not_styling_is_dropped_rather_than_printed() {
|
||||
// A cursor move, an erase, an OSC window title with its bell, and a
|
||||
// bare two-character escape.
|
||||
let text = format!("a{ESC}[2Jb{ESC}[Kc{ESC}]0;a title{BELL}d{ESC}=e");
|
||||
assert_eq!(styled(&text).text, "abcde");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_carriage_return_rewrites_its_line_as_it_does_on_a_terminal() {
|
||||
assert_eq!(styled("10%\r50%\rdone\n").text, "done\n");
|
||||
assert_eq!(styled("kept\r\nfirst\rlast").text, "kept\nlast");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sequence_cut_off_mid_stream_takes_no_text_with_it() {
|
||||
assert_eq!(styled(&format!("text {ESC}[3")).text, "text ");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unstyled_text_costs_no_spans_at_all() {
|
||||
assert_eq!(styled("nothing to do here").spans.len(), 0);
|
||||
assert_eq!(styled(&format!("a{ESC}[2Jb")).spans.len(), 0);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user