This commit is contained in:
2026-04-08 23:28:50 -04:00
parent edabc22431
commit bdf08ce52c
11 changed files with 417 additions and 158 deletions
+11 -5
View File
@@ -39,12 +39,14 @@ impl From<Lit> for Token {
pub type TokenInst = Spanned<Token>;
pub struct Tokens<'a> {
file: usize,
text: Peekable<CharIndices<'a>>,
}
impl<'a> Tokens<'a> {
pub fn new(code: &'a str) -> Self {
pub fn new(code: &'a str, file: usize) -> Self {
Self {
file,
text: code.char_indices().peekable(),
}
}
@@ -55,7 +57,11 @@ impl Iterator for Tokens<'_> {
fn next(&mut self) -> Option<Self::Item> {
let (i, c) = self.text.next()?;
let mut span = Span { first: i, last: i };
let mut span = Span {
start: i,
end: i,
file: self.file,
};
Some(Spanned {
inner: match c {
'=' => Token::Equal,
@@ -78,7 +84,7 @@ impl Iterator for Tokens<'_> {
&& c.is_alphanumeric()
{
s.push(*c);
span.last = *i;
span.end = *i;
self.text.next();
}
Lit::Number(s).into()
@@ -89,7 +95,7 @@ impl Iterator for Tokens<'_> {
&& !matches!(c, '"')
{
s.push(*c);
span.last = *i;
span.end = *i;
self.text.next();
}
self.text.next();
@@ -104,7 +110,7 @@ impl Iterator for Tokens<'_> {
)
{
s.push(*c);
span.last = *i;
span.end = *i;
self.text.next();
}
match s.as_str() {