iris/lang
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

work

Browse Source
This commit is contained in:
iris
2026-04-10 16:13:45 -04:00
parent bdf08ce52c
commit 29316e6353
16 changed files with 520 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/parser/cursor/token.rs
+128 -134
View File
@@ -1,38 +1,31 @@
use super::{Span, Spanned};
use super::{Lit, Span, Spanned};
use std::{iter::Peekable, str::CharIndices};
#[derive(PartialEq)]
pub enum Token {
// symbols
Equal,
Colon,
Semicolon,
OpenCurly,
CloseCurly,
OpenParen,
CloseParen,
Dash,
Plus,
Asterisk,
Slash,
// keywords
Let,
Fn,
// other
Ident(String),
Lit(Lit),
}
#[derive(PartialEq)]
pub enum Lit {
Number(String),
Bool(bool),
String(String),
}
impl From<Lit> for Token {
fn from(value: Lit) -> Self {
Self::Lit(value)
def_tokens! {
symbol {
'=' => Equal,
':' => Colon,
';' => Semicolon,
'{' => OpenCurly,
'}' => CloseCurly,
'(' => OpenParen,
')' => CloseParen,
'-' => Dash,
'+' => Plus,
'*' => Asterisk,
'/' => Slash,
}
keyword {
Let: "let",
Fn: "fn",
If: "if",
Loop: "loop",
While: "while",
For: "for",
}
other {
Ident(String),
Lit(Lit),
}
}
@@ -40,14 +33,14 @@ pub type TokenInst = Spanned<Token>;
pub struct Tokens<'a> {
file: usize,
text: Peekable<CharIndices<'a>>,
chars: Peekable<CharIndices<'a>>,
}
impl<'a> Tokens<'a> {
pub fn new(code: &'a str, file: usize) -> Self {
Self {
file,
text: code.char_indices().peekable(),
chars: code.char_indices().peekable(),
}
}
}
@@ -56,112 +49,113 @@ impl Iterator for Tokens<'_> {
type Item = Spanned<Token>;
fn next(&mut self) -> Option<Self::Item> {
let (i, c) = self.text.next()?;
let (i, c) = self.chars.next()?;
let mut span = Span {
start: i,
end: i,
file: self.file,
};
Some(Spanned {
inner: match c {
'=' => Token::Equal,
'{' => Token::OpenCurly,
'}' => Token::CloseCurly,
'(' => Token::OpenParen,
')' => Token::CloseParen,
':' => Token::Colon,
';' => Token::Semicolon,
'-' => Token::Dash,
'+' => Token::Plus,
'*' => Token::Asterisk,
'/' => Token::Slash,
'\n' | ' ' => {
return self.next();
}
'0'..='9' => {
let mut s = c.to_string();
while let Some((i, c)) = self.text.peek()
&& c.is_alphanumeric()
{
s.push(*c);
span.end = *i;
self.text.next();
}
Lit::Number(s).into()
}
'"' => {
let mut s = c.to_string();
while let Some((i, c)) = self.text.peek()
&& !matches!(c, '"')
{
s.push(*c);
span.end = *i;
self.text.next();
}
self.text.next();
Lit::String(s).into()
}
_ => {
let mut s = c.to_string();
while let Some((i, c)) = self.text.peek()
&& !matches!(
c,
'=' | '{' | '}' | '(' | ')' | ':' | ';' | '\n' | '"' | ' '
)
{
s.push(*c);
span.end = *i;
self.text.next();
}
match s.as_str() {
"let" => Token::Let,
"fn" => Token::Fn,
"true" => Lit::Bool(true).into(),
"false" => Lit::Bool(false).into(),
_ => Token::Ident(s),
}
}
},
span,
})
}
}
impl std::fmt::Display for Token {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Token::Equal => write!(f, "="),
Token::Colon => write!(f, ":"),
Token::Plus => write!(f, "+"),
Token::Dash => write!(f, "-"),
Token::Asterisk => write!(f, "*"),
Token::Slash => write!(f, "/"),
Token::Semicolon => write!(f, ";"),
Token::OpenCurly => write!(f, "{{"),
Token::CloseCurly => write!(f, "}}"),
Token::OpenParen => write!(f, "("),
Token::CloseParen => write!(f, ")"),
Token::Let => write!(f, "let"),
Token::Fn => write!(f, "fn"),
Token::Ident(s) => write!(f, "{s}"),
Token::Lit(lit) => write!(f, "{lit}"),
}?;
Ok(())
}
}
impl std::fmt::Display for Lit {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Lit::Number(n) => write!(f, "{n}"),
Lit::Bool(b) => write!(f, "{b}"),
Lit::String(s) => write!(f, "{s}"),
if let Some(inner) = from_char(c) {
return Some(Spanned { inner, span });
}
if c.is_whitespace() {
return self.next();
}
let inner = match c {
'0'..='9' => {
let mut s = c.to_string();
while let Some((i, c)) = self.chars.peek()
&& c.is_alphanumeric()
{
s.push(*c);
span.end = *i;
self.chars.next();
}
Lit::Number(s).into()
}
'"' => {
let mut s = c.to_string();
while let Some((i, c)) = self.chars.peek()
&& !matches!(c, '"')
{
s.push(*c);
span.end = *i;
self.chars.next();
}
self.chars.next();
Lit::String(s).into()
}
_ => {
let mut s = c.to_string();
while let Some((i, c)) = self.chars.peek()
&& c.is_alphanumeric()
{
s.push(*c);
span.end = *i;
self.chars.next();
}
match s.as_str() {
"true" => Lit::Bool(true).into(),
"false" => Lit::Bool(false).into(),
_ => from_str(s),
}
}
};
Some(Spanned { inner, span })
}
}
impl std::fmt::Debug for Token {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "'{self}'")
}
macro_rules! expand_sym_names {
({
$($sym_char:expr => $sym_name:ident,)*
}) => {
$($sym_name,)*
};
({
$($sym_char:expr => $sym_res:tt,)*
}) => {
expand_sym_names!($sym_res)
};
}
use expand_sym_names;
macro_rules! def_tokens {
{
symbol $syms:tt
keyword {
$($kw_name:ident: $kw_str:expr,)*
}
other {
$($other_name:ident($data:ty),)*
}
} => {
#[derive(PartialEq)]
pub enum Token {
$($syms,)*
$($kw_name,)*
$($other_name($data),)*
}
fn from_char(c: char) -> Option<Token> {
match c {
$($sym_char => Some(Token::$sym_res),)*
_ => None,
}
}
fn from_str(s: String) -> Token {
match s.as_str() {
$($kw_str => Token::$kw_name,)*
_ => Token::Ident(s),
}
}
impl std::fmt::Display for Token {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
$(Token::$sym_res => write!(f, "{}", $sym_char),)*
$(Token::$kw_name => write!(f, $kw_str),)*
$(Token::$other_name(v) => write!(f, "{v}"),)*
}
}
}
};
}
use def_tokens;