use crate::parser::cursor::LitTy; use super::{Span, Spanned}; use std::{iter::Peekable, str::CharIndices}; def_tokens! { symbol { Dot: ".", Comma: ",", Equal: "=", Colon: ":", Semicolon: ";", Plus: "+", Dash: "-", Asterisk: "*", Slash: "/", OpenParen: "(", CloseParen: ")", OpenSquare: "[", CloseSquare: "]", OpenCurly: "{", CloseCurly: "}", Arrow: "->", DoubleArrow: "=>", PlusEqual: "+=", DashEqual: "-=", AsteriskEqual: "*=", SlashEqual: "/=", Hash: "#", } keyword { Let: "let", Import: "import", Fn: "fn", If: "if", Loop: "loop", While: "while", For: "for", Match: "match", Break: "break", Asm: "asm", } other { Ident(String), Lit(LitTy), } } pub type TokenInst = Spanned; pub struct Tokens<'a> { file: usize, chars: Peekable>, } impl<'a> Tokens<'a> { pub fn new(code: &'a str, file: usize) -> Self { Self { file, chars: code.char_indices().peekable(), } } } impl Iterator for Tokens<'_> { type Item = Spanned; fn next(&mut self) -> Option { let (i, c) = self.chars.next()?; let mut span = Span { start: i, end: i, file: self.file, }; if c.is_whitespace() { return self.next(); } macro_rules! then { (_ => $def:expr, $($char:expr => $to:expr,)*) => { match self.chars.peek() { $(Some((_, $char)) => { self.chars.next(); $to },)* _ => $def, } }; } let inner = match c { '.' => Token::Dot, ',' => Token::Comma, '(' => Token::OpenParen, ')' => Token::CloseParen, '[' => Token::OpenSquare, ']' => Token::CloseSquare, '{' => Token::OpenCurly, '}' => Token::CloseCurly, '#' => Token::Hash, '+' => then! { _ => Token::Plus, '=' => Token::PlusEqual, }, '-' => then! { _ => Token::Dash, '=' => Token::DashEqual, '>' => Token::Arrow, }, '*' => then! { _ => Token::Asterisk, '=' => Token::AsteriskEqual, }, '/' => then! { _ => Token::Slash, '=' => Token::SlashEqual, }, ':' => Token::Colon, ';' => Token::Semicolon, '=' => then! { _ => Token::Equal, '>' => Token::DoubleArrow, }, '0'..='9' => { let mut s = c.to_string(); while let Some((i, c)) = self.chars.peek() && c.is_alphanumeric() { s.push(*c); span.end = *i; self.chars.next(); } LitTy::Number(s).into() } '"' => { let mut s = String::new(); while let Some((i, c)) = self.chars.next() && !matches!(c, '"') { s.push(c); span.end = i; } LitTy::String(s).into() } _ => { let mut s = c.to_string(); while let Some((i, c)) = self.chars.peek() && c.is_alphanumeric() { s.push(*c); span.end = *i; self.chars.next(); } match s.as_str() { "true" => LitTy::Bool(true).into(), "false" => LitTy::Bool(false).into(), _ => from_str(s), } } }; Some(Spanned { inner, span }) } } macro_rules! def_tokens { { symbol { $($sym_name:ident: $sym_str:expr,)* } keyword { $($kw_name:ident: $kw_str:expr,)* } other { $($other_name:ident($data:ty),)* } } => { #[derive(PartialEq)] pub enum Token { $($sym_name,)* $($kw_name,)* $($other_name($data),)* } fn from_str(s: String) -> Token { match s.as_str() { $($kw_str => Token::$kw_name,)* _ => Token::Ident(s), } } impl std::fmt::Display for Token { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { $(Token::$sym_name => write!(f, "{}", $sym_str),)* $(Token::$kw_name => write!(f, $kw_str),)* $(Token::$other_name(v) => write!(f, "{v}"),)* } } } }; } use def_tokens;