parser3

src/parser/cursor/token.rs

@@ -0,0 +1,161 @@
+use super::{Span, Spanned};
+use std::{iter::Peekable, str::CharIndices};
+
+#[derive(PartialEq)]
+pub enum Token {
+    // symbols
+    Equal,
+    Colon,
+    Semicolon,
+    OpenCurly,
+    CloseCurly,
+    OpenParen,
+    CloseParen,
+    Dash,
+    Plus,
+    Asterisk,
+    Slash,
+    // keywords
+    Let,
+    Fn,
+    // other
+    Ident(String),
+    Lit(Lit),
+}
+
+#[derive(PartialEq)]
+pub enum Lit {
+    Number(String),
+    Bool(bool),
+    String(String),
+}
+
+impl From<Lit> for Token {
+    fn from(value: Lit) -> Self {
+        Self::Lit(value)
+    }
+}
+
+pub type TokenInst = Spanned<Token>;
+
+pub struct Tokens<'a> {
+    text: Peekable<CharIndices<'a>>,
+}
+
+impl<'a> Tokens<'a> {
+    pub fn new(code: &'a str) -> Self {
+        Self {
+            text: code.char_indices().peekable(),
+        }
+    }
+}
+
+impl Iterator for Tokens<'_> {
+    type Item = Spanned<Token>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let (i, c) = self.text.next()?;
+        let mut span = Span { first: i, last: i };
+        Some(Spanned {
+            inner: match c {
+                '=' => Token::Equal,
+                '{' => Token::OpenCurly,
+                '}' => Token::CloseCurly,
+                '(' => Token::OpenParen,
+                ')' => Token::CloseParen,
+                ':' => Token::Colon,
+                ';' => Token::Semicolon,
+                '-' => Token::Dash,
+                '+' => Token::Plus,
+                '*' => Token::Asterisk,
+                '/' => Token::Slash,
+                '\n' | ' ' => {
+                    return self.next();
+                }
+                '0'..='9' => {
+                    let mut s = c.to_string();
+                    while let Some((i, c)) = self.text.peek()
+                        && c.is_alphanumeric()
+                    {
+                        s.push(*c);
+                        span.last = *i;
+                        self.text.next();
+                    }
+                    Lit::Number(s).into()
+                }
+                '"' => {
+                    let mut s = c.to_string();
+                    while let Some((i, c)) = self.text.peek()
+                        && !matches!(c, '"')
+                    {
+                        s.push(*c);
+                        span.last = *i;
+                        self.text.next();
+                    }
+                    self.text.next();
+                    Lit::String(s).into()
+                }
+                _ => {
+                    let mut s = c.to_string();
+                    while let Some((i, c)) = self.text.peek()
+                        && !matches!(
+                            c,
+                            '=' | '{' | '}' | '(' | ')' | ':' | ';' | '\n' | '"' | ' '
+                        )
+                    {
+                        s.push(*c);
+                        span.last = *i;
+                        self.text.next();
+                    }
+                    match s.as_str() {
+                        "let" => Token::Let,
+                        "fn" => Token::Fn,
+                        "true" => Lit::Bool(true).into(),
+                        "false" => Lit::Bool(false).into(),
+                        _ => Token::Ident(s),
+                    }
+                }
+            },
+            span,
+        })
+    }
+}
+
+impl std::fmt::Display for Token {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Token::Equal => write!(f, "="),
+            Token::Colon => write!(f, ":"),
+            Token::Plus => write!(f, "+"),
+            Token::Dash => write!(f, "-"),
+            Token::Asterisk => write!(f, "*"),
+            Token::Slash => write!(f, "/"),
+            Token::Semicolon => write!(f, ";"),
+            Token::OpenCurly => write!(f, "{{"),
+            Token::CloseCurly => write!(f, "}}"),
+            Token::OpenParen => write!(f, "("),
+            Token::CloseParen => write!(f, ")"),
+            Token::Let => write!(f, "let"),
+            Token::Fn => write!(f, "fn"),
+            Token::Ident(s) => write!(f, "{s}"),
+            Token::Lit(lit) => write!(f, "{lit}"),
+        }?;
+        Ok(())
+    }
+}
+
+impl std::fmt::Display for Lit {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Lit::Number(n) => write!(f, "{n}"),
+            Lit::Bool(b) => write!(f, "{b}"),
+            Lit::String(s) => write!(f, "{s}"),
+        }
+    }
+}
+
+impl std::fmt::Debug for Token {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "'{self}'")
+    }
+}