lang/src/parser/cursor/token.rs

use crate::parser::cursor::LitTy;

use super::{Span, Spanned};
use std::{iter::Peekable, str::CharIndices};

def_tokens! {
    symbol {
        Dot: ".",
        Comma: ",",
        Equal: "=",
        Colon: ":",
        Semicolon: ";",
        Plus: "+",
        Dash: "-",
        Asterisk: "*",
        Slash: "/",
        OpenParen: "(",
        CloseParen: ")",
        OpenSquare: "[",
        CloseSquare: "]",
        OpenCurly: "{",
        CloseCurly: "}",
        Arrow: "->",
        DoubleArrow: "=>",
        PlusEqual: "+=",
        DashEqual: "-=",
        AsteriskEqual: "*=",
        SlashEqual: "/=",
        Hash: "#",
    }
    keyword {
        Let: "let",
        Import: "import",
        Fn: "fn",
        If: "if",
        Loop: "loop",
        While: "while",
        For: "for",
        Match: "match",
        Break: "break",
        Asm: "asm",
    }
    other {
        Ident(String),
        Lit(LitTy),
    }
}

pub type TokenInst = Spanned<Token>;

pub struct Tokens<'a> {
    file: usize,
    chars: Peekable<CharIndices<'a>>,
}

impl<'a> Tokens<'a> {
    pub fn new(code: &'a str, file: usize) -> Self {
        Self {
            file,
            chars: code.char_indices().peekable(),
        }
    }
}

impl Iterator for Tokens<'_> {
    type Item = Spanned<Token>;

    fn next(&mut self) -> Option<Self::Item> {
        let (i, c) = self.chars.next()?;
        let mut span = Span {
            start: i,
            end: i,
            file: self.file,
        };
        if c.is_whitespace() {
            return self.next();
        }
        macro_rules! then {
            (_ => $def:expr, $($char:expr => $to:expr,)*) => {
                match self.chars.peek() {
                    $(Some((_, $char)) => {
                        self.chars.next();
                        $to
                    },)*
                    _ => $def,
                }
            };
        }
        let inner = match c {
            '.' => Token::Dot,
            ',' => Token::Comma,
            '(' => Token::OpenParen,
            ')' => Token::CloseParen,
            '[' => Token::OpenSquare,
            ']' => Token::CloseSquare,
            '{' => Token::OpenCurly,
            '}' => Token::CloseCurly,
            '#' => Token::Hash,
            '+' => then! {
                _ => Token::Plus,
                '=' => Token::PlusEqual,
            },
            '-' => then! {
                _ => Token::Dash,
                '=' => Token::DashEqual,
                '>' => Token::Arrow,
            },
            '*' => then! {
                _ => Token::Asterisk,
                '=' => Token::AsteriskEqual,
            },
            '/' => then! {
                _ => Token::Slash,
                '=' => Token::SlashEqual,
            },
            ':' => Token::Colon,
            ';' => Token::Semicolon,
            '=' => then! {
                _ => Token::Equal,
                '>' => Token::DoubleArrow,
            },
            '0'..='9' => {
                let mut s = c.to_string();
                while let Some((i, c)) = self.chars.peek()
                    && c.is_alphanumeric()
                {
                    s.push(*c);
                    span.end = *i;
                    self.chars.next();
                }
                LitTy::Number(s).into()
            }
            '"' => {
                let mut s = String::new();
                while let Some((i, c)) = self.chars.next()
                    && !matches!(c, '"')
                {
                    s.push(c);
                    span.end = i;
                }
                LitTy::String(s).into()
            }
            _ => {
                let mut s = c.to_string();
                while let Some((i, c)) = self.chars.peek()
                    && c.is_alphanumeric()
                {
                    s.push(*c);
                    span.end = *i;
                    self.chars.next();
                }
                match s.as_str() {
                    "true" => LitTy::Bool(true).into(),
                    "false" => LitTy::Bool(false).into(),
                    _ => from_str(s),
                }
            }
        };
        Some(Spanned { inner, span })
    }
}

macro_rules! def_tokens {
    {
        symbol {
            $($sym_name:ident: $sym_str:expr,)*
        }
        keyword {
            $($kw_name:ident: $kw_str:expr,)*
        }
        other {
            $($other_name:ident($data:ty),)*
        }
    } => {
        #[derive(PartialEq)]
        pub enum Token {
            $($sym_name,)*
            $($kw_name,)*
            $($other_name($data),)*
        }
        fn from_str(s: String) -> Token {
            match s.as_str() {
                $($kw_str => Token::$kw_name,)*
                _ => Token::Ident(s),
            }
        }
        impl std::fmt::Display for Token {
            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                match self {
                    $(Token::$sym_name => write!(f, "{}", $sym_str),)*
                    $(Token::$kw_name => write!(f, $kw_str),)*
                    $(Token::$other_name(v) => write!(f, "{v}"),)*
                }
            }
        }
    };
}
use def_tokens;