mylang_lexer/transition/
symbol.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
use mylang_token::{Pos, Token};

use crate::state::symbol::SymbolState;

pub enum SymbolLexResult {
    Continued(SymbolState),
    Interrupted(Token),
}

pub fn symbol_lex(symbol_state: &SymbolState, (pos, c): (Pos, char)) -> SymbolLexResult {
    match c {
        c if c.is_ascii() && !c.is_ascii_whitespace() => {
            SymbolLexResult::Continued(symbol_state.append_char(pos, c))
        }

        _ => SymbolLexResult::Interrupted(symbol_state.tokenize()),
    }
}