use std::fmt::Display;
use crate::parser::ErrorAt;
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum Token {
OpenBracket,
ClosingBracket,
Word(String),
String(String),
Number(f64),
Comment(String),
Assign,
Colon,
}
pub(crate) type TokenAt = (Token, (usize, usize));
impl Display for Token {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Token::OpenBracket => write!(f, "opening bracket"),
Token::ClosingBracket => write!(f, "closing bracket"),
Token::Word(w) => write!(f, "word \"{w}\""),
Token::String(s) => write!(f, "string \"{s}\""),
Token::Number(n) => write!(f, "number {n}"),
Token::Assign => write!(f, "assignment operator"),
Token::Colon => write!(f, "colon"),
Token::Comment(_) => write!(f, "comment"),
}
}
}
/// Tokenize a block of characters.
pub(crate) fn tokenize(code: &str) -> Result<Vec<TokenAt>, ErrorAt> {
let mut chars = Vec::new();
let mut line = 1;
let mut char_num = 0;
for char in code.chars().peekable() {
char_num += 1;
if char == '\n' {
line += 1;
char_num = 0;
};
chars.push((char, (line, char_num)))
}
let mut chars = chars.iter().peekable();
let mut tokens = Vec::new();
while let Some((char, at)) = chars.next() {
match char {
char if char == &'"' => {
let mut string = "".to_string();
let last_pos = at;
// TODO: Correct ranges.
loop {
match chars.next() {
Some((c, _)) if c == &'\\' => match chars.next() {
Some(('n', _)) => string.push('\n'),
Some((escaped, _)) => string.push(*escaped),
None => todo!(),
},
Some((char, _)) if char == &'"' => {
break;
}
None => {
return Err(("Unfinished string at file end".to_owned(), *last_pos))
}
Some((char, _)) => string.push(*char),
}
}
tokens.push((Token::String(string), (0, 0)));
}
char if char == &'(' => tokens.push((Token::OpenBracket, *at)),
char if char == &')' => tokens.push((Token::ClosingBracket, *at)),
char if char == &':' => tokens.push((Token::Colon, *at)),
char if char == &'#' => {
let mut comment = "".to_string();
loop {
let Some((char, _)) = chars.next() else {
break
};
if char == &'\n' {
break;
}
comment.push(*char);
}
let len = comment.len();
tokens.push((Token::Comment(comment), (at.0, at.1 + len)));
}
char if char == &'=' => {
tokens.push((Token::Assign, *at));
}
char if char.is_whitespace() => (),
char => {
let mut word = char.to_string();
while chars.peek().map(|(c, _)| c) != Some(&':')
&& chars.peek().map(|(c, _)| c) != Some(&'(')
&& chars.peek().map(|(c, _)| c) != Some(&')')
&& chars.peek().map(|(c, _)| c) != Some(&'=')
&& chars.peek().map(|(c, _)| c) != Some(&'\n')
&& chars.peek().map(|(c, _)| c) != Some(&'#')
&& chars.peek().map(|(c, _)| c) != Some(&' ')
{
let Some((next, _)) = chars.next() else {
break
};
word.push(*next);
}
if let Some(string) = word.strip_prefix('\'') {
tokens.push((Token::String(string.to_string().replace("\\n", "\n")), *at));
} else if word.find('.').is_some() {
tokens.push((Token::String(word), *at));
} else if let Ok(num) = word.parse::<f64>() {
tokens.push((Token::Number(num), *at));
} else {
tokens.push((Token::Word(word), *at));
}
}
}
}
Ok(tokens)
}