rewrite zern tokenizer in zern :)

This commit is contained in:
2025-12-25 21:04:34 +01:00
parent 07b46a987b
commit 270386da95
3 changed files with 273 additions and 10 deletions

View File

@@ -314,16 +314,6 @@ impl Tokenizer {
while self.peek().is_ascii_digit() {
self.advance();
}
if self.peek() == '.'
&& self.current + 1 < self.source.len()
&& self.source[self.current + 1].is_ascii_digit()
{
self.advance();
while self.peek().is_ascii_digit() {
self.advance();
}
}
}
self.add_token(TokenType::Number);