rewrite zern tokenizer in zern :)

This commit is contained in:
2025-12-25 21:04:34 +01:00
parent 07b46a987b
commit 270386da95
3 changed files with 273 additions and 10 deletions

View File

@@ -424,6 +424,14 @@ func array.free[xs: array] : void
mem.free(data)
mem.free(xs)
func array.pop[xs: array] : i64
let size: i64 = array.size(xs)
if size == 0
dbg.panic("array.pop on empty array")
let x: i64 = array.nth(xs, size - 1)
mem.write64(xs + 16, size - 1)
return x
func array.slice[xs: array, start: i64, length: i64] : array
if start < 0 | length < 0 | start + length > array.size(xs)
dbg.panic("array.slice out of bounds")

View File

@@ -314,16 +314,6 @@ impl Tokenizer {
while self.peek().is_ascii_digit() {
self.advance();
}
if self.peek() == '.'
&& self.current + 1 < self.source.len()
&& self.source[self.current + 1].is_ascii_digit()
{
self.advance();
while self.peek().is_ascii_digit() {
self.advance();
}
}
}
self.add_token(TokenType::Number);