diff --git a/examples/selfhost/tokenizer.zr b/examples/selfhost/tokenizer.zr new file mode 100644 index 0000000..cb76baf --- /dev/null +++ b/examples/selfhost/tokenizer.zr @@ -0,0 +1,408 @@ +const TT_LEFT_PAREN = 1 +const TT_RIGHT_PAREN = 2 +const TT_LEFT_BRACKET = 3 +const TT_RIGHT_BRACKET = 4 +const TT_COMMA = 5 +const TT_PLUS = 6 +const TT_PLUS_EQUAL = 7 +const TT_MINUS = 8 +const TT_MINUS_EQUAL = 9 +const TT_STAR = 10 +const TT_SLASH = 11 +const TT_MOD = 12 +const TT_XOR = 13 +const TT_BANG = 14 +const TT_COLON = 15 +const TT_BIT_AND = 16 +const TT_BIT_OR = 17 +const TT_LOGICAL_AND = 18 +const TT_LOGICAL_OR = 19 +const TT_DOUBLE_DOT = 20 +const TT_SHIFT_LEFT = 21 +const TT_SHIFT_RIGHT = 22 +const TT_ARROW = 23 +const TT_TILDE = 24 + +const TT_EQUAL = 25 +const TT_DOUBLE_EQUAL = 26 +const TT_NOT_EQUAL = 27 +const TT_GREATER = 28 +const TT_GREATER_EQUAL = 29 +const TT_LESS = 30 +const TT_LESS_EQUAL = 31 + +const TT_IDENTIFIER = 32 +const TT_STRING_LITERAL = 33 +const TT_CHAR_LITERAL = 34 +const TT_INT_LITERAL = 35 +const TT_TRUE = 36 +const TT_FALSE = 37 + +const TT_KW_CONST = 38 +const TT_KW_IF = 39 +const TT_KW_ELSE = 40 +const TT_KW_WHILE = 41 +const TT_KW_FOR = 42 +const TT_KW_IN = 43 +const TT_KW_FUNC = 44 +const TT_KW_RETURN = 45 +const TT_KW_BREAK = 46 +const TT_KW_CONTINUE = 47 +const TT_KW_EXTERN = 48 +const TT_KW_EXPORT = 49 +const TT_KW_STRUCT = 50 +const TT_KW_NEW = 51 +const TT_KW_AS = 52 + +const TT_INDENT = 53 +const TT_DEDENT = 54 +const TT_EOF = 55 + +struct Token + token_type: i64 + lexeme: str + loc_filename: str + loc_line: i64 + loc_column: i64 + +struct Tokenizer + source: str + source_len: i64 + tokens: Array + indent_stack: Array + current_indent: i64 + start: i64 + current: i64 + loc_filename: str + loc_line: i64 + loc_column: i64 + +func Tokenizer.init[t: Tokenizer, filename: str, source: str] : void + t->source = source + t->source_len = str.len(source) + t->tokens = [] + t->indent_stack = [0] + t->loc_filename = filename + t->loc_line = 1 + t->loc_column = 1 + +func Tokenizer.tokenize[t: Tokenizer] : Array + while !t->eof() + t->start = t->current + t->scan_token() + + t->add_token(TT_EOF, "") + return t->tokens + +func Tokenizer.scan_token[t: Tokenizer] : void + c := t->advance() + if c == '(' + t->add_token(TT_LEFT_PAREN, "(") + else if c == ')' + t->add_token(TT_RIGHT_PAREN, ")") + else if c == '[' + t->add_token(TT_LEFT_BRACKET, "[") + else if c == ']' + t->add_token(TT_RIGHT_BRACKET, "]") + else if c == '+' + if t->match_char('=') + t->add_token(TT_PLUS_EQUAL, "+=") + else + t->add_token(TT_PLUS, "+") + else if c == '*' + t->add_token(TT_STAR, "*") + else if c == ',' + t->add_token(TT_COMMA, ",") + else if c == '%' + t->add_token(TT_MOD, "%") + else if c == '^' + t->add_token(TT_XOR, "^") + else if c == ':' + t->add_token(TT_COLON, ":") + else if c == '~' + t->add_token(TT_TILDE, "~") + else if c == '-' + if t->match_char('=') + t->add_token(TT_MINUS_EQUAL, "-=") + else if t->match_char('>') + t->add_token(TT_ARROW, "->") + else + t->add_token(TT_MINUS, "-") + else if c == '.' + if t->match_char('.') + t->add_token(TT_DOUBLE_DOT, "..") + else + t->error("expected '.' after '.'") + else if c == '/' + if t->match_char('/') + while !t->eof() && t->peek() != '\n' + t->advance() + else + t->add_token(TT_SLASH, "/") + else if c == '&' + if t->match_char('&') + t->add_token(TT_LOGICAL_AND, "&&") + else + t->add_token(TT_BIT_AND, "&") + else if c == '|' + if t->match_char('|') + t->add_token(TT_LOGICAL_OR, "||") + else + t->add_token(TT_BIT_OR, "|") + else if c == '!' + if t->match_char('=') + t->add_token(TT_NOT_EQUAL, "!=") + else + t->add_token(TT_BANG, "!") + else if c == '=' + if t->match_char('=') + t->add_token(TT_DOUBLE_EQUAL, "==") + else + t->add_token(TT_EQUAL, "=") + else if c == '>' + if t->match_char('>') + t->add_token(TT_SHIFT_RIGHT, ">>") + else if t->match_char('=') + t->add_token(TT_GREATER_EQUAL, ">=") + else + t->add_token(TT_GREATER, ">") + else if c == '<' + if t->match_char('<') + t->add_token(TT_SHIFT_LEFT, "<<") + else if t->match_char('=') + t->add_token(TT_LESS_EQUAL, "<=") + else + t->add_token(TT_LESS, "<") + else if c == '\'' + if t->eof() + t->error("unterminated char literal") + + t->match_char('\\') + t->advance() + + if !t->match_char('\'') + t->error("expected ' after char literal") + + lexeme := t->source->substr(t->start, t->current - t->start) + t->add_token(TT_CHAR_LITERAL, lexeme) + else if c == '"' + start_filename := t->loc_filename + start_line := t->loc_line + start_column := t->loc_column + + while !t->eof() + if t->peek() == '\\' + t->advance() + else if t->peek() == '"' + break + else if t->peek() == '\n' + t->loc_line += 1 + t->loc_column = 0 + + t->advance() + + if t->eof() + t->loc_filename = start_filename + t->loc_line = start_line + t->loc_column = start_column + t->error("unterminated string") + + t->advance() + lexeme := t->source->substr(t->start, t->current - t->start) + t->add_token(TT_STRING_LITERAL, lexeme) + else if c == ' ' || c == '\r' + // ignore + else if c == '\n' + t->loc_line += 1 + t->loc_column = 1 + t->handle_indentation() + else if c >= '0' && c <= '9' + t->scan_number() + else if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' + t->scan_identifier() + else + t->error("unexpected character") + +func Tokenizer.handle_indentation[t: Tokenizer] : void + if t->peek() == '\n' + return 0 + + new_indent := t->count_indentation() + + if new_indent > t->current_indent + t->indent_stack->push(new_indent) + t->add_token(TT_INDENT, "") + else if new_indent < t->current_indent + while t->indent_stack->size > 0 && t->indent_stack->last() > new_indent + t->indent_stack->pop() + t->add_token(TT_DEDENT, "") + if t->indent_stack->size == 0 || t->indent_stack->last() != new_indent + t->error("invalid indentation") + + t->current_indent = new_indent + +func Tokenizer.count_indentation[t: Tokenizer] : i64 + count := 0 + while t->peek() == ' ' + count += 1 + t->advance() + return count + +func Tokenizer.scan_number[t: Tokenizer] : void + if t->match_char('x') + while t->peek()->is_hex_digit() + t->advance() + else if t->match_char('o') + while '0' <= t->peek() && t->peek() <= '7' + t->advance() + else + while t->peek()->is_digit() + t->advance() + + lexeme := t->source->substr(t->start, t->current - t->start) + t->add_token(TT_INT_LITERAL, lexeme) + +func str.match_at[s: str, start: i64, keyword: str] : bool + i := 0 + while keyword[i] != 0 + if s[start + i] != keyword[i] + return false + i += 1 + return true + +func Tokenizer.scan_identifier[t: Tokenizer] : void + while t->peek()->is_alphanumeric() || t->peek() == '_' || t->peek() == '.' + t->advance() + + len := t->current - t->start + + if len == 5 && t->source->match_at(t->start, "const") + t->add_token(TT_KW_CONST, "const") + else if len == 2 && t->source->match_at(t->start, "if") + t->add_token(TT_KW_IF, "if") + else if len == 4 && t->source->match_at(t->start, "else") + t->add_token(TT_KW_ELSE, "else") + else if len == 5 && t->source->match_at(t->start, "while") + t->add_token(TT_KW_WHILE, "while") + else if len == 3 && t->source->match_at(t->start, "for") + t->add_token(TT_KW_FOR, "for") + else if len == 2 && t->source->match_at(t->start, "in") + t->add_token(TT_KW_IN, "in") + else if len == 4 && t->source->match_at(t->start, "func") + t->add_token(TT_KW_FUNC, "func") + else if len == 6 && t->source->match_at(t->start, "return") + t->add_token(TT_KW_RETURN, "return") + else if len == 5 && t->source->match_at(t->start, "break") + t->add_token(TT_KW_BREAK, "break") + else if len == 8 && t->source->match_at(t->start, "continue") + t->add_token(TT_KW_CONTINUE, "continue") + else if len == 6 && t->source->match_at(t->start, "extern") + t->add_token(TT_KW_EXTERN, "extern") + else if len == 6 && t->source->match_at(t->start, "export") + t->add_token(TT_KW_EXPORT, "export") + else if len == 6 && t->source->match_at(t->start, "struct") + t->add_token(TT_KW_STRUCT, "struct") + else if len == 3 && t->source->match_at(t->start, "new") + t->add_token(TT_KW_NEW, "new") + else if len == 2 && t->source->match_at(t->start, "as") + t->add_token(TT_KW_AS, "as") + else if len == 4 && t->source->match_at(t->start, "true") + t->add_token(TT_TRUE, "true") + else if len == 5 && t->source->match_at(t->start, "false") + t->add_token(TT_FALSE, "false") + else + lexeme := t->source->substr(t->start, len) + t->add_token(TT_IDENTIFIER, lexeme) + +func Tokenizer.match_char[t: Tokenizer, expected: u8] : bool + if t->eof() || t->peek() != expected + return false + t->current += 1 + t->loc_column += 1 + return true + +func Tokenizer.add_token[t: Tokenizer, token_type: i64, lexeme: str] : void + if token_type == TT_STRING_LITERAL || token_type == TT_CHAR_LITERAL + lexeme = t->unescape(lexeme) + + token := new* Token + token->token_type = token_type + token->lexeme = lexeme + token->loc_filename = t->loc_filename + token->loc_line = t->loc_line + token->loc_column = t->loc_column + t->tokens->push(token) + +func Tokenizer.unescape[t: Tokenizer, s: str] : str + b := new str.Builder + i := 0 + len := s->len() + while i < len + if s[i] == '\\' + i += 1 + if i >= len + t->error("unexpected end of escape sequence") + + if s[i] == 'n' + b->append_char('\n') + else if s[i] == 'r' + b->append_char('\r') + else if s[i] == 't' + b->append_char('\t') + else if s[i] == '0' + b->append_char('\0') + else if s[i] == '\\' + b->append_char('\\') + else if s[i] == '\'' + b->append_char('\'') + else if s[i] == '"' + b->append_char('"') + else + t->error("unknown escape sequence") + else + b->append_char(s[i]) + i += 1 + + result := b->build() + b->destroy() + return result + +func Tokenizer.advance[t: Tokenizer] : u8 + c := t->source[t->current] + t->current += 1 + t->loc_column += 1 + return c + +func Tokenizer.peek[t: Tokenizer] : u8 + if t->eof() + return '\0' + return t->source[t->current] + +func Tokenizer.eof[t: Tokenizer] : bool + return t->current >= t->source_len + +func Tokenizer.error[t: Tokenizer, msg: str] : void + io.print(t->loc_filename) + io.print(":") + io.print_i64(t->loc_line) + io.print(":") + io.print_i64(t->loc_column) + io.print(" ERROR: ") + io.println(msg) + os.exit(1) + +func main[argc: i64, argv: ptr] : i64 + path := mem.read64(argv + 8) as str + ~source, ok := io.read_text_file(path) + if !ok + panic("file not found") + + filename := os.basename(path) + + tokenizer := new Tokenizer + tokenizer->init(filename, source) + + tokens := tokenizer->tokenize() + for i in 0..tokens->size + t := tokens->nth(i) as Token + io.printf("Token(token_type=%d, lexeme=\"%s\", loc=\"%s:%d:%d\")\n", t->token_type, t->lexeme, t->loc_filename, t->loc_line, t->loc_column) diff --git a/src/std/std.zr b/src/std/std.zr index 286b8a7..78aee04 100644 --- a/src/std/std.zr +++ b/src/std/std.zr @@ -849,7 +849,7 @@ func Array.new_preallocated_and_zeroed[size: i64] : Array func Array.nth[xs: Array, n: i64] : any if n < 0 || n >= xs->size - panic("array.nth out of bounds") + panic("Array.nth out of bounds") return mem.read64(xs->data + n * 8) func Array.set[xs: Array, n: i64, x: any] : void @@ -882,14 +882,19 @@ func Array.free_with_items[xs: Array] : void func Array.pop[xs: Array] : any if xs->size == 0 - panic("array.pop on empty array") - x : any = xs->nth(xs->size - 1) + panic("Array.pop on empty array") + x : any = Array.last(xs) xs->size = xs->size - 1 return x +func Array.last[xs: Array] : any + if xs->size == 0 + panic("Array.last on empty array") + return xs->nth(xs->size - 1) + func Array.slice[xs: Array, start: i64, length: i64] : Array if start < 0 || length < 0 || start + length > xs->size - panic("array.slice out of bounds") + panic("Array.slice out of bounds") new_array := Array.new_preallocated_and_zeroed(length) mem.copy(xs->data + start * 8, new_array->data, length * 8)