toni/zern
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move escape sequence parsing to tokenizer

Browse Source
This commit is contained in:
Antoni Piasecki
2026-03-11 18:30:31 +01:00
parent 01b98a1d45
commit a07e2dc8dd
3 changed files with 75 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
examples/x11.zr
View File

@@ -1,35 +0,0 @@
// needs to be compiled with -m -C="-lX11"
extern XOpenDisplay
extern XDefaultRootWindow
extern XCreateSimpleWindow
extern XMapWindow
extern XSelectInput
extern XNextEvent
extern XBlackPixel
extern XWhitePixel
extern XSetForeground
extern XCreateGC
extern XDefaultScreen
extern XDrawString
func main[] : i64
let dpy: ptr = XOpenDisplay(0)
let screen: ptr = XDefaultScreen(dpy)
let white: ptr = XWhitePixel(dpy, screen)
let black: ptr = XBlackPixel(dpy, screen)
let win: ptr = XCreateSimpleWindow(dpy, XDefaultRootWindow(dpy), 0, 0, 200, 100, 0, black, white)
XSelectInput(dpy, win, 1 << 15)
XMapWindow(dpy, win)
let gc: ptr = XCreateGC(dpy, win, 0, 0)
XSetForeground(dpy, gc, black)
let ev: ptr = mem.alloc(256)
while true
XNextEvent(dpy, ev)
if ev[0] == 12
XDrawString(dpy, win, gc, 20, 50, "Hello, World!", 13)

9
src/codegen_x86_64.rs
View File

@@ -444,12 +444,7 @@ _builtin_environ:
} }
TokenType::String => { TokenType::String => {
// TODO: actual string parsing in the tokenizer // TODO: actual string parsing in the tokenizer
let value = &token.lexeme[1..token.lexeme.len() - 1] let value = &token.lexeme[1..token.lexeme.len() - 1];
.replace("\\n", "\n")
.replace("\\r", "\r")
.replace("\\t", "\t")
.replace("\\033", "\x1b")
.replace("\\0", "\0");
let label = format!("str_{:03}", self.data_counter); let label = format!("str_{:03}", self.data_counter);
@@ -461,7 +456,7 @@ _builtin_environ:
.map(|x| (x as u8).to_string()) .map(|x| (x as u8).to_string())
.collect::<Vec<String>>() .collect::<Vec<String>>()
.join(","); .join(",");
emit!(&mut self.data_section, " {} db {},0", label, charcodes,); emit!(&mut self.data_section, " {} db {},0", label, charcodes);
} }
emit!(&mut self.output, " mov rax, {}", label); emit!(&mut self.output, " mov rax, {}", label);
self.data_counter += 1; self.data_counter += 1;

112
src/tokenizer.rs
View File

@@ -149,26 +149,26 @@ impl Tokenizer {
fn scan_token(&mut self) -> Result<(), ZernError> { fn scan_token(&mut self) -> Result<(), ZernError> {
match self.advance() { match self.advance() {
'(' => self.add_token(TokenType::LeftParen), '(' => self.add_token(TokenType::LeftParen)?,
')' => self.add_token(TokenType::RightParen), ')' => self.add_token(TokenType::RightParen)?,
'[' => self.add_token(TokenType::LeftBracket), '[' => self.add_token(TokenType::LeftBracket)?,
']' => self.add_token(TokenType::RightBracket), ']' => self.add_token(TokenType::RightBracket)?,
'+' => self.add_token(TokenType::Plus), '+' => self.add_token(TokenType::Plus)?,
'*' => self.add_token(TokenType::Star), '*' => self.add_token(TokenType::Star)?,
',' => self.add_token(TokenType::Comma), ',' => self.add_token(TokenType::Comma)?,
'%' => self.add_token(TokenType::Mod), '%' => self.add_token(TokenType::Mod)?,
'^' => self.add_token(TokenType::Xor), '^' => self.add_token(TokenType::Xor)?,
':' => self.add_token(TokenType::Colon), ':' => self.add_token(TokenType::Colon)?,
'-' => { '-' => {
if self.match_char('>') { if self.match_char('>') {
self.add_token(TokenType::Arrow) self.add_token(TokenType::Arrow)?
} else { } else {
self.add_token(TokenType::Minus) self.add_token(TokenType::Minus)?
} }
} }
'.' => { '.' => {
if self.match_char('.') { if self.match_char('.') {
self.add_token(TokenType::DoubleDot) self.add_token(TokenType::DoubleDot)?
} else { } else {
return error!(self.loc, "expected '.' after '.'"); return error!(self.loc, "expected '.' after '.'");
} }
@@ -179,67 +179,67 @@ impl Tokenizer {
self.advance(); self.advance();
} }
} else { } else {
self.add_token(TokenType::Slash) self.add_token(TokenType::Slash)?
} }
} }
'&' => { '&' => {
if self.match_char('&') { if self.match_char('&') {
self.add_token(TokenType::LogicalAnd) self.add_token(TokenType::LogicalAnd)?
} else { } else {
self.add_token(TokenType::BitAnd) self.add_token(TokenType::BitAnd)?
} }
} }
'|' => { '|' => {
if self.match_char('>') { if self.match_char('>') {
self.add_token(TokenType::Pipe); self.add_token(TokenType::Pipe)?
} else if self.match_char('|') { } else if self.match_char('|') {
self.add_token(TokenType::LogicalOr); self.add_token(TokenType::LogicalOr)?
} else { } else {
self.add_token(TokenType::BitOr); self.add_token(TokenType::BitOr)?
} }
} }
'!' => { '!' => {
if self.match_char('=') { if self.match_char('=') {
self.add_token(TokenType::NotEqual) self.add_token(TokenType::NotEqual)?
} else { } else {
self.add_token(TokenType::Bang) self.add_token(TokenType::Bang)?
} }
} }
'=' => { '=' => {
if self.match_char('=') { if self.match_char('=') {
self.add_token(TokenType::DoubleEqual) self.add_token(TokenType::DoubleEqual)?
} else { } else {
self.add_token(TokenType::Equal) self.add_token(TokenType::Equal)?
} }
} }
'>' => { '>' => {
if self.match_char('>') { if self.match_char('>') {
self.add_token(TokenType::ShiftRight); self.add_token(TokenType::ShiftRight)?
} else if self.match_char('=') { } else if self.match_char('=') {
self.add_token(TokenType::GreaterEqual) self.add_token(TokenType::GreaterEqual)?
} else { } else {
self.add_token(TokenType::Greater) self.add_token(TokenType::Greater)?
} }
} }
'<' => { '<' => {
if self.match_char('<') { if self.match_char('<') {
self.add_token(TokenType::ShiftLeft); self.add_token(TokenType::ShiftLeft)?
} else if self.match_char('=') { } else if self.match_char('=') {
self.add_token(TokenType::LessEqual) self.add_token(TokenType::LessEqual)?
} else { } else {
self.add_token(TokenType::Less) self.add_token(TokenType::Less)?
} }
} }
// TODO: escape sequences
'\'' => { '\'' => {
if self.eof() { if self.eof() {
return error!(self.loc, "unterminated char literal"); return error!(self.loc, "unterminated char literal");
} }
_ = self.match_char('\\'); // if its an escape sequence skip \ and read one more
self.advance(); self.advance();
if !self.match_char('\'') { if !self.match_char('\'') {
return error!(self.loc, "expected ' after char literal"); return error!(self.loc, "expected ' after char literal");
} }
self.add_token(TokenType::Char); self.add_token(TokenType::Char)?
} }
'"' => { '"' => {
while !self.eof() && self.peek() != '"' { while !self.eof() && self.peek() != '"' {
@@ -255,7 +255,7 @@ impl Tokenizer {
} }
self.advance(); self.advance();
self.add_token(TokenType::String); self.add_token(TokenType::String)?
} }
' ' | '\r' => {} ' ' | '\r' => {}
'\n' => { '\n' => {
@@ -263,8 +263,8 @@ impl Tokenizer {
self.loc.column = 1; self.loc.column = 1;
self.handle_indentation()?; self.handle_indentation()?;
} }
'0'..='9' => self.scan_number(), '0'..='9' => self.scan_number()?,
'A'..='Z' | 'a'..='z' | '_' => self.scan_identifier(), 'A'..='Z' | 'a'..='z' | '_' => self.scan_identifier()?,
_ => return error!(self.loc, "unexpected character"), _ => return error!(self.loc, "unexpected character"),
} }
Ok(()) Ok(())
@@ -318,7 +318,7 @@ impl Tokenizer {
count count
} }
fn scan_number(&mut self) { fn scan_number(&mut self) -> Result<(), ZernError> {
if self.match_char('x') { if self.match_char('x') {
while self.peek().is_ascii_hexdigit() { while self.peek().is_ascii_hexdigit() {
self.advance(); self.advance();
@@ -333,10 +333,10 @@ impl Tokenizer {
} }
} }
self.add_token(TokenType::Number); self.add_token(TokenType::Number)
} }
fn scan_identifier(&mut self) { fn scan_identifier(&mut self) -> Result<(), ZernError> {
while self.peek().is_alphanumeric() || self.peek() == '_' || self.peek() == '.' { while self.peek().is_alphanumeric() || self.peek() == '_' || self.peek() == '.' {
self.advance(); self.advance();
} }
@@ -374,13 +374,47 @@ impl Tokenizer {
} }
} }
fn add_token(&mut self, token_type: TokenType) { fn add_token(&mut self, token_type: TokenType) -> Result<(), ZernError> {
let lexeme: String = self.source[self.start..self.current].iter().collect(); let mut lexeme: String = self.source[self.start..self.current].iter().collect();
if token_type == TokenType::Char || token_type == TokenType::String {
lexeme = self.unescape(&lexeme)?;
}
self.tokens.push(Token { self.tokens.push(Token {
token_type, token_type,
lexeme, lexeme,
loc: self.loc.clone(), loc: self.loc.clone(),
}); });
Ok(())
}
fn unescape(&self, s: &str) -> Result<String, ZernError> {
let mut result = String::with_capacity(s.len());
let mut chars = s.chars();
while let Some(c) = chars.next() {
if c != '\\' {
result.push(c);
continue;
}
match chars.next() {
Some('n') => result.push('\n'),
Some('r') => result.push('\r'),
Some('t') => result.push('\t'),
Some('\\') => result.push('\\'),
Some('\'') => result.push('\''),
Some('"') => result.push('"'),
Some(c) => {
return error!(
self.loc.clone(),
format!("unknown escape sequence: \\{}", c)
);
}
None => return error!(self.loc.clone(), "unexpected end of escape sequence"),
}
}
Ok(result)
} }
fn advance(&mut self) -> char { fn advance(&mut self) -> char {