diff --git a/examples/x11.zr b/examples/x11.zr deleted file mode 100644 index e25b506..0000000 --- a/examples/x11.zr +++ /dev/null @@ -1,35 +0,0 @@ -// needs to be compiled with -m -C="-lX11" -extern XOpenDisplay -extern XDefaultRootWindow -extern XCreateSimpleWindow -extern XMapWindow -extern XSelectInput -extern XNextEvent -extern XBlackPixel -extern XWhitePixel -extern XSetForeground -extern XCreateGC -extern XDefaultScreen -extern XDrawString - -func main[] : i64 - let dpy: ptr = XOpenDisplay(0) - let screen: ptr = XDefaultScreen(dpy) - - let white: ptr = XWhitePixel(dpy, screen) - let black: ptr = XBlackPixel(dpy, screen) - - let win: ptr = XCreateSimpleWindow(dpy, XDefaultRootWindow(dpy), 0, 0, 200, 100, 0, black, white) - - XSelectInput(dpy, win, 1 << 15) - XMapWindow(dpy, win) - - let gc: ptr = XCreateGC(dpy, win, 0, 0) - XSetForeground(dpy, gc, black) - - let ev: ptr = mem.alloc(256) - - while true - XNextEvent(dpy, ev) - if ev[0] == 12 - XDrawString(dpy, win, gc, 20, 50, "Hello, World!", 13) diff --git a/src/codegen_x86_64.rs b/src/codegen_x86_64.rs index d0d8937..8805d69 100644 --- a/src/codegen_x86_64.rs +++ b/src/codegen_x86_64.rs @@ -444,12 +444,7 @@ _builtin_environ: } TokenType::String => { // TODO: actual string parsing in the tokenizer - let value = &token.lexeme[1..token.lexeme.len() - 1] - .replace("\\n", "\n") - .replace("\\r", "\r") - .replace("\\t", "\t") - .replace("\\033", "\x1b") - .replace("\\0", "\0"); + let value = &token.lexeme[1..token.lexeme.len() - 1]; let label = format!("str_{:03}", self.data_counter); @@ -461,7 +456,7 @@ _builtin_environ: .map(|x| (x as u8).to_string()) .collect::>() .join(","); - emit!(&mut self.data_section, " {} db {},0", label, charcodes,); + emit!(&mut self.data_section, " {} db {},0", label, charcodes); } emit!(&mut self.output, " mov rax, {}", label); self.data_counter += 1; diff --git a/src/tokenizer.rs b/src/tokenizer.rs index b31a68f..6fe4698 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -149,26 +149,26 @@ impl Tokenizer { fn scan_token(&mut self) -> Result<(), ZernError> { match self.advance() { - '(' => self.add_token(TokenType::LeftParen), - ')' => self.add_token(TokenType::RightParen), - '[' => self.add_token(TokenType::LeftBracket), - ']' => self.add_token(TokenType::RightBracket), - '+' => self.add_token(TokenType::Plus), - '*' => self.add_token(TokenType::Star), - ',' => self.add_token(TokenType::Comma), - '%' => self.add_token(TokenType::Mod), - '^' => self.add_token(TokenType::Xor), - ':' => self.add_token(TokenType::Colon), + '(' => self.add_token(TokenType::LeftParen)?, + ')' => self.add_token(TokenType::RightParen)?, + '[' => self.add_token(TokenType::LeftBracket)?, + ']' => self.add_token(TokenType::RightBracket)?, + '+' => self.add_token(TokenType::Plus)?, + '*' => self.add_token(TokenType::Star)?, + ',' => self.add_token(TokenType::Comma)?, + '%' => self.add_token(TokenType::Mod)?, + '^' => self.add_token(TokenType::Xor)?, + ':' => self.add_token(TokenType::Colon)?, '-' => { if self.match_char('>') { - self.add_token(TokenType::Arrow) + self.add_token(TokenType::Arrow)? } else { - self.add_token(TokenType::Minus) + self.add_token(TokenType::Minus)? } } '.' => { if self.match_char('.') { - self.add_token(TokenType::DoubleDot) + self.add_token(TokenType::DoubleDot)? } else { return error!(self.loc, "expected '.' after '.'"); } @@ -179,67 +179,67 @@ impl Tokenizer { self.advance(); } } else { - self.add_token(TokenType::Slash) + self.add_token(TokenType::Slash)? } } '&' => { if self.match_char('&') { - self.add_token(TokenType::LogicalAnd) + self.add_token(TokenType::LogicalAnd)? } else { - self.add_token(TokenType::BitAnd) + self.add_token(TokenType::BitAnd)? } } '|' => { if self.match_char('>') { - self.add_token(TokenType::Pipe); + self.add_token(TokenType::Pipe)? } else if self.match_char('|') { - self.add_token(TokenType::LogicalOr); + self.add_token(TokenType::LogicalOr)? } else { - self.add_token(TokenType::BitOr); + self.add_token(TokenType::BitOr)? } } '!' => { if self.match_char('=') { - self.add_token(TokenType::NotEqual) + self.add_token(TokenType::NotEqual)? } else { - self.add_token(TokenType::Bang) + self.add_token(TokenType::Bang)? } } '=' => { if self.match_char('=') { - self.add_token(TokenType::DoubleEqual) + self.add_token(TokenType::DoubleEqual)? } else { - self.add_token(TokenType::Equal) + self.add_token(TokenType::Equal)? } } '>' => { if self.match_char('>') { - self.add_token(TokenType::ShiftRight); + self.add_token(TokenType::ShiftRight)? } else if self.match_char('=') { - self.add_token(TokenType::GreaterEqual) + self.add_token(TokenType::GreaterEqual)? } else { - self.add_token(TokenType::Greater) + self.add_token(TokenType::Greater)? } } '<' => { if self.match_char('<') { - self.add_token(TokenType::ShiftLeft); + self.add_token(TokenType::ShiftLeft)? } else if self.match_char('=') { - self.add_token(TokenType::LessEqual) + self.add_token(TokenType::LessEqual)? } else { - self.add_token(TokenType::Less) + self.add_token(TokenType::Less)? } } - // TODO: escape sequences '\'' => { if self.eof() { return error!(self.loc, "unterminated char literal"); } + _ = self.match_char('\\'); // if its an escape sequence skip \ and read one more self.advance(); if !self.match_char('\'') { return error!(self.loc, "expected ' after char literal"); } - self.add_token(TokenType::Char); + self.add_token(TokenType::Char)? } '"' => { while !self.eof() && self.peek() != '"' { @@ -255,7 +255,7 @@ impl Tokenizer { } self.advance(); - self.add_token(TokenType::String); + self.add_token(TokenType::String)? } ' ' | '\r' => {} '\n' => { @@ -263,8 +263,8 @@ impl Tokenizer { self.loc.column = 1; self.handle_indentation()?; } - '0'..='9' => self.scan_number(), - 'A'..='Z' | 'a'..='z' | '_' => self.scan_identifier(), + '0'..='9' => self.scan_number()?, + 'A'..='Z' | 'a'..='z' | '_' => self.scan_identifier()?, _ => return error!(self.loc, "unexpected character"), } Ok(()) @@ -318,7 +318,7 @@ impl Tokenizer { count } - fn scan_number(&mut self) { + fn scan_number(&mut self) -> Result<(), ZernError> { if self.match_char('x') { while self.peek().is_ascii_hexdigit() { self.advance(); @@ -333,10 +333,10 @@ impl Tokenizer { } } - self.add_token(TokenType::Number); + self.add_token(TokenType::Number) } - fn scan_identifier(&mut self) { + fn scan_identifier(&mut self) -> Result<(), ZernError> { while self.peek().is_alphanumeric() || self.peek() == '_' || self.peek() == '.' { self.advance(); } @@ -374,13 +374,47 @@ impl Tokenizer { } } - fn add_token(&mut self, token_type: TokenType) { - let lexeme: String = self.source[self.start..self.current].iter().collect(); + fn add_token(&mut self, token_type: TokenType) -> Result<(), ZernError> { + let mut lexeme: String = self.source[self.start..self.current].iter().collect(); + + if token_type == TokenType::Char || token_type == TokenType::String { + lexeme = self.unescape(&lexeme)?; + } + self.tokens.push(Token { token_type, lexeme, loc: self.loc.clone(), }); + Ok(()) + } + + fn unescape(&self, s: &str) -> Result { + let mut result = String::with_capacity(s.len()); + let mut chars = s.chars(); + + while let Some(c) = chars.next() { + if c != '\\' { + result.push(c); + continue; + } + match chars.next() { + Some('n') => result.push('\n'), + Some('r') => result.push('\r'), + Some('t') => result.push('\t'), + Some('\\') => result.push('\\'), + Some('\'') => result.push('\''), + Some('"') => result.push('"'), + Some(c) => { + return error!( + self.loc.clone(), + format!("unknown escape sequence: \\{}", c) + ); + } + None => return error!(self.loc.clone(), "unexpected end of escape sequence"), + } + } + Ok(result) } fn advance(&mut self) -> char {