intercal/not-python-old.2020-08-27
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add more expressions to parser, add EOL and newline tokens

Browse Source 
* Lexer recognizes semicolons as EOL tokens, and newline tokens
* Parser can be configured to ignore newlines (which is also used
  internally)
* Newlines are allowed in lists, tuples, and parenthesized expressions
* Add a bunch of tests for the new stuff

Signed-off-by: Alek Ratzloff <alekratz@gmail.com>
This commit is contained in:
Alek Ratzloff
2020-05-05 16:38:07 -04:00
parent d9edf21d16
commit a4f289fb53
6 changed files with 274 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/syn/lexer.rs
View File

@@ -46,7 +46,7 @@ impl<'t> Lexer<'t> {
fn skip_whitespace(&mut self) {
while let Some(c) = self.curr_char() {
if !c.is_whitespace() {
if !c.is_whitespace() || c == '\n' {
break;
} else {
self.adv_char();
@@ -96,6 +96,8 @@ impl<'t> Lexer<'t> {
|(?P<lbrace>\{)
|(?P<rbrace>\})
|(?P<comma>,)
|(?P<eol>;)
|(?P<newline>\n)
"#).ignore_whitespace(true)
.build()
.unwrap();
@@ -133,6 +135,8 @@ impl<'t> Lexer<'t> {
("gt", TokenKind::Gt),
("eq", TokenKind::Eq),
("eol", TokenKind::Eol),
("newline", TokenKind::Newline),
];
self.skip_whitespace();
@@ -206,7 +210,7 @@ mod test {
assert!(matches!(lexer.next_token(), Ok(None)));
assert!(lexer.is_eof());
let mut lexer = Lexer::new(" \n \n \n\r\n\t ");
let mut lexer = Lexer::new(" \t \r \r\r\t\t ");
assert!(matches!(lexer.next_token(), Ok(None)));
assert!(lexer.is_eof());
}
@@ -263,6 +267,14 @@ mod test {
);
}
#[test]
fn test_eol() {
test_token!("\n;",
TokenKind::Newline, "\n",
TokenKind::Eol, ";"
);
}
#[test]
fn test_symbols() {
test_token!("(", TokenKind::LParen);