intercal/not-python-old.2020-08-27
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add expression parsing

Browse Source 
Signed-off-by: Alek Ratzloff <alekratz@gmail.com>
This commit is contained in:
Alek Ratzloff
2020-05-02 18:42:01 -04:00
parent 28d29c2270
commit d9edf21d16
9 changed files with 542 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
src/syn/lexer.rs
View File

@@ -72,6 +72,23 @@ impl<'t> Lexer<'t> {
|(?P<sym>:[a-zA-Z_][a-zA-Z0-9_]*)
|(?P<hex_num>0[xX][0-9a-fA-F]+)
|(?P<dec_num>[0-9]+)
|(?P<dq_str>"([^\\"]|\\[ntr0"'])*")
|(?P<sq_str>'([^\\"]|\\[ntr0"'])*')
|(?P<arrow>->)
|(?P<eqeq>==)
|(?P<bangeq>!=)
|(?P<lteq><=)
|(?P<gteq>>=)
|(?P<lt><)
|(?P<gt>>)
|(?P<eq>=)
|(?P<plus>\+)
|(?P<minus>-)
|(?P<splat>\*)
|(?P<fslash>/)
|(?P<bang>!)
|(?P<lparen>\()
|(?P<rparen>\))
|(?P<lbracket>\[)
@@ -79,14 +96,6 @@ impl<'t> Lexer<'t> {
|(?P<lbrace>\{)
|(?P<rbrace>\})
|(?P<comma>,)
|(?P<arrow>->)
|(?P<eq>=)
|(?P<plus>\+)
|(?P<minus>-)
|(?P<splat>\*)
|(?P<fslash>/)
|(?P<dq_str>"([^\\"]|\\[ntr0"'])*")
|(?P<sq_str>'([^\\"]|\\[ntr0"'])*')
"#).ignore_whitespace(true)
.build()
.unwrap();
@@ -113,8 +122,16 @@ impl<'t> Lexer<'t> {
("minus", TokenKind::Minus),
("splat", TokenKind::Splat),
("fslash", TokenKind::FSlash),
("bang", TokenKind::Bang),
("arrow", TokenKind::Arrow),
("eqeq", TokenKind::EqEq),
("bangeq", TokenKind::BangEq),
("lteq", TokenKind::LtEq),
("gteq", TokenKind::GtEq),
("lt", TokenKind::Lt),
("gt", TokenKind::Gt),
("eq", TokenKind::Eq),
];
@@ -160,6 +177,25 @@ impl<'t> Lexer<'t> {
mod test {
use super::*;
macro_rules! test_token {
($text:expr, $($token_kind:expr, $token_text:expr),+ $(,)?) => {{
let text = $text;
let mut lexer = Lexer::new(text);
$(
let token = lexer.next_token().expect("token").expect("token");
assert_eq!(token.kind(), $token_kind);
assert_eq!(token.text_at(text), $token_text);
)+
assert!(lexer.is_eof());
}};
($text:expr, $token_kind:expr) => {{
test_token!($text, $token_kind, $text);
}};
}
#[test]
fn test_next_token_eof() {
let mut lexer = Lexer::new("");
@@ -175,27 +211,16 @@ mod test {
assert!(lexer.is_eof());
}
macro_rules! test_token {
($text:expr, $token_kind:expr, $token_text:expr) => {{
let text = $text;
let mut lexer = Lexer::new(text);
let token = lexer.next_token().expect("token").expect("token");
assert_eq!(token.kind(), $token_kind);
assert_eq!(token.text_at(text), $token_text);
}};
($text:expr, $token_kind:expr) => {{
test_token!($text, $token_kind, $text);
}};
}
#[test]
fn test_ident_token() {
test_token!("ident", TokenKind::Ident);
test_token!("OtherIdent", TokenKind::Ident);
test_token!("other_ident", TokenKind::Ident);
test_token!("ident1234", TokenKind::Ident);
test_token!("RETURN", TokenKind::Ident);
test_token!(
"ident OtherIdent other_ident ident1234 RETURN",
TokenKind::Ident, "ident",
TokenKind::Ident, "OtherIdent",
TokenKind::Ident, "other_ident",
TokenKind::Ident, "ident1234",
TokenKind::Ident, "RETURN",
);
}
#[test]
@@ -205,16 +230,19 @@ mod test {
#[test]
fn test_num_token() {
test_token!("1234", TokenKind::Num);
test_token!("4321", TokenKind::Num);
test_token!("123498765", TokenKind::Num);
test_token!("432156789", TokenKind::Num);
test_token!("0xdcbaBEEF", TokenKind::Num);
test_token!("0xabcdFEED", TokenKind::Num);
test_token!("0XdcbaBEEF", TokenKind::Num);
test_token!("0XabcdFEED", TokenKind::Num);
test_token!("0X123456789DCBAbeef", TokenKind::Num);
test_token!("0xABCDfeed192837465", TokenKind::Num);
test_token!(
"1234 4321 123498765 432156789 0xdcbaBEEF 0xabcdFEED 0XdcbaBEEF 0XabcdFEED 0X123456789DCBAbeef 0xABCDfeed192837465",
TokenKind::Num, "1234",
TokenKind::Num, "4321",
TokenKind::Num, "123498765",
TokenKind::Num, "432156789",
TokenKind::Num, "0xdcbaBEEF",
TokenKind::Num, "0xabcdFEED",
TokenKind::Num, "0XdcbaBEEF",
TokenKind::Num, "0XabcdFEED",
TokenKind::Num, "0X123456789DCBAbeef",
TokenKind::Num, "0xABCDfeed192837465",
);
}
#[test]
@@ -227,14 +255,16 @@ mod test {
#[test]
fn test_sym_token() {
test_token!(":symbol", TokenKind::Sym);
test_token!(":OtherSymbol", TokenKind::Sym);
test_token!(":other_symbol", TokenKind::Sym);
test_token!(":symbol1234", TokenKind::Sym);
test_token!(":symbol :OtherSymbol :other_symbol :symbol1234",
TokenKind::Sym, ":symbol",
TokenKind::Sym, ":OtherSymbol",
TokenKind::Sym, ":other_symbol",
TokenKind::Sym, ":symbol1234",
);
}
#[test]
fn test_single_char_symbols() {
fn test_symbols() {
test_token!("(", TokenKind::LParen);
test_token!(")", TokenKind::RParen);
test_token!("{", TokenKind::LBrace);
@@ -246,11 +276,15 @@ mod test {
test_token!("-", TokenKind::Minus);
test_token!("*", TokenKind::Splat);
test_token!("/", TokenKind::FSlash);
}
test_token!("!", TokenKind::Bang);
#[test]
fn test_op_tokens() {
test_token!("=", TokenKind::Eq);
test_token!("!=", TokenKind::BangEq);
test_token!("==", TokenKind::EqEq);
test_token!("<=", TokenKind::LtEq);
test_token!(">=", TokenKind::GtEq);
test_token!("<", TokenKind::Lt);
test_token!(">", TokenKind::Gt);
test_token!("->", TokenKind::Arrow);
}
}