intercal/not-python-old.2020-08-27
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add more expressions to parser, add EOL and newline tokens

Browse Source 
* Lexer recognizes semicolons as EOL tokens, and newline tokens
* Parser can be configured to ignore newlines (which is also used
  internally)
* Newlines are allowed in lists, tuples, and parenthesized expressions
* Add a bunch of tests for the new stuff

Signed-off-by: Alek Ratzloff <alekratz@gmail.com>
This commit is contained in:
Alek Ratzloff
2020-05-05 16:38:07 -04:00
parent d9edf21d16
commit a4f289fb53
6 changed files with 274 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

249
src/syn/parser.rs
View File

@@ -37,6 +37,28 @@ impl<'t> Parser<'t> {
pub fn pos(&self) -> Pos {
self.span().start
}
pub fn set_skip_newlines(&mut self, skip: bool) -> Result<bool> {
let prev = mem::replace(&mut self.skip_newlines, skip);
match self.skip_newlines() {
Ok(()) => Ok(prev),
Err(e) => {
self.skip_newlines = prev;
Err(e)
}
}
}
pub fn is_skip_newlines(&self) -> bool {
self.skip_newlines
}
fn skip_newlines(&mut self) -> Result<()> {
if self.is_skip_newlines() && self.is_token_kind(TokenKind::Newline) {
self.adv_token()?;
}
Ok(())
}
}
////////////////////////////////////////////////////////////////////////////////
@@ -61,14 +83,28 @@ macro_rules! bin_expr {
impl<'t> Parser<'t> {
pub fn next_expr(&mut self) -> Result<Expr> {
self.next_bin_add_expr()
self.next_bin_cmp_expr()
}
// == < > <= >=
bin_expr!(
next_bin_cmp_expr,
&[
TokenKind::EqEq,
TokenKind::Lt,
TokenKind::Gt,
TokenKind::LtEq,
TokenKind::GtEq
],
next_bin_add_expr
);
// + -
bin_expr!(
next_bin_add_expr,
&[TokenKind::Plus, TokenKind::Minus],
next_bin_mul_expr
);
// * /
bin_expr!(
next_bin_mul_expr,
&[TokenKind::FSlash, TokenKind::Splat],
@@ -80,6 +116,7 @@ impl<'t> Parser<'t> {
let start = un_op.span();
let expr = self.next_un_expr()?;
let end = expr.span();
Ok(UnExpr {
op: un_op.kind().into(),
expr,
@@ -116,7 +153,9 @@ impl<'t> Parser<'t> {
}
.into(),
TokenKind::LBracket => {
let prev_skip = self.set_skip_newlines(true)?;
let expr_list = self.next_expr_list(TokenKind::RBracket)?;
self.set_skip_newlines(prev_skip)?;
let end_token =
self.expect_token_kind(TokenKind::RBracket, "end of list (right bracket)")?;
let span = token.span().union(end_token.span());
@@ -128,10 +167,14 @@ impl<'t> Parser<'t> {
}
TokenKind::LBrace => todo!(),
TokenKind::LParen => {
let prev_skip = self.set_skip_newlines(true)?;
let first = self.next_expr()?;
if let Some(_) = self.match_token_kind(TokenKind::Comma)? {
let mut list = self.next_expr_list(TokenKind::RParen)?;
self.set_skip_newlines(prev_skip)?;
let end_token = self.expect_token_kind(TokenKind::RParen, "end of tuple")?;
let span = first.span().union(end_token.span());
list.insert(0, first);
Expr::Base(
@@ -142,6 +185,7 @@ impl<'t> Parser<'t> {
.into(),
)
} else {
self.set_skip_newlines(prev_skip)?;
self.expect_token_kind(TokenKind::RParen, "end of expression")?;
first
}
@@ -173,7 +217,21 @@ impl<'t> Parser<'t> {
////////////////////////////////////////////////////////////////////////////////
fn adv_token(&mut self) -> Result<Option<Token>> {
let next_token = self.lexer.next_token()?;
let mut next_token = self.lexer.next_token()?;
if self.is_skip_newlines() {
while next_token.map(|t| t.kind() == TokenKind::Newline).unwrap_or(false) {
next_token = self.lexer.next_token()?;
}
if self.is_token_kind(TokenKind::Newline) {
self.curr_token = next_token;
while next_token.map(|t| t.kind() == TokenKind::Newline).unwrap_or(false) {
next_token = self.lexer.next_token()?;
}
}
}
Ok(mem::replace(&mut self.curr_token, next_token))
}
@@ -317,9 +375,8 @@ mod test {
}
macro_rules! test_parser {
($text:expr, $($tail:tt)+) => {{
let mut parser = Parser::try_from($text).unwrap();
($parser:expr, $($tail:tt)+) => {{
let mut parser = $parser;
test_parser!(@TAIL, &mut parser, $($tail)+);
}};
@@ -332,8 +389,12 @@ mod test {
test_parser!(@TAIL, $parser, $($tail)*);
}};
(@TAIL, $parser:expr) => {};
(@TAIL, $parser:expr,) => {};
(@TAIL, $parser:expr) => {
assert!($parser.is_eof());
};
(@TAIL, $parser:expr,) => {
assert!($parser.is_eof());
};
}
fn bin_expr(lhs: Expr, op: BinOp, rhs: Expr) -> Expr {
@@ -369,7 +430,7 @@ mod test {
#[test]
fn test_base_expr() {
test_parser!(
"1 x 'value' :sym",
Parser::try_from("1 x 'value' :sym").unwrap(),
// 1
next_expr,
base_expr(BaseExprKind::Num),
@@ -383,9 +444,12 @@ mod test {
next_expr,
base_expr(BaseExprKind::Sym)
);
}
#[test]
fn test_list_expr() {
test_parser!(
"[1, x, 'value', :sym]",
Parser::try_from("[1, x, 'value', :sym]").unwrap(),
next_expr,
base_expr(BaseExprKind::List(vec![
base_expr(BaseExprKind::Num),
@@ -396,7 +460,7 @@ mod test {
);
test_parser!(
"[1, x, 'value', :sym,]",
Parser::try_from("[1, x, 'value', :sym,]").unwrap(),
next_expr,
base_expr(BaseExprKind::List(vec![
base_expr(BaseExprKind::Num),
@@ -408,15 +472,49 @@ mod test {
}
#[test]
fn test_compound_expr() {
fn test_tuple_expr() {
test_parser!(
r#"
x + 2
1 + 2 * 3 - 4 / 5
1+2*3-4/5
1 - -1 * 8
1--1*8
"#,
Parser::try_from("(1, :sym)").unwrap(),
next_expr,
base_expr(BaseExprKind::Tuple(vec![
base_expr(BaseExprKind::Num),
base_expr(BaseExprKind::Sym),
]))
);
test_parser!(
Parser::try_from("(:sym,)").unwrap(),
next_expr,
base_expr(BaseExprKind::Tuple(vec![
base_expr(BaseExprKind::Sym),
]))
);
test_parser!(
Parser::try_from("(:sym)").unwrap(),
next_expr,
base_expr(BaseExprKind::Sym)
);
}
#[test]
fn test_bin_expr() {
test_parser!(
{
let mut parser = Parser::try_from(
r#"x + 2
1 + 2 * 3 - 4 / 5
1+2*3-4/5
1 - -1 * 8
1--1*8
1 + 1 == 2
3 >= 1 + 2
"#,
)
.unwrap();
parser.set_skip_newlines(true).unwrap();
parser
},
// x + 2
next_expr,
bin_expr(
@@ -484,9 +582,122 @@ mod test {
BinOp::Times,
base_expr(BaseExprKind::Num)
)
),
// 1 + 1 == 2
next_expr,
bin_expr(
bin_expr(
base_expr(BaseExprKind::Num),
BinOp::Plus,
base_expr(BaseExprKind::Num)
),
BinOp::EqEq,
base_expr(BaseExprKind::Num)
),
// 3 >= 1 + 2
next_expr,
bin_expr(
base_expr(BaseExprKind::Num),
BinOp::GtEq,
bin_expr(
base_expr(BaseExprKind::Num),
BinOp::Plus,
base_expr(BaseExprKind::Num)
)
)
);
}
#[test]
fn test_multiline_exprs() {
test_parser!(
Parser::try_from(r"[
1,
x,
['value', :value],
:sym
]").unwrap(),
next_expr,
base_expr(BaseExprKind::List(vec![
base_expr(BaseExprKind::Num),
base_expr(BaseExprKind::Ident),
base_expr(BaseExprKind::List(vec![
base_expr(BaseExprKind::Str),
base_expr(BaseExprKind::Sym),
])),
base_expr(BaseExprKind::Sym),
]))
);
test_parser!(
Parser::try_from(r"(
1,
x,
[
'value',
:value,
],
:sym
)").unwrap(),
next_expr,
base_expr(BaseExprKind::Tuple(vec![
base_expr(BaseExprKind::Num),
base_expr(BaseExprKind::Ident),
base_expr(BaseExprKind::List(vec![
base_expr(BaseExprKind::Str),
base_expr(BaseExprKind::Sym),
])),
base_expr(BaseExprKind::Sym),
]))
);
test_parser!(
Parser::try_from(r"(
+ 1
+ 2
+ 3
+ 4
)").unwrap(),
next_expr,
bin_expr(
un_expr(UnOp::Plus, base_expr(BaseExprKind::Num)),
BinOp::Plus,
bin_expr(
base_expr(BaseExprKind::Num),
BinOp::Plus,
bin_expr(
base_expr(BaseExprKind::Num),
BinOp::Plus,
base_expr(BaseExprKind::Num)
)
)
)
);
//
test_parser!(
Parser::try_from(r"(
+ 1
+ 2
+ 3
+ 4
,
)").unwrap(),
next_expr,
base_expr(BaseExprKind::Tuple(vec![
bin_expr(
un_expr(UnOp::Plus, base_expr(BaseExprKind::Num)),
BinOp::Plus,
bin_expr(
base_expr(BaseExprKind::Num),
BinOp::Plus,
bin_expr(
base_expr(BaseExprKind::Num),
BinOp::Plus,
base_expr(BaseExprKind::Num)
)
)
)
]))
);
}
}