Add Stmt and incorporate meta statements

This helps split up expressions, meta calls (like includes) that can be
expanded into more expressions.

Signed-off-by: Alek Ratzloff <alekratz@gmail.com>

src/compile.rs

@@ -17,14 +17,39 @@ impl<'s> Compile<'s> {
         }
     }
 
-    pub fn compile(&mut self, expr_list: &Vec<SpExpr>) -> Vec<Inst> {
+    pub fn compile(&mut self, stmt_list: Vec<SpStmt>) -> Vec<Inst> {
+        // Compile meta-statements
+        let expr_list = self.compile_meta(stmt_list);
+        self.compile_expr_list(&expr_list)
+    }
+
+    /// Compile meta-expressions, like includes.
+    fn compile_meta(&mut self, stmt_list: Vec<SpStmt>) -> Vec<SpExpr> {
+        let mut expr_list = Vec::new();
+        for stmt in stmt_list {
+            match stmt.into_inner() {
+                Stmt::Include(path) => {
+                    let stmt_list = self.parse_include(&path);
+                    expr_list.extend(self.compile_meta(stmt_list));
+                }
+                Stmt::Expr(expr) => expr_list.push(expr),
+            }
+        }
+        expr_list
+    }
+
+    fn parse_include(&mut self, _path: &str) -> Vec<SpStmt> {
+        todo!("includes")
+    }
+
+    fn compile_expr_list(&mut self, expr_list: &Vec<SpExpr>) -> Vec<Inst> {
         // Scoping is done here.
         // Local scopes are implicit. If a variable is assigned to at
         // all in the current scope, it's considered to be a local.
         let mut code = Vec::new();
         for expr in expr_list {
-            self.discover_locals(expr);
+            self.discover_locals(&expr);
-            let thunk = self.compile_expr(expr);
+            let thunk = self.compile_expr(&expr);
             code.extend(thunk.flatten());
         }
         code
@@ -34,13 +59,14 @@ impl<'s> Compile<'s> {
         match expr.inner() {
             Expr::Atom(atom) => self.compile_atom(atom),
             // this gets compiled whenever it gets evaluated
-            Expr::Quote(exprs) => {
+            Expr::Quote(stmts) => {
                 self.scope_stack.push_scope();
-                let compiled = Rc::new(self.compile(exprs));
+                // TODO - is self.compile the right thing to do here?
+                let compiled = Rc::new(self.compile(stmts.clone()));
                 let locals = self.scope_stack.pop_scope().unwrap();
                 let quote = self
                     .quote_table
-                    .insert(expr.span(), locals, exprs.clone(), compiled);
+                    .insert(expr.span(), locals, stmts.clone(), compiled);
                 Inst::PushValue(Value::Quote(quote)).into()
             }
         }

src/main.rs

@@ -36,9 +36,9 @@ fn main() -> Result {
     };
 
     let mut parser = Parser::from(text.as_str());
-    let mut exprs = Vec::new();
+    let mut stmts = Vec::new();
     while !parser.is_eof() {
-        exprs.extend(parser.next_expr_list()?);
+        stmts.extend(parser.next_stmt_list()?);
     }
 
     let mut machine = MachineBuilder::default()
@@ -46,7 +46,7 @@ fn main() -> Result {
         .max_arena_objects(opt.max_arena_objects)
         .finish();
 
-    machine.eval(&exprs)?;
+    machine.eval(stmts)?;
 
     Ok(())
 }

src/object.rs

@@ -3,7 +3,7 @@
 // want it to clog the warnings yet.
 #![allow(dead_code)]
 
-use crate::syn::ast::SpExpr;
+use crate::syn::ast::SpStmt;
 use crate::vm::{error::Result, machine::Machine};
 use crate::{scope::Scope, syn::span::Span, vm::inst::Inst};
 use std::cell::RefCell;
@@ -33,7 +33,7 @@ impl Quote {
 /// A table of compiled quotes, their expression trees, and their spans.
 #[derive(Debug, Clone, Default)]
 pub struct QuoteTable {
-    table: Vec<(Span, Scope, Vec<SpExpr>, Rc<Vec<Inst>>)>,
+    table: Vec<(Span, Scope, Vec<SpStmt>, Rc<Vec<Inst>>)>,
 }
 
 impl QuoteTable {
@@ -45,7 +45,7 @@ impl QuoteTable {
         &mut self,
         span: Span,
         scope: Scope,
-        quote: Vec<SpExpr>,
+        quote: Vec<SpStmt>,
         compiled: Rc<Vec<Inst>>,
     ) -> Quote {
         let next = Quote(self.table.len());
@@ -53,7 +53,7 @@ impl QuoteTable {
         next
     }
 
-    pub fn get(&self, quote: Quote) -> &(Span, Scope, Vec<SpExpr>, Rc<Vec<Inst>>) {
+    pub fn get(&self, quote: Quote) -> &(Span, Scope, Vec<SpStmt>, Rc<Vec<Inst>>) {
         &self.table[quote.0]
     }
 }

src/syn/ast.rs

@@ -1,21 +1,52 @@
 use crate::object::{Float, Int, Str};
 use crate::syn::span::*;
 
+/// A single statement, which may be a meta-expression or a normal expression.
 #[derive(Debug, Clone, PartialEq)]
-pub enum Expr {
+pub enum Stmt {
-    Atom(SpAtom),
+    /// A meta-expression that includes an AST.
-    Quote(Vec<SpExpr>),
+    Include(String),
+
+    /// A normal expression.
+    Expr(SpExpr),
 }
 
+/// A normal expression.
+///
+/// An expression may be an atom (word, float, int, string, etc), or it may be a
+/// quoted expression.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Expr {
+    /// An atom expression.
+    Atom(SpAtom),
+
+    /// A quoted expression list.
+    Quote(Vec<SpStmt>),
+}
+
+/// A single, finite value.
 #[derive(Debug, Clone, PartialEq)]
 pub enum Atom {
+    /// An assignment atom. This is basically a word that starts with the `:`
+    /// character.
     Assign(Str),
+
+    /// A word, which is pretty much anything that is not a number.
     Word(Str),
+
+    /// A floating point number.
     Float(Float),
+
+    /// An integer.
     Int(Int),
+
+    /// A string.
     Str(Str),
+
+    /// An application of a function.
     Apply,
 }
 
 pub type SpAtom = Spanned<Atom>;
 pub type SpExpr = Spanned<Expr>;
+pub type SpStmt = Spanned<Stmt>;

src/syn/parser.rs

@@ -83,16 +83,46 @@ impl<'t> Parser<'t> {
         }
     }
 
-    /// Gets all expressions until EOF is reached, or until a quote end is reached.
+    pub fn next_stmt_list(&mut self) -> Result<Vec<SpStmt>> {
-    pub fn next_expr_list(&mut self) -> Result<Vec<SpExpr>> {
+        let mut stmts = Vec::new();
-        let mut exprs = Vec::new();
         while let Some(peek) = self.peek()? {
             match peek.inner() {
                 Token::RQuote => break,
-                _ => exprs.push(self.next_expr()?),
+                _ => stmts.push(self.next_stmt()?),
+            }
+        }
+
+        Ok(stmts)
+    }
+
+    pub fn next_stmt(&mut self) -> Result<SpStmt> {
+        match self.peek()? {
+            Some(peek) if *peek.inner() == Token::Meta => self.next_meta(),
+            _ => {
+                let expr = self.next_expr()?;
+                let span = expr.span();
+                Ok(SpStmt::new(span, Stmt::Expr(expr)))
+            }
+        }
+    }
+
+    pub fn next_meta(&mut self) -> Result<SpStmt> {
+        let (meta_span, _token) = self.expect_any_token(&[Token::Meta])?.into_split();
+        let text = meta_span.text_at(self.lexer.text());
+        match text {
+            "%include" => {
+                // get the include location string
+                let (path_span, _token) = self.expect_any_token(&[Token::Str])?.into_split();
+                let path = unescape_string(path_span.text_at(self.lexer.text()));
+                Ok(SpStmt::new(meta_span.union(path_span), Stmt::Include(path)))
+            }
+            _ => {
+                todo!(
+                    "put a warning message here for an unknown meta statement {:?}",
+                    text
+                )
             }
         }
-        Ok(exprs)
     }
 
     pub fn next_expr(&mut self) -> Result<SpExpr> {
@@ -109,10 +139,10 @@ impl<'t> Parser<'t> {
 
     pub fn next_quote(&mut self) -> Result<SpExpr> {
         let start = self.expect_any_token(&[Token::LQuote])?;
-        let exprs = self.next_expr_list()?;
+        let stmts = self.next_stmt_list()?;
         let end = self.expect_any_token(&[Token::RQuote])?;
         let span = start.span().union(end.span());
-        Ok(SpExpr::new(span, Expr::Quote(exprs)))
+        Ok(SpExpr::new(span, Expr::Quote(stmts)))
     }
 
     pub fn next_atom(&mut self) -> Result<SpAtom> {
@@ -217,7 +247,7 @@ macro_rules! make_quote {
 macro_rules! make_quote_vec {
     ($($expr:expr),+ $(,)?) => {{
         vec![$(
-            SpExpr::new(Default::default(), $expr)
+            SpStmt::new(Default::default(), Stmt::Expr(SpExpr::new(Default::default(), $expr)))
         ),+]
     }};
 }

src/vm/machine.rs

@@ -1,7 +1,7 @@
 use crate::compile::Compile;
 use crate::object::*;
 use crate::scope::*;
-use crate::syn::ast::SpExpr;
+use crate::syn::ast::SpStmt;
 use crate::vm::{error::*, inst::*};
 use std::cell::RefCell;
 use std::collections::BTreeMap;
@@ -175,10 +175,10 @@ impl Machine {
     // Eval
     // /////////////////////////////////////////////////////////////////////////
 
-    pub fn eval(&mut self, exprs: &Vec<SpExpr>) -> Result<()> {
+    pub fn eval(&mut self, stmts: Vec<SpStmt>) -> Result<()> {
         self.scope_stack.push_scope();
         let mut compile = Compile::new(&mut self.scope_stack, &mut self.quote_table);
-        let code = Rc::new(compile.compile(exprs));
+        let code = Rc::new(compile.compile(stmts));
         let locals = self.scope_stack.pop_scope().unwrap();
         self.call_stack.push(
             QuoteFrame {