Add branching and more coherent method of function calls

We're turing-complete, babey!

* Call stack for functions now differentiates between native and quote
  calls
* Better API for builtin functions allowing for more control
* Example showing off branches

Signed-off-by: Alek Ratzloff <alekratz@gmail.com>

examples/branch.sy

@@ -0,0 +1,7 @@
+0 [ "fail - 0 is not truthy" . ] [ "OK" . ] if!
+1 [ "OK" . ] [ "fail - 1 is truthy" . ] if!
+1.1 [ "OK" . ] [ "fail - 1.1 is truthy" . ] if!
+0.1 [ "OK" . ] [ "fail - 0.1 is truthy" . ] if!
+-0.1 [ "OK" . ] [ "fail - -0.1 is truthy" . ] if!
+-0.0 [ "fail - -0.0 is not truthy" . ] [ "OK" . ] if!
+0.0 [ "fail - 0.0 is not truthy" . ] [ "OK" . ] if!

src/compile.rs

@@ -2,6 +2,7 @@ use crate::object::{QuoteTable, Value};
 use crate::scope::*;
 use crate::syn::ast::*;
 use crate::vm::inst::*;
+use std::rc::Rc;
 
 pub struct Compile<'s> {
     scope_stack: &'s mut ScopeStack,
@@ -35,7 +36,7 @@ impl<'s> Compile<'s> {
             // this gets compiled whenever it gets evaluated
             Expr::Quote(exprs) => {
                 self.scope_stack.push_scope();
-                let compiled = self.compile(exprs);
+                let compiled = Rc::new(self.compile(exprs));
                 let locals = self.scope_stack.pop_scope().unwrap();
                 let quote = self
                     .quote_table

src/object.rs

@@ -4,7 +4,7 @@
 #![allow(dead_code)]
 
 use crate::syn::ast::SpExpr;
-use crate::vm::machine::Machine;
+use crate::vm::{error::Result, machine::Machine};
 use crate::{scope::Scope, syn::span::Span, vm::inst::Inst};
 use std::cell::RefCell;
 use std::collections::{BTreeMap, HashMap};
@@ -33,7 +33,7 @@ impl Quote {
 /// A table of compiled quotes, their expression trees, and their spans.
 #[derive(Debug, Clone, Default)]
 pub struct QuoteTable {
-    table: Vec<(Span, Scope, Vec<SpExpr>, Vec<Inst>)>,
+    table: Vec<(Span, Scope, Vec<SpExpr>, Rc<Vec<Inst>>)>,
 }
 
 impl QuoteTable {
@@ -46,14 +46,14 @@ impl QuoteTable {
         span: Span,
         scope: Scope,
         quote: Vec<SpExpr>,
-        compiled: Vec<Inst>,
+        compiled: Rc<Vec<Inst>>,
     ) -> Quote {
         let next = Quote(self.table.len());
         self.table.push((span, scope, quote, compiled));
         next
     }
 
-    pub fn get(&self, quote: Quote) -> &(Span, Scope, Vec<SpExpr>, Vec<Inst>) {
+    pub fn get(&self, quote: Quote) -> &(Span, Scope, Vec<SpExpr>, Rc<Vec<Inst>>) {
         &self.table[quote.0]
     }
 }
@@ -86,6 +86,19 @@ impl Value {
             BuiltinFn(_) => "builtin function",
         }
     }
+
+    pub fn is_truthy(&self) -> bool {
+        use Value::*;
+        match self {
+            Array(a) => a.len() > 0,
+            Float(f) => *f != 0.0,
+            Int(i) => *i != 0,
+            Str(s) => s.len() > 0,
+            Quote(_) => true,
+            ObjPtr(_) => true,
+            BuiltinFn(_) => true,
+        }
+    }
 }
 
 impl Display for Value {
@@ -103,8 +116,40 @@ impl Display for Value {
     }
 }
 
+/// The intent of a builtin function once the function proper returns.
+///
+/// When a builtin function itself returns, it may want to remain on the call
+/// stack, or do some kind of cleanup before it's finished. So, a function may
+/// pre-empt itself with the understanding that it will eventually return.
+#[derive(Debug, Clone, Copy)]
+pub enum BuiltinExit {
+    /// Pop this builtin off the call stack, and then call the given quote.
+    Call(Quote),
+
+    /// Continue with the assumption that some other function has been pushed to
+    /// the call stack, updating the reentry for this function call to the
+    /// specified constant.
+    Resume(usize),
+
+    /// Pops this builtin off the call stack, returning like normal.
+    Return,
+}
+
 #[derive(Clone)]
-pub struct BuiltinFn(Rc<dyn FnMut(&mut Machine)>);
+pub struct BuiltinFn(Rc<BuiltinFnPtr>);
+
+impl BuiltinFn {
+    pub fn new(fun: Rc<BuiltinFnPtr>) -> Self {
+        BuiltinFn(fun)
+    }
+
+    pub fn call(&self, machine: &mut Machine, reentry: usize) -> Result<BuiltinExit> {
+        (self.0)(machine, reentry)
+    }
+}
+
+//pub type BuiltinFnPtr = dyn Fn(&mut Machine, usize) -> Result<BuiltinExit>;
+pub type BuiltinFnPtr = fn(&mut Machine, usize) -> Result<BuiltinExit>;
 
 impl Debug for BuiltinFn {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {

src/syn/lexer.rs

@@ -5,10 +5,10 @@ use regex::{Regex, RegexBuilder};
 lazy_static! {
     static ref LEX_PAT: Regex = RegexBuilder::new(
         r#"^(
-            (?P<assign>=[a-zA-Z_?\-*+/.'@$%^&|][0-9a-zA-Z_?\-*+/=.'@$%^&|]*)
+            (?P<float>[-+]?[0-9]+\.[0-9]+([eE][+\-][0-9]+)?)
+        |   (?P<int>[-+]?[0-9]+)
+        |   (?P<assign>=[a-zA-Z_?\-*+/.'@$%^&|][0-9a-zA-Z_?\-*+/=.'@$%^&|]*)
         |   (?P<word>[a-zA-Z_?\-*+/=.'@$%^&|][0-9a-zA-Z_?\-*+/=.'@$%^&|]*)
-        |   (?P<float>[0-9]+\.[0-9]+([eE][+\-][0-9]+)?)
-        |   (?P<int>[0-9]+)
         |   (?P<lquote>\[)
         |   (?P<rquote>\])
         |   (?P<colon>:)
@@ -195,17 +195,17 @@ mod test {
 
     #[test]
     fn test_numbers() {
-        let mut lexer = Lexer::new(r"1 12 123 9 98 987 987654321 1248 9764321 1.2 2.3");
+        let mut lexer = Lexer::new(r"1 -12 123 -9 98 987 -987654321 1248 9764321 -1.2 2.3");
         assert_token!(lexer, Token::Int, "1");
-        assert_token!(lexer, Token::Int, "12");
+        assert_token!(lexer, Token::Int, "-12");
         assert_token!(lexer, Token::Int, "123");
-        assert_token!(lexer, Token::Int, "9");
+        assert_token!(lexer, Token::Int, "-9");
         assert_token!(lexer, Token::Int, "98");
         assert_token!(lexer, Token::Int, "987");
-        assert_token!(lexer, Token::Int, "987654321");
+        assert_token!(lexer, Token::Int, "-987654321");
         assert_token!(lexer, Token::Int, "1248");
         assert_token!(lexer, Token::Int, "9764321");
-        assert_token!(lexer, Token::Float, "1.2");
+        assert_token!(lexer, Token::Float, "-1.2");
         assert_token!(lexer, Token::Float, "2.3");
         assert!(lexer.is_eof());
     }

src/vm/machine.rs

@@ -7,14 +7,38 @@ use std::cell::RefCell;
 use std::collections::BTreeMap;
 use std::rc::Rc;
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
-pub struct Frame {
+pub enum Frame {
+    Native(NativeFrame),
+    Quote(QuoteFrame),
+}
+
+impl From<NativeFrame> for Frame {
+    fn from(other: NativeFrame) -> Self {
+        Frame::Native(other)
+    }
+}
+
+impl From<QuoteFrame> for Frame {
+    fn from(other: QuoteFrame) -> Self {
+        Frame::Quote(other)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct NativeFrame {
+    fun: BuiltinFn,
+    reentry: usize,
+}
+
+#[derive(Debug, Clone)]
+pub struct QuoteFrame {
     locals: BTreeMap<Word, Option<Value>>,
-    code: Vec<Inst>, // TODO - deduplicate this with some kind of shared pointer
+    code: Rc<Vec<Inst>>, // TODO - deduplicate this with some kind of shared pointer
     pc: usize,
 }
 
-impl Frame {
+impl QuoteFrame {
     pub fn inst(&self) -> Option<&Inst> {
         self.code.get(self.pc)
     }
@@ -23,6 +47,7 @@ impl Frame {
 /// The current state of a VM.
 #[derive(Debug)]
 pub struct Machine {
+    globals: BTreeMap<Word, Value>,
     stack: Vec<Value>,
     max_stack_size: Option<usize>,
     arena: Rc<RefCell<Arena>>,
@@ -32,13 +57,19 @@ pub struct Machine {
 }
 
 impl Machine {
-    pub fn new(max_stack_size: Option<usize>, arena: Arena) -> Self {
+    pub fn new(
+        globals: BTreeMap<Word, Value>,
+        max_stack_size: Option<usize>,
+        arena: Arena,
+        scope_stack: ScopeStack,
+    ) -> Self {
         Machine {
+            globals,
             stack: Default::default(),
             max_stack_size,
             arena: Rc::new(RefCell::new(arena)),
             quote_table: Default::default(),
-            scope_stack: Default::default(),
+            scope_stack,
             call_stack: Default::default(),
         }
     }
@@ -79,13 +110,32 @@ impl Machine {
         self.call_stack
             .iter()
             .rev()
-            .filter_map(|frame| frame.locals.get(word))
+            .filter_map(|frame| {
-            .next()?
+                if let Frame::Quote(frame) = frame {
-            .as_ref()
+                    frame.locals.get(word)
+                } else {
+                    None
+                }
+            })
+            .filter_map(|value| value.as_ref())
+            .next()
+            .or_else(|| self.globals.get(word))
     }
 
     pub fn store_local(&mut self, word: Word, value: Value) {
-        let last = self.call_stack.last_mut().expect("no call stack");
+        let last = self
+            .call_stack
+            .iter_mut()
+            .rev()
+            .filter_map(|frame| {
+                if let Frame::Quote(frame) = frame {
+                    Some(frame)
+                } else {
+                    None
+                }
+            })
+            .next()
+            .expect("no call stack");
         last.locals.insert(word, Some(value));
     }
 
@@ -93,12 +143,32 @@ impl Machine {
         self.call_stack.last()
     }
 
-    pub fn frame_mut(&mut self) -> Option<&mut Frame> {
+    pub fn call_quote(&mut self, quote: Quote) {
-        self.call_stack.last_mut()
+        let (_span, locals, _expr, code) = self.quote_table.get(quote);
+
+        // create a new stack frame
+        self.call_stack.push(
+            QuoteFrame {
+                locals: locals.keys().map(|w| (*w, None)).collect(),
+                code: code.clone(),
+                pc: 0,
+            }
+            .into(),
+        );
     }
 
-    fn pc(&self) -> usize {
+    pub fn call_native(&mut self, native: BuiltinFn) {
-        self.frame().unwrap().pc
+        self.call_stack.push(Frame::Native(NativeFrame {
+            fun: native,
+            reentry: 0,
+        }));
+    }
+
+    fn pc(&self) -> Option<usize> {
+        match self.frame() {
+            Some(Frame::Quote(frame)) => Some(frame.pc),
+            _ => None,
+        }
     }
 
     // /////////////////////////////////////////////////////////////////////////
@@ -108,13 +178,16 @@ impl Machine {
     pub fn eval(&mut self, exprs: &Vec<SpExpr>) -> Result<()> {
         self.scope_stack.push_scope();
         let mut compile = Compile::new(&mut self.scope_stack, &mut self.quote_table);
-        let code = compile.compile(exprs);
+        let code = Rc::new(compile.compile(exprs));
         let locals = self.scope_stack.pop_scope().unwrap();
-        self.call_stack.push(Frame {
+        self.call_stack.push(
+            QuoteFrame {
                 locals: locals.keys().map(|w| (*w, None)).collect(),
                 code,
                 pc: 0,
-        });
+            }
+            .into(),
+        );
 
         // Run instructions
         loop {
@@ -122,7 +195,8 @@ impl Machine {
             if self.call_stack.is_empty() {
                 break;
             }
-            let frame = self.frame().unwrap();
+            match self.frame().unwrap() {
+                Frame::Quote(frame) => {
                     let inst = frame.inst();
                     if let Some(inst) = inst {
                         let inst = inst.clone();
@@ -131,13 +205,46 @@ impl Machine {
                         self.call_stack.pop();
                     }
                 }
+                Frame::Native(frame) => {
+                    // there's a more obvious way to do this, but sadly, it does
+                    // not play nicely with the borrow checker. so we have to
+                    // record values and index items so we don't borrow anything
+                    // mutably, simultaneously.
+                    let current_frame = self.call_stack.len() - 1;
+                    let fun = frame.fun.clone();
+                    let reentry = frame.reentry;
+                    let exit = self.do_call_native(fun, reentry)?;
+
+                    match exit {
+                        BuiltinExit::Call(quote) => {
+                            self.call_stack.pop();
+                            self.call_quote(quote);
+                        }
+                        BuiltinExit::Resume(reentry) => {
+                            if let Frame::Native(frame) = &mut self.call_stack[current_frame] {
+                                frame.reentry = reentry;
+                            } else {
+                                panic!("expected native call frame but it was replaced before we could get to it");
+                            }
+                        }
+                        BuiltinExit::Return => {
+                            self.call_stack.pop();
+                        }
+                    }
+                }
+            }
+        }
 
         Ok(())
     }
 
+    fn do_call_native(&mut self, native: BuiltinFn, reentry: usize) -> Result<BuiltinExit> {
+        native.call(self, reentry)
+    }
+
     fn eval_inst(&mut self, inst: Inst) -> Result<()> {
-        let mut new_frame = None;
+        let current_frame = self.call_stack.len() - 1;
-        let next_pc = self.pc() + 1;
+        let next_pc = self.pc().unwrap() + 1;
         match inst {
             Inst::PushValue(value) => self.stack_push(value)?,
             Inst::Load(word) => {
@@ -153,33 +260,26 @@ impl Machine {
                 let value = self.stack_pop()?;
                 self.store_local(word, value);
             }
-            Inst::Call => {
+            Inst::Call => match self.stack_pop()? {
-                let value = self.stack_pop()?;
+                Value::Quote(quote) => {
-                let quote = if let Value::Quote(quote) = value {
+                    self.call_quote(quote);
-                    quote
-                } else {
-                    return Err(RuntimeError::CannotCall(value.name().to_string()));
-                };
-                let (_span, locals, _expr, code) = self.quote_table.get(quote);
-
-                // create a new stack frame
-                new_frame = Some(Frame {
-                    locals: locals.keys().map(|w| (*w, None)).collect(),
-                    code: code.clone(),
-                    pc: 0,
-                });
                 }
+                Value::BuiltinFn(builtin) => {
+                    self.call_native(builtin);
+                }
+                value => return Err(RuntimeError::CannotCall(value.name().to_string())),
+            },
             Inst::Print => {
                 let value = self.stack_pop()?;
                 println!("{}", value);
             }
         };
 
-        self.frame_mut().unwrap().pc = next_pc;
+        // Update this frame's PC
-
+        if let Frame::Quote(frame) = &mut self.call_stack[current_frame] {
-        // Push the new stack frame *after* updating the PC
+            frame.pc = next_pc;
-        if let Some(frame) = new_frame {
+        } else {
-            self.call_stack.push(frame);
+            panic!("attempted to update call stack PC, but current recorded frame was removed before we could get to it");
         }
 
         Ok(())
@@ -188,6 +288,8 @@ impl Machine {
 
 #[derive(Debug, Default)]
 pub struct MachineBuilder {
+    globals: BTreeMap<Word, Value>,
+    scope_stack: ScopeStack,
     max_stack_size: Option<usize>,
     max_arena_objects: Option<usize>,
 }
@@ -203,7 +305,60 @@ impl MachineBuilder {
         self
     }
 
-    pub fn finish(self) -> Machine {
+    /// Registers all builtins for calling on the machine.
-        Machine::new(self.max_stack_size, Arena::new(self.max_arena_objects))
+    fn register_builtins(&mut self) {
+        self.scope_stack.push_scope();
+        self.register_builtin_fun("panic", |machine, _| {
+            println!("!!! panic");
+            println!("!!! top of stack");
+            for (i, value) in machine.stack().iter().enumerate().rev() {
+                println!("!!!   {}. {:?}", i, value);
+            }
+            println!("!!! bottom of stack");
+            panic!();
+        });
+
+        self.register_builtin_fun("if", |machine, _| {
+            let if_false = machine.stack_pop()?;
+            let if_true = machine.stack_pop()?;
+            let condition = machine.stack_pop()?;
+            let value = if condition.is_truthy() {
+                if_true
+            } else {
+                if_false
+            };
+
+            if let Value::Quote(quote) = value {
+                Ok(BuiltinExit::Call(quote))
+            } else {
+                return Err(RuntimeError::CannotCall(
+                    "if statement requires quote value".to_string(),
+                ));
+            }
+        });
+    }
+
+    fn register_builtin_fun(
+        &mut self,
+        name: &str,
+        fun: fn(&mut Machine, usize) -> Result<BuiltinExit>,
+    ) {
+        self.register_global(name, Value::BuiltinFn(BuiltinFn::new(Rc::new(fun))));
+    }
+
+    fn register_global(&mut self, name: &str, value: Value) {
+        let word = self.scope_stack.insert_local(name);
+        self.globals.insert(word, value);
+    }
+
+    pub fn finish(mut self) -> Machine {
+        self.register_builtins();
+
+        Machine::new(
+            self.globals,
+            self.max_stack_size,
+            Arena::new(self.max_arena_objects),
+            self.scope_stack,
+        )
     }
 }