Add lists

This introduces:

* new syntax for list literals, put comma-separated values between
  braces for your new list
* new syntax for indexing, do `foo[index]` to get the value in `foo` at
  `index`. Lists also allow negative indices too. Any type that wants to
  be indexed can include their own __index__ function as well.
* new VM instruction, BuildList. List literals were a lot easier to
  implement using this rather than creating a new list, creating a
  temporary stack value, and then duplicating + pushing to that
  temporary value over and over.

Signed-off-by: Alek Ratzloff <alekratz@gmail.com>

src/ast.rs

@@ -1,5 +1,5 @@
 // This is an auto-generated file. Any changes made to this file may be overwritten.
-// This file was created at: 2024-09-18 09:28:21
+// This file was created at: 2024-09-30 16:14:27
 #![allow(dead_code)]
 use std::fmt::Debug;
 use std::any::Any;
@@ -11,8 +11,10 @@ pub trait ExprVisitor {
     fn visit_unary_expr(&mut self, expr: &UnaryExpr) -> Result<(), Box<dyn std::error::Error>>;
     fn visit_call_expr(&mut self, expr: &CallExpr) -> Result<(), Box<dyn std::error::Error>>;
     fn visit_get_expr(&mut self, expr: &GetExpr) -> Result<(), Box<dyn std::error::Error>>;
+    fn visit_index_expr(&mut self, expr: &IndexExpr) -> Result<(), Box<dyn std::error::Error>>;
     fn visit_primary_expr(&mut self, expr: &PrimaryExpr) -> Result<(), Box<dyn std::error::Error>>;
     fn visit_function_expr(&mut self, expr: &FunctionExpr) -> Result<(), Box<dyn std::error::Error>>;
+    fn visit_list_expr(&mut self, expr: &ListExpr) -> Result<(), Box<dyn std::error::Error>>;
 }
 
 pub trait StmtVisitor {
@@ -114,6 +116,27 @@ impl Expr for GetExpr {
     }
 }
 
+#[derive(Debug)]
+pub struct IndexExpr {
+    pub expr: ExprP,
+    pub index: ExprP,
+    pub rbracket: Token,
+}
+
+impl Expr for IndexExpr {
+    fn accept(&self, visitor: &mut dyn ExprVisitor) -> Result<(), Box<dyn std::error::Error>> {
+        visitor.visit_index_expr(self)
+    }
+
+    fn as_any(self: Box<Self>) -> Box<dyn Any> {
+        self
+    }
+
+    fn as_any_ref(&self) -> &dyn Any {
+        self
+    }
+}
+
 #[derive(Debug)]
 pub struct PrimaryExpr {
     pub token: Token,
@@ -156,6 +179,27 @@ impl Expr for FunctionExpr {
     }
 }
 
+#[derive(Debug)]
+pub struct ListExpr {
+    pub lbracket: Token,
+    pub exprs: Vec<ExprP>,
+    pub rbracket: Token,
+}
+
+impl Expr for ListExpr {
+    fn accept(&self, visitor: &mut dyn ExprVisitor) -> Result<(), Box<dyn std::error::Error>> {
+        visitor.visit_list_expr(self)
+    }
+
+    fn as_any(self: Box<Self>) -> Box<dyn Any> {
+        self
+    }
+
+    fn as_any_ref(&self) -> &dyn Any {
+        self
+    }
+}
+
 pub trait Stmt: Debug + Any {
     fn accept(&self, visitor: &mut dyn StmtVisitor) -> Result<(), Box<dyn std::error::Error>>;
     fn as_any(self: Box<Self>) -> Box<dyn Any>;

src/compiler.rs

@@ -106,6 +106,12 @@ impl ExprVisitor for LineNumber {
         Ok(())
     }
 
+    fn visit_index_expr(&mut self, expr: &IndexExpr) -> Result<()> {
+        expr.expr.accept(self).unwrap();
+        self.update_end(expr.rbracket.line);
+        Ok(())
+    }
+
     fn visit_primary_expr(&mut self, expr: &PrimaryExpr) -> Result<()> {
         self.update_start(expr.token.line);
         self.update_end(expr.token.line);
@@ -117,6 +123,12 @@ impl ExprVisitor for LineNumber {
         self.update_end(expr.rbrace.line);
         Ok(())
     }
+
+    fn visit_list_expr(&mut self, expr: &ListExpr) -> Result<()> {
+        self.update_start(expr.lbracket.line);
+        self.update_end(expr.rbracket.line);
+        Ok(())
+    }
 }
 
 fn expr_line_number(expr: &dyn Expr) -> LineRange {
@@ -238,6 +250,10 @@ impl ExprVisitor for LocalAssignCollector {
         Ok(())
     }
 
+    fn visit_index_expr(&mut self, _expr: &IndexExpr) -> Result<()> {
+        Ok(())
+    }
+
     fn visit_primary_expr(&mut self, _expr: &PrimaryExpr) -> Result<()> {
         Ok(())
     }
@@ -246,6 +262,11 @@ impl ExprVisitor for LocalAssignCollector {
         // don't visit function expr, we're only collecting local assigns
         Ok(())
     }
+
+    fn visit_list_expr(&mut self, _expr: &ListExpr) -> Result<()> {
+        // there shouldn't be any local assignments inside of a list expression
+        Ok(())
+    }
 }
 
 #[derive(Default)]
@@ -326,6 +347,12 @@ impl ExprVisitor for LocalNameCollector {
         Ok(())
     }
 
+    fn visit_index_expr(&mut self, expr: &IndexExpr) -> Result<()> {
+        expr.expr.accept(self)?;
+        expr.index.accept(self)?;
+        Ok(())
+    }
+
     fn visit_primary_expr(&mut self, expr: &PrimaryExpr) -> Result<()> {
         if expr.token.kind == TokenKind::Name {
             self.names.insert(expr.token.text.to_string());
@@ -337,6 +364,13 @@ impl ExprVisitor for LocalNameCollector {
         // don't visit function expr, we're only collecting local assigns
         Ok(())
     }
+
+    fn visit_list_expr(&mut self, expr: &ListExpr) -> Result<()> {
+        for expr in &expr.exprs {
+            expr.accept(self)?;
+        }
+        Ok(())
+    }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -863,6 +897,15 @@ impl ExprVisitor for Compiler {
         Ok(())
     }
 
+    fn visit_index_expr(&mut self, expr: &IndexExpr) -> Result<()> {
+        self.compile_expr(&expr.expr)?;
+        let constant_id = self.insert_constant(Str::create("__index__"))?;
+        self.emit(expr_line_number(expr), Op::GetAttr(constant_id));
+        self.compile_expr(&expr.index)?;
+        self.emit(expr_line_number(expr), Op::Call(1));
+        Ok(())
+    }
+
     fn visit_primary_expr(&mut self, expr: &PrimaryExpr) -> Result<()> {
         match expr.token.kind {
             TokenKind::Name => {
@@ -990,4 +1033,16 @@ impl ExprVisitor for Compiler {
 
         Ok(())
     }
+
+    fn visit_list_expr(&mut self, expr: &ListExpr) -> Result<()> {
+        let line = expr_line_number(expr);
+
+        for expr in &expr.exprs {
+            self.compile_expr(expr)?;
+        }
+
+        self.emit(line, Op::BuildList(expr.exprs.len() as ListLen));
+
+        Ok(())
+    }
 }

src/disassemble.rs

@@ -99,6 +99,11 @@ fn disassemble_chunk(chunk: &Chunk, constants: &Vec<ObjP>, globals: &Vec<String>
                 arg = format!("{depth}");
                 info = format!("slot {slot} (name unknown)");
             }
+            Op::BuildList(len) => {
+                op_str = "BUILD_LIST";
+                arg = format!("{len}");
+                info = String::new();
+            }
             Op::Nop => {
                 op_str = "NOP";
                 arg = String::new();

src/obj.rs

@@ -1,5 +1,4 @@
-// TODO obj.rs - remove the warning suppression
+//! Object implementations.
-//#![allow(dead_code)]
 
 mod macros;
 // Leave this comment here - macros must come first
@@ -7,6 +6,7 @@ pub mod bool;
 pub mod float;
 pub mod function;
 pub mod int;
+pub mod list;
 pub mod str;
 pub mod ty;
 
@@ -41,7 +41,7 @@ pub mod prelude {
     pub use crate::obj::ObjP;
 
     pub use crate::obj::function::{BuiltinFunction, Method, UserFunction};
-    pub use crate::obj::{bool::Bool, float::Float, int::Int, str::Str, ty::Ty};
+    pub use crate::obj::{bool::Bool, float::Float, int::Int, list::List, str::Str, ty::Ty};
     pub use crate::obj::{Nil, Obj};
 
     // Other auxiliary types and functions

src/obj/list.rs

@@ -0,0 +1,209 @@
+use std::fmt::{self, Debug, Display};
+
+use gc::{Finalize, Trace};
+
+use crate::obj::macros::*;
+use crate::obj::prelude::*;
+use crate::obj::BaseObj;
+use crate::vm::Vm;
+
+#[derive(Trace, Finalize)]
+pub struct List {
+    base: BaseObj,
+    list: Vec<ObjP>,
+}
+
+impl List {
+    pub fn new(list: Vec<ObjP>) -> Self {
+        List {
+            base: Default::default(),
+            list,
+        }
+    }
+
+    pub fn list(&self) -> &Vec<ObjP> {
+        &self.list
+    }
+
+    pub fn list_mut(&mut self) -> &mut Vec<ObjP> {
+        &mut self.list
+    }
+
+    impl_create!(list: Vec<ObjP>);
+}
+
+impl Display for List {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        Debug::fmt(self, fmt)
+    }
+}
+
+impl Debug for List {
+    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+        // NOTE : this function should not actually be called by the runtime, since we should be
+        // calling `to_repr` on all children in the list, which requires VM context.
+        write!(fmt, "[")?;
+        for i in 0..self.list.len() - 1 {
+            write!(fmt, "{:?}, ", self.list[i].borrow())?;
+        }
+        if let Some(last) = self.list.last() {
+            write!(fmt, "{}]", last.borrow())
+        } else {
+            write!(fmt, "]")
+        }
+    }
+}
+
+impl Object for List {
+    fn equals(&self, other: &dyn Object) -> bool {
+        if let Some(other) = other.as_any().downcast_ref::<List>() {
+            self.list.len() == other.list.len()
+                && self
+                    .list
+                    .iter()
+                    .zip(other.list.iter())
+                    .all(|(me, you)| me.borrow().equals(&*you.borrow()))
+        } else {
+            false
+        }
+    }
+
+    impl_base_obj!(List);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// List function implementations
+////////////////////////////////////////////////////////////////////////////////
+impl List {
+    pub(crate) fn to_repr(vm: &mut Vm, state: FunctionState) -> FunctionResult {
+        // This function is a bit more complicated than the rest because it needs to effectively
+        // loop over the elements and call them, without using a loop. Thus, we use a sort-of state
+        // machine with the function state.
+        //
+        // When we begin, we check if the list is empty. If that's the case, then we just return
+        // the "empty list" string. Otherwise, we push two "locals", the string we're building, and
+        // the current list index, to the stack. Then, we call `to_repr` on the first item in the
+        // list, and yield execution to the VM with state 0.
+        //
+        // When function resumes, we get the return value off the top of the stack, append it to
+        // the current string, increment the index, and continue, until the string is fully built.
+
+        // This function needs to keep track of the string that we're building, plus the current
+        // index, on the VM stack.
+        let this_ptr = vm.frame_stack()[0].clone();
+        let this = this_ptr.borrow();
+        let this = this.as_any().downcast_ref::<List>().unwrap();
+
+        match state {
+            FunctionState::Begin => {
+                // empty list, exit early
+                if this.list().len() == 0 {
+                    return FunctionResult::ReturnPush(Str::create("[]"));
+                }
+
+                let string = Str::create("[");
+                vm.push(string);
+                let index = Int::create(0);
+                vm.push(index);
+
+                let item = this.list()[0].clone();
+                let method = item
+                    .borrow()
+                    .get_vtable_attr(item.clone(), "to_repr")
+                    .expect("no to_repr");
+                vm.push(method.clone());
+                method.borrow().call(vm, 0);
+                FunctionResult::Yield(0)
+            }
+            FunctionState::Resume(0) => {
+                let build_str = vm.frame_stack()[1].clone();
+                // putting the "1 +" in front so we don't forget that it's there
+                let index =
+                    1 + with_obj_downcast(vm.frame_stack()[2].clone(), Int::int_value) as usize;
+                let repr_str = vm.pop();
+
+                if index == this.list().len() {
+                    // if this is the last item in the list, then we're done
+                    let new_str = format!("{}{}]", build_str.borrow(), repr_str.borrow());
+                    FunctionResult::ReturnPush(Str::create(new_str))
+                } else {
+                    // otherwise, continue building the string and calling to_repr
+                    let new_str = format!("{}{}, ", build_str.borrow(), repr_str.borrow());
+                    vm.frame_stack_mut()[1] = Str::create(new_str);
+                    vm.frame_stack_mut()[2] = Int::create(index as i64);
+                    let item = this.list()[index].clone();
+                    let method = item
+                        .borrow()
+                        .get_vtable_attr(item.clone(), "to_repr")
+                        .expect("no to_repr");
+                    vm.push(method.clone());
+                    method.borrow().call(vm, 0);
+                    FunctionResult::Yield(0)
+                }
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    impl_do_call!(to_list);
+
+    pub(crate) fn init(_vm: &mut Vm, _state: FunctionState) -> FunctionResult {
+        // This is a no-op. We don't want the user-exposed `__init__` function to do anything,
+        // instantiation is done in the `__call__` function.
+        FunctionResult::ReturnPush(Nil::create())
+    }
+
+    pub(crate) fn index(vm: &mut Vm, _state: FunctionState) -> FunctionResult {
+        let this = vm.frame_stack()[0].clone();
+        let index_obj = vm.frame_stack()[1].clone();
+
+        let index = if let Some(index_obj) = index_obj.borrow().as_any().downcast_ref::<Int>() {
+            index_obj.int_value()
+        } else {
+            // TODO List::index - throw an exception when the index object is not an integer
+            // BLOCKED-ON: exceptions
+            todo!("throw an exception when the index object is not an integer")
+        };
+
+        let item = with_obj_downcast(this, |list: &List| {
+            let mut index = index;
+
+            // backtrack index lookup
+            if index < 0 {
+                index = list.list().len() as i64 + index;
+            }
+
+            if index < 0 || index as usize >= list.list().len() {
+                // TODO List::index - throw an exception when the index is out of range
+                // BLOCKED-ON: exceptions
+                todo!("throw an exception when the list index is out of range")
+            } else {
+                list.list()[index as usize].clone()
+            }
+        });
+
+        FunctionResult::ReturnPush(item)
+    }
+
+    pub(crate) fn push(vm: &mut Vm, _state: FunctionState) -> FunctionResult {
+        let this = vm.frame_stack()[0].clone();
+        let arg = vm.frame_stack()[1].clone();
+        with_obj_downcast_mut(this, |list: &mut List| list.list_mut().push(arg));
+        FunctionResult::ReturnPush(Nil::create())
+    }
+
+    pub(crate) fn pop(vm: &mut Vm, _state: FunctionState) -> FunctionResult {
+        let this = vm.frame_stack()[0].clone();
+        let last = if let Some(last) =
+            with_obj_downcast_mut(this, |list: &mut List| list.list_mut().pop())
+        {
+            last
+        } else {
+            // TODO List::pop - throw an exception when the list object is empty
+            // BLOCKED-ON: exceptions
+            todo!("throw an exception when the list is empty and there is nothing to pop")
+        };
+
+        FunctionResult::ReturnPush(last)
+    }
+}

src/obj/ty.rs

@@ -184,6 +184,21 @@ pub fn init_types() {
         Obj {
             //__call__ => BuiltinFunction::create("__call__",
         },
+        List {
+            // Conversion methods
+            to_repr => BuiltinFunction::create("to_repr", List::to_repr, 1),
+
+            // Constructor
+            __call__ => BuiltinFunction::create("__call__", List::do_call, 2),
+            __init__ => BuiltinFunction::create("__init__", List::init, 2),
+
+            // Operators
+            __index__ => BuiltinFunction::create("__index__", List::index, 2),
+
+            // Methods
+            push => BuiltinFunction::create("push", List::push, 2),
+            pop => BuiltinFunction::create("pop", List::pop, 1),
+        },
         Str {
             // Conversion methods
             to_str => BuiltinFunction::create("to_str", Str::to_str, 1),
@@ -198,7 +213,9 @@ pub fn init_types() {
             // Operators
             __add__ => BuiltinFunction::create("__add__", Str::add, 2),
             __mul__ => BuiltinFunction::create("__mul__", Str::mul, 2),
-            // .lower, .upper, .slice, etc
+
+            // Methods
+            // TODO Str methods - .lower, .upper, .slice, etc
         },
         Int {
             // Conversion methods

src/parser.rs

@@ -743,6 +743,16 @@ impl Parser {
                     .expect("expect name after '.'", TokenKind::Name)?
                     .clone();
                 expr = Box::new(GetExpr { expr, name });
+            } else if self.mat(TokenKind::LBracket)? {
+                let index = self.expr()?;
+                let rbracket = self
+                    .expect("expect ']' after index expression", TokenKind::RBracket)?
+                    .clone();
+                expr = Box::new(IndexExpr {
+                    expr,
+                    index,
+                    rbracket,
+                })
             } else {
                 break;
             }
@@ -801,6 +811,8 @@ impl Parser {
                 self.expect("expect ')' after expression", TokenKind::RParen)?;
             }
             Ok(expr)
+        } else if self.mat(TokenKind::LBracket)? {
+            self.list()
         } else {
             Err(self.error(format!("unexpected token {:?}", self.current.kind)))
         }
@@ -854,6 +866,32 @@ impl Parser {
         params.push((name, ty));
         Ok(())
     }
+
+    fn list(&mut self) -> Result<ExprP> {
+        let lbracket = self.prev.clone().unwrap();
+        let mut exprs = Vec::new();
+
+        if !self.check(TokenKind::RBracket) {
+            exprs.push(self.expr()?);
+            while self.mat(TokenKind::Comma)? {
+                // allow trailing comma
+                if self.check(TokenKind::RBracket) {
+                    break;
+                }
+                exprs.push(self.expr()?);
+            }
+        }
+
+        let rbracket = self
+            .expect("expect ']' after list items", TokenKind::RBracket)?
+            .clone();
+
+        Ok(Box::new(ListExpr {
+            lbracket,
+            exprs,
+            rbracket,
+        }))
+    }
 }
 
 #[cfg(test)]

src/vm.rs

@@ -29,6 +29,9 @@ pub enum Op {
     Return,
     CloseOver { depth: ShortOpArg, slot: ShortOpArg },
 
+    // Type-specific instructions
+    BuildList(ListLen),
+
     // VM control
     Nop,
     Halt,
@@ -46,6 +49,7 @@ pub type ConstantId = LongOpArg;
 pub type GlobalId = LongOpArg;
 pub type Argc = LongOpArg;
 pub type FrameDepth = ShortOpArg;
+pub type ListLen = LongOpArg;
 
 #[derive(Debug, Clone)]
 pub struct Local {
@@ -147,6 +151,12 @@ impl Vm {
         &self.stack()[self.frame().stack_base..]
     }
 
+    /// Mutably gets the current stack, starting at the frame's stack base.
+    pub fn frame_stack_mut(&mut self) -> &mut [ObjP] {
+        let index = self.frame().stack_base;
+        &mut self.stack_mut()[index..]
+    }
+
     /// Current stack frame.
     pub fn frame(&self) -> &Frame {
         self.frames.last().unwrap()
@@ -388,6 +398,12 @@ impl Vm {
                     fun.push_capture(value);
                     self.push(make_ptr(fun));
                 }
+                Op::BuildList(len) => {
+                    let index = self.stack().len() - len as usize;
+                    let list_items = self.stack_mut().split_off(index);
+                    let list = List::create(list_items);
+                    self.push(list);
+                }
                 Op::Nop => {
                     continue;
                 }

tools/genast.py

@@ -11,8 +11,10 @@ GENERATE = [
             "Unary      -> op: Token, expr: ExprP",
             "Call       -> expr: ExprP, args: Vec<ExprP>, rparen: Token",
             "Get        -> expr: ExprP, name: Token",
+            "Index      -> expr: ExprP, index: ExprP, rbracket: Token",
             "Primary    -> token: Token",
             "Function   -> lparen: Token, params: Vec<(Token <COMMA> Option<ExprP>)>, return_type: Option<ExprP>, body: Vec<StmtP>, rbrace: Token",
+            "List       -> lbracket: Token, exprs: Vec<ExprP>, rbracket: Token",
         ],
     ),
     (