From 4a7644b84a066524e5e01992baae1f33fe04a91b Mon Sep 17 00:00:00 2001 From: Alek Ratzloff Date: Thu, 3 Oct 2024 11:37:00 -0700 Subject: [PATCH] Update AST generator to be more structured We use objects and builders to make AST items now, it's a lot less clunky that doing string manipulation everywhere in the Python file. It's also more flexible and will open up more options in the future for things like enum AST items Signed-off-by: Alek Ratzloff --- tools/genast.py | 293 ++++++++++++++++++++++++++++++------------------ 1 file changed, 183 insertions(+), 110 deletions(-) diff --git a/tools/genast.py b/tools/genast.py index 793b485..cacd676 100755 --- a/tools/genast.py +++ b/tools/genast.py @@ -1,112 +1,187 @@ #!/usr/bin/env python3 +import abc import sys import time from pathlib import Path +from typing import Self + + +class SB: + "String builder" + + def __init__(self, init: str = ""): + self.value = init + self.padding = 0 + + def indent(self) -> None: + self.padding += 1 + + def dedent(self) -> None: + self.padding -= 1 + assert self.padding >= 0 + + def line(self, line: str = ""): + self.write(f"{line}\n") + + def write(self, value: str): + if not self.value or self.value[-1] == "\n": + self.value += " " * self.padding + self.value += value + + def __str__(self) -> str: + return self.value + + +class Generator(metaclass=abc.ABCMeta): + @abc.abstractmethod + def generate_members(self) -> str: + pass + + @abc.abstractmethod + def generate_visitor(self) -> str: + pass + + +class GenerateStruct(Generator): + def __init__(self, base: str, name: str, rules: list[str]) -> None: + self.base = base + self.name = name + self.rules = rules + + def generate_members(self) -> str: + sb = SB() + sb.line("#[derive(Debug)]") + sb.line(f"pub struct {self.name}{self.base} " + "{") + sb.indent() + for member in self.rules: + sb.write(f"pub {member}") + sb.line(",") + sb.dedent() + sb.line("}") + sb.line() + + sb.line(f"impl {self.base} for {self.name}{self.base} " + "{") + sb.indent() + sb.line( + f"fn accept(&self, visitor: &mut dyn {self.base}Visitor) -> Result<(), Box> " + + "{" + ) + sb.indent() + sb.line(f"visitor.visit_{self.name.lower()}_{self.base.lower()}(self)") + + sb.dedent() + sb.line("}") + sb.line() + sb.line("fn as_any(self: Box) -> Box { self }") + sb.line("fn as_any_ref(&self) -> &dyn Any { self }") + + sb.dedent() + sb.write("}") + + return str(sb) + + def generate_visitor(self) -> str: + return f"fn visit_{self.name.lower()}_{self.base.lower()}(&mut self, {self.base.lower()}: &{self.name}{self.base}) -> Result<(), Box>;" + + +class GenerateGroup(Generator): + def __init__(self, name: str, group: list[Generator] | None = None) -> None: + self.name = name + if not group: + group = [] + self.group = group + + def generate(self) -> str: + sb = SB() + + sb.line(self.generate_visitor()) + sb.line(self.generate_members()) + + return str(sb) + + def add_struct(self, name: str, rules: list[str]) -> Self: + self.group += [GenerateStruct(self.name, name, rules)] + return self + + def generate_members(self) -> str: + sb = SB() + + # Trait and acceptor + sb.line(f"pub trait {self.name}: Debug + Any " + "{") + sb.indent() + sb.line( + f"fn accept(&self, visitor: &mut dyn {self.name}Visitor) -> Result<(), Box>;" + ) + sb.line("fn as_any(self: Box) -> Box;") + sb.line("fn as_any_ref(&self) -> &dyn Any;") + sb.dedent() + sb.line("}") + sb.line() + + # Type + sb.line(f"pub type {self.name}P = Box;") + + # All members and rules + for g in self.group: + sb.line(g.generate_members()) + sb.line() + + return str(sb) + + def generate_visitor(self) -> str: + sb = SB() + + # Visitor + sb.line(f"pub trait {self.name}Visitor " + "{") + sb.indent() + for g in self.group: + sb.line(g.generate_visitor()) + sb.dedent() + sb.line("}") + + return str(sb) + GENERATE = [ - ( - "Expr", + # Expr + GenerateGroup("Expr") + .add_struct("Binary", ["lhs: ExprP", "op: Token", "rhs: ExprP"]) + .add_struct("Unary", ["op: Token", "expr: ExprP"]) + .add_struct("Call", ["expr: ExprP", "args: Vec", "rparen: Token"]) + .add_struct("Get", ["expr: ExprP", "name: Token"]) + .add_struct("Index", ["expr: ExprP", "index: ExprP", "rbracket: Token"]) + .add_struct("Primary", ["token: Token"]) + .add_struct( + "Function", [ - "Binary -> lhs: ExprP, op: Token, rhs: ExprP", - "Unary -> op: Token, expr: ExprP", - "Call -> expr: ExprP, args: Vec, rparen: Token", - "Get -> expr: ExprP, name: Token", - "Index -> expr: ExprP, index: ExprP, rbracket: Token", - "Primary -> token: Token", - "Function -> lparen: Token, params: Vec<(Token Option)>, return_type: Option, body: Vec, rbrace: Token", - "List -> lbracket: Token, exprs: Vec, rbracket: Token", + "lparen: Token", + "params: Vec<(Token, Option)>", + "return_type: Option", + "body: Vec", + "rbrace: Token", ], - ), - ( - "Stmt", + ) + .add_struct("List", ["lbracket: Token", "exprs: Vec", "rbracket: Token"]), + # Stmt + GenerateGroup("Stmt") + .add_struct("Import", ["import_kw: Token", "module: Token"]) + .add_struct("Expr", ["expr: ExprP"]) + .add_struct("Assign", ["lhs: Token", "rhs: ExprP"]) + .add_struct("Set", ["expr: ExprP", "name: Token", "rhs: ExprP"]) + .add_struct("Block", ["lbrace: Token", "stmts: Vec", "rbrace: Token"]) + .add_struct("Return", ["return_kw: Token", "expr: Option"]) + .add_struct( + "If", [ - "Expr -> expr: ExprP", - "Assign -> lhs: Token, rhs: ExprP", - "Set -> expr: ExprP, name: Token, rhs: ExprP", - "Block -> lbrace: Token, stmts: Vec, rbrace: Token", - "Return -> return_kw: Token, expr: Option", - "If -> if_kw: Token, condition: ExprP, then_branch: BlockStmt, else_branch: Vec", + "if_kw: Token", + "condition: ExprP", + "then_branch: BlockStmt", + "else_branch: Vec", ], ), ] -class FileWriter: - def __init__(self, path: Path) -> None: - self.path = path - self.fp = open(path, "w") - - def write(self, text: str = "") -> None: - self.fp.write(text) - - def line(self, line: str = "") -> None: - self.fp.write(line) - self.fp.write("\n") - - -def define_visitor(wr: FileWriter, base: str, type_lines: list[str]): - types = [] - for line in type_lines: - parts = line.split("->") - members = [p.strip().replace("", ",") for p in parts[1].split(",")] - types += [(parts[0].strip(), members)] - - wr.line(f"pub trait {base}Visitor " + "{") - - for type, members in types: - wr.line( - f" fn visit_{type.lower()}_{base.lower()}(&mut self, {base.lower()}: &{type}{base}) -> Result<(), Box>;" - ) - wr.line("}") - wr.line() - - -def define_ast(wr: FileWriter, base: str, type_lines: list[str]): - types = [] - for line in type_lines: - parts = line.split("->") - members = [p.strip().replace("", ",") for p in parts[1].split(",")] - types += [(parts[0].strip(), members)] - - visitor = f"{base}Visitor" - - wr.line(f"pub trait {base}: Debug + Any " + "{") - wr.line( - f" fn accept(&self, visitor: &mut dyn {visitor}) -> Result<(), Box>;" - ) - wr.line(" fn as_any(self: Box) -> Box;") - wr.line(" fn as_any_ref(&self) -> &dyn Any;") - wr.line("}") - wr.line() - wr.line(f"pub type {base}P = Box;") - wr.line() - - for type, members in types: - wr.line("#[derive(Debug)]") - wr.line(f"pub struct {type}{base} " + "{") - for member in members: - wr.line(f" pub {member},") - wr.line("}") - wr.line() - wr.line(f"impl {base} for {type}{base} " + "{") - wr.line( - f" fn accept(&self, visitor: &mut dyn {visitor}) -> Result<(), Box> " - + "{" - ) - wr.line(f" visitor.visit_{type.lower()}_{base.lower()}(self)") - wr.line(" }") - wr.line() - wr.line(" fn as_any(self: Box) -> Box {") - wr.line(" self") - wr.line(" }") - wr.line() - wr.line(" fn as_any_ref(&self) -> &dyn Any {") - wr.line(" self") - wr.line(" }") - wr.line("}") - wr.line() - - def main(): if len(sys.argv) == 1: path = Path(__file__).parent / "../src/ast.rs" @@ -116,29 +191,27 @@ def main(): path = path.resolve() print(f"Using {path}") - wr = FileWriter(path) + sb = SB() # File headers - wr.line( + sb.line( "// This is an auto-generated file. Any changes made to this file may be overwritten." ) - wr.line("// This file was created at: " + time.strftime("%Y-%m-%d %H:%M:%S")) - wr.line("#![allow(dead_code)]") + sb.line("// This file was created at: " + time.strftime("%Y-%m-%d %H:%M:%S")) + sb.line("#![allow(dead_code)]") # Imports - wr.line("use std::fmt::Debug;") - wr.line("use std::any::Any;") - wr.line() - wr.line("use crate::token::Token;") - wr.line() + sb.line("use std::fmt::Debug;") + sb.line("use std::any::Any;") + sb.line() + sb.line("use crate::token::Token;") + sb.line() - # Visitors - for base, types in GENERATE: - define_visitor(wr, base, types) + for g in GENERATE: + sb.line(g.generate()) - # AST - for base, types in GENERATE: - define_ast(wr, base, types) + with open(path, "w") as fp: + fp.write(str(sb).strip()) if __name__ == "__main__":