Initial commit with lexer

Signed-off-by: Alek Ratzloff <alekratz@gmail.com>

src/syn/token.rs

@@ -0,0 +1,68 @@
+use crate::syn::span::*;
+use std::fmt::{Display, Formatter, self};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TokenKind {
+    Ident,
+    Num,
+    Str,
+    Sym,
+
+    LParen,
+    RParen,
+    LBrace,
+    RBrace,
+    LBracket,
+    RBracket,
+    Comma,
+
+    Eq,
+    Arrow,
+
+    KwReturn,
+}
+
+impl Display for TokenKind {
+    fn fmt(&self, fmt: &mut Formatter) -> fmt::Result {
+        use TokenKind::*;
+        let s = match self {
+            Ident => "identifier",
+            Num => "number",
+            Str => "string",
+            Sym => "symbol",
+
+            LParen => "left paren",
+            RParen => "right paren",
+            LBrace => "left brace",
+            RBrace => "right brace",
+            LBracket => "left bracket",
+            RBracket => "right bracket",
+            Comma => "comma",
+
+            Eq => "equals",
+            Arrow => "arrow",
+
+            KwReturn => "return keyword",
+        };
+        Display::fmt(s, fmt)
+    }
+}
+
+pub struct Token {
+    kind: TokenKind,
+    span: Span,
+}
+
+impl Token {
+    pub fn new(kind: TokenKind, span: Span) -> Self {
+        Token { kind, span, }
+    }
+
+    pub fn kind(&self) -> TokenKind {
+        self.kind
+    }
+}
+
+impl Spanned for Token {
+    fn span(&self) -> Span { self.span }
+}