Add meta token

Signed-off-by: Alek Ratzloff <alekratz@gmail.com>

src/syn/lexer.rs

@@ -8,6 +8,7 @@ lazy_static! {
             (?P<float>[-+]?[0-9]+\.[0-9]+([eE][+\-][0-9]+)?)
         |   (?P<int>[-+]?[0-9]+)
         |   (?P<assign>=[a-zA-Z_?\-*+/.'@$%^&|][0-9a-zA-Z_?\-*+/=.'@$%^&|]*)
+        |   (?P<meta>%[a-zA-Z0-9\-_]+)
         |   (?P<word>[a-zA-Z_?\-*+/=.'@$%^&|][0-9a-zA-Z_?\-*+/=.'@$%^&|]*)
         |   (?P<lquote>\[)
         |   (?P<rquote>\])
@@ -89,6 +90,8 @@ impl<'t> Lexer<'t> {
             self.end = self.end.next_str(cap.get(0).unwrap().as_str());
             let sp_token = if let Some(_) = cap.name("assign") {
                 self.make_token(Token::Assign)
+            } else if let Some(_) = cap.name("meta") {
+                self.make_token(Token::Meta)
             } else if let Some(_) = cap.name("word") {
                 self.make_token(Token::Word)
             } else if let Some(_) = cap.name("float") {
@@ -193,6 +196,19 @@ mod test {
         assert!(lexer.is_eof());
     }
 
+    #[test]
+    fn test_meta() {
+        let mut lexer = Lexer::new(r"%meta %meta1 %include %include1029 %10239meta % %%");
+        assert_token!(lexer, Token::Meta, "%meta");
+        assert_token!(lexer, Token::Meta, "%meta1");
+        assert_token!(lexer, Token::Meta, "%include");
+        assert_token!(lexer, Token::Meta, "%include1029");
+        assert_token!(lexer, Token::Meta, "%10239meta");
+        assert_token!(lexer, Token::Word, "%");
+        assert_token!(lexer, Token::Word, "%%");
+        assert!(lexer.is_eof());
+    }
+
     #[test]
     fn test_numbers() {
         let mut lexer = Lexer::new(r"1 -12 123 -9 98 987 -987654321 1248 9764321 -1.2 2.3");

src/syn/token.rs

@@ -29,6 +29,9 @@ pub enum Token {
 
     /// Apply.
     Apply,
+
+    /// Meta
+    Meta,
 }
 
 impl Token {
@@ -44,6 +47,7 @@ impl Token {
             RQuote => "quote end",
             Colon => "colon",
             Apply => "apply",
+            Meta => "meta",
         }
     }
 }