WIP more lexer work/testing

hank/HankLexer.hx

@@ -34,8 +34,12 @@ enum HankToken {
     TTripleComma;
     // Other
     TInclude(p:String);
+    TWord(w:String);
     TEof;
 }
+class N {
+    public static var n = 0;
+}
 
 /**
  Lexer for valid tokens inside a Hank script.
@@ -84,15 +88,19 @@ class HankLexer extends Lexer implements RuleBuilder {
             lexer.token(include);
             TInclude(buf.toString());
         },
+        "." => {
+            buf = new StringBuf();
+            lexer.token(word);
+            TWord(buf.toString());
+        },
         "" => TEof
     ];
 
     public static var lineComment = @:rule [
-        '\\n' => {
+        '\n' => {
             lexer.curPos().pmax;
         },
         '[^"]' => {
-            trace('heyyyyya');
 			buf.add(lexer.current);
 			lexer.token(lineComment);
         }
@@ -100,11 +108,9 @@ class HankLexer extends Lexer implements RuleBuilder {
 
     public static var blockComment = @:rule [
         '\\*/' => {
-            trace("OUT");
             lexer.curPos().pmax;
         },
         '[^"]' => {
-            trace(lexer.current);
 			buf.add(lexer.current);
 			lexer.token(blockComment);
         }
@@ -118,7 +124,19 @@ class HankLexer extends Lexer implements RuleBuilder {
             lexer.curPos().pmax;
         },
         '[^"]' => {
-            trace(lexer.current);
+			buf.add(lexer.current);
+			lexer.token(include);
+        }
+    ];
+
+    public static var word = @:rule [
+        '\n' => {
+            lexer.curPos().pmax;
+        },
+        ' ' => {
+            lexer.curPos().pmax;
+        },
+        '[^"]' => {
 			buf.add(lexer.current);
 			lexer.token(include);
         }

tests/HankLexerTest.hx

@@ -20,5 +20,12 @@ class HankLexerTest extends utest.Test {
         var lexer = new HankLexer(ByteData.ofString(File.getContent('examples/main/main.hank')), 'testScript');
         var ts = new LexerTokenSource(lexer, HankLexer.tok);
         HankAssert.equals(TInclude("extra.hank"), ts.token());
+        HankAssert.equals(TNewline, ts.token());
+        HankAssert.equals(TArrow, ts.token());
+        HankAssert.equals(TWord("start"), ts.token());
+        HankAssert.equals(TLineComment(" This syntax moves the game flow to a new section."), ts.token());
+        for (i in 0...100) {
+            trace(ts.token());
+        }
     }
 }