WIP lexer work

hank/HankLexer.hx

@@ -34,7 +34,8 @@ enum HankToken {
     TTripleComma;
     // Other
     TInclude(p:String);
-    TWord(w:String);
+    TChar(c:String);
+    TWhitespace;
     TEof;
 }
 class N {
@@ -88,12 +89,9 @@ class HankLexer extends Lexer implements RuleBuilder {
             lexer.token(include);
             TInclude(buf.toString());
         },
-        "." => {
-            buf = new StringBuf();
-            lexer.token(word);
-            TWord(buf.toString());
-        },
-        "" => TEof
+        "[ \t]" => TWhitespace,
+        "" => TEof,
+        "." => TChar(lexer.current)
     ];
 
     public static var lineComment = @:rule [
@@ -128,18 +126,4 @@ class HankLexer extends Lexer implements RuleBuilder {
 			lexer.token(include);
         }
     ];
-
-    public static var word = @:rule [
-        '\n' => {
-            lexer.curPos().pmax;
-        },
-        ' ' => {
-            lexer.curPos().pmax;
-        },
-        '[^"]' => {
-			buf.add(lexer.current);
-			lexer.token(include);
-        }
-    ];
-
 }

tests/HankLexerTest.hx

@@ -1,11 +1,9 @@
 package tests;
 
-import haxe.io.Bytes;
 import sys.io.File;
 import byte.ByteData;
 
 import utest.Test;
-import utest.Assert;
 
 import hxparse.LexerTokenSource;
 
@@ -15,6 +13,34 @@ import hank.HankLexer.HankToken;
 import tests.HankAssert;
 
 class HankLexerTest extends utest.Test {
+
+    var nextSaved: HankToken = null;
+
+    /**
+     Helper: Retrieve the next meaningful lexer token (non-whitespace, non-text)
+    **/
+    function next(ts: LexerTokenSource) {
+        if (nextSaved != null) {
+            var temp = nextSaved;
+            nextSaved = null;
+            return temp;
+        }
+        var text = '';
+        var token: HankToken;
+        do {
+            switch (token) {
+                case TChar(c):
+                    text += c;
+            }
+            
+        } while(true);
+
+
+        switch (ts.next()) {
+            case TCh
+            other =>
+        }
+    }
     
     public function testLexMainExample() {
         var lexer = new HankLexer(ByteData.ofString(File.getContent('examples/main/main.hank')), 'testScript');
@@ -22,6 +48,7 @@ class HankLexerTest extends utest.Test {
         HankAssert.equals(TInclude("extra.hank"), ts.token());
         HankAssert.equals(TNewline, ts.token());
         HankAssert.equals(TArrow, ts.token());
+        HankAssert.equals(TWhitespace, ts.token());
         HankAssert.equals(TWord("start"), ts.token());
         HankAssert.equals(TLineComment(" This syntax moves the game flow to a new section."), ts.token());
         for (i in 0...100) {