WIP implementing a Hank lexer

Co-authored-by: Zicklag <zicklag@katharostech.com>

hank/HankLexer.hx

@@ -0,0 +1,103 @@
+// This file is based on @Zicklag's gist: https://gist.github.com/zicklag/c2a6060452759ce13864e43135e856f3
+
+package hank;
+
+import hxparse.Lexer;
+import hxparse.RuleBuilder;
+
+enum HankToken {
+    // Brackets
+    TParenOpen;
+    TParenClose;
+    TSquareOpen;
+    TSquareClose;
+    TCurlyOpen;
+    TCurlyClose;
+    // Symbols
+    TStar;
+    TPlus;
+    TDash;
+    TBang;
+    TTilde;
+    TArrow;
+    TEqual;
+    TDoubleEqual;
+    TTripleEqual;
+    TGlue;
+    TLineComment(s:String);
+    TBlockComment(s:String);
+    TNewline;
+    // HInterface
+    TBacktick;
+    TTripleBacktick;
+    TComma;
+    TTripleComma;
+    // Other
+    TEof;
+}
+
+/**
+ Lexer for valid tokens inside a Hank script.
+**/
+class HankLexer extends Lexer implements RuleBuilder {
+    static var buf: StringBuf;
+
+    public static var tok = @:rule [
+        // Brackets
+        "(" => TParenOpen,
+        ")" => TParenClose,
+        "[" => TSquareOpen,
+        "]" => TSquareClose,
+        "{" => TCurlyOpen,
+        "}" => TCurlyClose,
+        // Symbols
+        "*" => TStar,
+        "+" => TPlus,
+        "-" => TDash,
+        "!" => TBang,
+        "~" => TTilde,
+        "->" => TArrow,
+        "=" => TEqual,
+        "==" => TDoubleEqual,
+        "===" => TTripleEqual,
+        "<>" => TGlue,
+        "//" => {
+            buf = new StringBuf();
+            lexer.token(lineComment);
+            TLineComment(buf.toString());
+        },
+        "/*" => {
+            buf = new StringBuf();
+            lexer.token(blockComment);
+            TBlockComment(buf.toString());
+        },
+        "\n" => TNewline,
+        // HInterface
+        "`" => TBacktick,
+        "```" => TTripleBacktick,
+        "," => TComma,
+        ",,," => TTripleComma,
+        // Other
+        "" => TEof
+    ];
+
+    public static var lineComment = @:rule [
+        '\n' => {
+            lexer.curPos().pmax;
+        },
+        '[^"]' => {
+			buf.add(lexer.current);
+			lexer.token(lineComment);
+        }
+    ];
+
+    public static var blockComment = @:rule [
+        '*/' => {
+            lexer.curPos().pmax;
+        },
+        '[^"]' => {
+			buf.add(lexer.current);
+			lexer.token(blockComment);
+        }
+    ];
+}

haxelib.json

@@ -6,9 +6,10 @@
     "description": "Narrative scripting language for HaxeFlixel games based on Inkle's Ink engine",
     "version": "0.0.6",
     "releasenote": "It isn't safe to use this library yet.",
-    "contributors": ["NQNStudios"],
+    "contributors": ["NQNStudios", "Zicklag"],
     "dependencies": {
         "hscript": "",
+        "hxparse": "",
         "utest": ""
     }
 }

test.hxml

@@ -1,4 +1,5 @@
 -lib hscript
 -lib utest
+-lib hxparse
 -main tests.TestMain
 --interp

tests/HankLexerTest.hx

@@ -0,0 +1,23 @@
+package tests;
+
+import haxe.io.Bytes;
+import sys.io.File;
+import byte.ByteData;
+
+import utest.Test;
+import utest.Assert;
+
+import hxparse.LexerTokenSource;
+
+import hank.HankLexer;
+
+
+class HankLexerTest extends utest.Test {
+    
+    public function testLexMainExample() {
+        trace('constructing');
+        var lexer = new HankLexer(ByteData.ofString(File.getContent('examples/main/main.hank')), 'testScript');
+        var ts = new LexerTokenSource(lexer, HankLexer.tok);
+        trace(ts.token());
+    }
+}

tests/TestMain.hx

@@ -3,6 +3,6 @@ import utest.Test;
 
 class TestMain extends Test {
     public static function main() {
-        utest.UTest.run([new HInterfaceTest()]);
+        utest.UTest.run([new HankLexerTest(), new HInterfaceTest()]);
     }
 }