diff --git a/hank/HankLexer.hx b/hank/HankLexer.hx index eee0d93..44d158d 100644 --- a/hank/HankLexer.hx +++ b/hank/HankLexer.hx @@ -34,7 +34,8 @@ enum HankToken { TTripleComma; // Other TInclude(p:String); - TWord(w:String); + TChar(c:String); + TWhitespace; TEof; } class N { @@ -88,12 +89,9 @@ class HankLexer extends Lexer implements RuleBuilder { lexer.token(include); TInclude(buf.toString()); }, - "." => { - buf = new StringBuf(); - lexer.token(word); - TWord(buf.toString()); - }, - "" => TEof + "[ \t]" => TWhitespace, + "" => TEof, + "." => TChar(lexer.current) ]; public static var lineComment = @:rule [ @@ -128,18 +126,4 @@ class HankLexer extends Lexer implements RuleBuilder { lexer.token(include); } ]; - - public static var word = @:rule [ - '\n' => { - lexer.curPos().pmax; - }, - ' ' => { - lexer.curPos().pmax; - }, - '[^"]' => { - buf.add(lexer.current); - lexer.token(include); - } - ]; - } \ No newline at end of file diff --git a/tests/HankLexerTest.hx b/tests/HankLexerTest.hx index 23f4fe2..30fd1bf 100644 --- a/tests/HankLexerTest.hx +++ b/tests/HankLexerTest.hx @@ -1,11 +1,9 @@ package tests; -import haxe.io.Bytes; import sys.io.File; import byte.ByteData; import utest.Test; -import utest.Assert; import hxparse.LexerTokenSource; @@ -15,6 +13,34 @@ import hank.HankLexer.HankToken; import tests.HankAssert; class HankLexerTest extends utest.Test { + + var nextSaved: HankToken = null; + + /** + Helper: Retrieve the next meaningful lexer token (non-whitespace, non-text) + **/ + function next(ts: LexerTokenSource) { + if (nextSaved != null) { + var temp = nextSaved; + nextSaved = null; + return temp; + } + var text = ''; + var token: HankToken; + do { + switch (token) { + case TChar(c): + text += c; + } + + } while(true); + + + switch (ts.next()) { + case TCh + other => + } + } public function testLexMainExample() { var lexer = new HankLexer(ByteData.ofString(File.getContent('examples/main/main.hank')), 'testScript'); @@ -22,6 +48,7 @@ class HankLexerTest extends utest.Test { HankAssert.equals(TInclude("extra.hank"), ts.token()); HankAssert.equals(TNewline, ts.token()); HankAssert.equals(TArrow, ts.token()); + HankAssert.equals(TWhitespace, ts.token()); HankAssert.equals(TWord("start"), ts.token()); HankAssert.equals(TLineComment(" This syntax moves the game flow to a new section."), ts.token()); for (i in 0...100) {