Added a constructor for the lexer and tests to prove it works with buf

diff --git a/source/lexer.scm b/source/lexer.scm
index 11ea5992cd2f7316521918715063b32a3538cdf3..e74f80344395370ddbe1d7628249b4fed8f5f13f 100644 (file)
--- a/source/lexer.scm
+++ b/source/lexer.scm
@@ -3,6 +3,9 @@
          (uses parse-utils)
          (uses buf))
 
+(define (dlang/lexer input)
+  (buf (buf input read-char) dlang/tokenize))
+
 (define (dlang/tokenize in)
   (let ((ch (buf-lookahead! in 1)))
     (cond

diff --git a/tests/test_lexer.scm b/tests/test_lexer.scm
index 03dfe8c145d812a1e8bbeb7abb99814302904d8e..99787f6a300b7004c3cff5211a6410bd8c32047d 100644 (file)
--- a/tests/test_lexer.scm
+++ b/tests/test_lexer.scm
@@ -2,6 +2,36 @@
 (declare (unit test_lexer)
          (uses lexer))
 
+; dlang/lexer
+;------------------------------------------------------------------------------
+(def-test "dlang/lexer should create a new lexer from the given port"
+  (call-with-input-string "abc"
+    (lambda (input)
+      (define buffer (dlang/lexer input))
+      (and (buf? buffer)
+           (buf? (buf-src buffer))
+           (token? (buf-lookahead! buffer 1))))))
+
+(def-test "dlang/lexer should create a capable of returning a sequence of tokens"
+  (call-with-input-string "abc 123 $foo"
+    (lambda (input)
+      (define buffer (dlang/lexer input))
+      (and (buf? buffer)
+           (buf? (buf-src buffer))
+           (token? (buf-lookahead! buffer 1))
+           (token? (buf-lookahead! buffer 2))
+           (token? (buf-lookahead! buffer 3))))))
+
+(def-test "dlang/lexer should create a lexer that returns the #!eof when EOF reached"
+  (call-with-input-string "abc"
+    (lambda (input)
+      (define buffer (dlang/lexer input))
+      (and (buf? buffer)
+           (buf? (buf-src buffer))
+           (token? (buf-lookahead! buffer 1))
+           (eof-object? (buf-lookahead! buffer 2))
+           (eof-object? (buf-lookahead! buffer 3))))))
+
 ; dlang/tokenize
 ;------------------------------------------------------------------------------
 (def-test "dlang/tokenize should recognize whitespace"