From ff780c2ad33822827c9fcedca190dead93a5a2dc Mon Sep 17 00:00:00 2001
From: "Michael D. Lowis" <mike@mdlowis.com>
Date: Wed, 11 Jul 2012 23:04:55 -0400
Subject: [PATCH] Added a constructor for the lexer and tests to prove it works
 with buf

---
 source/lexer.scm     |  3 +++
 tests/test_lexer.scm | 30 ++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/source/lexer.scm b/source/lexer.scm
index 11ea599..e74f803 100644
--- a/source/lexer.scm
+++ b/source/lexer.scm
@@ -3,6 +3,9 @@
          (uses parse-utils)
          (uses buf))
 
+(define (dlang/lexer input)
+  (buf (buf input read-char) dlang/tokenize))
+
 (define (dlang/tokenize in)
   (let ((ch (buf-lookahead! in 1)))
     (cond
diff --git a/tests/test_lexer.scm b/tests/test_lexer.scm
index 03dfe8c..99787f6 100644
--- a/tests/test_lexer.scm
+++ b/tests/test_lexer.scm
@@ -2,6 +2,36 @@
 (declare (unit test_lexer)
          (uses lexer))
 
+; dlang/lexer
+;------------------------------------------------------------------------------
+(def-test "dlang/lexer should create a new lexer from the given port"
+  (call-with-input-string "abc"
+    (lambda (input)
+      (define buffer (dlang/lexer input))
+      (and (buf? buffer)
+           (buf? (buf-src buffer))
+           (token? (buf-lookahead! buffer 1))))))
+
+(def-test "dlang/lexer should create a capable of returning a sequence of tokens"
+  (call-with-input-string "abc 123 $foo"
+    (lambda (input)
+      (define buffer (dlang/lexer input))
+      (and (buf? buffer)
+           (buf? (buf-src buffer))
+           (token? (buf-lookahead! buffer 1))
+           (token? (buf-lookahead! buffer 2))
+           (token? (buf-lookahead! buffer 3))))))
+
+(def-test "dlang/lexer should create a lexer that returns the #!eof when EOF reached"
+  (call-with-input-string "abc"
+    (lambda (input)
+      (define buffer (dlang/lexer input))
+      (and (buf? buffer)
+           (buf? (buf-src buffer))
+           (token? (buf-lookahead! buffer 1))
+           (eof-object? (buf-lookahead! buffer 2))
+           (eof-object? (buf-lookahead! buffer 3))))))
+
 ; dlang/tokenize
 ;------------------------------------------------------------------------------
 (def-test "dlang/tokenize should recognize whitespace"
-- 
2.54.0