Finished unit tests for lexer

diff --git a/source/lexer.scm b/source/lexer.scm
index 5e0eaf7128c8fdd49a0cb456a32532577a7a0486..11ea5992cd2f7316521918715063b32a3538cdf3 100644 (file)
--- a/source/lexer.scm
+++ b/source/lexer.scm
@@ -83,7 +83,7 @@
 
 (define (dlang/exponent in)
   (string-append
-    ;(string (char-match-one-of in '(#\e #\E)))
+    ;(string (char-match-one-of in "eE"))
     (string
       (if (char=? (buf-lookahead! in 1) #\e)
         (char-match in #\e) (char-match in #\E)))
@@ -95,15 +95,19 @@
   (token 'character
     (string-append
       (string (char-match in #\'))
-      (string (buf-consume! in))
+      (if (eof-object? (buf-lookahead! in 1))
+        (error "Unexpected EOF while parsing character literal")
+        (string (buf-consume! in)))
       (string (char-match in #\')) )))
 
 (define (dlang/string in)
-  (token 'string
-    (string-append
-      (string (char-match in #\"))
-      ;(accumulate-till in string-append "" #\")
-      (string (char-match in #\")) )))
+  (define text (string (char-match in #\")))
+  (while (and (not (eof-object? (buf-lookahead! in 1)))
+              (not (char=? #\newline (buf-lookahead! in 1)))
+              (not (char=? #\" (buf-lookahead! in 1))))
+    (set! text (string-append text (string (buf-consume! in)))))
+  (set! text (string-append text (string (char-match in #\"))))
+  (token 'string text))
 
 (define (dlang/symbol in)
   (token 'symbol

diff --git a/tests/test_lexer.scm b/tests/test_lexer.scm
index 80a7c960178653d9d16c6486f4288e9040580bc7..03dfe8c145d812a1e8bbeb7abb99814302904d8e 100644 (file)
--- a/tests/test_lexer.scm
+++ b/tests/test_lexer.scm
@@ -385,9 +385,94 @@
 
 ; dlang/character
 ;------------------------------------------------------------------------------
+(def-test "dlang/character should recognize a character"
+  (call-with-input-string "'a'"
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (define result (dlang/character buffer))
+      (and (token? result)
+           (equal? 'character (token-type result))
+           (equal? "'a'" (token-text result))))))
+
+(def-test "dlang/character should error when missing first single quote"
+  (call-with-input-string "a'"
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (check-error "Expected ''', received 'a' instead"
+        (dlang/character buffer)))))
+
+(def-test "dlang/character should error when missing second single quote"
+  (call-with-input-string "'a"
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (check-error "Expected ''', received EOF instead"
+        (dlang/character buffer)))))
+
+(def-test "dlang/character should error when EOF reached"
+  (call-with-input-string "'"
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (check-error "Unexpected EOF while parsing character literal"
+        (dlang/character buffer)))))
 
 ; dlang/string
 ;------------------------------------------------------------------------------
+(def-test "dlang/string should recognize an empty string"
+  (call-with-input-string "\"\""
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (define result (dlang/string buffer))
+      (and (token? result)
+           (equal? 'string (token-type result))
+           (equal? "\"\"" (token-text result))))))
+
+(def-test "dlang/string should recognize a string of length 1"
+  (call-with-input-string "\"a\""
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (define result (dlang/string buffer))
+      (and (token? result)
+           (equal? 'string (token-type result))
+           (equal? "\"a\"" (token-text result))))))
+
+(def-test "dlang/string should recognize a string of length 2"
+  (call-with-input-string "\"ab\""
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (define result (dlang/string buffer))
+      (and (token? result)
+           (equal? 'string (token-type result))
+           (equal? "\"ab\"" (token-text result))))))
+
+(def-test "dlang/string should recognize a string of length 3"
+  (call-with-input-string "\"abc\""
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (define result (dlang/string buffer))
+      (and (token? result)
+           (equal? 'string (token-type result))
+           (equal? "\"abc\"" (token-text result))))))
+
+(def-test "dlang/string should error when missing first double quote"
+  (call-with-input-string "a\""
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (check-error "Expected '\"', received 'a' instead"
+        (dlang/string buffer)))))
+
+(def-test "dlang/string should error when missing second double quote"
+  (call-with-input-string "\"a"
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (check-error "Expected '\"', received EOF instead"
+        (dlang/string buffer)))))
+
+(def-test "dlang/string should error when EOF reached"
+  (call-with-input-string "\""
+    (lambda (input)
+      (define buffer (buf input read-char))
+      (check-error "Expected '\"', received EOF instead"
+        (dlang/string buffer)))))
 
 ; dlang/symbol
 ;------------------------------------------------------------------------------