Updated tests for lexer

diff --git a/source/dllexer/dllexer.cpp b/source/dllexer/dllexer.cpp
index 98c9d8c93bcdebaf454dcdec921ef65d836f5e5b..03827a416a5d8a3a3c34df3bd35c08f32be43ed9 100644 (file)
--- a/source/dllexer/dllexer.cpp
+++ b/source/dllexer/dllexer.cpp
@@ -16,12 +16,6 @@ bool DLLexer::isWhiteSpace(void)
            (lookahead(1) == '\n');
 }
 
-bool DLLexer::isLetter(void)
-{
-    return  ((lookahead(1) >= 'a') && (lookahead(1) <= 'z')) ||
-            ((lookahead(1) >= 'A') && (lookahead(1) <= 'Z'));
-}
-
 bool DLLexer::isDigit(char lach)
 {
     return ((lach >= '0') && (lach <= '9'));
@@ -34,6 +28,15 @@ bool DLLexer::isStringChar(void)
              && (lookahead(1) != '\n'));
 }
 
+bool DLLexer::isIDChar(void)
+{
+    return ( !isWhiteSpace() &&
+            ('(' != lookahead(1)) &&
+            (')' != lookahead(1)) &&
+            ('#' != lookahead(1)) &&
+            (EOF != lookahead(1)) );
+}
+
 void DLLexer::terminator(std::string term)
 {
     terminator_string = term;
@@ -100,13 +103,13 @@ Token DLLexer::next(void)
         // Consume parentheses
         else if (lookahead(1) == '(')
         {
-            ret = Token( LPAR, "(", line, column );
             consume();
+            ret = Token( LPAR, "(", line, column );
         }
         else if (lookahead(1) == ')')
         {
-            ret = Token( RPAR, ")", line, column );
             consume();
+            ret = Token( RPAR, ")", line, column );
         }
 
         // Everything else (except the unescaped terminator) is considered an ID
@@ -157,11 +160,12 @@ void DLLexer::Id(Token& tok)
         oss << lookahead(1);
         consume();
     }
-    while( !isWhiteSpace() &&
-           ('(' != lookahead(1)) &&
-           (')' != lookahead(1)) &&
-           ('#' != lookahead(1)) &&
-           (EOF != lookahead(1)) );
+    while( isIDChar() );
+    //while( !isWhiteSpace() &&
+    //       ('(' != lookahead(1)) &&
+    //       (')' != lookahead(1)) &&
+    //       ('#' != lookahead(1)) &&
+    //       (EOF != lookahead(1)) );
     //while(isLetter() || isDigit() || lookahead(1) == '_');
     tok = Token(ID, oss.str(), line, column);
 }
@@ -302,7 +306,7 @@ void DLLexer::Symbol(Token& tok)
         oss << lookahead(1);
         consume();
     }
-    while(isLetter() || isDigit(lookahead(1)) || lookahead(1) == '_');
+    while( isIDChar() );
     tok = Token( SYMBOL, oss.str(), line, column );
 }
 

diff --git a/source/dllexer/dllexer.h b/source/dllexer/dllexer.h
index 391e5fbbcbd126555b7f67ffb3a80e023c7db7cb..a068bede96536c4963dc38c01402b38f9970790d 100644 (file)
--- a/source/dllexer/dllexer.h
+++ b/source/dllexer/dllexer.h
@@ -47,7 +47,7 @@ class DLLexer : public LLNLexer {
     public:
         DLLexer(std::istream& in);
         bool isWhiteSpace(void);
-        bool isLetter(void);
+        bool isIDChar(void);
         bool isDigit(char lach);
         bool isStringChar(void);
         void WS(void);

diff --git a/tests/test_dllexer.cpp b/tests/test_dllexer.cpp
index bd30d0cc33591ef1343abb826810c79a49b7ba5f..84619f9c88998f6a8e4efd1e92d3c0ac65893477 100644 (file)
--- a/tests/test_dllexer.cpp
+++ b/tests/test_dllexer.cpp
@@ -216,61 +216,76 @@ namespace {
         delete lexer;
     }
 
-    ////-------------------------------------------------------------------------
-    //// Test String Recognition
-    ////-------------------------------------------------------------------------
-    //TEST(Recognize_Valid_Strings)
-    //{
-    //    std::string input(
-    //        // Make Sure we recognize all valid characters for a symbol
-    //        "\"\" \n"
-    //        "\"a\" \n"
-    //        "\"\\a\" \n"
-    //    );
-    //    eTokenTypes expected[] = {
-    //        STRING, STRING, STRING, (eTokenTypes)EOF
-    //    };
-    //    TestLexerWithInput( input, expected );
-    //}
+    //-------------------------------------------------------------------------
+    // Test String Recognition
+    //-------------------------------------------------------------------------
+    TEST(Recognize_Valid_Strings)
+    {
+        DLLexer* lexer = SetupLexer(
+            // Make Sure we recognize all valid characters for a symbol
+            "\"\" \n"
+            "\"a\" \n"
+            "\"\\a\" \n"
+        );
+        CHECK_TOKEN( STRING, "",     1,   2 );
+        CHECK_TOKEN( STRING, "a",    2,   3 );
+        CHECK_TOKEN( STRING, "\\a",  3,   4 );
+        CHECK_TOKEN( EOF,    "",    -1,  -1 );
+        delete lexer;
+    }
 
-    ////-------------------------------------------------------------------------
-    //// Test Symbol Recognition
-    ////-------------------------------------------------------------------------
-    //TEST(Recognize_Valid_Symbols)
-    //{
-    //    std::string input(
-    //        // Make Sure we recognize all valid characters for a symbol
-    //        "$abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_\n"
-    //        "$ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_\n"
-    //        "$a_123\n"
-    //        "$a123\n"
-    //        "$a_\n"
-    //        "$a_a\n"
-    //    );
-    //    eTokenTypes expected[] = {
-    //        SYMBOL, SYMBOL, SYMBOL, SYMBOL, SYMBOL, SYMBOL, (eTokenTypes)EOF
-    //    };
-    //    TestLexerWithInput( input, expected );
-    //}
+    //-------------------------------------------------------------------------
+    // Test Symbol Recognition
+    //-------------------------------------------------------------------------
+    TEST(Recognize_Valid_Symbols)
+    {
+        DLLexer* lexer = SetupLexer(
+            // Make Sure we recognize all valid characters for a symbol
+            "$abc\n"
+            "$ABC\n"
+            "$a_123\n"
+            "$a123\n"
+            "$a_\n"
+            "$a_a\n"
+        );
+        CHECK_TOKEN( SYMBOL, "abc",    1,   4 );
+        CHECK_TOKEN( SYMBOL, "ABC",    2,   4 );
+        CHECK_TOKEN( SYMBOL, "a_123",  3,   6 );
+        CHECK_TOKEN( SYMBOL, "a123",   4,   5 );
+        CHECK_TOKEN( SYMBOL, "a_",     5,   3 );
+        CHECK_TOKEN( SYMBOL, "a_a",    6,   4 );
+        CHECK_TOKEN( EOF,    "",       -1,  -1 );
+        delete lexer;
+    }
 
-    ////-------------------------------------------------------------------------
-    //// Recognize Valid IDs
-    ////-------------------------------------------------------------------------
-    //TEST(Recognize_Valid_IDs)
-    //{
-    //    std::string input(
-    //        // Make Sure we recognize all valid characters for an ID
-    //        "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_\n"
-    //        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_\n"
-    //        "a_123\n"
-    //        "a123\n"
-    //        "a_\n"
-    //        "a_a\n"
-    //    );
-    //    eTokenTypes expected[] = { ID, ID, ID, ID, ID, ID, (eTokenTypes)EOF };
-    //    TestLexerWithInput( input, expected );
-    //}
+    //-------------------------------------------------------------------------
+    // Test Expression Terminator Recognition
+    //-------------------------------------------------------------------------
+    TEST(Recognize_Default_Expression_Terminator)
+    {
+        DLLexer* lexer = SetupLexer("end");
+        CHECK_TOKEN( TERM, "end",  1,  3 );
+        CHECK_TOKEN( EOF,  "",    -1, -1 );
+        delete lexer;
+    }
+
+    TEST(Recognize_Overridden_Expression_Terminator)
+    {
+        DLLexer* lexer = SetupLexer(";");
+        lexer->terminator( ";" );
+        CHECK_TOKEN( TERM, ";",    1,  1 );
+        CHECK_TOKEN( EOF,  "",    -1, -1 );
+        delete lexer;
+    }
 
+    TEST(Recognize_Overridden_Punctuation_Expression_Terminator)
+    {
+        DLLexer* lexer = SetupLexer(")");
+        lexer->terminator( ")" );
+        CHECK_TOKEN( TERM, ")",    1,  1 );
+        CHECK_TOKEN( EOF,  "",    -1, -1 );
+        delete lexer;
+    }
 
     //-------------------------------------------------------------------------
     // Test General Lexer Corner Cases