From: Mike D. Lowis <mike@mdlowis.com>
Date: Fri, 6 Apr 2012 17:08:23 +0000 (-0400)
Subject: Added tests for lexer and parser
X-Git-Url: https://git.mdlowis.com/?a=commitdiff_plain;h=24a6ec5a68e4e7bd89e1c2a67c30daf146f79e45;p=archive%2Fdlang.git

Added tests for lexer and parser
---

diff --git a/source/dllexer/dllexer.cpp b/source/dllexer/dllexer.cpp
index 0ff7ad9..ebf4896 100644
--- a/source/dllexer/dllexer.cpp
+++ b/source/dllexer/dllexer.cpp
@@ -65,6 +65,12 @@ bool DLLexer::isStringChar(void)
 Token DLLexer::next(void)
 {
     Token ret;
+
+    // Prime the buffer. For empty input strings this results in us finding
+    // the EOF and skipping the loop
+    (void)lookahead(1);
+
+    // If we have non-EOF chars tehn process them
     while ( !eof() && (ret.type() == EOF) )
     {
         if (isWhiteSpace())
@@ -116,6 +122,7 @@ Token DLLexer::next(void)
             SingleCharOp(ret);
         }
     }
+
     return ret;
 }
 
@@ -157,7 +164,7 @@ void DLLexer::Number(Token& tok, bool isNegative)
     // Get the first part of the number
     oss << FloatingPoint(isNegative);
 
-    // Get teh exponent if we have one
+    // Get the exponent if we have one
     if ( lookahead(1) == 'e')
     {
         // consume the 'e'
diff --git a/tests/test_dllexer.cpp b/tests/test_dllexer.cpp
index ed1b86f..93c64cf 100644
--- a/tests/test_dllexer.cpp
+++ b/tests/test_dllexer.cpp
@@ -56,6 +56,9 @@ void TestLexerThrowsException(std::string& input)
 //-----------------------------------------------------------------------------
 namespace {
 
+    //-------------------------------------------------------------------------
+    // Recognize Individual Operators
+    //-------------------------------------------------------------------------
     TEST(Recognize_And_Ignore_Whitespace)
     {
         std::string input("foo \t\r\n foo");
@@ -63,6 +66,9 @@ namespace {
         TestLexerWithInput( input, expected );
     }
 
+    //-------------------------------------------------------------------------
+    // Recognize Individual Operators
+    //-------------------------------------------------------------------------
     TEST(Recognize_And_Ignore_Comments)
     {
         std::string input(
@@ -78,6 +84,9 @@ namespace {
         TestLexerWithInput( input, expected );
     }
 
+    //-------------------------------------------------------------------------
+    // Recognize Individual Operators
+    //-------------------------------------------------------------------------
     TEST(Recognize_Valid_IDs)
     {
         std::string input(
@@ -93,6 +102,9 @@ namespace {
         TestLexerWithInput( input, expected );
     }
 
+    //-------------------------------------------------------------------------
+    // Test Number Recognition
+    //-------------------------------------------------------------------------
     TEST(Recognize_Valid_Numbers)
     {
         std::string input(
@@ -118,6 +130,9 @@ namespace {
         TestLexerThrowsException( missing_exp );
     }
 
+    //-------------------------------------------------------------------------
+    // Test Character Recognition
+    //-------------------------------------------------------------------------
     TEST(Recognize_Valid_Characters)
     {
         std::string input(
@@ -130,6 +145,9 @@ namespace {
         TestLexerWithInput( input, expected );
     }
 
+    //-------------------------------------------------------------------------
+    // Test String Recognition
+    //-------------------------------------------------------------------------
     TEST(Recognize_Valid_Strings)
     {
         std::string input(
@@ -144,6 +162,9 @@ namespace {
         TestLexerWithInput( input, expected );
     }
 
+    //-------------------------------------------------------------------------
+    // Test Symbol Recognition
+    //-------------------------------------------------------------------------
     TEST(Recognize_Valid_Symbols)
     {
         std::string input(
@@ -161,26 +182,211 @@ namespace {
         TestLexerWithInput( input, expected );
     }
 
-    TEST(Recognize_Valid_Operators)
+    //-------------------------------------------------------------------------
+    // Test Individual Operators
+    //-------------------------------------------------------------------------
+    TEST(Recognize_Left_Bracket)
     {
-        std::string input(
-            // Recognize single character operators
-            "[ ] ( ) { } , + * / . % - "
-            // Recognize multi character operators and similar single char ones
-            "= == ! != < <= > >= | || && : := @ @="
-        );
-        eTokenTypes expected[] = {
-            LBRACK, RBRACK, LPAR, RPAR, LBRACE, RBRACE, COMMA, ADD, MUL, DIV,
-            MEMB, MACRO, SUB, ASSIGN, EQ, NOT, NE, LT, LTE, GT, GTE, PIPE, OR,
-            AND, SEP, DEFN, MAP, IMPORT,
-            (eTokenTypes)EOF
-        };
+        std::string input("[");
+        eTokenTypes expected[] = { LBRACK, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Right_Bracket)
+    {
+        std::string input("]");
+        eTokenTypes expected[] = { RBRACK, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Left_Paren)
+    {
+        std::string input("(");
+        eTokenTypes expected[] = { LPAR, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Right_Paren)
+    {
+        std::string input(")");
+        eTokenTypes expected[] = { RPAR, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Left_Brace)
+    {
+        std::string input("{");
+        eTokenTypes expected[] = { LBRACE, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Right_Brace)
+    {
+        std::string input("}");
+        eTokenTypes expected[] = { RBRACE, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Comma)
+    {
+        std::string input(",");
+        eTokenTypes expected[] = { COMMA, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Addition_Op)
+    {
+        std::string input("+");
+        eTokenTypes expected[] = { ADD, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Multiplication_Op)
+    {
+        std::string input("*");
+        eTokenTypes expected[] = { MUL, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Division_Op)
+    {
+        std::string input("/");
+        eTokenTypes expected[] = { DIV, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Member_Accessor_Op)
+    {
+        std::string input(".");
+        eTokenTypes expected[] = { MEMB, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Macro_Op)
+    {
+        std::string input("%");
+        eTokenTypes expected[] = { MACRO, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Subtraction_Op)
+    {
+        std::string input("-");
+        eTokenTypes expected[] = { SUB, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Assignment_Op)
+    {
+        std::string input("=");
+        eTokenTypes expected[] = { ASSIGN, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Equal_Comparison_Op)
+    {
+        std::string input("==");
+        eTokenTypes expected[] = { EQ, (eTokenTypes)EOF };
         TestLexerWithInput( input, expected );
     }
 
+    TEST(Recognize_Not_Op)
+    {
+        std::string input("!");
+        eTokenTypes expected[] = { NOT, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Not_Equal_Op)
+    {
+        std::string input("!=");
+        eTokenTypes expected[] = { NE, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Less_Than_Op)
+    {
+        std::string input("<");
+        eTokenTypes expected[] = { LT, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Less_Than_Equal_Op)
+    {
+        std::string input("<=");
+        eTokenTypes expected[] = { LTE, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Greater_Than_Op)
+    {
+        std::string input(">");
+        eTokenTypes expected[] = { GT, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Greater_Than_Or_Equal_Op)
+    {
+        std::string input(">=");
+        eTokenTypes expected[] = { GTE, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Pipe)
+    {
+        std::string input("|");
+        eTokenTypes expected[] = { PIPE, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Or_Op)
+    {
+        std::string input("||");
+        eTokenTypes expected[] = { OR, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_And_Op)
+    {
+        std::string input("&&");
+        eTokenTypes expected[] = { AND, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Seperator)
+    {
+        std::string input(":");
+        eTokenTypes expected[] = { SEP, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Definition_Op)
+    {
+        std::string input(":=");
+        eTokenTypes expected[] = { DEFN, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Map_Op)
+    {
+        std::string input("@");
+        eTokenTypes expected[] = { MAP, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Recognize_Import_Op)
+    {
+        std::string input("@=");
+        eTokenTypes expected[] = { IMPORT, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    //-------------------------------------------------------------------------
+    // Test Exceptional Cases
+    //-------------------------------------------------------------------------
     TEST(Throw_Exceptions_For_Exceptional_Cases)
     {
-        // Make sure invalud number literals throw exceptions where appropriate
+        // Make sure invalid number literals throw exceptions where appropriate
         std::string num_exception1("1.0e-");
         TestLexerThrowsException( num_exception1 );
 
@@ -201,4 +407,22 @@ namespace {
         std::string multi_op_exception1("&");
         TestLexerThrowsException( multi_op_exception1 );
     }
+
+    //-------------------------------------------------------------------------
+    // Test General Corner Cases
+    //-------------------------------------------------------------------------
+    TEST(Handle_An_Empty_Input_Stream)
+    {
+        std::string input("");
+        eTokenTypes expected[] = { (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
+    TEST(Handle_Recognition_At_The_End_Of_Input)
+    {
+        std::string input("[");
+        eTokenTypes expected[] = { LBRACK, (eTokenTypes)EOF };
+        TestLexerWithInput( input, expected );
+    }
+
 }
diff --git a/tests/test_dlparser.cpp b/tests/test_dlparser.cpp
new file mode 100644
index 0000000..8785bc5
--- /dev/null
+++ b/tests/test_dlparser.cpp
@@ -0,0 +1,398 @@
+// Unit Test Framework Includes
+#include "UnitTest++.h"
+
+// Supporting Includes
+#include <sstream>
+#include "exception.h"
+#include "ivisitor.h"
+
+// File To Test
+#include "dlparser.h"
+
+using namespace UnitTest;
+
+//-----------------------------------------------------------------------------
+// Helper Functions
+//-----------------------------------------------------------------------------
+class TreeTester : public IVisitor {
+    public:
+        eTokenTypes* expected_types;
+        unsigned int cur_index;
+        TreeTester(eTokenTypes expected[]) : IVisitor(), expected_types(expected), cur_index(0) {}
+    private:
+        void afterChildren(AST* cur, int depth) {
+            CHECK_EQUAL( expected_types[ cur_index ], cur->type() );
+            cur_index = (expected_types[cur_index] == PROGRAM) ? cur_index : (cur_index + 1);
+        }
+
+        // Not Used Here
+        void beforeVisit(AST* cur, int depth) {}
+        void afterVisit(AST* cur, int depth) {}
+        void beforeChildren(AST* cur, int depth) {}
+        void beforeChild(AST* cur, int depth) {}
+        void afterChild(AST* cur, int depth) {}
+};
+
+
+
+//-----------------------------------------------------------------------------
+// Helper Functions
+//-----------------------------------------------------------------------------
+void TestParserWithInput(std::string& input, eTokenTypes expected_types[])
+{
+    // Setup
+    std::istringstream input_stream(input);
+    DLParser parser;
+    TreeTester tester(expected_types);
+
+    // Parse the test input
+    parser.input( new DLLexer( input_stream ) );
+    parser.parse();
+
+    // Test the tree against expected format
+    parser.process( tester );
+}
+
+//void TestParserThrowsException(std::string& input)
+//{
+//    // Setup
+//    //std::istringstream input_stream(input);
+//    //DLLexer* lexer = new DLLexer(input_stream);
+//
+//    //CHECK_THROW( lexer->next(), Exception );
+//
+//    // Cleanup
+//    //delete lexer;
+//}
+
+//-----------------------------------------------------------------------------
+// Begin Unit Tests
+//-----------------------------------------------------------------------------
+namespace {
+
+    //-------------------------------------------------------------------------
+    // Test Parsing of Data Type Literals
+    //-------------------------------------------------------------------------
+
+    // Vector Literals
+    //----------------
+    TEST(Parse_An_Empty_Vector)
+    {
+        std::string input("[]");
+        eTokenTypes expected[] = {
+            VECTOR,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_An_Vector_With_One_Item)
+    {
+        std::string input("[1]");
+        eTokenTypes expected[] = {
+            NUM, VECTOR,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Vector_Of_Two_Items)
+    {
+        std::string input("[1,2]");
+        eTokenTypes expected[] = {
+            NUM, NUM, VECTOR,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Vector_Of_Three_Items)
+    {
+        std::string input("[1,2,3]");
+        eTokenTypes expected[] = {
+            NUM, NUM, NUM, VECTOR,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Vector_With_Trailing_Commas)
+    {
+        std::string input("[1,]");
+        eTokenTypes expected[] = {
+            NUM, VECTOR,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // List Literals
+    //--------------
+    TEST(Parse_A_List)
+    {
+        std::string input("()");
+        eTokenTypes expected[] = {
+            LIST,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_An_List_With_One_Item)
+    {
+        std::string input("(1,)"); // Comma is required here
+        eTokenTypes expected[] = {
+            NUM, LIST,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_List_Of_Two_Items)
+    {
+        std::string input("(1,2)");
+        eTokenTypes expected[] = {
+            NUM, NUM, LIST,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_List_Of_Three_Items)
+    {
+        std::string input("(1,2,3)");
+        eTokenTypes expected[] = {
+            NUM, NUM, NUM, LIST,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_List_With_Trailing_Commas)
+    {
+        std::string input("(1,)");
+        eTokenTypes expected[] = {
+            NUM, LIST,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_List_With_Heterogeneous_Elements)
+    {
+        std::string input("(1,$foo,bar)");
+        eTokenTypes expected[] = {
+            NUM, SYMBOL, ID, LIST,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // Block Literals
+    //---------------
+    TEST(Parse_A_Block_With_No_Parameters)
+    {
+        std::string input("{}");
+        eTokenTypes expected[] = {
+            PARAMS, BLOCK, FUNC,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Block_With_One_Param)
+    {
+        std::string input("{|a|}");
+        eTokenTypes expected[] = {
+            ID, PARAMS, BLOCK, FUNC,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Block_With_Two_Params)
+    {
+        std::string input("{|a,b|}");
+        eTokenTypes expected[] = {
+            ID, ID, PARAMS, BLOCK, FUNC,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Block_With_Two_Params_That_Performs_An_Addition_Operation)
+    {
+        std::string input("{|a,b| a + b}");
+        eTokenTypes expected[] = {
+            ID, ID, PARAMS, ID, ID, ADD, BLOCK, FUNC,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // Map Literals
+    //-------------
+    TEST(Parse_An_Empty_Map)
+    {
+        std::string input("@{}");
+        eTokenTypes expected[] = {
+            MAP,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Map_With_One_Entry)
+    {
+        std::string input("@{ $foo : 42 }");
+        eTokenTypes expected[] = {
+            SYMBOL, NUM, SEP, MAP,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Map_With_Two_Entries)
+    {
+        std::string input("@{ $foo : 42, $bar : 42 }");
+        eTokenTypes expected[] = {
+            SYMBOL, NUM, SEP, SYMBOL, NUM, SEP, MAP,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Maps_Should_Allow_A_Trailing_Comma)
+    {
+        std::string input("@{ $foo : 42, $bar : 42, }");
+        eTokenTypes expected[] = {
+            SYMBOL, NUM, SEP, SYMBOL, NUM, SEP, MAP,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // Id Literals
+    //------------
+    TEST(Parse_An_Id)
+    {
+        std::string input("foo");
+        eTokenTypes expected[] = {
+            ID,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // Number Literals
+    //----------------
+    TEST(Parse_A_Num)
+    {
+        std::string input("42");
+        eTokenTypes expected[] = {
+            NUM,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // Character Literals
+    //-------------------
+    TEST(Parse_A_Char)
+    {
+        std::string input("'a'");
+        eTokenTypes expected[] = {
+            CHAR,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // String Literals
+    //----------------
+    TEST(Parse_A_String)
+    {
+        std::string input("\"\"");
+        eTokenTypes expected[] = {
+            STRING,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // Symbol Literals
+    //----------------
+    TEST(Parse_A_Symbol)
+    {
+        std::string input("$foo");
+        eTokenTypes expected[] = {
+            SYMBOL,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    //-------------------------------------------------------------------------
+    // Test Operators And Operator Precedence
+    //-------------------------------------------------------------------------
+
+    // Member Accessor
+    //----------------
+    TEST(Parse_A_Member_Acces_One_Level_Deep)
+    {
+        std::string input("foo.bar");
+        eTokenTypes expected[] = {
+            ID, ID, MEMB,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    TEST(Parse_A_Member_Acces_Two_Levels_Deep)
+    {
+        std::string input("foo.bar.somethin");
+        eTokenTypes expected[] = {
+            ID, ID, ID, MEMB, MEMB,
+            PROGRAM
+        };
+        TestParserWithInput( input, expected );
+    }
+
+    // Grouping Expression
+    //--------------------
+    //TEST(Parse_A_Grouping_Expression_With_A_Lower_Priority_Statement)
+    //{
+    //    std::string input("(foo.bar).somethin");
+    //    eTokenTypes expected[] = {
+    //        ID, ID, MEMB, ID, MEMB,
+    //        PROGRAM
+    //    };
+    //    TestParserWithInput( input, expected );
+    //}
+
+    //TEST(Parse_A_Grouping_Expression_With_A_Higher_Priority_Statement)
+    //{
+    //    std::string input("(1+1).somethin");
+    //    eTokenTypes expected[] = {
+    //        NUM, NUM, ADD, ID, MEMB,
+    //        PROGRAM
+    //    };
+    //    TestParserWithInput( input, expected );
+    //}
+
+    // Function Application
+    //---------------------
+
+    // Collection Access
+    //------------------
+
+    //-------------------------------------------------------------------------
+    // Test General Corner Cases
+    //-------------------------------------------------------------------------
+    TEST(Parse_An_Empty_Program)
+    {
+        std::string input("");
+        eTokenTypes expected[] = { PROGRAM };
+        TestParserWithInput( input, expected );
+    }
+}