From 254bd8d3dd48cdffe969626e016c08c696a647a9 Mon Sep 17 00:00:00 2001 From: "Mike D. Lowis" Date: Wed, 30 May 2012 12:56:51 -0400 Subject: [PATCH] Updated tests for lexer --- source/dllexer/dllexer.cpp | 32 +++++----- source/dllexer/dllexer.h | 2 +- tests/test_dllexer.cpp | 119 +++++++++++++++++++++---------------- 3 files changed, 86 insertions(+), 67 deletions(-) diff --git a/source/dllexer/dllexer.cpp b/source/dllexer/dllexer.cpp index 98c9d8c..03827a4 100644 --- a/source/dllexer/dllexer.cpp +++ b/source/dllexer/dllexer.cpp @@ -16,12 +16,6 @@ bool DLLexer::isWhiteSpace(void) (lookahead(1) == '\n'); } -bool DLLexer::isLetter(void) -{ - return ((lookahead(1) >= 'a') && (lookahead(1) <= 'z')) || - ((lookahead(1) >= 'A') && (lookahead(1) <= 'Z')); -} - bool DLLexer::isDigit(char lach) { return ((lach >= '0') && (lach <= '9')); @@ -34,6 +28,15 @@ bool DLLexer::isStringChar(void) && (lookahead(1) != '\n')); } +bool DLLexer::isIDChar(void) +{ + return ( !isWhiteSpace() && + ('(' != lookahead(1)) && + (')' != lookahead(1)) && + ('#' != lookahead(1)) && + (EOF != lookahead(1)) ); +} + void DLLexer::terminator(std::string term) { terminator_string = term; @@ -100,13 +103,13 @@ Token DLLexer::next(void) // Consume parentheses else if (lookahead(1) == '(') { - ret = Token( LPAR, "(", line, column ); consume(); + ret = Token( LPAR, "(", line, column ); } else if (lookahead(1) == ')') { - ret = Token( RPAR, ")", line, column ); consume(); + ret = Token( RPAR, ")", line, column ); } // Everything else (except the unescaped terminator) is considered an ID @@ -157,11 +160,12 @@ void DLLexer::Id(Token& tok) oss << lookahead(1); consume(); } - while( !isWhiteSpace() && - ('(' != lookahead(1)) && - (')' != lookahead(1)) && - ('#' != lookahead(1)) && - (EOF != lookahead(1)) ); + while( isIDChar() ); + //while( !isWhiteSpace() && + // ('(' != lookahead(1)) && + // (')' != lookahead(1)) && + // ('#' != lookahead(1)) && + // (EOF != lookahead(1)) ); //while(isLetter() || isDigit() || lookahead(1) == '_'); tok = Token(ID, oss.str(), line, column); } @@ -302,7 +306,7 @@ void DLLexer::Symbol(Token& tok) oss << lookahead(1); consume(); } - while(isLetter() || isDigit(lookahead(1)) || lookahead(1) == '_'); + while( isIDChar() ); tok = Token( SYMBOL, oss.str(), line, column ); } diff --git a/source/dllexer/dllexer.h b/source/dllexer/dllexer.h index 391e5fb..a068bed 100644 --- a/source/dllexer/dllexer.h +++ b/source/dllexer/dllexer.h @@ -47,7 +47,7 @@ class DLLexer : public LLNLexer { public: DLLexer(std::istream& in); bool isWhiteSpace(void); - bool isLetter(void); + bool isIDChar(void); bool isDigit(char lach); bool isStringChar(void); void WS(void); diff --git a/tests/test_dllexer.cpp b/tests/test_dllexer.cpp index bd30d0c..84619f9 100644 --- a/tests/test_dllexer.cpp +++ b/tests/test_dllexer.cpp @@ -216,61 +216,76 @@ namespace { delete lexer; } - ////------------------------------------------------------------------------- - //// Test String Recognition - ////------------------------------------------------------------------------- - //TEST(Recognize_Valid_Strings) - //{ - // std::string input( - // // Make Sure we recognize all valid characters for a symbol - // "\"\" \n" - // "\"a\" \n" - // "\"\\a\" \n" - // ); - // eTokenTypes expected[] = { - // STRING, STRING, STRING, (eTokenTypes)EOF - // }; - // TestLexerWithInput( input, expected ); - //} + //------------------------------------------------------------------------- + // Test String Recognition + //------------------------------------------------------------------------- + TEST(Recognize_Valid_Strings) + { + DLLexer* lexer = SetupLexer( + // Make Sure we recognize all valid characters for a symbol + "\"\" \n" + "\"a\" \n" + "\"\\a\" \n" + ); + CHECK_TOKEN( STRING, "", 1, 2 ); + CHECK_TOKEN( STRING, "a", 2, 3 ); + CHECK_TOKEN( STRING, "\\a", 3, 4 ); + CHECK_TOKEN( EOF, "", -1, -1 ); + delete lexer; + } - ////------------------------------------------------------------------------- - //// Test Symbol Recognition - ////------------------------------------------------------------------------- - //TEST(Recognize_Valid_Symbols) - //{ - // std::string input( - // // Make Sure we recognize all valid characters for a symbol - // "$abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_\n" - // "$ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_\n" - // "$a_123\n" - // "$a123\n" - // "$a_\n" - // "$a_a\n" - // ); - // eTokenTypes expected[] = { - // SYMBOL, SYMBOL, SYMBOL, SYMBOL, SYMBOL, SYMBOL, (eTokenTypes)EOF - // }; - // TestLexerWithInput( input, expected ); - //} + //------------------------------------------------------------------------- + // Test Symbol Recognition + //------------------------------------------------------------------------- + TEST(Recognize_Valid_Symbols) + { + DLLexer* lexer = SetupLexer( + // Make Sure we recognize all valid characters for a symbol + "$abc\n" + "$ABC\n" + "$a_123\n" + "$a123\n" + "$a_\n" + "$a_a\n" + ); + CHECK_TOKEN( SYMBOL, "abc", 1, 4 ); + CHECK_TOKEN( SYMBOL, "ABC", 2, 4 ); + CHECK_TOKEN( SYMBOL, "a_123", 3, 6 ); + CHECK_TOKEN( SYMBOL, "a123", 4, 5 ); + CHECK_TOKEN( SYMBOL, "a_", 5, 3 ); + CHECK_TOKEN( SYMBOL, "a_a", 6, 4 ); + CHECK_TOKEN( EOF, "", -1, -1 ); + delete lexer; + } - ////------------------------------------------------------------------------- - //// Recognize Valid IDs - ////------------------------------------------------------------------------- - //TEST(Recognize_Valid_IDs) - //{ - // std::string input( - // // Make Sure we recognize all valid characters for an ID - // "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_\n" - // "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_\n" - // "a_123\n" - // "a123\n" - // "a_\n" - // "a_a\n" - // ); - // eTokenTypes expected[] = { ID, ID, ID, ID, ID, ID, (eTokenTypes)EOF }; - // TestLexerWithInput( input, expected ); - //} + //------------------------------------------------------------------------- + // Test Expression Terminator Recognition + //------------------------------------------------------------------------- + TEST(Recognize_Default_Expression_Terminator) + { + DLLexer* lexer = SetupLexer("end"); + CHECK_TOKEN( TERM, "end", 1, 3 ); + CHECK_TOKEN( EOF, "", -1, -1 ); + delete lexer; + } + + TEST(Recognize_Overridden_Expression_Terminator) + { + DLLexer* lexer = SetupLexer(";"); + lexer->terminator( ";" ); + CHECK_TOKEN( TERM, ";", 1, 1 ); + CHECK_TOKEN( EOF, "", -1, -1 ); + delete lexer; + } + TEST(Recognize_Overridden_Punctuation_Expression_Terminator) + { + DLLexer* lexer = SetupLexer(")"); + lexer->terminator( ")" ); + CHECK_TOKEN( TERM, ")", 1, 1 ); + CHECK_TOKEN( EOF, "", -1, -1 ); + delete lexer; + } //------------------------------------------------------------------------- // Test General Lexer Corner Cases -- 2.52.0