From: Mike D. Lowis <mike@mdlowis.com>
Date: Mon, 2 Apr 2012 17:42:59 +0000 (-0400)
Subject: Made consume and match functionality of Ilexer abstract and add LLNLexer class for... 
X-Git-Url: https://git.mdlowis.com/?a=commitdiff_plain;h=818c00c412d9ef34d211d1282977f180f41293b8;p=archive%2Fparse-utils.git

Made consume and match functionality of Ilexer abstract and add LLNLexer class for infinite lookahead lexers
---

diff --git a/source/lexer/ilexer.cpp b/source/lexer/ilexer.cpp
index bd6c333..66e8645 100644
--- a/source/lexer/ilexer.cpp
+++ b/source/lexer/ilexer.cpp
@@ -3,7 +3,7 @@
 
 using namespace std;
 
-ILexer::ILexer(istream& in) : line(-1), column(-1), in_stream(in), current(in_stream.get())
+ILexer::ILexer(istream& in) : line(-1), column(-1), in_stream(in)
 {
 }
 
@@ -16,38 +16,3 @@ bool ILexer::eof(void)
     return in_stream.eof();
 }
 
-void ILexer::consume(void)
-{
-    if(in_stream.eof())
-    {
-        current = EOF;
-    }
-    else
-    {
-        current = in_stream.get();
-        if(current == '\n')
-        {
-            line++;
-            column = 0;
-        }
-        else
-        {
-            column++;
-        }
-    }
-}
-
-void ILexer::match(char x) {
-    if ( current == x)
-    {
-        consume();
-    }
-    else
-    {
-        Exception ex(line,column);
-        ex << "Unexpected character. Expected " << x << ", received " << current << ".";
-        throw ex;
-    }
-}
-
-
diff --git a/source/lexer/ilexer.h b/source/lexer/ilexer.h
index 034e8da..accf67d 100644
--- a/source/lexer/ilexer.h
+++ b/source/lexer/ilexer.h
@@ -12,14 +12,13 @@ class ILexer
         int line;
         int column;
         std::istream& in_stream;
-        char current;
 
     public:
         ILexer(std::istream& in);
         virtual ~ILexer();
-        void consume(void);
-        void match(char x);
-        bool eof(void);
+        virtual void consume(void) = 0;
+        virtual void match(char x) = 0;
+        virtual bool eof(void);
         virtual Token next(void) = 0;
 };
 
diff --git a/source/lexer/llnlexer/llnlexer.cpp b/source/lexer/llnlexer/llnlexer.cpp
new file mode 100644
index 0000000..553f303
--- /dev/null
+++ b/source/lexer/llnlexer/llnlexer.cpp
@@ -0,0 +1,63 @@
+#include "llnlexer.h"
+#include "exception.h"
+
+LLNLexer::LLNLexer(std::istream& in) : ILexer(in), cur_idx(0)
+{
+}
+
+LLNLexer::~LLNLexer()
+{
+}
+
+void LLNLexer::consume(void)
+{
+    cur_idx++;
+    if(cur_idx == la_buffer.size())
+    {
+        cur_idx = 0;
+        la_buffer.clear();
+    }
+    sync(1);
+}
+
+void LLNLexer::match(char match)
+{
+    if( lookahead(1) == match )
+    {
+        consume();
+    }
+    else
+    {
+        Exception ex(line,column);
+        ex << "Unexpected character. Expected '" << match << "', received '" << lookahead(1) << "'.";
+        throw ex;
+    }
+}
+
+void LLNLexer::sync(unsigned int i)
+{
+    unsigned int next_index = cur_idx + i - 1;
+    unsigned int max_index = (la_buffer.size() == 0) ? 0 : (la_buffer.size() - 1);
+    if( next_index >= max_index )
+    {
+        fill( next_index - max_index);
+    }
+}
+
+void LLNLexer::fill(unsigned int n)
+{
+    unsigned int i = 0;
+    for (i = 0; i <= n; i++)
+    {
+        la_buffer.push_back( in_stream.get() );
+    }
+}
+
+char LLNLexer::lookahead(unsigned int i)
+{
+    sync(i);
+    return la_buffer.at( cur_idx + i - 1 );
+}
+
+
+
diff --git a/source/lexer/llnlexer/llnlexer.h b/source/lexer/llnlexer/llnlexer.h
new file mode 100644
index 0000000..d7f0dbf
--- /dev/null
+++ b/source/lexer/llnlexer/llnlexer.h
@@ -0,0 +1,26 @@
+#ifndef LLNLEXER_H
+#define LLNLEXER_H
+
+#include <vector>
+#include "ilexer.h"
+
+class LLNLexer : public ILexer
+{
+    protected:
+        unsigned int cur_idx;
+        std::vector<char> la_buffer;
+
+    public:
+        LLNLexer(std::istream& in);
+        virtual ~LLNLexer();
+
+        void consume(void);
+        void match(char type);
+        void sync(unsigned int i);
+        void fill(unsigned int n);
+        char lookahead(unsigned int i);
+
+        Token next(void) = 0;
+};
+
+#endif