From 3b562ddb15e62cbb878422ddfa4e5299478cc328 Mon Sep 17 00:00:00 2001
From: "Mike D. Lowis" <mike.lowis@gentex.com>
Date: Thu, 28 May 2015 16:53:32 -0400
Subject: [PATCH] Removed unneccessary tokens from the lexer

---
 build.rb        |  9 +++------
 source/lexer.l  |  6 ++----
 source/main.c   | 38 ++++++++++++++------------------------
 source/parser.h | 16 +++++++---------
 4 files changed, 26 insertions(+), 43 deletions(-)

diff --git a/build.rb b/build.rb
index 6071448..372ed05 100755
--- a/build.rb
+++ b/build.rb
@@ -19,15 +19,12 @@ runtime_libs = ['build/lib/libcds.a']
 
 # Build the parser
 main_env.CFile('source/lex.yy.c', 'source/lexer.l')
-main_env.Program('parser', FileList['source/*.c'] + ['source/lex.yy.c'] + runtime_libs)
+parser_srcs = (FileList['source/*.c'] + ['source/lex.yy.c']).uniq
+main_env.Program('parser',  parser_srcs + runtime_libs)
 
 #------------------------------------------------------------------------------
 # Test Build Targets
 #------------------------------------------------------------------------------
 if Opts[:profile].include? "test"
-#  compiler_libs = ['build/lib/libparse-test.a'] + runtime_libs
-#  test_env.Library('build/lib/libparse-test.a', FileList['source/libparse/*.c'])
-#  test_env.Program('build/bin/sclpl-test', FileList['source/sclpl/*.c'] + compiler_libs)
-#  test_env.Command('RSpec', [], 'CMD' => [
-#      'rspec', '--pattern', 'spec/**{,/*/**}/*_spec.rb', '--format', 'documentation'])
+  # Do nothing for now
 end
diff --git a/source/lexer.l b/source/lexer.l
index 5331c31..3d13e1b 100644
--- a/source/lexer.l
+++ b/source/lexer.l
@@ -21,8 +21,7 @@ int yywrap(void){ return 1; }
 "="                         TOKEN(ASSIGN);
 "<-"                        TOKEN(ASSIGN);
 
-"."                         TOKEN(END);
-","                         TOKEN(COMMA);
+"."                         TOKEN(PERIOD);
 ":"                         TOKEN(COLON);
 "|"                         TOKEN(PIPE);
 "("                         TOKEN(LPAR);
@@ -34,11 +33,10 @@ int yywrap(void){ return 1; }
 "{"                         TOKEN(LBRACE);
 "}"                         TOKEN(RBRACE);
 
-"self"                      TOKEN(SELF);
-"true"|"false"              TOKEN(BOOL);
 [0-9]+                      TOKEN(NUM);
 '[^\\]'                     TOKEN(CHAR);
 \"[^"]*\"                   TOKEN(STRING);
+$[_a-zA-Z][_a-zA-Z0-9]*     TOKEN(SYMBOL);
 
 [+-]+                       TOKEN(BINOP);
 [_a-zA-Z][_a-zA-Z0-9]*      TOKEN(ID);
diff --git a/source/main.c b/source/main.c
index 1aab45e..51ceab8 100644
--- a/source/main.c
+++ b/source/main.c
@@ -119,7 +119,7 @@ static void hashset(void);
 
 static void expression(void) {
     keyword_send();
-    optional(END);
+    optional(PERIOD);
 }
 
 static void keyword_send(void) {
@@ -167,9 +167,8 @@ static void literal(void) {
     switch (Current) {
         case ID:      shift_reduce(ID, 0u);     push_reduce(UNARY_MSG, 1); break;
         case NUM:     shift_reduce(NUM, 0u);    break;
-        case SELF:    shift_reduce(SELF, 0u);   break;
         case STRING:  shift_reduce(STRING, 0u); break;
-        case BOOL:    shift_reduce(BOOL, 0u);   break;
+        case SYMBOL:  shift_reduce(SYMBOL, 0u); break;
         case CHAR:    shift_reduce(CHAR, 0u);   break;
         case LBRACK:  array();                  break;
         case LBRACE:  object();                 break;
@@ -182,12 +181,9 @@ static void literal(void) {
 static void array(void) {
     int count = 0;
     expect(LBRACK);
-    if (!accept(RBRACK)) {
-        do {
-            optional(COMMA);
-            expression();
-            count++;
-        } while(accept(COMMA));
+    while (!accept(RBRACK)) {
+        expression();
+        count++;
     }
     expect(RBRACK);
     push_reduce(ARRAY, count);
@@ -196,15 +192,12 @@ static void array(void) {
 static void hashmap(void) {
     int count = 0;
     expect(ALBRACE);
-    if (!accept(RBRACE)) {
-        do {
-            optional(COMMA);
-            shift_reduce(STRING, 0);
-            expect(COLON);
-            expression();
-            push_reduce(PAIR, 2);
-            count++;
-        } while(accept(COMMA));
+    while (!accept(RBRACE)) {
+        shift_reduce(STRING, 0);
+        expect(COLON);
+        expression();
+        push_reduce(PAIR, 2);
+        count++;
     }
     expect(RBRACE);
     push_reduce(HASHMAP, count);
@@ -214,12 +207,9 @@ static void hashset(void)
 {
     int count = 0;
     expect(ALBRACK);
-    if (!accept(RBRACK)) {
-        do {
-            optional(COMMA);
-            expression();
-            count++;
-        } while(accept(COMMA));
+    while (!accept(RBRACK)) {
+        expression();
+        count++;
     }
     expect(RBRACK);
     push_reduce(HASHSET, count);
diff --git a/source/parser.h b/source/parser.h
index c89fc60..2acadd6 100644
--- a/source/parser.h
+++ b/source/parser.h
@@ -8,10 +8,9 @@
 #define UNKNOWN 0
 
 #define NUM     1
-#define SELF    2
-#define STRING  3
-#define BOOL    4
-#define CHAR    5
+#define STRING  2
+#define CHAR    3
+#define SYMBOL  4
 
 #define LPAR    10
 #define RPAR    11
@@ -22,11 +21,10 @@
 #define LBRACE  16
 #define RBRACE  17
 
-#define COMMA   20
-#define COLON   21
-#define END     22
-#define BINOP   23
-#define PIPE    24
+#define COLON   20
+#define PERIOD  21
+#define BINOP   22
+#define PIPE    23
 
 #define RETURN  30
 #define ASSIGN  31
-- 
2.52.0