Added line and column data to tokens during lexical analysis

diff --git a/source/sclpl/lexer.c b/source/sclpl/lexer.c
index d4982351f5dc00aa19133da4e0065a1d70d109f3..2bd31f5196a7f79c5ea4b4c528b359d878d5c7e3 100644 (file)
--- a/source/sclpl/lexer.c
+++ b/source/sclpl/lexer.c
@@ -10,7 +10,7 @@
 #include <stdlib.h>
 #include <errno.h>
 
-static lex_tok_t* lexer_make_token(char* text);
+static lex_tok_t* lexer_make_token(size_t line, size_t col, char* text);
 static lex_tok_t* lexer_punc(char* text);
 static lex_tok_t* lexer_char(char* text);
 static lex_tok_t* lexer_radix_int(char* text);
@@ -48,9 +48,11 @@ lexer_t* lexer_new(char* p_prompt, FILE* p_input) {
 
 lex_tok_t* lexer_read(lexer_t* p_lexer) {
     lex_tok_t* p_tok = NULL;
-    char* text = scanner_read(p_lexer->scanner);
+    size_t line;
+    size_t col;
+    char* text = scanner_read(p_lexer->scanner, &line, &col);
     if (NULL != text) {
-        p_tok = lexer_make_token(text);
+        p_tok = lexer_make_token(line, col, text);
         free(text);
     }
     return p_tok;
@@ -60,7 +62,7 @@ void lexer_skipline(lexer_t* p_lexer) {
     scanner_getline(p_lexer->scanner);
 }
 
-static lex_tok_t* lexer_make_token(char* text) {
+static lex_tok_t* lexer_make_token(size_t line, size_t col, char* text) {
     lex_tok_t* p_tok = NULL;
     if (0 == strcmp(text,"end")) {
         p_tok = lex_tok_new(T_END, NULL);
@@ -80,6 +82,11 @@ static lex_tok_t* lexer_make_token(char* text) {
     } else {
         p_tok = lexer_var(text);
     }
+    /* If we found a valid token then fill in the location details */
+    if (NULL != p_tok) {
+        p_tok->line = line;
+        p_tok->col  = col;
+    }
     return p_tok;
 }
 

diff --git a/source/sclpl/lexer.h b/source/sclpl/lexer.h
index 1f345e731c8067b7f5df09550b93c204efb7f4b2..04fa1172c67a5525b6535ab7db04a8dff160f35e 100644 (file)
--- a/source/sclpl/lexer.h
+++ b/source/sclpl/lexer.h
@@ -25,7 +25,7 @@ typedef struct {
     lex_tok_type_t type;
     const char* file;
     size_t line;
-    size_t column;
+    size_t col;
     void* value;
 } lex_tok_t;
 

diff --git a/source/sclpl/main.c b/source/sclpl/main.c
index 7afc1fd2160ed458d9742cc44c0b1b1ac45fb91e..9b77876da47ca5caf81c4d5a8b606e60181c880c 100644 (file)
--- a/source/sclpl/main.c
+++ b/source/sclpl/main.c
@@ -240,7 +240,7 @@ str_t* token_file(str_t* in) {
     lexer_t* p_lexer = lexer_new(NULL, input);
     lex_tok_t* token;
     while(NULL != (token = lexer_read(p_lexer))) {
-        pprint_token(output, token);
+        pprint_token(output, token, true);
         mem_release(token);
     }
     mem_release(p_lexer);

diff --git a/source/sclpl/pprint.c b/source/sclpl/pprint.c
index 36bf2d8356c858ccfe7e2a968cf7b988670c7773..6c3c146bbbc3b4de14ef1534fe9bd8b1d606fac3 100644 (file)
--- a/source/sclpl/pprint.c
+++ b/source/sclpl/pprint.c
@@ -69,8 +69,12 @@ void pprint_token_value(FILE* file, lex_tok_t* token) {
     }
 }
 
-void pprint_token(FILE* file, lex_tok_t* token)
+void pprint_token(FILE* file, lex_tok_t* token, bool print_loc)
 {
+    if (print_loc) {
+        fprintf(file, "%zu:", token->line);
+        fprintf(file, "%zu:", token->col);
+    }
     pprint_token_type(file, token);
     if (token->type < T_LBRACE) {
         fprintf(file, ":");
@@ -84,7 +88,7 @@ void pprint_tree(FILE* file, tree_t* tree, int depth)
 {
     print_indent(file, depth);
     if (tree->tag == ATOM) {
-        pprint_token(file, tree->ptr.tok);
+        pprint_token(file, tree->ptr.tok, false);
     } else {
         fputs("(tree", file);
         vec_t* p_vec = tree->ptr.vec;

diff --git a/source/sclpl/pprint.h b/source/sclpl/pprint.h
index 9d54159e1f2d0dc4463ab97595a04cf999d4cb37..2693ff41ffc389b5db755e60fb23b804a33e7680 100644 (file)
--- a/source/sclpl/pprint.h
+++ b/source/sclpl/pprint.h
@@ -14,7 +14,7 @@ void pprint_token_type(FILE* file, lex_tok_t* token);
 
 void pprint_token_value(FILE* file, lex_tok_t* token);
 
-void pprint_token(FILE* file, lex_tok_t* token);
+void pprint_token(FILE* file, lex_tok_t* token, bool print_loc);
 
 void pprint_tree(FILE* file, tree_t* tree, int depth);
 

diff --git a/source/sclpl/scanner.c b/source/sclpl/scanner.c
index 9db651b6eeabda608d41a70701967e3e6762bb5e..50005348de45d3a873ffdddc43ca719cf387ea1f 100644 (file)
--- a/source/sclpl/scanner.c
+++ b/source/sclpl/scanner.c
@@ -17,14 +17,17 @@ scanner_t* scanner_new(char* p_prompt, FILE* p_file) {
     scanner_t* p_scanner = (scanner_t*)mem_allocate(sizeof(scanner_t), &scanner_free);
     p_scanner->p_line   = NULL;
     p_scanner->index    = 0;
+    p_scanner->line     = 0;
     p_scanner->p_input  = p_file;
     p_scanner->p_prompt = p_prompt;
     return p_scanner;
 }
 
-char* scanner_read(scanner_t* p_scanner) {
+char* scanner_read(scanner_t* p_scanner, size_t* line, size_t* column) {
     char* p_tok = NULL;
     scanner_skip_ws(p_scanner);
+    *line   = p_scanner->line;
+    *column = p_scanner->index+1;
     if (!scanner_eof(p_scanner)) {
         if (scanner_oneof(p_scanner, "()[]{};,'")) {
             p_tok = scanner_dup(p_scanner, p_scanner->index, 1);
@@ -126,6 +129,8 @@ void scanner_getline(scanner_t* p_scanner) {
         p_scanner->p_line[index++] = (c == EOF) ? '\0' : c;
         p_scanner->p_line[index++] = '\0';
         p_scanner->index = 0;
+        /* Increment line count */
+        p_scanner->line++;
     }
 }
 

diff --git a/source/sclpl/scanner.h b/source/sclpl/scanner.h
index 31afceabdc983f14380edfc7d79ea96ed396aa50..1d052855eeae486404eb0a3ec90d0686466154ce 100644 (file)
--- a/source/sclpl/scanner.h
+++ b/source/sclpl/scanner.h
@@ -13,13 +13,14 @@
 typedef struct {
     char* p_line;
     size_t index;
+    size_t line;
     FILE* p_input;
     char* p_prompt;
 } scanner_t;
 
 scanner_t* scanner_new(char* p_prompt, FILE* p_file);
 
-char* scanner_read(scanner_t* p_scanner);
+char* scanner_read(scanner_t* p_scanner, size_t* line, size_t* col);
 
 bool scanner_eof(scanner_t* p_scanner);
 

diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 409f27a33715acbb4c2113578aa0d3fc4fd3b88f..6913d976e4075c33026c01083bbab23f417eee6c 100644 (file)
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -9,7 +9,7 @@ def cli(options, input = "")
 end
 
 def lexer(input)
-  cli(['--tokens'], input).scan(/^(T_[A-Z]+(:("[^"]*"|[^\n]+))?)/m).map {|m| m[0] }
+  cli(['--tokens'], input).scan(/^\d+:\d+:(T_[A-Z]+(:("[^"]*"|[^\n]+))?)/m).map {|m| m[0] }
 end
 
 def re_structure( token_array, offset = 0 )