]> git.mdlowis.com Git - proto/sclpl.git/commitdiff
Implemented Token Parsing
authorMike D. Lowis <mike@mdlowis.com>
Fri, 29 Mar 2013 16:17:20 +0000 (12:17 -0400)
committerMike D. Lowis <mike@mdlowis.com>
Fri, 29 Mar 2013 16:17:20 +0000 (12:17 -0400)
premake4.lua
source/common/tokens/tokens.c [new file with mode: 0644]
source/common/tokens/tokens.h [moved from source/common/tokens.h with 90% similarity]
source/lexer/lex.c
source/parser/main.c
source/parser/tok.c [new file with mode: 0644]
source/parser/tok.h [new file with mode: 0644]
source/runtime/types/types.c

index 5bd8aca2b9295ac21f1c74d99a3a85be477d4632..f58a2e2030aabd55f401579d47b00ffea9d92119 100644 (file)
@@ -12,16 +12,28 @@ project "sclpl-rt"
     kind "SharedLib"
     language "C"
     location "build"
-    files { "source/runtime/**.*" }
+    files {
+        "source/runtime/**.*"
+    }
 
 project "sclpl-rt-tests"
     kind "ConsoleApp"
     language "C++"
     location "build"
-    links { "UnitTest++", "sclpl-rt" }
-    includedirs { "source/runtime/**", "tools/UnitTest++/**" }
-    files { "tests/runtime/*.c*" }
-    postbuildcommands { "./sclpl-rt-tests" }
+    links {
+        "UnitTest++",
+        "sclpl-rt"
+    }
+    includedirs {
+        "source/runtime/**",
+        "tools/UnitTest++/**"
+    }
+    files {
+        "tests/runtime/*.c*"
+    }
+    postbuildcommands {
+        "./sclpl-rt-tests"
+    }
 
 -------------------------------------------------------------------------------
 -- SCLPL Lexer
@@ -32,22 +44,31 @@ project "sclpl-lex"
     location "build"
     includedirs {
         "source/lexer/**",
-        "source/common/",
-        "source/runtime/**"
+        "source/runtime/**",
+        "source/common/**",
     }
     files {
         "source/lexer/**.*",
-        "source/runtime/collector/**.*"
+        "source/common/**.*",
     }
 
 project "sclpl-lex-tests"
     kind "ConsoleApp"
     language "C++"
     location "build"
-    links { "UnitTest++" }
-    includedirs { "source/lexer/**", "tools/UnitTest++/**" }
-    files { "tests/lexer/*.c*" }
-    postbuildcommands { "./sclpl-lex-tests" }
+    links {
+        "UnitTest++"
+    }
+    includedirs {
+        "source/lexer/**",
+        "tools/UnitTest++/**"
+    }
+    files {
+        "tests/lexer/*.c*"
+    }
+    postbuildcommands {
+        "./sclpl-lex-tests"
+    }
 
 -------------------------------------------------------------------------------
 -- SCLPL Parser
@@ -59,7 +80,7 @@ project "sclpl-parse"
     includedirs {
         "source/lexer/**",
         "source/runtime/**",
-        "source/common/**"
+        "source/common/**",
     }
     files {
         "source/parser/**.*",
@@ -71,10 +92,19 @@ project "sclpl-parse-tests"
     kind "ConsoleApp"
     language "C++"
     location "build"
-    links { "UnitTest++" }
-    includedirs { "source/parser/**", "tools/UnitTest++/**" }
-    files { "tests/parser/*.c*" }
-    postbuildcommands { "./sclpl-parse-tests" }
+    links {
+        "UnitTest++"
+    }
+    includedirs {
+        "source/parser/**",
+        "tools/UnitTest++/**"
+    }
+    files {
+        "tests/parser/*.c*"
+    }
+    postbuildcommands {
+        "./sclpl-parse-tests"
+    }
 
 -------------------------------------------------------------------------------
 -- UnitTest++ - A C/C++ unit testing library
diff --git a/source/common/tokens/tokens.c b/source/common/tokens/tokens.c
new file mode 100644 (file)
index 0000000..8985269
--- /dev/null
@@ -0,0 +1,22 @@
+/**
+    @file tokens.c
+    @brief See header for details
+    $Revision$
+    $HeadURL$
+*/
+#include "tokens.h"
+
+const char* Token_Types[TOK_MAX] = {
+    "EOF",    /* TOK_EOF */
+    "ID",     /* TOK_ID  */
+    "NUM",    /* TOK_NUM */
+    "LPAREN", /* TOK_LPAR */
+    "RPAREN", /* TOK_RPAR */
+    "LBRACK", /* TOK_LBRACK */
+    "RBRACK", /* TOK_RBRACK */
+    "LBRACE", /* TOK_LBRACE */
+    "RBRACE", /* TOK_RBRACE */
+    "TERM",   /* TOK_TERM */
+    "BOOL",   /* TOK_BOOL */
+};
+
similarity index 90%
rename from source/common/tokens.h
rename to source/common/tokens/tokens.h
index fb28466bc2e5cefedce9fd43e39655b200fa4be9..ad6009edad79db1d38c3a8f0781e941f863eb101 100644 (file)
@@ -22,4 +22,6 @@ typedef enum {
     TOK_MAX    = 11,
 } tok_type_t;
 
+extern const char* Token_Types[TOK_MAX];
+
 #endif /* TOKENS_H */
index cc697af9095a961383260362de040fc9b291d6cf..c5819791d809ca5fd6957d4499a3d3d469617cbc 100644 (file)
@@ -11,6 +11,7 @@
 #include "classes.h"
 #include "file.h"
 #include "buf.h"
+#include "tokens.h"
 
 /* Prototypes
  *****************************************************************************/
@@ -33,20 +34,6 @@ static void identifier(void);
  *****************************************************************************/
 jmp_buf Jump_Point;
 
-const char* Types[TOK_MAX] = {
-    "EOF",    /* TOK_EOF */
-    "ID",     /* TOK_ID  */
-    "NUM",    /* TOK_NUM */
-    "LPAREN", /* TOK_LPAR */
-    "RPAREN", /* TOK_RPAR */
-    "LBRACK", /* TOK_LBRACK */
-    "RBRACK", /* TOK_RBRACK */
-    "LBRACE", /* TOK_LBRACE */
-    "RBRACE", /* TOK_RBRACE */
-    "TERM",   /* TOK_TERM */
-    "BOOL",   /* TOK_BOOL */
-};
-
 const lex_keyword_t Keywords[] = {
     { "end",   TOK_TERM },
     { "true",  TOK_BOOL },
@@ -66,7 +53,7 @@ static void accept(void)
 
 static void accept_char(tok_type_t tok)
 {
-    tok_set_type( Types[tok] );
+    tok_set_type( Token_Types[tok] );
     tok_consume();
     tok_accept();
 }
@@ -137,7 +124,7 @@ static void keyword(void)
     {
         if (0 == strcmp( p_text, Keywords[i].p_text ))
         {
-            tok_set_type( Types[ Keywords[i].type ] );
+            tok_set_type( Token_Types[ Keywords[i].type ] );
             break;
         }
         i++;
@@ -169,7 +156,7 @@ static void punctuation(void)
 
 static void number(void)
 {
-    tok_set_type(Types[TOK_NUM]);
+    tok_set_type(Token_Types[TOK_NUM]);
     if (matches('0'))
     {
         tok_consume();
@@ -218,7 +205,7 @@ static void exponent(void)
 
 static void identifier(void)
 {
-    tok_set_type(Types[TOK_ID]);
+    tok_set_type(Token_Types[TOK_ID]);
     while (!token_end())
         tok_consume();
     accept();
index 2bd0d10ea2de02c03b72e9d21beb040b554d56af..2e60382a0400d29a502480279dc0eb5f7d76af4d 100644 (file)
@@ -1,4 +1,5 @@
 #include <stdio.h>
+#include "tok.h"
 
 int parse_files(int num_files, char** fnames);
 int parse_input(char* outfile);
@@ -29,9 +30,25 @@ int parse_files(int num_files, char** fnames)
 
 int parse_input(char* outfile)
 {
-    //tok_open(outfile);
-
-    //tok_close();
+    int ret = 0;
+    if (tok_source(outfile))
+    {
+        while (!tok_eof())
+        {
+            tok_t* p_tok = tok_read();
+            if (NULL != p_tok)
+            {
+                printf( "%s:%d:%d:\t%d\t%s",
+                        p_tok->p_file_name,
+                        p_tok->line,
+                        p_tok->column,
+                        p_tok->type,
+                        p_tok->p_text );
+            }
+        }
+        tok_finish();
+    }
+    return ret;
 }
 
 
diff --git a/source/parser/tok.c b/source/parser/tok.c
new file mode 100644 (file)
index 0000000..b4f63b9
--- /dev/null
@@ -0,0 +1,169 @@
+/**
+    @file tok.c
+    @brief See header for details
+    $Revision$
+    $HeadURL$
+*/
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "tok.h"
+#include "tokens.h"
+
+#define FIELD_TYPE       0
+#define FIELD_LINE_NUM   1
+#define FIELD_COLUMN_NUM 2
+#define FIELD_TEXT       3
+#define FIELD_MAX        4
+
+#define MAX_INPUT_STR 1024
+FILE* Handle = NULL;
+tok_t Token  = { 0 };
+char  Buffer[MAX_INPUT_STR];
+
+bool tok_source(char* fname)
+{
+    if (NULL == fname)
+    {
+        Handle = stdin;
+    }
+    else
+    {
+        Handle = fopen(fname,"r");
+    }
+    return (NULL != Handle);
+}
+
+bool tok_eof(void)
+{
+    bool ret = true;
+    if (NULL != Handle)
+    {
+        ret = feof( Handle );
+    }
+    return ret;
+}
+
+void tok_finish(void)
+{
+    fclose(Handle);
+}
+
+tok_t* tok_read(void)
+{
+    tok_t* p_tok = NULL;
+    if (NULL != Handle)
+    {
+        fgets(Buffer, MAX_INPUT_STR, Handle);
+        if ('\0' != Buffer[0])
+        {
+            if ('@' == Buffer[0])
+            {
+                tok_read_fname();
+                p_tok = tok_read();
+            }
+            else
+            {
+                p_tok = tok_build_token();
+            }
+            Buffer[0] = '\0';
+        }
+    }
+    return p_tok;
+}
+
+void tok_read_fname(void)
+{
+    uint32_t index = 0;
+    Buffer[strlen(Buffer)-1] = '\0';
+    while(' ' != Buffer[index]) index++;
+    char* new_str = (char*)malloc( strlen(&Buffer[index+1]) );
+    strcpy( new_str, &Buffer[index+1] );
+    Token.p_file_name = new_str;
+}
+
+tok_t* tok_build_token(void)
+{
+    tok_t* ret = NULL;
+    uint32_t index;
+    uint32_t start = 0;
+    uint32_t end = 0;
+
+    /* Look for and read all of the fields */
+    for (index = 0; index < FIELD_MAX; index++)
+    {
+        /* Advance to the next field */
+        bool last_field = tok_next_field( &end );
+        /* copy the filed data */
+        tok_read_field( index, &Buffer[start] );
+        /* advance to next field or exit if last field */
+        if (last_field) break;
+        else start = ++end;
+    }
+
+    /* Copy the token to the heap */
+    if (index == FIELD_TEXT)
+    {
+        ret = (tok_t*)calloc(1,sizeof(tok_t));
+        *ret = Token;
+    }
+
+    return ret;
+}
+
+bool tok_next_field(uint32_t* end)
+{
+    bool last_field = false;
+    while (('\t' != Buffer[*end]) &&
+           !(('\r' == Buffer[*end]) ||
+             ('\0' == Buffer[*end])))
+    {
+        (*end)++;
+    }
+    last_field = (('\r' == Buffer[*end]) ||
+                  ('\0' == Buffer[*end])) ? true : false;
+    Buffer[*end] = '\0';
+    return last_field;
+}
+
+void tok_read_field(uint32_t index, char* str)
+{
+    uint32_t type;
+    switch (index)
+    {
+        case FIELD_TYPE:
+            for (type = 0; type < TOK_MAX; type++)
+            {
+                if (0 == strcmp(Token_Types[type],str))
+                {
+                    Token.type = (tok_type_t)type;
+                    break;
+                }
+            }
+            break;
+
+        case FIELD_LINE_NUM:
+            Token.line = atoi( str );
+            break;
+
+        case FIELD_COLUMN_NUM:
+            Token.column = atoi( str );
+            break;
+
+        case FIELD_TEXT:
+            Token.p_text = (char*)malloc(strlen(str));
+            strcpy( Token.p_text, str );
+            break;
+
+        default:
+            tok_fatal_error(1);
+            break;
+    }
+}
+
+void tok_fatal_error(uint32_t err_code)
+{
+    fprintf(stderr,"Fatal Error\n");
+    exit(err_code);
+}
+
diff --git a/source/parser/tok.h b/source/parser/tok.h
new file mode 100644 (file)
index 0000000..2070070
--- /dev/null
@@ -0,0 +1,33 @@
+/**
+    @file tok.h
+    @brief TODO: Describe this file
+    $Revision$
+    $HeadURL$
+*/
+#ifndef TOK_H
+#define TOK_H
+
+#include <stdbool.h>
+#include <stdint.h>
+#include "tokens.h"
+
+typedef struct
+{
+    tok_type_t type;
+    char* p_text;
+    char* p_file_name;
+    long line;
+    long column;
+} tok_t;
+
+bool tok_source(char* fname);
+bool tok_eof(void);
+void tok_finish(void);
+tok_t* tok_read(void);
+void tok_read_fname(void);
+tok_t* tok_build_token(void);
+bool tok_next_field(uint32_t* end);
+void tok_read_field(uint32_t index, char* str);
+void tok_fatal_error(uint32_t err_code);
+
+#endif /* TOK_H */
index fa08d499b41b25c83353918a146648f7a004ffef..f0e77fe428890985c8a994293911c4ada9efae39 100644 (file)
@@ -5,6 +5,7 @@
     $HeadURL$
 */
 #include "types.h"
+//#include "gc.h"
 
 var_t new_num(double val)
 {