From 5c6eea20be389391a83b515caaf81189a8ae61d6 Mon Sep 17 00:00:00 2001 From: "Mike D. Lowis" Date: Fri, 29 Mar 2013 12:17:20 -0400 Subject: [PATCH] Implemented Token Parsing --- premake4.lua | 64 ++++++++--- source/common/tokens/tokens.c | 22 ++++ source/common/{ => tokens}/tokens.h | 2 + source/lexer/lex.c | 23 +--- source/parser/main.c | 23 +++- source/parser/tok.c | 169 ++++++++++++++++++++++++++++ source/parser/tok.h | 33 ++++++ source/runtime/types/types.c | 1 + 8 files changed, 299 insertions(+), 38 deletions(-) create mode 100644 source/common/tokens/tokens.c rename source/common/{ => tokens}/tokens.h (90%) create mode 100644 source/parser/tok.c create mode 100644 source/parser/tok.h diff --git a/premake4.lua b/premake4.lua index 5bd8aca..f58a2e2 100644 --- a/premake4.lua +++ b/premake4.lua @@ -12,16 +12,28 @@ project "sclpl-rt" kind "SharedLib" language "C" location "build" - files { "source/runtime/**.*" } + files { + "source/runtime/**.*" + } project "sclpl-rt-tests" kind "ConsoleApp" language "C++" location "build" - links { "UnitTest++", "sclpl-rt" } - includedirs { "source/runtime/**", "tools/UnitTest++/**" } - files { "tests/runtime/*.c*" } - postbuildcommands { "./sclpl-rt-tests" } + links { + "UnitTest++", + "sclpl-rt" + } + includedirs { + "source/runtime/**", + "tools/UnitTest++/**" + } + files { + "tests/runtime/*.c*" + } + postbuildcommands { + "./sclpl-rt-tests" + } ------------------------------------------------------------------------------- -- SCLPL Lexer @@ -32,22 +44,31 @@ project "sclpl-lex" location "build" includedirs { "source/lexer/**", - "source/common/", - "source/runtime/**" + "source/runtime/**", + "source/common/**", } files { "source/lexer/**.*", - "source/runtime/collector/**.*" + "source/common/**.*", } project "sclpl-lex-tests" kind "ConsoleApp" language "C++" location "build" - links { "UnitTest++" } - includedirs { "source/lexer/**", "tools/UnitTest++/**" } - files { "tests/lexer/*.c*" } - postbuildcommands { "./sclpl-lex-tests" } + links { + "UnitTest++" + } + includedirs { + "source/lexer/**", + "tools/UnitTest++/**" + } + files { + "tests/lexer/*.c*" + } + postbuildcommands { + "./sclpl-lex-tests" + } ------------------------------------------------------------------------------- -- SCLPL Parser @@ -59,7 +80,7 @@ project "sclpl-parse" includedirs { "source/lexer/**", "source/runtime/**", - "source/common/**" + "source/common/**", } files { "source/parser/**.*", @@ -71,10 +92,19 @@ project "sclpl-parse-tests" kind "ConsoleApp" language "C++" location "build" - links { "UnitTest++" } - includedirs { "source/parser/**", "tools/UnitTest++/**" } - files { "tests/parser/*.c*" } - postbuildcommands { "./sclpl-parse-tests" } + links { + "UnitTest++" + } + includedirs { + "source/parser/**", + "tools/UnitTest++/**" + } + files { + "tests/parser/*.c*" + } + postbuildcommands { + "./sclpl-parse-tests" + } ------------------------------------------------------------------------------- -- UnitTest++ - A C/C++ unit testing library diff --git a/source/common/tokens/tokens.c b/source/common/tokens/tokens.c new file mode 100644 index 0000000..8985269 --- /dev/null +++ b/source/common/tokens/tokens.c @@ -0,0 +1,22 @@ +/** + @file tokens.c + @brief See header for details + $Revision$ + $HeadURL$ +*/ +#include "tokens.h" + +const char* Token_Types[TOK_MAX] = { + "EOF", /* TOK_EOF */ + "ID", /* TOK_ID */ + "NUM", /* TOK_NUM */ + "LPAREN", /* TOK_LPAR */ + "RPAREN", /* TOK_RPAR */ + "LBRACK", /* TOK_LBRACK */ + "RBRACK", /* TOK_RBRACK */ + "LBRACE", /* TOK_LBRACE */ + "RBRACE", /* TOK_RBRACE */ + "TERM", /* TOK_TERM */ + "BOOL", /* TOK_BOOL */ +}; + diff --git a/source/common/tokens.h b/source/common/tokens/tokens.h similarity index 90% rename from source/common/tokens.h rename to source/common/tokens/tokens.h index fb28466..ad6009e 100644 --- a/source/common/tokens.h +++ b/source/common/tokens/tokens.h @@ -22,4 +22,6 @@ typedef enum { TOK_MAX = 11, } tok_type_t; +extern const char* Token_Types[TOK_MAX]; + #endif /* TOKENS_H */ diff --git a/source/lexer/lex.c b/source/lexer/lex.c index cc697af..c581979 100644 --- a/source/lexer/lex.c +++ b/source/lexer/lex.c @@ -11,6 +11,7 @@ #include "classes.h" #include "file.h" #include "buf.h" +#include "tokens.h" /* Prototypes *****************************************************************************/ @@ -33,20 +34,6 @@ static void identifier(void); *****************************************************************************/ jmp_buf Jump_Point; -const char* Types[TOK_MAX] = { - "EOF", /* TOK_EOF */ - "ID", /* TOK_ID */ - "NUM", /* TOK_NUM */ - "LPAREN", /* TOK_LPAR */ - "RPAREN", /* TOK_RPAR */ - "LBRACK", /* TOK_LBRACK */ - "RBRACK", /* TOK_RBRACK */ - "LBRACE", /* TOK_LBRACE */ - "RBRACE", /* TOK_RBRACE */ - "TERM", /* TOK_TERM */ - "BOOL", /* TOK_BOOL */ -}; - const lex_keyword_t Keywords[] = { { "end", TOK_TERM }, { "true", TOK_BOOL }, @@ -66,7 +53,7 @@ static void accept(void) static void accept_char(tok_type_t tok) { - tok_set_type( Types[tok] ); + tok_set_type( Token_Types[tok] ); tok_consume(); tok_accept(); } @@ -137,7 +124,7 @@ static void keyword(void) { if (0 == strcmp( p_text, Keywords[i].p_text )) { - tok_set_type( Types[ Keywords[i].type ] ); + tok_set_type( Token_Types[ Keywords[i].type ] ); break; } i++; @@ -169,7 +156,7 @@ static void punctuation(void) static void number(void) { - tok_set_type(Types[TOK_NUM]); + tok_set_type(Token_Types[TOK_NUM]); if (matches('0')) { tok_consume(); @@ -218,7 +205,7 @@ static void exponent(void) static void identifier(void) { - tok_set_type(Types[TOK_ID]); + tok_set_type(Token_Types[TOK_ID]); while (!token_end()) tok_consume(); accept(); diff --git a/source/parser/main.c b/source/parser/main.c index 2bd0d10..2e60382 100644 --- a/source/parser/main.c +++ b/source/parser/main.c @@ -1,4 +1,5 @@ #include +#include "tok.h" int parse_files(int num_files, char** fnames); int parse_input(char* outfile); @@ -29,9 +30,25 @@ int parse_files(int num_files, char** fnames) int parse_input(char* outfile) { - //tok_open(outfile); - - //tok_close(); + int ret = 0; + if (tok_source(outfile)) + { + while (!tok_eof()) + { + tok_t* p_tok = tok_read(); + if (NULL != p_tok) + { + printf( "%s:%d:%d:\t%d\t%s", + p_tok->p_file_name, + p_tok->line, + p_tok->column, + p_tok->type, + p_tok->p_text ); + } + } + tok_finish(); + } + return ret; } diff --git a/source/parser/tok.c b/source/parser/tok.c new file mode 100644 index 0000000..b4f63b9 --- /dev/null +++ b/source/parser/tok.c @@ -0,0 +1,169 @@ +/** + @file tok.c + @brief See header for details + $Revision$ + $HeadURL$ +*/ +#include +#include +#include +#include "tok.h" +#include "tokens.h" + +#define FIELD_TYPE 0 +#define FIELD_LINE_NUM 1 +#define FIELD_COLUMN_NUM 2 +#define FIELD_TEXT 3 +#define FIELD_MAX 4 + +#define MAX_INPUT_STR 1024 +FILE* Handle = NULL; +tok_t Token = { 0 }; +char Buffer[MAX_INPUT_STR]; + +bool tok_source(char* fname) +{ + if (NULL == fname) + { + Handle = stdin; + } + else + { + Handle = fopen(fname,"r"); + } + return (NULL != Handle); +} + +bool tok_eof(void) +{ + bool ret = true; + if (NULL != Handle) + { + ret = feof( Handle ); + } + return ret; +} + +void tok_finish(void) +{ + fclose(Handle); +} + +tok_t* tok_read(void) +{ + tok_t* p_tok = NULL; + if (NULL != Handle) + { + fgets(Buffer, MAX_INPUT_STR, Handle); + if ('\0' != Buffer[0]) + { + if ('@' == Buffer[0]) + { + tok_read_fname(); + p_tok = tok_read(); + } + else + { + p_tok = tok_build_token(); + } + Buffer[0] = '\0'; + } + } + return p_tok; +} + +void tok_read_fname(void) +{ + uint32_t index = 0; + Buffer[strlen(Buffer)-1] = '\0'; + while(' ' != Buffer[index]) index++; + char* new_str = (char*)malloc( strlen(&Buffer[index+1]) ); + strcpy( new_str, &Buffer[index+1] ); + Token.p_file_name = new_str; +} + +tok_t* tok_build_token(void) +{ + tok_t* ret = NULL; + uint32_t index; + uint32_t start = 0; + uint32_t end = 0; + + /* Look for and read all of the fields */ + for (index = 0; index < FIELD_MAX; index++) + { + /* Advance to the next field */ + bool last_field = tok_next_field( &end ); + /* copy the filed data */ + tok_read_field( index, &Buffer[start] ); + /* advance to next field or exit if last field */ + if (last_field) break; + else start = ++end; + } + + /* Copy the token to the heap */ + if (index == FIELD_TEXT) + { + ret = (tok_t*)calloc(1,sizeof(tok_t)); + *ret = Token; + } + + return ret; +} + +bool tok_next_field(uint32_t* end) +{ + bool last_field = false; + while (('\t' != Buffer[*end]) && + !(('\r' == Buffer[*end]) || + ('\0' == Buffer[*end]))) + { + (*end)++; + } + last_field = (('\r' == Buffer[*end]) || + ('\0' == Buffer[*end])) ? true : false; + Buffer[*end] = '\0'; + return last_field; +} + +void tok_read_field(uint32_t index, char* str) +{ + uint32_t type; + switch (index) + { + case FIELD_TYPE: + for (type = 0; type < TOK_MAX; type++) + { + if (0 == strcmp(Token_Types[type],str)) + { + Token.type = (tok_type_t)type; + break; + } + } + break; + + case FIELD_LINE_NUM: + Token.line = atoi( str ); + break; + + case FIELD_COLUMN_NUM: + Token.column = atoi( str ); + break; + + case FIELD_TEXT: + Token.p_text = (char*)malloc(strlen(str)); + strcpy( Token.p_text, str ); + break; + + default: + tok_fatal_error(1); + break; + } +} + +void tok_fatal_error(uint32_t err_code) +{ + fprintf(stderr,"Fatal Error\n"); + exit(err_code); +} + diff --git a/source/parser/tok.h b/source/parser/tok.h new file mode 100644 index 0000000..2070070 --- /dev/null +++ b/source/parser/tok.h @@ -0,0 +1,33 @@ +/** + @file tok.h + @brief TODO: Describe this file + $Revision$ + $HeadURL$ +*/ +#ifndef TOK_H +#define TOK_H + +#include +#include +#include "tokens.h" + +typedef struct +{ + tok_type_t type; + char* p_text; + char* p_file_name; + long line; + long column; +} tok_t; + +bool tok_source(char* fname); +bool tok_eof(void); +void tok_finish(void); +tok_t* tok_read(void); +void tok_read_fname(void); +tok_t* tok_build_token(void); +bool tok_next_field(uint32_t* end); +void tok_read_field(uint32_t index, char* str); +void tok_fatal_error(uint32_t err_code); + +#endif /* TOK_H */ diff --git a/source/runtime/types/types.c b/source/runtime/types/types.c index fa08d49..f0e77fe 100644 --- a/source/runtime/types/types.c +++ b/source/runtime/types/types.c @@ -5,6 +5,7 @@ $HeadURL$ */ #include "types.h" +//#include "gc.h" var_t new_num(double val) { -- 2.49.0