From 8dbeb4f38aab1faed2ce1ef78e2a8fce30ec9faa Mon Sep 17 00:00:00 2001 From: "Mike D. Lowis" Date: Tue, 26 Feb 2013 20:17:01 -0500 Subject: [PATCH] updated new design using tok module --- source/lexer/classes.h | 1 - source/lexer/lex.c | 117 ++++++++++++++++++++--------------------- source/lexer/lex.h | 27 ++-------- source/lexer/main.c | 3 +- source/lexer/tok.c | 49 +++++++++++++++++ source/lexer/tok.h | 28 ++++++++++ 6 files changed, 140 insertions(+), 85 deletions(-) create mode 100644 source/lexer/tok.c create mode 100644 source/lexer/tok.h diff --git a/source/lexer/classes.h b/source/lexer/classes.h index cb935de..c2b10b0 100644 --- a/source/lexer/classes.h +++ b/source/lexer/classes.h @@ -17,6 +17,5 @@ bool hex_digit(void); bool token_end(void); bool matches(char ch); bool matches_any(char* str); -bool one_or_more(predicate_t pfn); #endif /* CLASSES_H */ diff --git a/source/lexer/lex.c b/source/lexer/lex.c index 9cc160c..88e36d3 100644 --- a/source/lexer/lex.c +++ b/source/lexer/lex.c @@ -7,15 +7,30 @@ #include #include #include "lex.h" +#include "tok.h" #include "classes.h" #include "file.h" #include "buf.h" +/* Prototypes + *****************************************************************************/ +static void accept(void); +static void accept_char(tok_type_t tok); +static void abort(void); +static void reset(void); +static void match_and_consume(char ch); +static bool one_or_more(predicate_t pfn); +static void punctuation(void); +static void number(void); +static void hexadecimal(void); +static void floating_point(void); +static void identifier(void); + /* Global Variables *****************************************************************************/ -tok_t Token; jmp_buf Jump_Point; -const char* Token_Strings[TOK_MAX] = { + +const char* Types[TOK_MAX] = { "EOF", /* TOK_EOF */ "ID", /* TOK_ID */ "NUM", /* TOK_NUM */ @@ -30,69 +45,52 @@ const char* Token_Strings[TOK_MAX] = { /* Control Functions *****************************************************************************/ -void record_position(void) +static void accept(void) { - Token.line = file_line(); - Token.column = file_column(); -} - -void set_type(tok_type_t type) -{ - Token.type = Token_Strings[type]; + if (!token_end()) + abort(); + else + tok_accept(); } -void consume(void) +static void accept_char(tok_type_t tok) { - buf_put( file_get() ); + tok_set_type( Types[tok] ); + tok_consume(); + tok_accept(); } -void match_consume(char ch) +static void abort(void) { - if (matches(ch)) - consume(); - else - abort(); + longjmp(Jump_Point,1); } -void prepare_for_token(void) +static void reset(void) { - (void)memset(&Token,0,sizeof(Token)); while( whitespace() ) - (void)file_get(); - record_position(); + tok_discard(); + tok_reset(); } -void accept_char(tok_type_t type) +static void match_and_consume(char ch) { - set_type(type); - consume(); - accept(); -} - -void accept(void) -{ - if (!token_end()) - abort(); + if (matches(ch)) + tok_consume(); else - Token.str = buf_accept(); -} - -void abort(void) -{ - longjmp(Jump_Point,1); + abort(); } -bool one_or_more(predicate_t pfn) +static bool one_or_more(predicate_t pfn) { if (!pfn()) abort(); - while (pfn()) consume(); + while (pfn()) tok_consume(); } /* Token Matching Functions *****************************************************************************/ -tok_t next_token(void) +void next_token(tok_t* p_token) { - prepare_for_token(); + reset(); if (!file_eof()) { /* Mark our starting point so we can resume if we abort */ @@ -115,30 +113,31 @@ tok_t next_token(void) } /* the keyword "end" is actually a TOK_TERM */ - if (0 == strcmp(Token.str,"end")) - set_type(TOK_TERM); + if (0 == strcmp(tok_string(),"end")) + tok_set_type(Types[TOK_TERM]); + } - return Token; + tok_copy( p_token ); } -void punctuation(void) +static void punctuation(void) { switch (file_peek()) { - case '(': accept_char( TOK_LPAR ); break; - case ')': accept_char( TOK_RPAR ); break; + case '(': accept_char( TOK_LPAR ); break; + case ')': accept_char( TOK_RPAR ); break; case '[': accept_char( TOK_LBRACK ); break; case ']': accept_char( TOK_RBRACK ); break; case '{': accept_char( TOK_LBRACE ); break; case '}': accept_char( TOK_RBRACE ); break; - case ';': accept_char( TOK_TERM ); break; + case ';': accept_char( TOK_TERM ); break; default: identifier(); break; } } -void number(void) +static void number(void) { - set_type(TOK_NUM); + tok_set_type(Types[TOK_NUM]); if (matches('h')) { hexadecimal(); @@ -148,35 +147,35 @@ void number(void) floating_point(); if (matches_any("eE")) { - consume(); + tok_consume(); floating_point(); } } accept(); } -void hexadecimal(void) +static void hexadecimal(void) { - match_consume('h'); + match_and_consume('h'); one_or_more( hex_digit ); } -void floating_point(void) +static void floating_point(void) { - if (matches('-')) consume(); + if (matches('-')) tok_consume(); one_or_more( digit ); if (matches('.')) { - consume(); + tok_consume(); one_or_more( digit ); } } -void identifier(void) +static void identifier(void) { - set_type(TOK_ID); + tok_set_type(Types[TOK_ID]); while (!token_end()) - consume(); + tok_consume(); accept(); } diff --git a/source/lexer/lex.h b/source/lexer/lex.h index 5bfd45a..ef4a08d 100644 --- a/source/lexer/lex.h +++ b/source/lexer/lex.h @@ -7,16 +7,9 @@ #ifndef LEX_H #define LEX_H +#include "tok.h" #include "classes.h" -typedef struct -{ - int line; - int column; - const char* type; - char* str; -} tok_t; - typedef enum { TOK_EOF = 0, TOK_ID = 1, @@ -29,22 +22,8 @@ typedef enum { TOK_RBRACE = 8, TOK_TERM = 9, TOK_MAX = 10, -} tok_type_t; +} lex_tok_t; -tok_t next_token(void); -void punctuation(void); -void record_position(void); -void identifier(void); -void number(void); -void hexadecimal(void); -void floating_point(void); -void set_type(tok_type_t type); -void consume(void); -void match_consume(char ch); -void prepare_for_token(void); -void accept_char(tok_type_t type); -void accept(void); -void abort(void); -bool one_or_more(predicate_t pfn); +void next_token(tok_t* p_token); #endif /* LEX_H */ diff --git a/source/lexer/main.c b/source/lexer/main.c index 41e2f8e..9fc64d2 100644 --- a/source/lexer/main.c +++ b/source/lexer/main.c @@ -48,10 +48,11 @@ int lex_files(int argc, char** argv) int lex_input(FILE* outfile) { + tok_t token; int ret = 0; while (!file_eof()) { - tok_t token = next_token(); + next_token( &token ); if (token.type != NULL) fprintf(outfile, "%s\t%d\t%d\t%s\n", token.type, token.line, token.column, token.str); free(token.str); diff --git a/source/lexer/tok.c b/source/lexer/tok.c new file mode 100644 index 0000000..84b2302 --- /dev/null +++ b/source/lexer/tok.c @@ -0,0 +1,49 @@ +/** + @file tok.c + @brief See header for details + $Revision$ + $HeadURL$ +*/ +#include +#include "tok.h" +#include "buf.h" + +tok_t Token; + +void tok_reset(void) +{ + (void)memset(&Token,0,sizeof(Token)); + Token.line = file_line(); + Token.column = file_column(); +} + +void tok_copy(tok_t* p_token) +{ + *(p_token) = Token; +} + +void tok_set_type(const char* p_str) +{ + Token.type = p_str; +} + +char* tok_string(void) +{ + return Token.str; +} + +void tok_consume(void) +{ + buf_put( file_get() ); +} + +void tok_discard(void) +{ + (void)file_get(); +} + +void tok_accept(void) +{ + Token.str = buf_accept(); +} + diff --git a/source/lexer/tok.h b/source/lexer/tok.h new file mode 100644 index 0000000..2860301 --- /dev/null +++ b/source/lexer/tok.h @@ -0,0 +1,28 @@ +/** + @file tok.h + @brief TODO: Describe this file + $Revision$ + $HeadURL$ +*/ +#ifndef TOK_H +#define TOK_H + +typedef int tok_type_t; + +typedef struct +{ + int line; + int column; + const char* type; + char* str; +} tok_t; + +void tok_reset(void); +void tok_copy(tok_t* p_token); +void tok_set_type(const char* p_str); +char* tok_string(void); +void tok_consume(void); +void tok_discard(void); +void tok_accept(void); + +#endif /* TOK_H */ -- 2.52.0