From ff01ce7cc007b2b5fe007c348bb161de8c04c1a8 Mon Sep 17 00:00:00 2001 From: "Michael D. Lowis" Date: Mon, 11 Mar 2019 15:44:54 -0400 Subject: [PATCH] updated token printing to reflect new token types --- example.src | 2 +- source/pprint.c | 35 ++++++--- source/rt/gc.c | 178 -------------------------------------------- source/rt/main.c | 9 --- spec/lexer_spec.rb | 46 +++++------- spec/spec_helper.rb | 2 +- 6 files changed, 48 insertions(+), 224 deletions(-) delete mode 100644 source/rt/gc.c delete mode 100644 source/rt/main.c diff --git a/example.src b/example.src index 2b1dcd7..0012407 100644 --- a/example.src +++ b/example.src @@ -27,7 +27,7 @@ type type_union = union { } fun main(args string[]) int { - let foo int = 123 + let foo int = 123u var bar int = 123 {123} 123 diff --git a/source/pprint.c b/source/pprint.c index 011706c..b4e3344 100644 --- a/source/pprint.c +++ b/source/pprint.c @@ -6,13 +6,14 @@ static void print_indent(FILE* file, int depth) { } static const char* token_type_to_string(int type) { + #define TOK(name) case (name): return #name switch(type) { - case T_STRING: return "T_STRING"; - case T_CHAR: return "T_CHAR"; - case T_INT: return "T_INT"; - case T_FLOAT: return "T_FLOAT"; - case T_BOOL: return "T_BOOL"; - case T_ID: return "T_ID"; + TOK(T_NONE); TOK(T_ERROR); TOK(T_END_FILE); TOK(T_PACKAGE); + TOK(T_REQUIRES); TOK(T_PROVIDES); TOK(T_LET); TOK(T_VAR); + TOK(T_FUN); TOK(T_TYPE); TOK(T_STRUCT); TOK(T_UNION); + TOK(T_RETURN); TOK(T_IF); TOK(T_ELSE); TOK(T_ID); + TOK(T_CHAR); TOK(T_INT); TOK(T_FLOAT); TOK(T_BOOL); + TOK(T_STRING); case '{': return "T_LBRACE"; case '}': return "T_RBRACE"; case '[': return "T_LBRACK"; @@ -24,9 +25,9 @@ static const char* token_type_to_string(int type) { case '&': return "T_AMP"; case '\'': return "T_SQUOTE"; case '"': return "T_DQUOTE"; - case T_END_FILE: return "T_END_FILE"; default: return "???"; } + #undef TOK } static void print_char(FILE* file, char ch) { @@ -48,19 +49,35 @@ static void print_char(FILE* file, char ch) { } void pprint_token_type(FILE* file, Tok* token) { - fprintf(file, "%s", token_type_to_string(token->type)); + if (token->type > 256) + fprintf(file, "%s", token_type_to_string(token->type)); + else + fprintf(file, "%c", token->type); } void pprint_token_value(FILE* file, Tok* token) { + #define TOK(name) case (name): fprintf(file, "%s", #name); break switch(token->type) { + /* value tokens */ case T_STRING: fprintf(file, "\"%s\"", token->value.text); break; case T_ID: fprintf(file, "%s", token->value.text); break; case T_CHAR: print_char(file, token->value.character); break; case T_INT: fprintf(file, "%ld", token->value.integer); break; case T_FLOAT: fprintf(file, "%f", token->value.floating); break; case T_BOOL: fprintf(file, "%s", (token->value.boolean)?"true":"false"); break; - default: fprintf(file, "???"); break; + + /* keyword tokens */ + TOK(T_NONE); TOK(T_ERROR); TOK(T_END_FILE); TOK(T_PACKAGE); + TOK(T_REQUIRES); TOK(T_PROVIDES); TOK(T_LET); TOK(T_VAR); + TOK(T_FUN); TOK(T_TYPE); TOK(T_STRUCT); TOK(T_UNION); TOK(T_RETURN); + TOK(T_IF); TOK(T_ELSE); + + /* evertything else */ + default: + fprintf(file, "???"); + break; } + #undef TOK } void pprint_token(FILE* file, Tok* token, bool print_loc) diff --git a/source/rt/gc.c b/source/rt/gc.c deleted file mode 100644 index 3dec0ed..0000000 --- a/source/rt/gc.c +++ /dev/null @@ -1,178 +0,0 @@ -#include -#include -#include - -#define NENTRIES 4096u -#define DIRTY_BIT 1u - -/* garbage collected block definition */ -typedef struct obj_t { - void (*dtor)(void*); /* destructor function for cleaning up on free() */ - size_t refs; /* count of references to this object */ - uint8_t data[]; /* user data portion of the block */ -} obj_t; - -/* log entry definition */ -typedef struct { - void* ref; /* pointer to the updated cell */ - void* val; /* pointer to the original cell value */ -} entry_t; - -size_t EntryCount = 0; -entry_t EntryLog[NENTRIES] = {0}; - -void gc_collect(void) { - -} - -void* gc_alloc(size_t sz, void (*dtor)(void*)) { - obj_t* obj = malloc(sizeof(obj_t) + sz); - if (obj) { - obj->dtor = dtor; - obj->refs = 0u; - EntryLog[EntryCount].ref = NULL; - EntryLog[EntryCount].val = obj->data; - if (++EntryCount == NENTRIES) - gc_collect(); - } else { - perror("refcreate() failed :"); - abort(); - } - return obj->data; -} - -void gc_setref(void** ref, void* val) { - if (!((uintptr_t)*ref & DIRTY_BIT)) { - EntryLog[EntryCount].ref = ref; - EntryLog[EntryCount].val = val; - if (!((uintptr_t)*ref & DIRTY_BIT)) { - *(uintptr_t*)ref |= DIRTY_BIT; - EntryCount++; - } - if (EntryCount == NENTRIES) - gc_collect(); - } - *ref = val; -} - -void* gc_getref(void** ref) { - return (void*)((uintptr_t)*ref & ~DIRTY_BIT); -} - -/*****************************************************************************/ - -typedef struct hash_entry_t { - struct hash_entry_t* next; - uintptr_t hash; -} hash_entry_t; - -typedef struct { - size_t size; - size_t bktid; - hash_entry_t** buckets; -} hash_t; - -static void hash_set(hash_t* hash, hash_entry_t* entry); - -#define NUM_PRIMES (sizeof(Primes)/sizeof(unsigned int)) - -static unsigned int Primes[] = { - 5, 13, 23, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, - 49157, 98317, 196613, 393241, 786433, 1572869, 3145739, 6291469, - 12582917, 25165843, 50331653, 100663319, 201326611, 402653189, - 805306457, 1610612741 -}; - -static void hash_init(hash_t* hash) { - hash->size = 0; - hash->bktid = 0; - hash->buckets = (hash_entry_t**)calloc(sizeof(hash_entry_t*), Primes[hash->bktid]); -} - -static void find_entry(hash_entry_t** parent, hash_entry_t** current, hash_entry_t* entry) { - while(*current != NULL) { - if (((*current)->hash == entry->hash) && - ((*current)->object == entry->object)) - break; - *parent = *current; - *current = (*current)->next; - } -} - -static void rehash(hash_t* hash) { - unsigned int oldcount, i; - hash_entry_t** oldbuckets; - hash_entry_t *node, *entry; - if ((hash->bktid + 1) < NUM_PRIMES) { - oldcount = hash->bktid++; - oldbuckets = hash->buckets; - hash->buckets = (hash_entry_t**)calloc(sizeof(hash_entry_t*), Primes[hash->bktid]); - hash->size = 0; - /* Iterate over all of the old buckets */ - for (i = 0; i < Primes[oldcount]; i++) { - node = oldbuckets[i]; - /* re-insert all entries in the bucket into the new bucket table */ - while (node != NULL) { - entry = node; - node = entry->next; - hash_set(hash, entry); - } - } - /* Free the old bucket table */ - free(oldbuckets); - } -} - -static uint64_t hash64(uint64_t key) { - key = (~key) + (key << 21); - key = key ^ (key >> 24); - key = (key + (key << 3)) + (key << 8); - key = key ^ (key >> 14); - key = (key + (key << 2)) + (key << 4); - key = key ^ (key >> 28); - key = key + (key << 31); - return key; -} - -static void hash_set(hash_t* hash, hash_entry_t* entry) { - unsigned int index; - hash_entry_t *parent, *node, *deadite; - if (hash->size >= Primes[hash->bktid]) - rehash(hash); - entry->hash = hash64((uint64_t)(entry->object)); - index = (entry->hash % Primes[hash->bktid]); - parent = NULL; - node = hash->buckets[index]; - deadite = NULL; - find_entry(&parent, &node, entry); - if ((parent == NULL) && (node == NULL)) { - hash->buckets[index] = entry; - entry->next = NULL; - hash->size++; - } else if (node == NULL) { - parent->next = entry; - entry->next = NULL; - hash->size++; - } -} - -static hash_entry_t* hash_del(hash_t* hash, hash_entry_t* entry) { - unsigned int index; - hash_entry_t *parent, *node, *ret = NULL; - entry->hash = hash64((uint64_t)(entry->object)); - index = (entry->hash % Primes[hash->bktid]); - parent = NULL; - node = hash->buckets[index]; - find_entry(&parent, &node, entry); - if (node != NULL) { - ret = node; - node = node->next; - if (parent != NULL) - parent->next = node; - else - hash->buckets[index] = node; - hash->size--; - } - return ret; -} - diff --git a/source/rt/main.c b/source/rt/main.c deleted file mode 100644 index fbb53ce..0000000 --- a/source/rt/main.c +++ /dev/null @@ -1,9 +0,0 @@ -#include - -uintptr_t* StackBot; - -int main(int argc, char** argv) { - { StackBot = (uintptr_t*)&(int){0}; } - extern int usermain(int argc, char** argv); - return usermain(argc, argv); -} diff --git a/spec/lexer_spec.rb b/spec/lexer_spec.rb index 1a7c93c..a9add8b 100644 --- a/spec/lexer_spec.rb +++ b/spec/lexer_spec.rb @@ -3,81 +3,75 @@ require 'spec_helper' describe "lexer" do context "punctuation" do it "should recognize [" do - expect(lexer('[')).to eq ['T_LBRACK'] + expect(lexer('[')).to eq ['['] end it "should recognize ]" do - expect(lexer(']')).to eq ['T_RBRACK'] + expect(lexer(']')).to eq [']'] end it "should recognize (" do - expect(lexer('(')).to eq ['T_LPAR'] + expect(lexer('(')).to eq ['('] end it "should recognize )" do - expect(lexer(')')).to eq ['T_RPAR'] + expect(lexer(')')).to eq [')'] end it "should recognize {" do - expect(lexer('{')).to eq ['T_LBRACE'] + expect(lexer('{')).to eq ['{'] end it "should recognize }" do - expect(lexer('}')).to eq ['T_RBRACE'] + expect(lexer('}')).to eq ['}'] end it "should recognize '" do - expect(lexer('\'')).to eq ['T_SQUOTE'] + expect(lexer('\'')).to eq ['\''] end it "should recognize ," do - expect(lexer(',')).to eq ['T_COMMA'] + expect(lexer(',')).to eq [','] end it "should recognize :" do - expect(lexer(':')).to eq ['T_COLON'] + expect(lexer(':')).to eq [':'] end it "should recognize all punctuation" do - expect(lexer('[](){}\',')).to eq( - ["T_LBRACK", "T_RBRACK", "T_LPAR", "T_RPAR", "T_LBRACE", "T_RBRACE", - "T_SQUOTE", "T_COMMA"]) + expect(lexer('[](){}\',')).to eq( ["[", "]", "(", ")", "{", "}", "'", ","]) end it "should recognize [ after an identifier" do - expect(lexer('foo[')).to eq(['T_ID:foo', 'T_LBRACK']) + expect(lexer('foo[')).to eq(['T_ID:foo', '[']) end it "should recognize ] after an identifier" do - expect(lexer('foo]')).to eq(['T_ID:foo', 'T_RBRACK']) + expect(lexer('foo]')).to eq(['T_ID:foo', ']']) end it "should recognize ( after an identifier" do - expect(lexer('foo(')).to eq(['T_ID:foo', 'T_LPAR']) + expect(lexer('foo(')).to eq(['T_ID:foo', '(']) end it "should recognize ) after an identifier" do - expect(lexer('foo)')).to eq(['T_ID:foo', 'T_RPAR']) + expect(lexer('foo)')).to eq(['T_ID:foo', ')']) end it "should recognize { after an identifier" do - expect(lexer('foo{')).to eq(['T_ID:foo', 'T_LBRACE']) + expect(lexer('foo{')).to eq(['T_ID:foo', '{']) end it "should recognize } after an identifier" do - expect(lexer('foo}')).to eq(['T_ID:foo', 'T_RBRACE']) + expect(lexer('foo}')).to eq(['T_ID:foo', '}']) end - it "should recognize } after an identifier" do - expect(lexer('foo\'')).to eq(['T_ID:foo', 'T_SQUOTE']) + it "should recognize ' after an identifier" do + expect(lexer('foo\'')).to eq(['T_ID:foo', '\'']) end - it "should recognize } after an identifier" do - expect(lexer('foo,')).to eq(['T_ID:foo', 'T_COMMA']) - end - - it "should recognize } after an identifier" do - expect(lexer('foo')).to eq(['T_ID:foo']) + it "should recognize , after an identifier" do + expect(lexer('foo,')).to eq(['T_ID:foo', ',']) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 37828ae..52929bb 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -9,7 +9,7 @@ def cli(options, input = "") end def lexer(input) - cli(['-Atok'], input).scan(/^\d+:\d+:(T_[A-Z]+(:("[^"]*"|[^\n]+))?)/m).map {|m| m[0] } + cli(['-Atok'], input).scan(/^\d+:\d+:((:|[^:\n]+)(:("[^"]*"|[^\n]+))?)/m).map {|m| m[0] } end def re_structure( token_array, offset = 0 ) -- 2.54.0