}
fun main(args string[]) int {
- let foo int = 123
+ let foo int = 123u
var bar int = 123
{123}
123
}
static const char* token_type_to_string(int type) {
+ #define TOK(name) case (name): return #name
switch(type) {
- case T_STRING: return "T_STRING";
- case T_CHAR: return "T_CHAR";
- case T_INT: return "T_INT";
- case T_FLOAT: return "T_FLOAT";
- case T_BOOL: return "T_BOOL";
- case T_ID: return "T_ID";
+ TOK(T_NONE); TOK(T_ERROR); TOK(T_END_FILE); TOK(T_PACKAGE);
+ TOK(T_REQUIRES); TOK(T_PROVIDES); TOK(T_LET); TOK(T_VAR);
+ TOK(T_FUN); TOK(T_TYPE); TOK(T_STRUCT); TOK(T_UNION);
+ TOK(T_RETURN); TOK(T_IF); TOK(T_ELSE); TOK(T_ID);
+ TOK(T_CHAR); TOK(T_INT); TOK(T_FLOAT); TOK(T_BOOL);
+ TOK(T_STRING);
case '{': return "T_LBRACE";
case '}': return "T_RBRACE";
case '[': return "T_LBRACK";
case '&': return "T_AMP";
case '\'': return "T_SQUOTE";
case '"': return "T_DQUOTE";
- case T_END_FILE: return "T_END_FILE";
default: return "???";
}
+ #undef TOK
}
static void print_char(FILE* file, char ch) {
}
void pprint_token_type(FILE* file, Tok* token) {
- fprintf(file, "%s", token_type_to_string(token->type));
+ if (token->type > 256)
+ fprintf(file, "%s", token_type_to_string(token->type));
+ else
+ fprintf(file, "%c", token->type);
}
void pprint_token_value(FILE* file, Tok* token) {
+ #define TOK(name) case (name): fprintf(file, "%s", #name); break
switch(token->type) {
+ /* value tokens */
case T_STRING: fprintf(file, "\"%s\"", token->value.text); break;
case T_ID: fprintf(file, "%s", token->value.text); break;
case T_CHAR: print_char(file, token->value.character); break;
case T_INT: fprintf(file, "%ld", token->value.integer); break;
case T_FLOAT: fprintf(file, "%f", token->value.floating); break;
case T_BOOL: fprintf(file, "%s", (token->value.boolean)?"true":"false"); break;
- default: fprintf(file, "???"); break;
+
+ /* keyword tokens */
+ TOK(T_NONE); TOK(T_ERROR); TOK(T_END_FILE); TOK(T_PACKAGE);
+ TOK(T_REQUIRES); TOK(T_PROVIDES); TOK(T_LET); TOK(T_VAR);
+ TOK(T_FUN); TOK(T_TYPE); TOK(T_STRUCT); TOK(T_UNION); TOK(T_RETURN);
+ TOK(T_IF); TOK(T_ELSE);
+
+ /* evertything else */
+ default:
+ fprintf(file, "???");
+ break;
}
+ #undef TOK
}
void pprint_token(FILE* file, Tok* token, bool print_loc)
+++ /dev/null
-#include <stdint.h>
-#include <stdlib.h>
-#include <stdio.h>
-
-#define NENTRIES 4096u
-#define DIRTY_BIT 1u
-
-/* garbage collected block definition */
-typedef struct obj_t {
- void (*dtor)(void*); /* destructor function for cleaning up on free() */
- size_t refs; /* count of references to this object */
- uint8_t data[]; /* user data portion of the block */
-} obj_t;
-
-/* log entry definition */
-typedef struct {
- void* ref; /* pointer to the updated cell */
- void* val; /* pointer to the original cell value */
-} entry_t;
-
-size_t EntryCount = 0;
-entry_t EntryLog[NENTRIES] = {0};
-
-void gc_collect(void) {
-
-}
-
-void* gc_alloc(size_t sz, void (*dtor)(void*)) {
- obj_t* obj = malloc(sizeof(obj_t) + sz);
- if (obj) {
- obj->dtor = dtor;
- obj->refs = 0u;
- EntryLog[EntryCount].ref = NULL;
- EntryLog[EntryCount].val = obj->data;
- if (++EntryCount == NENTRIES)
- gc_collect();
- } else {
- perror("refcreate() failed :");
- abort();
- }
- return obj->data;
-}
-
-void gc_setref(void** ref, void* val) {
- if (!((uintptr_t)*ref & DIRTY_BIT)) {
- EntryLog[EntryCount].ref = ref;
- EntryLog[EntryCount].val = val;
- if (!((uintptr_t)*ref & DIRTY_BIT)) {
- *(uintptr_t*)ref |= DIRTY_BIT;
- EntryCount++;
- }
- if (EntryCount == NENTRIES)
- gc_collect();
- }
- *ref = val;
-}
-
-void* gc_getref(void** ref) {
- return (void*)((uintptr_t)*ref & ~DIRTY_BIT);
-}
-
-/*****************************************************************************/
-
-typedef struct hash_entry_t {
- struct hash_entry_t* next;
- uintptr_t hash;
-} hash_entry_t;
-
-typedef struct {
- size_t size;
- size_t bktid;
- hash_entry_t** buckets;
-} hash_t;
-
-static void hash_set(hash_t* hash, hash_entry_t* entry);
-
-#define NUM_PRIMES (sizeof(Primes)/sizeof(unsigned int))
-
-static unsigned int Primes[] = {
- 5, 13, 23, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593,
- 49157, 98317, 196613, 393241, 786433, 1572869, 3145739, 6291469,
- 12582917, 25165843, 50331653, 100663319, 201326611, 402653189,
- 805306457, 1610612741
-};
-
-static void hash_init(hash_t* hash) {
- hash->size = 0;
- hash->bktid = 0;
- hash->buckets = (hash_entry_t**)calloc(sizeof(hash_entry_t*), Primes[hash->bktid]);
-}
-
-static void find_entry(hash_entry_t** parent, hash_entry_t** current, hash_entry_t* entry) {
- while(*current != NULL) {
- if (((*current)->hash == entry->hash) &&
- ((*current)->object == entry->object))
- break;
- *parent = *current;
- *current = (*current)->next;
- }
-}
-
-static void rehash(hash_t* hash) {
- unsigned int oldcount, i;
- hash_entry_t** oldbuckets;
- hash_entry_t *node, *entry;
- if ((hash->bktid + 1) < NUM_PRIMES) {
- oldcount = hash->bktid++;
- oldbuckets = hash->buckets;
- hash->buckets = (hash_entry_t**)calloc(sizeof(hash_entry_t*), Primes[hash->bktid]);
- hash->size = 0;
- /* Iterate over all of the old buckets */
- for (i = 0; i < Primes[oldcount]; i++) {
- node = oldbuckets[i];
- /* re-insert all entries in the bucket into the new bucket table */
- while (node != NULL) {
- entry = node;
- node = entry->next;
- hash_set(hash, entry);
- }
- }
- /* Free the old bucket table */
- free(oldbuckets);
- }
-}
-
-static uint64_t hash64(uint64_t key) {
- key = (~key) + (key << 21);
- key = key ^ (key >> 24);
- key = (key + (key << 3)) + (key << 8);
- key = key ^ (key >> 14);
- key = (key + (key << 2)) + (key << 4);
- key = key ^ (key >> 28);
- key = key + (key << 31);
- return key;
-}
-
-static void hash_set(hash_t* hash, hash_entry_t* entry) {
- unsigned int index;
- hash_entry_t *parent, *node, *deadite;
- if (hash->size >= Primes[hash->bktid])
- rehash(hash);
- entry->hash = hash64((uint64_t)(entry->object));
- index = (entry->hash % Primes[hash->bktid]);
- parent = NULL;
- node = hash->buckets[index];
- deadite = NULL;
- find_entry(&parent, &node, entry);
- if ((parent == NULL) && (node == NULL)) {
- hash->buckets[index] = entry;
- entry->next = NULL;
- hash->size++;
- } else if (node == NULL) {
- parent->next = entry;
- entry->next = NULL;
- hash->size++;
- }
-}
-
-static hash_entry_t* hash_del(hash_t* hash, hash_entry_t* entry) {
- unsigned int index;
- hash_entry_t *parent, *node, *ret = NULL;
- entry->hash = hash64((uint64_t)(entry->object));
- index = (entry->hash % Primes[hash->bktid]);
- parent = NULL;
- node = hash->buckets[index];
- find_entry(&parent, &node, entry);
- if (node != NULL) {
- ret = node;
- node = node->next;
- if (parent != NULL)
- parent->next = node;
- else
- hash->buckets[index] = node;
- hash->size--;
- }
- return ret;
-}
-
+++ /dev/null
-#include <stdint.h>
-
-uintptr_t* StackBot;
-
-int main(int argc, char** argv) {
- { StackBot = (uintptr_t*)&(int){0}; }
- extern int usermain(int argc, char** argv);
- return usermain(argc, argv);
-}
describe "lexer" do
context "punctuation" do
it "should recognize [" do
- expect(lexer('[')).to eq ['T_LBRACK']
+ expect(lexer('[')).to eq ['[']
end
it "should recognize ]" do
- expect(lexer(']')).to eq ['T_RBRACK']
+ expect(lexer(']')).to eq [']']
end
it "should recognize (" do
- expect(lexer('(')).to eq ['T_LPAR']
+ expect(lexer('(')).to eq ['(']
end
it "should recognize )" do
- expect(lexer(')')).to eq ['T_RPAR']
+ expect(lexer(')')).to eq [')']
end
it "should recognize {" do
- expect(lexer('{')).to eq ['T_LBRACE']
+ expect(lexer('{')).to eq ['{']
end
it "should recognize }" do
- expect(lexer('}')).to eq ['T_RBRACE']
+ expect(lexer('}')).to eq ['}']
end
it "should recognize '" do
- expect(lexer('\'')).to eq ['T_SQUOTE']
+ expect(lexer('\'')).to eq ['\'']
end
it "should recognize ," do
- expect(lexer(',')).to eq ['T_COMMA']
+ expect(lexer(',')).to eq [',']
end
it "should recognize :" do
- expect(lexer(':')).to eq ['T_COLON']
+ expect(lexer(':')).to eq [':']
end
it "should recognize all punctuation" do
- expect(lexer('[](){}\',')).to eq(
- ["T_LBRACK", "T_RBRACK", "T_LPAR", "T_RPAR", "T_LBRACE", "T_RBRACE",
- "T_SQUOTE", "T_COMMA"])
+ expect(lexer('[](){}\',')).to eq( ["[", "]", "(", ")", "{", "}", "'", ","])
end
it "should recognize [ after an identifier" do
- expect(lexer('foo[')).to eq(['T_ID:foo', 'T_LBRACK'])
+ expect(lexer('foo[')).to eq(['T_ID:foo', '['])
end
it "should recognize ] after an identifier" do
- expect(lexer('foo]')).to eq(['T_ID:foo', 'T_RBRACK'])
+ expect(lexer('foo]')).to eq(['T_ID:foo', ']'])
end
it "should recognize ( after an identifier" do
- expect(lexer('foo(')).to eq(['T_ID:foo', 'T_LPAR'])
+ expect(lexer('foo(')).to eq(['T_ID:foo', '('])
end
it "should recognize ) after an identifier" do
- expect(lexer('foo)')).to eq(['T_ID:foo', 'T_RPAR'])
+ expect(lexer('foo)')).to eq(['T_ID:foo', ')'])
end
it "should recognize { after an identifier" do
- expect(lexer('foo{')).to eq(['T_ID:foo', 'T_LBRACE'])
+ expect(lexer('foo{')).to eq(['T_ID:foo', '{'])
end
it "should recognize } after an identifier" do
- expect(lexer('foo}')).to eq(['T_ID:foo', 'T_RBRACE'])
+ expect(lexer('foo}')).to eq(['T_ID:foo', '}'])
end
- it "should recognize } after an identifier" do
- expect(lexer('foo\'')).to eq(['T_ID:foo', 'T_SQUOTE'])
+ it "should recognize ' after an identifier" do
+ expect(lexer('foo\'')).to eq(['T_ID:foo', '\''])
end
- it "should recognize } after an identifier" do
- expect(lexer('foo,')).to eq(['T_ID:foo', 'T_COMMA'])
- end
-
- it "should recognize } after an identifier" do
- expect(lexer('foo')).to eq(['T_ID:foo'])
+ it "should recognize , after an identifier" do
+ expect(lexer('foo,')).to eq(['T_ID:foo', ','])
end
end
end
def lexer(input)
- cli(['-Atok'], input).scan(/^\d+:\d+:(T_[A-Z]+(:("[^"]*"|[^\n]+))?)/m).map {|m| m[0] }
+ cli(['-Atok'], input).scan(/^\d+:\d+:((:|[^:\n]+)(:("[^"]*"|[^\n]+))?)/m).map {|m| m[0] }
end
def re_structure( token_array, offset = 0 )
projects / proto / sclpl.git / commitdiff