# HG changeset patch # User William Astle # Date 1379271960 21600 # Node ID 856caf91ffaa929b9898e958428992f5f0c13723 # Parent 6112c67728baf741996ca91d8f13f6980f76f172 Added token list structure and switched some stuff to use it Swithced to using a token list structure instead of manually fiddling pointers throughout the macro expansion code. Also fixed up some problematic things related to stringification and concatenation. diff -r 6112c67728ba -r 856caf91ffaa lwcc/lex.c --- a/lwcc/lex.c Sat Sep 14 22:42:53 2013 -0600 +++ b/lwcc/lex.c Sun Sep 15 13:06:00 2013 -0600 @@ -596,7 +596,9 @@ { if (!pp -> lexstr) preproc_throw_error(pp, "Invalid character constant"); - break; + ttype = TOK_ERROR; + strval = strbuf_end(strbuf); + goto out; } cl++; strbuf_add(strbuf, c); @@ -604,10 +606,15 @@ } strbuf_add(strbuf, c); } - if (cl == 0 && !pp -> lexstr) - preproc_throw_error(pp, "Invalid character constant"); strval = strbuf_end(strbuf); - ttype = TOK_CHR_LIT; + if (cl == 0) + { + ttype = TOK_ERROR; + if (!pp -> lexstr) + preproc_throw_error(pp, "Invalid character constant"); + } + else + ttype = TOK_CHR_LIT; goto out; case '"': @@ -617,7 +624,15 @@ for (;;) { c = preproc_lex_fetch_byte(pp); - if (c == CPP_EOF || c == CPP_EOL || c == '"') + if (c == CPP_EOF || c == CPP_EOL) + { + ttype = TOK_ERROR; + strval = strbuf_end(strbuf); + if (!pp -> lexstr) + preproc_throw_error(pp, "Invalid string constant"); + goto out; + } + if (c == '"') break; if (c == '\\') { @@ -625,9 +640,11 @@ c = preproc_lex_fetch_byte(pp); if (c == CPP_EOF || c == CPP_EOL) { + ttype = TOK_ERROR; if (!pp -> lexstr) preproc_throw_error(pp, "Invalid string constant"); - break; + strval = strbuf_end(strbuf); + goto out; } cl++; strbuf_add(strbuf, c); diff -r 6112c67728ba -r 856caf91ffaa lwcc/preproc.c --- a/lwcc/preproc.c Sat Sep 14 22:42:53 2013 -0600 +++ b/lwcc/preproc.c Sun Sep 15 13:06:00 2013 -0600 @@ -254,8 +254,7 @@ static void dir_define(struct preproc_info *pp) { - struct token *tl = NULL; - struct token *ttl; + struct token_list *tl = NULL; struct token *ct; int nargs = -1; int vargs = 0; @@ -335,6 +334,7 @@ baddefine: preproc_throw_error(pp, "bad #define"); baddefine2: + token_list_destroy(tl); skip_eol(pp); lw_free(mname); while (nargs > 0) @@ -348,12 +348,7 @@ ct = preproc_next_token(pp); if (ct -> ttype == TOK_EOL) break; - if (!tl) - tl = ct; - else - ttl -> next = ct; - ttl = ct; - pp -> curtok = NULL; // tell *_next_token* not to clobber token + token_list_append(tl, token_dup(ct)); } out: if (strcmp(mname, "defined") == 0) @@ -767,10 +762,11 @@ /* Below here is the logic for expanding a macro */ -static char *stringify(struct token *tl) +static char *stringify(struct token_list *tli) { struct strbuf *s; int ws = 0; + struct token *tl = tli -> head; s = strbuf_new(); strbuf_add(s, '"'); @@ -804,212 +800,141 @@ return strbuf_end(s); } -/* return list to tokens as a result of ## expansion */ -static struct token *paste_tokens(struct preproc_info *pp, struct symtab_e *s, struct token **arglist, struct token *t1, struct token *t2) +static int macro_arg(struct symtab_e *s, char *str) { - struct token *rl = NULL, *rlt; - struct token *s1, *s2; - struct token *ws; int i; + if (strcmp(str, "__VA_ARGS__") == 0) + i = s -> nargs; + else + for (i = 0; i < s -> nargs; i++) + if (strcmp(s -> params[i], str) == 0) + break; + if (i == s -> nargs) + if (s -> vargs == 0) + return -1; + return i; +} + +/* return list to tokens as a result of ## expansion */ +static struct token_list *paste_tokens(struct preproc_info *pp, struct symtab_e *s, struct token_list **arglist, struct token *t1, struct token *t2) +{ + struct token_list *left; + struct token_list *right; char *tstr; + struct token *ttok; + int i; if (t1 -> ttype == TOK_IDENT) { - if (strcmp(t1 -> strval, "__VA_ARGS__") == 0) - { - i = s -> nargs; - } - else + i = macro_arg(s, t1 -> strval); + if (i == -1) { - for (i = 0; i < s -> nargs; i++) - { - if (strcmp(s -> params[i], t1 -> strval) == 0) - break; - } - } - if ((i == s -> nargs) && !(s -> vargs)) - { - s1 = token_dup(t1); + left = token_list_create(); + token_list_append(left, token_dup(t1)); } else { - /* find last non-whitespace token */ - ws = NULL; - for (t1 = s -> tl; t1; t1 = t1 -> next) - { - if (t1 -> ttype != TOK_WSPACE) - ws = t1; - } - if (!ws) - { - s1 = NULL; - } - else - { - if (ws != s -> tl) - { - /* output extra tokens */ - for (t1 = s -> tl; t1 -> next != ws; t1 = t1 -> next) - { - if (!rl) - { - rl = token_dup(t1); - rlt = rl; - } - else - { - rlt -> next = token_dup(t1); - rlt = rlt -> next; - } - } - } - s1 = token_dup(ws); - } + left = token_list_dup(arglist[i]); } } else { - s1 = token_dup(t1); + left = token_list_create(); + token_list_append(left, token_dup(t1)); } + // munch trailing white space + while (left -> tail && left -> tail -> ttype == TOK_WSPACE) + { + token_list_remove(left -> tail); + } + if (t2 -> ttype == TOK_IDENT) { - if (strcmp(t1 -> strval, "__VA_ARGS__") == 0) + i = macro_arg(s, t2 -> strval); + if (i == -1) { - i = s -> nargs; + right = token_list_create(); + token_list_append(right, token_dup(t2)); } else { - for (i = 0; i < s -> nargs; i++) - { - if (strcmp(s -> params[i], t1 -> strval) == 0) - break; - } - } - if ((i == s -> nargs) && !(s -> vargs)) - { - s2 = token_dup(t2); - t2 = NULL; - } - else - { - /* find last non-whitespace token */ - ws = NULL; - for (t2 = s -> tl; t2; t2 = t2 -> next) - { - if (t2 -> ttype != TOK_WSPACE) - { - ws = t2; - t2 = t2 -> next; - break; - } - } - if (!ws) - { - s2 = NULL; - } - else - { - s2 = token_dup(ws); - } + right = token_list_dup(arglist[i]); } } else { - s2 = token_dup(t2); + right = token_list_create(); + token_list_append(right, token_dup(t2)); + } + // munch leading white space + while (right -> head && right -> head -> ttype == TOK_WSPACE) + { + token_list_remove(right -> head); + } + + // nothing to append at all + if (left -> head != NULL && right -> head == NULL) + { + // right arg is empty - use left + token_list_destroy(right); + return left; + } + if (left -> head == NULL && right -> head != NULL) + { + // left arg is empty, use right + token_list_destroy(left); + return right; + } + if (left -> head == NULL && right -> head == NULL) + { + // both empty, use left + token_list_destroy(right); + return left; } - /* here, s1 is NULL if no left operand or a duplicated token for the actual left operand */ - /* here, s2 is NULL if no right operand or a duplicated token for the actual right operand */ - /* here, t2 points to a possibly empty list of extra tokens to output after the concatenated tokens */ - /* here, rl,rlt is a possibly non-empty list of tokens preceding the concatenation */ - - /* tokens combine if the combination exactly matches "combinelist", in which case the string values are - concatenated and the new token type is used to create a new token. If the tokens do not combine, - s1 and s2 are returned in sequence. */ + // both non-empty - past left tail with right head + // then past the right list onto the left + tstr = lw_alloc(strlen(left -> tail -> strval) + strlen(right -> head -> strval) + 1); + strcpy(tstr, left -> tail -> strval); + strcat(tstr, right -> head -> strval); - if (!s1 && s2) - { - if (!rl) - rl = s2; - else - rlt -> next = s2; - rlt = s2; - } - else if (s1 && !s2) - { - if (!rl) - rl = s1; - else - rlt -> next = s1; - rlt = s1; - } - else if (s1 && s2) + pp -> lexstr = tstr; + pp -> lexstrloc = 0; + + ttok = preproc_lex_next_token(pp); + if (ttok -> ttype != TOK_ERROR && pp -> lexstr[pp -> lexstrloc] == 0) { - tstr = lw_alloc(strlen(s1 -> strval) + strlen(s2 -> strval) + 1); - strcpy(tstr, s1 -> strval); - strcat(tstr, s2 -> strval); - /* now try to lex the string */ - pp -> lexstr = tstr; - pp -> lexstrloc = 0; - t1 = preproc_lex_next_token(pp); - if (pp -> lexstr[pp -> lexstrloc]) - { - // doesn't make a new token - pass through the original two - if (!rl) - rl = s1; - else - rlt -> next = s1; - s1 -> next = s2; - rlt = s2; - } - else - { - // does make a new token - t1 -> fn = s1 -> fn; - t1 -> column = s1 -> column; - t1 -> lineno = s1 -> lineno; - if (!rl) - rl = t1; - else - rlt -> next = t1; - rlt = t1; - } - lw_free(tstr); - pp -> lexstr = NULL; + // we have a new token here + token_list_remove(left -> tail); + token_list_remove(right -> head); + token_list_append(left, token_dup(ttok)); } - - /* add in any extra tokens */ - while (t2) + lw_free(tstr); + pp -> lexstr = NULL; + pp -> lexstrloc = 0; + for (ttok = right -> head; ttok; ttok = ttok -> next) { - if (!rl) - { - rl = token_dup(t2); - rlt = rl; - } - else - { - rlt -> next = token_dup(t2); - rlt = rlt -> next; - } - t2 = t2 -> next; + token_list_append(left, token_dup(ttok)); } - - return rl; + token_list_destroy(right); + return left; } - static int expand_macro(struct preproc_info *pp, char *mname) { struct symtab_e *s; struct token *t, *t2, *t3; - struct token **arglist = NULL; int nargs = 0; struct expand_e *e; - struct token **exparglist = NULL; + struct token_list **exparglist = NULL; + struct token_list **arglist = NULL; int i; int pcount; char *tstr; - + struct token_list *expand_list; + int repl; + struct token_list *rtl; + s = symtab_find(pp, mname); if (!s) return 0; @@ -1056,8 +981,8 @@ // parse parameters here t = preproc_next_token_nws(pp); nargs = 1; - arglist = lw_alloc(sizeof(struct token *)); - arglist[0] = NULL; + arglist = lw_alloc(sizeof(struct token_list *)); + arglist[0] = token_list_create(); t2 = NULL; while (t -> ttype != TOK_CPAREN) @@ -1079,22 +1004,13 @@ if (!(s -> vargs) || (nargs > s -> nargs)) { nargs++; - arglist = lw_realloc(arglist, sizeof(struct token *) * nargs); - arglist[nargs - 1] = NULL; + arglist = lw_realloc(arglist, sizeof(struct token_list *) * nargs); + arglist[nargs - 1] = token_list_create(); t2 = NULL; continue; } } - if (t2) - { - t2 -> next = token_dup(t); - t2 = t2 -> next; - } - else - { - t2 = token_dup(t); - arglist[nargs - 1] = t2; - } + token_list_append(arglist[nargs - 1], token_dup(t)); } if (s -> vargs) @@ -1113,172 +1029,124 @@ } /* now calculate the pre-expansions of the arguments */ - exparglist = lw_alloc(nargs); + exparglist = lw_alloc(nargs * sizeof(struct token_list *)); for (i = 0; i < nargs; i++) { - t2 = NULL; - exparglist[i] = NULL; + exparglist[i] = token_list_create(); // NOTE: do nothing if empty argument - if (arglist[i] == NULL) + if (arglist[i] == NULL || arglist[i] -> head == NULL) continue; - pp -> sourcelist = arglist[i]; + pp -> sourcelist = arglist[i]->head; for (;;) { t = preproc_next_processed_token(pp); if (t -> ttype == TOK_EOF) break; - if (t2) - { - t2 -> next = token_dup(t); - t2 = t2 -> next; - } - else - { - t2 = token_dup(t); - exparglist[i] = t2; - } + token_list_append(exparglist[i], token_dup(t)); } } expandmacro: - t2 = NULL; - t3 = NULL; - - for (t = s -> tl; t; t = t -> next) + expand_list = token_list_dup(s -> tl); + + // scan for stringification and handle it + repl = 0; + while (repl == 0) { -again: - if (t -> ttype != TOK_WSPACE && t -> next) + for (t = expand_list -> head; t; t = t -> next) { - struct token *ct1, *ct2; - - for (ct1 = t -> next; ct1 && ct1 -> ttype == TOK_WSPACE; ct1 = ct1 -> next) + if (t -> ttype == TOK_HASH && t -> next && t -> next -> ttype == TOK_IDENT) { - if (ct1 -> ttype == TOK_DBLHASH) + i = macro_arg(s, t -> next -> strval); + if (i != -1) { - // possible concatenation here - for (ct2 = ct1 -> next; ct2 && ct2 -> ttype == TOK_WSPACE; ct2 = ct2 -> next) - /* do nothing */ ; - if (ct2) - { - // we have concatenation here so we paste str1 and str2 together and see what we get - // if we get NULL, the past didn't make a valid token - ct1 = paste_tokens(pp, s, arglist, t, ct2); - if (ct1) - { - if (t2) - { - t2 -> next = ct1; - } - else - { - t3 = ct1; - } - for (t2 = ct1; t2 -> next; t2 = t2 -> next) - /* do nothing */ ; - - /* because of the level of control structures, move to next token and restart loop */ - t = ct2 -> next; - goto again; - } - goto nopaste; - } + repl = 1; + tstr = stringify(arglist[i]); + token_list_remove(t -> next); + token_list_insert(expand_list, t, token_create(TOK_STRING, tstr, t -> lineno, t -> column, t -> fn)); + token_list_remove(t); + lw_free(tstr); + break; } } } + } + + + // scan for concatenation and handle it -nopaste: - if (t -> ttype == TOK_HASH) + for (t = expand_list -> head; t; t = t -> next) + { + if (t -> ttype == TOK_DBLHASH) { - if (t -> next && t -> next -> ttype == TOK_IDENT) + // have a concatenation operator here + for (t2 = t -> prev; t2; t2 = t2 -> prev) + { + if (t2 -> ttype != TOK_WSPACE) + break; + } + for (t3 = t -> next; t3; t3 = t3 -> next); { - if (strcmp(t -> next -> strval, "__VA_ARGS__") == 0) - { - i = nargs; - } - else - { - for (i = 0; i < nargs; i++) - { - if (strcmp(t -> next -> strval, s -> params[i]) == 0) - break; - } - } - if (!((i == s -> nargs) && !(s -> vargs))) - { - // we have a stringification here - t = t -> next; - tstr = stringify(arglist[i]); - if (t2) - { - t2 = token_create(TOK_STRING, tstr, t -> lineno, t -> column, t -> fn); - t2 = t2 -> next; - } - else - { - t3 = token_create(TOK_STRING, tstr, t -> lineno, t -> column, t -> fn); - t2 = t3; - } - lw_free(tstr); - continue; - } + if (t3 -> ttype != TOK_WSPACE) + break; } + // if no non-whitespace before or after, ignore it + if (!t2 || !t3) + continue; + // eat the whitespace before and after + while (t -> prev != t2) + token_list_remove(t -> prev); + while (t -> next != t3) + token_list_remove(t -> next); + // now paste t -> prev with t -> next and replace t with the result + // continue scanning for ## at t -> next -> next + t3 = t -> next -> next; + + rtl = paste_tokens(pp, s, arglist, t -> prev, t -> next); + token_list_remove(t -> next); + token_list_remove(t -> prev); + t2 = t -> prev; + token_list_remove(t); + for (t = rtl -> head; t; t = t -> next) + { + token_list_insert(expand_list, t2, token_dup(t)); + } + t = t3 -> prev; + token_list_destroy(rtl); } + } + + // now scan for arguments and expand them + for (t = expand_list -> head; t; t = t -> next) + { + again: if (t -> ttype == TOK_IDENT) { /* identifiers might need expansion to arguments */ - if (strcmp(t -> strval, "__VA_ARGS__") == 0) - { - i = s -> nargs; - } - else - { - for (i = 0; i < nargs; i++) - { - if (strcmp(t -> strval, s -> params[i]) == 0) - break; - } - } - if ((i == s -> nargs) && !(s -> vargs)) + i = macro_arg(s, t -> strval); + if (i != -1) { - struct token *te; - // expand argument - for (te = exparglist[i]; te; te = te -> next) - { - if (t2) - { - t2 -> next = token_dup(te); - t2 = t2 -> next; - } - else - { - t3 = token_dup(te); - t2 = t2; - } - } - continue; + t3 = t -> next; + for (t2 = exparglist[i] -> tail; t2; t2 = t2 -> prev) + token_list_insert(expand_list, t, token_dup(t2)); + token_list_remove(t); + t = t3; + goto again; } } - if (t2) - { - t2 -> next = token_dup(t); - t2 = t2 -> next; - } - else - { - t3 = token_dup(t); - t2 = t3; - } } /* put the new expansion in front of the input, if relevant; if we expanded to nothing, no need to create an expansion record or put anything into the input queue */ - if (t3) + if (expand_list -> head) { - t2 -> next = token_create(TOK_ENDEXPAND, "", -1, -1, ""); - t2 -> next -> next = pp -> tokqueue; - pp -> tokqueue = t3; - + token_list_append(expand_list, token_create(TOK_ENDEXPAND, "", -1, -1, "")); + + // move the expanded list into the token queue + for (t = expand_list -> tail; t; t = t -> prev) + preproc_unget_token(pp, token_dup(t)); + /* set up expansion record */ e = lw_alloc(sizeof(struct expand_e)); e -> next = pp -> expand_list; @@ -1287,10 +1155,11 @@ } /* now clean up */ + token_list_destroy(expand_list); for (i = 0; i < nargs; i++) { - lw_free(arglist[i]); - lw_free(exparglist[i]); + token_list_destroy(arglist[i]); + token_list_destroy(exparglist[i]); } lw_free(arglist); lw_free(exparglist); diff -r 6112c67728ba -r 856caf91ffaa lwcc/symbol.c --- a/lwcc/symbol.c Sat Sep 14 22:42:53 2013 -0600 +++ b/lwcc/symbol.c Sun Sep 15 13:06:00 2013 -0600 @@ -31,19 +31,13 @@ void symbol_free(struct symtab_e *s) { int i; - struct token *t; - + lw_free(s -> name); for (i = 0; i < s -> nargs; i++) lw_free(s -> params[i]); lw_free(s -> params); - while (s -> tl) - { - t = s -> tl; - s -> tl = t -> next; - token_free(t); - } + token_list_destroy(s -> tl); } struct symtab_e *symtab_find(struct preproc_info *pp, char *name) @@ -77,7 +71,7 @@ } } -void symtab_define(struct preproc_info *pp, char *name, struct token *def, int nargs, char **params, int vargs) +void symtab_define(struct preproc_info *pp, char *name, struct token_list *def, int nargs, char **params, int vargs) { struct symtab_e *s; diff -r 6112c67728ba -r 856caf91ffaa lwcc/symbol.h --- a/lwcc/symbol.h Sat Sep 14 22:42:53 2013 -0600 +++ b/lwcc/symbol.h Sun Sep 15 13:06:00 2013 -0600 @@ -28,7 +28,7 @@ struct symtab_e { char *name; // symbol name - struct token *tl; // token list the name is defined as, NULL for none + struct token_list *tl; // token list the name is defined as, NULL for none int nargs; // number named of arguments - -1 for object like macro int vargs; // set if macro has varargs style char **params; // the names of the parameters @@ -37,6 +37,6 @@ struct symtab_e *symtab_find(struct preproc_info *, char *); void symtab_undef(struct preproc_info *, char *); -void symtab_define(struct preproc_info *, char *, struct token *, int, char **, int); +void symtab_define(struct preproc_info *, char *, struct token_list *, int, char **, int); #endif // symbol_h_seen___ diff -r 6112c67728ba -r 856caf91ffaa lwcc/token.c --- a/lwcc/token.c Sat Sep 14 22:42:53 2013 -0600 +++ b/lwcc/token.c Sun Sep 15 13:06:00 2013 -0600 @@ -132,3 +132,112 @@ if (t -> strval) fprintf(f, "%s", t -> strval); } + +/* token list management */ +struct token_list *token_list_create(void) +{ + struct token_list *tl; + tl = lw_alloc(sizeof(struct token_list)); + tl -> head = NULL; + tl -> tail = NULL; + return tl; +} + +void token_list_destroy(struct token_list *tl) +{ + if (tl == NULL) + return; + while (tl -> head) + { + tl -> tail = tl -> head; + tl -> head = tl -> head -> next; + token_free(tl -> tail); + lw_free(tl -> tail); + } + lw_free(tl); +} + +void token_list_append(struct token_list *tl, struct token *tok) +{ + tok -> list = tl; + if (tl -> head == NULL) + { + tl -> head = tl -> tail = tok; + tok -> next = tok -> prev = NULL; + return; + } + tl -> tail -> next = tok; + tok -> prev = tl -> tail; + tl -> tail = tok; + tok -> next = NULL; + return; +} + +void token_list_remove(struct token *tok) +{ + if (tok -> list == NULL) + return; + + if (tok -> prev) + tok -> prev -> next = tok -> next; + if (tok -> next) + tok -> next -> prev = tok -> prev; + if (tok == tok -> list -> head) + tok -> list -> head = tok -> next; + if (tok == tok -> list -> tail) + tok -> list -> tail = tok -> prev; + tok -> list = NULL; +} + +void token_list_prepend(struct token_list *tl, struct token *tok) +{ + tok -> list = tl; + if (tl -> head == NULL) + { + tl -> head = tl -> tail = tok; + tok -> next = tok -> prev = NULL; + } + tl -> head -> prev = tok; + tok -> next = tl -> head; + tl -> head = tok; + tok -> prev = NULL; +} + +void token_list_insert(struct token_list *tl, struct token *after, struct token *newt) +{ + struct token *t; + + if (after == NULL || tl -> head == NULL) + { + token_list_prepend(tl, newt); + return; + } + + for (t = tl -> head; t && t != after; t = t -> next) + /* do nothing */ ; + if (!t) + { + token_list_append(tl, newt); + return; + } + newt -> prev = t; + newt -> next = t -> next; + if (t -> next) + t -> next -> prev = newt; + else + tl -> tail = newt; + t -> next = newt; +} + +struct token_list *token_list_dup(struct token_list *tl) +{ + struct token_list *nl; + struct token *t; + + nl = token_list_create(); + for (t = tl -> head; t; t = t -> next) + { + token_list_append(nl, token_dup(t)); + } + return nl; +} diff -r 6112c67728ba -r 856caf91ffaa lwcc/token.h --- a/lwcc/token.h Sat Sep 14 22:42:53 2013 -0600 +++ b/lwcc/token.h Sun Sep 15 13:06:00 2013 -0600 @@ -93,6 +93,7 @@ TOK_ARROW, TOK_ENDEXPAND, TOK_STARTEXPAND, + TOK_ERROR, TOK_MAX }; @@ -102,23 +103,35 @@ char *strval; // the token value if relevant struct token *prev; // previous token in a list struct token *next; // next token in a list + struct token_list *list;// pointer to head of list descriptor this token is on int lineno; // line number token came from int column; // character column token came from const char *fn; // file name token came from }; +struct token_list +{ + struct token *head; // the head of the list + struct token *tail; // the tail of the list +}; + extern void token_free(struct token *); extern struct token *token_create(int, char *strval, int, int, const char *); extern struct token *token_dup(struct token *); /* add a token to the end of a list */ -extern void token_append(struct token *, struct token *); +extern void token_list_append(struct token_list *, struct token *); /* add a token to the start of a list */ -extern void token_prepend(struct token *, struct token *); +extern void token_list_prepend(struct token_list *, struct token *); /* remove individual token from whatever list it is on */ -extern void token_remove(struct token *); +extern void token_list_remove(struct token *); /* replace token with list of tokens specified */ -extern void token_replace(struct token *, struct token *); +extern void token_list_insert(struct token_list *, struct token *, struct token *); +/* duplicate a list to a new list pointer */ +extern struct token_list *token_list_dup(struct token_list *); /* print a token out */ +extern struct token_list *token_list_create(void); +extern void token_list_destroy(struct token_list *); + extern void token_print(struct token *, FILE *); #endif // token_h_seen___