# HG changeset patch
# User William Astle <lost@l-w.ca>
# Date 1379553472 21600
# Node ID 54f213c8fb81ade22f569c635da0c53712634645
# Parent  d85d173ba120b5a5aea92a6d5970812d8120b861
Various bugfixes and output tuning

Tuned output of preprocessor to include line markers similar to the ones
added by the gcc preprocessor.

Also, many fixes for various bits of dumbosity leading to misbehaviour and
crashing.

diff -r d85d173ba120 -r 54f213c8fb81 lwcc/cpp-main.c
--- a/lwcc/cpp-main.c	Tue Sep 17 19:33:41 2013 -0600
+++ b/lwcc/cpp-main.c	Wed Sep 18 19:17:52 2013 -0600
@@ -27,6 +27,7 @@
 
 #include <lw_stringlist.h>
 #include <lw_cmdline.h>
+#include <lw_string.h>
 
 #include "cpp.h"
 
@@ -134,28 +135,73 @@
 	}
 	lw_stringlist_destroy(input_files);
 	
-//	symbol_dump();
 	exit(retval);
 }
 
+static void print_line_marker(FILE *fp, int line, const char *fn, int flag)
+{
+	fprintf(fp, "\n# %d \"", line);
+	while (*fn)
+	{
+		if (*fn < 32 || *fn == 34 || *fn > 126)
+		{
+			fprintf(fp, "\\%03o", *fn);
+		}
+		else
+		{
+			fprintf(fp, "%c", *fn);
+		}
+		fn++;
+	}
+	fprintf(fp, "\" %d\n", flag);
+}
+
 int process_file(const char *fn)
 {
 	struct preproc_info *pp;
 	struct token *tok = NULL;
-	
+	int last_line = 0;
+	char *last_fn = NULL;
+		
 	pp = preproc_init(fn);
 	if (!pp)
 		return -1;
-	
+
+	print_line_marker(output_fp, 1, fn, 1);
+	last_fn = lw_strdup(fn);	
 	for (;;)
 	{
 		tok = preproc_next(pp);
 		if (tok -> ttype == TOK_EOF)
 			break;
+		if (strcmp(tok -> fn, last_fn) != 0)
+		{
+			int lt = 1;
+			if (tok -> lineno != 1)
+			{
+				lt = 2;
+			}
+			lw_free(last_fn);
+			last_fn = lw_strdup(tok -> fn);
+			last_line = tok -> lineno;
+			print_line_marker(output_fp, last_line, last_fn, lt);
+		}
+		else
+		{
+			while (tok -> lineno > last_line)
+			{
+				fprintf(output_fp, "\n");
+				last_line++;
+			}
+		}
 		token_print(tok, output_fp);
+		if (tok -> ttype == TOK_EOL)
+			last_line++;
 		token_free(tok);
 	}
 	token_free(tok);
+	lw_free(last_fn);
+//	symtab_dump(pp);
 	preproc_finish(pp);
 	return 0;
 }
diff -r d85d173ba120 -r 54f213c8fb81 lwcc/cpp.c
--- a/lwcc/cpp.c	Tue Sep 17 19:33:41 2013 -0600
+++ b/lwcc/cpp.c	Wed Sep 18 19:17:52 2013 -0600
@@ -54,7 +54,10 @@
 	pp -> fn = strpool_strdup(pp -> strpool, fn);
 	pp -> fp = fp;
 	pp -> ra = CPP_NOUNG;
+	pp -> unget = CPP_NOUNG;
 	pp -> ppeolseen = 1;
+	pp -> lineno = 1;
+	pp -> n = NULL;
 	return pp;
 }
 
@@ -146,14 +149,16 @@
 	fprintf(stderr, "WARNING: %s\n", m);
 }
 
-static void preproc_throw_message(void (*cb)(const char *), const char *m, va_list args)
+static void preproc_throw_message(struct preproc_info *pp, void (*cb)(const char *), const char *m, va_list args)
 {
-	int s;
+	int s, s2;
 	char *b;
 	
+	s2 = snprintf(NULL, 0, "(%s:%d:%d) ", pp -> fn, pp -> lineno, pp -> column);
 	s = vsnprintf(NULL, 0, m, args);
-	b = lw_alloc(s + 1);
-	vsnprintf(b, s + 1, m, args);
+	b = lw_alloc(s + s2 + 1);
+	snprintf(b, s2 + 1, "(%s:%d:%d) ", pp -> fn, pp -> lineno, pp -> column);
+	vsnprintf(b + s2, s + 1, m, args);
 	(*cb)(b);
 	lw_free(b);
 }
@@ -162,7 +167,7 @@
 {
 	va_list args;
 	va_start(args, m);
-	preproc_throw_message(pp -> errorcb ? pp -> errorcb : preproc_throw_error_default, m, args);
+	preproc_throw_message(pp, pp -> errorcb ? pp -> errorcb : preproc_throw_error_default, m, args);
 	va_end(args);
 	exit(1);
 }
@@ -171,6 +176,6 @@
 {
 	va_list args;
 	va_start(args, m);
-	preproc_throw_message(pp -> warningcb ? pp -> warningcb : preproc_throw_warning_default, m, args);
+	preproc_throw_message(pp, pp -> warningcb ? pp -> warningcb : preproc_throw_warning_default, m, args);
 	va_end(args);
 }
diff -r d85d173ba120 -r 54f213c8fb81 lwcc/lex.c
--- a/lwcc/lex.c	Tue Sep 17 19:33:41 2013 -0600
+++ b/lwcc/lex.c	Wed Sep 18 19:17:52 2013 -0600
@@ -191,7 +191,7 @@
    by fetch_byte(). Theoretically, an unlimited number of characters can
    be unfetched. Line and column counting may be incorrect if unfetched
    characters cross a token boundary. */
-static void preproc_lex_unfetch_byte(struct preproc_info *pp, int c)
+void preproc_lex_unfetch_byte(struct preproc_info *pp, int c)
 {
 	if (pp -> ungetbufl >= pp -> ungetbufs)
 	{
@@ -275,7 +275,7 @@
 */
 
 
-static int preproc_lex_fetch_byte(struct preproc_info *pp)
+int preproc_lex_fetch_byte(struct preproc_info *pp)
 {
 	int c;
 	c = fetch_byte(pp);
@@ -318,7 +318,7 @@
 			for (;;)
 			{
 				c2 = fetch_byte(pp);
-				if (c2 == CPP_EOL || c2 == CPP_EOF)
+				if (c2 == CPP_EOF)
 				{
 					preproc_lex_unfetch_byte(pp, c);
 					break;
@@ -362,6 +362,12 @@
 			c = CPP_EOL;
 		}
 	}
+
+	if (pp -> lineno != sline)
+	{
+		sline = pp -> lineno;
+		scol = pp -> column;
+	}
 	
 	if (c == CPP_EOF)
 	{
@@ -639,6 +645,7 @@
 strlit:
 		/* string literal */
 		strbuf = strbuf_new();
+		strbuf_add(strbuf, '"');
 		for (;;)
 		{
 			c = preproc_lex_fetch_byte(pp);
@@ -670,6 +677,7 @@
 			}
 			strbuf_add(strbuf, c);
 		}
+		strbuf_add(strbuf, '"');
 		strval = strbuf_end(strbuf);
 		ttype = TOK_STR_LIT;
 		goto out;
diff -r d85d173ba120 -r 54f213c8fb81 lwcc/preproc.c
--- a/lwcc/preproc.c	Tue Sep 17 19:33:41 2013 -0600
+++ b/lwcc/preproc.c	Wed Sep 18 19:17:52 2013 -0600
@@ -40,18 +40,22 @@
 struct token *preproc_next_processed_token(struct preproc_info *pp)
 {
 	struct token *ct;
-	
+
 again:
 	ct = preproc_next_token(pp);
 	if (ct -> ttype == TOK_EOF)
 		return ct;
 	if (ct -> ttype == TOK_EOL)
+	{
 		pp -> ppeolseen = 1;
-		
+		return ct;
+	}
+
 	if (ct -> ttype == TOK_HASH && pp -> ppeolseen == 1)
 	{
 		// preprocessor directive 
 		process_directive(pp);
+		goto again;
 	}
 	// if we're in a false section, don't return the token; keep scanning
 	if (pp -> skip_level)
@@ -108,7 +112,7 @@
 {
 	struct token *t;
 
-	t = preproc_next_token(pp);
+	t = preproc_next_token_nws(pp);
 	if (t -> ttype != TOK_EOL)
 		preproc_throw_warning(pp, "Extra text after preprocessor directive");
 	skip_eol(pp);
@@ -344,7 +348,8 @@
 		lw_free(arglist);
 		return;
 	}
-	
+
+	tl = token_list_create();	
 	for (;;)
 	{
 		ct = preproc_next_token(pp);
@@ -459,13 +464,13 @@
 	struct preproc_info *fs;
 	
 	ct = preproc_next_token_nws(pp);
-	if (ct -> ttype == TOK_STRING)
+	if (ct -> ttype == TOK_STR_LIT)
 	{
 usrinc:
 		sys = strlen(ct -> strval);
 		fn = lw_alloc(sys - 1);
 		memcpy(fn, ct -> strval + 1, sys - 2);
-		fn[sys - 1] = 0;
+		fn[sys - 2] = 0;
 		sys = 0;
 		goto doinc;
 	}
@@ -474,19 +479,18 @@
 		strbuf = strbuf_new();
 		for (;;)
 		{
-			ct = preproc_next_token(pp);
-			if (ct -> ttype == TOK_GT)
-				break;
-			if (ct -> ttype == TOK_EOL)
+			int c;
+			c = preproc_lex_fetch_byte(pp);
+			if (c == CPP_EOL)
 			{
+				preproc_lex_unfetch_byte(pp, c);
 				preproc_throw_error(pp, "Bad #include");
 				lw_free(strbuf_end(strbuf));
-				return;
+				break;
 			}
-			for (i = 0; ct -> strval[i]; ct++)
-			{
-				strbuf_add(strbuf, ct -> strval[i]);
-			}
+			if (c == '>')
+				break;
+			strbuf_add(strbuf, c);
 		}
 		ct = preproc_next_token_nws(pp);
 		if (ct -> ttype != TOK_EOL)
@@ -505,7 +509,7 @@
 		preproc_unget_token(pp, ct);
 		// computed include
 		ct = preproc_next_processed_token_nws(pp);
-		if (ct -> ttype == TOK_STRING)
+		if (ct -> ttype == TOK_STR_LIT)
 			goto usrinc;
 		else if (ct -> ttype == TOK_LT)
 		{
@@ -550,7 +554,7 @@
 	fp = fopen(fn, "rb");
 	if (!fp)
 	{
-		preproc_throw_error(pp, "Cannot open #include file - this is fatal");
+		preproc_throw_error(pp, "Cannot open #include file %s - this is fatal", fn);
 		exit(1);
 	}
 	
@@ -558,13 +562,14 @@
 	fs = lw_alloc(sizeof(struct preproc_info));
 	*fs = *pp;
 	fs -> n = pp -> filestack;
+	pp -> curtok = NULL;
 	pp -> filestack = fs;
 	pp -> fn = fn;
 	pp -> fp = fp;
 	pp -> ra = CPP_NOUNG;
 	pp -> ppeolseen = 1;
 	pp -> eolstate = 0;
-	pp -> lineno = 0;
+	pp -> lineno = 1;
 	pp -> column = 0;
 	pp -> qseen = 0;
 	pp -> ungetbufl = 0;
@@ -577,7 +582,7 @@
 	pp -> found_level = 0;
 	pp -> else_level = 0;
 	pp -> else_skip_level = 0;
-	
+	pp -> tokqueue = NULL;	
 	// now get on with processing
 }
 
@@ -612,7 +617,7 @@
 		pp -> lineno = lineno;
 		return;
 	}
-	if (ct -> ttype != TOK_STRING)
+	if (ct -> ttype != TOK_STR_LIT)
 	{
 		preproc_throw_error(pp, "Bad #line");
 		skip_eol(pp);
@@ -920,9 +925,11 @@
 static long eval_expr(struct preproc_info *pp)
 {
 	long rv;
+	struct token *t;
 	
 	rv = eval_expr_real(pp, 0);
-	if (pp -> curtok -> ttype != TOK_EOL)
+	t = preproc_next_token_nws(pp);
+	if (t -> ttype != TOK_EOL)
 	{
 		preproc_throw_error(pp, "Bad expression");
 		skip_eol(pp);
@@ -1091,7 +1098,7 @@
 		}
 		for (ws = 0; tl -> strval[ws]; ws++)
 		{
-			if (tl -> ttype == TOK_STRING || tl -> ttype == TOK_CHR_LIT)
+			if (tl -> ttype == TOK_STR_LIT || tl -> ttype == TOK_CHR_LIT)
 			{
 				if (tl -> strval[ws] == '"' || tl -> strval[ws] == '\\')
 					strbuf_add(s, '\\');
@@ -1367,13 +1374,14 @@
 					repl = 1;
 					tstr = stringify(arglist[i]);
 					token_list_remove(t -> next);
-					token_list_insert(expand_list, t, token_create(TOK_STRING, tstr, t -> lineno, t -> column, t -> fn));
+					token_list_insert(expand_list, t, token_create(TOK_STR_LIT, tstr, t -> lineno, t -> column, t -> fn));
 					token_list_remove(t);
 					lw_free(tstr);
 					break;
 				}
 			}
 		}
+		repl = 1;
 	}
 
 
diff -r d85d173ba120 -r 54f213c8fb81 lwcc/symbol.c
--- a/lwcc/symbol.c	Tue Sep 17 19:33:41 2013 -0600
+++ b/lwcc/symbol.c	Wed Sep 18 19:17:52 2013 -0600
@@ -74,13 +74,58 @@
 void symtab_define(struct preproc_info *pp, char *name, struct token_list *def, int nargs, char **params, int vargs)
 {
 	struct symtab_e *s;
-	
+	int i;
+		
 	s = lw_alloc(sizeof(struct symtab_e));
 	s -> name = lw_strdup(name);
 	s -> tl = def;
 	s -> nargs = nargs;
-	s -> params = params;
+	s -> params = NULL;
+	if (params)
+	{
+		s -> params = lw_alloc(sizeof(char *) * nargs);
+		for (i = 0; i < nargs; i++)
+			s -> params[i] = lw_strdup(params[i]);
+	}
 	s -> vargs = vargs;
 	s -> next = pp -> sh;
 	pp -> sh = s;
 }
+
+void symtab_dump(struct preproc_info *pp)
+{
+	struct symtab_e *s;
+	struct token *t;
+	int i;
+		
+	for (s = pp -> sh; s; s = s -> next)
+	{
+		printf("%s", s -> name);
+		if (s -> nargs >= 0)
+		{
+			printf("(");
+			for (i = 0; i < s -> nargs; i++)
+			{
+				if (i)
+					printf(",");
+				printf("%s", s -> params[i]);
+			}
+			if (s -> vargs)
+			{
+				if (s -> nargs)
+					printf(",");
+				printf("...");
+			}
+			printf(")");
+		}
+		printf(" => ");
+		if (s -> tl)
+		{
+			for (t = s -> tl -> head; t; t = t -> next)
+			{
+				token_print(t, stdout);
+			}
+		}
+		printf("\n");
+	}
+}
diff -r d85d173ba120 -r 54f213c8fb81 lwcc/token.c
--- a/lwcc/token.c	Tue Sep 17 19:33:41 2013 -0600
+++ b/lwcc/token.c	Wed Sep 18 19:17:52 2013 -0600
@@ -41,6 +41,7 @@
 	t -> fn = fn;
 	t -> next = NULL;
 	t -> prev = NULL;
+	t -> list = NULL;
 	return t;
 }
 
@@ -55,11 +56,16 @@
 	struct token *t2;
 	
 	t2 = lw_alloc(sizeof(struct token));
-	(*t2) = (*t);
+	t2 -> ttype = t -> ttype;
+	t2 -> lineno = t -> lineno;
+	t2 -> column = t -> column;
+	t2 -> list = NULL;
 	t2 -> next = NULL;
 	t2 -> prev = NULL;
 	if (t -> strval)
 		t2 -> strval = lw_strdup(t -> strval);
+	else
+		t2 -> strval = NULL;
 	return t2;
 }
 
@@ -152,7 +158,6 @@
 		tl -> tail = tl -> head;
 		tl -> head = tl -> head -> next;
 		token_free(tl -> tail);
-		lw_free(tl -> tail);
 	}
 	lw_free(tl);
 }
diff -r d85d173ba120 -r 54f213c8fb81 lwcc/token.h
--- a/lwcc/token.h	Tue Sep 17 19:33:41 2013 -0600
+++ b/lwcc/token.h	Wed Sep 18 19:17:52 2013 -0600
@@ -39,7 +39,7 @@
 	TOK_WSPACE,
 	TOK_IDENT,
 	TOK_NUMBER,
-	TOK_STRING,
+//	TOK_STRING,
 	TOK_CHAR,
 	TOK_DIV,
 	TOK_ADD,