diff --git a/compiler/lex.c b/compiler/lex.c index 30fc41a..850ba04 100644 --- a/compiler/lex.c +++ b/compiler/lex.c @@ -228,6 +228,7 @@ intern_identifier(LexState *ls, u8 *ident) static LexToken identifier(LexState *ls) { + LexToken token = { .loc = ls->cur_loc }; /* this gets copied to the hash table arena, no problem */ u8 ident_buf[MAX_IDENT_SIZE]; usize i = 0; @@ -246,16 +247,16 @@ identifier(LexState *ls) if (chr.ok) backup(ls, 1); - return (LexToken) { - .id = T_IDENT, - .ident = {intern_identifier(ls, ident_buf), i}, - .len = i, - }; + token.id = T_IDENT; + token.ident = (Str){intern_identifier(ls, ident_buf), i}; + token.len = i; + return token; } static LexToken string_literal(LexState *ls) { + LexToken token = { .loc = ls->cur_loc }; isize str_buf_len = STRING_LITERAL_BASE_SIZE; u8 *str_buf = malloc(str_buf_len); isize i = 0; @@ -284,11 +285,10 @@ string_literal(LexState *ls) str_buf = nil; } - return (LexToken) { - .id = T_STRING, - .str = {str_buf, i}, - .len = i, - }; + token.id = T_STRING; + token.str = (Str){str_buf, i}; + token.len = i; + return token; err: return make_error(); } @@ -304,7 +304,7 @@ err: static LexToken number_literal(LexState *ls) { - LexToken t = { .id = T_NUMBER }; + LexToken token = { .id = T_NUMBER, .loc = ls->cur_loc }; u64 number = 0; u8 base = 10; @@ -313,7 +313,7 @@ number_literal(LexState *ls) if (chr.val == '0') { chr = read_chr(ls); /* skip 0 prefix */ if (!chr.ok) { /* EOF edge case */ - return t; /* 0 */ + return token; /* 0 */ } switch (chr.val) { case 'b': @@ -332,7 +332,7 @@ number_literal(LexState *ls) } //lex_error(ls, "unknown numeric prefix '0%c'", chr.val); /* start of another token */ - return t; /* 0 */ + return token; /* 0 */ } chr = read_chr(ls); if (!chr.ok) { @@ -367,8 +367,8 @@ number_literal(LexState *ls) if (chr.ok) backup(ls, 1); - t.inumber = number; - return t; + token.inumber = number; + return token; overflow: lex_error(ls, "integer literal is too big (2^64 max)"); return make_error(); @@ -378,7 +378,7 @@ static LexToken keyword(LexToken *t) { #define kwcmp(ident, kw, tid) \ - {if (Str_equal(ident, kw)) return (LexToken){ .id = tid, .len = kw.len };} + {if (Str_equal(ident, kw)) return (LexToken){ .id = tid, .len = kw.len, .loc = t->loc };} Str ident = t->ident; --ident.len; diff --git a/compiler/lex.h b/compiler/lex.h index 6d46a5f..6589d46 100644 --- a/compiler/lex.h +++ b/compiler/lex.h @@ -56,6 +56,7 @@ typedef struct { double floatn; }; isize len; /* Size in bytes of this token */ + Location loc; /* Start position of this token in the file or stream */ } LexToken; typedef HashMapStr(i8) IdentsBucket;