tocariimaa/rutile
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lexer: make each token return its location

Browse source
This commit is contained in:
tocariimaa 2025-01-12 23:13:17 -03:00
parent 2a83bc00f8
commit 7f5bcf4f77
2 changed files with 17 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
compiler/lex.c
View file

@ -228,6 +228,7 @@ intern_identifier(LexState *ls, u8 *ident)
static LexToken
identifier(LexState *ls)
{
LexToken token = { .loc = ls->cur_loc };
/* this gets copied to the hash table arena, no problem */
u8 ident_buf[MAX_IDENT_SIZE];
usize i = 0;
@ -246,16 +247,16 @@ identifier(LexState *ls)
if (chr.ok)
backup(ls, 1);
return (LexToken) {
.id = T_IDENT,
.ident = {intern_identifier(ls, ident_buf), i},
.len = i,
};
token.id = T_IDENT;
token.ident = (Str){intern_identifier(ls, ident_buf), i};
token.len = i;
return token;
}
static LexToken
string_literal(LexState *ls)
{
LexToken token = { .loc = ls->cur_loc };
isize str_buf_len = STRING_LITERAL_BASE_SIZE;
u8 *str_buf = malloc(str_buf_len);
isize i = 0;
@ -284,11 +285,10 @@ string_literal(LexState *ls)
str_buf = nil;
}
return (LexToken) {
.id = T_STRING,
.str = {str_buf, i},
.len = i,
};
token.id = T_STRING;
token.str = (Str){str_buf, i};
token.len = i;
return token;
err:
return make_error();
}
@ -304,7 +304,7 @@ err:
static LexToken
number_literal(LexState *ls)
{
LexToken t = { .id = T_NUMBER };
LexToken token = { .id = T_NUMBER, .loc = ls->cur_loc };
u64 number = 0;
u8 base = 10;
@ -313,7 +313,7 @@ number_literal(LexState *ls)
if (chr.val == '0') {
chr = read_chr(ls); /* skip 0 prefix */
if (!chr.ok) { /* EOF edge case */
return t; /* 0 */
return token; /* 0 */
}
switch (chr.val) {
case 'b':
@ -332,7 +332,7 @@ number_literal(LexState *ls)
}
//lex_error(ls, "unknown numeric prefix '0%c'", chr.val);
/* start of another token */
return t; /* 0 */
return token; /* 0 */
}
chr = read_chr(ls);
if (!chr.ok) {
@ -367,8 +367,8 @@ number_literal(LexState *ls)
if (chr.ok)
backup(ls, 1);
t.inumber = number;
return t;
token.inumber = number;
return token;
overflow:
lex_error(ls, "integer literal is too big (2^64 max)");
return make_error();
@ -378,7 +378,7 @@ static LexToken
keyword(LexToken *t)
{
#define kwcmp(ident, kw, tid) \
{if (Str_equal(ident, kw)) return (LexToken){ .id = tid, .len = kw.len };}
{if (Str_equal(ident, kw)) return (LexToken){ .id = tid, .len = kw.len, .loc = t->loc };}
Str ident = t->ident;
--ident.len;

1
compiler/lex.h
View file

@ -56,6 +56,7 @@ typedef struct {
double floatn;
};
isize len; /* Size in bytes of this token */
Location loc; /* Start position of this token in the file or stream */
} LexToken;
typedef HashMapStr(i8) IdentsBucket;