ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new handling for strings in lexer

This commit is contained in:
Allen Webster 2017-04-17 19:34:54 -04:00
parent bc4c866cf0
commit 9d1756ad2b
5 changed files with 1666 additions and 992 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

738
4cpp/4cpp_lexer.h
View File

@ -340,6 +340,8 @@ cpp__table_match(String_And_Flag *table, i32_4tech count, char *s, i32_4tech len
return(result); return(result);
} }
#define LEXER_TB(n) ((n) & (sizeof(S.tb)-1))
FCPP_LINK Cpp_Lex_Result FCPP_LINK Cpp_Lex_Result
cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech size, Cpp_Token_Array *token_array_out){ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech size, Cpp_Token_Array *token_array_out){
Cpp_Lex_Data S = *S_ptr; Cpp_Lex_Data S = *S_ptr;
@ -359,7 +361,11 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
DfrCase(3); DfrCase(3);
DfrCase(4); DfrCase(4);
DfrCase(5); DfrCase(5);
DfrCase(6);
DfrCase(7); DfrCase(7);
DfrCase(8);
DfrCase(9);
DfrCase(10);
} }
for (;;){ for (;;){
@ -401,11 +407,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
for (; S.fsm.state < LS_count && S.pos < end_pos;){ for (; S.fsm.state < LS_count && S.pos < end_pos;){
c = chunk[S.pos++]; c = chunk[S.pos++];
S.tb[(S.tb_pos++) & (sizeof(S.tb)-1)] = c; S.tb[LEXER_TB(S.tb_pos++)] = c;
i32_4tech i = S.fsm.state + eq_classes[c]; i32_4tech i = S.fsm.state + eq_classes[c];
S.fsm.state = fsm_table[i]; S.fsm.state = fsm_table[i];
S.fsm.multi_line |= multiline_state_table[S.fsm.state];
} }
S.fsm.emit_token = (S.fsm.state >= LS_count); S.fsm.emit_token = (S.fsm.state >= LS_count);
} }
@ -432,65 +437,68 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
switch (S.fsm.state){ switch (S.fsm.state){
case LS_default: case LS_default:
switch (c){ {
case 0: S.fsm.emit_token = 0; break; switch (c){
case 0: S.fsm.emit_token = 0; break;
#define OperCase(op,t) case op: S.token.type = t; break; #define OperCase(op,t) case op: S.token.type = t; break;
OperCase('{', CPP_TOKEN_BRACE_OPEN); OperCase('{', CPP_TOKEN_BRACE_OPEN);
OperCase('}', CPP_TOKEN_BRACE_CLOSE); OperCase('}', CPP_TOKEN_BRACE_CLOSE);
OperCase('[', CPP_TOKEN_BRACKET_OPEN); OperCase('[', CPP_TOKEN_BRACKET_OPEN);
OperCase(']', CPP_TOKEN_BRACKET_CLOSE); OperCase(']', CPP_TOKEN_BRACKET_CLOSE);
OperCase('(', CPP_TOKEN_PARENTHESE_OPEN); OperCase('(', CPP_TOKEN_PARENTHESE_OPEN);
OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE); OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE);
OperCase('~', CPP_TOKEN_TILDE); OperCase('~', CPP_TOKEN_TILDE);
OperCase(',', CPP_TOKEN_COMMA); OperCase(',', CPP_TOKEN_COMMA);
OperCase(';', CPP_TOKEN_SEMICOLON); OperCase(';', CPP_TOKEN_SEMICOLON);
OperCase('?', CPP_TOKEN_TERNARY_QMARK); OperCase('?', CPP_TOKEN_TERNARY_QMARK);
OperCase('@', CPP_TOKEN_JUNK); OperCase('@', CPP_TOKEN_JUNK);
#undef OperCase #undef OperCase
case '\\': case '\\':
if (S.pp_state == LSPP_default || S.pp_state == LSPP_no_strings){ if (S.pp_state == LSPP_default || S.pp_state == LSPP_no_strings){
S.token.type = CPP_TOKEN_JUNK; S.token.type = CPP_TOKEN_JUNK;
} }
else{ else{
S.pos_overide = S.pos; S.pos_overide = S.pos;
S.white_done = 0; S.white_done = false;
for (;;){ for (;;){
for (; S.white_done == 0 && S.pos < end_pos;){ for (; !S.white_done && S.pos < end_pos;){
c = chunk[S.pos++]; c = chunk[S.pos++];
if (!(c == ' ' || c == '\t' || c == '\r' || c == '\v' || c == '\f')){ if (!(c == ' ' || c == '\t' || c == '\r' || c == '\v' || c == '\f')){
S.white_done = 1; S.white_done = true;
}
}
if (!S.white_done){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(1, LexResult_NeedChunk);
}
else{
break;
} }
} }
if (S.white_done == 0){ if (c == '\n'){
S.chunk_pos += size; S.fsm.emit_token = 0;
token_array_out->count = token_i; S.pos_overide = 0;
DfrYield(1, LexResult_NeedChunk); }
else{
S.token.type = CPP_TOKEN_JUNK;
} }
else break;
}
if (c == '\n'){
S.fsm.emit_token = 0;
S.pos_overide = 0;
}
else{
S.token.type = CPP_TOKEN_JUNK;
} }
break;
} }
break;
}
if (c != '@' && c != '\\'){ if (c != '@' && c != '\\'){
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
} }
break; }break;
case LS_identifier: case LS_identifier:
{ {
@ -523,15 +531,16 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
}break; }break;
case LS_pound: case LS_pound:
S.token.flags = 0; {
switch (c){ S.token.flags = 0;
case '#': S.token.type = CPP_PP_CONCAT; break; switch (c){
default: case '#': S.token.type = CPP_PP_CONCAT; break;
S.token.type = CPP_PP_STRINGIFY; default:
--S.pos; S.token.type = CPP_PP_STRINGIFY;
break; --S.pos;
} break;
break; }
}break;
case LS_pp: case LS_pp:
{ {
@ -572,307 +581,423 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
case LS_number: case LS_number:
case LS_number0: case LS_number0:
case LS_hex: case LS_hex:
S.fsm.int_state = LSINT_default; {
S.fsm.emit_token = 0; S.fsm.state = LSINT_default;
--S.pos; S.fsm.emit_token = 0;
for (;;){ --S.pos;
for (; S.fsm.int_state < LSINT_count && S.pos < end_pos;){ for (;;){
c = chunk[S.pos++]; for (; S.fsm.state < LSINT_count && S.pos < end_pos;){
S.fsm.int_state = int_fsm_table[S.fsm.int_state + int_fsm_eq_classes[c]]; c = chunk[S.pos++];
} S.fsm.state = int_fsm_table[S.fsm.state + int_fsm_eq_classes[c]];
S.fsm.emit_token = (S.fsm.int_state >= LSINT_count); }
S.fsm.emit_token = (S.fsm.state >= LSINT_count);
if (S.fsm.emit_token == 0){ if (S.fsm.emit_token == 0){
S.chunk_pos += size; S.chunk_pos += size;
token_array_out->count = token_i; token_array_out->count = token_i;
DfrYield(5, LexResult_NeedChunk); DfrYield(5, LexResult_NeedChunk);
}
else{
break;
}
} }
else break; --S.pos;
}
--S.pos;
S.token.type = CPP_TOKEN_INTEGER_CONSTANT; S.token.type = CPP_TOKEN_INTEGER_CONSTANT;
S.token.flags = 0; S.token.flags = 0;
break; }break;
case LS_float: case LS_float:
case LS_crazy_float0: case LS_crazy_float0:
case LS_crazy_float1: case LS_crazy_float1:
S.token.type = CPP_TOKEN_FLOATING_CONSTANT; {
S.token.flags = 0; S.token.type = CPP_TOKEN_FLOATING_CONSTANT;
switch (c){ S.token.flags = 0;
case 'f': case 'F': switch (c){
case 'l': case 'L':break; case 'f': case 'F': case 'l': case 'L': break;
default: default: --S.pos; break;
--S.pos; }
break; }break;
}
break;
case LS_char: case LS_string_raw:
case LS_char_slashed: {
S.token.type = CPP_TOKEN_JUNK; S.tb_pos = 0;
if (c == '\''){ S.delim_length = 0;
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
}
S.token.flags = 0;
break;
case LS_char_multiline: S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.type = CPP_TOKEN_JUNK;
if (c == '\''){
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
}
S.token.flags = CPP_TFLAG_MULTILINE;
break;
case LS_string: S.fsm.state = LSSTR_default;
case LS_string_slashed: S.fsm.flags = 0;
S.token.type = CPP_TOKEN_JUNK; for (;;){
if (S.pp_state == LSPP_include){ for (; S.fsm.state < LSSTR_count && S.pos < end_pos;){
if (c == '>' || c == '"'){ c = chunk[S.pos++];
S.tb[LEXER_TB(S.tb_pos++)] = c;
S.fsm.state = raw_str_table[S.fsm.state + raw_str_eq_classes[c]];
S.fsm.flags |= raw_str_flags[S.fsm.state];
}
S.fsm.emit_token = (S.fsm.state >= LSSTR_count);
if (S.fsm.emit_token == 0){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(7, LexResult_NeedChunk);
}
else{
u8_4tech emit_state = S.fsm.state - LSSTR_count;
switch (emit_state){
case LSSTR_default:
{
S.token.type = CPP_TOKEN_JUNK;
goto doublebreak;
}break;
case LSSTR_get_delim:
{
if (S.tb_pos <= 17){
S.delim_length = S.tb_pos-1;
for (i32_4tech n = 0; n < S.delim_length; ++n){
S.raw_delim[n] = S.tb[n];
}
S.tb_pos = 0;
}
else{
S.token.type = CPP_TOKEN_JUNK;
--S.pos;
goto doublebreak;
}
}break;
case LSSTR_check_delim:
{
if (c == 0){
goto doublebreak;
}
else if (S.tb_pos >= S.delim_length){
u32_4tech m = S.tb_pos - S.delim_length - 2;
if (S.tb[LEXER_TB(m)] == ')'){
b32_4tech is_match = true;
++m;
for (i32_4tech n = 0; n < S.delim_length; ++n, ++m){
if (S.tb[LEXER_TB(m)] != S.raw_delim[n]){
is_match = false;
break;
}
}
if (is_match){
goto doublebreak;
}
}
}
}break;
}
S.fsm.state = LSSTR_get_delim;
}
}
doublebreak:;
S.token.flags = (S.fsm.flags)?(CPP_TFLAG_MULTILINE):(0);
}break;
case LS_string_normal:
{
S.fsm.state = LSSTR_default;
S.fsm.flags = 0;
for (;;){
for (; S.fsm.state < LSSTR_count && S.pos < end_pos;){
c = chunk[S.pos++];
S.fsm.state = normal_str_table[S.fsm.state + normal_str_eq_classes[c]];
S.fsm.flags |= normal_str_flags[S.fsm.state];
}
S.fsm.emit_token = (S.fsm.state >= LSSTR_count);
if (S.fsm.emit_token == 0){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(8, LexResult_NeedChunk);
}
else{
break;
}
}
S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.flags = (S.fsm.flags)?(CPP_TFLAG_MULTILINE):(0);
if (c == '\n'){
--S.pos;
}
}break;
case LS_string_include:
{
S.fsm.state = LSSTR_default;
for (;;){
for (; S.fsm.state < LSSTR_include_count && S.pos < end_pos;){
c = chunk[S.pos++];
S.fsm.state = include_str_table[S.fsm.state + include_str_eq_classes[c]];
}
S.fsm.emit_token = (S.fsm.state >= LSSTR_include_count);
if (S.fsm.emit_token == 0){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(6, LexResult_NeedChunk);
}
else{
break;
}
}
S.fsm.state -= LSSTR_include_count;
if (S.fsm.state == LSSTR_default){
S.token.type = CPP_PP_INCLUDE_FILE; S.token.type = CPP_PP_INCLUDE_FILE;
} }
} else{
else{ S.token.type = CPP_TOKEN_JUNK;
if (c == '"'){
S.token.type = CPP_TOKEN_STRING_CONSTANT;
} }
} S.token.flags = 0;
S.token.flags = 0;
break;
case LS_string_multiline: if (c == '\n'){
S.token.type = CPP_TOKEN_JUNK; --S.pos;
if (c == '"'){ }
S.token.type = CPP_TOKEN_STRING_CONSTANT; }break;
}
S.token.flags = CPP_TFLAG_MULTILINE; case LS_char:
break; {
S.fsm.state = LSSTR_default;
S.fsm.flags = 0;
for (;;){
for (; S.fsm.state < LSSTR_count && S.pos < end_pos;){
c = chunk[S.pos++];
S.fsm.state = normal_char_table[S.fsm.state + normal_char_eq_classes[c]];
S.fsm.flags |= normal_char_flags[S.fsm.state];
}
S.fsm.emit_token = (S.fsm.state >= LSSTR_count);
if (S.fsm.emit_token == 0){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(9, LexResult_NeedChunk);
}
else{
break;
}
}
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
S.token.flags = (S.fsm.flags)?(CPP_TFLAG_MULTILINE):(0);
if (c == '\n'){
--S.pos;
}
}break;
case LS_comment_pre: case LS_comment_pre:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '=': S.token.type = CPP_TOKEN_DIVEQ; break; switch (c){
default: case '=': S.token.type = CPP_TOKEN_DIVEQ; break;
S.token.type = CPP_TOKEN_DIV; default:
--S.pos; S.token.type = CPP_TOKEN_DIV;
break; --S.pos;
} break;
break; }
}break;
case LS_comment: case LS_comment:
case LS_comment_slashed: case LS_comment_slashed:
S.token.type = CPP_TOKEN_COMMENT; {
S.token.flags = 0; S.token.type = CPP_TOKEN_COMMENT;
--S.pos; S.token.flags = 0;
break; --S.pos;
}break;
case LS_comment_block: case LS_comment_block:
case LS_comment_block_ending: case LS_comment_block_ending:
S.token.type = CPP_TOKEN_COMMENT; {
S.token.flags = 0; S.token.type = CPP_TOKEN_COMMENT;
break; S.token.flags = 0;
}break;
case LS_error_message: case LS_error_message:
S.token.type = CPP_PP_ERROR_MESSAGE; {
S.token.flags = 0; S.token.type = CPP_PP_ERROR_MESSAGE;
--S.pos; S.token.flags = 0;
break; --S.pos;
}break;
case LS_dot: case LS_dot:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '*': S.token.type = CPP_TOKEN_PTRDOT; break; switch (c){
default: case '*': S.token.type = CPP_TOKEN_PTRDOT; break;
S.token.type = CPP_TOKEN_DOT; default:
--S.pos; S.token.type = CPP_TOKEN_DOT;
break; --S.pos;
} break;
break; }
}break;
case LS_ellipsis: case LS_ellipsis:
switch (c){ {
case '.': switch (c){
S.token.flags = CPP_TFLAG_IS_OPERATOR; case '.':
S.token.type = CPP_TOKEN_ELLIPSIS; S.token.flags = CPP_TFLAG_IS_OPERATOR;
break; S.token.type = CPP_TOKEN_ELLIPSIS;
break;
default: default:
S.token.type = CPP_TOKEN_JUNK; S.token.type = CPP_TOKEN_JUNK;
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_less: case LS_less:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '=': S.token.type = CPP_TOKEN_LESSEQ; break; switch (c){
default: case '=': S.token.type = CPP_TOKEN_LESSEQ; break;
S.token.type = CPP_TOKEN_LESS; default:
--S.pos; S.token.type = CPP_TOKEN_LESS;
break; --S.pos;
} break;
break; }
}break;
case LS_less_less: case LS_less_less:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '=': S.token.type = CPP_TOKEN_LSHIFTEQ; break; switch (c){
default: case '=': S.token.type = CPP_TOKEN_LSHIFTEQ; break;
S.token.type = CPP_TOKEN_LSHIFT; default:
--S.pos; S.token.type = CPP_TOKEN_LSHIFT;
break; --S.pos;
} break;
break; }
}break;
case LS_more: case LS_more:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '=': S.token.type = CPP_TOKEN_GRTREQ; break; switch (c){
default: case '=': S.token.type = CPP_TOKEN_GRTREQ; break;
S.token.type = CPP_TOKEN_GRTR; default:
--S.pos; S.token.type = CPP_TOKEN_GRTR;
break; --S.pos;
} break;
break; }
}break;
case LS_more_more: case LS_more_more:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '=': S.token.type = CPP_TOKEN_RSHIFTEQ; break; switch (c){
default: case '=': S.token.type = CPP_TOKEN_RSHIFTEQ; break;
S.token.type = CPP_TOKEN_RSHIFT; default:
--S.pos; S.token.type = CPP_TOKEN_RSHIFT;
break; --S.pos;
} break;
break; }
}break;
case LS_minus: case LS_minus:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '-': S.token.type = CPP_TOKEN_DECREMENT; break; switch (c){
case '=': S.token.type = CPP_TOKEN_SUBEQ; break; case '-': S.token.type = CPP_TOKEN_DECREMENT; break;
default: case '=': S.token.type = CPP_TOKEN_SUBEQ; break;
S.token.type = CPP_TOKEN_MINUS; default:
--S.pos; S.token.type = CPP_TOKEN_MINUS;
break; --S.pos;
} break;
break; }
}break;
case LS_arrow: case LS_arrow:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '*': S.token.type = CPP_TOKEN_PTRARROW; break; switch (c){
default: case '*': S.token.type = CPP_TOKEN_PTRARROW; break;
S.token.type = CPP_TOKEN_ARROW; default:
--S.pos; S.token.type = CPP_TOKEN_ARROW;
break; --S.pos;
} break;
break; }
}break;
case LS_and: case LS_and:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '&': S.token.type = CPP_TOKEN_AND; break; switch (c){
case '=': S.token.type = CPP_TOKEN_ANDEQ; break; case '&': S.token.type = CPP_TOKEN_AND; break;
default: case '=': S.token.type = CPP_TOKEN_ANDEQ; break;
S.token.type = CPP_TOKEN_AMPERSAND; default:
--S.pos; S.token.type = CPP_TOKEN_AMPERSAND;
break; --S.pos;
} break;
break; }
}break;
case LS_or: case LS_or:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '|': S.token.type = CPP_TOKEN_OR; break; switch (c){
case '=': S.token.type = CPP_TOKEN_OREQ; break; case '|': S.token.type = CPP_TOKEN_OR; break;
default: case '=': S.token.type = CPP_TOKEN_OREQ; break;
S.token.type = CPP_TOKEN_BIT_OR; default:
--S.pos; S.token.type = CPP_TOKEN_BIT_OR;
break; --S.pos;
} break;
break; }
}break;
case LS_plus: case LS_plus:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case '+': S.token.type = CPP_TOKEN_INCREMENT; break; switch (c){
case '=': S.token.type = CPP_TOKEN_ADDEQ; break; case '+': S.token.type = CPP_TOKEN_INCREMENT; break;
default: case '=': S.token.type = CPP_TOKEN_ADDEQ; break;
S.token.type = CPP_TOKEN_PLUS; default:
--S.pos; S.token.type = CPP_TOKEN_PLUS;
break; --S.pos;
} break;
break; }
}break;
case LS_colon: case LS_colon:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ S.token.flags = CPP_TFLAG_IS_OPERATOR;
case ':': S.token.type = CPP_TOKEN_SCOPE; break; switch (c){
default: case ':': S.token.type = CPP_TOKEN_SCOPE; break;
S.token.type = CPP_TOKEN_COLON; default:
--S.pos; S.token.type = CPP_TOKEN_COLON;
break; --S.pos;
} break;
break; }
}break;
case LS_star: case LS_single_op:
S.token.flags = CPP_TFLAG_IS_OPERATOR; {
switch (c){ u32_4tech plain_version = 0;
case '=': S.token.type = CPP_TOKEN_MULEQ; break; u32_4tech eq_version = 0;
default: S.token.flags = CPP_TFLAG_IS_OPERATOR;
S.token.type = CPP_TOKEN_STAR; switch (S.tb[0]){
--S.pos; case '*': plain_version = CPP_TOKEN_STAR; eq_version = CPP_TOKEN_MULEQ; break;
break; case '%': plain_version = CPP_TOKEN_MOD; eq_version = CPP_TOKEN_MODEQ; break;
} case '^': plain_version = CPP_TOKEN_BIT_XOR; eq_version = CPP_TOKEN_XOREQ; break;
break; case '=': plain_version = CPP_TOKEN_EQ; eq_version = CPP_TOKEN_EQEQ; break;
case '!': plain_version = CPP_TOKEN_NOT; eq_version = CPP_TOKEN_NOTEQ; break;
}
case LS_modulo: S.token.type = eq_version;
S.token.flags = CPP_TFLAG_IS_OPERATOR; if (c != '='){
switch (c){ S.token.type = plain_version;
case '=': S.token.type = CPP_TOKEN_MODEQ; break; --S.pos;
default: }
S.token.type = CPP_TOKEN_MOD; }break;
--S.pos;
break;
}
break;
case LS_caret:
S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){
case '=': S.token.type = CPP_TOKEN_XOREQ; break;
default:
S.token.type = CPP_TOKEN_BIT_XOR;
--S.pos;
break;
}
break;
case LS_eq:
S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){
case '=': S.token.type = CPP_TOKEN_EQEQ; break;
default:
S.token.type = CPP_TOKEN_EQ;
--S.pos;
break;
}
break;
case LS_bang:
S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){
case '=': S.token.type = CPP_TOKEN_NOTEQ; break;
default:
S.token.type = CPP_TOKEN_NOT;
--S.pos;
break;
}
break;
} }
if (S.pos > S.chunk_pos && chunk[S.pos-1] == 0){ if (S.pos > S.chunk_pos && chunk[S.pos-1] == 0){
@ -940,7 +1065,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
if (S.pos == end_pos){ if (S.pos == end_pos){
S.chunk_pos += size; S.chunk_pos += size;
token_array_out->count = token_i; token_array_out->count = token_i;
DfrYield(7, LexResult_NeedChunk); DfrYield(10, LexResult_NeedChunk);
} }
token_array_out->count = token_i; token_array_out->count = token_i;
DfrYield(2, LexResult_NeedTokenMemory); DfrYield(2, LexResult_NeedTokenMemory);
@ -1182,6 +1307,7 @@ The start and end points are based on the edited region of the file before the e
if (end_pos > array->tokens[range.end_token_index].start){ if (end_pos > array->tokens[range.end_token_index].start){
++range.end_token_index; ++range.end_token_index;
} }
++range.end_token_index;
if (range.end_token_index < 0){ if (range.end_token_index < 0){
range.end_token_index = 0; range.end_token_index = 0;
} }

992
4cpp/4cpp_lexer_tables.c
View File

File diff suppressed because it is too large Load Diff

134
4cpp/4cpp_lexer_types.h
View File

@ -75,32 +75,25 @@ ENUM(uint32_t, Cpp_Token_Type){
CPP_TOKEN_SEMICOLON = 46, CPP_TOKEN_SEMICOLON = 46,
CPP_TOKEN_ELLIPSIS = 47, CPP_TOKEN_ELLIPSIS = 47,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_STAR = 48, CPP_TOKEN_STAR = 48,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_AMPERSAND = 49, CPP_TOKEN_AMPERSAND = 49,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_TILDE = 50, CPP_TOKEN_TILDE = 50,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_PLUS = 51, CPP_TOKEN_PLUS = 51,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_MINUS = 52, CPP_TOKEN_MINUS = 52,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_INCREMENT = 53, CPP_TOKEN_INCREMENT = 53,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_DECREMENT = 54, CPP_TOKEN_DECREMENT = 54,
// NOTE(allen): Precedence 1, LtoR // NOTE(allen): Precedence 1, LtoR
@ -309,9 +302,8 @@ STRUCT Cpp_Relex_Range{
struct Cpp_Lex_FSM{ struct Cpp_Lex_FSM{
uint8_t state; uint8_t state;
uint8_t int_state;
uint8_t emit_token; uint8_t emit_token;
uint8_t multi_line; uint8_t flags;
}; };
static Cpp_Lex_FSM null_lex_fsm = {0}; static Cpp_Lex_FSM null_lex_fsm = {0};
@ -322,23 +314,29 @@ DOC_SEE(cpp_lex_data_init)
HIDE_MEMBERS() */ HIDE_MEMBERS() */
STRUCT Cpp_Lex_Data{ STRUCT Cpp_Lex_Data{
char tb[32]; char tb[32];
int32_t tb_pos; i32_4tech tb_pos;
int32_t token_start; i32_4tech token_start;
int32_t pos; i32_4tech pos;
int32_t pos_overide; i32_4tech pos_overide;
int32_t chunk_pos; i32_4tech chunk_pos;
Cpp_Lex_FSM fsm; Cpp_Lex_FSM fsm;
uint8_t white_done; u8_4tech white_done;
uint8_t pp_state; u8_4tech pp_state;
uint8_t completed; u8_4tech completed;
Cpp_Token token; Cpp_Token token;
int32_t ignore_string_delims; char raw_delim[16];
i32_4tech delim_length;
int32_t __pc__; b8_4tech str_raw;
b8_4tech str_include;
i32_4tech ignore_string_delims;
i32_4tech __pc__;
}; };
/* DOC(Cpp_Lex_Result is returned from the lexing engine to indicate why it stopped lexing.) */ /* DOC(Cpp_Lex_Result is returned from the lexing engine to indicate why it stopped lexing.) */
@ -395,45 +393,44 @@ ENUM_INTERNAL(uint8_t, Cpp_Lex_State){
LS_pound = 2, LS_pound = 2,
LS_pp = 3, LS_pp = 3,
LS_ppdef = 4, LS_ppdef = 4,
LS_char = 5, LS_string_R = 5,
LS_char_multiline = 6, LS_string_LUu8 = 6,
LS_char_slashed = 7, LS_string_u = 7,
LS_string = 8, LS_number = 8,
LS_string_multiline = 9, LS_number0 = 9,
LS_string_slashed = 10, LS_float = 10,
LS_number = 11, LS_crazy_float0 = 11,
LS_number0 = 12, LS_crazy_float1 = 12,
LS_float = 13, LS_hex = 13,
LS_crazy_float0 = 14, LS_comment_pre = 14,
LS_crazy_float1 = 15, LS_comment = 15,
LS_hex = 16, LS_comment_slashed = 16,
LS_comment_pre = 17, LS_comment_block = 17,
LS_comment = 18, LS_comment_block_ending = 18,
LS_comment_slashed = 19, LS_dot = 19,
LS_comment_block = 20, LS_ellipsis = 20,
LS_comment_block_ending = 21, LS_less = 21,
LS_dot = 22, LS_less_less = 22,
LS_ellipsis = 23, LS_more = 23,
LS_less = 24, LS_more_more = 24,
LS_less_less = 25, LS_minus = 25,
LS_more = 26, LS_arrow = 26,
LS_more_more = 27, LS_and = 27,
LS_minus = 28, LS_or = 28,
LS_arrow = 29, LS_plus = 29,
LS_and = 30, LS_colon = 30,
LS_or = 31, LS_single_op = 31,
LS_plus = 32, LS_error_message = 32,
LS_colon = 33,
LS_star = 34,
LS_modulo = 35,
LS_caret = 36,
LS_eq = 37,
LS_bang = 38,
LS_error_message = 39,
// //
LS_count = 40 LS_count = 33,
LS_char = 34,
}; };
// NOTE(allen): These provide names that match the overloaded meanings of string states.
#define LS_string_raw LS_string_R
#define LS_string_normal LS_string_LUu8
#define LS_string_include LS_string_u
ENUM_INTERNAL(uint8_t, Cpp_Lex_Int_State){ ENUM_INTERNAL(uint8_t, Cpp_Lex_Int_State){
LSINT_default, LSINT_default,
LSINT_u, LSINT_u,
@ -447,6 +444,19 @@ ENUM_INTERNAL(uint8_t, Cpp_Lex_Int_State){
LSINT_count LSINT_count
}; };
ENUM_INTERNAL(uint8_t, Cpp_Lex_Str_State){
LSSTR_default,
LSSTR_escape,
LSSTR_multiline,
//
LSSTR_count
};
#define LSSTR_include_count 1
#define LSSTR_error LSSTR_escape
#define LSSTR_get_delim LSSTR_escape
#define LSSTR_check_delim LSSTR_count
ENUM_INTERNAL(uint8_t, Cpp_Lex_PP_State){ ENUM_INTERNAL(uint8_t, Cpp_Lex_PP_State){
LSPP_default, LSPP_default,
LSPP_include, LSPP_include,

BIN
4ed_site.ctm
View File

Binary file not shown.

774
meta/fsm_table_generator.cpp
View File

File diff suppressed because it is too large Load Diff