1566 lines
45 KiB
C++
1566 lines
45 KiB
C++
|
|
// TOP
|
|
|
|
#ifndef FCPP_NEW_LEXER_INC
|
|
#define FCPP_NEW_LEXER_INC
|
|
|
|
#include "../4cpp_lexer_types.h"
|
|
|
|
namespace new_lex{
|
|
//
|
|
|
|
#define lexer_link static
|
|
|
|
// TODO(allen): revisit this keyword data declaration system
|
|
struct String_And_Flag{
|
|
char *str;
|
|
fcpp_u32 flags;
|
|
};
|
|
|
|
struct String_List{
|
|
String_And_Flag *data;
|
|
int count;
|
|
};
|
|
|
|
struct Sub_Match_List_Result{
|
|
int index;
|
|
fcpp_i32 new_pos;
|
|
};
|
|
|
|
|
|
// TODO(allen): shift towards storing in a context
|
|
static String_And_Flag int_suf_strings[] = {
|
|
{"ull"}, {"ULL"},
|
|
{"llu"}, {"LLU"},
|
|
{"ll"}, {"LL"},
|
|
{"l"}, {"L"},
|
|
{"u"}, {"U"}
|
|
};
|
|
|
|
#define lexer_string_list(x) {x, (sizeof(x)/sizeof(*x))}
|
|
|
|
static String_List int_sufs = lexer_string_list(int_suf_strings);
|
|
|
|
static String_And_Flag float_suf_strings[] = {
|
|
{"f"}, {"F"},
|
|
{"l"}, {"L"}
|
|
};
|
|
static String_List float_sufs = lexer_string_list(float_suf_strings);
|
|
|
|
static String_And_Flag bool_lit_strings[] = {
|
|
{"true"}, {"false"}
|
|
};
|
|
static String_List bool_lits = lexer_string_list(bool_lit_strings);
|
|
|
|
static String_And_Flag keyword_strings[] = {
|
|
{"and", CPP_TOKEN_AND},
|
|
{"and_eq", CPP_TOKEN_ANDEQ},
|
|
{"bitand", CPP_TOKEN_BIT_AND},
|
|
{"bitor", CPP_TOKEN_BIT_OR},
|
|
{"or", CPP_TOKEN_OR},
|
|
{"or_eq", CPP_TOKEN_OREQ},
|
|
{"sizeof", CPP_TOKEN_SIZEOF},
|
|
{"alignof", CPP_TOKEN_ALIGNOF},
|
|
{"decltype", CPP_TOKEN_DECLTYPE},
|
|
{"throw", CPP_TOKEN_THROW},
|
|
{"new", CPP_TOKEN_NEW},
|
|
{"delete", CPP_TOKEN_DELETE},
|
|
{"xor", CPP_TOKEN_BIT_XOR},
|
|
{"xor_eq", CPP_TOKEN_XOREQ},
|
|
{"not", CPP_TOKEN_NOT},
|
|
{"not_eq", CPP_TOKEN_NOTEQ},
|
|
{"typeid", CPP_TOKEN_TYPEID},
|
|
{"compl", CPP_TOKEN_BIT_NOT},
|
|
|
|
{"void", CPP_TOKEN_KEY_TYPE},
|
|
{"bool", CPP_TOKEN_KEY_TYPE},
|
|
{"char", CPP_TOKEN_KEY_TYPE},
|
|
{"int", CPP_TOKEN_KEY_TYPE},
|
|
{"float", CPP_TOKEN_KEY_TYPE},
|
|
{"double", CPP_TOKEN_KEY_TYPE},
|
|
|
|
{"long", CPP_TOKEN_KEY_MODIFIER},
|
|
{"short", CPP_TOKEN_KEY_MODIFIER},
|
|
{"unsigned", CPP_TOKEN_KEY_MODIFIER},
|
|
|
|
{"const", CPP_TOKEN_KEY_QUALIFIER},
|
|
{"volatile", CPP_TOKEN_KEY_QUALIFIER},
|
|
|
|
{"asm", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"break", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"case", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"catch", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"continue", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"default", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"do", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"else", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"for", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"goto", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"if", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"return", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"switch", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"try", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"while", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
{"static_assert", CPP_TOKEN_KEY_CONTROL_FLOW},
|
|
|
|
{"const_cast", CPP_TOKEN_KEY_CAST},
|
|
{"dynamic_cast", CPP_TOKEN_KEY_CAST},
|
|
{"reinterpret_cast", CPP_TOKEN_KEY_CAST},
|
|
{"static_cast", CPP_TOKEN_KEY_CAST},
|
|
|
|
{"class", CPP_TOKEN_KEY_TYPE_DECLARATION},
|
|
{"enum", CPP_TOKEN_KEY_TYPE_DECLARATION},
|
|
{"struct", CPP_TOKEN_KEY_TYPE_DECLARATION},
|
|
{"typedef", CPP_TOKEN_KEY_TYPE_DECLARATION},
|
|
{"union", CPP_TOKEN_KEY_TYPE_DECLARATION},
|
|
{"template", CPP_TOKEN_KEY_TYPE_DECLARATION},
|
|
{"typename", CPP_TOKEN_KEY_TYPE_DECLARATION},
|
|
|
|
{"friend", CPP_TOKEN_KEY_ACCESS},
|
|
{"namespace", CPP_TOKEN_KEY_ACCESS},
|
|
{"private", CPP_TOKEN_KEY_ACCESS},
|
|
{"protected", CPP_TOKEN_KEY_ACCESS},
|
|
{"public", CPP_TOKEN_KEY_ACCESS},
|
|
{"using", CPP_TOKEN_KEY_ACCESS},
|
|
|
|
{"extern", CPP_TOKEN_KEY_LINKAGE},
|
|
{"export", CPP_TOKEN_KEY_LINKAGE},
|
|
{"inline", CPP_TOKEN_KEY_LINKAGE},
|
|
{"static", CPP_TOKEN_KEY_LINKAGE},
|
|
{"virtual", CPP_TOKEN_KEY_LINKAGE},
|
|
|
|
{"alignas", CPP_TOKEN_KEY_OTHER},
|
|
{"explicit", CPP_TOKEN_KEY_OTHER},
|
|
{"noexcept", CPP_TOKEN_KEY_OTHER},
|
|
{"nullptr", CPP_TOKEN_KEY_OTHER},
|
|
{"operator", CPP_TOKEN_KEY_OTHER},
|
|
{"register", CPP_TOKEN_KEY_OTHER},
|
|
{"this", CPP_TOKEN_KEY_OTHER},
|
|
{"thread_local", CPP_TOKEN_KEY_OTHER},
|
|
};
|
|
static String_List keywords = lexer_string_list(keyword_strings);
|
|
|
|
static String_And_Flag op_strings[] = {
|
|
{"...", CPP_TOKEN_ELLIPSIS},
|
|
{"<<=", CPP_TOKEN_LSHIFTEQ},
|
|
{">>=", CPP_TOKEN_RSHIFTEQ},
|
|
{"->*", CPP_TOKEN_PTRARROW},
|
|
{"<<", CPP_TOKEN_LSHIFT},
|
|
{">>", CPP_TOKEN_RSHIFT},
|
|
{"&&", CPP_TOKEN_AND},
|
|
{"||", CPP_TOKEN_OR},
|
|
{"->", CPP_TOKEN_ARROW},
|
|
{"++", CPP_TOKEN_INCREMENT},
|
|
{"--", CPP_TOKEN_DECREMENT},
|
|
{"::", CPP_TOKEN_SCOPE},
|
|
{"+=", CPP_TOKEN_ADDEQ},
|
|
{"-=", CPP_TOKEN_SUBEQ},
|
|
{"*=", CPP_TOKEN_MULEQ},
|
|
{"/=", CPP_TOKEN_DIVEQ},
|
|
{"%=", CPP_TOKEN_MODEQ},
|
|
{"&=", CPP_TOKEN_ANDEQ},
|
|
{"|=", CPP_TOKEN_OREQ},
|
|
{"^=", CPP_TOKEN_XOREQ},
|
|
{"==", CPP_TOKEN_EQEQ},
|
|
{">=", CPP_TOKEN_GRTREQ},
|
|
{"<=", CPP_TOKEN_LESSEQ},
|
|
{"!=", CPP_TOKEN_NOTEQ},
|
|
{".*", CPP_TOKEN_PTRDOT},
|
|
{"{", CPP_TOKEN_BRACE_OPEN},
|
|
{"}", CPP_TOKEN_BRACE_CLOSE},
|
|
{"[", CPP_TOKEN_BRACKET_OPEN},
|
|
{"]", CPP_TOKEN_BRACKET_CLOSE},
|
|
{"(", CPP_TOKEN_PARENTHESE_OPEN},
|
|
{")", CPP_TOKEN_PARENTHESE_CLOSE},
|
|
{"<", CPP_TOKEN_LESS},
|
|
{">", CPP_TOKEN_GRTR},
|
|
{"+", CPP_TOKEN_PLUS},
|
|
{"-", CPP_TOKEN_MINUS},
|
|
{"!", CPP_TOKEN_NOT},
|
|
{"~", CPP_TOKEN_TILDE},
|
|
{"*", CPP_TOKEN_STAR},
|
|
{"&", CPP_TOKEN_AMPERSAND},
|
|
{"|", CPP_TOKEN_BIT_OR},
|
|
{"^", CPP_TOKEN_BIT_XOR},
|
|
{"=", CPP_TOKEN_EQ},
|
|
{",", CPP_TOKEN_COMMA},
|
|
{":", CPP_TOKEN_COLON},
|
|
{";", CPP_TOKEN_SEMICOLON},
|
|
{"/", CPP_TOKEN_DIV},
|
|
{"?", CPP_TOKEN_TERNARY_QMARK},
|
|
{"%", CPP_TOKEN_MOD},
|
|
{".", CPP_TOKEN_DOT},
|
|
};
|
|
static String_List ops = lexer_string_list(op_strings);
|
|
|
|
static String_And_Flag pp_op_strings[] = {
|
|
{"##", CPP_PP_CONCAT},
|
|
{"#", CPP_PP_STRINGIFY},
|
|
};
|
|
static String_List pp_ops = lexer_string_list(pp_op_strings);
|
|
|
|
static String_And_Flag preprop_strings[] = {
|
|
{"include", CPP_PP_INCLUDE},
|
|
{"INCLUDE", CPP_PP_INCLUDE},
|
|
{"ifndef", CPP_PP_IFNDEF},
|
|
{"IFNDEF", CPP_PP_IFNDEF},
|
|
{"define", CPP_PP_DEFINE},
|
|
{"DEFINE", CPP_PP_DEFINE},
|
|
{"import", CPP_PP_IMPORT},
|
|
{"IMPORT", CPP_PP_IMPORT},
|
|
{"pragma", CPP_PP_PRAGMA},
|
|
{"PRAGMA", CPP_PP_PRAGMA},
|
|
{"undef", CPP_PP_UNDEF},
|
|
{"UNDEF", CPP_PP_UNDEF},
|
|
{"endif", CPP_PP_ENDIF},
|
|
{"ENDIF", CPP_PP_ENDIF},
|
|
{"error", CPP_PP_ERROR},
|
|
{"ERROR", CPP_PP_ERROR},
|
|
{"ifdef", CPP_PP_IFDEF},
|
|
{"IFDEF", CPP_PP_IFDEF},
|
|
{"using", CPP_PP_USING},
|
|
{"USING", CPP_PP_USING},
|
|
{"else", CPP_PP_ELSE},
|
|
{"ELSE", CPP_PP_ELSE},
|
|
{"elif", CPP_PP_ELIF},
|
|
{"ELIF", CPP_PP_ELIF},
|
|
{"line", CPP_PP_LINE},
|
|
{"LINE", CPP_PP_LINE},
|
|
{"if", CPP_PP_IF},
|
|
{"IF", CPP_PP_IF},
|
|
};
|
|
static String_List preprops = lexer_string_list(preprop_strings);
|
|
|
|
lexer_link Sub_Match_List_Result
|
|
sub_match_list(char *chunk, int size, int pos, String_List list, int sub_size){
|
|
Sub_Match_List_Result result;
|
|
String str_main;
|
|
char *str_check;
|
|
int i,l;
|
|
|
|
result.index = -1;
|
|
result.new_pos = pos;
|
|
str_main = make_string(chunk + pos, size - pos);
|
|
if (sub_size > 0){
|
|
str_main = substr(str_main, 0, sub_size);
|
|
for (i = 0; i < list.count; ++i){
|
|
str_check = list.data[i].str;
|
|
if (match(str_main, str_check)){
|
|
result.index = i;
|
|
result.new_pos = pos + sub_size;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
else{
|
|
for (i = 0; i < list.count; ++i){
|
|
str_check = list.data[i].str;
|
|
if (match_part(str_main, str_check, &l)){
|
|
result.index = i;
|
|
result.new_pos = pos + l;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
|
|
lexer_link Cpp_Get_Token_Result
|
|
cpp_get_token(Cpp_Token_Stack *token_stack, int pos){
|
|
Cpp_Get_Token_Result result = {};
|
|
Cpp_Token *token_array = token_stack->tokens;
|
|
Cpp_Token *token = 0;
|
|
int first = 0;
|
|
int count = token_stack->count;
|
|
int last = count;
|
|
int this_start = 0, next_start = 0;
|
|
|
|
if (count > 0){
|
|
for (;;){
|
|
result.token_index = (first + last)/2;
|
|
token = token_array + result.token_index;
|
|
|
|
this_start = token->start;
|
|
|
|
if (result.token_index + 1 < count){
|
|
next_start = (token + 1)->start;
|
|
}
|
|
else{
|
|
next_start = this_start + token->size;
|
|
}
|
|
if (this_start <= pos && pos < next_start){
|
|
break;
|
|
}
|
|
else if (pos < this_start){
|
|
last = result.token_index;
|
|
}
|
|
else{
|
|
first = result.token_index + 1;
|
|
}
|
|
if (first == last){
|
|
result.token_index = first;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (result.token_index == count){
|
|
--result.token_index;
|
|
result.in_whitespace = 1;
|
|
}
|
|
else{
|
|
if (token->start + token->size <= pos){
|
|
result.in_whitespace = 1;
|
|
}
|
|
}
|
|
}
|
|
else{
|
|
result.token_index = -1;
|
|
result.in_whitespace = 1;
|
|
}
|
|
|
|
return(result);
|
|
}
|
|
|
|
lexer_link void
|
|
cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token_i, int shift_amount){
|
|
Cpp_Token *token = stack->tokens + from_token_i;
|
|
int count = stack->count, i;
|
|
|
|
for (i = from_token_i; i < count; ++i, ++token){
|
|
token->start += shift_amount;
|
|
}
|
|
}
|
|
|
|
enum Lex_State{
|
|
LS_default,
|
|
LS_identifier,
|
|
LS_pound,
|
|
LS_pp,
|
|
LS_char,
|
|
LS_char_slashed,
|
|
LS_string,
|
|
LS_string_slashed,
|
|
LS_number,
|
|
LS_number0,
|
|
LS_float,
|
|
LS_crazy_float0,
|
|
LS_crazy_float1,
|
|
LS_hex,
|
|
LS_comment_pre,
|
|
LS_comment,
|
|
LS_comment_slashed,
|
|
LS_comment_block,
|
|
LS_comment_block_ending,
|
|
LS_dot,
|
|
LS_ellipsis,
|
|
LS_less,
|
|
LS_less_less,
|
|
LS_more,
|
|
LS_more_more,
|
|
LS_minus,
|
|
LS_arrow,
|
|
LS_and,
|
|
LS_or,
|
|
LS_plus,
|
|
LS_colon,
|
|
LS_star,
|
|
LS_modulo,
|
|
LS_caret,
|
|
LS_eq,
|
|
LS_bang,
|
|
LS_error_message,
|
|
//
|
|
LS_count
|
|
};
|
|
|
|
enum Lex_Int_State{
|
|
LSINT_default,
|
|
LSINT_u,
|
|
LSINT_l,
|
|
LSINT_L,
|
|
LSINT_ul,
|
|
LSINT_uL,
|
|
LSINT_ll,
|
|
LSINT_extra
|
|
};
|
|
|
|
enum Lex_INC_State{
|
|
LSINC_default,
|
|
LSINC_quotes,
|
|
LSINC_pointy,
|
|
LSINC_junk,
|
|
};
|
|
|
|
enum Lex_PP_State{
|
|
LSPP_default,
|
|
LSPP_include,
|
|
LSPP_macro_identifier,
|
|
LSPP_identifier,
|
|
LSPP_body_if,
|
|
LSPP_body,
|
|
LSPP_number,
|
|
LSPP_error,
|
|
LSPP_junk,
|
|
//
|
|
LSPP_count
|
|
};
|
|
|
|
enum Pos_Update_Rule{
|
|
PUR_none,
|
|
PUR_unget_whitespace
|
|
};
|
|
|
|
lexer_link Lex_PP_State
|
|
cpp_pp_directive_to_state(Cpp_Token_Type type){
|
|
Lex_PP_State result = LSPP_default;
|
|
switch (type){
|
|
case CPP_PP_INCLUDE:
|
|
case CPP_PP_IMPORT:
|
|
case CPP_PP_USING:
|
|
result = LSPP_include;
|
|
break;
|
|
|
|
case CPP_PP_DEFINE:
|
|
result = LSPP_macro_identifier;
|
|
break;
|
|
|
|
case CPP_PP_UNDEF:
|
|
case CPP_PP_IFDEF:
|
|
case CPP_PP_IFNDEF:
|
|
result = LSPP_identifier;
|
|
break;
|
|
|
|
case CPP_PP_IF:
|
|
case CPP_PP_ELIF:
|
|
result = LSPP_body_if;
|
|
break;
|
|
|
|
case CPP_PP_PRAGMA:
|
|
result = LSPP_body;
|
|
break;
|
|
|
|
case CPP_PP_LINE:
|
|
result = LSPP_number;
|
|
break;
|
|
|
|
case CPP_PP_ERROR:
|
|
result = LSPP_error;
|
|
break;
|
|
|
|
case CPP_PP_UNKNOWN:
|
|
case CPP_PP_ELSE:
|
|
case CPP_PP_ENDIF:
|
|
result = LSPP_junk;
|
|
break;
|
|
}
|
|
return(result);
|
|
}
|
|
|
|
lexer_link Cpp_Token_Merge
|
|
cpp_attempt_token_merge(Cpp_Token prev_token, Cpp_Token next_token){
|
|
Cpp_Token_Merge result = {(Cpp_Token_Type)0};
|
|
if (next_token.type == CPP_TOKEN_COMMENT && prev_token.type == CPP_TOKEN_COMMENT &&
|
|
next_token.flags == prev_token.flags && next_token.state_flags == prev_token.state_flags){
|
|
result.did_merge = 1;
|
|
prev_token.size = next_token.start + next_token.size - prev_token.start;
|
|
result.new_token = prev_token;
|
|
}
|
|
else if (next_token.type == CPP_TOKEN_JUNK && prev_token.type == CPP_TOKEN_JUNK &&
|
|
next_token.flags == prev_token.flags && next_token.state_flags == prev_token.state_flags){
|
|
result.did_merge = 1;
|
|
prev_token.size = next_token.start + next_token.size - prev_token.start;
|
|
result.new_token = prev_token;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
lexer_link void
|
|
cpp_push_token_nonalloc(Cpp_Token *out_tokens, int *token_i, Cpp_Token token){
|
|
Cpp_Token_Merge merge = {(Cpp_Token_Type)0};
|
|
Cpp_Token prev_token = {(Cpp_Token_Type)0};
|
|
|
|
if (*token_i > 0){
|
|
prev_token = out_tokens[*token_i - 1];
|
|
merge = new_lex::cpp_attempt_token_merge(prev_token, token);
|
|
if (merge.did_merge){
|
|
out_tokens[*token_i - 1] = merge.new_token;
|
|
}
|
|
}
|
|
|
|
if (!merge.did_merge){
|
|
out_tokens[(*token_i)++] = token;
|
|
}
|
|
}
|
|
|
|
struct Whitespace_FSM{
|
|
unsigned char pp_state;
|
|
unsigned char white_done;
|
|
};
|
|
|
|
struct Lex_FSM{
|
|
unsigned char state;
|
|
unsigned char int_state;
|
|
unsigned char emit_token;
|
|
unsigned char multi_line;
|
|
unsigned char completed;
|
|
};
|
|
|
|
struct Lex_Data{
|
|
char *tb;
|
|
int tb_pos;
|
|
int token_start;
|
|
|
|
int pos;
|
|
int pos_overide;
|
|
|
|
Lex_FSM fsm;
|
|
Whitespace_FSM wfsm;
|
|
unsigned char pp_state;
|
|
unsigned char completed;
|
|
|
|
Cpp_Token token;
|
|
|
|
int __pc__;
|
|
};
|
|
|
|
Whitespace_FSM
|
|
whitespace_skip_fsm(Whitespace_FSM wfsm, char c){
|
|
if (wfsm.pp_state != LSPP_default){
|
|
if (c == '\n') wfsm.pp_state = LSPP_default;
|
|
}
|
|
if (!(c == ' ' || c == '\n' || c == '\t' || c == '\r' || c == '\f' || c == '\v')){
|
|
wfsm.white_done = 1;
|
|
}
|
|
return(wfsm);
|
|
}
|
|
|
|
Lex_FSM
|
|
int_fsm(Lex_FSM fsm, char c){
|
|
switch (fsm.int_state){
|
|
case LSINT_default:
|
|
switch (c){
|
|
case 'u': case 'U': fsm.int_state = LSINT_u; break;
|
|
case 'l': fsm.int_state = LSINT_l; break;
|
|
case 'L': fsm.int_state = LSINT_L; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LSINT_u:
|
|
switch (c){
|
|
case 'l': fsm.int_state = LSINT_ul; break;
|
|
case 'L': fsm.int_state = LSINT_uL; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LSINT_l:
|
|
switch (c){
|
|
case 'l': fsm.int_state = LSINT_ll; break;
|
|
case 'U': case 'u': fsm.int_state = LSINT_extra; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LSINT_L:
|
|
switch (c){
|
|
case 'L': fsm.int_state = LSINT_ll; break;
|
|
case 'U': case 'u': fsm.int_state = LSINT_extra; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LSINT_ul:
|
|
switch (c){
|
|
case 'l': fsm.int_state = LSINT_extra; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LSINT_uL:
|
|
switch (c){
|
|
case 'L': fsm.int_state = LSINT_extra; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LSINT_ll:
|
|
switch (c){
|
|
case 'u': case 'U': fsm.int_state = LSINT_extra; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LSINT_extra:
|
|
fsm.emit_token = 1;
|
|
break;
|
|
}
|
|
return(fsm);
|
|
}
|
|
|
|
Lex_FSM
|
|
main_fsm(Lex_FSM fsm, unsigned char pp_state, char c){
|
|
if (c == 0) fsm.emit_token = 1;
|
|
else
|
|
switch (pp_state){
|
|
case LSPP_error:
|
|
fsm.state = LS_error_message;
|
|
if (c == '\n') fsm.emit_token = 1;
|
|
break;
|
|
|
|
case LSPP_include:
|
|
switch (fsm.state){
|
|
case LSINC_default:
|
|
switch (c){
|
|
case '"': fsm.state = LSINC_quotes; break;
|
|
case '<': fsm.state = LSINC_pointy; break;
|
|
default: fsm.state = LSINC_junk; break;
|
|
}
|
|
break;
|
|
|
|
case LSINC_quotes:
|
|
if (c == '"') fsm.emit_token = 1;
|
|
break;
|
|
|
|
case LSINC_pointy:
|
|
if (c == '>') fsm.emit_token = 1;
|
|
break;
|
|
|
|
case LSINC_junk:
|
|
if (c == '\n') fsm.emit_token = 1;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
default:
|
|
switch (fsm.state){
|
|
case LS_default:
|
|
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'){
|
|
fsm.state = LS_identifier;
|
|
}
|
|
else if (c >= '1' && c <= '9'){
|
|
fsm.state = LS_number;
|
|
}
|
|
else if (c == '0'){
|
|
fsm.state = LS_number0;
|
|
}
|
|
else switch (c){
|
|
case '\'': fsm.state = LS_char; break;
|
|
case '"': fsm.state = LS_string; break;
|
|
|
|
case '/': fsm.state = LS_comment_pre; break;
|
|
|
|
case '.': fsm.state = LS_dot; break;
|
|
|
|
case '<': fsm.state = LS_less; break;
|
|
case '>': fsm.state = LS_more; break;
|
|
|
|
case '-': fsm.state = LS_minus; break;
|
|
|
|
case '&': fsm.state = LS_and; break;
|
|
case '|': fsm.state = LS_or; break;
|
|
|
|
case '+': fsm.state = LS_plus; break;
|
|
|
|
case ':': fsm.state = LS_colon; break;
|
|
|
|
case '*': fsm.state = LS_star; break;
|
|
|
|
case '%': fsm.state = LS_modulo; break;
|
|
case '^': fsm.state = LS_caret; break;
|
|
|
|
case '=': fsm.state = LS_eq; break;
|
|
case '!': fsm.state = LS_bang; break;
|
|
|
|
case '#': fsm.state = LS_pound; break;
|
|
|
|
#define OperCase(op,type) case op: fsm.emit_token = 1; break;
|
|
OperCase('{', CPP_TOKEN_BRACE_OPEN);
|
|
OperCase('}', CPP_TOKEN_BRACE_CLOSE);
|
|
|
|
OperCase('[', CPP_TOKEN_BRACKET_OPEN);
|
|
OperCase(']', CPP_TOKEN_BRACKET_CLOSE);
|
|
|
|
OperCase('(', CPP_TOKEN_PARENTHESE_OPEN);
|
|
OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE);
|
|
|
|
OperCase('~', CPP_TOKEN_TILDE);
|
|
OperCase(',', CPP_TOKEN_COMMA);
|
|
OperCase(';', CPP_TOKEN_SEMICOLON);
|
|
OperCase('?', CPP_TOKEN_TERNARY_QMARK);
|
|
|
|
OperCase('@', CPP_TOKEN_JUNK);
|
|
OperCase('$', CPP_TOKEN_JUNK);
|
|
OperCase('\\', CPP_TOKEN_JUNK);
|
|
#undef OperCase
|
|
}
|
|
break;
|
|
|
|
case LS_identifier:
|
|
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
|
|
fsm.emit_token = 1;
|
|
}
|
|
break;
|
|
|
|
case LS_pound:
|
|
if (pp_state == LSPP_default){
|
|
if (c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v'){
|
|
fsm.state = LS_pound;
|
|
}
|
|
else if (c == '\n'){
|
|
fsm.emit_token = 1;
|
|
}
|
|
else{
|
|
fsm.state = LS_pp;
|
|
}
|
|
}
|
|
else{
|
|
switch (c){
|
|
case '#': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case LS_pp:
|
|
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
|
|
fsm.emit_token = 1;
|
|
}
|
|
break;
|
|
|
|
case LS_char:
|
|
switch(c){
|
|
case '\'': fsm.emit_token = 1; break;
|
|
case '\\': fsm.state = LS_char_slashed; break;
|
|
}
|
|
break;
|
|
|
|
case LS_char_slashed:
|
|
switch (c){
|
|
case '\r': case '\f': case '\v': break;
|
|
case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
|
|
default: fsm.state = LS_char; break;
|
|
}
|
|
break;
|
|
|
|
case LS_string:
|
|
switch(c){
|
|
case '\"': fsm.emit_token = 1; break;
|
|
case '\\': fsm.state = LS_string_slashed; break;
|
|
}
|
|
break;
|
|
|
|
case LS_string_slashed:
|
|
switch (c){
|
|
case '\r': case '\f': case '\v': break;
|
|
case '\n': fsm.state = LS_string; fsm.multi_line |= 1; break;
|
|
default: fsm.state = LS_string; break;
|
|
}
|
|
break;
|
|
|
|
case LS_number:
|
|
if (c >= '0' && c <= '9'){
|
|
fsm.state = LS_number;
|
|
}
|
|
else{
|
|
switch (c){
|
|
case '.': fsm.state = LS_float; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case LS_number0:
|
|
if (c >= '0' && c <= '9'){
|
|
fsm.state = LS_number;
|
|
}
|
|
else if (c == 'x'){
|
|
fsm.state = LS_hex;
|
|
}
|
|
else if (c == '.'){
|
|
fsm.state = LS_float;
|
|
}
|
|
else{
|
|
fsm.emit_token = 1;
|
|
}
|
|
break;
|
|
|
|
case LS_float:
|
|
if (!(c >= '0' && c <= '9')){
|
|
switch (c){
|
|
case 'e': fsm.state = LS_crazy_float0; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case LS_crazy_float0:
|
|
{
|
|
if ((c >= '0' && c <= '9') || c == '-'){
|
|
fsm.state = LS_crazy_float1;
|
|
}
|
|
else{
|
|
fsm.emit_token = 1;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case LS_crazy_float1:
|
|
{
|
|
if (!(c >= '0' && c <= '9')){
|
|
fsm.emit_token = 1;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case LS_hex:
|
|
if (!(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F')){
|
|
fsm.emit_token = 1;
|
|
}
|
|
break;
|
|
|
|
case LS_dot:
|
|
if (c >= '0' && c <= '9'){
|
|
fsm.state = LS_float;
|
|
}
|
|
else
|
|
switch (c){
|
|
case '.': fsm.state = LS_ellipsis; break;
|
|
case '*': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_ellipsis: fsm.emit_token = 1; break;
|
|
|
|
case LS_less:
|
|
switch (c){
|
|
case '<': fsm.state = LS_less_less; break;
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_less_less:
|
|
switch (c){
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_more:
|
|
switch (c){
|
|
case '>': fsm.state = LS_more_more; break;
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_more_more:
|
|
switch (c){
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_comment_pre:
|
|
switch (c){
|
|
case '/': fsm.state = LS_comment; break;
|
|
case '*': fsm.state = LS_comment_block; break;
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_comment:
|
|
switch (c){
|
|
case '\\': fsm.state = LS_comment_slashed; break;
|
|
case '\n': fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_comment_slashed:
|
|
switch (c){
|
|
case '\r': case '\f': case '\v': break;
|
|
default: fsm.state = LS_comment; break;
|
|
}
|
|
break;
|
|
|
|
case LS_comment_block:
|
|
switch (c){
|
|
case '*': fsm.state = LS_comment_block_ending; break;
|
|
}
|
|
break;
|
|
|
|
case LS_comment_block_ending:
|
|
switch (c){
|
|
case '*': fsm.state = LS_comment_block_ending; break;
|
|
case '/': fsm.emit_token = 1; break;
|
|
default: fsm.state = LS_comment_block; break;
|
|
}
|
|
break;
|
|
|
|
case LS_minus:
|
|
switch (c){
|
|
case '>': fsm.state = LS_arrow; break;
|
|
case '-': fsm.emit_token = 1; break;
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_arrow:
|
|
switch (c){
|
|
case '*': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_and:
|
|
switch (c){
|
|
case '&': fsm.emit_token = 1; break;
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_or:
|
|
switch (c){
|
|
case '|': fsm.emit_token = 1; break;
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_plus:
|
|
switch (c){
|
|
case '+': fsm.emit_token = 1; break;
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_colon:
|
|
switch (c){
|
|
case ':': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_star:
|
|
switch (c){
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_modulo:
|
|
switch (c){
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_caret:
|
|
switch (c){
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_eq:
|
|
switch (c){
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
|
|
case LS_bang:
|
|
switch (c){
|
|
case '=': fsm.emit_token = 1; break;
|
|
default: fsm.emit_token = 1; break;
|
|
}
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
return(fsm);
|
|
}
|
|
|
|
#define DrCase(PC) case PC: goto resumespot_##PC
|
|
|
|
#define DrYield(PC, n) {\
|
|
token_stack_out->count = token_i;\
|
|
*S_ptr = S; S_ptr->__pc__ = PC; return(n); resumespot_##PC:; }
|
|
|
|
#define DrReturn(n) {\
|
|
token_stack_out->count = token_i;\
|
|
*S_ptr = S; S_ptr->__pc__ = -1; return(n); }
|
|
|
|
lexer_link int
|
|
cpp_lex_nonalloc(Lex_Data *S_ptr, char *chunk, int size, Cpp_Token_Stack *token_stack_out){
|
|
Lex_Data S = *S_ptr;
|
|
|
|
Cpp_Token *out_tokens = token_stack_out->tokens;
|
|
int token_i = token_stack_out->count;
|
|
int max_token_i = token_stack_out->max_count;
|
|
|
|
Lex_FSM fsm = {0};
|
|
Pos_Update_Rule pos_update_rule = PUR_none;
|
|
|
|
char c = 0;
|
|
|
|
int end_pos = size + S.pos;
|
|
chunk -= S.pos;
|
|
|
|
switch (S.__pc__){
|
|
DrCase(1);
|
|
DrCase(2);
|
|
DrCase(3);
|
|
DrCase(4);
|
|
DrCase(5);
|
|
}
|
|
|
|
for (;;){
|
|
S.wfsm.white_done = 0;
|
|
S.wfsm.pp_state = S.pp_state;
|
|
for(;;){
|
|
for (; S.wfsm.white_done == 0 && S.pos < end_pos;){
|
|
c = chunk[S.pos++];
|
|
S.wfsm = whitespace_skip_fsm(S.wfsm, c);
|
|
}
|
|
if (S.wfsm.white_done == 0){
|
|
DrYield(4, 1);
|
|
}
|
|
else break;
|
|
}
|
|
--S.pos;
|
|
S.pp_state = S.wfsm.pp_state;
|
|
|
|
S.token_start = S.pos;
|
|
S.tb_pos = 0;
|
|
S.fsm = {0};
|
|
for(;;){
|
|
for (; S.fsm.emit_token == 0 && S.pos < end_pos;){
|
|
c = chunk[S.pos++];
|
|
S.tb[S.tb_pos++] = c;
|
|
|
|
fsm = S.fsm;
|
|
fsm = main_fsm(fsm, S.pp_state, c);
|
|
S.fsm = fsm;
|
|
}
|
|
if (S.fsm.emit_token == 0){
|
|
DrYield(3, 1);
|
|
}
|
|
else break;
|
|
}
|
|
|
|
Assert(S.fsm.emit_token == 1);
|
|
|
|
if (c != 0){
|
|
pos_update_rule = PUR_none;
|
|
if (S.pp_state == LSPP_include){
|
|
switch (S.fsm.state){
|
|
case LSINC_default:break;
|
|
|
|
case LSINC_quotes:
|
|
case LSINC_pointy:
|
|
S.token.type = CPP_TOKEN_INCLUDE_FILE;
|
|
S.token.flags = 0;
|
|
break;
|
|
|
|
case LSINC_junk:
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
S.token.flags = 0;
|
|
break;
|
|
}
|
|
}
|
|
else switch (S.fsm.state){
|
|
case LS_default:
|
|
switch (c){
|
|
#define OperCase(op,t) case op: S.token.type = t; break;
|
|
OperCase('{', CPP_TOKEN_BRACE_OPEN);
|
|
OperCase('}', CPP_TOKEN_BRACE_CLOSE);
|
|
|
|
OperCase('[', CPP_TOKEN_BRACKET_OPEN);
|
|
OperCase(']', CPP_TOKEN_BRACKET_CLOSE);
|
|
|
|
OperCase('(', CPP_TOKEN_PARENTHESE_OPEN);
|
|
OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE);
|
|
|
|
OperCase('~', CPP_TOKEN_TILDE);
|
|
OperCase(',', CPP_TOKEN_COMMA);
|
|
OperCase(';', CPP_TOKEN_SEMICOLON);
|
|
OperCase('?', CPP_TOKEN_TERNARY_QMARK);
|
|
|
|
OperCase('@', CPP_TOKEN_JUNK);
|
|
OperCase('$', CPP_TOKEN_JUNK);
|
|
#undef OperCase
|
|
|
|
case '\\':
|
|
if (S.pp_state == LSPP_default){
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
}
|
|
else{
|
|
S.pos_overide = S.pos;
|
|
S.wfsm.white_done = 0;
|
|
for (;;){
|
|
for (; S.wfsm.white_done == 0 && S.pos < end_pos;){
|
|
c = chunk[S.pos++];
|
|
if (!(c == ' ' || c == '\t' || c == '\r' || c == '\v' || c == '\f')) S.wfsm.white_done = 1;
|
|
}
|
|
|
|
if (S.wfsm.white_done == 0){
|
|
DrYield(1, 1);
|
|
}
|
|
else break;
|
|
}
|
|
|
|
if (c == '\n'){
|
|
S.fsm.emit_token = 0;
|
|
S.pos_overide = 0;
|
|
}
|
|
else{
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
if (c != '@' && c != '$' && c != '\\'){
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
}
|
|
break;
|
|
|
|
case LS_identifier:
|
|
{
|
|
--S.pos;
|
|
|
|
int word_size = S.pos - S.token_start;
|
|
|
|
if (S.pp_state == LSPP_body_if){
|
|
if (match(make_string(S.tb, word_size), make_lit_string("defined"))){
|
|
S.token.type = CPP_TOKEN_DEFINED;
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR | CPP_TFLAG_IS_KEYWORD;
|
|
break;
|
|
}
|
|
}
|
|
|
|
Sub_Match_List_Result sub_match;
|
|
sub_match = sub_match_list(S.tb, S.tb_pos, 0, bool_lits, word_size);
|
|
|
|
if (sub_match.index != -1){
|
|
S.token.type = CPP_TOKEN_BOOLEAN_CONSTANT;
|
|
S.token.flags = CPP_TFLAG_IS_KEYWORD;
|
|
}
|
|
else{
|
|
sub_match = sub_match_list(S.tb, S.tb_pos, 0, keywords, word_size);
|
|
|
|
if (sub_match.index != -1){
|
|
String_And_Flag data = keywords.data[sub_match.index];
|
|
S.token.type = (Cpp_Token_Type)data.flags;
|
|
S.token.flags = CPP_TFLAG_IS_KEYWORD;
|
|
}
|
|
else{
|
|
S.token.type = CPP_TOKEN_IDENTIFIER;
|
|
S.token.flags = 0;
|
|
}
|
|
}
|
|
}break;
|
|
|
|
case LS_pound:
|
|
S.token.flags = 0;
|
|
switch (c){
|
|
case '#': S.token.type = CPP_PP_CONCAT; break;
|
|
default:
|
|
S.token.type = CPP_PP_STRINGIFY;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_pp:
|
|
{
|
|
--S.pos;
|
|
int start = 1;
|
|
|
|
c = S.tb[start];
|
|
while (start < S.tb_pos && (c == ' ' || c == '\t' || c == '\r' || c == '\v' || c == '\f')){
|
|
++start;
|
|
c = S.tb[start];
|
|
}
|
|
|
|
int word_size = S.tb_pos - start - 1;
|
|
Sub_Match_List_Result match;
|
|
match = sub_match_list(S.tb, S.tb_pos, start, preprops, word_size);
|
|
|
|
if (match.index != -1){
|
|
String_And_Flag data = preprops.data[match.index];
|
|
S.token.type = (Cpp_Token_Type)data.flags;
|
|
S.token.flags = CPP_TFLAG_PP_DIRECTIVE;
|
|
S.pp_state = (unsigned char)cpp_pp_directive_to_state(S.token.type);
|
|
}
|
|
else{
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
S.token.flags = 0;
|
|
}
|
|
}break;
|
|
|
|
case LS_number:
|
|
case LS_number0:
|
|
case LS_hex:
|
|
S.fsm.int_state = LSINT_default;
|
|
|
|
{
|
|
S.fsm.emit_token = 0;
|
|
--S.pos;
|
|
for (;;){
|
|
for (; S.fsm.emit_token == 0 && S.pos < end_pos;){
|
|
c = chunk[S.pos++];
|
|
S.fsm = int_fsm(S.fsm, c);
|
|
}
|
|
if (S.fsm.emit_token == 0){
|
|
DrYield(5, 1);
|
|
}
|
|
else break;
|
|
}
|
|
--S.pos;
|
|
}
|
|
|
|
S.token.type = CPP_TOKEN_INTEGER_CONSTANT;
|
|
S.token.flags = 0;
|
|
break;
|
|
|
|
case LS_float:
|
|
case LS_crazy_float0:
|
|
case LS_crazy_float1:
|
|
S.token.type = CPP_TOKEN_FLOATING_CONSTANT;
|
|
S.token.flags = 0;
|
|
switch (c){
|
|
case 'f': case 'F':
|
|
case 'l': case 'L':break;
|
|
default: --S.pos; break;
|
|
}
|
|
break;
|
|
|
|
case LS_char:
|
|
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
|
|
S.token.flags = 0;
|
|
break;
|
|
|
|
case LS_string:
|
|
S.token.type = CPP_TOKEN_STRING_CONSTANT;
|
|
S.token.flags = 0;
|
|
break;
|
|
|
|
case LS_comment_pre:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_DIVEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_DIV;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_comment: case LS_comment_block_ending:
|
|
S.token.type = CPP_TOKEN_COMMENT;
|
|
S.token.flags = 0;
|
|
pos_update_rule = PUR_unget_whitespace;
|
|
break;
|
|
|
|
case LS_error_message:
|
|
S.token.type = CPP_TOKEN_ERROR_MESSAGE;
|
|
S.token.flags = 0;
|
|
pos_update_rule = PUR_unget_whitespace;
|
|
break;
|
|
|
|
case LS_dot:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '*': S.token.type = CPP_TOKEN_PTRDOT; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_DOT;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_ellipsis:
|
|
switch (c){
|
|
case '.':
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
S.token.type = CPP_TOKEN_ELLIPSIS;
|
|
break;
|
|
|
|
default:
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_less:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_LESSEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_LESS;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_less_less:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_LSHIFTEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_LSHIFT;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_more:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_GRTREQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_GRTR;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_more_more:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_RSHIFTEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_RSHIFT;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_minus:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '-': S.token.type = CPP_TOKEN_DECREMENT; break;
|
|
case '=': S.token.type = CPP_TOKEN_SUBEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_MINUS;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_arrow:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '*': S.token.type = CPP_TOKEN_PTRARROW; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_ARROW;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_and:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '&': S.token.type = CPP_TOKEN_AND; break;
|
|
case '=': S.token.type = CPP_TOKEN_ANDEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_AMPERSAND;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_or:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '|': S.token.type = CPP_TOKEN_OR; break;
|
|
case '=': S.token.type = CPP_TOKEN_OREQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_BIT_OR;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_plus:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '+': S.token.type = CPP_TOKEN_INCREMENT; break;
|
|
case '=': S.token.type = CPP_TOKEN_ADDEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_PLUS;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_colon:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case ':': S.token.type = CPP_TOKEN_SCOPE; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_COLON;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_star:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_MULEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_STAR;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_modulo:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_MODEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_MOD;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_caret:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_XOREQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_BIT_XOR;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_eq:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_EQEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_EQ;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
|
|
case LS_bang:
|
|
S.token.flags = CPP_TFLAG_IS_OPERATOR;
|
|
switch (c){
|
|
case '=': S.token.type = CPP_TOKEN_NOTEQ; break;
|
|
default:
|
|
S.token.type = CPP_TOKEN_NOT;
|
|
--S.pos;
|
|
break;
|
|
}
|
|
break;
|
|
}
|
|
|
|
switch (pos_update_rule){
|
|
case PUR_unget_whitespace:
|
|
c = chunk[--S.pos];
|
|
while (c == ' ' || c == '\n' || c == '\t' || c == '\r' || c == '\v' || c == '\f'){
|
|
c = chunk[--S.pos];
|
|
}
|
|
++S.pos;
|
|
break;
|
|
}
|
|
|
|
if ((S.token.flags & CPP_TFLAG_PP_DIRECTIVE) == 0){
|
|
switch (S.pp_state){
|
|
case LSPP_include:
|
|
if (S.token.type != CPP_TOKEN_INCLUDE_FILE){
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
}
|
|
S.pp_state = LSPP_junk;
|
|
break;
|
|
|
|
case LSPP_macro_identifier:
|
|
if (S.fsm.state != LS_identifier){
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
S.pp_state = LSPP_junk;
|
|
}
|
|
else{
|
|
S.pp_state = LSPP_body;
|
|
}
|
|
break;
|
|
|
|
case LSPP_identifier:
|
|
if (S.fsm.state != LS_identifier){
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
}
|
|
S.pp_state = LSPP_junk;
|
|
break;
|
|
|
|
case LSPP_number:
|
|
if (S.token.type != CPP_TOKEN_INTEGER_CONSTANT){
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
S.pp_state = LSPP_junk;
|
|
}
|
|
else{
|
|
S.pp_state = LSPP_include;
|
|
}
|
|
break;
|
|
|
|
case LSPP_junk:
|
|
S.token.type = CPP_TOKEN_JUNK;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (S.fsm.emit_token){
|
|
S.token.start = S.token_start;
|
|
if (S.pos_overide){
|
|
S.token.size = S.pos_overide - S.token_start;
|
|
S.pos_overide = 0;
|
|
}
|
|
else{
|
|
S.token.size = S.pos - S.token_start;
|
|
}
|
|
S.token.flags |= (S.fsm.multi_line)?(CPP_TFLAG_MULTILINE):(0);
|
|
if ((S.token.flags & CPP_TFLAG_PP_DIRECTIVE) == 0){
|
|
S.token.flags |= (S.pp_state != LSPP_default)?(CPP_TFLAG_PP_BODY):(0);
|
|
}
|
|
S.token.state_flags = S.pp_state;
|
|
|
|
cpp_push_token_nonalloc(out_tokens, &token_i, S.token);
|
|
if (token_i == max_token_i){
|
|
DrYield(2, 2);
|
|
}
|
|
}
|
|
}
|
|
// NOTE(allen): else case for "if (c != 0) {...}
|
|
else{
|
|
S.completed = 1;
|
|
break;
|
|
}
|
|
}
|
|
|
|
DrReturn(0);
|
|
}
|
|
|
|
#undef DrYield
|
|
#undef DrReturn
|
|
#undef DrCase
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
// BOTTOM
|