ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

eliminate the Cpp_File that drove me crazy

This commit is contained in:
Allen Webster 2016-07-16 14:40:26 -04:00
parent 89b6fe663d
commit 7ec039edfd
9 changed files with 330 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
4coder_default_include.cpp
View File

@ -1819,11 +1819,11 @@ CUSTOM_COMMAND_SIG(open_all_code){
// was originally, so that new appends overwrite old ones.
dir.size = dir_size;
append(&dir, info->filename);
exec_command(app, cmdid_interactive_open);
app->create_buffer(app, dir.str, dir.size, 0);
}
}
}
app->free_file_list(app, list);
}

404
4cpp_lexer.h
View File

@ -78,14 +78,6 @@ struct Cpp_Read_Result{
char has_result;
};
Cpp_File
data_as_cpp_file(Data data){
Cpp_File result;
result.data = (char*)data.data;
result.size = data.size;
return(result);
}
// TODO(allen): revisit this keyword data declaration system
struct String_And_Flag{
char *str;
@ -125,27 +117,27 @@ is_keyword(Cpp_Token_Type type){
return (type >= CPP_TOKEN_KEY_TYPE && type <= CPP_TOKEN_KEY_OTHER);
}
FCPP_LINK Sub_Match_List_Result sub_match_list(Cpp_File file, int pos, String_List list, int sub_size);
FCPP_LINK Sub_Match_List_Result sub_match_list(char *data, int size, int pos, String_List list, int sub_size);
FCPP_LINK Seek_Result seek_unescaped_eol(char *data, int size, int pos);
FCPP_LINK Seek_Result seek_unescaped_delim(char *data, int size, int pos, char delim);
FCPP_LINK Seek_Result seek_block_comment_end(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_whitespace(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_junk_line(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_operator(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_operator(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body);
inline Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos) { return cpp_read_alpha_numeric(file, pos, 0); }
FCPP_LINK Cpp_Read_Result cpp_read_number(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_string_litteral(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_character_litteral(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_line_comment(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_block_comment(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_preprocessor(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_include_file(Cpp_File file, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body);
inline Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos) { return cpp_read_pp_default_mode(file, pos, 0); }
FCPP_LINK Cpp_Read_Result cpp_read_whitespace(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_junk_line(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_operator(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_operator(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_alpha_numeric(char *data, int size, int pos, bool in_if_body);
inline Cpp_Read_Result cpp_read_alpha_numeric(char *data, int size, int pos) { return cpp_read_alpha_numeric(data, size, pos, 0); }
FCPP_LINK Cpp_Read_Result cpp_read_number(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_string_litteral(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_character_litteral(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_line_comment(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_block_comment(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_preprocessor(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_include_file(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_default_mode(char *data, int size, int pos, bool in_if_body);
inline Cpp_Read_Result cpp_read_pp_default_mode(char *data, int size, int pos) { return cpp_read_pp_default_mode(data, size, pos, 0); }
FCPP_LINK Cpp_Token_Merge cpp_attempt_token_merge(Cpp_Token prev, Cpp_Token next);
@ -154,18 +146,18 @@ FCPP_LINK bool cpp_push_token_nonalloc(Cpp_Token_Stack *stack, Cpp_Token token);
inline Cpp_Lex_Data cpp_lex_data_zero() { Cpp_Lex_Data data = {(Cpp_Preprocessor_State)0}; return(data); }
FCPP_LINK Cpp_Read_Result cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex);
FCPP_LINK Cpp_Read_Result cpp_lex_step(char *data, int size, Cpp_Lex_Data *lex);
FCPP_LINK int cpp_lex_file_token_count(Cpp_File file);
FCPP_LINK Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack, Cpp_Lex_Data data);
inline Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack) { return cpp_lex_file_nonalloc(file, stack, cpp_lex_data_zero()); }
FCPP_LINK int cpp_lex_file_token_count(char *data, int size);
FCPP_LINK Cpp_Lex_Data cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *stack, Cpp_Lex_Data lex_data);
inline Cpp_Lex_Data cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *stack) { return cpp_lex_file_nonalloc(data, size, stack, cpp_lex_data_zero()); }
FCPP_LINK Cpp_Get_Token_Result cpp_get_token(Cpp_Token_Stack *stack, int pos);
FCPP_LINK int cpp_get_end_token(Cpp_Token_Stack *stack, int end);
FCPP_LINK void cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token, int amount);
FCPP_LINK Cpp_Relex_State cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance);
FCPP_LINK Cpp_Relex_State cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance);
FCPP_LINK bool cpp_relex_nonalloc_main(Cpp_Relex_State state, Cpp_Token_Stack *stack);
#ifndef FCPP_FORBID_MALLOC
@ -174,10 +166,10 @@ FCPP_LINK void cpp_free_token_stack(Cpp_Token_Stack stack);
FCPP_LINK void cpp_resize_token_stack(Cpp_Token_Stack *stack, int new_max);
FCPP_LINK void cpp_push_token(Cpp_Token_Stack *stack, Cpp_Token token);
FCPP_LINK void cpp_lex_file(Cpp_File file, Cpp_Token_Stack *stack);
FCPP_LINK bool cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount, int extra_tolerance);
inline void cpp_relex_file(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount)
{ cpp_relex_file_limited(file, stack, start_i, end_i, amount, -1); }
FCPP_LINK void cpp_lex_file(char *data, int size, Cpp_Token_Stack *stack);
FCPP_LINK bool cpp_relex_file_limited(char *data, int size, Cpp_Token_Stack *stack, int start_i, int end_i, int amount, int extra_tolerance);
inline void cpp_relex_file(char *data, int size, Cpp_Token_Stack *stack, int start_i, int end_i, int amount)
{ cpp_relex_file_limited(data, size, stack, start_i, end_i, amount, -1); }
#endif
#define FCPP_STRING_LIST(x) {x, FCPP_COUNT(x)}
@ -393,7 +385,7 @@ FCPP_GLOBAL String_List preprops = FCPP_STRING_LIST(preprop_strings);
#define _TentativeAssert FCPP_ASSERT
FCPP_LINK Sub_Match_List_Result
sub_match_list(Cpp_File file, int pos, String_List list, int sub_size){
sub_match_list(char *data, int size, int pos, String_List list, int sub_size){
Sub_Match_List_Result result;
String str_main;
char *str_check;
@ -401,7 +393,7 @@ sub_match_list(Cpp_File file, int pos, String_List list, int sub_size){
result.index = -1;
result.new_pos = pos;
str_main = make_string(file.data + pos, file.size - pos);
str_main = make_string(data + pos, size - pos);
if (sub_size > 0){
str_main = substr(str_main, 0, sub_size);
for (i = 0; i < list.count; ++i){
@ -505,11 +497,11 @@ seek_block_comment_end(char *data, int size, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_whitespace(Cpp_File file, int pos){
cpp_read_whitespace(char *data, int size, int pos){
Cpp_Read_Result result = {};
while (pos < file.size && char_is_whitespace(file.data[pos])){
if (file.data[pos] == '\n'){
while (pos < size && char_is_whitespace(data[pos])){
if (data[pos] == '\n'){
result.newline = 1;
}
++pos;
@ -521,16 +513,16 @@ cpp_read_whitespace(Cpp_File file, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_junk_line(Cpp_File file, int pos){
cpp_read_junk_line(char *data, int size, int pos){
Cpp_Read_Result result = {};
result.token.start = pos;
result.token.type = CPP_TOKEN_JUNK;
bool comment_end = 0;
while (pos < file.size && file.data[pos] != '\n'){
if (file.data[pos] == '/' && pos + 1 < file.size){
if (file.data[pos + 1] == '/' ||
file.data[pos + 1] == '*'){
while (pos < size && data[pos] != '\n'){
if (data[pos] == '/' && pos + 1 < size){
if (data[pos + 1] == '/' ||
data[pos + 1] == '*'){
comment_end = 1;
break;
}
@ -543,10 +535,10 @@ cpp_read_junk_line(Cpp_File file, int pos){
result.token.size = pos - result.token.start;
}
else{
while (pos > 0 && file.data[pos - 1] == '\r'){
while (pos > 0 && data[pos - 1] == '\r'){
--pos;
}
if (pos > 0 && file.data[pos - 1] == '\\'){
if (pos > 0 && data[pos - 1] == '\\'){
--pos;
}
result.pos = pos;
@ -557,13 +549,13 @@ cpp_read_junk_line(Cpp_File file, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_operator(Cpp_File file, int pos){
cpp_read_operator(char *data, int size, int pos){
Cpp_Read_Result result = {};
result.pos = pos;
result.token.start = pos;
Sub_Match_List_Result match;
match = sub_match_list(file, result.token.start, ops, -1);
match = sub_match_list(data, size, result.token.start, ops, -1);
if (match.index != -1){
result.pos = match.new_pos;
@ -581,13 +573,13 @@ cpp_read_operator(Cpp_File file, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_pp_operator(Cpp_File file, int pos){
cpp_read_pp_operator(char *data, int size, int pos){
Cpp_Read_Result result = {};
result.pos = pos;
result.token.start = pos;
Sub_Match_List_Result match;
match = sub_match_list(file, result.token.start, pp_ops, -1);
match = sub_match_list(data, size, result.token.start, pp_ops, -1);
_Assert(match.index != -1);
result.pos = match.new_pos;
@ -598,13 +590,13 @@ cpp_read_pp_operator(Cpp_File file, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){
cpp_read_alpha_numeric(char *data, int size, int pos, bool in_if_body){
Cpp_Read_Result result = {};
result.pos = pos;
result.token.start = pos;
while (result.pos < file.size &&
char_is_alpha_numeric(file.data[result.pos])){
while (result.pos < size &&
char_is_alpha_numeric(data[result.pos])){
++result.pos;
}
@ -614,7 +606,7 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){
if (in_if_body){
String word;
word.size = result.token.size;
word.str = file.data + result.token.start;
word.str = data + result.token.start;
if (match(word, "defined")){
result.token.type = CPP_TOKEN_DEFINED;
result.token.flags |= CPP_TFLAG_IS_OPERATOR;
@ -624,14 +616,14 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){
if (result.token.type == CPP_TOKEN_JUNK){
Sub_Match_List_Result match;
match = sub_match_list(file, result.token.start, bool_lits, result.token.size);
match = sub_match_list(data, size, result.token.start, bool_lits, result.token.size);
if (match.index != -1){
result.token.type = CPP_TOKEN_BOOLEAN_CONSTANT;
result.token.flags |= CPP_TFLAG_IS_KEYWORD;
}
else{
match = sub_match_list(file, result.token.start, keywords, result.token.size);
match = sub_match_list(data, size, result.token.start, keywords, result.token.size);
if (match.index != -1){
String_And_Flag data = keywords.data[match.index];
@ -648,7 +640,7 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){
}
FCPP_LINK Cpp_Read_Result
cpp_read_number(Cpp_File file, int pos){
cpp_read_number(char *data, int size, int pos){
Cpp_Read_Result result = {};
result.pos = pos;
result.token.start = pos;
@ -659,9 +651,9 @@ cpp_read_number(Cpp_File file, int pos){
bool is_hex = 0;
bool is_zero = 0;
if (file.data[pos] == '0'){
if (pos+1 < file.size){
char next = file.data[pos+1];
if (data[pos] == '0'){
if (pos+1 < size){
char next = data[pos+1];
if (next == 'x'){
is_hex = 1;
is_integer = 1;
@ -684,7 +676,7 @@ cpp_read_number(Cpp_File file, int pos){
is_integer = 1;
}
}
else if (file.data[pos] == '.'){
else if (data[pos] == '.'){
is_float = 1;
}
@ -696,20 +688,20 @@ cpp_read_number(Cpp_File file, int pos){
char character;
do{
++result.pos;
if (result.pos >= file.size){
if (result.pos >= size){
break;
}
character = file.data[result.pos];
character = data[result.pos];
} while(char_is_hex(character));
}
else if (is_oct){
char character;
do{
++result.pos;
if (result.pos >= file.size){
if (result.pos >= size){
break;
}
character = file.data[result.pos];
character = data[result.pos];
}while(char_is_numeric(character));
}
else{
@ -718,11 +710,11 @@ cpp_read_number(Cpp_File file, int pos){
while (1){
++result.pos;
if (result.pos >= file.size){
if (result.pos >= size){
break;
}
bool is_good = 0;
char character = file.data[result.pos];
char character = data[result.pos];
if (character >= '0' && character <= '9'){
is_good = 1;
}
@ -744,11 +736,11 @@ cpp_read_number(Cpp_File file, int pos){
while (1){
++result.pos;
if (result.pos >= file.size){
if (result.pos >= size){
break;
}
is_good = 0;
character = file.data[result.pos];
character = data[result.pos];
if (character >= '0' && character <= '9'){
is_good = 1;
}
@ -771,7 +763,7 @@ cpp_read_number(Cpp_File file, int pos){
if (is_integer){
Sub_Match_List_Result match =
sub_match_list(file, result.pos, int_sufs, -1);
sub_match_list(data, size, result.pos, int_sufs, -1);
if (match.index != -1){
result.pos = match.new_pos;
}
@ -780,7 +772,7 @@ cpp_read_number(Cpp_File file, int pos){
}
else if (is_float){
Sub_Match_List_Result match =
sub_match_list(file, result.pos, float_sufs, -1);
sub_match_list(data, size, result.pos, float_sufs, -1);
if (match.index != -1){
result.pos = match.new_pos;
}
@ -795,69 +787,69 @@ cpp_read_number(Cpp_File file, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_string_litteral(Cpp_File file, int pos){
cpp_read_string_litteral(char *data, int size, int pos){
Cpp_Read_Result result = {};
result.token.start = pos;
_Assert(file.data[pos] == '"');
Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '"');
_Assert(data[pos] == '"');
Seek_Result seek = seek_unescaped_delim(data, size, pos, '"');
pos = seek.pos;
if (seek.new_line){
result.token.flags |= CPP_TFLAG_MULTILINE;
}
result.token.size = pos - result.token.start;
result.token.type = CPP_TOKEN_STRING_CONSTANT;
result.pos = pos;
return result;
}
FCPP_LINK Cpp_Read_Result
cpp_read_character_litteral(Cpp_File file, int pos){
cpp_read_character_litteral(char *data, int size, int pos){
Cpp_Read_Result result = {};
result.token.start = pos;
_Assert(file.data[pos] == '\'');
Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '\'');
_Assert(data[pos] == '\'');
Seek_Result seek = seek_unescaped_delim(data, size, pos, '\'');
pos = seek.pos;
if (seek.new_line){
result.token.flags |= CPP_TFLAG_MULTILINE;
}
result.token.size = pos - result.token.start;
result.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
result.pos = pos;
return result;
}
FCPP_LINK Cpp_Read_Result
cpp_read_line_comment(Cpp_File file, int pos){
cpp_read_line_comment(char *data, int size, int pos){
Cpp_Read_Result result = {};
result.token.start = pos;
_Assert(file.data[pos] == '/' && file.data[pos + 1] == '/');
_Assert(data[pos] == '/' && data[pos + 1] == '/');
pos += 2;
while (pos < file.size){
if (file.data[pos] == '\n'){
while (pos < size){
if (data[pos] == '\n'){
break;
}
if (file.data[pos] == '\\'){
if (pos + 1 < file.size &&
file.data[pos + 1] == '\n'){
if (data[pos] == '\\'){
if (pos + 1 < size &&
data[pos + 1] == '\n'){
++pos;
}
else if (pos + 2 < file.size &&
file.data[pos + 1] == '\r' &&
file.data[pos + 2] == '\n'){
else if (pos + 2 < size &&
data[pos + 1] == '\r' &&
data[pos + 2] == '\n'){
pos += 2;
}
}
++pos;
}
if (pos > 0 && file.data[pos-1] == '\r'){
if (pos > 0 && data[pos-1] == '\r'){
--pos;
}
result.token.size = pos - result.token.start;
@ -867,16 +859,16 @@ cpp_read_line_comment(Cpp_File file, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_block_comment(Cpp_File file, int pos){
cpp_read_block_comment(char *data, int size, int pos){
Cpp_Read_Result result = {};
result.token.start = pos;
_Assert(file.data[pos] == '/' && file.data[pos + 1] == '*');
_Assert(data[pos] == '/' && data[pos + 1] == '*');
pos += 2;
while (pos < file.size){
if (file.data[pos] == '*' &&
pos + 1 < file.size &&
file.data[pos+1] == '/'){
while (pos < size){
if (data[pos] == '*' &&
pos + 1 < size &&
data[pos+1] == '/'){
break;
}
++pos;
@ -889,43 +881,42 @@ cpp_read_block_comment(Cpp_File file, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_preprocessor(Cpp_File file, int pos){
_Assert(file.data[pos] == '#');
cpp_read_preprocessor(char *data, int size, int pos){
_Assert(data[pos] == '#');
Cpp_Read_Result result = {};
result.token.start = pos;
result.token.type = CPP_PP_UNKNOWN;
result.token.flags |= CPP_TFLAG_PP_DIRECTIVE;
++pos;
while (pos < file.size &&
(file.data[pos] == ' ' ||
file.data[pos] == '\t')){
while (pos < size &&
(data[pos] == ' ' ||
data[pos] == '\t')){
++pos;
}
Sub_Match_List_Result match
= sub_match_list(file, pos, preprops, -1);
= sub_match_list(data, size, pos, preprops, -1);
if (match.index != -1){
result.token.size = match.new_pos - result.token.start;
result.token.type = (Cpp_Token_Type)preprops.data[match.index].flags;
result.pos = match.new_pos;
}
else{
while (pos < file.size &&
!char_is_whitespace(file.data[pos])){
while (pos < size && !char_is_whitespace(data[pos])){
++pos;
}
result.token.size = pos - result.token.start;
result.pos = pos;
}
return result;
}
FCPP_LINK Cpp_Read_Result
cpp_read_pp_include_file(Cpp_File file, int pos){
char start = file.data[pos];
cpp_read_pp_include_file(char *data, int size, int pos){
char start = data[pos];
_Assert(start == '<' || start == '"');
Cpp_Read_Result result = {};
@ -942,22 +933,20 @@ cpp_read_pp_include_file(Cpp_File file, int pos){
}
++pos;
while (pos < file.size && file.data[pos] != end){
if (file.data[pos] == '\n'){
while (pos < size && data[pos] != end){
if (data[pos] == '\n'){
result.token.type = CPP_TOKEN_JUNK;
result.token.flags |= CPP_TFLAG_BAD_ENDING;
break;
}
if (file.data[pos] == '\\'){
// TODO(allen): Not sure that this is 100% correct.
if (pos + 1 < file.size &&
file.data[pos + 1] == '\n'){
if (data[pos] == '\\'){
if (pos + 1 < size && data[pos + 1] == '\n'){
++pos;
result.token.flags |= CPP_TFLAG_MULTILINE;
}
else if (pos + 2 < file.size &&
file.data[pos + 1] == '\r' &&
file.data[pos + 2] == '\n'){
else if (pos + 2 < size &&
data[pos + 1] == '\r' &&
data[pos + 2] == '\n'){
pos += 2;
result.token.flags |= CPP_TFLAG_MULTILINE;
}
@ -966,7 +955,7 @@ cpp_read_pp_include_file(Cpp_File file, int pos){
}
if (result.token.type != CPP_TOKEN_JUNK){
if (pos < file.size){
if (pos < size){
++pos;
}
}
@ -978,55 +967,55 @@ cpp_read_pp_include_file(Cpp_File file, int pos){
}
FCPP_LINK Cpp_Read_Result
cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body){
char current = file.data[pos];
cpp_read_pp_default_mode(char *data, int size, int pos, bool in_if_body){
char current = data[pos];
Cpp_Read_Result result;
if (char_is_numeric(current)){
result = cpp_read_number(file, pos);
result = cpp_read_number(data, size, pos);
}
else if (char_is_alpha(current)){
result = cpp_read_alpha_numeric(file, pos, in_if_body);
result = cpp_read_alpha_numeric(data, size, pos, in_if_body);
}
else if (current == '.'){
if (pos + 1 < file.size){
char next = file.data[pos + 1];
if (pos + 1 < size){
char next = data[pos + 1];
if (char_is_numeric(next)){
result = cpp_read_number(file, pos);
result = cpp_read_number(data, size, pos);
}
else{
result = cpp_read_operator(file, pos);
result = cpp_read_operator(data, size, pos);
}
}
else{
result = cpp_read_operator(file, pos);
result = cpp_read_operator(data, size, pos);
}
}
else if (current == '/'){
if (pos + 1 < file.size){
char next = file.data[pos + 1];
if (pos + 1 < size){
char next = data[pos + 1];
if (next == '/'){
result = cpp_read_line_comment(file, pos);
result = cpp_read_line_comment(data, size, pos);
}
else if (next == '*'){
result = cpp_read_block_comment(file, pos);
result = cpp_read_block_comment(data, size, pos);
}
else{
result = cpp_read_operator(file, pos);
result = cpp_read_operator(data, size, pos);
}
}
else{
result = cpp_read_operator(file, pos);
result = cpp_read_operator(data, size, pos);
}
}
else if (current == '"'){
result = cpp_read_string_litteral(file, pos);
result = cpp_read_string_litteral(data, size, pos);
}
else if (current == '\''){
result = cpp_read_character_litteral(file, pos);
result = cpp_read_character_litteral(data, size, pos);
}
else{
result = cpp_read_operator(file, pos);
result = cpp_read_operator(data, size, pos);
}
return result;
@ -1084,16 +1073,16 @@ cpp_push_token_nonalloc(Cpp_Token_Stack *token_stack, Cpp_Token token){
}
FCPP_LINK Cpp_Read_Result
cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
cpp_lex_step(char *data, int size, Cpp_Lex_Data *lex_data){
Cpp_Lex_Data lex = *lex_data;
Cpp_Read_Result result = {};
bool has_result = 1;
fcpp_u16 state_flags = cpp_token_set_pp_state(0, lex.pp_state);
char current = file.data[lex.pos];
char current = data[lex.pos];
if (char_is_whitespace(current)){
result = cpp_read_whitespace(file, lex.pos);
result = cpp_read_whitespace(data, size, lex.pos);
lex.pos = result.pos;
if (result.newline && lex.pp_state != CPP_LEX_PP_DEFAULT){
lex.pp_state = CPP_LEX_PP_DEFAULT;
@ -1105,7 +1094,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
if (lex.pp_state == CPP_LEX_PP_DEFAULT){
// TODO(allen): Not first hard of the line? Then it's junk.
if (current == '#'){
result = cpp_read_preprocessor(file, lex.pos);
result = cpp_read_preprocessor(data, size, lex.pos);
lex.pos = result.pos;
switch (result.token.type){
case CPP_PP_INCLUDE:
@ -1143,7 +1132,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
}
}
else{
result = cpp_read_pp_default_mode(file, lex.pos);
result = cpp_read_pp_default_mode(data, size, lex.pos);
lex.pos = result.pos;
}
}
@ -1152,10 +1141,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
if (current == '\\'){
fcpp_i32 seek = lex.pos;
++seek;
while (seek < file.size && file.data[seek] == '\r'){
while (seek < size && data[seek] == '\r'){
++seek;
}
if ((seek < file.size && file.data[seek] == '\n') || seek >= file.size){
if ((seek < size && data[seek] == '\n') || seek >= size){
lex.pos = seek + 1;
has_result = 0;
}
@ -1177,7 +1166,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
lex.pp_state = CPP_LEX_PP_JUNK;
}
else{
result = cpp_read_alpha_numeric(file, lex.pos);
result = cpp_read_alpha_numeric(data, size, lex.pos);
result.token.flags |= CPP_TFLAG_PP_BODY;
lex.pos = result.pos;
lex.pp_state = CPP_LEX_PP_JUNK;
@ -1190,7 +1179,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
lex.pp_state = CPP_LEX_PP_JUNK;
}
else{
result = cpp_read_alpha_numeric(file, lex.pos);
result = cpp_read_alpha_numeric(data, size, lex.pos);
result.token.flags |= CPP_TFLAG_PP_BODY;
lex.pos = result.pos;
lex.pp_state = CPP_LEX_PP_BODY;
@ -1203,7 +1192,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
lex.pp_state = CPP_LEX_PP_JUNK;
}
else{
result = cpp_read_pp_include_file(file, lex.pos);
result = cpp_read_pp_include_file(data, size, lex.pos);
lex.pos = result.pos;
lex.pp_state = CPP_LEX_PP_JUNK;
}
@ -1211,10 +1200,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
case CPP_LEX_PP_BODY:
if (current == '#'){
result = cpp_read_pp_operator(file, lex.pos);
result = cpp_read_pp_operator(data, size, lex.pos);
}
else{
result = cpp_read_pp_default_mode(file, lex.pos);
result = cpp_read_pp_default_mode(data, size, lex.pos);
}
lex.pos = result.pos;
result.token.flags |= CPP_TFLAG_PP_BODY;
@ -1222,10 +1211,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
case CPP_LEX_PP_BODY_IF:
if (current == '#'){
result = cpp_read_pp_operator(file, lex.pos);
result = cpp_read_pp_operator(data, size, lex.pos);
}
else{
result = cpp_read_pp_default_mode(file, lex.pos, 1);
result = cpp_read_pp_default_mode(data, size, lex.pos, 1);
}
lex.pos = result.pos;
result.token.flags |= CPP_TFLAG_PP_BODY;
@ -1237,7 +1226,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
lex.pp_state = CPP_LEX_PP_JUNK;
}
else{
result = cpp_read_number(file, lex.pos);
result = cpp_read_number(data, size, lex.pos);
lex.pos = result.pos;
result.token.flags |= CPP_TFLAG_PP_BODY;
lex.pp_state = CPP_LEX_PP_INCLUDE;
@ -1245,7 +1234,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
break;
case CPP_LEX_PP_ERROR:
result = cpp_read_junk_line(file, lex.pos);
result = cpp_read_junk_line(data, size, lex.pos);
lex.pos = result.pos;
result.token.type = CPP_TOKEN_ERROR_MESSAGE;
result.token.flags |= CPP_TFLAG_PP_BODY;
@ -1254,21 +1243,21 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
default:
{
bool took_comment = 0;
if (current == '/' && lex.pos + 1 < file.size){
if (file.data[lex.pos + 1] == '/'){
result = cpp_read_line_comment(file, lex.pos);
if (current == '/' && lex.pos + 1 < size){
if (data[lex.pos + 1] == '/'){
result = cpp_read_line_comment(data, size, lex.pos);
lex.pp_state = CPP_LEX_PP_DEFAULT;
lex.pos = result.pos;
took_comment = 1;
}else if (file.data[lex.pos + 1] == '*'){
result = cpp_read_block_comment(file, lex.pos);
}else if (data[lex.pos + 1] == '*'){
result = cpp_read_block_comment(data, size, lex.pos);
lex.pos = result.pos;
took_comment = 1;
}
}
if (!took_comment){
result = cpp_read_junk_line(file, lex.pos);
result = cpp_read_junk_line(data, size, lex.pos);
lex.pos = result.pos;
result.token.flags |= CPP_TFLAG_PP_BODY;
}
@ -1287,12 +1276,12 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
}
FCPP_LINK int
cpp_lex_file_token_count(Cpp_File file){
cpp_lex_file_token_count(char *data, int size){
int count = 0;
Cpp_Lex_Data lex = {};
Cpp_Token token = {};
while (lex.pos < file.size){
Cpp_Read_Result step_result = cpp_lex_step(file, &lex);
while (lex.pos < size){
Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex);
if (step_result.has_result){
if (count > 0){
@ -1315,21 +1304,21 @@ cpp_lex_file_token_count(Cpp_File file){
}
FCPP_LINK Cpp_Lex_Data
cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *token_stack_out, Cpp_Lex_Data data){
while (data.pos < file.size){
Cpp_Lex_Data prev_lex = data;
Cpp_Read_Result step_result = cpp_lex_step(file, &data);
cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *token_stack_out, Cpp_Lex_Data lex_data){
while (lex_data.pos < size){
Cpp_Lex_Data prev_lex = lex_data;
Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex_data);
if (step_result.has_result){
if (!cpp_push_token_nonalloc(token_stack_out, step_result.token)){
data = prev_lex;
return data;
lex_data = prev_lex;
return lex_data;
}
}
}
data.complete = 1;
return data;
lex_data.complete = 1;
return lex_data;
}
FCPP_LINK Cpp_Get_Token_Result
@ -1395,10 +1384,11 @@ cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token_i, int amount){
}
FCPP_LINK Cpp_Relex_State
cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack,
cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack,
int start, int end, int amount, int tolerance){
Cpp_Relex_State state;
state.file = file;
state.data = data;
state.size = size;
state.stack = stack;
state.start = start;
state.end = end;
@ -1460,12 +1450,12 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, Cpp_Token_Stack *relex_stack, in
lex.pos = state->relex_start;
int relex_end_i = state->end_token_i;
Cpp_Token match_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i);
Cpp_Token match_token = cpp__get_token(stack, tokens, state->size, relex_end_i);
Cpp_Token end_token = match_token;
bool went_too_far = 0;
for (;;){
Cpp_Read_Result read = cpp_lex_step(state->file, &lex);
Cpp_Read_Result read = cpp_lex_step(state->data, state->size, &lex);
if (read.has_result){
if (read.token.start == end_token.start &&
read.token.size == end_token.size &&
@ -1477,14 +1467,14 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, Cpp_Token_Stack *relex_stack, in
while (lex.pos > end_token.start && relex_end_i < stack->count){
++relex_end_i;
end_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i);
end_token = cpp__get_token(stack, tokens, state->size, relex_end_i);
}
if (relex_stack->count == relex_stack->max_count){
went_too_far = 1;
break;
}
}
if (lex.pos >= state->file.size) break;
if (lex.pos >= state->size) break;
}
if (!went_too_far){
@ -1557,10 +1547,10 @@ cpp_push_token(Cpp_Token_Stack *token_stack, Cpp_Token token){
}
FCPP_LINK void
cpp_lex_file(Cpp_File file, Cpp_Token_Stack *token_stack_out){
cpp_lex_file(char *data, int size, Cpp_Token_Stack *token_stack_out){
Cpp_Lex_Data lex = {};
while (lex.pos < file.size){
Cpp_Read_Result step_result = cpp_lex_step(file, &lex);
while (lex.pos < size){
Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex);
if (step_result.has_result){
cpp_push_token(token_stack_out, step_result.token);
}
@ -1568,7 +1558,7 @@ cpp_lex_file(Cpp_File file, Cpp_Token_Stack *token_stack_out){
}
FCPP_LINK bool
cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
cpp_relex_file_limited(char *data, int size, Cpp_Token_Stack *stack,
int start, int end, int amount, int tolerance){
#if 0
int start_token_i, end_token_i;
@ -1583,18 +1573,18 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
++end_token_i;
}
cpp_shift_token_starts(token_stack, end_token_i, amount);
int relex_start_i = start_token_i - 1;
if (relex_start_i < 0){
relex_start_i = 0;
}
int end_guess_i = end_token_i + 1;
if (end_guess_i > token_stack->count){
--end_guess_i;
}
#endif
int relex_start_i;
int end_token_i, end_guess_i;
{
@ -1605,26 +1595,26 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
else{
relex_start_i = result.token_index-1;
}
result = cpp_get_token(stack, end);
if (result.token_index < 0) result.token_index = 0;
else if (end > stack->tokens[result.token_index].start) ++result.token_index;
end_token_i = result.token_index;
end_guess_i = result.token_index+1;
}
int relex_start = stack->tokens[relex_start_i].start;
if (start < relex_start) relex_start = start;
cpp_shift_token_starts(stack, end_token_i, amount);
Cpp_Token_Stack relex_stack = cpp_make_token_stack((end_guess_i - relex_start_i + 1) * 3 / 2);
Cpp_Lex_Data lex = {};
lex.pp_state = cpp_token_get_pp_state(stack->tokens[relex_start_i].state_flags);
lex.pos = relex_start;
bool went_too_far = 0;
while (1){
Cpp_Read_Result result = cpp_lex_step(file, &lex);
Cpp_Read_Result result = cpp_lex_step(data, size, &lex);
if (result.has_result){
if (end_guess_i < stack->count &&
result.token.start == stack->tokens[end_guess_i].start &&
@ -1641,20 +1631,20 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
}
}
}
if (lex.pos >= file.size){
if (lex.pos >= size){
break;
}
if (tolerance >= 0 && relex_stack.count + relex_start_i >= end_guess_i + tolerance){
went_too_far = 1;
break;
}
}
if (!went_too_far){
int relex_end_i = end_guess_i;
if (relex_stack.count > 0){
if (relex_start_i > 0){
Cpp_Token_Merge merge = cpp_attempt_token_merge(stack->tokens[relex_start_i - 1],
@ -1664,7 +1654,7 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
relex_stack.tokens[0] = merge.new_token;
}
}
if (relex_end_i < stack->count){
Cpp_Token_Merge merge = cpp_attempt_token_merge(relex_stack.tokens[relex_stack.count - 1],
stack->tokens[relex_end_i]);
@ -1674,10 +1664,10 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
}
}
}
int token_delete_amount = relex_end_i - relex_start_i;
int token_shift_amount = relex_stack.count - token_delete_amount;
if (token_shift_amount != 0){
int new_token_count = stack->count + token_shift_amount;
if (new_token_count > stack->max_count){
@ -1687,24 +1677,24 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
}
cpp_resize_token_stack(stack, new_max);
}
if (relex_end_i < stack->count){
FCPP_MEM_MOVE(stack->tokens + relex_end_i + token_shift_amount,
stack->tokens + relex_end_i, sizeof(Cpp_Token)*(stack->count - relex_end_i));
}
stack->count += token_shift_amount;
}
FCPP_MEM_COPY(stack->tokens + relex_start_i, relex_stack.tokens, sizeof(Cpp_Token)*relex_stack.count);
cpp_free_token_stack(relex_stack);
}
else{
cpp_shift_token_starts(stack, end_token_i, -amount);
cpp_free_token_stack(relex_stack);
}
return went_too_far;
}
#endif

11
4cpp_lexer_types.h
View File

@ -170,13 +170,6 @@ enum Cpp_Token_Type{
CPP_TOKEN_TYPE_COUNT
};
// TODO(allen): This is a dumb redundant type... probably just
// move towards using String for this everywhere eventually.
struct Cpp_File{
char *data;
int size;
};
struct Cpp_Token{
Cpp_Token_Type type;
int start, size;
@ -235,7 +228,9 @@ struct Cpp_Get_Token_Result{
};
struct Cpp_Relex_State{
Cpp_File file;
char *data;
int size;
Cpp_Token_Stack *stack;
int start, end, amount;
int start_token_i;

18
4ed_file_view.cpp
View File

@ -1177,9 +1177,8 @@ Job_Callback_Sig(job_full_lex){
Editing_File *file = (Editing_File*)data[0];
General_Memory *general = (General_Memory*)data[1];
Cpp_File cpp_file;
cpp_file.data = file->state.buffer.data;
cpp_file.size = file->state.buffer.size;
char *text_data = file->state.buffer.data;
i32 text_size = file->state.buffer.size;
i32 buffer_size = file->state.buffer.size;
buffer_size = (buffer_size + 3)&(~3);
@ -1243,9 +1242,9 @@ Job_Callback_Sig(job_full_lex){
Cpp_Lex_Data status = {};
do{
for (i32 r = 2048; r > 0 && status.pos < cpp_file.size; --r){
for (i32 r = 2048; r > 0 && status.pos < text_size; --r){
Cpp_Lex_Data prev_lex = status;
Cpp_Read_Result step_result = cpp_lex_step(cpp_file, &status);
Cpp_Read_Result step_result = cpp_lex_step(text_data, text_size, &status);
if (step_result.has_result){
if (!cpp_push_token_nonalloc(&tokens, step_result.token)){
@ -1257,7 +1256,7 @@ Job_Callback_Sig(job_full_lex){
}
}
if (status.pos >= cpp_file.size){
if (status.pos >= text_size){
status.complete = 1;
}
else{
@ -1361,14 +1360,13 @@ file_relex_parallel(System_Functions *system,
b32 result = true;
b32 inline_lex = !file->state.still_lexing;
if (inline_lex){
Cpp_File cpp_file;
cpp_file.data = file->state.buffer.data;
cpp_file.size = file->state.buffer.size;
char *data = file->state.buffer.data;
i32 size = file->state.buffer.size;
Cpp_Token_Stack *stack = &file->state.token_stack;
Cpp_Relex_State state =
cpp_relex_nonalloc_start(cpp_file, stack,
cpp_relex_nonalloc_start(data, size, stack,
start_i, end_i, amount, 100);
Temp_Memory temp = begin_temp_memory(part);

155
4ed_metagen.cpp
View File

@ -710,14 +710,14 @@ perform_doc_parse(Partition *part, String doc_string, Documentation *doc){
}
static int
get_type_doc_string(Cpp_File file, Cpp_Token *tokens, int i,
get_type_doc_string(char *data, Cpp_Token *tokens, int i,
String *doc_string){
int result = false;
if (i > 0){
Cpp_Token *prev_token = tokens + i - 1;
if (prev_token->type == CPP_TOKEN_COMMENT){
*doc_string = make_string(file.data + prev_token->start, prev_token->size);
*doc_string = make_string(data + prev_token->start, prev_token->size);
if (check_and_fix_docs(doc_string)){
result = true;
}
@ -729,13 +729,13 @@ get_type_doc_string(Cpp_File file, Cpp_Token *tokens, int i,
static int
parse_struct(Partition *part, int is_struct,
Cpp_File file, Cpp_Token *tokens, int count,
char *data, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr,
Struct_Member *top_member);
static int
parse_struct_member(Partition *part,
Cpp_File file, Cpp_Token *tokens, int count,
char *data, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr,
Struct_Member *member){
@ -745,7 +745,7 @@ parse_struct_member(Partition *part,
int i = (int)(token - tokens);
String doc_string = {0};
get_type_doc_string(file, tokens, i, &doc_string);
get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i;
Cpp_Token *start_token = token;
@ -779,18 +779,18 @@ parse_struct_member(Partition *part,
}
}
String name = make_string(file.data + token_j->start, token_j->size);
String name = make_string(data + token_j->start, token_j->size);
name = skip_chop_whitespace(name);
int type_start = start_token->start;
int type_end = token_j->start;
String type = make_string(file.data + type_start, type_end - type_start);
String type = make_string(data + type_start, type_end - type_start);
type = skip_chop_whitespace(type);
type_start = token_j->start + token_j->size;
type_end = token->start;
String type_postfix = make_string(file.data + type_start, type_end - type_start);
String type_postfix = make_string(data + type_start, type_end - type_start);
type_postfix = skip_chop_whitespace(type_postfix);
++token;
@ -811,7 +811,7 @@ parse_struct_member(Partition *part,
static Struct_Member*
parse_struct_next_member(Partition *part,
Cpp_File file, Cpp_Token *tokens, int count,
char *data, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr){
Struct_Member *result = 0;
@ -821,11 +821,11 @@ parse_struct_next_member(Partition *part,
for (; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER ||
(token->flags & CPP_TFLAG_IS_KEYWORD)){
String lexeme = make_string(file.data + token->start, token->size);
String lexeme = make_string(data + token->start, token->size);
if (match(lexeme, make_lit_string("struct"))){
Struct_Member *member = push_struct(part, Struct_Member);
if (parse_struct(part, true, file, tokens, count, &token, member)){
if (parse_struct(part, true, data, tokens, count, &token, member)){
result = member;
break;
}
@ -835,7 +835,7 @@ parse_struct_next_member(Partition *part,
}
else if (match(lexeme, make_lit_string("union"))){
Struct_Member *member = push_struct(part, Struct_Member);
if (parse_struct(part, false, file, tokens, count, &token, member)){
if (parse_struct(part, false, data, tokens, count, &token, member)){
result = member;
break;
}
@ -845,7 +845,7 @@ parse_struct_next_member(Partition *part,
}
else{
Struct_Member *member = push_struct(part, Struct_Member);
if (parse_struct_member(part, file, tokens, count, &token, member)){
if (parse_struct_member(part, data, tokens, count, &token, member)){
result = member;
break;
}
@ -866,7 +866,7 @@ parse_struct_next_member(Partition *part,
static int
parse_struct(Partition *part, int is_struct,
Cpp_File file, Cpp_Token *tokens, int count,
char *data, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr,
Struct_Member *top_member){
@ -876,7 +876,7 @@ parse_struct(Partition *part, int is_struct,
int i = (int)(token - tokens);
String doc_string = {0};
get_type_doc_string(file, tokens, i, &doc_string);
get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i;
@ -899,7 +899,7 @@ parse_struct(Partition *part, int is_struct,
String name = {0};
if (j != start_i){
name = make_string(file.data + token_j->start, token_j->size);
name = make_string(data + token_j->start, token_j->size);
name = skip_chop_whitespace(name);
}
@ -913,7 +913,7 @@ parse_struct(Partition *part, int is_struct,
++token;
Struct_Member *new_member =
parse_struct_next_member(part, file, tokens, count, &token);
parse_struct_next_member(part, data, tokens, count, &token);
if (new_member){
top_member->first_child = new_member;
@ -921,7 +921,7 @@ parse_struct(Partition *part, int is_struct,
Struct_Member *head_member = new_member;
for(;;){
new_member =
parse_struct_next_member(part, file, tokens, count, &token);
parse_struct_next_member(part, data, tokens, count, &token);
if (new_member){
head_member->next_sibling = new_member;
head_member = new_member;
@ -1124,7 +1124,7 @@ print_see_also(FILE *file, Documentation *doc){
}
static int
parse_enum(Partition *part, Cpp_File file,
parse_enum(Partition *part, char *data,
Cpp_Token *tokens, int count,
Cpp_Token **token_ptr, int start_i,
Enum_Set flag_set, int flag_index){
@ -1143,7 +1143,7 @@ parse_enum(Partition *part, Cpp_File file,
}
}
String name = make_string(file.data + token_j->start, token_j->size);
String name = make_string(data + token_j->start, token_j->size);
name = skip_chop_whitespace(name);
for (; i < count; ++i, ++token){
@ -1162,9 +1162,9 @@ parse_enum(Partition *part, Cpp_File file,
}
else if (token->type == CPP_TOKEN_IDENTIFIER){
String doc_string = {0};
get_type_doc_string(file, tokens, i, &doc_string);
get_type_doc_string(data, tokens, i, &doc_string);
String name = make_string(file.data + token->start, token->size);
String name = make_string(data + token->start, token->size);
name = skip_chop_whitespace(name);
String value = {0};
@ -1185,7 +1185,7 @@ parse_enum(Partition *part, Cpp_File file,
int val_start = start_token->start + start_token->size;
int val_end = token->start;
value = make_string(file.data + val_start, val_end - val_start);
value = make_string(data + val_start, val_end - val_start);
value = skip_chop_whitespace(value);
--i;
@ -1282,7 +1282,7 @@ allocate_argument_breakdown(int count){
}
static Argument_Breakdown
do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token){
do_parameter_parse(char *data, Cpp_Token *args_start_token, Cpp_Token *token){
int arg_index = 0;
Cpp_Token *arg_token = args_start_token + 1;
int param_string_start = arg_token->start;
@ -1303,7 +1303,7 @@ do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token)
arg_token->type == CPP_TOKEN_PARENTHESE_CLOSE){
int size = arg_token->start - param_string_start;
String param_string = make_string(file.data + param_string_start, size);
String param_string = make_string(data + param_string_start, size);
param_string = chop_whitespace(param_string);
breakdown.param_string[arg_index] = param_string;
@ -1313,7 +1313,7 @@ do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token)
if (param_name_token->type == CPP_TOKEN_IDENTIFIER){
int start = param_name_token->start;
int size = param_name_token->size;
breakdown.param_name[arg_index] = make_string(file.data + start, size);
breakdown.param_name[arg_index] = make_string(data + start, size);
break;
}
}
@ -1363,7 +1363,7 @@ do_function_parse_check(int *index, Cpp_Token **token_ptr, int count){
static int
do_function_get_doc(int *index, Cpp_Token **token_ptr, int count,
Cpp_File file, String *doc_string){
char *data, String *doc_string){
int result = false;
int i = *index;
@ -1371,7 +1371,7 @@ do_function_get_doc(int *index, Cpp_Token **token_ptr, int count,
for (; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_COMMENT){
String lexeme = make_string(file.data + token->start, token->size);
String lexeme = make_string(data + token->start, token->size);
if (check_and_fix_docs(&lexeme)){
*doc_string = lexeme;
result = true;
@ -1391,7 +1391,7 @@ do_function_get_doc(int *index, Cpp_Token **token_ptr, int count,
static int
do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_start_token,
Cpp_File file, Function_Set function_set, int sig_count){
char *data, Function_Set function_set, int sig_count){
int result = false;
int i = *index;
@ -1399,10 +1399,10 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s
Cpp_Token *args_start_token = token+1;
function_set.name[sig_count] = make_string(file.data + token->start, token->size);
function_set.name[sig_count] = make_string(data + token->start, token->size);
int size = token->start - ret_start_token->start;
String ret = make_string(file.data + ret_start_token->start, size);
String ret = make_string(data + ret_start_token->start, size);
ret = chop_whitespace(ret);
function_set.ret[sig_count] = ret;
@ -1415,12 +1415,12 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s
if (i < count){
int size = token->start + token->size - args_start_token->start;;
function_set.args[sig_count] =
make_string(file.data + args_start_token->start, size);
make_string(data + args_start_token->start, size);
function_set.valid[sig_count] = true;
result = true;
Argument_Breakdown *breakdown = &function_set.breakdown[sig_count];
*breakdown = do_parameter_parse(file, args_start_token, token);
*breakdown = do_parameter_parse(data, args_start_token, token);
}
*index = i;
@ -1430,7 +1430,7 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s
}
static int
do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File file,
do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, char *data,
Function_Set function_set, int sig_count){
int result = false;
@ -1438,7 +1438,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi
Cpp_Token *token = *token_ptr;
{
function_set.marker[sig_count] = make_string(file.data + token->start, token->size);
function_set.marker[sig_count] = make_string(data + token->start, token->size);
int j = i;
Cpp_Token *jtoken = token;
@ -1446,7 +1446,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi
if (do_function_parse_check(&j, &jtoken, count)){
if (token->type == CPP_TOKEN_IDENTIFIER){
String doc_string = {0};
if (do_function_get_doc(&j, &jtoken, count, file, &doc_string)){
if (do_function_get_doc(&j, &jtoken, count, data, &doc_string)){
function_set.doc_string[sig_count] = doc_string;
}
}
@ -1457,7 +1457,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi
Cpp_Token *ret_start_token = token;
if (do_function_parse_check(&i, &token, count)){
if (do_function_parse(&i, &token, count, ret_start_token,
file, function_set, sig_count)){
data, function_set, sig_count)){
result = true;
}
}
@ -1499,7 +1499,7 @@ do_macro_parse_check(int *index, Cpp_Token **token_ptr, int count){
static int
do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
Cpp_File file, Function_Set macro_set, int sig_count){
char *data, Function_Set macro_set, int sig_count){
int result = false;
int i = *index;
@ -1508,7 +1508,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
if (i > 0){
Cpp_Token *doc_token = token-1;
String doc_string = make_string(file.data + doc_token->start, doc_token->size);
String doc_string = make_string(data + doc_token->start, doc_token->size);
if (check_and_fix_docs(&doc_string)){
macro_set.doc_string[sig_count] = doc_string;
@ -1520,7 +1520,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
}
if (i < count && (token->flags & CPP_TFLAG_PP_BODY)){
macro_set.name[sig_count] = make_string(file.data + token->start, token->size);
macro_set.name[sig_count] = make_string(data + token->start, token->size);
++i, ++token;
if (i < count){
@ -1534,10 +1534,10 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
if (i < count){
int start = args_start_token->start;
int end = token->start + token->size;
macro_set.args[sig_count] = make_string(file.data + start, end - start);
macro_set.args[sig_count] = make_string(data + start, end - start);
Argument_Breakdown *breakdown = &macro_set.breakdown[sig_count];
*breakdown = do_parameter_parse(file, args_start_token, token);
*breakdown = do_parameter_parse(data, args_start_token, token);
++i, ++token;
if (i < count){
@ -1556,7 +1556,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
start = body_start->start;
end = body_end->start + body_end->size;
macro_set.body[sig_count] = make_string(file.data + start, end - start);
macro_set.body[sig_count] = make_string(data + start, end - start);
}
}
@ -1745,12 +1745,11 @@ generate_custom_headers(){
String *code = &string_code;
Cpp_Token_Stack *token_stack = &string_tokens;
Cpp_File file;
file.data = code->str;
file.size = code->size;
char *data = code->str;
int size = code->size;
*token_stack = cpp_make_token_stack(1024);
cpp_lex_file(file, token_stack);
cpp_lex_file(data, size, token_stack);
int count = token_stack->count;
@ -1760,7 +1759,7 @@ generate_custom_headers(){
for (int i = 0; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER &&
!(token->flags & CPP_TFLAG_PP_BODY)){
String lexeme = make_string(file.data + token->start, token->size);
String lexeme = make_string(data + token->start, token->size);
String_Function_Marker marker =
do_string_function_marker_check(lexeme);
@ -1786,9 +1785,7 @@ generate_custom_headers(){
String *code = &string_code;
Cpp_Token_Stack *token_stack = &string_tokens;
Cpp_File file;
file.data = code->str;
file.size = code->size;
char *data = code->str;
int count = token_stack->count;
Cpp_Token *tokens = token_stack->tokens;
@ -1797,21 +1794,21 @@ generate_custom_headers(){
for (int i = 0; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER &&
!(token->flags & CPP_TFLAG_PP_BODY)){
String lexeme = make_string(file.data + token->start, token->size);
String lexeme = make_string(data + token->start, token->size);
String_Function_Marker marker =
do_string_function_marker_check(lexeme);
if (marker.parse_function){
if (do_full_function_parse(&i, &token, count, file,
if (do_full_function_parse(&i, &token, count, data,
string_function_set, string_sig_count)){
++string_sig_count;
}
}
else if (marker.parse_doc){
if (do_macro_parse_check(&i, &token, count)){
do_macro_parse(&i, &token, count,
file, string_function_set, string_sig_count);
do_macro_parse(&i, &token, count, data,
string_function_set, string_sig_count);
++string_sig_count;
}
}
@ -1834,13 +1831,11 @@ generate_custom_headers(){
String *code = &code_data[J];
Parse *parse = &parses[J];
// TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!!
Cpp_File file;
file.data = code->str;
file.size = code->size;
char *data = code->str;
int size = code->size;
parse->tokens = cpp_make_token_stack(512);
cpp_lex_file(file, &parse->tokens);
cpp_lex_file(data, size, &parse->tokens);
int count = parse->tokens.count;
Cpp_Token *tokens = parse->tokens.tokens;
@ -1849,7 +1844,7 @@ generate_custom_headers(){
for (int i = 0; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER &&
!(token->flags & CPP_TFLAG_PP_BODY)){
String lexeme = make_string(file.data + token->start, token->size);
String lexeme = make_string(data + token->start, token->size);
if (match(lexeme, "API_EXPORT")){
if (do_function_parse_check(&i, &token, count)){
++line_count;
@ -1867,10 +1862,7 @@ generate_custom_headers(){
String *code = &code_data[J];
Parse *parse = &parses[J];
// TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!!
Cpp_File file;
file.data = code->str;
file.size = code->size;
char *data = code->str;
int count = parse->tokens.count;
Cpp_Token *tokens = parse->tokens.tokens;
@ -1880,9 +1872,9 @@ generate_custom_headers(){
for (int i = 0; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER &&
!(token->flags & CPP_TFLAG_PP_BODY)){
String lexeme = make_string(file.data + token->start, token->size);
String lexeme = make_string(data + token->start, token->size);
if (match(lexeme, "API_EXPORT")){
do_full_function_parse(&i, &token, count, file, function_set, sig_count);
do_full_function_parse(&i, &token, count, data, function_set, sig_count);
if (!function_set.valid[sig_count]){
zero_index(function_set, sig_count);
// TODO(allen): get warning file name and line numbers
@ -1989,14 +1981,11 @@ generate_custom_headers(){
String type_code = file_dump("4coder_types.h");
// TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!!
Cpp_File type_file;
type_file.data = type_code.str;
type_file.size = type_code.size;
char *data = type_code.str;
int size = type_code.size;
Cpp_Token_Stack types_tokens = cpp_make_token_stack(512);
cpp_lex_file(type_file, &types_tokens);
cpp_lex_file(data, size, &types_tokens);
int typedef_count = 0;
int struct_count = 0;
@ -2021,7 +2010,7 @@ generate_custom_headers(){
(token->type == CPP_TOKEN_KEY_TYPE_DECLARATION ||
token->type == CPP_TOKEN_IDENTIFIER)){
String lexeme = make_string(type_file.data + token->start, token->size);
String lexeme = make_string(data + token->start, token->size);
int match_index = 0;
if (string_set_match(type_spec_keys, ArrayCount(type_spec_keys),
lexeme, &match_index)){
@ -2077,7 +2066,7 @@ generate_custom_headers(){
(token->type == CPP_TOKEN_KEY_TYPE_DECLARATION ||
token->type == CPP_TOKEN_IDENTIFIER)){
String lexeme = make_string(type_file.data + token->start, token->size);
String lexeme = make_string(data + token->start, token->size);
int match_index = 0;
if (string_set_match(type_spec_keys, ArrayCount(type_spec_keys),
lexeme, &match_index)){
@ -2085,7 +2074,7 @@ generate_custom_headers(){
case 0: //typedef
{
String doc_string = {0};
get_type_doc_string(type_file, tokens, i, &doc_string);
get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i;
Cpp_Token *start_token = token;
@ -2105,12 +2094,12 @@ generate_custom_headers(){
}
}
String name = make_string(type_file.data + token_j->start, token_j->size);
String name = make_string(data + token_j->start, token_j->size);
name = skip_chop_whitespace(name);
int type_start = start_token->start + start_token->size;
int type_end = token_j->start;
String type = make_string(type_file.data + type_start, type_end - type_start);
String type = make_string(data + type_start, type_end - type_start);
type = skip_chop_whitespace(type);
typedef_set.type[typedef_index] = type;
@ -2123,7 +2112,7 @@ generate_custom_headers(){
case 1: case 2: //struct/union
{
if (parse_struct(part, (match_index == 1),
type_file, tokens, count, &token,
data, tokens, count, &token,
struct_set.structs + struct_index)){
++struct_index;
}
@ -2133,7 +2122,7 @@ generate_custom_headers(){
case 3: //ENUM
{
String doc_string = {0};
get_type_doc_string(type_file, tokens, i, &doc_string);
get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i;
@ -2143,7 +2132,7 @@ generate_custom_headers(){
}
}
if (parse_enum(part, type_file,
if (parse_enum(part, data,
tokens, count,
&token, start_i,
enum_set, enum_index)){
@ -2156,7 +2145,7 @@ generate_custom_headers(){
case 4: //FLAGENUM
{
String doc_string = {0};
get_type_doc_string(type_file, tokens, i, &doc_string);
get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i;
@ -2166,7 +2155,7 @@ generate_custom_headers(){
}
}
if (parse_enum(part, type_file,
if (parse_enum(part, data,
tokens, count,
&token, start_i,
flag_set, flag_index)){

4
build.bat
View File

@ -1,6 +1,6 @@
@echo off
REM "build_exp.bat" /Zi
REM "build_all.bat" /DFRED_SUPER /DFRED_INTERNAL /Zi
"build_all.bat" /DFRED_INTERNAL /Zi
"build_all.bat" /DFRED_SUPER /DFRED_INTERNAL /Zi
REM "build_all.bat" /DFRED_INTERNAL /Zi
REM "build_all.bat" /O2 /Zi

2
power/4coder_casey.cpp
View File

@ -687,6 +687,8 @@ CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking)
CLI_OverlapWithConflict);
}
exec_command(app, change_active_panel);
prev_location = null_location;
}
#if 0

15
test/4cpp_new_lexer.h
View File

@ -1032,10 +1032,11 @@ cpp_lex_size_nonalloc(Lex_Data *S_ptr,
}
lexer_link Cpp_Relex_State
cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack,
cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack,
int start, int end, int amount, int tolerance){
Cpp_Relex_State state;
state.file = file;
state.data = data;
state.size = size;
state.stack = stack;
state.start = start;
state.end = end;
@ -1107,7 +1108,7 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state,
lex.pos = state->relex_start;
int relex_end_i = state->end_token_i;
Cpp_Token match_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i);
Cpp_Token match_token = cpp__get_token(stack, tokens, state->size, relex_end_i);
Cpp_Token end_token = match_token;
int went_too_far = false;
@ -1115,9 +1116,9 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state,
for (;;){
int result =
cpp_lex_size_nonalloc(&lex,
state->file.data,
state->file.size,
state->file.size,
state->data,
state->size,
state->size,
relex_stack, 1);
switch (result){
@ -1134,7 +1135,7 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state,
while (lex.pos > end_token.start && relex_end_i < stack->count){
++relex_end_i;
end_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i);
end_token = cpp__get_token(stack, tokens, state->size, relex_end_i);
}
}
break;

74
test/experiment.cpp
View File

@ -13,7 +13,7 @@
// TOP
#include "../4ed_meta.h"
#define FCPP_STRING_IMPLEMENTATION
#define FSTRING_IMPLEMENTATION
#include "../4coder_string.h"
#include "../4cpp_types.h"
@ -94,13 +94,12 @@ system_set_file_list(File_List *file_list, String directory){
append(&dir, directory);
char trail_str[] = "\\*";
append(&dir, trail_str);
char *c_str_dir = make_c_str(dir);
terminate_with_null(&dir);
WIN32_FIND_DATA find_data;
HANDLE search;
search = FindFirstFileA(c_str_dir, &find_data);
search = FindFirstFileA(dir.str, &find_data);
if (search != INVALID_HANDLE_VALUE){
i32 count = 0;
i32 file_count = 0;
@ -116,19 +115,19 @@ system_set_file_list(File_List *file_list, String directory){
more_files = FindNextFile(search, &find_data);
}while(more_files);
FindClose(search);
i32 required_size = count + file_count * sizeof(File_Info);
if (file_list->block_size < required_size){
Win32FreeMemory(file_list->block);
file_list->block = Win32GetMemory(required_size);
file_list->block_size = required_size;
}
file_list->infos = (File_Info*)file_list->block;
char *name = (char*)(file_list->infos + file_count);
if (file_list->block){
search = FindFirstFileA(c_str_dir, &find_data);
search = FindFirstFileA(dir.str, &find_data);
if (search != INVALID_HANDLE_VALUE){
File_Info *info = file_list->infos;
more_files = 1;
@ -137,21 +136,21 @@ system_set_file_list(File_List *file_list, String directory){
!match(find_data.cFileName, "..")){
info->folder = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
info->filename.str = name;
i32 i = 0;
for(;find_data.cFileName[i];++i) *name++ = find_data.cFileName[i];
info->filename.size = i;
info->filename.memory_size = info->filename.size + 1;
*name++ = 0;
replace_char(info->filename, '\\', '/');
replace_char(&info->filename, '\\', '/');
++info;
}
more_files = FindNextFile(search, &find_data);
}while(more_files);
FindClose(search);
file_list->count = file_count;
}else{
Win32FreeMemory(file_list->block);
file_list->block = 0;
@ -212,7 +211,6 @@ run_experiment(Experiment *exp, char *filename, int verbose,
int chunks, int max_tokens){
String extension = {};
Data file_data;
Cpp_File file_cpp;
new_lex::Lex_Data ld = {0};
int pass;
int k, chunk_size, is_last;
@ -232,8 +230,8 @@ run_experiment(Experiment *exp, char *filename, int verbose,
memset(exp->correct_stack.tokens, 0, TOKEN_ARRAY_SIZE);
memset(exp->testing_stack.tokens, 0, TOKEN_ARRAY_SIZE);
file_cpp.data = (char*)file_data.data;
file_cpp.size = file_data.size;
char *data = (char*)file_data.data;
int size = file_data.size;
ld.tb = (char*)malloc(file_data.size + 1);
@ -241,7 +239,7 @@ run_experiment(Experiment *exp, char *filename, int verbose,
i64 start;
start = __rdtsc();
cpp_lex_file_nonalloc(file_cpp, &exp->correct_stack, lex_data);
cpp_lex_file_nonalloc(data, size, &exp->correct_stack, lex_data);
time.handcoded += (__rdtsc() - start);
if (max_tokens == 0){
@ -357,8 +355,8 @@ run_experiment(Experiment *exp, char *filename, int verbose,
" %.*s original %.*s testing\n",
j,
correct->start, correct->size, testing->start, testing->size,
correct->size, file_cpp.data + correct->start,
testing->size, file_cpp.data + testing->start);
correct->size, data + correct->start,
testing->size, data + testing->start);
}
}
@ -389,15 +387,15 @@ show_time(Times t, int repeats, char *type){
f32 speed_up = ((f32)t.handcoded) / (t.fsm);
printf(
"\n%s time for %d repeates\n"
OUTLINE("%lld")
OUTLINE("%lld")
OUTLINE("%f"),
OUTLINE("%lld")
OUTLINE("%lld")
OUTLINE("%f"),
type,
repeats,
OUTLINE_VAR(i64, t.handcoded),
OUTLINE_VAR(i64, t.fsm),
OUTLINE_VAR(f32, speed_up)
);
);
}
#define BASE_DIR "w:/4ed/data/test/"
@ -413,27 +411,27 @@ int main(){
int chunks = (chunk_start > 0 && chunk_start <= chunk_end);
int c = 0;
char test_directory[] = BASE_DIR;
File_List all_files = {};
Experiment exp = {};
Experiment chunk_exp = {};
Times exp_t = {};
Times chunk_exp_t = {};
init_test_stack(&exp.correct_stack);
init_test_stack(&exp.testing_stack);
init_test_stack(&chunk_exp.correct_stack);
init_test_stack(&chunk_exp.testing_stack);
AllowLocal(test_directory);
AllowLocal(all_files);
#if SINGLE_ITEM
(void)(repeats);
(void)(verbose_level);
if (chunks){
begin_t(&chunk_exp_t);
printf("With chunks of %d\n", chunks);
@ -442,16 +440,16 @@ int main(){
}
end_t(&chunk_exp_t);
}
begin_t(&exp_t);
printf("Unchunked\n");
run_experiment(&exp, BASE_DIR TEST_FILE, 1, 0, token_limit);
end_t(&exp_t);
#else
system_set_file_list(&all_files, make_lit_string(test_directory));
for (int j = 0; j < repeats; ++j){
for (int i = 0; i < all_files.count; ++i){
if (all_files.infos[i].folder == 0){
@ -477,21 +475,21 @@ int main(){
}
}
#endif
if (chunks){
printf("chunks of sizes %d through %d tested\n", chunk_start, chunk_end);
printf("chunked passed %d / %d tests\n", chunk_exp.passed_total, chunk_exp.test_total);
}
printf("unchunk passed %d / %d tests\n", exp.passed_total, exp.test_total);
if (passed(exp) && (chunks == 0 || passed(chunk_exp))){
if (chunks){
show_time(chunk_exp_t, repeats, "Chunked");
}
show_time(exp_t, repeats, "Unchunked");
}
return(0);
}