diff --git a/4coder_default_include.cpp b/4coder_default_include.cpp index d07e2b86..e574cf10 100644 --- a/4coder_default_include.cpp +++ b/4coder_default_include.cpp @@ -1819,11 +1819,11 @@ CUSTOM_COMMAND_SIG(open_all_code){ // was originally, so that new appends overwrite old ones. dir.size = dir_size; append(&dir, info->filename); - exec_command(app, cmdid_interactive_open); + app->create_buffer(app, dir.str, dir.size, 0); } } } - + app->free_file_list(app, list); } diff --git a/4cpp_lexer.h b/4cpp_lexer.h index da136e44..486c587f 100644 --- a/4cpp_lexer.h +++ b/4cpp_lexer.h @@ -78,14 +78,6 @@ struct Cpp_Read_Result{ char has_result; }; -Cpp_File -data_as_cpp_file(Data data){ - Cpp_File result; - result.data = (char*)data.data; - result.size = data.size; - return(result); -} - // TODO(allen): revisit this keyword data declaration system struct String_And_Flag{ char *str; @@ -125,27 +117,27 @@ is_keyword(Cpp_Token_Type type){ return (type >= CPP_TOKEN_KEY_TYPE && type <= CPP_TOKEN_KEY_OTHER); } -FCPP_LINK Sub_Match_List_Result sub_match_list(Cpp_File file, int pos, String_List list, int sub_size); +FCPP_LINK Sub_Match_List_Result sub_match_list(char *data, int size, int pos, String_List list, int sub_size); FCPP_LINK Seek_Result seek_unescaped_eol(char *data, int size, int pos); FCPP_LINK Seek_Result seek_unescaped_delim(char *data, int size, int pos, char delim); FCPP_LINK Seek_Result seek_block_comment_end(char *data, int size, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_whitespace(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_junk_line(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_operator(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_pp_operator(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body); -inline Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos) { return cpp_read_alpha_numeric(file, pos, 0); } -FCPP_LINK Cpp_Read_Result cpp_read_number(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_string_litteral(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_character_litteral(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_line_comment(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_block_comment(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_preprocessor(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_pp_include_file(Cpp_File file, int pos); -FCPP_LINK Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body); -inline Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos) { return cpp_read_pp_default_mode(file, pos, 0); } +FCPP_LINK Cpp_Read_Result cpp_read_whitespace(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_junk_line(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_operator(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_pp_operator(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_alpha_numeric(char *data, int size, int pos, bool in_if_body); +inline Cpp_Read_Result cpp_read_alpha_numeric(char *data, int size, int pos) { return cpp_read_alpha_numeric(data, size, pos, 0); } +FCPP_LINK Cpp_Read_Result cpp_read_number(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_string_litteral(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_character_litteral(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_line_comment(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_block_comment(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_preprocessor(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_pp_include_file(char *data, int size, int pos); +FCPP_LINK Cpp_Read_Result cpp_read_pp_default_mode(char *data, int size, int pos, bool in_if_body); +inline Cpp_Read_Result cpp_read_pp_default_mode(char *data, int size, int pos) { return cpp_read_pp_default_mode(data, size, pos, 0); } FCPP_LINK Cpp_Token_Merge cpp_attempt_token_merge(Cpp_Token prev, Cpp_Token next); @@ -154,18 +146,18 @@ FCPP_LINK bool cpp_push_token_nonalloc(Cpp_Token_Stack *stack, Cpp_Token token); inline Cpp_Lex_Data cpp_lex_data_zero() { Cpp_Lex_Data data = {(Cpp_Preprocessor_State)0}; return(data); } -FCPP_LINK Cpp_Read_Result cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex); +FCPP_LINK Cpp_Read_Result cpp_lex_step(char *data, int size, Cpp_Lex_Data *lex); -FCPP_LINK int cpp_lex_file_token_count(Cpp_File file); -FCPP_LINK Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack, Cpp_Lex_Data data); -inline Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack) { return cpp_lex_file_nonalloc(file, stack, cpp_lex_data_zero()); } +FCPP_LINK int cpp_lex_file_token_count(char *data, int size); +FCPP_LINK Cpp_Lex_Data cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *stack, Cpp_Lex_Data lex_data); +inline Cpp_Lex_Data cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *stack) { return cpp_lex_file_nonalloc(data, size, stack, cpp_lex_data_zero()); } FCPP_LINK Cpp_Get_Token_Result cpp_get_token(Cpp_Token_Stack *stack, int pos); FCPP_LINK int cpp_get_end_token(Cpp_Token_Stack *stack, int end); FCPP_LINK void cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token, int amount); -FCPP_LINK Cpp_Relex_State cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance); +FCPP_LINK Cpp_Relex_State cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance); FCPP_LINK bool cpp_relex_nonalloc_main(Cpp_Relex_State state, Cpp_Token_Stack *stack); #ifndef FCPP_FORBID_MALLOC @@ -174,10 +166,10 @@ FCPP_LINK void cpp_free_token_stack(Cpp_Token_Stack stack); FCPP_LINK void cpp_resize_token_stack(Cpp_Token_Stack *stack, int new_max); FCPP_LINK void cpp_push_token(Cpp_Token_Stack *stack, Cpp_Token token); -FCPP_LINK void cpp_lex_file(Cpp_File file, Cpp_Token_Stack *stack); -FCPP_LINK bool cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount, int extra_tolerance); -inline void cpp_relex_file(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount) -{ cpp_relex_file_limited(file, stack, start_i, end_i, amount, -1); } +FCPP_LINK void cpp_lex_file(char *data, int size, Cpp_Token_Stack *stack); +FCPP_LINK bool cpp_relex_file_limited(char *data, int size, Cpp_Token_Stack *stack, int start_i, int end_i, int amount, int extra_tolerance); +inline void cpp_relex_file(char *data, int size, Cpp_Token_Stack *stack, int start_i, int end_i, int amount) +{ cpp_relex_file_limited(data, size, stack, start_i, end_i, amount, -1); } #endif #define FCPP_STRING_LIST(x) {x, FCPP_COUNT(x)} @@ -393,7 +385,7 @@ FCPP_GLOBAL String_List preprops = FCPP_STRING_LIST(preprop_strings); #define _TentativeAssert FCPP_ASSERT FCPP_LINK Sub_Match_List_Result -sub_match_list(Cpp_File file, int pos, String_List list, int sub_size){ +sub_match_list(char *data, int size, int pos, String_List list, int sub_size){ Sub_Match_List_Result result; String str_main; char *str_check; @@ -401,7 +393,7 @@ sub_match_list(Cpp_File file, int pos, String_List list, int sub_size){ result.index = -1; result.new_pos = pos; - str_main = make_string(file.data + pos, file.size - pos); + str_main = make_string(data + pos, size - pos); if (sub_size > 0){ str_main = substr(str_main, 0, sub_size); for (i = 0; i < list.count; ++i){ @@ -505,11 +497,11 @@ seek_block_comment_end(char *data, int size, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_whitespace(Cpp_File file, int pos){ +cpp_read_whitespace(char *data, int size, int pos){ Cpp_Read_Result result = {}; - while (pos < file.size && char_is_whitespace(file.data[pos])){ - if (file.data[pos] == '\n'){ + while (pos < size && char_is_whitespace(data[pos])){ + if (data[pos] == '\n'){ result.newline = 1; } ++pos; @@ -521,16 +513,16 @@ cpp_read_whitespace(Cpp_File file, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_junk_line(Cpp_File file, int pos){ +cpp_read_junk_line(char *data, int size, int pos){ Cpp_Read_Result result = {}; result.token.start = pos; result.token.type = CPP_TOKEN_JUNK; bool comment_end = 0; - while (pos < file.size && file.data[pos] != '\n'){ - if (file.data[pos] == '/' && pos + 1 < file.size){ - if (file.data[pos + 1] == '/' || - file.data[pos + 1] == '*'){ + while (pos < size && data[pos] != '\n'){ + if (data[pos] == '/' && pos + 1 < size){ + if (data[pos + 1] == '/' || + data[pos + 1] == '*'){ comment_end = 1; break; } @@ -543,10 +535,10 @@ cpp_read_junk_line(Cpp_File file, int pos){ result.token.size = pos - result.token.start; } else{ - while (pos > 0 && file.data[pos - 1] == '\r'){ + while (pos > 0 && data[pos - 1] == '\r'){ --pos; } - if (pos > 0 && file.data[pos - 1] == '\\'){ + if (pos > 0 && data[pos - 1] == '\\'){ --pos; } result.pos = pos; @@ -557,13 +549,13 @@ cpp_read_junk_line(Cpp_File file, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_operator(Cpp_File file, int pos){ +cpp_read_operator(char *data, int size, int pos){ Cpp_Read_Result result = {}; result.pos = pos; result.token.start = pos; Sub_Match_List_Result match; - match = sub_match_list(file, result.token.start, ops, -1); + match = sub_match_list(data, size, result.token.start, ops, -1); if (match.index != -1){ result.pos = match.new_pos; @@ -581,13 +573,13 @@ cpp_read_operator(Cpp_File file, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_pp_operator(Cpp_File file, int pos){ +cpp_read_pp_operator(char *data, int size, int pos){ Cpp_Read_Result result = {}; result.pos = pos; result.token.start = pos; Sub_Match_List_Result match; - match = sub_match_list(file, result.token.start, pp_ops, -1); + match = sub_match_list(data, size, result.token.start, pp_ops, -1); _Assert(match.index != -1); result.pos = match.new_pos; @@ -598,13 +590,13 @@ cpp_read_pp_operator(Cpp_File file, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){ +cpp_read_alpha_numeric(char *data, int size, int pos, bool in_if_body){ Cpp_Read_Result result = {}; result.pos = pos; result.token.start = pos; - while (result.pos < file.size && - char_is_alpha_numeric(file.data[result.pos])){ + while (result.pos < size && + char_is_alpha_numeric(data[result.pos])){ ++result.pos; } @@ -614,7 +606,7 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){ if (in_if_body){ String word; word.size = result.token.size; - word.str = file.data + result.token.start; + word.str = data + result.token.start; if (match(word, "defined")){ result.token.type = CPP_TOKEN_DEFINED; result.token.flags |= CPP_TFLAG_IS_OPERATOR; @@ -624,14 +616,14 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){ if (result.token.type == CPP_TOKEN_JUNK){ Sub_Match_List_Result match; - match = sub_match_list(file, result.token.start, bool_lits, result.token.size); + match = sub_match_list(data, size, result.token.start, bool_lits, result.token.size); if (match.index != -1){ result.token.type = CPP_TOKEN_BOOLEAN_CONSTANT; result.token.flags |= CPP_TFLAG_IS_KEYWORD; } else{ - match = sub_match_list(file, result.token.start, keywords, result.token.size); + match = sub_match_list(data, size, result.token.start, keywords, result.token.size); if (match.index != -1){ String_And_Flag data = keywords.data[match.index]; @@ -648,7 +640,7 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){ } FCPP_LINK Cpp_Read_Result -cpp_read_number(Cpp_File file, int pos){ +cpp_read_number(char *data, int size, int pos){ Cpp_Read_Result result = {}; result.pos = pos; result.token.start = pos; @@ -659,9 +651,9 @@ cpp_read_number(Cpp_File file, int pos){ bool is_hex = 0; bool is_zero = 0; - if (file.data[pos] == '0'){ - if (pos+1 < file.size){ - char next = file.data[pos+1]; + if (data[pos] == '0'){ + if (pos+1 < size){ + char next = data[pos+1]; if (next == 'x'){ is_hex = 1; is_integer = 1; @@ -684,7 +676,7 @@ cpp_read_number(Cpp_File file, int pos){ is_integer = 1; } } - else if (file.data[pos] == '.'){ + else if (data[pos] == '.'){ is_float = 1; } @@ -696,20 +688,20 @@ cpp_read_number(Cpp_File file, int pos){ char character; do{ ++result.pos; - if (result.pos >= file.size){ + if (result.pos >= size){ break; } - character = file.data[result.pos]; + character = data[result.pos]; } while(char_is_hex(character)); } else if (is_oct){ char character; do{ ++result.pos; - if (result.pos >= file.size){ + if (result.pos >= size){ break; } - character = file.data[result.pos]; + character = data[result.pos]; }while(char_is_numeric(character)); } else{ @@ -718,11 +710,11 @@ cpp_read_number(Cpp_File file, int pos){ while (1){ ++result.pos; - if (result.pos >= file.size){ + if (result.pos >= size){ break; } bool is_good = 0; - char character = file.data[result.pos]; + char character = data[result.pos]; if (character >= '0' && character <= '9'){ is_good = 1; } @@ -744,11 +736,11 @@ cpp_read_number(Cpp_File file, int pos){ while (1){ ++result.pos; - if (result.pos >= file.size){ + if (result.pos >= size){ break; } is_good = 0; - character = file.data[result.pos]; + character = data[result.pos]; if (character >= '0' && character <= '9'){ is_good = 1; } @@ -771,7 +763,7 @@ cpp_read_number(Cpp_File file, int pos){ if (is_integer){ Sub_Match_List_Result match = - sub_match_list(file, result.pos, int_sufs, -1); + sub_match_list(data, size, result.pos, int_sufs, -1); if (match.index != -1){ result.pos = match.new_pos; } @@ -780,7 +772,7 @@ cpp_read_number(Cpp_File file, int pos){ } else if (is_float){ Sub_Match_List_Result match = - sub_match_list(file, result.pos, float_sufs, -1); + sub_match_list(data, size, result.pos, float_sufs, -1); if (match.index != -1){ result.pos = match.new_pos; } @@ -795,69 +787,69 @@ cpp_read_number(Cpp_File file, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_string_litteral(Cpp_File file, int pos){ +cpp_read_string_litteral(char *data, int size, int pos){ Cpp_Read_Result result = {}; result.token.start = pos; - - _Assert(file.data[pos] == '"'); - Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '"'); + + _Assert(data[pos] == '"'); + Seek_Result seek = seek_unescaped_delim(data, size, pos, '"'); pos = seek.pos; if (seek.new_line){ result.token.flags |= CPP_TFLAG_MULTILINE; } - + result.token.size = pos - result.token.start; result.token.type = CPP_TOKEN_STRING_CONSTANT; result.pos = pos; - + return result; } FCPP_LINK Cpp_Read_Result -cpp_read_character_litteral(Cpp_File file, int pos){ +cpp_read_character_litteral(char *data, int size, int pos){ Cpp_Read_Result result = {}; result.token.start = pos; - - _Assert(file.data[pos] == '\''); - Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '\''); + + _Assert(data[pos] == '\''); + Seek_Result seek = seek_unescaped_delim(data, size, pos, '\''); pos = seek.pos; if (seek.new_line){ result.token.flags |= CPP_TFLAG_MULTILINE; } - + result.token.size = pos - result.token.start; result.token.type = CPP_TOKEN_CHARACTER_CONSTANT; result.pos = pos; - + return result; } FCPP_LINK Cpp_Read_Result -cpp_read_line_comment(Cpp_File file, int pos){ +cpp_read_line_comment(char *data, int size, int pos){ Cpp_Read_Result result = {}; result.token.start = pos; - - _Assert(file.data[pos] == '/' && file.data[pos + 1] == '/'); - + + _Assert(data[pos] == '/' && data[pos + 1] == '/'); + pos += 2; - while (pos < file.size){ - if (file.data[pos] == '\n'){ + while (pos < size){ + if (data[pos] == '\n'){ break; } - if (file.data[pos] == '\\'){ - if (pos + 1 < file.size && - file.data[pos + 1] == '\n'){ + if (data[pos] == '\\'){ + if (pos + 1 < size && + data[pos + 1] == '\n'){ ++pos; } - else if (pos + 2 < file.size && - file.data[pos + 1] == '\r' && - file.data[pos + 2] == '\n'){ + else if (pos + 2 < size && + data[pos + 1] == '\r' && + data[pos + 2] == '\n'){ pos += 2; } } ++pos; } - if (pos > 0 && file.data[pos-1] == '\r'){ + if (pos > 0 && data[pos-1] == '\r'){ --pos; } result.token.size = pos - result.token.start; @@ -867,16 +859,16 @@ cpp_read_line_comment(Cpp_File file, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_block_comment(Cpp_File file, int pos){ +cpp_read_block_comment(char *data, int size, int pos){ Cpp_Read_Result result = {}; result.token.start = pos; - _Assert(file.data[pos] == '/' && file.data[pos + 1] == '*'); + _Assert(data[pos] == '/' && data[pos + 1] == '*'); pos += 2; - while (pos < file.size){ - if (file.data[pos] == '*' && - pos + 1 < file.size && - file.data[pos+1] == '/'){ + while (pos < size){ + if (data[pos] == '*' && + pos + 1 < size && + data[pos+1] == '/'){ break; } ++pos; @@ -889,43 +881,42 @@ cpp_read_block_comment(Cpp_File file, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_preprocessor(Cpp_File file, int pos){ - _Assert(file.data[pos] == '#'); +cpp_read_preprocessor(char *data, int size, int pos){ + _Assert(data[pos] == '#'); Cpp_Read_Result result = {}; result.token.start = pos; result.token.type = CPP_PP_UNKNOWN; result.token.flags |= CPP_TFLAG_PP_DIRECTIVE; - + ++pos; - while (pos < file.size && - (file.data[pos] == ' ' || - file.data[pos] == '\t')){ + while (pos < size && + (data[pos] == ' ' || + data[pos] == '\t')){ ++pos; } - + Sub_Match_List_Result match - = sub_match_list(file, pos, preprops, -1); - + = sub_match_list(data, size, pos, preprops, -1); + if (match.index != -1){ result.token.size = match.new_pos - result.token.start; result.token.type = (Cpp_Token_Type)preprops.data[match.index].flags; result.pos = match.new_pos; } else{ - while (pos < file.size && - !char_is_whitespace(file.data[pos])){ + while (pos < size && !char_is_whitespace(data[pos])){ ++pos; } result.token.size = pos - result.token.start; result.pos = pos; } - + return result; } FCPP_LINK Cpp_Read_Result -cpp_read_pp_include_file(Cpp_File file, int pos){ - char start = file.data[pos]; +cpp_read_pp_include_file(char *data, int size, int pos){ + char start = data[pos]; _Assert(start == '<' || start == '"'); Cpp_Read_Result result = {}; @@ -942,22 +933,20 @@ cpp_read_pp_include_file(Cpp_File file, int pos){ } ++pos; - while (pos < file.size && file.data[pos] != end){ - if (file.data[pos] == '\n'){ + while (pos < size && data[pos] != end){ + if (data[pos] == '\n'){ result.token.type = CPP_TOKEN_JUNK; result.token.flags |= CPP_TFLAG_BAD_ENDING; break; } - if (file.data[pos] == '\\'){ - // TODO(allen): Not sure that this is 100% correct. - if (pos + 1 < file.size && - file.data[pos + 1] == '\n'){ + if (data[pos] == '\\'){ + if (pos + 1 < size && data[pos + 1] == '\n'){ ++pos; result.token.flags |= CPP_TFLAG_MULTILINE; } - else if (pos + 2 < file.size && - file.data[pos + 1] == '\r' && - file.data[pos + 2] == '\n'){ + else if (pos + 2 < size && + data[pos + 1] == '\r' && + data[pos + 2] == '\n'){ pos += 2; result.token.flags |= CPP_TFLAG_MULTILINE; } @@ -966,7 +955,7 @@ cpp_read_pp_include_file(Cpp_File file, int pos){ } if (result.token.type != CPP_TOKEN_JUNK){ - if (pos < file.size){ + if (pos < size){ ++pos; } } @@ -978,55 +967,55 @@ cpp_read_pp_include_file(Cpp_File file, int pos){ } FCPP_LINK Cpp_Read_Result -cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body){ - char current = file.data[pos]; +cpp_read_pp_default_mode(char *data, int size, int pos, bool in_if_body){ + char current = data[pos]; Cpp_Read_Result result; if (char_is_numeric(current)){ - result = cpp_read_number(file, pos); + result = cpp_read_number(data, size, pos); } else if (char_is_alpha(current)){ - result = cpp_read_alpha_numeric(file, pos, in_if_body); + result = cpp_read_alpha_numeric(data, size, pos, in_if_body); } else if (current == '.'){ - if (pos + 1 < file.size){ - char next = file.data[pos + 1]; + if (pos + 1 < size){ + char next = data[pos + 1]; if (char_is_numeric(next)){ - result = cpp_read_number(file, pos); + result = cpp_read_number(data, size, pos); } else{ - result = cpp_read_operator(file, pos); + result = cpp_read_operator(data, size, pos); } } else{ - result = cpp_read_operator(file, pos); + result = cpp_read_operator(data, size, pos); } } else if (current == '/'){ - if (pos + 1 < file.size){ - char next = file.data[pos + 1]; + if (pos + 1 < size){ + char next = data[pos + 1]; if (next == '/'){ - result = cpp_read_line_comment(file, pos); + result = cpp_read_line_comment(data, size, pos); } else if (next == '*'){ - result = cpp_read_block_comment(file, pos); + result = cpp_read_block_comment(data, size, pos); } else{ - result = cpp_read_operator(file, pos); + result = cpp_read_operator(data, size, pos); } } else{ - result = cpp_read_operator(file, pos); + result = cpp_read_operator(data, size, pos); } } else if (current == '"'){ - result = cpp_read_string_litteral(file, pos); + result = cpp_read_string_litteral(data, size, pos); } else if (current == '\''){ - result = cpp_read_character_litteral(file, pos); + result = cpp_read_character_litteral(data, size, pos); } else{ - result = cpp_read_operator(file, pos); + result = cpp_read_operator(data, size, pos); } return result; @@ -1084,16 +1073,16 @@ cpp_push_token_nonalloc(Cpp_Token_Stack *token_stack, Cpp_Token token){ } FCPP_LINK Cpp_Read_Result -cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ +cpp_lex_step(char *data, int size, Cpp_Lex_Data *lex_data){ Cpp_Lex_Data lex = *lex_data; Cpp_Read_Result result = {}; bool has_result = 1; fcpp_u16 state_flags = cpp_token_set_pp_state(0, lex.pp_state); - char current = file.data[lex.pos]; + char current = data[lex.pos]; if (char_is_whitespace(current)){ - result = cpp_read_whitespace(file, lex.pos); + result = cpp_read_whitespace(data, size, lex.pos); lex.pos = result.pos; if (result.newline && lex.pp_state != CPP_LEX_PP_DEFAULT){ lex.pp_state = CPP_LEX_PP_DEFAULT; @@ -1105,7 +1094,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ if (lex.pp_state == CPP_LEX_PP_DEFAULT){ // TODO(allen): Not first hard of the line? Then it's junk. if (current == '#'){ - result = cpp_read_preprocessor(file, lex.pos); + result = cpp_read_preprocessor(data, size, lex.pos); lex.pos = result.pos; switch (result.token.type){ case CPP_PP_INCLUDE: @@ -1143,7 +1132,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ } } else{ - result = cpp_read_pp_default_mode(file, lex.pos); + result = cpp_read_pp_default_mode(data, size, lex.pos); lex.pos = result.pos; } } @@ -1152,10 +1141,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ if (current == '\\'){ fcpp_i32 seek = lex.pos; ++seek; - while (seek < file.size && file.data[seek] == '\r'){ + while (seek < size && data[seek] == '\r'){ ++seek; } - if ((seek < file.size && file.data[seek] == '\n') || seek >= file.size){ + if ((seek < size && data[seek] == '\n') || seek >= size){ lex.pos = seek + 1; has_result = 0; } @@ -1177,7 +1166,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ lex.pp_state = CPP_LEX_PP_JUNK; } else{ - result = cpp_read_alpha_numeric(file, lex.pos); + result = cpp_read_alpha_numeric(data, size, lex.pos); result.token.flags |= CPP_TFLAG_PP_BODY; lex.pos = result.pos; lex.pp_state = CPP_LEX_PP_JUNK; @@ -1190,7 +1179,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ lex.pp_state = CPP_LEX_PP_JUNK; } else{ - result = cpp_read_alpha_numeric(file, lex.pos); + result = cpp_read_alpha_numeric(data, size, lex.pos); result.token.flags |= CPP_TFLAG_PP_BODY; lex.pos = result.pos; lex.pp_state = CPP_LEX_PP_BODY; @@ -1203,7 +1192,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ lex.pp_state = CPP_LEX_PP_JUNK; } else{ - result = cpp_read_pp_include_file(file, lex.pos); + result = cpp_read_pp_include_file(data, size, lex.pos); lex.pos = result.pos; lex.pp_state = CPP_LEX_PP_JUNK; } @@ -1211,10 +1200,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ case CPP_LEX_PP_BODY: if (current == '#'){ - result = cpp_read_pp_operator(file, lex.pos); + result = cpp_read_pp_operator(data, size, lex.pos); } else{ - result = cpp_read_pp_default_mode(file, lex.pos); + result = cpp_read_pp_default_mode(data, size, lex.pos); } lex.pos = result.pos; result.token.flags |= CPP_TFLAG_PP_BODY; @@ -1222,10 +1211,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ case CPP_LEX_PP_BODY_IF: if (current == '#'){ - result = cpp_read_pp_operator(file, lex.pos); + result = cpp_read_pp_operator(data, size, lex.pos); } else{ - result = cpp_read_pp_default_mode(file, lex.pos, 1); + result = cpp_read_pp_default_mode(data, size, lex.pos, 1); } lex.pos = result.pos; result.token.flags |= CPP_TFLAG_PP_BODY; @@ -1237,7 +1226,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ lex.pp_state = CPP_LEX_PP_JUNK; } else{ - result = cpp_read_number(file, lex.pos); + result = cpp_read_number(data, size, lex.pos); lex.pos = result.pos; result.token.flags |= CPP_TFLAG_PP_BODY; lex.pp_state = CPP_LEX_PP_INCLUDE; @@ -1245,7 +1234,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ break; case CPP_LEX_PP_ERROR: - result = cpp_read_junk_line(file, lex.pos); + result = cpp_read_junk_line(data, size, lex.pos); lex.pos = result.pos; result.token.type = CPP_TOKEN_ERROR_MESSAGE; result.token.flags |= CPP_TFLAG_PP_BODY; @@ -1254,21 +1243,21 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ default: { bool took_comment = 0; - if (current == '/' && lex.pos + 1 < file.size){ - if (file.data[lex.pos + 1] == '/'){ - result = cpp_read_line_comment(file, lex.pos); + if (current == '/' && lex.pos + 1 < size){ + if (data[lex.pos + 1] == '/'){ + result = cpp_read_line_comment(data, size, lex.pos); lex.pp_state = CPP_LEX_PP_DEFAULT; lex.pos = result.pos; took_comment = 1; - }else if (file.data[lex.pos + 1] == '*'){ - result = cpp_read_block_comment(file, lex.pos); + }else if (data[lex.pos + 1] == '*'){ + result = cpp_read_block_comment(data, size, lex.pos); lex.pos = result.pos; took_comment = 1; } } if (!took_comment){ - result = cpp_read_junk_line(file, lex.pos); + result = cpp_read_junk_line(data, size, lex.pos); lex.pos = result.pos; result.token.flags |= CPP_TFLAG_PP_BODY; } @@ -1287,12 +1276,12 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ } FCPP_LINK int -cpp_lex_file_token_count(Cpp_File file){ +cpp_lex_file_token_count(char *data, int size){ int count = 0; Cpp_Lex_Data lex = {}; Cpp_Token token = {}; - while (lex.pos < file.size){ - Cpp_Read_Result step_result = cpp_lex_step(file, &lex); + while (lex.pos < size){ + Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex); if (step_result.has_result){ if (count > 0){ @@ -1315,21 +1304,21 @@ cpp_lex_file_token_count(Cpp_File file){ } FCPP_LINK Cpp_Lex_Data -cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *token_stack_out, Cpp_Lex_Data data){ - while (data.pos < file.size){ - Cpp_Lex_Data prev_lex = data; - Cpp_Read_Result step_result = cpp_lex_step(file, &data); +cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *token_stack_out, Cpp_Lex_Data lex_data){ + while (lex_data.pos < size){ + Cpp_Lex_Data prev_lex = lex_data; + Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex_data); if (step_result.has_result){ if (!cpp_push_token_nonalloc(token_stack_out, step_result.token)){ - data = prev_lex; - return data; + lex_data = prev_lex; + return lex_data; } } } - data.complete = 1; - return data; + lex_data.complete = 1; + return lex_data; } FCPP_LINK Cpp_Get_Token_Result @@ -1395,10 +1384,11 @@ cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token_i, int amount){ } FCPP_LINK Cpp_Relex_State -cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, +cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance){ Cpp_Relex_State state; - state.file = file; + state.data = data; + state.size = size; state.stack = stack; state.start = start; state.end = end; @@ -1460,12 +1450,12 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, Cpp_Token_Stack *relex_stack, in lex.pos = state->relex_start; int relex_end_i = state->end_token_i; - Cpp_Token match_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); + Cpp_Token match_token = cpp__get_token(stack, tokens, state->size, relex_end_i); Cpp_Token end_token = match_token; bool went_too_far = 0; for (;;){ - Cpp_Read_Result read = cpp_lex_step(state->file, &lex); + Cpp_Read_Result read = cpp_lex_step(state->data, state->size, &lex); if (read.has_result){ if (read.token.start == end_token.start && read.token.size == end_token.size && @@ -1477,14 +1467,14 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, Cpp_Token_Stack *relex_stack, in while (lex.pos > end_token.start && relex_end_i < stack->count){ ++relex_end_i; - end_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); + end_token = cpp__get_token(stack, tokens, state->size, relex_end_i); } if (relex_stack->count == relex_stack->max_count){ went_too_far = 1; break; } } - if (lex.pos >= state->file.size) break; + if (lex.pos >= state->size) break; } if (!went_too_far){ @@ -1557,10 +1547,10 @@ cpp_push_token(Cpp_Token_Stack *token_stack, Cpp_Token token){ } FCPP_LINK void -cpp_lex_file(Cpp_File file, Cpp_Token_Stack *token_stack_out){ +cpp_lex_file(char *data, int size, Cpp_Token_Stack *token_stack_out){ Cpp_Lex_Data lex = {}; - while (lex.pos < file.size){ - Cpp_Read_Result step_result = cpp_lex_step(file, &lex); + while (lex.pos < size){ + Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex); if (step_result.has_result){ cpp_push_token(token_stack_out, step_result.token); } @@ -1568,7 +1558,7 @@ cpp_lex_file(Cpp_File file, Cpp_Token_Stack *token_stack_out){ } FCPP_LINK bool -cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, +cpp_relex_file_limited(char *data, int size, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance){ #if 0 int start_token_i, end_token_i; @@ -1583,18 +1573,18 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, ++end_token_i; } cpp_shift_token_starts(token_stack, end_token_i, amount); - + int relex_start_i = start_token_i - 1; if (relex_start_i < 0){ relex_start_i = 0; } - + int end_guess_i = end_token_i + 1; if (end_guess_i > token_stack->count){ --end_guess_i; } #endif - + int relex_start_i; int end_token_i, end_guess_i; { @@ -1605,26 +1595,26 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, else{ relex_start_i = result.token_index-1; } - + result = cpp_get_token(stack, end); if (result.token_index < 0) result.token_index = 0; else if (end > stack->tokens[result.token_index].start) ++result.token_index; end_token_i = result.token_index; end_guess_i = result.token_index+1; } - + int relex_start = stack->tokens[relex_start_i].start; if (start < relex_start) relex_start = start; - + cpp_shift_token_starts(stack, end_token_i, amount); Cpp_Token_Stack relex_stack = cpp_make_token_stack((end_guess_i - relex_start_i + 1) * 3 / 2); Cpp_Lex_Data lex = {}; lex.pp_state = cpp_token_get_pp_state(stack->tokens[relex_start_i].state_flags); lex.pos = relex_start; bool went_too_far = 0; - + while (1){ - Cpp_Read_Result result = cpp_lex_step(file, &lex); + Cpp_Read_Result result = cpp_lex_step(data, size, &lex); if (result.has_result){ if (end_guess_i < stack->count && result.token.start == stack->tokens[end_guess_i].start && @@ -1641,20 +1631,20 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, } } } - - if (lex.pos >= file.size){ + + if (lex.pos >= size){ break; } - + if (tolerance >= 0 && relex_stack.count + relex_start_i >= end_guess_i + tolerance){ went_too_far = 1; break; } } - + if (!went_too_far){ int relex_end_i = end_guess_i; - + if (relex_stack.count > 0){ if (relex_start_i > 0){ Cpp_Token_Merge merge = cpp_attempt_token_merge(stack->tokens[relex_start_i - 1], @@ -1664,7 +1654,7 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, relex_stack.tokens[0] = merge.new_token; } } - + if (relex_end_i < stack->count){ Cpp_Token_Merge merge = cpp_attempt_token_merge(relex_stack.tokens[relex_stack.count - 1], stack->tokens[relex_end_i]); @@ -1674,10 +1664,10 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, } } } - + int token_delete_amount = relex_end_i - relex_start_i; int token_shift_amount = relex_stack.count - token_delete_amount; - + if (token_shift_amount != 0){ int new_token_count = stack->count + token_shift_amount; if (new_token_count > stack->max_count){ @@ -1687,24 +1677,24 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, } cpp_resize_token_stack(stack, new_max); } - + if (relex_end_i < stack->count){ FCPP_MEM_MOVE(stack->tokens + relex_end_i + token_shift_amount, stack->tokens + relex_end_i, sizeof(Cpp_Token)*(stack->count - relex_end_i)); } - + stack->count += token_shift_amount; } - + FCPP_MEM_COPY(stack->tokens + relex_start_i, relex_stack.tokens, sizeof(Cpp_Token)*relex_stack.count); cpp_free_token_stack(relex_stack); } - + else{ cpp_shift_token_starts(stack, end_token_i, -amount); cpp_free_token_stack(relex_stack); } - + return went_too_far; } #endif diff --git a/4cpp_lexer_types.h b/4cpp_lexer_types.h index b25a6d1a..89a431a7 100644 --- a/4cpp_lexer_types.h +++ b/4cpp_lexer_types.h @@ -170,13 +170,6 @@ enum Cpp_Token_Type{ CPP_TOKEN_TYPE_COUNT }; -// TODO(allen): This is a dumb redundant type... probably just -// move towards using String for this everywhere eventually. -struct Cpp_File{ - char *data; - int size; -}; - struct Cpp_Token{ Cpp_Token_Type type; int start, size; @@ -235,7 +228,9 @@ struct Cpp_Get_Token_Result{ }; struct Cpp_Relex_State{ - Cpp_File file; + char *data; + int size; + Cpp_Token_Stack *stack; int start, end, amount; int start_token_i; diff --git a/4ed_file_view.cpp b/4ed_file_view.cpp index 8842567b..d308f94e 100644 --- a/4ed_file_view.cpp +++ b/4ed_file_view.cpp @@ -1177,9 +1177,8 @@ Job_Callback_Sig(job_full_lex){ Editing_File *file = (Editing_File*)data[0]; General_Memory *general = (General_Memory*)data[1]; - Cpp_File cpp_file; - cpp_file.data = file->state.buffer.data; - cpp_file.size = file->state.buffer.size; + char *text_data = file->state.buffer.data; + i32 text_size = file->state.buffer.size; i32 buffer_size = file->state.buffer.size; buffer_size = (buffer_size + 3)&(~3); @@ -1243,9 +1242,9 @@ Job_Callback_Sig(job_full_lex){ Cpp_Lex_Data status = {}; do{ - for (i32 r = 2048; r > 0 && status.pos < cpp_file.size; --r){ + for (i32 r = 2048; r > 0 && status.pos < text_size; --r){ Cpp_Lex_Data prev_lex = status; - Cpp_Read_Result step_result = cpp_lex_step(cpp_file, &status); + Cpp_Read_Result step_result = cpp_lex_step(text_data, text_size, &status); if (step_result.has_result){ if (!cpp_push_token_nonalloc(&tokens, step_result.token)){ @@ -1257,7 +1256,7 @@ Job_Callback_Sig(job_full_lex){ } } - if (status.pos >= cpp_file.size){ + if (status.pos >= text_size){ status.complete = 1; } else{ @@ -1361,14 +1360,13 @@ file_relex_parallel(System_Functions *system, b32 result = true; b32 inline_lex = !file->state.still_lexing; if (inline_lex){ - Cpp_File cpp_file; - cpp_file.data = file->state.buffer.data; - cpp_file.size = file->state.buffer.size; + char *data = file->state.buffer.data; + i32 size = file->state.buffer.size; Cpp_Token_Stack *stack = &file->state.token_stack; Cpp_Relex_State state = - cpp_relex_nonalloc_start(cpp_file, stack, + cpp_relex_nonalloc_start(data, size, stack, start_i, end_i, amount, 100); Temp_Memory temp = begin_temp_memory(part); diff --git a/4ed_metagen.cpp b/4ed_metagen.cpp index 184f544a..ccc6d4ce 100644 --- a/4ed_metagen.cpp +++ b/4ed_metagen.cpp @@ -710,14 +710,14 @@ perform_doc_parse(Partition *part, String doc_string, Documentation *doc){ } static int -get_type_doc_string(Cpp_File file, Cpp_Token *tokens, int i, +get_type_doc_string(char *data, Cpp_Token *tokens, int i, String *doc_string){ int result = false; if (i > 0){ Cpp_Token *prev_token = tokens + i - 1; if (prev_token->type == CPP_TOKEN_COMMENT){ - *doc_string = make_string(file.data + prev_token->start, prev_token->size); + *doc_string = make_string(data + prev_token->start, prev_token->size); if (check_and_fix_docs(doc_string)){ result = true; } @@ -729,13 +729,13 @@ get_type_doc_string(Cpp_File file, Cpp_Token *tokens, int i, static int parse_struct(Partition *part, int is_struct, - Cpp_File file, Cpp_Token *tokens, int count, + char *data, Cpp_Token *tokens, int count, Cpp_Token **token_ptr, Struct_Member *top_member); static int parse_struct_member(Partition *part, - Cpp_File file, Cpp_Token *tokens, int count, + char *data, Cpp_Token *tokens, int count, Cpp_Token **token_ptr, Struct_Member *member){ @@ -745,7 +745,7 @@ parse_struct_member(Partition *part, int i = (int)(token - tokens); String doc_string = {0}; - get_type_doc_string(file, tokens, i, &doc_string); + get_type_doc_string(data, tokens, i, &doc_string); int start_i = i; Cpp_Token *start_token = token; @@ -779,18 +779,18 @@ parse_struct_member(Partition *part, } } - String name = make_string(file.data + token_j->start, token_j->size); + String name = make_string(data + token_j->start, token_j->size); name = skip_chop_whitespace(name); int type_start = start_token->start; int type_end = token_j->start; - String type = make_string(file.data + type_start, type_end - type_start); + String type = make_string(data + type_start, type_end - type_start); type = skip_chop_whitespace(type); type_start = token_j->start + token_j->size; type_end = token->start; - String type_postfix = make_string(file.data + type_start, type_end - type_start); + String type_postfix = make_string(data + type_start, type_end - type_start); type_postfix = skip_chop_whitespace(type_postfix); ++token; @@ -811,7 +811,7 @@ parse_struct_member(Partition *part, static Struct_Member* parse_struct_next_member(Partition *part, - Cpp_File file, Cpp_Token *tokens, int count, + char *data, Cpp_Token *tokens, int count, Cpp_Token **token_ptr){ Struct_Member *result = 0; @@ -821,11 +821,11 @@ parse_struct_next_member(Partition *part, for (; i < count; ++i, ++token){ if (token->type == CPP_TOKEN_IDENTIFIER || (token->flags & CPP_TFLAG_IS_KEYWORD)){ - String lexeme = make_string(file.data + token->start, token->size); + String lexeme = make_string(data + token->start, token->size); if (match(lexeme, make_lit_string("struct"))){ Struct_Member *member = push_struct(part, Struct_Member); - if (parse_struct(part, true, file, tokens, count, &token, member)){ + if (parse_struct(part, true, data, tokens, count, &token, member)){ result = member; break; } @@ -835,7 +835,7 @@ parse_struct_next_member(Partition *part, } else if (match(lexeme, make_lit_string("union"))){ Struct_Member *member = push_struct(part, Struct_Member); - if (parse_struct(part, false, file, tokens, count, &token, member)){ + if (parse_struct(part, false, data, tokens, count, &token, member)){ result = member; break; } @@ -845,7 +845,7 @@ parse_struct_next_member(Partition *part, } else{ Struct_Member *member = push_struct(part, Struct_Member); - if (parse_struct_member(part, file, tokens, count, &token, member)){ + if (parse_struct_member(part, data, tokens, count, &token, member)){ result = member; break; } @@ -866,7 +866,7 @@ parse_struct_next_member(Partition *part, static int parse_struct(Partition *part, int is_struct, - Cpp_File file, Cpp_Token *tokens, int count, + char *data, Cpp_Token *tokens, int count, Cpp_Token **token_ptr, Struct_Member *top_member){ @@ -876,7 +876,7 @@ parse_struct(Partition *part, int is_struct, int i = (int)(token - tokens); String doc_string = {0}; - get_type_doc_string(file, tokens, i, &doc_string); + get_type_doc_string(data, tokens, i, &doc_string); int start_i = i; @@ -899,7 +899,7 @@ parse_struct(Partition *part, int is_struct, String name = {0}; if (j != start_i){ - name = make_string(file.data + token_j->start, token_j->size); + name = make_string(data + token_j->start, token_j->size); name = skip_chop_whitespace(name); } @@ -913,7 +913,7 @@ parse_struct(Partition *part, int is_struct, ++token; Struct_Member *new_member = - parse_struct_next_member(part, file, tokens, count, &token); + parse_struct_next_member(part, data, tokens, count, &token); if (new_member){ top_member->first_child = new_member; @@ -921,7 +921,7 @@ parse_struct(Partition *part, int is_struct, Struct_Member *head_member = new_member; for(;;){ new_member = - parse_struct_next_member(part, file, tokens, count, &token); + parse_struct_next_member(part, data, tokens, count, &token); if (new_member){ head_member->next_sibling = new_member; head_member = new_member; @@ -1124,7 +1124,7 @@ print_see_also(FILE *file, Documentation *doc){ } static int -parse_enum(Partition *part, Cpp_File file, +parse_enum(Partition *part, char *data, Cpp_Token *tokens, int count, Cpp_Token **token_ptr, int start_i, Enum_Set flag_set, int flag_index){ @@ -1143,7 +1143,7 @@ parse_enum(Partition *part, Cpp_File file, } } - String name = make_string(file.data + token_j->start, token_j->size); + String name = make_string(data + token_j->start, token_j->size); name = skip_chop_whitespace(name); for (; i < count; ++i, ++token){ @@ -1162,9 +1162,9 @@ parse_enum(Partition *part, Cpp_File file, } else if (token->type == CPP_TOKEN_IDENTIFIER){ String doc_string = {0}; - get_type_doc_string(file, tokens, i, &doc_string); + get_type_doc_string(data, tokens, i, &doc_string); - String name = make_string(file.data + token->start, token->size); + String name = make_string(data + token->start, token->size); name = skip_chop_whitespace(name); String value = {0}; @@ -1185,7 +1185,7 @@ parse_enum(Partition *part, Cpp_File file, int val_start = start_token->start + start_token->size; int val_end = token->start; - value = make_string(file.data + val_start, val_end - val_start); + value = make_string(data + val_start, val_end - val_start); value = skip_chop_whitespace(value); --i; @@ -1282,7 +1282,7 @@ allocate_argument_breakdown(int count){ } static Argument_Breakdown -do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token){ +do_parameter_parse(char *data, Cpp_Token *args_start_token, Cpp_Token *token){ int arg_index = 0; Cpp_Token *arg_token = args_start_token + 1; int param_string_start = arg_token->start; @@ -1303,7 +1303,7 @@ do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token) arg_token->type == CPP_TOKEN_PARENTHESE_CLOSE){ int size = arg_token->start - param_string_start; - String param_string = make_string(file.data + param_string_start, size); + String param_string = make_string(data + param_string_start, size); param_string = chop_whitespace(param_string); breakdown.param_string[arg_index] = param_string; @@ -1313,7 +1313,7 @@ do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token) if (param_name_token->type == CPP_TOKEN_IDENTIFIER){ int start = param_name_token->start; int size = param_name_token->size; - breakdown.param_name[arg_index] = make_string(file.data + start, size); + breakdown.param_name[arg_index] = make_string(data + start, size); break; } } @@ -1363,7 +1363,7 @@ do_function_parse_check(int *index, Cpp_Token **token_ptr, int count){ static int do_function_get_doc(int *index, Cpp_Token **token_ptr, int count, - Cpp_File file, String *doc_string){ + char *data, String *doc_string){ int result = false; int i = *index; @@ -1371,7 +1371,7 @@ do_function_get_doc(int *index, Cpp_Token **token_ptr, int count, for (; i < count; ++i, ++token){ if (token->type == CPP_TOKEN_COMMENT){ - String lexeme = make_string(file.data + token->start, token->size); + String lexeme = make_string(data + token->start, token->size); if (check_and_fix_docs(&lexeme)){ *doc_string = lexeme; result = true; @@ -1391,7 +1391,7 @@ do_function_get_doc(int *index, Cpp_Token **token_ptr, int count, static int do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_start_token, - Cpp_File file, Function_Set function_set, int sig_count){ + char *data, Function_Set function_set, int sig_count){ int result = false; int i = *index; @@ -1399,10 +1399,10 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s Cpp_Token *args_start_token = token+1; - function_set.name[sig_count] = make_string(file.data + token->start, token->size); + function_set.name[sig_count] = make_string(data + token->start, token->size); int size = token->start - ret_start_token->start; - String ret = make_string(file.data + ret_start_token->start, size); + String ret = make_string(data + ret_start_token->start, size); ret = chop_whitespace(ret); function_set.ret[sig_count] = ret; @@ -1415,12 +1415,12 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s if (i < count){ int size = token->start + token->size - args_start_token->start;; function_set.args[sig_count] = - make_string(file.data + args_start_token->start, size); + make_string(data + args_start_token->start, size); function_set.valid[sig_count] = true; result = true; Argument_Breakdown *breakdown = &function_set.breakdown[sig_count]; - *breakdown = do_parameter_parse(file, args_start_token, token); + *breakdown = do_parameter_parse(data, args_start_token, token); } *index = i; @@ -1430,7 +1430,7 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s } static int -do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File file, +do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, char *data, Function_Set function_set, int sig_count){ int result = false; @@ -1438,7 +1438,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi Cpp_Token *token = *token_ptr; { - function_set.marker[sig_count] = make_string(file.data + token->start, token->size); + function_set.marker[sig_count] = make_string(data + token->start, token->size); int j = i; Cpp_Token *jtoken = token; @@ -1446,7 +1446,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi if (do_function_parse_check(&j, &jtoken, count)){ if (token->type == CPP_TOKEN_IDENTIFIER){ String doc_string = {0}; - if (do_function_get_doc(&j, &jtoken, count, file, &doc_string)){ + if (do_function_get_doc(&j, &jtoken, count, data, &doc_string)){ function_set.doc_string[sig_count] = doc_string; } } @@ -1457,7 +1457,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi Cpp_Token *ret_start_token = token; if (do_function_parse_check(&i, &token, count)){ if (do_function_parse(&i, &token, count, ret_start_token, - file, function_set, sig_count)){ + data, function_set, sig_count)){ result = true; } } @@ -1499,7 +1499,7 @@ do_macro_parse_check(int *index, Cpp_Token **token_ptr, int count){ static int do_macro_parse(int *index, Cpp_Token **token_ptr, int count, - Cpp_File file, Function_Set macro_set, int sig_count){ + char *data, Function_Set macro_set, int sig_count){ int result = false; int i = *index; @@ -1508,7 +1508,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count, if (i > 0){ Cpp_Token *doc_token = token-1; - String doc_string = make_string(file.data + doc_token->start, doc_token->size); + String doc_string = make_string(data + doc_token->start, doc_token->size); if (check_and_fix_docs(&doc_string)){ macro_set.doc_string[sig_count] = doc_string; @@ -1520,7 +1520,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count, } if (i < count && (token->flags & CPP_TFLAG_PP_BODY)){ - macro_set.name[sig_count] = make_string(file.data + token->start, token->size); + macro_set.name[sig_count] = make_string(data + token->start, token->size); ++i, ++token; if (i < count){ @@ -1534,10 +1534,10 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count, if (i < count){ int start = args_start_token->start; int end = token->start + token->size; - macro_set.args[sig_count] = make_string(file.data + start, end - start); + macro_set.args[sig_count] = make_string(data + start, end - start); Argument_Breakdown *breakdown = ¯o_set.breakdown[sig_count]; - *breakdown = do_parameter_parse(file, args_start_token, token); + *breakdown = do_parameter_parse(data, args_start_token, token); ++i, ++token; if (i < count){ @@ -1556,7 +1556,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count, start = body_start->start; end = body_end->start + body_end->size; - macro_set.body[sig_count] = make_string(file.data + start, end - start); + macro_set.body[sig_count] = make_string(data + start, end - start); } } @@ -1745,12 +1745,11 @@ generate_custom_headers(){ String *code = &string_code; Cpp_Token_Stack *token_stack = &string_tokens; - Cpp_File file; - file.data = code->str; - file.size = code->size; + char *data = code->str; + int size = code->size; *token_stack = cpp_make_token_stack(1024); - cpp_lex_file(file, token_stack); + cpp_lex_file(data, size, token_stack); int count = token_stack->count; @@ -1760,7 +1759,7 @@ generate_custom_headers(){ for (int i = 0; i < count; ++i, ++token){ if (token->type == CPP_TOKEN_IDENTIFIER && !(token->flags & CPP_TFLAG_PP_BODY)){ - String lexeme = make_string(file.data + token->start, token->size); + String lexeme = make_string(data + token->start, token->size); String_Function_Marker marker = do_string_function_marker_check(lexeme); @@ -1786,9 +1785,7 @@ generate_custom_headers(){ String *code = &string_code; Cpp_Token_Stack *token_stack = &string_tokens; - Cpp_File file; - file.data = code->str; - file.size = code->size; + char *data = code->str; int count = token_stack->count; Cpp_Token *tokens = token_stack->tokens; @@ -1797,21 +1794,21 @@ generate_custom_headers(){ for (int i = 0; i < count; ++i, ++token){ if (token->type == CPP_TOKEN_IDENTIFIER && !(token->flags & CPP_TFLAG_PP_BODY)){ - String lexeme = make_string(file.data + token->start, token->size); + String lexeme = make_string(data + token->start, token->size); String_Function_Marker marker = do_string_function_marker_check(lexeme); if (marker.parse_function){ - if (do_full_function_parse(&i, &token, count, file, + if (do_full_function_parse(&i, &token, count, data, string_function_set, string_sig_count)){ ++string_sig_count; } } else if (marker.parse_doc){ if (do_macro_parse_check(&i, &token, count)){ - do_macro_parse(&i, &token, count, - file, string_function_set, string_sig_count); + do_macro_parse(&i, &token, count, data, + string_function_set, string_sig_count); ++string_sig_count; } } @@ -1834,13 +1831,11 @@ generate_custom_headers(){ String *code = &code_data[J]; Parse *parse = &parses[J]; - // TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!! - Cpp_File file; - file.data = code->str; - file.size = code->size; + char *data = code->str; + int size = code->size; parse->tokens = cpp_make_token_stack(512); - cpp_lex_file(file, &parse->tokens); + cpp_lex_file(data, size, &parse->tokens); int count = parse->tokens.count; Cpp_Token *tokens = parse->tokens.tokens; @@ -1849,7 +1844,7 @@ generate_custom_headers(){ for (int i = 0; i < count; ++i, ++token){ if (token->type == CPP_TOKEN_IDENTIFIER && !(token->flags & CPP_TFLAG_PP_BODY)){ - String lexeme = make_string(file.data + token->start, token->size); + String lexeme = make_string(data + token->start, token->size); if (match(lexeme, "API_EXPORT")){ if (do_function_parse_check(&i, &token, count)){ ++line_count; @@ -1867,10 +1862,7 @@ generate_custom_headers(){ String *code = &code_data[J]; Parse *parse = &parses[J]; - // TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!! - Cpp_File file; - file.data = code->str; - file.size = code->size; + char *data = code->str; int count = parse->tokens.count; Cpp_Token *tokens = parse->tokens.tokens; @@ -1880,9 +1872,9 @@ generate_custom_headers(){ for (int i = 0; i < count; ++i, ++token){ if (token->type == CPP_TOKEN_IDENTIFIER && !(token->flags & CPP_TFLAG_PP_BODY)){ - String lexeme = make_string(file.data + token->start, token->size); + String lexeme = make_string(data + token->start, token->size); if (match(lexeme, "API_EXPORT")){ - do_full_function_parse(&i, &token, count, file, function_set, sig_count); + do_full_function_parse(&i, &token, count, data, function_set, sig_count); if (!function_set.valid[sig_count]){ zero_index(function_set, sig_count); // TODO(allen): get warning file name and line numbers @@ -1989,14 +1981,11 @@ generate_custom_headers(){ String type_code = file_dump("4coder_types.h"); - - // TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!! - Cpp_File type_file; - type_file.data = type_code.str; - type_file.size = type_code.size; + char *data = type_code.str; + int size = type_code.size; Cpp_Token_Stack types_tokens = cpp_make_token_stack(512); - cpp_lex_file(type_file, &types_tokens); + cpp_lex_file(data, size, &types_tokens); int typedef_count = 0; int struct_count = 0; @@ -2021,7 +2010,7 @@ generate_custom_headers(){ (token->type == CPP_TOKEN_KEY_TYPE_DECLARATION || token->type == CPP_TOKEN_IDENTIFIER)){ - String lexeme = make_string(type_file.data + token->start, token->size); + String lexeme = make_string(data + token->start, token->size); int match_index = 0; if (string_set_match(type_spec_keys, ArrayCount(type_spec_keys), lexeme, &match_index)){ @@ -2077,7 +2066,7 @@ generate_custom_headers(){ (token->type == CPP_TOKEN_KEY_TYPE_DECLARATION || token->type == CPP_TOKEN_IDENTIFIER)){ - String lexeme = make_string(type_file.data + token->start, token->size); + String lexeme = make_string(data + token->start, token->size); int match_index = 0; if (string_set_match(type_spec_keys, ArrayCount(type_spec_keys), lexeme, &match_index)){ @@ -2085,7 +2074,7 @@ generate_custom_headers(){ case 0: //typedef { String doc_string = {0}; - get_type_doc_string(type_file, tokens, i, &doc_string); + get_type_doc_string(data, tokens, i, &doc_string); int start_i = i; Cpp_Token *start_token = token; @@ -2105,12 +2094,12 @@ generate_custom_headers(){ } } - String name = make_string(type_file.data + token_j->start, token_j->size); + String name = make_string(data + token_j->start, token_j->size); name = skip_chop_whitespace(name); int type_start = start_token->start + start_token->size; int type_end = token_j->start; - String type = make_string(type_file.data + type_start, type_end - type_start); + String type = make_string(data + type_start, type_end - type_start); type = skip_chop_whitespace(type); typedef_set.type[typedef_index] = type; @@ -2123,7 +2112,7 @@ generate_custom_headers(){ case 1: case 2: //struct/union { if (parse_struct(part, (match_index == 1), - type_file, tokens, count, &token, + data, tokens, count, &token, struct_set.structs + struct_index)){ ++struct_index; } @@ -2133,7 +2122,7 @@ generate_custom_headers(){ case 3: //ENUM { String doc_string = {0}; - get_type_doc_string(type_file, tokens, i, &doc_string); + get_type_doc_string(data, tokens, i, &doc_string); int start_i = i; @@ -2143,7 +2132,7 @@ generate_custom_headers(){ } } - if (parse_enum(part, type_file, + if (parse_enum(part, data, tokens, count, &token, start_i, enum_set, enum_index)){ @@ -2156,7 +2145,7 @@ generate_custom_headers(){ case 4: //FLAGENUM { String doc_string = {0}; - get_type_doc_string(type_file, tokens, i, &doc_string); + get_type_doc_string(data, tokens, i, &doc_string); int start_i = i; @@ -2166,7 +2155,7 @@ generate_custom_headers(){ } } - if (parse_enum(part, type_file, + if (parse_enum(part, data, tokens, count, &token, start_i, flag_set, flag_index)){ diff --git a/build.bat b/build.bat index 8f9a1dcd..d4d32294 100644 --- a/build.bat +++ b/build.bat @@ -1,6 +1,6 @@ @echo off REM "build_exp.bat" /Zi -REM "build_all.bat" /DFRED_SUPER /DFRED_INTERNAL /Zi -"build_all.bat" /DFRED_INTERNAL /Zi +"build_all.bat" /DFRED_SUPER /DFRED_INTERNAL /Zi +REM "build_all.bat" /DFRED_INTERNAL /Zi REM "build_all.bat" /O2 /Zi diff --git a/power/4coder_casey.cpp b/power/4coder_casey.cpp index 649073bf..70acebbf 100644 --- a/power/4coder_casey.cpp +++ b/power/4coder_casey.cpp @@ -687,6 +687,8 @@ CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking) CLI_OverlapWithConflict); } exec_command(app, change_active_panel); + + prev_location = null_location; } #if 0 diff --git a/test/4cpp_new_lexer.h b/test/4cpp_new_lexer.h index b3bc7fa1..d5d2f8e0 100644 --- a/test/4cpp_new_lexer.h +++ b/test/4cpp_new_lexer.h @@ -1032,10 +1032,11 @@ cpp_lex_size_nonalloc(Lex_Data *S_ptr, } lexer_link Cpp_Relex_State -cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, +cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance){ Cpp_Relex_State state; - state.file = file; + state.data = data; + state.size = size; state.stack = stack; state.start = start; state.end = end; @@ -1107,7 +1108,7 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, lex.pos = state->relex_start; int relex_end_i = state->end_token_i; - Cpp_Token match_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); + Cpp_Token match_token = cpp__get_token(stack, tokens, state->size, relex_end_i); Cpp_Token end_token = match_token; int went_too_far = false; @@ -1115,9 +1116,9 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, for (;;){ int result = cpp_lex_size_nonalloc(&lex, - state->file.data, - state->file.size, - state->file.size, + state->data, + state->size, + state->size, relex_stack, 1); switch (result){ @@ -1134,7 +1135,7 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, while (lex.pos > end_token.start && relex_end_i < stack->count){ ++relex_end_i; - end_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); + end_token = cpp__get_token(stack, tokens, state->size, relex_end_i); } } break; diff --git a/test/experiment.cpp b/test/experiment.cpp index 730817c5..c2c8483e 100644 --- a/test/experiment.cpp +++ b/test/experiment.cpp @@ -13,7 +13,7 @@ // TOP #include "../4ed_meta.h" -#define FCPP_STRING_IMPLEMENTATION +#define FSTRING_IMPLEMENTATION #include "../4coder_string.h" #include "../4cpp_types.h" @@ -94,13 +94,12 @@ system_set_file_list(File_List *file_list, String directory){ append(&dir, directory); char trail_str[] = "\\*"; append(&dir, trail_str); - - char *c_str_dir = make_c_str(dir); - + terminate_with_null(&dir); + WIN32_FIND_DATA find_data; HANDLE search; - search = FindFirstFileA(c_str_dir, &find_data); - + search = FindFirstFileA(dir.str, &find_data); + if (search != INVALID_HANDLE_VALUE){ i32 count = 0; i32 file_count = 0; @@ -116,19 +115,19 @@ system_set_file_list(File_List *file_list, String directory){ more_files = FindNextFile(search, &find_data); }while(more_files); FindClose(search); - + i32 required_size = count + file_count * sizeof(File_Info); if (file_list->block_size < required_size){ Win32FreeMemory(file_list->block); file_list->block = Win32GetMemory(required_size); file_list->block_size = required_size; } - + file_list->infos = (File_Info*)file_list->block; char *name = (char*)(file_list->infos + file_count); if (file_list->block){ - search = FindFirstFileA(c_str_dir, &find_data); - + search = FindFirstFileA(dir.str, &find_data); + if (search != INVALID_HANDLE_VALUE){ File_Info *info = file_list->infos; more_files = 1; @@ -137,21 +136,21 @@ system_set_file_list(File_List *file_list, String directory){ !match(find_data.cFileName, "..")){ info->folder = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; info->filename.str = name; - + i32 i = 0; for(;find_data.cFileName[i];++i) *name++ = find_data.cFileName[i]; info->filename.size = i; info->filename.memory_size = info->filename.size + 1; *name++ = 0; - replace_char(info->filename, '\\', '/'); + replace_char(&info->filename, '\\', '/'); ++info; } more_files = FindNextFile(search, &find_data); }while(more_files); FindClose(search); - + file_list->count = file_count; - + }else{ Win32FreeMemory(file_list->block); file_list->block = 0; @@ -212,7 +211,6 @@ run_experiment(Experiment *exp, char *filename, int verbose, int chunks, int max_tokens){ String extension = {}; Data file_data; - Cpp_File file_cpp; new_lex::Lex_Data ld = {0}; int pass; int k, chunk_size, is_last; @@ -232,8 +230,8 @@ run_experiment(Experiment *exp, char *filename, int verbose, memset(exp->correct_stack.tokens, 0, TOKEN_ARRAY_SIZE); memset(exp->testing_stack.tokens, 0, TOKEN_ARRAY_SIZE); - file_cpp.data = (char*)file_data.data; - file_cpp.size = file_data.size; + char *data = (char*)file_data.data; + int size = file_data.size; ld.tb = (char*)malloc(file_data.size + 1); @@ -241,7 +239,7 @@ run_experiment(Experiment *exp, char *filename, int verbose, i64 start; start = __rdtsc(); - cpp_lex_file_nonalloc(file_cpp, &exp->correct_stack, lex_data); + cpp_lex_file_nonalloc(data, size, &exp->correct_stack, lex_data); time.handcoded += (__rdtsc() - start); if (max_tokens == 0){ @@ -357,8 +355,8 @@ run_experiment(Experiment *exp, char *filename, int verbose, " %.*s original %.*s testing\n", j, correct->start, correct->size, testing->start, testing->size, - correct->size, file_cpp.data + correct->start, - testing->size, file_cpp.data + testing->start); + correct->size, data + correct->start, + testing->size, data + testing->start); } } @@ -389,15 +387,15 @@ show_time(Times t, int repeats, char *type){ f32 speed_up = ((f32)t.handcoded) / (t.fsm); printf( "\n%s time for %d repeates\n" - OUTLINE("%lld") - OUTLINE("%lld") - OUTLINE("%f"), + OUTLINE("%lld") + OUTLINE("%lld") + OUTLINE("%f"), type, repeats, OUTLINE_VAR(i64, t.handcoded), OUTLINE_VAR(i64, t.fsm), OUTLINE_VAR(f32, speed_up) - ); + ); } #define BASE_DIR "w:/4ed/data/test/" @@ -413,27 +411,27 @@ int main(){ int chunks = (chunk_start > 0 && chunk_start <= chunk_end); int c = 0; - + char test_directory[] = BASE_DIR; File_List all_files = {}; Experiment exp = {}; Experiment chunk_exp = {}; Times exp_t = {}; Times chunk_exp_t = {}; - + init_test_stack(&exp.correct_stack); init_test_stack(&exp.testing_stack); - + init_test_stack(&chunk_exp.correct_stack); init_test_stack(&chunk_exp.testing_stack); - + AllowLocal(test_directory); AllowLocal(all_files); - + #if SINGLE_ITEM (void)(repeats); (void)(verbose_level); - + if (chunks){ begin_t(&chunk_exp_t); printf("With chunks of %d\n", chunks); @@ -442,16 +440,16 @@ int main(){ } end_t(&chunk_exp_t); } - + begin_t(&exp_t); printf("Unchunked\n"); run_experiment(&exp, BASE_DIR TEST_FILE, 1, 0, token_limit); end_t(&exp_t); - + #else - + system_set_file_list(&all_files, make_lit_string(test_directory)); - + for (int j = 0; j < repeats; ++j){ for (int i = 0; i < all_files.count; ++i){ if (all_files.infos[i].folder == 0){ @@ -477,21 +475,21 @@ int main(){ } } #endif - + if (chunks){ printf("chunks of sizes %d through %d tested\n", chunk_start, chunk_end); printf("chunked passed %d / %d tests\n", chunk_exp.passed_total, chunk_exp.test_total); } - + printf("unchunk passed %d / %d tests\n", exp.passed_total, exp.test_total); - + if (passed(exp) && (chunks == 0 || passed(chunk_exp))){ if (chunks){ show_time(chunk_exp_t, repeats, "Chunked"); } show_time(exp_t, repeats, "Unchunked"); } - + return(0); }