From c551c081a76986b6a34970ef49058ce3a2dcfe4c Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 18 Sep 2016 11:24:43 -0400 Subject: [PATCH] now officially running through the gap buffer with 100 of features working!!!! --- 4cpp_lexer.h | 122 ------------------------------ 4ed_api_implementation.cpp | 4 +- 4ed_app_target.cpp | 2 +- 4ed_file_view.cpp | 93 +++++++---------------- buffer/4coder_buffer_abstract.cpp | 21 +++-- 5 files changed, 45 insertions(+), 197 deletions(-) diff --git a/4cpp_lexer.h b/4cpp_lexer.h index 89f9e51f..ca863081 100644 --- a/4cpp_lexer.h +++ b/4cpp_lexer.h @@ -1125,42 +1125,6 @@ cpp_index_array(Cpp_Token_Array *array, int32_t file_size, int32_t index){ return(result); } -#if 0 -FCPP_INTERNAL Cpp_Relex_State -cpp_relex_nonalloc_start(Cpp_Token_Array *array, int32_t start, int32_t end, int32_t tolerance){ - Cpp_Relex_State state; - state.array = array; - state.start = start; - state.end = end; - - Cpp_Get_Token_Result result = cpp_get_token(array, start); - - state.start_token_i = result.token_index-1; - if (state.start_token_i < 0){ - state.start_token_i = 0; - } - - result = cpp_get_token(array, end); - - state.end_token_i = result.token_index; - if (end > array->tokens[state.end_token_i].start){ - ++state.end_token_i; - } - if (state.end_token_i < 0){ - state.end_token_i = 0; - } - - state.relex_start = array->tokens[state.start_token_i].start; - if (start < state.relex_start){ - state.relex_start = start; - } - - state.space_request = state.end_token_i - state.start_token_i + tolerance + 1; - - return(state); -} -#endif - FCPP_INTERNAL Cpp_Relex_Range cpp_get_relex_range(Cpp_Token_Array *array, int32_t start_pos, int32_t end_pos){ Cpp_Relex_Range range = {0}; @@ -1301,7 +1265,6 @@ cpp_relex_step(Cpp_Relex_Data *S_ptr, char *chunk, int32_t chunk_size, int32_t f } double_break:; - DrReturn(LexResult_Finished); } @@ -1371,91 +1334,6 @@ cpp_relex_abort(Cpp_Relex_Data *S_ptr, Cpp_Token_Array *array){ cpp_shift_token_starts(array, S_ptr->original_end_token_index, -S_ptr->character_shift_amount); } -#if 0 -// TODO(allen): rename shift_amount to character_shift_amount -FCPP_INTERNAL Cpp_Lex_Result -cpp_relex_nonalloc_main(Cpp_Relex_Data *S_ptr, - - Cpp_Relex_Range range, int32_t shift_amount, - - char *chunk, int32_t chunk_size, int32_t size, - - Cpp_Token_Array *array, - - Cpp_Token_Array *relex_array, - int32_t *relex_end_out, - - char *spare){ - - cpp_shift_token_starts(array, range.end_token_index, shift_amount); - Cpp_Token match_token = cpp_index_array(array, size, range.end_token_index); - Cpp_Token end_token = match_token; - - Cpp_Relex_State result = LexResult_Finished; - - Cpp_Token *tokens = array->tokens; - int32_t relex_end_index = range.end_token_index; - - if (state->relex_start < size){ - Cpp_Lex_Data lex = cpp_lex_data_init(spare); - lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags); - lex.pos = state->relex_start; - - // TODO(allen): This can be better I suspect. - for (;;){ - int32_t step_result = - cpp_lex_nonalloc_no_null_out_limit(&lex, data, size, size, - relex_array, 1); - - switch (step_result){ - case LexResult_HitTokenLimit: - { - Cpp_Token token = relex_array->tokens[relex_array->count-1]; - if (token.start == end_token.start && - token.size == end_token.size && - token.flags == end_token.flags && - token.state_flags == end_token.state_flags){ - --relex_array->count; - goto double_break; - } - - while (lex.pos > end_token.start && relex_end_i < array->count){ - ++relex_end_i; - end_token = cpp_index_array(array, size, relex_end_i); - } - } - break; - - case LexResult_NeedChunk: Assert(!"Invalid path"); break; - - case LexResult_NeedTokenMemory: - result = LexResult_NeedTokenMemory; - goto double_break; - - case LexResult_Finished: - goto double_break; - } - } - double_break:; - } - - switch (result){ - case LexResult_Finished: - { - *relex_end_out = relex_end_i; - }break; - - case LexResult_NeedTokenMemory: - { - cpp_shift_token_starts(array, state->end_token_i, -shift_amount); - }break; - - default: Assert(!"Invalid path"); - } - - return(result); -} -#endif #if !defined(FCPP_FORBID_MALLOC) diff --git a/4ed_api_implementation.cpp b/4ed_api_implementation.cpp index fba9cd37..8112a5c4 100644 --- a/4ed_api_implementation.cpp +++ b/4ed_api_implementation.cpp @@ -23,7 +23,7 @@ fill_buffer_summary(Buffer_Summary *buffer, Editing_File *file, Working_Set *wor buffer->ready = file_is_ready(file); buffer->buffer_id = file->id.id; - buffer->size = file->state.buffer.size; + buffer->size = buffer_size(&file->state.buffer); buffer->line_count = file->state.buffer.line_count; buffer->file_name_len = file->name.source_path.size; @@ -724,7 +724,6 @@ DOC_SEE(Buffer_Setting_ID) switch (setting){ case BufferSetting_Lex: { -#if BUFFER_EXPERIMENT_SCALPEL <= 0 if (file->settings.tokens_exist){ if (!value){ file_kill_tokens(system, &models->mem.general, file); @@ -735,7 +734,6 @@ DOC_SEE(Buffer_Setting_ID) file_first_lex_parallel(system, &models->mem.general, file); } } -#endif }break; case BufferSetting_WrapLine: diff --git a/4ed_app_target.cpp b/4ed_app_target.cpp index 0491ad0a..715b683c 100644 --- a/4ed_app_target.cpp +++ b/4ed_app_target.cpp @@ -19,7 +19,7 @@ #include "4coder_custom.h" -#define BUFFER_EXPERIMENT_SCALPEL 0 +#define BUFFER_EXPERIMENT_SCALPEL 1 #include "4ed_math.h" diff --git a/4ed_file_view.cpp b/4ed_file_view.cpp index d86323ff..8460db63 100644 --- a/4ed_file_view.cpp +++ b/4ed_file_view.cpp @@ -1123,10 +1123,9 @@ Job_Callback_Sig(job_full_lex){ General_Memory *general = (General_Memory*)data[1]; char *text_data = file->state.buffer.data; - i32 text_size = file->state.buffer.size; + i32 text_size = buffer_size(&file->state.buffer); - i32 buffer_size = file->state.buffer.size; - buffer_size = (buffer_size + 3)&(~3); + i32 buffer_size = (text_size + 3)&(~3); while (memory->size < buffer_size*2){ system->grow_thread_memory(memory); @@ -1224,7 +1223,6 @@ file_kill_tokens(System_Functions *system, file->state.token_array = null_cpp_token_array; } -#if BUFFER_EXPERIMENT_SCALPEL <= 0 internal void file_first_lex_parallel(System_Functions *system, General_Memory *general, Editing_File *file){ @@ -1243,7 +1241,6 @@ file_first_lex_parallel(System_Functions *system, file->state.lex_job = system->post_job(BACKGROUND_THREADS, job); } } -#endif internal b32 file_relex_parallel(System_Functions *system, @@ -1260,6 +1257,7 @@ file_relex_parallel(System_Functions *system, b32 result = true; b32 inline_lex = !file->state.still_lexing; if (inline_lex){ + Buffer_Type *buffer = &file->state.buffer; i32 extra_tolerance = 100; Cpp_Token_Array *array = &file->state.token_array; @@ -1275,33 +1273,32 @@ file_relex_parallel(System_Functions *system, relex_array.max_count = relex_space_size; relex_array.tokens = push_array(part, Cpp_Token, relex_array.max_count); - i32 size = file->state.buffer.size; + i32 size = buffer_size(buffer); char *spare = push_array(part, char, size+1); Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, spare); - char *chunk = file->state.buffer.data; - i32 chunk_size = 1024; - i32 chunk_index = 0; + char *chunks[3]; + i32 chunk_sizes[3]; - int32_t start_position = cpp_relex_start_position(&state); + chunks[0] = buffer->data; + chunk_sizes[0] = buffer->size1; - if (start_position == size){ - chunk = 0; - chunk_size = 0; - cpp_relex_declare_first_chunk_position(&state, size); - } - else{ - chunk_index = start_position / chunk_size; - - int32_t chunk_start_position = chunk_index*1024; - if (chunk_start_position + chunk_size > size){ - chunk_size = size - chunk_start_position; - } - - cpp_relex_declare_first_chunk_position(&state, chunk_start_position); - - chunk += chunk_start_position; + chunks[1] = buffer->data + buffer->size1 + buffer->gap_size; + chunk_sizes[1] = buffer->size2; + + chunks[2] = 0; + chunk_sizes[2] = 0; + + int32_t chunk_index = 0; + + char *chunk = chunks[chunk_index]; + int32_t chunk_size = chunk_sizes[chunk_index]; + + while (!cpp_relex_is_start_chunk(&state, chunk, chunk_size)){ + ++chunk_index; + chunk = chunks[chunk_index]; + chunk_size = chunk_sizes[chunk_index]; } for(;;){ @@ -1310,15 +1307,10 @@ file_relex_parallel(System_Functions *system, switch (lex_result){ case LexResult_NeedChunk: - { - ++chunk_index; - chunk += chunk_size; - - int32_t chunk_start_position = chunk_index*1024; - if (chunk_start_position + chunk_size > size){ - chunk_size = size - chunk_start_position; - } - }break; + ++chunk_index; + chunk = chunks[chunk_index]; + chunk_size = chunk_sizes[chunk_index]; + break; case LexResult_NeedTokenMemory: inline_lex = 0; @@ -1342,35 +1334,6 @@ file_relex_parallel(System_Functions *system, } cpp_relex_complete(&state, array, &relex_array); - -#if 0 - i32 delete_amount = relex_end - state.start_token_i; - i32 shift_amount = relex_space.count - delete_amount; - - if (shift_amount != 0){ - i32 new_count = array->count + shift_amount; - if (new_count > array->max_count){ - i32 new_max = LargeRoundUp(new_count, Kbytes(1)); - array->tokens = (Cpp_Token*) - general_memory_reallocate(general, array->tokens, - array->count*sizeof(Cpp_Token), - new_max*sizeof(Cpp_Token)); - array->max_count = new_max; - } - - i32 shift_size = array->count - relex_end; - if (shift_size > 0){ - Cpp_Token *old_base = array->tokens + relex_end; - memmove(old_base + shift_amount, old_base, - sizeof(Cpp_Token)*shift_size); - } - - array->count += shift_amount; - } - - memcpy(state.array->tokens + state.start_token_i, relex_space.tokens, - sizeof(Cpp_Token)*relex_space.count); -#endif } else{ cpp_relex_abort(&state, array); @@ -2164,12 +2127,10 @@ file_do_single_edit(System_Functions *system, file_edit_cursor_fix(system, part, general, file, layout, desc, 0); -#if BUFFER_EXPERIMENT_SCALPEL <= 0 // NOTE(allen): token fixing if (file->settings.tokens_exist){ file_relex_parallel(system, mem, file, start, end, shift_amount); } -#endif } internal void diff --git a/buffer/4coder_buffer_abstract.cpp b/buffer/4coder_buffer_abstract.cpp index 96e80cc9..e3410c5e 100644 --- a/buffer/4coder_buffer_abstract.cpp +++ b/buffer/4coder_buffer_abstract.cpp @@ -599,8 +599,13 @@ internal_4tech Partial_Cursor buffer_partial_from_pos(Buffer_Type *buffer, int pos){ Partial_Cursor result = {0}; - if (pos > buffer->size) pos = buffer->size; - if (pos < 0) pos = 0; + int32_t size = buffer_size(buffer); + if (pos > size){ + pos = size; + } + if (pos < 0){ + pos = 0; + } int line_index = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count); result.pos = pos; @@ -616,8 +621,13 @@ buffer_cursor_from_pos(Buffer_Type *buffer, int pos, float *wraps, Full_Cursor result; int line_index; - if (pos > buffer->size) pos = buffer->size; - if (pos < 0) pos = 0; + int32_t size = buffer_size(buffer); + if (pos > size){ + pos = size; + } + if (pos < 0){ + pos = 0; + } line_index = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count); result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height); @@ -635,8 +645,9 @@ buffer_partial_from_line_character(Buffer_Type *buffer, int line, int character) if (line_index >= buffer->line_count) line_index = buffer->line_count - 1; if (line_index < 0) line_index = 0; + int32_t size = buffer_size(buffer); int this_start = buffer->line_starts[line_index]; - int max_character = (buffer->size-this_start) + 1; + int max_character = (size-this_start) + 1; if (line_index+1 < buffer->line_count){ int next_start = buffer->line_starts[line_index+1]; max_character = (next_start-this_start);