now officially running through the gap buffer with 100 of features working!!!!

4cpp_lexer.h

@@ -1125,42 +1125,6 @@ cpp_index_array(Cpp_Token_Array *array, int32_t file_size, int32_t index){
     return(result);
 }
 
-#if 0
-FCPP_INTERNAL Cpp_Relex_State
-cpp_relex_nonalloc_start(Cpp_Token_Array *array, int32_t start, int32_t end, int32_t tolerance){
-    Cpp_Relex_State state;
-    state.array = array;
-    state.start = start;
-    state.end = end;
-    
-    Cpp_Get_Token_Result result = cpp_get_token(array, start);
-    
-    state.start_token_i = result.token_index-1;
-    if (state.start_token_i < 0){
-        state.start_token_i = 0;
-    }
-    
-    result = cpp_get_token(array, end);
-    
-    state.end_token_i = result.token_index;
-    if (end > array->tokens[state.end_token_i].start){
-        ++state.end_token_i;
-    }
-    if (state.end_token_i < 0){
-        state.end_token_i = 0;
-    }
-    
-    state.relex_start = array->tokens[state.start_token_i].start;
-    if (start < state.relex_start){
-        state.relex_start = start;
-    }
-    
-    state.space_request = state.end_token_i - state.start_token_i + tolerance + 1;
-    
-    return(state);
-}
-#endif
-
 FCPP_INTERNAL Cpp_Relex_Range
 cpp_get_relex_range(Cpp_Token_Array *array, int32_t start_pos, int32_t end_pos){
     Cpp_Relex_Range range = {0};
@@ -1301,7 +1265,6 @@ cpp_relex_step(Cpp_Relex_Data *S_ptr, char *chunk, int32_t chunk_size, int32_t f
     }
     
     double_break:;
-    
     DrReturn(LexResult_Finished);
 }
 
@@ -1371,91 +1334,6 @@ cpp_relex_abort(Cpp_Relex_Data *S_ptr, Cpp_Token_Array *array){
     cpp_shift_token_starts(array, S_ptr->original_end_token_index, -S_ptr->character_shift_amount);
 }
 
-#if 0
-// TODO(allen): rename shift_amount to character_shift_amount
-FCPP_INTERNAL Cpp_Lex_Result
-cpp_relex_nonalloc_main(Cpp_Relex_Data *S_ptr,
-                        
-                        Cpp_Relex_Range range, int32_t shift_amount, 
-                        
-                        char *chunk, int32_t chunk_size, int32_t size,
-                        
-                        Cpp_Token_Array *array,
-                        
-                        Cpp_Token_Array *relex_array,
-                        int32_t *relex_end_out,
-                        
-                        char *spare){
-    
-    cpp_shift_token_starts(array, range.end_token_index, shift_amount);
-    Cpp_Token match_token = cpp_index_array(array, size, range.end_token_index);
-    Cpp_Token end_token = match_token;
-    
-    Cpp_Relex_State result = LexResult_Finished;
-    
-    Cpp_Token *tokens = array->tokens;
-    int32_t relex_end_index = range.end_token_index;
-    
-    if (state->relex_start < size){
-        Cpp_Lex_Data lex = cpp_lex_data_init(spare);
-        lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags);
-        lex.pos = state->relex_start;
-        
-        // TODO(allen): This can be better I suspect.
-        for (;;){
-            int32_t step_result = 
-                cpp_lex_nonalloc_no_null_out_limit(&lex, data, size, size,
-                                                   relex_array, 1);
-            
-            switch (step_result){
-                case LexResult_HitTokenLimit:
-                {
-                    Cpp_Token token = relex_array->tokens[relex_array->count-1];
-                    if (token.start == end_token.start &&
-                        token.size == end_token.size &&
-                        token.flags == end_token.flags &&
-                        token.state_flags == end_token.state_flags){
-                        --relex_array->count;
-                        goto double_break;
-                    }
-                    
-                    while (lex.pos > end_token.start && relex_end_i < array->count){
-                        ++relex_end_i;
-                        end_token = cpp_index_array(array, size, relex_end_i);
-                    }
-                }
-                break;
-                
-                case LexResult_NeedChunk: Assert(!"Invalid path"); break;
-                
-                case LexResult_NeedTokenMemory:
-                result = LexResult_NeedTokenMemory;
-                goto double_break;
-                
-                case LexResult_Finished:
-                goto double_break;
-            }
-        }
-        double_break:;
-    }
-    
-    switch (result){
-        case LexResult_Finished:
-        {
-            *relex_end_out = relex_end_i;
-        }break;
-        
-        case LexResult_NeedTokenMemory:
-        {
-            cpp_shift_token_starts(array, state->end_token_i, -shift_amount);
-        }break;
-        
-        default: Assert(!"Invalid path");
-    }
-    
-    return(result);
-}
-#endif
 
 #if !defined(FCPP_FORBID_MALLOC)
 

4ed_api_implementation.cpp

@@ -23,7 +23,7 @@ fill_buffer_summary(Buffer_Summary *buffer, Editing_File *file, Working_Set *wor
         buffer->ready = file_is_ready(file);
         
         buffer->buffer_id = file->id.id;
-        buffer->size = file->state.buffer.size;
+        buffer->size = buffer_size(&file->state.buffer);
         buffer->line_count = file->state.buffer.line_count;
         
         buffer->file_name_len = file->name.source_path.size;
@@ -724,7 +724,6 @@ DOC_SEE(Buffer_Setting_ID)
         switch (setting){
             case BufferSetting_Lex:
             {
-#if BUFFER_EXPERIMENT_SCALPEL <= 0
                 if (file->settings.tokens_exist){
                     if (!value){
                         file_kill_tokens(system, &models->mem.general, file);
@@ -735,7 +734,6 @@ DOC_SEE(Buffer_Setting_ID)
                         file_first_lex_parallel(system, &models->mem.general, file);
                     }
                 }
-#endif
             }break;
             
             case BufferSetting_WrapLine:

4ed_app_target.cpp

@@ -19,7 +19,7 @@
 
 #include "4coder_custom.h"
 
-#define BUFFER_EXPERIMENT_SCALPEL 0
+#define BUFFER_EXPERIMENT_SCALPEL 1
 
 #include "4ed_math.h"
 

4ed_file_view.cpp

@@ -1123,10 +1123,9 @@ Job_Callback_Sig(job_full_lex){
     General_Memory *general = (General_Memory*)data[1];
     
     char *text_data = file->state.buffer.data;
-    i32 text_size = file->state.buffer.size;
+    i32 text_size = buffer_size(&file->state.buffer);
     
-    i32 buffer_size = file->state.buffer.size;
+    i32 buffer_size = (text_size + 3)&(~3);
-    buffer_size = (buffer_size + 3)&(~3);
     
     while (memory->size < buffer_size*2){
         system->grow_thread_memory(memory);
@@ -1224,7 +1223,6 @@ file_kill_tokens(System_Functions *system,
     file->state.token_array = null_cpp_token_array;
 }
 
-#if BUFFER_EXPERIMENT_SCALPEL <= 0
 internal void
 file_first_lex_parallel(System_Functions *system,
                         General_Memory *general, Editing_File *file){
@@ -1243,7 +1241,6 @@ file_first_lex_parallel(System_Functions *system,
         file->state.lex_job = system->post_job(BACKGROUND_THREADS, job);
     }
 }
-#endif
 
 internal b32
 file_relex_parallel(System_Functions *system,
@@ -1260,6 +1257,7 @@ file_relex_parallel(System_Functions *system,
     b32 result = true;
     b32 inline_lex = !file->state.still_lexing;
     if (inline_lex){
+        Buffer_Type *buffer = &file->state.buffer;
         i32 extra_tolerance = 100;
         
         Cpp_Token_Array *array = &file->state.token_array;
@@ -1275,33 +1273,32 @@ file_relex_parallel(System_Functions *system,
         relex_array.max_count = relex_space_size;
         relex_array.tokens = push_array(part, Cpp_Token, relex_array.max_count);
         
-        i32 size = file->state.buffer.size;
+        i32 size = buffer_size(buffer);
         char *spare = push_array(part, char, size+1);
         
         Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, spare);
         
-        char *chunk = file->state.buffer.data;
+        char *chunks[3];
-        i32 chunk_size = 1024;
+        i32 chunk_sizes[3];
-        i32 chunk_index = 0;
         
-        int32_t start_position = cpp_relex_start_position(&state);
+        chunks[0] = buffer->data;
+        chunk_sizes[0] = buffer->size1;
         
-        if (start_position == size){
+        chunks[1] = buffer->data + buffer->size1 + buffer->gap_size;
-            chunk = 0;
+        chunk_sizes[1] = buffer->size2;
-            chunk_size = 0;
-            cpp_relex_declare_first_chunk_position(&state, size);
-        }
-        else{
-            chunk_index = start_position / chunk_size;
         
-            int32_t chunk_start_position = chunk_index*1024;
+        chunks[2] = 0;
-            if (chunk_start_position + chunk_size > size){
+        chunk_sizes[2] = 0;
-                chunk_size = size - chunk_start_position;
-            }
         
-            cpp_relex_declare_first_chunk_position(&state, chunk_start_position);
+        int32_t chunk_index = 0;
         
-            chunk += chunk_start_position;
+        char *chunk = chunks[chunk_index];
+        int32_t chunk_size = chunk_sizes[chunk_index];
+        
+        while (!cpp_relex_is_start_chunk(&state, chunk, chunk_size)){
+            ++chunk_index;
+            chunk = chunks[chunk_index];
+            chunk_size = chunk_sizes[chunk_index];
         }
         
         for(;;){
@@ -1310,15 +1307,10 @@ file_relex_parallel(System_Functions *system,
             
             switch (lex_result){
                 case LexResult_NeedChunk:
-                {
+                ++chunk_index;
-                    ++chunk_index;
+                chunk = chunks[chunk_index];
-                    chunk += chunk_size;
+                chunk_size = chunk_sizes[chunk_index];
-                    
+                break;
-                    int32_t chunk_start_position = chunk_index*1024;
-                    if (chunk_start_position + chunk_size > size){
-                        chunk_size = size - chunk_start_position;
-                    }
-                }break;
                 
                 case LexResult_NeedTokenMemory:
                 inline_lex = 0;
@@ -1342,35 +1334,6 @@ file_relex_parallel(System_Functions *system,
             }
             
             cpp_relex_complete(&state, array, &relex_array);
-            
-#if 0
-            i32 delete_amount = relex_end - state.start_token_i;
-            i32 shift_amount = relex_space.count - delete_amount;
-            
-            if (shift_amount != 0){
-                i32 new_count = array->count + shift_amount;
-                if (new_count > array->max_count){
-                    i32 new_max = LargeRoundUp(new_count, Kbytes(1));
-                    array->tokens = (Cpp_Token*)
-                        general_memory_reallocate(general, array->tokens,
-                                                  array->count*sizeof(Cpp_Token),
-                                                  new_max*sizeof(Cpp_Token));
-                    array->max_count = new_max;
-                }
-                
-                i32 shift_size = array->count - relex_end;
-                if (shift_size > 0){
-                    Cpp_Token *old_base = array->tokens + relex_end;
-                    memmove(old_base + shift_amount, old_base,
-                            sizeof(Cpp_Token)*shift_size);
-                }
-                
-                array->count += shift_amount;
-            }
-            
-            memcpy(state.array->tokens + state.start_token_i, relex_space.tokens,
-                   sizeof(Cpp_Token)*relex_space.count);
-#endif
         }
         else{
             cpp_relex_abort(&state, array);
@@ -2164,12 +2127,10 @@ file_do_single_edit(System_Functions *system,
     
     file_edit_cursor_fix(system, part, general, file, layout, desc, 0);
     
-#if BUFFER_EXPERIMENT_SCALPEL <= 0
     // NOTE(allen): token fixing
     if (file->settings.tokens_exist){
         file_relex_parallel(system, mem, file, start, end, shift_amount);
     }
-#endif
 }
 
 internal void

buffer/4coder_buffer_abstract.cpp

@@ -599,8 +599,13 @@ internal_4tech Partial_Cursor
 buffer_partial_from_pos(Buffer_Type *buffer, int pos){
     Partial_Cursor result = {0};
     
-    if (pos > buffer->size) pos = buffer->size;
+    int32_t size = buffer_size(buffer);
-    if (pos < 0) pos = 0;
+    if (pos > size){
+        pos = size;
+    }
+    if (pos < 0){
+        pos = 0;
+    }
     
     int line_index = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count);
     result.pos = pos;
@@ -616,8 +621,13 @@ buffer_cursor_from_pos(Buffer_Type *buffer, int pos, float *wraps,
     Full_Cursor result;
     int line_index;
     
-    if (pos > buffer->size) pos = buffer->size;
+    int32_t size = buffer_size(buffer);
-    if (pos < 0) pos = 0;
+    if (pos > size){
+        pos = size;
+    }
+    if (pos < 0){
+        pos = 0;
+    }
     
     line_index = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count);
     result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height);
@@ -635,8 +645,9 @@ buffer_partial_from_line_character(Buffer_Type *buffer, int line, int character)
     if (line_index >= buffer->line_count) line_index = buffer->line_count - 1;
     if (line_index < 0) line_index = 0;
     
+    int32_t size = buffer_size(buffer);
     int this_start = buffer->line_starts[line_index];
-    int max_character = (buffer->size-this_start) + 1;
+    int max_character = (size-this_start) + 1;
     if (line_index+1 < buffer->line_count){
         int next_start = buffer->line_starts[line_index+1];
         max_character = (next_start-this_start);