From 8ad2b5bbffb4d140889b1c0233187074f13bdb4f Mon Sep 17 00:00:00 2001 From: Peter Slattery Date: Sun, 3 Aug 2025 18:03:40 -0700 Subject: [PATCH] Incrementally update global_code_index.name_hash --- code/4ed_api_implementation.cpp | 133 +++++++------- code/custom/4coder_code_index.cpp | 150 +++++++++++++-- code/custom/4coder_code_index.h | 9 +- code/custom/4coder_custom_hooks.cpp | 4 +- code/custom/4coder_default_bindings.cpp | 6 +- code/custom/4coder_default_include.cpp | 4 +- code/custom/4coder_string_pool.cpp | 129 +++++++++++++ code/custom/4coder_string_pool.h | 29 +++ code/custom/4coder_tree_sitter.cpp | 233 +++++++++++++----------- code/custom/4coder_tree_sitter.h | 2 + 10 files changed, 506 insertions(+), 193 deletions(-) create mode 100644 code/custom/4coder_string_pool.cpp create mode 100644 code/custom/4coder_string_pool.h diff --git a/code/4ed_api_implementation.cpp b/code/4ed_api_implementation.cpp index 6ae52c93..05999cdb 100644 --- a/code/4ed_api_implementation.cpp +++ b/code/4ed_api_implementation.cpp @@ -257,6 +257,7 @@ get_active_edit_behaviors(Models *models, Editing_File *file){ api(custom) function b32 buffer_replace_range(Application_Links *app, Buffer_ID buffer_id, Range_i64 range, String_Const_u8 string) { + ProfileScope(app, "buffer_replace_range"); Models *models = (Models*)app->cmd_context; Editing_File *file = imp_get_file(models, buffer_id); b32 result = false; @@ -338,7 +339,7 @@ buffer_seek_character_class(Application_Links *app, Buffer_ID buffer, Character_ Scratch_Block scratch(app); Gap_Buffer *gap_buffer = &file->state.buffer; List_String_Const_u8 chunks_list = buffer_get_chunks(scratch, gap_buffer); - + if (chunks_list.node_count > 0){ // TODO(allen): If you are reading this comment, then I haven't revisited this to tighten it up yet. // buffer_seek_character_class was originally implemented using the chunk indexer helper @@ -358,7 +359,7 @@ buffer_seek_character_class(Application_Links *app, Buffer_ID buffer, Character_ chunks.vals[chunks.count] = node->string; chunks.count += 1; } - + i64 size = buffer_size(gap_buffer); start_pos = clamp(-1, start_pos, size); Buffer_Chunk_Position pos = buffer_get_chunk_position(chunks, size, start_pos); @@ -734,22 +735,22 @@ buffer_get_setting(Application_Links *app, Buffer_ID buffer_id, Buffer_Setting_I { *value_out = file->settings.unimportant; }break; - + case BufferSetting_Unkillable: { *value_out = (file->settings.never_kill || file->settings.unkillable); }break; - + case BufferSetting_ReadOnly: { *value_out = file->settings.read_only; }break; - + case BufferSetting_RecordsHistory: { *value_out = history_is_activated(&file->state.history); }break; - + default: { result = false; @@ -777,17 +778,17 @@ buffer_set_setting(Application_Links *app, Buffer_ID buffer_id, Buffer_Setting_I file_set_unimportant(file, false); } }break; - + case BufferSetting_Unkillable: { file->settings.unkillable = (value != 0); }break; - + case BufferSetting_ReadOnly: { file->settings.read_only = (value != 0); }break; - + case BufferSetting_RecordsHistory: { if (value){ @@ -801,14 +802,14 @@ buffer_set_setting(Application_Links *app, Buffer_ID buffer_id, Buffer_Setting_I } } }break; - + default: { result = 0; }break; } } - + return(result); } @@ -854,7 +855,7 @@ buffer_save(Application_Links *app, Buffer_ID buffer_id, String_Const_u8 file_na { Models *models = (Models*)app->cmd_context; Editing_File *file = imp_get_file(models, buffer_id); - + b32 result = false; if (api_check_buffer(file)){ b32 skip_save = false; @@ -863,7 +864,7 @@ buffer_save(Application_Links *app, Buffer_ID buffer_id, String_Const_u8 file_na skip_save = true; } } - + if (!skip_save){ Thread_Context *tctx = app->tctx; Scratch_Block scratch(tctx); @@ -872,7 +873,7 @@ buffer_save(Application_Links *app, Buffer_ID buffer_id, String_Const_u8 file_na result = true; } } - + return(result); } @@ -891,16 +892,16 @@ buffer_kill(Application_Links *app, Buffer_ID buffer_id, Buffer_Kill_Flag flags) if (models->end_buffer != 0){ models->end_buffer(app, file->id); } - + buffer_unbind_name_low_level(working_set, file); if (file->canon.name_size != 0){ buffer_unbind_file(working_set, file); } file_free(tctx, models, file); working_set_free_file(&models->heap, working_set, file); - + Layout *layout = &models->layout; - + Node *order = &working_set->touch_order_sentinel; Node *file_node = order->next; for (Panel *panel = layout_get_first_open_panel(layout); @@ -919,7 +920,7 @@ buffer_kill(Application_Links *app, Buffer_ID buffer_id, Buffer_Kill_Flag flags) Assert(file_node != order); } } - + Child_Process_Container *child_processes = &models->child_processes; for (Node *node = child_processes->child_process_active_list.next; node != &child_processes->child_process_active_list; @@ -929,7 +930,7 @@ buffer_kill(Application_Links *app, Buffer_ID buffer_id, Buffer_Kill_Flag flags) child_process->out_file = 0; } } - + result = BufferKillResult_Killed; } else{ @@ -956,20 +957,20 @@ buffer_reopen(Application_Links *app, Buffer_ID buffer_id, Buffer_Reopen_Flag fl Plat_Handle handle = {}; if (system_load_handle(scratch, (char*)file->canon.name_space, &handle)){ File_Attributes attributes = system_load_attributes(handle); - + char *file_memory = push_array(scratch, char, (i32)attributes.size); - + if (file_memory != 0){ if (system_load_file(handle, file_memory, (i32)attributes.size)){ system_load_close(handle); - + // TODO(allen): try(perform a diff maybe apply edits in reopen) - + i32 line_numbers[16]; i32 column_numbers[16]; View *vptrs[16]; i32 vptr_count = 0; - + Layout *layout = &models->layout; for (Panel *panel = layout_get_first_open_panel(layout); panel != 0; @@ -985,15 +986,15 @@ buffer_reopen(Application_Links *app, Buffer_ID buffer_id, Buffer_Reopen_Flag fl ++vptr_count; } } - + Working_Set *working_set = &models->working_set; file_free(tctx, models, file); working_set_file_default_settings(working_set, file); file_create_from_string(tctx, models, file, SCu8(file_memory, attributes.size), attributes); - + for (i32 i = 0; i < vptr_count; ++i){ view_set_file(tctx, models, vptrs[i], file); - + vptrs[i]->file = file; i64 line = line_numbers[i]; i64 col = column_numbers[i]; @@ -1303,13 +1304,13 @@ panel_set_split(Application_Links *app, Panel_ID panel_id, Panel_Split_Kind kind { panel->split.v_f32 = clamp(0.f, t, 1.f); }break; - + case PanelSplitKind_FixedPixels_Max: case PanelSplitKind_FixedPixels_Min: { panel->split.v_i32 = i32_round32(t); }break; - + default: { print_message(app, string_u8_litexpr("Invalid split kind passed to panel_set_split, no change made to view layout")); @@ -1451,7 +1452,7 @@ view_get_setting(Application_Links *app, View_ID view_id, View_Setting_ID settin { Models *models = (Models*)app->cmd_context; View *view = imp_get_view(models, view_id); - + b32 result = false; if (api_check_view(view)){ result = true; @@ -1460,17 +1461,17 @@ view_get_setting(Application_Links *app, View_ID view_id, View_Setting_ID settin { *value_out = view->show_whitespace; }break; - + case ViewSetting_ShowScrollbar: { *value_out = !view->hide_scrollbar; }break; - + case ViewSetting_ShowFileBar: { *value_out = !view->hide_file_bar; }break; - + default: { result = false; @@ -1485,7 +1486,7 @@ view_set_setting(Application_Links *app, View_ID view_id, View_Setting_ID settin { Models *models = (Models*)app->cmd_context; View *view = imp_get_view(models, view_id); - + b32 result = false; if (api_check_view(view)){ result = true; @@ -1494,17 +1495,17 @@ view_set_setting(Application_Links *app, View_ID view_id, View_Setting_ID settin { view->show_whitespace = (b8)value; }break; - + case ViewSetting_ShowScrollbar: { view->hide_scrollbar = (b8)!value; }break; - + case ViewSetting_ShowFileBar: { view->hide_file_bar = (b8)!value; }break; - + default: { result = false; @@ -1637,7 +1638,7 @@ view_set_mark(Application_Links *app, View_ID view_id, Buffer_Seek seek) { Models *models = (Models*)app->cmd_context; View *view = imp_get_view(models, view_id); - + b32 result = false; if (api_check_view(view)){ Editing_File *file = view->file; @@ -1831,19 +1832,19 @@ get_managed_scope_with_multiple_dependencies(Application_Links *app, Managed_Sco { Models *models = (Models*)app->cmd_context; Lifetime_Allocator *lifetime_allocator = &models->lifetime_allocator; - + Scratch_Block scratch(app); - + // TODO(allen): revisit this struct Node_Ptr{ Node_Ptr *next; Lifetime_Object *object_ptr; }; - + Node_Ptr *first = 0; Node_Ptr *last = 0; i32 member_count = 0; - + b32 filled_array = true; for (i32 i = 0; i < count; i += 1){ Dynamic_Workspace *workspace = get_dynamic_workspace(models, scopes[i]); @@ -1851,13 +1852,13 @@ get_managed_scope_with_multiple_dependencies(Application_Links *app, Managed_Sco filled_array = false; break; } - + switch (workspace->user_type){ case DynamicWorkspace_Global: { // NOTE(allen): (global_scope INTERSECT X) == X for all X, therefore we emit nothing when a global group is in the key list. }break; - + case DynamicWorkspace_Unassociated: case DynamicWorkspace_Buffer: case DynamicWorkspace_View: @@ -1869,7 +1870,7 @@ get_managed_scope_with_multiple_dependencies(Application_Links *app, Managed_Sco new_node->object_ptr = object; member_count += 1; }break; - + case DynamicWorkspace_Intersected: { Lifetime_Key *key = (Lifetime_Key*)workspace->user_back_ptr; @@ -1884,14 +1885,14 @@ get_managed_scope_with_multiple_dependencies(Application_Links *app, Managed_Sco } } }break; - + default: { InvalidPath; }break; } } - + Managed_Scope result = 0; if (filled_array){ Lifetime_Object **object_ptr_array = push_array(scratch, Lifetime_Object*, member_count); @@ -1906,7 +1907,7 @@ get_managed_scope_with_multiple_dependencies(Application_Links *app, Managed_Sco Lifetime_Key *key = lifetime_get_or_create_intersection_key(lifetime_allocator, object_ptr_array, member_count); result = (Managed_Scope)key->dynamic_workspace.scope_id; } - + return(result); } @@ -2640,7 +2641,7 @@ buffer_set_face(Application_Links *app, Buffer_ID buffer_id, Face_ID id) { Models *models = (Models*)app->cmd_context; Editing_File *file = imp_get_file(models, buffer_id); - + b32 did_change = false; if (api_check_buffer(file)){ Face *face = font_set_face_from_id(&models->font_set, id); @@ -2912,13 +2913,13 @@ text_layout_create(Application_Links *app, Buffer_ID buffer_id, Rect_f32 rect, B if (api_check_buffer(file)){ Thread_Context *tctx = app->tctx; Face *face = file_get_face(models, file); - + Gap_Buffer *buffer = &file->state.buffer; - + Layout_Function *layout_func = file_get_layout_func(file); - + Vec2_f32 dim = rect_dim(rect); - + i64 line_count = buffer_line_count(buffer); i64 line_number = buffer_point.line_number; f32 y = -buffer_point.pixel_shift.y; @@ -2933,13 +2934,13 @@ text_layout_create(Application_Links *app, Buffer_ID buffer_id, Rect_f32 rect, B } y = next_y; } - + Range_i64 visible_line_number_range = Ii64(buffer_point.line_number, line_number); Range_i64 visible_range = Ii64(buffer_get_first_pos_from_line_number(buffer, visible_line_number_range.min), buffer_get_last_pos_from_line_number(buffer, visible_line_number_range.max)); - + i64 item_count = range_size_inclusive(visible_range); - + Arena arena = make_arena_system(); Arena *arena_ptr = push_array_zero(&arena, Arena, 1); *arena_ptr = arena; @@ -2992,16 +2993,16 @@ text_layout_line_on_screen(Application_Links *app, Text_Layout_ID layout_id, i64 if (layout == 0){ return(result); } - + Layout_Function *layout_func = layout->layout_func; - + Rect_f32 rect = layout->rect; if (range_contains_inclusive(layout->visible_line_number_range, line_number)){ Editing_File *file = imp_get_file(models, layout->buffer_id); if (api_check_buffer(file)){ f32 width = rect_width(rect); Face *face = file_get_face(models, file); - + for (i64 line_number_it = layout->visible_line_number_range.first;; line_number_it += 1){ Layout_Item_List line = file_get_line_layout(app->tctx, models, file, @@ -3013,7 +3014,7 @@ text_layout_line_on_screen(Application_Links *app, Text_Layout_ID layout_id, i64 } result.min = result.max; } - + result += rect.y0 - layout->point.pixel_shift.y; } } @@ -3023,7 +3024,7 @@ text_layout_line_on_screen(Application_Links *app, Text_Layout_ID layout_id, i64 else if (line_number > layout->visible_line_number_range.max){ result = If32(rect.y1, rect.y1); } - + return(result); } @@ -3037,14 +3038,14 @@ text_layout_character_on_screen(Application_Links *app, Text_Layout_ID layout_id if (api_check_buffer(file)){ Gap_Buffer *buffer = &file->state.buffer; i64 line_number = buffer_get_line_index(buffer, pos) + 1; - + if (range_contains_inclusive(layout->visible_line_number_range, line_number)){ Rect_f32 rect = layout->rect; f32 width = rect_width(rect); Face *face = file_get_face(models, file); - + Layout_Function *layout_func = layout->layout_func; - + f32 y = 0.f; Layout_Item_List line = {}; for (i64 line_number_it = layout->visible_line_number_range.first;; @@ -3057,7 +3058,7 @@ text_layout_character_on_screen(Application_Links *app, Text_Layout_ID layout_id } y += line.height; } - + // TODO(allen): optimization: This is some fairly heavy computation. We really // need to accelerate the (pos -> item) lookup within a single // Buffer_Layout_Item_List. @@ -3081,7 +3082,7 @@ text_layout_character_on_screen(Application_Links *app, Text_Layout_ID layout_id } } } - + Vec2_f32 shift = V2f32(rect.x0, rect.y0 + y) - layout->point.pixel_shift; result.p0 += shift; result.p1 += shift; diff --git a/code/custom/4coder_code_index.cpp b/code/custom/4coder_code_index.cpp index 224b61ea..76952cb5 100644 --- a/code/custom/4coder_code_index.cpp +++ b/code/custom/4coder_code_index.cpp @@ -15,6 +15,87 @@ delims_are_open_close_pair(Code_Index_Scope_Delim_Kind a, Code_Index_Scope_Delim return false; } +//////////////////////////////// +// NOTE(allen): Code_Index_Note_List + +function void +code_index_note_list_init(Code_Index_Note_List_New* list) +{ + list->sentinel_first.next = &list->sentinel_last; + list->sentinel_last.next = 0; + list->count = 0; +} + +function void +code_index_note_list_hash_init(Code_Index_Note_List_New* list) +{ + list->sentinel_first.next_in_hash = &list->sentinel_last; + list->sentinel_first.prev_in_hash = 0; + list->sentinel_last.next_in_hash = 0; + list->sentinel_last.prev_in_hash = &list->sentinel_first; + list->count = 0; +} + +function void +code_index_note_list_insert(Code_Index_Note_List_New* list, Code_Index_Note* prev, Code_Index_Note* note) +{ + Assert(prev != 0); + Code_Index_Note* next = prev->next; + prev->next = note; + note->next = next; + list->count += 1; +} + +function void +code_index_note_list_hash_insert(Code_Index_Note_List_New* list, Code_Index_Note* prev, Code_Index_Note* note) +{ + Assert(prev != 0); + Code_Index_Note* next = prev->next_in_hash; + prev->next_in_hash = note; + note->prev_in_hash = prev; + note->next_in_hash = next; + next->prev_in_hash = note; + list->count += 1; +} + +function void +code_index_note_list_hash_append(Code_Index_Note_List_New* list, Code_Index_Note* note) +{ + Code_Index_Note* prev = list->sentinel_last.prev_in_hash; + Code_Index_Note* next = &list->sentinel_last; + prev->next_in_hash = note; + note->prev_in_hash = prev; + note->next_in_hash = next; + next->prev_in_hash = note; + list->count += 1; +} + +function void +code_index_note_list_remove(Code_Index_Note_List_New* list, Code_Index_Note* prev, Code_Index_Note* note) +{ + Assert(note->next != 0); + Assert(list->count > 0); + Code_Index_Note* next = note->next; + prev->next = next; + list->count -= 1; + note->next = 0; +} + +function void +code_index_note_list_hash_remove(Code_Index_Note_List_New* list, Code_Index_Note* note) +{ + Assert(note->prev_in_hash != 0 && note->next_in_hash != 0); + Assert(list->count > 0); + Code_Index_Note* prev = note->prev_in_hash; + Code_Index_Note* next = note->next_in_hash; + prev->next_in_hash = next; + next->prev_in_hash = prev; + list->count -= 1; + + note->prev_in_hash = 0; + note->next_in_hash = 0; +} + //////////////////////////////// // NOTE(allen): Lookups @@ -49,19 +130,21 @@ code_index_get_nest(Code_Index_File *file, i64 pos) return result; } -function Code_Index_Note_List* -code_index__list_from_string(String_Const_u8 string){ +function Code_Index_Note_List_New* +code_index__list_from_string(String_Const_u8 string) +{ u64 hash = table_hash_u8(string.str, string.size); - Code_Index_Note_List *result = &global_code_index.name_hash[hash % ArrayCount(global_code_index.name_hash)]; + u64 index = hash % ArrayCount(global_code_index.name_hash); + Code_Index_Note_List_New *result = &global_code_index.name_hash[index]; return(result); } function Code_Index_Note* code_index_note_from_string(String_Const_u8 string){ - Code_Index_Note_List *list = code_index__list_from_string(string); + Code_Index_Note_List_New *list = code_index__list_from_string(string); Code_Index_Note *result = 0; - for (Code_Index_Note *node = list->first; - node != 0; + for (Code_Index_Note *node = list->sentinel_first.next_in_hash; + node != &list->sentinel_last; node = node->next_in_hash){ if (string_match(string, node->text)){ result = node; @@ -76,10 +159,16 @@ code_index_note_from_string(String_Const_u8 string){ // NOTE(allen): Global Code Index function void -code_index_init(void){ +code_index_init(void) +{ global_code_index.mutex = system_mutex_make(); global_code_index.node_arena = make_arena_system(KB(4)); global_code_index.buffer_to_index_file = make_table_u64_u64(global_code_index.node_arena.base_allocator, 500); + + for (int i = 0; i < ArrayCount(global_code_index.name_hash); i++) + { + code_index_note_list_hash_init(&global_code_index.name_hash[i]); + } } function Code_Index_File_Storage* @@ -121,13 +210,16 @@ code_index_unlock(void){ function void code_index__hash_file(Code_Index_File *file){ +// TODO(PS): @RemoveWholeFileHashing - this isn't necessary since, to support incremental updates +// we insert nodes into the hash table when the node is created +#if 0 for (Code_Index_Note *node = file->note_list.first; node != 0; node = node->next){ - Code_Index_Note_List *list = code_index__list_from_string(node->text); - zdll_push_back_NP_(list->first, list->last, node, next_in_hash, prev_in_hash); - list->count += 1; + Code_Index_Note_List_New *list = code_index__list_from_string(node->text); + code_index_note_list_hash_insert(list, list->sentinel_last.prev_in_hash, node); } +#endif } function void @@ -135,9 +227,8 @@ code_index__clear_file(Code_Index_File *file){ for (Code_Index_Note *node = file->note_list.first; node != 0; node = node->next){ - Code_Index_Note_List *list = code_index__list_from_string(node->text); - zdll_remove_NP_(list->first, list->last, node, next_in_hash, prev_in_hash); - list->count -= 1; + Code_Index_Note_List_New *list = code_index__list_from_string(node->text); + code_index_note_list_hash_remove(list, node); } } @@ -261,12 +352,45 @@ code_index_shift_list(Code_Index_Scope_Delim_List *list, Range_i64 old_range, u6 } } +function void +code_index_shift_list(Code_Index_Note_List *list, Range_i64 old_range, u64 new_size) +{ + i32 count = list->count; + Code_Index_Note *note = list->first; + for (i32 i = 0; i < count; i += 1, note = note->next) + { + if (old_range.min == old_range.max) + { + if (old_range.max <= note->pos.min) + { + note->pos.min = note->pos.min + new_size - (old_range.max - old_range.min); + note->pos.max = note->pos.max + new_size - (old_range.max - old_range.min); + } + } + else + { + if (old_range.min <= note->pos.min && note->pos.min < old_range.max) { + note->pos.min = old_range.first; + } else if (old_range.max <= note->pos.min) { + note->pos.min = note->pos.min + new_size - (old_range.max - old_range.min); + } + + if (old_range.min < note->pos.max && note->pos.max <= old_range.max) { + note->pos.max = old_range.first; + } else if (old_range.max <= note->pos.max) { + note->pos.max = note->pos.max + new_size - (old_range.max - old_range.min); + } + } + } +} + function void code_index_shift(Code_Index_File *file, Range_i64 old_range, u64 new_size) { // TODO(PS): @DontShiftNestList - This is unnecessary now that nest_list just gets fully rebuilt each edit code_index_shift_list(&file->nest_list, old_range, new_size); code_index_shift_list(&file->scope_delim_list, old_range, new_size); + code_index_shift_list(&file->note_list, old_range, new_size); } //////////////////////////////// diff --git a/code/custom/4coder_code_index.h b/code/custom/4coder_code_index.h index bd4590f8..6fe7c614 100644 --- a/code/custom/4coder_code_index.h +++ b/code/custom/4coder_code_index.h @@ -89,6 +89,12 @@ struct Code_Index_Note_List{ i32 count; }; +struct Code_Index_Note_List_New { + Code_Index_Note sentinel_first; + Code_Index_Note sentinel_last; + i32 count; +}; + struct Code_Index_File{ Code_Index_Scope_Delim_List scope_delim_list; Code_Index_Nest_List nest_list; @@ -96,6 +102,7 @@ struct Code_Index_File{ Buffer_ID buffer; Code_Index_Scope_Delim* scope_delim_free; Code_Index_Note* note_free; + String_Pool string_pool; }; struct Code_Index_File_Storage{ @@ -114,7 +121,7 @@ struct Code_Index{ Code_Index_File_Storage *storage_last; i32 storage_count; - Code_Index_Note_List name_hash[10000]; + Code_Index_Note_List_New name_hash[10000]; }; //////////////////////////////// diff --git a/code/custom/4coder_custom_hooks.cpp b/code/custom/4coder_custom_hooks.cpp index e7877679..e021a316 100644 --- a/code/custom/4coder_custom_hooks.cpp +++ b/code/custom/4coder_custom_hooks.cpp @@ -92,7 +92,7 @@ BUFFER_HOOK_SIG(custom_end_buffer){ /////////////////////////////////////////////////////////////////////////// BUFFER_EDIT_RANGE_SIG(custom_buffer_edit_range){ - ProfileScope(app, "default edit range"); + ProfileScope(app, "custom edit range"); Scratch_Block scratch(app); Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); @@ -101,6 +101,7 @@ BUFFER_EDIT_RANGE_SIG(custom_buffer_edit_range){ b8 trigger_code_index_update = false; { // Tree Sitter + ProfileScope(app, "Tree Sitter Shift"); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); // TODO(PS): if there's not tree_data, we actually want to block @@ -142,6 +143,7 @@ BUFFER_EDIT_RANGE_SIG(custom_buffer_edit_range){ buffer_shift_fade_ranges(buffer_id, old_range.max, (new_range.max - old_range.max)); { + ProfileScope(app, "Code Index Shift"); code_index_lock(); Code_Index_File *file = code_index_get_file(buffer_id); if (file != 0) { diff --git a/code/custom/4coder_default_bindings.cpp b/code/custom/4coder_default_bindings.cpp index 4d077748..ca832b6d 100644 --- a/code/custom/4coder_default_bindings.cpp +++ b/code/custom/4coder_default_bindings.cpp @@ -45,10 +45,10 @@ go_to_definition(Application_Links* app, String_Const_u8 lexeme, View_ID view) // and then loop if (string_match(go_to_definition_last_lexeme, lexeme)) { - Code_Index_Note_List* list = code_index__list_from_string(lexeme); + Code_Index_Note_List_New* list = code_index__list_from_string(lexeme); u64 i = 0; - for (Code_Index_Note *it = list->first; - it != 0; + for (Code_Index_Note *it = list->sentinel_first.next_in_hash; + it != &list->sentinel_last; it = it->next_in_hash, i++){ if (string_match(lexeme, it->text) && i > go_to_definition_last_lexeme_index){ note = it; diff --git a/code/custom/4coder_default_include.cpp b/code/custom/4coder_default_include.cpp index 0ecd1055..a043d23f 100644 --- a/code/custom/4coder_default_include.cpp +++ b/code/custom/4coder_default_include.cpp @@ -19,6 +19,7 @@ #include "4coder_table.h" #include "4coder_events.h" #include "4coder_types.h" +#include "4coder_string_pool.h" #include "4coder_doc_content_types.h" #include "4coder_default_colors.h" #define DYNAMIC_LINK_API @@ -30,6 +31,7 @@ #include "generated/command_metadata.h" #endif + #include "4coder_token.h" #include "generated/lexer_cpp.h" @@ -73,7 +75,7 @@ #include "4coder_stringf.cpp" #include "4coder_app_links_allocator.cpp" #include "4coder_system_allocator.cpp" - +#include "4coder_string_pool.cpp" #include "4coder_file.cpp" #define DYNAMIC_LINK_API diff --git a/code/custom/4coder_string_pool.cpp b/code/custom/4coder_string_pool.cpp new file mode 100644 index 00000000..da70e62b --- /dev/null +++ b/code/custom/4coder_string_pool.cpp @@ -0,0 +1,129 @@ + +function void +string_pool_init(String_Pool* pool) +{ + pool->free_first.next = &pool->free_last; + pool->free_last.prev = &pool->free_first; +} + +function String_Pool_Free_List* +free_string_inner(String_Pool* pool, String_Const_u8 str) +{ + String_Pool_Free_List* free_at = (String_Pool_Free_List*)str.str; + free_at->next = 0; free_at->prev = 0; + free_at->size = str.size; + + String_Pool_Free_List* prev = 0; + for (String_Pool_Free_List* at = pool->free_first.next; at != &pool->free_last; at = prev->next) + { + u8* addr = (u8*)at; + if (addr < (u8*)free_at) prev = at; + else break; + } + + if (prev) + { + String_Pool_Free_List* next = prev->next; + prev->next = free_at; + free_at->prev = prev; + free_at->next = next; + next->prev = free_at; + + b8 should_merge = (u8*)prev + prev->size == (u8*)free_at; + if (should_merge) free_at = prev; + } + else + { + String_Pool_Free_List* prev = &pool->free_first; + String_Pool_Free_List* next = pool->free_first.next; + prev->next = free_at; + free_at->prev = prev; + free_at->next = next; + next->prev = free_at; + } + + while ((u8*)free_at + free_at->size == (u8*)free_at->next) + { + String_Pool_Free_List* next = free_at->next; + free_at->size += next->size; + next->next->prev = free_at; + free_at->next = next->next; + } + + return free_at; +} + +function String_Pool_Free_List* +string_pool_push_buffer(String_Pool* pool, int size_provided, Arena* backing_arena) +{ + u64 next_buffer_size = pool->last_buffer_size * 2; + if (next_buffer_size == 0) next_buffer_size = KB(4); + String_Const_u8 buffer_data = string_const_u8_push(backing_arena, next_buffer_size); + pool->last_buffer_size = next_buffer_size; + + String_Pool_Buffer* buffer = (String_Pool_Buffer*)buffer_data.str; + buffer_data.str += sizeof(String_Pool_Buffer); + buffer_data.size -= sizeof(String_Pool_Buffer); + buffer->data = buffer_data; + buffer->next = pool->buffers; + pool->buffers = buffer; + + return free_string_inner(pool, buffer_data); +} + +function String_Const_u8 +alloc_string(String_Pool* pool, int size_provided, Arena* backing_arena) +{ + int size = ((size_provided + STRING_POOL_ALLOC_SIZE - 1) / STRING_POOL_ALLOC_SIZE) * STRING_POOL_ALLOC_SIZE; + + String_Pool_Free_List* free_at = pool->free_first.next; + while (free_at != &pool->free_last && free_at->size < size) free_at = free_at->next; + + if (free_at == &pool->free_last || free_at->size < size) + { + free_at = string_pool_push_buffer(pool, size, backing_arena); + } + Assert(free_at->size >= size); + + String_Const_u8 result; + result.str = (u8*)free_at; + result.size = size; + + String_Pool_Free_List* prev = free_at->prev; + String_Pool_Free_List* next = free_at->next; + if (free_at->size - size > 0) + { + u8* new_free_at_ptr = (u8*)free_at; + String_Pool_Free_List* new_free_at = (String_Pool_Free_List*)(new_free_at_ptr + size); + new_free_at->size = free_at->size - size; + + prev->next = new_free_at; + new_free_at->prev = prev; + new_free_at->next = next; + next->prev = new_free_at; + } + else + { + prev->next = next; + next->prev = prev; + } + + block_zero(result.str, result.size); + return result; +} + +function String_Const_u8 +alloc_string_copy(String_Pool* pool, String_Const_u8 src, Arena* backing_arena) +{ + String_Const_u8 dst = alloc_string(pool, src.size+1, backing_arena); + dst.size = src.size; + block_copy_dynamic_array(dst.str, src.str, src.size); + dst.str[src.size] = 0; + return dst; +} + +function void +free_string(String_Pool* pool, String_Const_u8 str) +{ + free_string_inner(pool, str); +} \ No newline at end of file diff --git a/code/custom/4coder_string_pool.h b/code/custom/4coder_string_pool.h new file mode 100644 index 00000000..aac08bb6 --- /dev/null +++ b/code/custom/4coder_string_pool.h @@ -0,0 +1,29 @@ +/* date = August 3rd 2025 5:01 pm */ + +#ifndef FCODER_STRING_POOL_H +#define FCODER_STRING_POOL_H + +#define STRING_POOL_ALLOC_SIZE 64 + +struct String_Pool_Free_List +{ + i64 size; + String_Pool_Free_List* next; + String_Pool_Free_List* prev; +}; + +struct String_Pool_Buffer +{ + String_Const_u8 data; + String_Pool_Buffer* next; +}; + +struct String_Pool +{ + String_Pool_Buffer* buffers; + String_Pool_Free_List free_first; + String_Pool_Free_List free_last; + u64 last_buffer_size; +}; + +#endif //FCODER_STRING_POOL_H diff --git a/code/custom/4coder_tree_sitter.cpp b/code/custom/4coder_tree_sitter.cpp index dc20fe77..af0199ab 100644 --- a/code/custom/4coder_tree_sitter.cpp +++ b/code/custom/4coder_tree_sitter.cpp @@ -482,6 +482,8 @@ tree_sitter_code_index_update_state_create( } else { state.index_arena = make_arena_system(KB(16)); state.index = push_array_zero(&state.index_arena, Code_Index_File, 1); + string_pool_init(&state.index->string_pool); + state.index->buffer = buffer_id; } state.buffer_contents = push_whole_buffer(app, scratch, state.buffer_id); @@ -489,9 +491,10 @@ tree_sitter_code_index_update_state_create( state.nest_stack_last = 0; state.last_note = 0; state.last_note_match_id = max_u32; + code_index_note_list_hash_init(&state.new_notes); state.ok = true; -/* Freeing all delims is undesired for incremental updates, and unnecessary if +/* @RemoveOldFreeingCode = Freeing all delims is undesired for incremental updates, and unnecessary if we're working with a wholly new index. The same will be true for notes soon. if (state.index->scope_delim_list.last) { state.index->scope_delim_list.last->next = state.index->scope_delim_free; @@ -548,40 +551,24 @@ tree_sitter_code_index_update_process_query_match( state->last_delim = delim; } } - else if (string_match(capture_name, SCu8("definition.class"))) - { - Code_Index_Note* note = code_index_new_note(state->index, &state->index_arena, CodeIndexNote_Type, type_range, state->nest_stack_last); - state->last_note_match_id = query_match.id; - code_index_note_insert(&state->index->note_list, state->last_note, note); - state->last_note = note; - } - else if (string_match(capture_name, SCu8("definition.function"))) - { - Code_Index_Note* note = code_index_new_note(state->index, &state->index_arena, CodeIndexNote_Function, type_range, state->nest_stack_last); - state->last_note_match_id = query_match.id; - code_index_note_insert(&state->index->note_list, state->last_note, note); - state->last_note = note; - } - else if (string_match(capture_name, SCu8("definition.method"))) - { - Code_Index_Note* note = code_index_new_note(state->index, &state->index_arena, CodeIndexNote_Function, type_range, state->nest_stack_last);; - state->last_note_match_id = query_match.id; - code_index_note_insert(&state->index->note_list, state->last_note, note); - state->last_note = note; - } - else if (string_match(capture_name, SCu8("definition.type"))) - { - Code_Index_Note* note = code_index_new_note(state->index, &state->index_arena, CodeIndexNote_Type, type_range, state->nest_stack_last); - state->last_note_match_id = query_match.id; - code_index_note_insert(&state->index->note_list, state->last_note, note); - state->last_note = note; - } else if (string_match(capture_name, SCu8("name"))) { - if (state->last_note != 0 && state->last_note_match_id == query_match.id) - { - state->last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start); - } + Range_i64 range = Ii64_size(type_range.start, type_range.end - type_range.start); + bool apply_name_to_last_note = state->new_notes.sentinel_last.prev_in_hash != 0; + apply_name_to_last_note &= state->last_note_match_id == query_match.id; + if (apply_name_to_last_note) state->new_notes.sentinel_last.prev_in_hash->pos = range; + } + else + { + Code_Index_Note_Kind kind; + if (string_match(capture_name, SCu8("definition.class"))) kind = CodeIndexNote_Type; + else if (string_match(capture_name, SCu8("definition.function"))) kind = CodeIndexNote_Function; + else if (string_match(capture_name, SCu8("definition.method"))) kind = CodeIndexNote_Function; + else if (string_match(capture_name, SCu8("definition.type"))) kind = CodeIndexNote_Type; + + Code_Index_Note* note = code_index_new_note(state->index, &state->index_arena, kind, type_range, state->nest_stack_last); + code_index_note_list_hash_append(&state->new_notes, note); + state->last_note_match_id = query_match.id; } } @@ -592,6 +579,7 @@ tree_sitter_code_index_update_complete( Arena* scratch, bool update_was_incremental ){ + ProfileScope(app, "Complete Code Index Update"); if (state->ok) { Code_Index_Nest* free_nests = state->index->nest_list.first; @@ -667,11 +655,34 @@ tree_sitter_code_index_update_complete( delim_at = delim_at->next; } - // TODO(PS): this should just be for the new notes - if a note wasn't - // updated incrementally this frame, then it already has it's text - for (Code_Index_Note* note = state->index->note_list.first; note != 0 && note->next != note; note = note->next) { - note->text = push_string_copy(&state->index_arena, string_substring(state->buffer_contents, note->pos)); + ProfileScope(app, "Update Global Name Table"); + Code_Index_Note* note = state->new_notes.sentinel_first.next_in_hash; + Code_Index_Note* next = 0; + for (int i = 0; i < state->new_notes.count; i++) + { + next = note->next_in_hash; + note->next = 0; note->next_in_hash = 0; note->prev_in_hash = 0; + + String_Const_u8 source = string_substring( + state->buffer_contents, + note->pos + ); + note->text = alloc_string_copy( + &state->index->string_pool, + source, + &state->index_arena + ); + note->file = state->index; + + Code_Index_Note_List_New* hash_list = code_index__list_from_string(note->text); + code_index_note_list_hash_append(hash_list, note); + + code_index_note_insert(&state->index->note_list, state->last_note, note); + state->last_note = note; + + note = next; + } } if (!update_was_incremental) @@ -771,14 +782,12 @@ tree_sitter_code_index_update_tick(Application_Links* app) modified_node != 0; modified_node = modified_node->next ){ + ProfileScope(app, "Incremental Parse"); Buffer_ID buffer_id = modified_node->buffer; Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); if (!tree_data || !tree_data->tree) continue; - Buffer_ID out_buffer = get_buffer_by_name(app, string_u8_litexpr("*tree*"), Access_Always); - buffer_replace_range(app, out_buffer, Ii64(0,buffer_get_size(app, out_buffer)), SCu8("")); - Range_i64 old_range = tree_data->last_update_old_range; Range_i64 new_range = tree_data->last_update_new_range; if (old_range == new_range) continue; @@ -877,94 +886,102 @@ tree_sitter_code_index_update_tick(Application_Links* app) tree_data->last_update_node_range = edit_range; // TODO(PS): TEMP - remove me once debugging is done // Free Scope Delimiters that fall within old_range - Code_Index_Scope_Delim* delim = state.index->scope_delim_list.first; - Code_Index_Scope_Delim* delim_before_range = 0; - Code_Index_Scope_Delim* after_range = 0; - while (delim) { - Code_Index_Scope_Delim* next = delim->next; - if (delim->pos.min < edit_range.min && delim->pos.max <= edit_range.min) + ProfileScope(app, "Free Scope Delimiters"); + Code_Index_Scope_Delim* delim = state.index->scope_delim_list.first; + Code_Index_Scope_Delim* delim_before_range = 0; + Code_Index_Scope_Delim* after_range = 0; + while (delim) { - delim_before_range = delim; + Code_Index_Scope_Delim* next = delim->next; + if (delim->pos.min < edit_range.min && delim->pos.max <= edit_range.min) + { + delim_before_range = delim; + } + if (range_overlap(delim->pos, edit_range)) + { + state.index->scope_delim_list.count -= 1; + zdll_remove( + state.index->scope_delim_list.first, + state.index->scope_delim_list.last, + delim + ); + code_index_free_scope_delim(state.index, delim); + } + if (delim->pos.min >= edit_range.max) break; + delim = next; } - if (range_overlap(delim->pos, edit_range)) - { - state.index->scope_delim_list.count -= 1; - zdll_remove( - state.index->scope_delim_list.first, - state.index->scope_delim_list.last, - delim - ); - code_index_free_scope_delim(state.index, delim); - } - if (delim->pos.min >= edit_range.max) break; - delim = next; + state.last_delim = delim_before_range; } - state.last_delim = delim_before_range; // Free Scope Notes that fall within old_range - Code_Index_Note* note = state.index->note_list.first; - Code_Index_Note* prev = 0; - Code_Index_Note* note_before_range = 0; - while (note) { - Code_Index_Note* next = note->next; - if (note->pos.min < edit_range.min && note->pos.max <= edit_range.min) + ProfileScope(app, "Free Notes"); + Code_Index_Note* note = state.index->note_list.first; + Code_Index_Note* prev = 0; + Code_Index_Note* note_before_range = 0; + while (note) { - note_before_range = note; - } - if (range_overlap(note->pos, edit_range)) - { - Code_Index_Note* new_first = state.index->note_list.first; - Code_Index_Note* new_last = state.index->note_list.last; - Code_Index_Note* new_next = next; - - if (note == state.index->note_list.first) new_first = next; - if (note == state.index->note_list.last) + Code_Index_Note* next = note->next; + if (note->pos.min < edit_range.min && note->pos.max <= edit_range.min) { - if (prev) new_last = prev; - else new_last = 0; + note_before_range = note; } + if (range_overlap(note->pos, edit_range)) + { + Code_Index_Note* new_first = state.index->note_list.first; + Code_Index_Note* new_last = state.index->note_list.last; + Code_Index_Note* new_next = next; - if (new_first == 0) Assert(new_last == 0 && state.index->note_list.count == 1); + if (note == state.index->note_list.first) new_first = next; + if (note == state.index->note_list.last) + { + if (prev) new_last = prev; + else new_last = 0; + } - state.index->note_list.count -= 1; - state.index->note_list.first = new_first; - state.index->note_list.last = new_last; - if (prev) prev->next = new_next; - code_index_free_note(state.index, note); + if (new_first == 0) Assert(new_last == 0 && state.index->note_list.count == 1); + + Code_Index_Note_List_New* list = code_index__list_from_string(note->text); + code_index_note_list_hash_remove(list, note); + + free_string(&state.index->string_pool, note->text); + + state.index->note_list.count -= 1; + state.index->note_list.first = new_first; + state.index->note_list.last = new_last; + if (prev) prev->next = new_next; + code_index_free_note(state.index, note); + } + else + { + prev = note; + } + if (note->pos.min >= edit_range.max) break; + note = next; } - else - { - prev = note; - } - if (note->pos.min >= edit_range.max) break; - note = next; + state.last_note = note_before_range; + if (state.index->note_list.count == 0) Assert(state.index->note_list.first == 0 && state.index->note_list.first == 0); } - state.last_note = note_before_range; - if (state.index->note_list.count == 0) Assert(state.index->note_list.first == 0 && state.index->note_list.first == 0); - for (int i = 0; i < query_count; i++) { - Tree_Sitter_Query_Cursor query = queries[i]; - TSQueryMatch query_match; - u32 capture_index; - while (state.ok && tree_sitter_query_continue(&query, &query_match, &capture_index)) + ProfileScope(app, "Perform Query"); + for (int i = 0; i < query_count; i++) { - tree_sitter_code_index_update_process_query_match( - &state, query, query_match, capture_index, scratch - ); + Tree_Sitter_Query_Cursor query = queries[i]; + TSQueryMatch query_match; + u32 capture_index; + while (state.ok && tree_sitter_query_continue(&query, &query_match, &capture_index)) + { + tree_sitter_code_index_update_process_query_match( + &state, query, query_match, capture_index, scratch + ); + } + tree_sitter_query_end(&query); } - tree_sitter_query_end(&query); } + tree_sitter_code_index_update_complete(app, &state, scratch, true); - - // @Report - for (Code_Index_Note* note = state.index->note_list.first; note != 0; note = note->next) - { - String_Const_u8 string = push_stringf(scratch, "Note: '%.*s'\n", string_expand(note->text)); - buffer_replace_range(app, out_buffer, Ii64(buffer_get_size(app, out_buffer)), string); - } } buffer_modified_set_clear(); diff --git a/code/custom/4coder_tree_sitter.h b/code/custom/4coder_tree_sitter.h index c816a943..4b0c7bc8 100644 --- a/code/custom/4coder_tree_sitter.h +++ b/code/custom/4coder_tree_sitter.h @@ -92,6 +92,8 @@ struct Tree_Sitter_Code_Index_Update_State Arena index_arena; Code_Index_File* index; + Code_Index_Note_List_New new_notes; + Code_Index_Nest_Stack* nest_stack_first = 0; Code_Index_Nest_Stack* nest_stack_last = 0;