diff --git a/code/custom/4coder_tree_sitter.cpp b/code/custom/4coder_tree_sitter.cpp index 0728fc77..19dcdf42 100644 --- a/code/custom/4coder_tree_sitter.cpp +++ b/code/custom/4coder_tree_sitter.cpp @@ -103,6 +103,34 @@ tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id) return tree_sitter_language_for_buffer(app, buffer_id, scratch); } +//////////////////////////////////////////////////////////////////// +// Async Management +//////////////////////////////////////////////////////////////////// + +function void +async_task_cancel_nowait(Application_Links* app, Async_System* async_system, Async_Task task) +{ + system_mutex_acquire(async_system->mutex); + Async_Node *node = async_get_pending_node(async_system, task); + if (node != 0) + { + dll_remove(&node->node); + async_system->task_count -= 1; + async_free_node(async_system, node); + } + else + { + node = async_get_running_node(async_system, task); + if (node != 0) + { + b32 *cancel_signal = &node->thread->cancel_signal; + atomic_write_b32(cancel_signal, true); + // async_task_wait__inner(app, async_system, task); + } + } + system_mutex_release(async_system->mutex); +} + ///////////////////////////////////////////// // Tree Sitter Hook Internals ///////////////////////////////////////////// @@ -264,10 +292,10 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id) tree_data->tree = new_tree; system_mutex_acquire(tree_data->tree_mutex); - Tree_Sitter_Code_Index_Parse_State parse_state = tree_sitter_code_index_update_single_buffer(app, buffer_id); - code_index_lock(); - code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index); - code_index_unlock(); + Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); + Async_Task* lex_task = scope_attachment(app, scope, buffer_tree_sitter_update_code_index_task_id, Async_Task); + if (lex_task != 0) async_task_cancel_nowait(app, &global_async_system, *lex_task); + *lex_task = async_task_no_dep(&global_async_system, tree_sitter_code_index_update_async, make_data_struct(&buffer_id)); // Force a frame refresh by requesting another frame animate_in_n_milliseconds(app, 0); @@ -289,6 +317,10 @@ tree_sitter_parse_async(Async_Context* actx, String_Const_u8 data) tree_sitter_parse_async__inner(actx, buffer_id); } +//////////////////////////////////////////////////////////////////// +// Code Index Updating +//////////////////////////////////////////////////////////////////// + function Range_i64 tree_sitter_node_to_range(TSNode node) { @@ -312,30 +344,34 @@ code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind k } function Tree_Sitter_Code_Index_Parse_State -tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID buffer_id) +tree_sitter_code_index_update_single_buffer(Async_Context* actx, Buffer_ID buffer_id) { - Scratch_Block scratch(app); + Application_Links* app = actx->app; - Tree_Sitter_Code_Index_Parse_State parse_state = {}; - parse_state.ok = false; - parse_state.buffer_id = buffer_id; - parse_state.language = tree_sitter_language_for_buffer(app, parse_state.buffer_id); - if (!parse_state.language) return parse_state; + acquire_global_frame_mutex(app); + Scratch_Block scratch(app); - parse_state.index_arena = make_arena_system(KB(16)); - parse_state.index = push_array_zero(&parse_state.index_arena, Code_Index_File, 1); - parse_state.buffer_contents = push_whole_buffer(app, scratch, parse_state.buffer_id); - parse_state.query = tree_sitter_query_init( - app, parse_state.buffer_id, parse_state.language->queries.ptr[Tree_Sitter_Language_Query_Tags] - ); - parse_state.nest_stack_first = 0; - parse_state.nest_stack_last = 0; - parse_state.last_note = 0; - parse_state.last_note_match_id = max_u32; + Tree_Sitter_Code_Index_Parse_State parse_state = {}; + parse_state.ok = false; + parse_state.buffer_id = buffer_id; + parse_state.language = tree_sitter_language_for_buffer(app, parse_state.buffer_id); + if (!parse_state.language) return parse_state; + parse_state.index_arena = make_arena_system(KB(16)); + parse_state.index = push_array_zero(&parse_state.index_arena, Code_Index_File, 1); + parse_state.buffer_contents = push_whole_buffer(app, scratch, parse_state.buffer_id); + parse_state.query = tree_sitter_query_init( + app, parse_state.buffer_id, parse_state.language->queries.ptr[Tree_Sitter_Language_Query_Tags] + ); + parse_state.nest_stack_first = 0; + parse_state.nest_stack_last = 0; + parse_state.last_note = 0; + parse_state.last_note_match_id = max_u32; + release_global_frame_mutex(app); + parse_state.ok = true; TSQueryMatch query_match; u32 capture_index; - while (tree_sitter_query_continue(&parse_state.query, &query_match, &capture_index)) + while (parse_state.ok && tree_sitter_query_continue(&parse_state.query, &query_match, &capture_index)) { TSQueryCapture type_capture = query_match.captures[capture_index]; @@ -408,116 +444,74 @@ tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID bu parse_state.last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start); } } - } - while (parse_state.nest_stack_last != 0) - { - Code_Index_Nest* nest = parse_state.nest_stack_last->nest; - if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest); - else code_index_push_nest(&parse_state.index->nest_list, nest); - nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list); - parse_state.nest_stack_last = parse_state.nest_stack_last->prev; + if (async_check_canceled(actx)) + { + parse_state.ok = false; + break; + } } - for (Code_Index_Note* note = parse_state.index->note_list.first; note != 0 && note->next != note; note = note->next) - { - note->text = push_string_copy(&parse_state.index_arena, string_substring(parse_state.buffer_contents, note->pos)); - } - - // Finish the Index - parse_state.index->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->nest_list); - parse_state.index->note_array = code_index_note_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->note_list); - tree_sitter_query_end(&parse_state.query); - parse_state.ok = true; + if (parse_state.ok) + { + while (parse_state.nest_stack_last != 0) + { + Code_Index_Nest* nest = parse_state.nest_stack_last->nest; + if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest); + else code_index_push_nest(&parse_state.index->nest_list, nest); + nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list); + parse_state.nest_stack_last = parse_state.nest_stack_last->prev; + } + + for (Code_Index_Note* note = parse_state.index->note_list.first; note != 0 && note->next != note; note = note->next) + { + note->text = push_string_copy(&parse_state.index_arena, string_substring(parse_state.buffer_contents, note->pos)); + } + + // Finish the Index + parse_state.index->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->nest_list); + parse_state.index->note_array = code_index_note_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->note_list); + + code_index_lock(); + code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index); + code_index_unlock(); + + buffer_clear_layout_cache(app, parse_state.buffer_id); + } + else + { + linalloc_clear(&parse_state.index_arena); + } + return parse_state; } -#define BUFFER_CODE_INDEX_UPDATES false +function void +tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data) +{ + if (data.size != sizeof(Buffer_ID)) return; + Buffer_ID buffer_id = *(Buffer_ID*)data.str; + tree_sitter_code_index_update_single_buffer(actx, buffer_id); +} function void tree_sitter_code_index_update_tick(Application_Links* app) { -#define BUFFERED_CODE_INDICES_CAP 8 - Tree_Sitter_Code_Index_Parse_State buffered_code_indices[BUFFERED_CODE_INDICES_CAP]; - u32 buffered_code_indices_count = 0; - - Tree_Sitter_Code_Index_Parse_State parse_state = {}; for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first; modified_node != 0; modified_node = modified_node->next ){ - parse_state = tree_sitter_code_index_update_single_buffer(app, modified_node->buffer); - if (parse_state.ok) - { - #if BUFFER_CODE_INDEX_UPDATES - buffered_code_indices[buffered_code_indices_count] = parse_state; - buffered_code_indices_count += 1; - #else - code_index_lock(); - code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index); - code_index_unlock(); - #endif - } - - if (buffered_code_indices_count >= BUFFERED_CODE_INDICES_CAP) - { - code_index_lock(); - for (int i = 0; i < buffered_code_indices_count; i++) - { - parse_state = buffered_code_indices[i]; - code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index); - } - buffered_code_indices_count = 0; - code_index_unlock(); - } - - buffer_clear_layout_cache(app, parse_state.buffer_id); + Buffer_ID buffer_id = modified_node->buffer; + Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); + Async_Task* lex_task = scope_attachment(app, scope, buffer_tree_sitter_update_code_index_task_id, Async_Task); + if (lex_task != 0) async_task_cancel_nowait(app, &global_async_system, *lex_task); + *lex_task = async_task_no_dep(&global_async_system, tree_sitter_code_index_update_async, make_data_struct(&buffer_id)); } - - if (buffered_code_indices_count > 0) - { - code_index_lock(); - for (int i = 0; i < buffered_code_indices_count; i++) - { - parse_state = buffered_code_indices[i]; - code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index); - } - code_index_unlock(); - } - buffer_modified_set_clear(); } -//////////////////////////////////////////////////////////////////// -// Async Management -//////////////////////////////////////////////////////////////////// - -function void -async_task_cancel_nowait(Application_Links* app, Async_System* async_system, Async_Task task) -{ - system_mutex_acquire(async_system->mutex); - Async_Node *node = async_get_pending_node(async_system, task); - if (node != 0) - { - dll_remove(&node->node); - async_system->task_count -= 1; - async_free_node(async_system, node); - } - else - { - node = async_get_running_node(async_system, task); - if (node != 0) - { - b32 *cancel_signal = &node->thread->cancel_signal; - atomic_write_b32(cancel_signal, true); - // async_task_wait__inner(app, async_system, task); - } - } - system_mutex_release(async_system->mutex); -} - //////////////////////////////////////////////////////////////////// // Token Highlighting //////////////////////////////////////////////////////////////////// diff --git a/code/custom/4coder_tree_sitter.h b/code/custom/4coder_tree_sitter.h index e5a79581..f6983cb9 100644 --- a/code/custom/4coder_tree_sitter.h +++ b/code/custom/4coder_tree_sitter.h @@ -92,6 +92,7 @@ global Tree_Sitter_Languages tree_sitter_languages; CUSTOM_ID(attachment, buffer_tree_sitter_data_id); CUSTOM_ID(attachment, buffer_tree_sitter_parse_task_id); +CUSTOM_ID(attachment, buffer_tree_sitter_update_code_index_task_id); b8 use_tree_sitter_code_indexing = true; b8 use_tree_sitter_token_coloring = true; @@ -104,6 +105,7 @@ function void tree_sitter_register_language(String_Const_u8 ext, TSLanguage* lan function Tree_Sitter_Language_Definition* tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id); function Tree_Sitter_Code_Index_Parse_State tree_sitter_code_index_update_single_buffer(Application_Links *app, Buffer_ID buffer_id); -function void tree_sitter_code_index_update_tick(Application_Links *app); +function void tree_sitter_code_index_update_tick(Application_Links *app); +function void tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data); #endif //FCODER_TREE_SITTER_H