Move tree-sitter code index updating to an async process to unblock the ui while processing large files

code/custom/4coder_custom_hooks.cpp

@@ -281,20 +281,16 @@ function void custom_render_buffer(
     f32 mark_thickness = (f32)def_get_config_u64(app, vars_save_string_lit("mark_thickness"));
 
     // NOTE(allen): Token colorizing
-    Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer);
+    Token_Array token_array;
-    Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
-    TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data);
-
-    Token_Array token_array = get_token_array_from_buffer(app, buffer);
     paint_text_color_fcolor(app, text_layout_id, visible_range, fcolor_id(defcolor_text_default)); // will get overridden by lang-specific token coloring below
-    if (token_array.tokens != 0 && tree)
+    if (use_tree_sitter_token_coloring)
     {
-      if (use_tree_sitter_token_coloring)
+      draw_tree_sitter_node_colors(app, text_layout_id, buffer);
-      {
+    }
-        draw_tree_sitter_node_colors(app, text_layout_id, buffer);
+    else
-      }
+    {
-      else
+      token_array = get_token_array_from_buffer(app, buffer);
-      {
+      if (token_array.tokens != 0) {
         draw_cpp_token_colors(app, text_layout_id, &token_array);
       }
     }
@@ -359,13 +355,17 @@ function void custom_render_buffer(
     // NOTE(allen): Whitespace highlight
     b64 show_whitespace = false;
     view_get_setting(app, view_id, ViewSetting_ShowWhitespace, &show_whitespace);
-    if (show_whitespace){
+    if (show_whitespace)
-        if (token_array.tokens == 0){
+    {
-            draw_whitespace_highlight(app, buffer, text_layout_id, cursor_roundness);
+      if (token_array.tokens == 0) token_array = get_token_array_from_buffer(app, buffer);
-        }
+      if (token_array.tokens == 0)
-        else{
+      {
-            draw_whitespace_highlight(app, text_layout_id, &token_array, cursor_roundness);
+          draw_whitespace_highlight(app, buffer, text_layout_id, cursor_roundness);
-        }
+      }
+      else
+      {
+          draw_whitespace_highlight(app, text_layout_id, &token_array, cursor_roundness);
+      }
     }
 
     // NOTE(allen): Cursor

code/custom/4coder_helper.cpp

@@ -61,6 +61,21 @@ get_token_array_from_buffer(Application_Links *app, Buffer_ID buffer){
     return(result);
 }
 
+function Token_Array
+get_token_array_from_buffer_no_wait(Application_Links *app, Buffer_ID buffer)
+{
+    Token_Array result = {};
+    Managed_Scope scope = buffer_get_managed_scope(app, buffer);
+    Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task);
+    if (lex_task_ptr == 0){
+      Token_Array *ptr = scope_attachment(app, scope, attachment_tokens, Token_Array);
+      if (ptr != 0){
+          result = *ptr;
+      }
+    }
+    return(result);
+}
+
 ////////////////////////////////
 
 function Buffer_Seek

code/custom/4coder_tree_sitter.cpp

@@ -103,6 +103,34 @@ tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id)
   return tree_sitter_language_for_buffer(app, buffer_id, scratch);
 }
 
+////////////////////////////////////////////////////////////////////
+// Async Management
+////////////////////////////////////////////////////////////////////
+
+function void
+async_task_cancel_nowait(Application_Links* app, Async_System* async_system, Async_Task task)
+{
+  system_mutex_acquire(async_system->mutex);
+  Async_Node *node = async_get_pending_node(async_system, task);
+  if (node != 0)
+  {
+    dll_remove(&node->node);
+    async_system->task_count -= 1;
+    async_free_node(async_system, node);
+  }
+  else
+  {
+    node = async_get_running_node(async_system, task);
+    if (node != 0)
+    {
+      b32 *cancel_signal = &node->thread->cancel_signal;
+      atomic_write_b32(cancel_signal, true);
+      // async_task_wait__inner(app, async_system, task);
+    }
+  }
+  system_mutex_release(async_system->mutex);
+}
+
 /////////////////////////////////////////////
 // Tree Sitter Hook Internals
 /////////////////////////////////////////////
@@ -264,10 +292,10 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id)
       tree_data->tree = new_tree;
       system_mutex_acquire(tree_data->tree_mutex);
 
-      Tree_Sitter_Code_Index_Parse_State parse_state = tree_sitter_code_index_update_single_buffer(app, buffer_id);
+      Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
-      code_index_lock();
+      Async_Task* lex_task = scope_attachment(app, scope, buffer_tree_sitter_update_code_index_task_id, Async_Task);
-      code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
+      if (lex_task != 0) async_task_cancel_nowait(app, &global_async_system, *lex_task);
-      code_index_unlock();
+      *lex_task = async_task_no_dep(&global_async_system, tree_sitter_code_index_update_async, make_data_struct(&buffer_id));
 
       // Force a frame refresh by requesting another frame
       animate_in_n_milliseconds(app, 0);
@@ -289,6 +317,10 @@ tree_sitter_parse_async(Async_Context* actx, String_Const_u8 data)
   tree_sitter_parse_async__inner(actx, buffer_id);
 }
 
+////////////////////////////////////////////////////////////////////
+// Code Index Updating
+////////////////////////////////////////////////////////////////////
+
 function Range_i64
 tree_sitter_node_to_range(TSNode node)
 {
@@ -312,30 +344,34 @@ code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind k
 }
 
 function Tree_Sitter_Code_Index_Parse_State
-tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID buffer_id)
+tree_sitter_code_index_update_single_buffer(Async_Context* actx, Buffer_ID buffer_id)
 {
-  Scratch_Block scratch(app);
+  Application_Links* app = actx->app;
 
-  Tree_Sitter_Code_Index_Parse_State parse_state = {};
+  acquire_global_frame_mutex(app);
-  parse_state.ok = false;
+    Scratch_Block scratch(app);
-  parse_state.buffer_id = buffer_id;
-  parse_state.language = tree_sitter_language_for_buffer(app, parse_state.buffer_id);
-  if (!parse_state.language) return parse_state;
 
-  parse_state.index_arena = make_arena_system(KB(16));
+    Tree_Sitter_Code_Index_Parse_State parse_state = {};
-  parse_state.index = push_array_zero(&parse_state.index_arena, Code_Index_File, 1);
+    parse_state.ok = false;
-  parse_state.buffer_contents = push_whole_buffer(app, scratch, parse_state.buffer_id);
+    parse_state.buffer_id = buffer_id;
-  parse_state.query = tree_sitter_query_init(
+    parse_state.language = tree_sitter_language_for_buffer(app, parse_state.buffer_id);
-    app, parse_state.buffer_id, parse_state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
+    if (!parse_state.language) return parse_state;
-  );
+    parse_state.index_arena = make_arena_system(KB(16));
-  parse_state.nest_stack_first = 0;
+    parse_state.index = push_array_zero(&parse_state.index_arena, Code_Index_File, 1);
-  parse_state.nest_stack_last = 0;
+    parse_state.buffer_contents = push_whole_buffer(app, scratch, parse_state.buffer_id);
-  parse_state.last_note = 0;
+    parse_state.query = tree_sitter_query_init(
-  parse_state.last_note_match_id = max_u32;
+      app, parse_state.buffer_id, parse_state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
+    );
+    parse_state.nest_stack_first = 0;
+    parse_state.nest_stack_last = 0;
+    parse_state.last_note = 0;
+    parse_state.last_note_match_id = max_u32;
+  release_global_frame_mutex(app);
 
+  parse_state.ok = true;
   TSQueryMatch query_match;
   u32 capture_index;
-  while (tree_sitter_query_continue(&parse_state.query, &query_match, &capture_index))
+  while (parse_state.ok && tree_sitter_query_continue(&parse_state.query, &query_match, &capture_index))
   {
     TSQueryCapture type_capture = query_match.captures[capture_index];
 
@@ -408,116 +444,74 @@ tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID bu
         parse_state.last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start);
       }
     }
-  }
 
-  while (parse_state.nest_stack_last != 0)
+    if (async_check_canceled(actx))
-  {
+    {
-    Code_Index_Nest* nest = parse_state.nest_stack_last->nest;
+      parse_state.ok = false;
-    if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest);
+      break;
-    else                   code_index_push_nest(&parse_state.index->nest_list, nest);
+    }
-    nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list);
-    parse_state.nest_stack_last = parse_state.nest_stack_last->prev;
   }
 
-  for (Code_Index_Note* note = parse_state.index->note_list.first; note != 0 && note->next != note; note = note->next)
-  {
-    note->text = push_string_copy(&parse_state.index_arena, string_substring(parse_state.buffer_contents, note->pos));
-  }
-
-  // Finish the Index
-  parse_state.index->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->nest_list);
-  parse_state.index->note_array = code_index_note_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->note_list);
-
   tree_sitter_query_end(&parse_state.query);
 
-  parse_state.ok = true;
+  if (parse_state.ok)
+  {
+    while (parse_state.nest_stack_last != 0)
+    {
+      Code_Index_Nest* nest = parse_state.nest_stack_last->nest;
+      if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest);
+      else                   code_index_push_nest(&parse_state.index->nest_list, nest);
+      nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list);
+      parse_state.nest_stack_last = parse_state.nest_stack_last->prev;
+    }
+
+    for (Code_Index_Note* note = parse_state.index->note_list.first; note != 0 && note->next != note; note = note->next)
+    {
+      note->text = push_string_copy(&parse_state.index_arena, string_substring(parse_state.buffer_contents, note->pos));
+    }
+
+    // Finish the Index
+    parse_state.index->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->nest_list);
+    parse_state.index->note_array = code_index_note_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->note_list);
+
+    code_index_lock();
+    code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
+    code_index_unlock();
+
+    buffer_clear_layout_cache(app, parse_state.buffer_id);
+  }
+  else
+  {
+    linalloc_clear(&parse_state.index_arena);
+  }
+
   return parse_state;
 }
 
-#define BUFFER_CODE_INDEX_UPDATES false
+function void
+tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data)
+{
+  if (data.size != sizeof(Buffer_ID)) return;
+  Buffer_ID buffer_id = *(Buffer_ID*)data.str;
+  tree_sitter_code_index_update_single_buffer(actx, buffer_id);
+}
 
 function void
 tree_sitter_code_index_update_tick(Application_Links* app)
 {
-#define BUFFERED_CODE_INDICES_CAP 8
-  Tree_Sitter_Code_Index_Parse_State buffered_code_indices[BUFFERED_CODE_INDICES_CAP];
-  u32 buffered_code_indices_count = 0;
-
-  Tree_Sitter_Code_Index_Parse_State parse_state = {};
   for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first;
        modified_node != 0;
        modified_node = modified_node->next
   ){
-    parse_state = tree_sitter_code_index_update_single_buffer(app, modified_node->buffer);
+    Buffer_ID buffer_id = modified_node->buffer;
-    if (parse_state.ok)
+    Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
-    {
+    Async_Task* lex_task = scope_attachment(app, scope, buffer_tree_sitter_update_code_index_task_id, Async_Task);
-      #if BUFFER_CODE_INDEX_UPDATES
+    if (lex_task != 0) async_task_cancel_nowait(app, &global_async_system, *lex_task);
-      buffered_code_indices[buffered_code_indices_count] = parse_state;
+    *lex_task = async_task_no_dep(&global_async_system, tree_sitter_code_index_update_async, make_data_struct(&buffer_id));
-      buffered_code_indices_count += 1;
-      #else
-      code_index_lock();
-      code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
-      code_index_unlock();
-      #endif
-    }
-
-    if (buffered_code_indices_count >= BUFFERED_CODE_INDICES_CAP)
-    {
-      code_index_lock();
-      for (int i = 0; i < buffered_code_indices_count; i++)
-      {
-        parse_state = buffered_code_indices[i];
-        code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
-      }
-      buffered_code_indices_count = 0;
-      code_index_unlock();
-    }
-
-    buffer_clear_layout_cache(app, parse_state.buffer_id);
   }
-
-  if (buffered_code_indices_count > 0)
-  {
-    code_index_lock();
-    for (int i = 0; i < buffered_code_indices_count; i++)
-    {
-      parse_state = buffered_code_indices[i];
-      code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
-    }
-    code_index_unlock();
-  }
-
   buffer_modified_set_clear();
 }
 
-////////////////////////////////////////////////////////////////////
-// Async Management
-////////////////////////////////////////////////////////////////////
-
-function void
-async_task_cancel_nowait(Application_Links* app, Async_System* async_system, Async_Task task)
-{
-  system_mutex_acquire(async_system->mutex);
-  Async_Node *node = async_get_pending_node(async_system, task);
-  if (node != 0)
-  {
-    dll_remove(&node->node);
-    async_system->task_count -= 1;
-    async_free_node(async_system, node);
-  }
-  else
-  {
-    node = async_get_running_node(async_system, task);
-    if (node != 0)
-    {
-      b32 *cancel_signal = &node->thread->cancel_signal;
-      atomic_write_b32(cancel_signal, true);
-      // async_task_wait__inner(app, async_system, task);
-    }
-  }
-  system_mutex_release(async_system->mutex);
-}
-
 ////////////////////////////////////////////////////////////////////
 // Token Highlighting
 ////////////////////////////////////////////////////////////////////
@@ -577,6 +571,7 @@ function void
 draw_tree_sitter_node_colors(Application_Links* app, Text_Layout_ID text_layout_id, Buffer_ID buffer_id)
 {
   Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
+  if (!lang) return;
   TSQuery* query = lang->queries.ptr[Tree_Sitter_Language_Query_Highlights];
 
   Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id);

code/custom/4coder_tree_sitter.h

@@ -92,6 +92,7 @@ global Tree_Sitter_Languages tree_sitter_languages;
 
 CUSTOM_ID(attachment, buffer_tree_sitter_data_id);
 CUSTOM_ID(attachment, buffer_tree_sitter_parse_task_id);
+CUSTOM_ID(attachment, buffer_tree_sitter_update_code_index_task_id);
 
 b8 use_tree_sitter_code_indexing = true;
 b8 use_tree_sitter_token_coloring = true;
@@ -104,6 +105,7 @@ function void tree_sitter_register_language(String_Const_u8 ext, TSLanguage* lan
 function Tree_Sitter_Language_Definition* tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id);
 
 function Tree_Sitter_Code_Index_Parse_State tree_sitter_code_index_update_single_buffer(Application_Links *app, Buffer_ID buffer_id);
-function void tree_sitter_code_index_update_tick(Application_Links *app);
+function void                               tree_sitter_code_index_update_tick(Application_Links *app);
+function void                               tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data);
 
 #endif //FCODER_TREE_SITTER_H