Move tree-sitter code index updating to an async process to unblock the ui while processing large files

code/custom/4coder_custom_hooks.cpp

@@ -281,20 +281,16 @@ function void custom_render_buffer(
     f32 mark_thickness = (f32)def_get_config_u64(app, vars_save_string_lit("mark_thickness"));
 
     // NOTE(allen): Token colorizing
-    Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer);
-    Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
-    TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data);
-
-    Token_Array token_array = get_token_array_from_buffer(app, buffer);
+    Token_Array token_array;
     paint_text_color_fcolor(app, text_layout_id, visible_range, fcolor_id(defcolor_text_default)); // will get overridden by lang-specific token coloring below
-    if (token_array.tokens != 0 && tree)
+    if (use_tree_sitter_token_coloring)
     {
-      if (use_tree_sitter_token_coloring)
-      {
-        draw_tree_sitter_node_colors(app, text_layout_id, buffer);
-      }
-      else
-      {
+      draw_tree_sitter_node_colors(app, text_layout_id, buffer);
+    }
+    else
+    {
+      token_array = get_token_array_from_buffer(app, buffer);
+      if (token_array.tokens != 0) {
         draw_cpp_token_colors(app, text_layout_id, &token_array);
       }
     }
@@ -359,13 +355,17 @@ function void custom_render_buffer(
     // NOTE(allen): Whitespace highlight
     b64 show_whitespace = false;
     view_get_setting(app, view_id, ViewSetting_ShowWhitespace, &show_whitespace);
-    if (show_whitespace){
-        if (token_array.tokens == 0){
-            draw_whitespace_highlight(app, buffer, text_layout_id, cursor_roundness);
-        }
-        else{
-            draw_whitespace_highlight(app, text_layout_id, &token_array, cursor_roundness);
-        }
+    if (show_whitespace)
+    {
+      if (token_array.tokens == 0) token_array = get_token_array_from_buffer(app, buffer);
+      if (token_array.tokens == 0)
+      {
+          draw_whitespace_highlight(app, buffer, text_layout_id, cursor_roundness);
+      }
+      else
+      {
+          draw_whitespace_highlight(app, text_layout_id, &token_array, cursor_roundness);
+      }
     }
 
     // NOTE(allen): Cursor

code/custom/4coder_helper.cpp

@@ -61,6 +61,21 @@ get_token_array_from_buffer(Application_Links *app, Buffer_ID buffer){
     return(result);
 }
 
+function Token_Array
+get_token_array_from_buffer_no_wait(Application_Links *app, Buffer_ID buffer)
+{
+    Token_Array result = {};
+    Managed_Scope scope = buffer_get_managed_scope(app, buffer);
+    Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task);
+    if (lex_task_ptr == 0){
+      Token_Array *ptr = scope_attachment(app, scope, attachment_tokens, Token_Array);
+      if (ptr != 0){
+          result = *ptr;
+      }
+    }
+    return(result);
+}
+
 ////////////////////////////////
 
 function Buffer_Seek
@@ -645,7 +660,7 @@ boundary_token(Application_Links *app, Buffer_ID buffer, Side side, Scan_Directi
                     }
                 }
             }break;
-            
+
             case Scan_Backward:
             {
                 result = 0;
@@ -772,17 +787,17 @@ seek_string(Application_Links *app, Buffer_ID buffer_id, i64 pos, i64 end, i64 m
         {
             seek_string_forward(app, buffer_id, pos, end, str, result);
         }break;
-        
+
         case BufferSeekString_Backward:
         {
             seek_string_backward(app, buffer_id, pos, min, str, result);
         }break;
-        
+
         case BufferSeekString_CaseInsensitive:
         {
             seek_string_insensitive_forward(app, buffer_id, pos, end, str, result);
         }break;
-        
+
         case BufferSeekString_Backward|BufferSeekString_CaseInsensitive:
         {
             seek_string_insensitive_backward(app, buffer_id, pos, min, str, result);
@@ -1248,12 +1263,12 @@ function Indent_Info
 get_indent_info_range(Application_Links *app, Buffer_ID buffer, Range_i64 range, i32 tab_width){
     Scratch_Block scratch(app);
     String_Const_u8 s = push_buffer_range(app, scratch, buffer, range);
-    
+
     Indent_Info info = {};
     info.first_char_pos = range.end;
     info.is_blank = true;
     info.all_space = true;
-    
+
     for (u64 i = 0; i < s.size; i += 1){
         u8 c = s.str[i];
         if (!character_is_whitespace(c)){
@@ -1272,7 +1287,7 @@ get_indent_info_range(Application_Links *app, Buffer_ID buffer, Range_i64 range,
             info.indent_pos += tab_width;
         }
     }
-    
+
     return(info);
 }
 
@@ -1329,17 +1344,17 @@ swap_lines(Application_Links *app, Buffer_ID buffer, i64 line_1, i64 line_2){
     if (1 <= line_1 && line_2 <= line_count){
         Range_i64 range_1 = get_line_pos_range(app, buffer, line_1);
         Range_i64 range_2 = get_line_pos_range(app, buffer, line_2);
-        
+
         Scratch_Block scratch(app);
-        
+
         String_Const_u8 text_1 = push_buffer_range(app, scratch, buffer, range_1);
         String_Const_u8 text_2 = push_buffer_range(app, scratch, buffer, range_2);
-        
+
         History_Group group = history_group_begin(app, buffer);
         buffer_replace_range(app, buffer, range_2, text_1);
         buffer_replace_range(app, buffer, range_1, text_2);
         history_group_end(group);
-        
+
         i64 shift = replace_range_shift(range_1, text_2.size);
         result.min = range_1.min;
         result.max = range_2.min + shift;
@@ -1505,13 +1520,13 @@ query_user_general(Application_Links *app, Query_Bar *bar, b32 force_number, Str
     if (start_query_bar(app, bar, 0) == 0){
         return(false);
     }
-    
+
     if (init_string.size > 0){
         String_u8 string = Su8(bar->string.str, bar->string.size, bar->string_capacity);
         string_append(&string, init_string);
         bar->string.size = string.string.size;
     }
-    
+
     b32 success = true;
     for (;;){
         User_Input in = get_next_input(app, EventPropertyGroup_Any,
@@ -1520,7 +1535,7 @@ query_user_general(Application_Links *app, Query_Bar *bar, b32 force_number, Str
             success = false;
             break;
         }
-        
+
         Scratch_Block scratch(app);
         b32 good_insert = false;
         String_Const_u8 insert_string = to_writable(&in);
@@ -1540,7 +1555,7 @@ query_user_general(Application_Links *app, Query_Bar *bar, b32 force_number, Str
                 good_insert = true;
             }
         }
-        
+
         if (in.event.kind == InputEventKind_KeyStroke &&
             (in.event.key.code == KeyCode_Return || in.event.key.code == KeyCode_Tab)){
             break;
@@ -1576,7 +1591,7 @@ query_user_general(Application_Links *app, Query_Bar *bar, b32 force_number, Str
             }
         }
     }
-    
+
     return(success);
 }
 
@@ -1668,23 +1683,23 @@ function void
 place_begin_and_end_on_own_lines(Application_Links *app, char *begin, char *end){
     View_ID view = get_active_view(app, Access_ReadWriteVisible);
     Buffer_ID buffer = view_get_buffer(app, view, Access_ReadWriteVisible);
-    
+
     Range_i64 range = get_view_range(app, view);
     Range_i64 lines = get_line_range_from_pos_range(app, buffer, range);
     range = get_pos_range_from_line_range(app, buffer, lines);
-    
+
     Scratch_Block scratch(app);
-    
+
     b32 min_line_blank = line_is_valid_and_blank(app, buffer, lines.min);
     b32 max_line_blank = line_is_valid_and_blank(app, buffer, lines.max);
-    
+
     if ((lines.min < lines.max) || (!min_line_blank)){
         String_Const_u8 begin_str = {};
         String_Const_u8 end_str = {};
-        
+
         i64 min_adjustment = 0;
         i64 max_adjustment = 0;
-        
+
         if (min_line_blank){
             begin_str = push_u8_stringf(scratch, "\n%s", begin);
             min_adjustment += 1;
@@ -1699,15 +1714,15 @@ place_begin_and_end_on_own_lines(Application_Links *app, char *begin, char *end)
             end_str = push_u8_stringf(scratch, "\n%s", end);
             max_adjustment += 1;
         }
-        
+
         max_adjustment += begin_str.size;
         Range_i64 new_pos = Ii64(range.min + min_adjustment, range.max + max_adjustment);
-        
+
         History_Group group = history_group_begin(app, buffer);
         buffer_replace_range(app, buffer, Ii64(range.min), begin_str);
         buffer_replace_range(app, buffer, Ii64(range.max + begin_str.size), end_str);
         history_group_end(group);
-        
+
         set_view_range(app, view, new_pos);
     }
     else{
@@ -1833,7 +1848,7 @@ view_disable_highlight_range(Application_Links *app, View_ID view){
 function void
 view_set_highlight_range(Application_Links *app, View_ID view, Range_i64 range){
     view_disable_highlight_range(app, view);
-    
+
     Buffer_ID buffer = view_get_buffer(app, view, Access_Always);
     Managed_Scope scope = view_get_managed_scope(app, view);
     Managed_Object *highlight = scope_attachment(app, scope, view_highlight_range, Managed_Object);
@@ -1853,7 +1868,7 @@ view_look_at_region(Application_Links *app, View_ID view, i64 major_pos, i64 min
     if (major_pos == range.max){
         bottom_major = true;
     }
-    
+
     Buffer_Cursor top = view_compute_cursor(app, view, seek_pos(range.min));
     if (top.line > 0){
         Buffer_Cursor bottom = view_compute_cursor(app, view, seek_pos(range.max));
@@ -1862,16 +1877,16 @@ view_look_at_region(Application_Links *app, View_ID view, i64 major_pos, i64 min
             f32 view_height = rect_height(region);
             f32 skirt_height = view_height*.1f;
             Range_f32 acceptable_y = If32(skirt_height, view_height*.9f);
-            
+
             f32 target_height = view_line_y_difference(app, view, bottom.line + 1, top.line);
-            
+
             f32 line_height = get_view_line_height(app, view);
             if (target_height + 2*line_height > view_height){
                 i64 major_line = bottom.line;
                 if (range.min == major_pos){
                     major_line = top.line;
                 }
-                
+
                 Buffer_Scroll scroll = view_get_buffer_scroll(app, view);
                 scroll.target.line_number = major_line;
                 scroll.target.pixel_shift.y = -skirt_height;
@@ -2357,7 +2372,7 @@ find_nest_side(Application_Links *app, Buffer_ID buffer, i64 pos,
                Find_Nest_Flag flags, Scan_Direction scan, Nest_Delimiter_Kind delim,
                Range_i64 *out){
     b32 result = false;
-    
+
     b32 balanced = HasFlag(flags, FindNest_Balanced);
     if (balanced){
         if ((delim == NestDelim_Open && scan == Scan_Forward) ||
@@ -2365,7 +2380,7 @@ find_nest_side(Application_Links *app, Buffer_ID buffer, i64 pos,
             balanced = false;
         }
     }
-    
+
     Managed_Scope scope = buffer_get_managed_scope(app, buffer);
     Token_Array *tokens = scope_attachment(app, scope, attachment_tokens, Token_Array);
     if (tokens != 0 && tokens->count > 0){
@@ -2374,17 +2389,17 @@ find_nest_side(Application_Links *app, Buffer_ID buffer, i64 pos,
         for (;;){
             Token *token = token_it_read(&it);
             Nest_Delimiter_Kind token_delim = get_nest_delimiter_kind(token->kind, flags);
-            
+
             if (level == 0 && token_delim == delim){
                 *out = Ii64_size(token->pos, token->size);
                 result = true;
                 break;
             }
-            
+
             if (balanced && token_delim != NestDelim_None){
                 level += (token_delim == delim)?-1:1;
             }
-            
+
             b32 good = false;
             if (scan == Scan_Forward){
                 good = token_it_inc(&it);
@@ -2397,7 +2412,7 @@ find_nest_side(Application_Links *app, Buffer_ID buffer, i64 pos,
             }
         }
     }
-    
+
     return(result);
 }
 
@@ -2476,7 +2491,7 @@ set_buffer_system_command(Application_Links *app, Child_Process_ID process, Buff
         clear_buffer(app, buffer);
         if (HasFlag(flags, CLI_SendEndSignal)){
             buffer_send_end_signal(app, buffer);
-            
+
             Buffer_Hook_Function *begin_buffer = (Buffer_Hook_Function*)get_custom_hook(app, HookID_BeginBuffer);
             if (begin_buffer != 0){
                 begin_buffer(app, buffer);

code/custom/4coder_tree_sitter.cpp

@@ -103,6 +103,34 @@ tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id)
   return tree_sitter_language_for_buffer(app, buffer_id, scratch);
 }
 
+////////////////////////////////////////////////////////////////////
+// Async Management
+////////////////////////////////////////////////////////////////////
+
+function void
+async_task_cancel_nowait(Application_Links* app, Async_System* async_system, Async_Task task)
+{
+  system_mutex_acquire(async_system->mutex);
+  Async_Node *node = async_get_pending_node(async_system, task);
+  if (node != 0)
+  {
+    dll_remove(&node->node);
+    async_system->task_count -= 1;
+    async_free_node(async_system, node);
+  }
+  else
+  {
+    node = async_get_running_node(async_system, task);
+    if (node != 0)
+    {
+      b32 *cancel_signal = &node->thread->cancel_signal;
+      atomic_write_b32(cancel_signal, true);
+      // async_task_wait__inner(app, async_system, task);
+    }
+  }
+  system_mutex_release(async_system->mutex);
+}
+
 /////////////////////////////////////////////
 // Tree Sitter Hook Internals
 /////////////////////////////////////////////
@@ -264,10 +292,10 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id)
       tree_data->tree = new_tree;
       system_mutex_acquire(tree_data->tree_mutex);
 
-      Tree_Sitter_Code_Index_Parse_State parse_state = tree_sitter_code_index_update_single_buffer(app, buffer_id);
-      code_index_lock();
-      code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
-      code_index_unlock();
+      Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
+      Async_Task* lex_task = scope_attachment(app, scope, buffer_tree_sitter_update_code_index_task_id, Async_Task);
+      if (lex_task != 0) async_task_cancel_nowait(app, &global_async_system, *lex_task);
+      *lex_task = async_task_no_dep(&global_async_system, tree_sitter_code_index_update_async, make_data_struct(&buffer_id));
 
       // Force a frame refresh by requesting another frame
       animate_in_n_milliseconds(app, 0);
@@ -289,6 +317,10 @@ tree_sitter_parse_async(Async_Context* actx, String_Const_u8 data)
   tree_sitter_parse_async__inner(actx, buffer_id);
 }
 
+////////////////////////////////////////////////////////////////////
+// Code Index Updating
+////////////////////////////////////////////////////////////////////
+
 function Range_i64
 tree_sitter_node_to_range(TSNode node)
 {
@@ -312,30 +344,34 @@ code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind k
 }
 
 function Tree_Sitter_Code_Index_Parse_State
-tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID buffer_id)
+tree_sitter_code_index_update_single_buffer(Async_Context* actx, Buffer_ID buffer_id)
 {
-  Scratch_Block scratch(app);
+  Application_Links* app = actx->app;
 
-  Tree_Sitter_Code_Index_Parse_State parse_state = {};
-  parse_state.ok = false;
-  parse_state.buffer_id = buffer_id;
-  parse_state.language = tree_sitter_language_for_buffer(app, parse_state.buffer_id);
-  if (!parse_state.language) return parse_state;
+  acquire_global_frame_mutex(app);
+    Scratch_Block scratch(app);
 
-  parse_state.index_arena = make_arena_system(KB(16));
-  parse_state.index = push_array_zero(&parse_state.index_arena, Code_Index_File, 1);
-  parse_state.buffer_contents = push_whole_buffer(app, scratch, parse_state.buffer_id);
-  parse_state.query = tree_sitter_query_init(
-    app, parse_state.buffer_id, parse_state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
-  );
-  parse_state.nest_stack_first = 0;
-  parse_state.nest_stack_last = 0;
-  parse_state.last_note = 0;
-  parse_state.last_note_match_id = max_u32;
+    Tree_Sitter_Code_Index_Parse_State parse_state = {};
+    parse_state.ok = false;
+    parse_state.buffer_id = buffer_id;
+    parse_state.language = tree_sitter_language_for_buffer(app, parse_state.buffer_id);
+    if (!parse_state.language) return parse_state;
+    parse_state.index_arena = make_arena_system(KB(16));
+    parse_state.index = push_array_zero(&parse_state.index_arena, Code_Index_File, 1);
+    parse_state.buffer_contents = push_whole_buffer(app, scratch, parse_state.buffer_id);
+    parse_state.query = tree_sitter_query_init(
+      app, parse_state.buffer_id, parse_state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
+    );
+    parse_state.nest_stack_first = 0;
+    parse_state.nest_stack_last = 0;
+    parse_state.last_note = 0;
+    parse_state.last_note_match_id = max_u32;
+  release_global_frame_mutex(app);
 
+  parse_state.ok = true;
   TSQueryMatch query_match;
   u32 capture_index;
-  while (tree_sitter_query_continue(&parse_state.query, &query_match, &capture_index))
+  while (parse_state.ok && tree_sitter_query_continue(&parse_state.query, &query_match, &capture_index))
   {
     TSQueryCapture type_capture = query_match.captures[capture_index];
 
@@ -408,116 +444,74 @@ tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID bu
         parse_state.last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start);
       }
     }
-  }
 
-  while (parse_state.nest_stack_last != 0)
-  {
-    Code_Index_Nest* nest = parse_state.nest_stack_last->nest;
-    if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest);
-    else                   code_index_push_nest(&parse_state.index->nest_list, nest);
-    nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list);
-    parse_state.nest_stack_last = parse_state.nest_stack_last->prev;
+    if (async_check_canceled(actx))
+    {
+      parse_state.ok = false;
+      break;
+    }
   }
 
-  for (Code_Index_Note* note = parse_state.index->note_list.first; note != 0 && note->next != note; note = note->next)
-  {
-    note->text = push_string_copy(&parse_state.index_arena, string_substring(parse_state.buffer_contents, note->pos));
-  }
-
-  // Finish the Index
-  parse_state.index->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->nest_list);
-  parse_state.index->note_array = code_index_note_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->note_list);
-
   tree_sitter_query_end(&parse_state.query);
 
-  parse_state.ok = true;
+  if (parse_state.ok)
+  {
+    while (parse_state.nest_stack_last != 0)
+    {
+      Code_Index_Nest* nest = parse_state.nest_stack_last->nest;
+      if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest);
+      else                   code_index_push_nest(&parse_state.index->nest_list, nest);
+      nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list);
+      parse_state.nest_stack_last = parse_state.nest_stack_last->prev;
+    }
+
+    for (Code_Index_Note* note = parse_state.index->note_list.first; note != 0 && note->next != note; note = note->next)
+    {
+      note->text = push_string_copy(&parse_state.index_arena, string_substring(parse_state.buffer_contents, note->pos));
+    }
+
+    // Finish the Index
+    parse_state.index->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->nest_list);
+    parse_state.index->note_array = code_index_note_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->note_list);
+
+    code_index_lock();
+    code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
+    code_index_unlock();
+
+    buffer_clear_layout_cache(app, parse_state.buffer_id);
+  }
+  else
+  {
+    linalloc_clear(&parse_state.index_arena);
+  }
+
   return parse_state;
 }
 
-#define BUFFER_CODE_INDEX_UPDATES false
+function void
+tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data)
+{
+  if (data.size != sizeof(Buffer_ID)) return;
+  Buffer_ID buffer_id = *(Buffer_ID*)data.str;
+  tree_sitter_code_index_update_single_buffer(actx, buffer_id);
+}
 
 function void
 tree_sitter_code_index_update_tick(Application_Links* app)
 {
-#define BUFFERED_CODE_INDICES_CAP 8
-  Tree_Sitter_Code_Index_Parse_State buffered_code_indices[BUFFERED_CODE_INDICES_CAP];
-  u32 buffered_code_indices_count = 0;
-
-  Tree_Sitter_Code_Index_Parse_State parse_state = {};
   for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first;
        modified_node != 0;
        modified_node = modified_node->next
   ){
-    parse_state = tree_sitter_code_index_update_single_buffer(app, modified_node->buffer);
-    if (parse_state.ok)
-    {
-      #if BUFFER_CODE_INDEX_UPDATES
-      buffered_code_indices[buffered_code_indices_count] = parse_state;
-      buffered_code_indices_count += 1;
-      #else
-      code_index_lock();
-      code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
-      code_index_unlock();
-      #endif
-    }
-
-    if (buffered_code_indices_count >= BUFFERED_CODE_INDICES_CAP)
-    {
-      code_index_lock();
-      for (int i = 0; i < buffered_code_indices_count; i++)
-      {
-        parse_state = buffered_code_indices[i];
-        code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
-      }
-      buffered_code_indices_count = 0;
-      code_index_unlock();
-    }
-
-    buffer_clear_layout_cache(app, parse_state.buffer_id);
+    Buffer_ID buffer_id = modified_node->buffer;
+    Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
+    Async_Task* lex_task = scope_attachment(app, scope, buffer_tree_sitter_update_code_index_task_id, Async_Task);
+    if (lex_task != 0) async_task_cancel_nowait(app, &global_async_system, *lex_task);
+    *lex_task = async_task_no_dep(&global_async_system, tree_sitter_code_index_update_async, make_data_struct(&buffer_id));
   }
-
-  if (buffered_code_indices_count > 0)
-  {
-    code_index_lock();
-    for (int i = 0; i < buffered_code_indices_count; i++)
-    {
-      parse_state = buffered_code_indices[i];
-      code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
-    }
-    code_index_unlock();
-  }
-
   buffer_modified_set_clear();
 }
 
-////////////////////////////////////////////////////////////////////
-// Async Management
-////////////////////////////////////////////////////////////////////
-
-function void
-async_task_cancel_nowait(Application_Links* app, Async_System* async_system, Async_Task task)
-{
-  system_mutex_acquire(async_system->mutex);
-  Async_Node *node = async_get_pending_node(async_system, task);
-  if (node != 0)
-  {
-    dll_remove(&node->node);
-    async_system->task_count -= 1;
-    async_free_node(async_system, node);
-  }
-  else
-  {
-    node = async_get_running_node(async_system, task);
-    if (node != 0)
-    {
-      b32 *cancel_signal = &node->thread->cancel_signal;
-      atomic_write_b32(cancel_signal, true);
-      // async_task_wait__inner(app, async_system, task);
-    }
-  }
-  system_mutex_release(async_system->mutex);
-}
-
 ////////////////////////////////////////////////////////////////////
 // Token Highlighting
 ////////////////////////////////////////////////////////////////////
@@ -577,6 +571,7 @@ function void
 draw_tree_sitter_node_colors(Application_Links* app, Text_Layout_ID text_layout_id, Buffer_ID buffer_id)
 {
   Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
+  if (!lang) return;
   TSQuery* query = lang->queries.ptr[Tree_Sitter_Language_Query_Highlights];
 
   Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id);

code/custom/4coder_tree_sitter.h

@@ -92,6 +92,7 @@ global Tree_Sitter_Languages tree_sitter_languages;
 
 CUSTOM_ID(attachment, buffer_tree_sitter_data_id);
 CUSTOM_ID(attachment, buffer_tree_sitter_parse_task_id);
+CUSTOM_ID(attachment, buffer_tree_sitter_update_code_index_task_id);
 
 b8 use_tree_sitter_code_indexing = true;
 b8 use_tree_sitter_token_coloring = true;
@@ -104,6 +105,7 @@ function void tree_sitter_register_language(String_Const_u8 ext, TSLanguage* lan
 function Tree_Sitter_Language_Definition* tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id);
 
 function Tree_Sitter_Code_Index_Parse_State tree_sitter_code_index_update_single_buffer(Application_Links *app, Buffer_ID buffer_id);
-function void tree_sitter_code_index_update_tick(Application_Links *app);
+function void                               tree_sitter_code_index_update_tick(Application_Links *app);
+function void                               tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data);
 
 #endif //FCODER_TREE_SITTER_H