tree sitter logic handles edits to a parsed buffer

- async_task_cancel_nowait implementation

code/custom/4coder_custom_hooks.cpp

@@ -84,6 +84,175 @@ BUFFER_HOOK_SIG(custom_end_buffer){
   return(0);
 }
 
+///////////////////////////////////////////////////////////////////////////
+// Buffer Edit Range
+///////////////////////////////////////////////////////////////////////////
+
+BUFFER_EDIT_RANGE_SIG(custom_buffer_edit_range){
+  // buffer_id, new_range, original_size
+  ProfileScope(app, "default edit range");
+
+  Range_i64 old_range = Ii64(old_cursor_range.min.pos, old_cursor_range.max.pos);
+
+  buffer_shift_fade_ranges(buffer_id, old_range.max, (new_range.max - old_range.max));
+
+  {
+    code_index_lock();
+    Code_Index_File *file = code_index_get_file(buffer_id);
+    if (file != 0) code_index_shift(file, old_range, range_size(new_range));
+    code_index_unlock();
+  }
+
+  i64 insert_size = range_size(new_range);
+  i64 text_shift = replace_range_shift(old_range, insert_size);
+
+  Scratch_Block scratch(app);
+
+  Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
+
+  { // Tree Sitter
+    Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
+    if (tree_data->tree)
+    {
+      Async_Task* tree_sitter_parse_task = scope_attachment(app, scope, buffer_tree_sitter_parse_task_id, Async_Task);
+      if (async_task_is_running_or_pending(&global_async_system, *tree_sitter_parse_task))
+      {
+        // NOTE(jack): We dont want to wait for ts_tree delete, as it is slow for large files
+        // (noticable spikes when it fires in 4coder_base_types.cpp ~200Kb)
+        // And if we restart the next task before it attempts the delete we may be able to
+        // avoid the delete entirely as the reference count may have increased from the next
+        // parse attempt
+        async_task_cancel_nowait(app, &global_async_system, *tree_sitter_parse_task);
+        *tree_sitter_parse_task = 0;
+      }
+
+      i64 new_end_line = get_line_number_from_pos(app, buffer_id, new_range.end);
+      i64 new_end_pos = new_range.end - get_line_start_pos(app, buffer_id, new_end_line);
+
+      TSInputEdit edit;
+      edit.start_byte = (u32)old_range.start;
+      edit.old_end_byte = (u32)old_range.end;
+      edit.new_end_byte = (u32)new_range.end;
+      edit.start_point = {
+        (u32)old_cursor_range.start.line,
+        (u32)old_cursor_range.start.col
+      };
+      edit.old_end_point = {
+        (u32)old_cursor_range.end.line,
+        (u32)old_cursor_range.end.col
+      };
+      // TODO(PS): jack says this works but looks wrong???
+      edit.new_end_point = {
+        (u32)new_end_line - 1,
+        (u32)new_end_pos + 1,
+      };
+
+      ts_tree_edit(tree_data->tree, &edit);
+      *tree_sitter_parse_task = async_task_no_dep(&global_async_system, tree_sitter_parse_async, make_data_struct(&buffer_id));
+    }
+  }
+
+
+  Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task);
+
+  Base_Allocator *allocator = managed_scope_allocator(app, scope);
+  b32 do_full_relex = false;
+
+  if (async_task_is_running_or_pending(&global_async_system, *lex_task_ptr))
+  {
+    async_task_cancel(app, &global_async_system, *lex_task_ptr);
+    buffer_unmark_as_modified(buffer_id);
+    do_full_relex = true;
+    *lex_task_ptr = 0;
+  }
+
+  Token_Array *ptr = scope_attachment(app, scope, attachment_tokens, Token_Array);
+  if (ptr != 0 && ptr->tokens != 0)
+  {
+    ProfileBlockNamed(app, "attempt resync", profile_attempt_resync);
+
+    i64 token_index_first = token_relex_first(ptr, old_range.first, 1);
+    i64 token_index_resync_guess =
+        token_relex_resync(ptr, old_range.one_past_last, 16);
+
+    if (token_index_resync_guess - token_index_first >= 4000)
+    {
+      do_full_relex = true;
+    }
+    else
+    {
+      Token *token_first = ptr->tokens + token_index_first;
+      Token *token_resync = ptr->tokens + token_index_resync_guess;
+
+      Range_i64 relex_range = Ii64(token_first->pos, token_resync->pos + token_resync->size + text_shift);
+      String_Const_u8 partial_text = push_buffer_range(app, scratch, buffer_id, relex_range);
+
+      Token_List relex_list = lex_full_input_cpp(scratch, partial_text);
+      if (relex_range.one_past_last < buffer_get_size(app, buffer_id))
+      {
+        token_drop_eof(&relex_list);
+      }
+
+      Token_Relex relex = token_relex(relex_list, relex_range.first - text_shift, ptr->tokens, token_index_first, token_index_resync_guess);
+
+      ProfileCloseNow(profile_attempt_resync);
+
+      if (!relex.successful_resync)
+      {
+        do_full_relex = true;
+      }
+      else
+      {
+        ProfileBlock(app, "apply resync");
+
+        i64 token_index_resync = relex.first_resync_index;
+
+        Range_i64 head = Ii64(0, token_index_first);
+        Range_i64 replaced = Ii64(token_index_first, token_index_resync);
+        Range_i64 tail = Ii64(token_index_resync, ptr->count);
+        i64 resynced_count = (token_index_resync_guess + 1) - token_index_resync;
+        i64 relexed_count = relex_list.total_count - resynced_count;
+        i64 tail_shift = relexed_count - (token_index_resync - token_index_first);
+
+        i64 new_tokens_count = ptr->count + tail_shift;
+        Token *new_tokens = base_array(allocator, Token, new_tokens_count);
+
+        Token *old_tokens = ptr->tokens;
+        block_copy_array_shift(new_tokens, old_tokens, head, 0);
+        token_fill_memory_from_list(new_tokens + replaced.first, &relex_list, relexed_count);
+        for (i64 i = 0, index = replaced.first; i < relexed_count; i += 1, index += 1)
+        {
+          new_tokens[index].pos += relex_range.first;
+        }
+        for (i64 i = tail.first; i < tail.one_past_last; i += 1)
+        {
+          old_tokens[i].pos += text_shift;
+        }
+        block_copy_array_shift(new_tokens, ptr->tokens, tail, tail_shift);
+
+        base_free(allocator, ptr->tokens);
+
+        ptr->tokens = new_tokens;
+        ptr->count = new_tokens_count;
+        ptr->max = new_tokens_count;
+
+        buffer_mark_as_modified(buffer_id);
+      }
+    }
+  }
+
+  if (do_full_relex)
+  {
+    *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_async,
+                                      make_data_struct(&buffer_id));
+  }
+
+  loco_on_buffer_edit(app, buffer_id, old_range, new_range);
+
+  // no meaning for return
+  return(0);
+}
+
 ///////////////////////////////////////////////////////////////////////////
 // Render Buffer
 ///////////////////////////////////////////////////////////////////////////

code/custom/4coder_default_bindings.cpp

@@ -520,32 +520,20 @@ custom_keyboard_bindings()
 }
 
 void
-custom_layer_init(Application_Links *app){
+custom_layer_init(Application_Links *app)
-    Thread_Context *tctx = get_thread_context(app);
+{
-    default_framework_init(app);
+  Thread_Context *tctx = get_thread_context(app);
+  default_framework_init(app);
 
-    set_all_default_hooks(app);
+  set_all_default_hooks(app);
-    modal_init(3, tctx);
+  modal_init(3, tctx);
 
-    set_custom_hook(app, HookID_BeginBuffer,  custom_begin_buffer);
+  set_custom_hook(app, HookID_BeginBuffer,     custom_begin_buffer);
-    set_custom_hook(app, HookID_EndBuffer,    custom_end_buffer);
+  set_custom_hook(app, HookID_BufferEditRange, custom_buffer_edit_range);
-    set_custom_hook(app, HookID_RenderCaller, custom_render_caller);
+  set_custom_hook(app, HookID_EndBuffer,       custom_end_buffer);
-
+  set_custom_hook(app, HookID_RenderCaller,    custom_render_caller);
-    custom_keyboard_bindings();
-
-    #if 0
-    mapping_init(tctx, &framework_mapping);
-    String_ID global_map_id = vars_save_string_lit("keys_global");
-    String_ID file_map_id = vars_save_string_lit("keys_file");
-    String_ID code_map_id = vars_save_string_lit("keys_code");
-#if OS_MAC
-    setup_mac_mapping(&framework_mapping, global_map_id, file_map_id, code_map_id);
-#else
-    setup_default_mapping(&framework_mapping, global_map_id, file_map_id, code_map_id);
-#endif
-	setup_essential_mapping(&framework_mapping, global_map_id, file_map_id, code_map_id);
-  #endif
 
+  custom_keyboard_bindings();
   tree_sitter_init(app);
 }
 

code/custom/4coder_tree_sitter.cpp

@@ -318,6 +318,34 @@ tree_sitter_code_index_update_tick(Application_Links* app)
 #endif
 }
 
+////////////////////////////////////////////////////////////////////
+// Async Management
+////////////////////////////////////////////////////////////////////
+
+function void
+async_task_cancel_nowait(Application_Links* app, Async_System* async_system, Async_Task task)
+{
+  system_mutex_acquire(async_system->mutex);
+  Async_Node *node = async_get_pending_node(async_system, task);
+  if (node != 0)
+  {
+    dll_remove(&node->node);
+    async_system->task_count -= 1;
+    async_free_node(async_system, node);
+  }
+  else
+  {
+    node = async_get_running_node(async_system, task);
+    if (node != 0)
+    {
+      b32 *cancel_signal = &node->thread->cancel_signal;
+      atomic_write_b32(cancel_signal, true);
+      // async_task_wait__inner(app, async_system, task);
+    }
+  }
+  system_mutex_release(async_system->mutex);
+}
+
 ////////////////////////////////////////////////////////////////////
 // Token Highlighting
 ////////////////////////////////////////////////////////////////////