Parse delimiters before creating the nests tree

code/custom/4coder_code_index.cpp

@@ -6,6 +6,15 @@
 
 global Code_Index global_code_index = {};
 
+function bool
+delims_are_open_close_pair(Code_Index_Scope_Delim_Kind a, Code_Index_Scope_Delim_Kind b)
+{
+  if (a == CodeIndexScopeDelim_ScopeOpen) return b == CodeIndexScopeDelim_ScopeClose;
+  if (a == CodeIndexScopeDelim_ParenOpen) return b == CodeIndexScopeDelim_ParenClose;
+  if (a == CodeIndexScopeDelim_BracketOpen) return b == CodeIndexScopeDelim_BracketClose;
+  return false;
+}
+
 ////////////////////////////////
 // NOTE(allen): Lookups
 
@@ -196,19 +205,26 @@ code_index_erase_file(Buffer_ID buffer){
   }
 }
 
-function Code_Index_File*
+function Code_Index_File_Storage*
-code_index_get_file(Buffer_ID buffer){
+code_index_get_file_storage(Buffer_ID buffer){
-  Code_Index_File *result = 0;
+  Code_Index_File_Storage *result = 0;
   Table_Lookup lookup = table_lookup(&global_code_index.buffer_to_index_file, buffer);
   if (lookup.found_match){
     u64 val = 0;
     table_read(&global_code_index.buffer_to_index_file, lookup, &val);
-    Code_Index_File_Storage *storage = (Code_Index_File_Storage*)IntAsPtr(val);
+    result = (Code_Index_File_Storage*)IntAsPtr(val);
-    result = storage->file;
   }
   return(result);
 }
 
+function Code_Index_File*
+code_index_get_file(Buffer_ID buffer){
+  Code_Index_File *result = 0;
+  Code_Index_File_Storage *storage = code_index_get_file_storage(buffer);
+  if (storage) result = storage->file;
+  return result;
+}
+
 function void
 index_shift(i64 *ptr, Range_i64 old_range, u64 new_size){
   i64 i = *ptr;

code/custom/4coder_code_index.h

@@ -7,6 +7,35 @@
 #if !defined(FCODER_CODE_INDEX_H)
 #define FCODER_CODE_INDEX_H
 
+typedef i32 Code_Index_Scope_Delim_Kind;
+enum {
+  CodeIndexScopeDelim_ScopeOpen,
+  CodeIndexScopeDelim_ScopeClose,
+  CodeIndexScopeDelim_ParenOpen,
+  CodeIndexScopeDelim_ParenClose,
+  CodeIndexScopeDelim_BracketOpen,
+  CodeIndexScopeDelim_BracketClose
+};
+
+struct Code_Index_Scope_Delim {
+  Code_Index_Scope_Delim_Kind kind;
+  i32 depth;
+  Range_i64 pos;
+
+  Code_Index_Scope_Delim* next;
+  Code_Index_Scope_Delim* prev;
+};
+
+struct Code_Index_Scope_Delim_List {
+  Code_Index_Scope_Delim* first;
+  Code_Index_Scope_Delim* last;
+};
+
+struct Code_Index_Scope_Delim_Ptr_Array {
+  Code_Index_Scope_Delim** ptrs;
+  i32 count;
+};
+
 struct Code_Index_Nest_List{
     struct Code_Index_Nest *first;
     struct Code_Index_Nest *last;
@@ -29,7 +58,10 @@ enum{
 struct Code_Index_Nest{
     Code_Index_Nest *next;
 
+    // TODO(PS): kind and delim are redundant, I don't want to break virtual indentation just yet
+    // so Im leaving them both here.
     Code_Index_Nest_Kind kind;
+    Code_Index_Scope_Delim_Kind delim;
     b32 is_closed;
     Range_i64 open;
     Range_i64 close;
@@ -73,6 +105,8 @@ struct Code_Index_Note_Ptr_Array{
 };
 
 struct Code_Index_File{
+    Code_Index_Scope_Delim_List scope_delim_list;
+    Code_Index_Scope_Delim_Ptr_Array scope_delim_array;
     Code_Index_Nest_List nest_list;
     Code_Index_Nest_Ptr_Array nest_array;
     Code_Index_Note_List note_list;

code/custom/4coder_tree_sitter.cpp

@@ -410,40 +410,31 @@ tree_sitter_code_index_update_process_query_match(
   const char* tmp = ts_query_capture_name_for_id(state->query.query, type_capture.index, &length);
   String_Const_u8 capture_name = SCu8((char*)tmp, length);
 
-  if (string_match(capture_name, SCu8("scope_begin")))
+  if (string_match(capture_name, SCu8("scope_begin")) || string_match(capture_name, SCu8("scope_end")))
   {
-    Code_Index_Nest* nest = push_array_zero(&state->index_arena, Code_Index_Nest, 1);
+    Code_Index_Scope_Delim_Kind kind;
-    nest->kind = CodeIndexNest_Scope;
+    String_Const_u8 delim_char = string_substring(state->buffer_contents, type_range);
-    nest->open = type_range;
+    bool skip = false;
-    nest->close = Ii64(max_i64);
+         if (delim_char.str[0] == '{') kind = CodeIndexScopeDelim_ScopeOpen;
-    nest->file = state->index;
+    else if (delim_char.str[0] == '}') kind = CodeIndexScopeDelim_ScopeClose;
-    if (state->nest_stack_last != 0)
+    else if (delim_char.str[0] == '(') kind = CodeIndexScopeDelim_ParenOpen;
+    else if (delim_char.str[0] == ')') kind = CodeIndexScopeDelim_ParenClose;
+    else if (delim_char.str[0] == '[') kind = CodeIndexScopeDelim_BracketOpen;
+    else if (delim_char.str[0] == ']') kind = CodeIndexScopeDelim_BracketClose;
+    else skip = true;
+
+    if (!skip)
     {
-      nest->parent = state->nest_stack_last->nest;
+      Code_Index_Scope_Delim* delim = push_array_zero(&state->index_arena, Code_Index_Scope_Delim, 1);
-      code_index_push_nest(&nest->parent->nest_list, nest);
+      delim->pos = type_range;
+      delim->kind = kind;
+
+      Code_Index_Scope_Delim_List* list = &state->index->scope_delim_list;
+      if (!list->first) list->first = delim;
+      if (list->last) list->last->next = delim;
+      delim->prev = list->last;
+      list->last = delim;
     }
-
-    Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1);
-    stack->nest = nest;
-    stack->prev = state->nest_stack_last;
-    stack->match_id = query_match.id;
-    if (state->nest_stack_last != 0) state->nest_stack_last->next = stack;
-    else                             state->nest_stack_first      = stack;
-    state->nest_stack_last = stack;
-  }
-  else if (string_match(capture_name, SCu8("scope_end")))
-  {
-    Assert(state->nest_stack_last != 0);
-    Assert(state->nest_stack_last->match_id == query_match.id);
-
-    Code_Index_Nest* nest = state->nest_stack_last->nest;
-    nest->close = type_range;
-    nest->is_closed = true;
-    nest->nest_array = code_index_nest_ptr_array_from_list(&state->index_arena, &nest->nest_list);
-    if (nest->parent == 0) code_index_push_nest(&state->index->nest_list, nest);
-
-    state->nest_stack_last = state->nest_stack_last->prev;
-    if (state->nest_stack_last != 0) state->nest_stack_last->next = 0;
   }
   else if (string_match(capture_name, SCu8("definition.class")))
   {
@@ -477,16 +468,89 @@ tree_sitter_code_index_update_process_query_match(
 function void
 tree_sitter_code_index_update_complete(
   Application_Links* app,
-  Tree_Sitter_Code_Index_Update_State* state
+  Tree_Sitter_Code_Index_Update_State* state,
+  Arena* scratch
 ){
   tree_sitter_query_end(&state->query);
   if (state->ok)
   {
+    Code_Index_Nest* free_nests = state->index->nest_list.first;
+    state->index->nest_list.first = 0;
+    state->index->nest_list.last = 0;
+
+    Code_Index_Scope_Delim* delim_at = state->index->scope_delim_list.first;
+    while (delim_at != 0)
+    {
+      bool is_open = (
+        delim_at->kind == CodeIndexScopeDelim_ScopeOpen ||
+        delim_at->kind == CodeIndexScopeDelim_ParenOpen ||
+        delim_at->kind == CodeIndexScopeDelim_BracketOpen
+      );
+      if (is_open)
+      {
+        if (!free_nests)
+        {
+          int count = 32;
+          free_nests = push_array_zero(&state->index_arena, Code_Index_Nest, count);
+          for (int i = 0; i < count-2; i++)
+          {
+            free_nests[i].next = &free_nests[i+1];
+          }
+        }
+
+        Code_Index_Nest* nest = free_nests;
+        if (nest->nest_list.first)
+        {
+          nest->nest_list.last->next = free_nests;
+          free_nests = nest->nest_list.first;
+        }
+        else
+        {
+          free_nests = nest->next;
+        }
+
+        nest->kind = CodeIndexNest_Scope;
+        nest->delim = delim_at->kind;
+        nest->open = delim_at->pos;
+        nest->close = Ii64(max_i64);
+        nest->file = state->index;
+        if (state->nest_stack_last != 0)
+        {
+          nest->parent = state->nest_stack_last->nest;
+          code_index_push_nest(&nest->parent->nest_list, nest);
+        }
+        else
+        {
+          code_index_push_nest(&state->index->nest_list, nest);
+        }
+
+        Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1);
+        stack->nest = nest;
+        stack->prev = state->nest_stack_last;
+        if (state->nest_stack_last != 0) state->nest_stack_last->next = stack;
+        else                             state->nest_stack_first      = stack;
+        state->nest_stack_last = stack;
+      }
+      else
+      {
+        if (state->nest_stack_last != 0 && delims_are_open_close_pair(state->nest_stack_last->nest->delim, delim_at->kind))
+        {
+          Code_Index_Nest* nest = state->nest_stack_last->nest;
+          nest->close = delim_at->pos;
+          nest->is_closed = true;
+          nest->nest_array = code_index_nest_ptr_array_from_list(&state->index_arena, &nest->nest_list);
+
+          state->nest_stack_last = state->nest_stack_last->prev;
+          if (state->nest_stack_last != 0) state->nest_stack_last->next = 0;
+        }
+      }
+
+      delim_at = delim_at->next;
+    }
+
     while (state->nest_stack_last != 0)
     {
       Code_Index_Nest* nest = state->nest_stack_last->nest;
-      if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest);
-      else                   code_index_push_nest(&state->index->nest_list, nest);
       nest->nest_array = code_index_nest_ptr_array_from_list(&state->index_arena, &nest->nest_list);
       state->nest_stack_last = state->nest_stack_last->prev;
     }
@@ -500,7 +564,6 @@ tree_sitter_code_index_update_complete(
     state->index->nest_array = code_index_nest_ptr_array_from_list(&state->index_arena, &state->index->nest_list);
     state->index->note_array = code_index_note_ptr_array_from_list(&state->index_arena, &state->index->note_list);
 
-
     code_index_lock();
     code_index_set_file(state->buffer_id, state->index_arena, state->index);
     code_index_unlock();
@@ -513,36 +576,6 @@ tree_sitter_code_index_update_complete(
   }
 }
 
-function Tree_Sitter_Code_Index_Update_State
-tree_sitter_code_index_update_full_file_async(Async_Context* actx, Buffer_ID buffer_id)
-{
-  Application_Links* app = actx->app;
-  Scratch_Block scratch(app);
-
-  acquire_global_frame_mutex(app);
-  Tree_Sitter_Code_Index_Update_State state = tree_sitter_code_index_update_state_create(
-    app,
-    buffer_id,
-    scratch
-  );
-  release_global_frame_mutex(app);
-
-  TSQueryMatch query_match;
-  u32 capture_index;
-  while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index))
-  {
-    tree_sitter_code_index_update_process_query_match(&state, query_match, capture_index, scratch);
-    if (async_check_canceled(actx))
-    {
-      state.ok = false;
-      break;
-    }
-  }
-
-  tree_sitter_code_index_update_complete(app, &state);
-  return state;
-}
-
 function Tree_Sitter_Code_Index_Update_State
 tree_sitter_code_index_update_full_file(Application_Links* app, Buffer_ID buffer_id)
 {
@@ -566,7 +599,7 @@ tree_sitter_code_index_update_full_file(Application_Links* app, Buffer_ID buffer
     );
   }
 
-  tree_sitter_code_index_update_complete(app, &state);
+  tree_sitter_code_index_update_complete(app, &state, scratch);
   return state;
 }
 
@@ -575,9 +608,72 @@ tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data)
 {
   if (data.size != sizeof(Buffer_ID)) return;
   Buffer_ID buffer_id = *(Buffer_ID*)data.str;
-  tree_sitter_code_index_update_full_file_async(actx, buffer_id);
+  Application_Links* app = actx->app;
+  Scratch_Block scratch(app);
+
+  acquire_global_frame_mutex(app);
+  Tree_Sitter_Code_Index_Update_State state = tree_sitter_code_index_update_state_create(
+    app,
+    buffer_id,
+    scratch
+  );
+  release_global_frame_mutex(app);
+
+  TSQueryMatch query_match;
+  u32 capture_index;
+  while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index))
+  {
+    tree_sitter_code_index_update_process_query_match(&state, query_match, capture_index, scratch);
+    if (async_check_canceled(actx))
+    {
+      state.ok = false;
+      break;
+    }
+  }
+
+  tree_sitter_code_index_update_complete(app, &state, scratch);
 }
 
+/*
+function void
+tree_sitter_code_index_update_incremental(Application_Links* app, Buffer_ID buffer_id)
+{
+  // TODO(PS): @Collapse with tree_sitter_code_index_update_state_create
+  Tree_Sitter_Code_Index_Update_State state = {};
+  state.ok = false;
+  state.buffer_id = buffer_id;
+  state.language = tree_sitter_language_for_buffer(app, state.buffer_id);
+  if (!state.language) return state;
+
+  Code_Index_File_Storage* storage = code_index_get_file_storage(state.buffer_id);
+  state.index_arena = storage->arena;
+  state.index = storage->file;
+  state.buffer_contents = push_whole_buffer(app, scratch, state.buffer_id);
+  state.query = tree_sitter_query_init(
+    app, state.buffer_id, state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
+  );
+  state.nest_stack_first = 0;
+  state.nest_stack_last = 0;
+  state.last_note = 0;
+  state.last_note_match_id = max_u32;
+  state.ok = true;
+
+  TSQueryMatch query_match;
+  u32 capture_index;
+  while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index))
+  {
+    tree_sitter_code_index_update_process_query_match(
+      &state,
+      query_match,
+      capture_index,
+      scratch
+    );
+  }
+
+  tree_sitter_code_index_update_complete(app, &state, scratch);
+}
+*/
+
 function void
 tree_sitter_code_index_update_tick(Application_Links* app)
 {
@@ -591,7 +687,11 @@ tree_sitter_code_index_update_tick(Application_Links* app)
     Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
     if (!tree_data || !tree_data->tree) continue;
 
+#if 1
     tree_sitter_code_index_update_full_file(app, buffer_id);
+#else
+    tree_sitter_code_index_update_incremental(app, buffer_id);
+#endif
   }
   buffer_modified_set_clear();
 }
@@ -658,6 +758,19 @@ draw_tree_sitter_node_colors(Application_Links* app, Text_Layout_ID text_layout_
   if (!lang) return;
   TSQuery* query = lang->queries.ptr[Tree_Sitter_Language_Query_Highlights];
 
+  Managed_ID color_id = managed_id_get(app, SCu8("colors"), SCu8("defcolor_str_constant"));
+  Code_Index_File* file = code_index_get_file(buffer_id);
+  if (file != 0)
+  {
+    Code_Index_Scope_Delim* delim = file->scope_delim_list.first;
+    while (delim != 0)
+    {
+      paint_text_color_fcolor(app, text_layout_id, delim->pos, fcolor_id(color_id));
+      if (delim->next == delim) break;
+      delim = delim->next;
+    }
+  }
+
   Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id);
   Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
   Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);

code/custom/languages/tree_sitter_cpp.h

@@ -21,9 +21,16 @@ String_Const_u8 TS_CPP_TAGS_QUERY_SCM = string_u8_litexpr(R"DONE(
 
 (class_specifier name: (type_identifier) @name) @definition.class
 
-(_ "{" @scope_begin "}" @scope_end )
+"{" @scope_begin
-(_ "(" @scope_begin ")" @scope_end )
+"(" @scope_begin
-(_ "[" @scope_begin "]" @scope_end )
+"[" @scope_begin
+"}" @scope_end
+")" @scope_end
+"]" @scope_end
+
+; (_ "{" @scope_begin "}" @scope_end )
+; (_ "(" @scope_begin ")" @scope_end )
+; (_ "[" @scope_begin "]" @scope_end )
 
 )DONE");