ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Buffering code index updates when there are many modified buffers within a single frame

This commit is contained in:
Peter Slattery 2025-07-13 13:46:55 -07:00
parent 18428ec90d
commit 5ed8767819
2 changed files with 128 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

178
code/custom/4coder_tree_sitter.cpp
View File

@ -155,16 +155,6 @@ tree_sitter_buffer_get_tree_copy(Buffer_Tree_Sitter_Data* tree_data)
// Queries // Queries
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////
struct Tree_Sitter_Query_Cursor
{
Buffer_ID buffer_id;
TSQuery* query;
TSQueryCursor* query_cursor;
TSTree* tree;
TSNode first_node;
bool ok;
};
function Tree_Sitter_Query_Cursor function Tree_Sitter_Query_Cursor
tree_sitter_query_init(Application_Links* app, Buffer_ID buffer_id, TSQuery* query) tree_sitter_query_init(Application_Links* app, Buffer_ID buffer_id, TSQuery* query)
{ {
@ -225,10 +215,6 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id)
acquire_global_frame_mutex(app); acquire_global_frame_mutex(app);
Scratch_Block scratch(app); Scratch_Block scratch(app);
String_Const_u8 buffer_name = push_buffer_unique_name(app, scratch, buffer_id); String_Const_u8 buffer_name = push_buffer_unique_name(app, scratch, buffer_id);
print_message(app, SCu8("Parsing "));
print_message(app, buffer_name);
print_message(app, SCu8("\n"));
Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id); Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
String_Const_u8 src = push_whole_buffer(app, &arena, buffer_id); String_Const_u8 src = push_whole_buffer(app, &arena, buffer_id);
Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
@ -269,7 +255,10 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id)
tree_data->tree = new_tree; tree_data->tree = new_tree;
system_mutex_acquire(tree_data->tree_mutex); system_mutex_acquire(tree_data->tree_mutex);
tree_sitter_code_index_update_single_buffer(app, buffer_id); Tree_Sitter_Code_Index_Parse_State parse_state = tree_sitter_code_index_update_single_buffer(app, buffer_id);
code_index_lock();
code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
code_index_unlock();
// Force a frame refresh by requesting another frame // Force a frame refresh by requesting another frame
animate_in_n_milliseconds(app, 0); animate_in_n_milliseconds(app, 0);
@ -300,14 +289,6 @@ tree_sitter_node_to_range(TSNode node)
return result; return result;
} }
struct Code_Index_Nest_Stack
{
Code_Index_Nest_Stack* prev;
Code_Index_Nest_Stack* next;
Code_Index_Nest* nest;
u32 match_id;
};
function Code_Index_Note* function Code_Index_Note*
code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind kind, Range_i64 range, Code_Index_Nest_Stack* parent) code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind kind, Range_i64 range, Code_Index_Nest_Stack* parent)
{ {
@ -321,141 +302,172 @@ code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind k
return result; return result;
} }
function void function Tree_Sitter_Code_Index_Parse_State
tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID buffer_id) tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID buffer_id)
{ {
Scratch_Block scratch(app); Scratch_Block scratch(app);
Arena arena = make_arena_system(KB(16)); Tree_Sitter_Code_Index_Parse_State parse_state = {};
Code_Index_File* index = push_array_zero(&arena, Code_Index_File, 1); parse_state.ok = false;
index->buffer = buffer_id; parse_state.buffer_id = buffer_id;
parse_state.language = tree_sitter_language_for_buffer(app, parse_state.buffer_id);
if (!parse_state.language) return parse_state;
String_Const_u8 buffer_contents = push_whole_buffer(app, scratch, buffer_id); parse_state.index_arena = make_arena_system(KB(16));
parse_state.index = push_array_zero(&parse_state.index_arena, Code_Index_File, 1);
Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id); parse_state.buffer_contents = push_whole_buffer(app, scratch, parse_state.buffer_id);
if (!lang) return; parse_state.query = tree_sitter_query_init(
app, parse_state.buffer_id, parse_state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
TSQuery* ts_query = lang->queries.ptr[Tree_Sitter_Language_Query_Tags];
Tree_Sitter_Query_Cursor query = tree_sitter_query_init(
app, buffer_id, ts_query
); );
parse_state.nest_stack_first = 0;
Code_Index_Nest_Stack* nest_stack_first = 0; parse_state.nest_stack_last = 0;
Code_Index_Nest_Stack* nest_stack_last = 0; parse_state.last_note = 0;
parse_state.last_note_match_id = max_u32;
Code_Index_Note* last_note = 0;
u32 last_note_match_id = max_u32;
TSQueryMatch query_match; TSQueryMatch query_match;
u32 capture_index; u32 capture_index;
while (tree_sitter_query_continue(&query, &query_match, &capture_index)) while (tree_sitter_query_continue(&parse_state.query, &query_match, &capture_index))
{ {
TSQueryCapture type_capture = query_match.captures[capture_index]; TSQueryCapture type_capture = query_match.captures[capture_index];
TSNode type_node = type_capture.node; TSNode type_node = type_capture.node;
Range_i64 type_range = tree_sitter_node_to_range(type_node); Range_i64 type_range = tree_sitter_node_to_range(type_node);
u32 length; u32 length;
const char* tmp = ts_query_capture_name_for_id(query.query, type_capture.index, &length); const char* tmp = ts_query_capture_name_for_id(parse_state.query.query, type_capture.index, &length);
String_Const_u8 capture_name = SCu8((char*)tmp, length); String_Const_u8 capture_name = SCu8((char*)tmp, length);
if (string_match(capture_name, SCu8("scope_begin"))) if (string_match(capture_name, SCu8("scope_begin")))
{ {
Code_Index_Nest* nest = push_array_zero(&arena, Code_Index_Nest, 1); Code_Index_Nest* nest = push_array_zero(&parse_state.index_arena, Code_Index_Nest, 1);
nest->kind = CodeIndexNest_Scope; nest->kind = CodeIndexNest_Scope;
nest->open = type_range; nest->open = type_range;
nest->close = Ii64(max_i64); nest->close = Ii64(max_i64);
nest->file = index; nest->file = parse_state.index;
if (nest_stack_last != 0) if (parse_state.nest_stack_last != 0)
{ {
nest->parent = nest_stack_last->nest; nest->parent = parse_state.nest_stack_last->nest;
code_index_push_nest(&nest->parent->nest_list, nest); code_index_push_nest(&nest->parent->nest_list, nest);
} }
Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1); Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1);
stack->nest = nest; stack->nest = nest;
stack->prev = nest_stack_last; stack->prev = parse_state.nest_stack_last;
stack->match_id = query_match.id; stack->match_id = query_match.id;
if (nest_stack_last != 0) nest_stack_last->next = stack; if (parse_state.nest_stack_last != 0) parse_state.nest_stack_last->next = stack;
else nest_stack_first = stack; else parse_state.nest_stack_first = stack;
nest_stack_last = stack; parse_state.nest_stack_last = stack;
} }
else if (string_match(capture_name, SCu8("scope_end"))) else if (string_match(capture_name, SCu8("scope_end")))
{ {
Assert(nest_stack_last != 0); Assert(parse_state.nest_stack_last != 0);
Assert(nest_stack_last->match_id == query_match.id); Assert(parse_state.nest_stack_last->match_id == query_match.id);
Code_Index_Nest* nest = nest_stack_last->nest; Code_Index_Nest* nest = parse_state.nest_stack_last->nest;
nest->close = type_range; nest->close = type_range;
nest->is_closed = true; nest->is_closed = true;
nest->nest_array = code_index_nest_ptr_array_from_list(&arena, &nest->nest_list); nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list);
if (nest->parent == 0) code_index_push_nest(&index->nest_list, nest); if (nest->parent == 0) code_index_push_nest(&parse_state.index->nest_list, nest);
nest_stack_last = nest_stack_last->prev; parse_state.nest_stack_last = parse_state.nest_stack_last->prev;
if (nest_stack_last != 0) nest_stack_last->next = 0; if (parse_state.nest_stack_last != 0) parse_state.nest_stack_last->next = 0;
} }
else if (string_match(capture_name, SCu8("definition.class"))) else if (string_match(capture_name, SCu8("definition.class")))
{ {
last_note = code_index_new_note(index, &arena, CodeIndexNote_Type, type_range, nest_stack_last); parse_state.last_note = code_index_new_note(parse_state.index, &parse_state.index_arena, CodeIndexNote_Type, type_range, parse_state.nest_stack_last);
last_note_match_id = query_match.id; parse_state.last_note_match_id = query_match.id;
} }
else if (string_match(capture_name, SCu8("definition.function"))) else if (string_match(capture_name, SCu8("definition.function")))
{ {
last_note = code_index_new_note(index, &arena, CodeIndexNote_Function, type_range, nest_stack_last); parse_state.last_note = code_index_new_note(parse_state.index, &parse_state.index_arena, CodeIndexNote_Function, type_range, parse_state.nest_stack_last);
last_note_match_id = query_match.id; parse_state.last_note_match_id = query_match.id;
} }
else if (string_match(capture_name, SCu8("definition.method"))) else if (string_match(capture_name, SCu8("definition.method")))
{ {
last_note = code_index_new_note(index, &arena, CodeIndexNote_Function, type_range, nest_stack_last);; parse_state.last_note = code_index_new_note(parse_state.index, &parse_state.index_arena, CodeIndexNote_Function, type_range, parse_state.nest_stack_last);;
last_note_match_id = query_match.id; parse_state.last_note_match_id = query_match.id;
} }
else if (string_match(capture_name, SCu8("definition.type"))) else if (string_match(capture_name, SCu8("definition.type")))
{ {
last_note = code_index_new_note(index, &arena, CodeIndexNote_Type, type_range, nest_stack_last); parse_state.last_note = code_index_new_note(parse_state.index, &parse_state.index_arena, CodeIndexNote_Type, type_range, parse_state.nest_stack_last);
last_note_match_id = query_match.id; parse_state.last_note_match_id = query_match.id;
} }
else if (string_match(capture_name, SCu8("name"))) else if (string_match(capture_name, SCu8("name")))
{ {
if (last_note != 0 && last_note_match_id == query_match.id) if (parse_state.last_note != 0 && parse_state.last_note_match_id == query_match.id)
{ {
last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start); parse_state.last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start);
} }
} }
} }
while (nest_stack_last != 0) while (parse_state.nest_stack_last != 0)
{ {
Code_Index_Nest* nest = nest_stack_last->nest; Code_Index_Nest* nest = parse_state.nest_stack_last->nest;
if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest); if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest);
else code_index_push_nest(&index->nest_list, nest); else code_index_push_nest(&parse_state.index->nest_list, nest);
nest->nest_array = code_index_nest_ptr_array_from_list(&arena, &nest->nest_list); nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list);
nest_stack_last = nest_stack_last->prev; parse_state.nest_stack_last = parse_state.nest_stack_last->prev;
} }
for (Code_Index_Note* note = index->note_list.first; note != 0 && note->next != note; note = note->next) for (Code_Index_Note* note = parse_state.index->note_list.first; note != 0 && note->next != note; note = note->next)
{ {
note->text = push_string_copy(&arena, string_substring(buffer_contents, note->pos)); note->text = push_string_copy(&parse_state.index_arena, string_substring(parse_state.buffer_contents, note->pos));
} }
// Finish the Index // Finish the Index
index->nest_array = code_index_nest_ptr_array_from_list(&arena, &index->nest_list); parse_state.index->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->nest_list);
index->note_array = code_index_note_ptr_array_from_list(&arena, &index->note_list); parse_state.index->note_array = code_index_note_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->note_list);
code_index_lock(); tree_sitter_query_end(&parse_state.query);
code_index_set_file(buffer_id, arena, index);
code_index_unlock();
buffer_clear_layout_cache(app, buffer_id);
tree_sitter_query_end(&query); parse_state.ok = true;
return parse_state;
} }
function void function void
tree_sitter_code_index_update_tick(Application_Links* app) tree_sitter_code_index_update_tick(Application_Links* app)
{ {
#define BUFFERED_CODE_INDICES_CAP 8
Tree_Sitter_Code_Index_Parse_State buffered_code_indices[BUFFERED_CODE_INDICES_CAP];
u32 buffered_code_indices_count = 0;
Tree_Sitter_Code_Index_Parse_State parse_state = {};
for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first; for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first;
modified_node != 0; modified_node != 0;
modified_node = modified_node->next modified_node = modified_node->next
){ ){
tree_sitter_code_index_update_single_buffer(app, modified_node->buffer); parse_state = tree_sitter_code_index_update_single_buffer(app, modified_node->buffer);
if (parse_state.ok)
{
buffered_code_indices[buffered_code_indices_count] = parse_state;
buffered_code_indices_count += 1;
}
if (buffered_code_indices_count >= BUFFERED_CODE_INDICES_CAP)
{
code_index_lock();
for (int i = 0; i < buffered_code_indices_count; i++)
{
parse_state = buffered_code_indices[i];
code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
}
buffered_code_indices_count = 0;
code_index_unlock();
}
buffer_clear_layout_cache(app, parse_state.buffer_id);
}
if (buffered_code_indices_count > 0)
{
code_index_lock();
for (int i = 0; i < buffered_code_indices_count; i++)
{
parse_state = buffered_code_indices[i];
code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index);
}
code_index_unlock();
} }
buffer_modified_set_clear(); buffer_modified_set_clear();

45
code/custom/4coder_tree_sitter.h
View File

@ -46,21 +46,42 @@ struct Buffer_Tree_Sitter_Data
System_Mutex tree_mutex; System_Mutex tree_mutex;
}; };
struct Tree_Sitter_Code_Index_Nest_Node struct Tree_Sitter_Query_Cursor
{ {
Tree_Sitter_Code_Index_Nest_Node* next; Buffer_ID buffer_id;
Tree_Sitter_Code_Index_Nest_Node* prev; TSQuery* query;
Tree_Sitter_Code_Index_Nest_Node* parent; TSQueryCursor* query_cursor;
Tree_Sitter_Code_Index_Nest_Node* child_first; TSTree* tree;
Tree_Sitter_Code_Index_Nest_Node* child_last; TSNode first_node;
bool ok;
Code_Index_Nest* nest;
}; };
struct Tree_Sitter_Code_Index_Nest_List struct Code_Index_Nest_Stack
{ {
Tree_Sitter_Code_Index_Nest_Node* first; Code_Index_Nest_Stack* prev;
Tree_Sitter_Code_Index_Nest_Node* last; Code_Index_Nest_Stack* next;
Code_Index_Nest* nest;
u32 match_id;
};
struct Tree_Sitter_Code_Index_Parse_State
{
Buffer_ID buffer_id;
String_Const_u8 buffer_contents;
Tree_Sitter_Language_Definition* language;
Tree_Sitter_Query_Cursor query;
Arena index_arena;
Code_Index_File* index;
Code_Index_Nest_Stack* nest_stack_first = 0;
Code_Index_Nest_Stack* nest_stack_last = 0;
Code_Index_Note* last_note = 0;
u32 last_note_match_id = max_u32;
b8 ok;
}; };
function TSQuery* tree_sitter_query_new(Application_Links* app, TSLanguage* language, String_Const_u8 query_string); function TSQuery* tree_sitter_query_new(Application_Links* app, TSLanguage* language, String_Const_u8 query_string);
@ -68,7 +89,7 @@ function void tree_sitter_register_language(String_Const_u8 ext, TSLanguage* lan
b8 use_tree_sitter_code_indexing = true; b8 use_tree_sitter_code_indexing = true;
b8 use_tree_sitter_token_coloring = true; b8 use_tree_sitter_token_coloring = true;
function void tree_sitter_code_index_update_single_buffer(Application_Links *app, Buffer_ID buffer_id); function Tree_Sitter_Code_Index_Parse_State tree_sitter_code_index_update_single_buffer(Application_Links *app, Buffer_ID buffer_id);
function void tree_sitter_code_index_update_tick(Application_Links *app); function void tree_sitter_code_index_update_tick(Application_Links *app);
#endif //FCODER_TREE_SITTER_H #endif //FCODER_TREE_SITTER_H