diff --git a/code/custom/4coder_tree_sitter.cpp b/code/custom/4coder_tree_sitter.cpp index d050acb6..db182f8f 100644 --- a/code/custom/4coder_tree_sitter.cpp +++ b/code/custom/4coder_tree_sitter.cpp @@ -155,16 +155,6 @@ tree_sitter_buffer_get_tree_copy(Buffer_Tree_Sitter_Data* tree_data) // Queries //////////////////////////////////////////////////////////////////// -struct Tree_Sitter_Query_Cursor -{ - Buffer_ID buffer_id; - TSQuery* query; - TSQueryCursor* query_cursor; - TSTree* tree; - TSNode first_node; - bool ok; -}; - function Tree_Sitter_Query_Cursor tree_sitter_query_init(Application_Links* app, Buffer_ID buffer_id, TSQuery* query) { @@ -225,10 +215,6 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id) acquire_global_frame_mutex(app); Scratch_Block scratch(app); String_Const_u8 buffer_name = push_buffer_unique_name(app, scratch, buffer_id); - print_message(app, SCu8("Parsing ")); - print_message(app, buffer_name); - print_message(app, SCu8("\n")); - Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id); String_Const_u8 src = push_whole_buffer(app, &arena, buffer_id); Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); @@ -269,7 +255,10 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id) tree_data->tree = new_tree; system_mutex_acquire(tree_data->tree_mutex); - tree_sitter_code_index_update_single_buffer(app, buffer_id); + Tree_Sitter_Code_Index_Parse_State parse_state = tree_sitter_code_index_update_single_buffer(app, buffer_id); + code_index_lock(); + code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index); + code_index_unlock(); // Force a frame refresh by requesting another frame animate_in_n_milliseconds(app, 0); @@ -300,14 +289,6 @@ tree_sitter_node_to_range(TSNode node) return result; } -struct Code_Index_Nest_Stack -{ - Code_Index_Nest_Stack* prev; - Code_Index_Nest_Stack* next; - Code_Index_Nest* nest; - u32 match_id; -}; - function Code_Index_Note* code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind kind, Range_i64 range, Code_Index_Nest_Stack* parent) { @@ -321,141 +302,172 @@ code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind k return result; } -function void +function Tree_Sitter_Code_Index_Parse_State tree_sitter_code_index_update_single_buffer(Application_Links* app, Buffer_ID buffer_id) { Scratch_Block scratch(app); - Arena arena = make_arena_system(KB(16)); - Code_Index_File* index = push_array_zero(&arena, Code_Index_File, 1); - index->buffer = buffer_id; + Tree_Sitter_Code_Index_Parse_State parse_state = {}; + parse_state.ok = false; + parse_state.buffer_id = buffer_id; + parse_state.language = tree_sitter_language_for_buffer(app, parse_state.buffer_id); + if (!parse_state.language) return parse_state; - String_Const_u8 buffer_contents = push_whole_buffer(app, scratch, buffer_id); - - Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id); - if (!lang) return; - - TSQuery* ts_query = lang->queries.ptr[Tree_Sitter_Language_Query_Tags]; - Tree_Sitter_Query_Cursor query = tree_sitter_query_init( - app, buffer_id, ts_query + parse_state.index_arena = make_arena_system(KB(16)); + parse_state.index = push_array_zero(&parse_state.index_arena, Code_Index_File, 1); + parse_state.buffer_contents = push_whole_buffer(app, scratch, parse_state.buffer_id); + parse_state.query = tree_sitter_query_init( + app, parse_state.buffer_id, parse_state.language->queries.ptr[Tree_Sitter_Language_Query_Tags] ); - - Code_Index_Nest_Stack* nest_stack_first = 0; - Code_Index_Nest_Stack* nest_stack_last = 0; - - Code_Index_Note* last_note = 0; - u32 last_note_match_id = max_u32; + parse_state.nest_stack_first = 0; + parse_state.nest_stack_last = 0; + parse_state.last_note = 0; + parse_state.last_note_match_id = max_u32; TSQueryMatch query_match; u32 capture_index; - while (tree_sitter_query_continue(&query, &query_match, &capture_index)) + while (tree_sitter_query_continue(&parse_state.query, &query_match, &capture_index)) { TSQueryCapture type_capture = query_match.captures[capture_index]; + TSNode type_node = type_capture.node; Range_i64 type_range = tree_sitter_node_to_range(type_node); u32 length; - const char* tmp = ts_query_capture_name_for_id(query.query, type_capture.index, &length); + const char* tmp = ts_query_capture_name_for_id(parse_state.query.query, type_capture.index, &length); String_Const_u8 capture_name = SCu8((char*)tmp, length); if (string_match(capture_name, SCu8("scope_begin"))) { - Code_Index_Nest* nest = push_array_zero(&arena, Code_Index_Nest, 1); + Code_Index_Nest* nest = push_array_zero(&parse_state.index_arena, Code_Index_Nest, 1); nest->kind = CodeIndexNest_Scope; nest->open = type_range; nest->close = Ii64(max_i64); - nest->file = index; - if (nest_stack_last != 0) + nest->file = parse_state.index; + if (parse_state.nest_stack_last != 0) { - nest->parent = nest_stack_last->nest; + nest->parent = parse_state.nest_stack_last->nest; code_index_push_nest(&nest->parent->nest_list, nest); } Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1); stack->nest = nest; - stack->prev = nest_stack_last; + stack->prev = parse_state.nest_stack_last; stack->match_id = query_match.id; - if (nest_stack_last != 0) nest_stack_last->next = stack; - else nest_stack_first = stack; - nest_stack_last = stack; + if (parse_state.nest_stack_last != 0) parse_state.nest_stack_last->next = stack; + else parse_state.nest_stack_first = stack; + parse_state.nest_stack_last = stack; } else if (string_match(capture_name, SCu8("scope_end"))) { - Assert(nest_stack_last != 0); - Assert(nest_stack_last->match_id == query_match.id); + Assert(parse_state.nest_stack_last != 0); + Assert(parse_state.nest_stack_last->match_id == query_match.id); - Code_Index_Nest* nest = nest_stack_last->nest; + Code_Index_Nest* nest = parse_state.nest_stack_last->nest; nest->close = type_range; nest->is_closed = true; - nest->nest_array = code_index_nest_ptr_array_from_list(&arena, &nest->nest_list); - if (nest->parent == 0) code_index_push_nest(&index->nest_list, nest); + nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list); + if (nest->parent == 0) code_index_push_nest(&parse_state.index->nest_list, nest); - nest_stack_last = nest_stack_last->prev; - if (nest_stack_last != 0) nest_stack_last->next = 0; + parse_state.nest_stack_last = parse_state.nest_stack_last->prev; + if (parse_state.nest_stack_last != 0) parse_state.nest_stack_last->next = 0; } else if (string_match(capture_name, SCu8("definition.class"))) { - last_note = code_index_new_note(index, &arena, CodeIndexNote_Type, type_range, nest_stack_last); - last_note_match_id = query_match.id; + parse_state.last_note = code_index_new_note(parse_state.index, &parse_state.index_arena, CodeIndexNote_Type, type_range, parse_state.nest_stack_last); + parse_state.last_note_match_id = query_match.id; } else if (string_match(capture_name, SCu8("definition.function"))) { - last_note = code_index_new_note(index, &arena, CodeIndexNote_Function, type_range, nest_stack_last); - last_note_match_id = query_match.id; + parse_state.last_note = code_index_new_note(parse_state.index, &parse_state.index_arena, CodeIndexNote_Function, type_range, parse_state.nest_stack_last); + parse_state.last_note_match_id = query_match.id; } else if (string_match(capture_name, SCu8("definition.method"))) { - last_note = code_index_new_note(index, &arena, CodeIndexNote_Function, type_range, nest_stack_last);; - last_note_match_id = query_match.id; + parse_state.last_note = code_index_new_note(parse_state.index, &parse_state.index_arena, CodeIndexNote_Function, type_range, parse_state.nest_stack_last);; + parse_state.last_note_match_id = query_match.id; } else if (string_match(capture_name, SCu8("definition.type"))) { - last_note = code_index_new_note(index, &arena, CodeIndexNote_Type, type_range, nest_stack_last); - last_note_match_id = query_match.id; + parse_state.last_note = code_index_new_note(parse_state.index, &parse_state.index_arena, CodeIndexNote_Type, type_range, parse_state.nest_stack_last); + parse_state.last_note_match_id = query_match.id; } else if (string_match(capture_name, SCu8("name"))) { - if (last_note != 0 && last_note_match_id == query_match.id) + if (parse_state.last_note != 0 && parse_state.last_note_match_id == query_match.id) { - last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start); + parse_state.last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start); } } } - while (nest_stack_last != 0) + while (parse_state.nest_stack_last != 0) { - Code_Index_Nest* nest = nest_stack_last->nest; + Code_Index_Nest* nest = parse_state.nest_stack_last->nest; if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest); - else code_index_push_nest(&index->nest_list, nest); - nest->nest_array = code_index_nest_ptr_array_from_list(&arena, &nest->nest_list); - nest_stack_last = nest_stack_last->prev; + else code_index_push_nest(&parse_state.index->nest_list, nest); + nest->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &nest->nest_list); + parse_state.nest_stack_last = parse_state.nest_stack_last->prev; } - for (Code_Index_Note* note = index->note_list.first; note != 0 && note->next != note; note = note->next) + for (Code_Index_Note* note = parse_state.index->note_list.first; note != 0 && note->next != note; note = note->next) { - note->text = push_string_copy(&arena, string_substring(buffer_contents, note->pos)); + note->text = push_string_copy(&parse_state.index_arena, string_substring(parse_state.buffer_contents, note->pos)); } // Finish the Index - index->nest_array = code_index_nest_ptr_array_from_list(&arena, &index->nest_list); - index->note_array = code_index_note_ptr_array_from_list(&arena, &index->note_list); + parse_state.index->nest_array = code_index_nest_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->nest_list); + parse_state.index->note_array = code_index_note_ptr_array_from_list(&parse_state.index_arena, &parse_state.index->note_list); - code_index_lock(); - code_index_set_file(buffer_id, arena, index); - code_index_unlock(); - buffer_clear_layout_cache(app, buffer_id); + tree_sitter_query_end(&parse_state.query); - tree_sitter_query_end(&query); + parse_state.ok = true; + return parse_state; } function void tree_sitter_code_index_update_tick(Application_Links* app) { +#define BUFFERED_CODE_INDICES_CAP 8 + Tree_Sitter_Code_Index_Parse_State buffered_code_indices[BUFFERED_CODE_INDICES_CAP]; + u32 buffered_code_indices_count = 0; + + Tree_Sitter_Code_Index_Parse_State parse_state = {}; for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first; modified_node != 0; modified_node = modified_node->next ){ - tree_sitter_code_index_update_single_buffer(app, modified_node->buffer); + parse_state = tree_sitter_code_index_update_single_buffer(app, modified_node->buffer); + if (parse_state.ok) + { + buffered_code_indices[buffered_code_indices_count] = parse_state; + buffered_code_indices_count += 1; + } + + if (buffered_code_indices_count >= BUFFERED_CODE_INDICES_CAP) + { + code_index_lock(); + for (int i = 0; i < buffered_code_indices_count; i++) + { + parse_state = buffered_code_indices[i]; + code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index); + } + buffered_code_indices_count = 0; + code_index_unlock(); + } + + buffer_clear_layout_cache(app, parse_state.buffer_id); + } + + if (buffered_code_indices_count > 0) + { + code_index_lock(); + for (int i = 0; i < buffered_code_indices_count; i++) + { + parse_state = buffered_code_indices[i]; + code_index_set_file(parse_state.buffer_id, parse_state.index_arena, parse_state.index); + } + code_index_unlock(); } buffer_modified_set_clear(); diff --git a/code/custom/4coder_tree_sitter.h b/code/custom/4coder_tree_sitter.h index 9611a85f..a0991777 100644 --- a/code/custom/4coder_tree_sitter.h +++ b/code/custom/4coder_tree_sitter.h @@ -46,21 +46,42 @@ struct Buffer_Tree_Sitter_Data System_Mutex tree_mutex; }; -struct Tree_Sitter_Code_Index_Nest_Node +struct Tree_Sitter_Query_Cursor { - Tree_Sitter_Code_Index_Nest_Node* next; - Tree_Sitter_Code_Index_Nest_Node* prev; - Tree_Sitter_Code_Index_Nest_Node* parent; - Tree_Sitter_Code_Index_Nest_Node* child_first; - Tree_Sitter_Code_Index_Nest_Node* child_last; - - Code_Index_Nest* nest; + Buffer_ID buffer_id; + TSQuery* query; + TSQueryCursor* query_cursor; + TSTree* tree; + TSNode first_node; + bool ok; }; -struct Tree_Sitter_Code_Index_Nest_List +struct Code_Index_Nest_Stack { - Tree_Sitter_Code_Index_Nest_Node* first; - Tree_Sitter_Code_Index_Nest_Node* last; + Code_Index_Nest_Stack* prev; + Code_Index_Nest_Stack* next; + Code_Index_Nest* nest; + u32 match_id; +}; + +struct Tree_Sitter_Code_Index_Parse_State +{ + Buffer_ID buffer_id; + String_Const_u8 buffer_contents; + + Tree_Sitter_Language_Definition* language; + Tree_Sitter_Query_Cursor query; + + Arena index_arena; + Code_Index_File* index; + + Code_Index_Nest_Stack* nest_stack_first = 0; + Code_Index_Nest_Stack* nest_stack_last = 0; + + Code_Index_Note* last_note = 0; + u32 last_note_match_id = max_u32; + + b8 ok; }; function TSQuery* tree_sitter_query_new(Application_Links* app, TSLanguage* language, String_Const_u8 query_string); @@ -68,7 +89,7 @@ function void tree_sitter_register_language(String_Const_u8 ext, TSLanguage* lan b8 use_tree_sitter_code_indexing = true; b8 use_tree_sitter_token_coloring = true; -function void tree_sitter_code_index_update_single_buffer(Application_Links *app, Buffer_ID buffer_id); +function Tree_Sitter_Code_Index_Parse_State tree_sitter_code_index_update_single_buffer(Application_Links *app, Buffer_ID buffer_id); function void tree_sitter_code_index_update_tick(Application_Links *app); #endif //FCODER_TREE_SITTER_H