From 5ccd6dd2ab79cab09cff1942a4cf077ef2835f41 Mon Sep 17 00:00:00 2001 From: Peter Slattery Date: Sun, 13 Jul 2025 11:19:29 -0700 Subject: [PATCH] Implemented tree_sitter_code_index_update_tick - populates Code_Index_Nests and Code_Index_Notes from each languages Tags query Switched tree_sitter_list_all_query_results to list Code_Index_Note's matching a Code_Index_Note_Kind in each file - remove old query types NESTS, FUNCTIONS, TYPES --- code/custom/4coder_tree_sitter.cpp | 245 ++++++++++++++++++----------- code/custom/4coder_tree_sitter.h | 6 - 2 files changed, 152 insertions(+), 99 deletions(-) diff --git a/code/custom/4coder_tree_sitter.cpp b/code/custom/4coder_tree_sitter.cpp index 77e6c200..f6deca6d 100644 --- a/code/custom/4coder_tree_sitter.cpp +++ b/code/custom/4coder_tree_sitter.cpp @@ -307,60 +307,162 @@ tree_sitter_node_to_range(TSNode node) return result; } +struct Code_Index_Nest_Stack +{ + Code_Index_Nest_Stack* prev; + Code_Index_Nest_Stack* next; + Code_Index_Nest* nest; + u32 match_id; +}; + +function Code_Index_Note* +code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind kind, Range_i64 range, Code_Index_Nest_Stack* parent) +{ + Code_Index_Note *result = push_array(arena, Code_Index_Note, 1); + sll_queue_push(index->note_list.first, index->note_list.last, result); + index->note_list.count += 1; + result->file = index; + result->note_kind = kind; + result->pos = range; + if (parent != 0) result->parent = parent->nest; + return result; +} + function void tree_sitter_code_index_update_tick(Application_Links* app) { Scratch_Block scratch(app); -#if 0 - // TODO(PS): this should be done when we register the language - if (!tree_sitter_cpp_index_query) - { - u32 error_offset; - TSQueryError query_error; - tree_sitter_cpp_index_query = ts_query_new( - tree_sitter_cpp(), TS_CPP_INDEX_QUERY, (u32)TS_CPP_INDEX_QUERY.size, &error_offset, &query_error - ); - if (!tree_sitter_cpp_index_query) - { - print_message(app, string_u8_litexpr("Failed to create cpp index query\n"); - } - } - for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first; modified_node != 0; modified_node = modified_node->next ){ Temp_Memory_Block temp(scratch); - Buffer_ID buffer_id = modified_node->buffer; + Buffer_ID buffer_id = modified_node->buffer; Arena arena = make_arena_system(KB(16)); Code_Index_File* index = push_array_zero(&arena, Code_Index_File, 1); index->buffer = buffer_id; - Tree_Sitter_Code_Index_Nest_List nests = {}; + String_Const_u8 buffer_contents = push_whole_buffer(app, scratch, buffer_id); - Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id); - Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); - TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data); - if (tree) + Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id); + if (!lang) continue; + + TSQuery* ts_query = lang->queries.ptr[Tree_Sitter_Language_Query_Tags]; + Tree_Sitter_Query_Cursor query = tree_sitter_query_init( + app, buffer_id, ts_query + ); + + Code_Index_Nest_Stack* nest_stack_first = 0; + Code_Index_Nest_Stack* nest_stack_last = 0; + + Code_Index_Note* last_note = 0; + u32 last_note_match_id = max_u32; + + TSQueryMatch query_match; + u32 capture_index; + while (tree_sitter_query_continue(&query, &query_match, &capture_index)) { - TSQueryCursor* query_cursor = ts_query_cursor_new(); - ts_query_cursor_exec(query_cursor, tree_sitter_cpp_index_query, ts_tree_root_node(tree)); + TSQueryCapture type_capture = query_match.captures[capture_index]; + TSNode type_node = type_capture.node; + Range_i64 type_range = tree_sitter_node_to_range(type_node); - TSQueryMatch query_match; - while (ts_query_cursor_next_match(query_cursor, &match)) + u32 length; + const char* tmp = ts_query_capture_name_for_id(query.query, type_capture.index, &length); + String_Const_u8 capture_name = SCu8((char*)tmp, length); + + if (string_match(capture_name, SCu8("scope_begin"))) { - TSQueryCapture type_capture = match.captures[0]; - TSNode type_node = type_capture.node; - Range_i64 type_range = tree_sitter_node_to_range(type_node); + Code_Index_Nest* nest = push_array_zero(&arena, Code_Index_Nest, 1); + nest->kind = CodeIndexNest_Scope; + nest->open = type_range; + nest->close = Ii64(max_i64); + nest->file = index; + if (nest_stack_last != 0) + { + nest->parent = nest_stack_last->nest; + code_index_push_nest(&nest->parent->nest_list, nest); + } - // TODO(PS): PICK UP HERE + Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1); + stack->nest = nest; + stack->prev = nest_stack_last; + stack->match_id = query_match.id; + if (nest_stack_last != 0) nest_stack_last->next = stack; + else nest_stack_first = stack; + nest_stack_last = stack; + } + else if (string_match(capture_name, SCu8("scope_end"))) + { + Assert(nest_stack_last != 0); + Assert(nest_stack_last->match_id == query_match.id); + + Code_Index_Nest* nest = nest_stack_last->nest; + nest->close = type_range; + nest->is_closed = true; + nest->nest_array = code_index_nest_ptr_array_from_list(&arena, &nest->nest_list); + if (nest->parent == 0) code_index_push_nest(&index->nest_list, nest); + + nest_stack_last = nest_stack_last->prev; + if (nest_stack_last != 0) nest_stack_last->next = 0; + } + else if (string_match(capture_name, SCu8("definition.class"))) + { + last_note = code_index_new_note(index, &arena, CodeIndexNote_Type, type_range, nest_stack_last); + last_note_match_id = query_match.id; + } + else if (string_match(capture_name, SCu8("definition.function"))) + { + last_note = code_index_new_note(index, &arena, CodeIndexNote_Function, type_range, nest_stack_last); + last_note_match_id = query_match.id; + } + else if (string_match(capture_name, SCu8("definition.method"))) + { + last_note = code_index_new_note(index, &arena, CodeIndexNote_Function, type_range, nest_stack_last);; + last_note_match_id = query_match.id; + } + else if (string_match(capture_name, SCu8("definition.type"))) + { + last_note = code_index_new_note(index, &arena, CodeIndexNote_Type, type_range, nest_stack_last); + last_note_match_id = query_match.id; + } + else if (string_match(capture_name, SCu8("name"))) + { + if (last_note != 0 && last_note_match_id == query_match.id) + { + last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start); + } } } - ts_tree_delete(tree); + + while (nest_stack_last != 0) + { + Code_Index_Nest* nest = nest_stack_last->nest; + if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest); + else code_index_push_nest(&index->nest_list, nest); + nest->nest_array = code_index_nest_ptr_array_from_list(&arena, &nest->nest_list); + nest_stack_last = nest_stack_last->prev; + } + + for (Code_Index_Note* note = index->note_list.first; note != 0 && note->next != note; note = note->next) + { + note->text = push_string_copy(&arena, string_substring(buffer_contents, note->pos)); + } + + // Finish the Index + index->nest_array = code_index_nest_ptr_array_from_list(&arena, &index->nest_list); + index->note_array = code_index_note_ptr_array_from_list(&arena, &index->note_list); + + code_index_lock(); + code_index_set_file(buffer_id, arena, index); + code_index_unlock(); + buffer_clear_layout_cache(app, buffer_id); + + tree_sitter_query_end(&query); } -#endif + + buffer_modified_set_clear(); } //////////////////////////////////////////////////////////////////// @@ -549,7 +651,7 @@ function void tree_sitter_list_all_query_results( Application_Links *app, Buffer_ID optional_target_buffer, - Tree_Sitter_Language_Query_Kind query_kind + Code_Index_Note_Kind note_kind ){ String_Const_u8 decls_name = string_u8_litexpr("*decls*"); Buffer_ID decls_buffer = get_buffer_by_name(app, decls_name, Access_Always); @@ -570,75 +672,32 @@ tree_sitter_list_all_query_results( Cursor insertion_cursor = make_cursor(push_array(scratch, u8, KB(256)), KB(256)); Buffer_Insertion out = begin_buffer_insertion_at_buffered(app, decls_buffer, 0, &insertion_cursor); + code_index_lock(); for (Buffer_ID buffer_it = get_buffer_next(app, 0, Access_Always); buffer_it != 0; buffer_it = get_buffer_next(app, buffer_it, Access_Always)) { Buffer_ID buffer = buffer_it; if (optional_target_buffer != 0) buffer = optional_target_buffer; - String_Const_u8 buffer_name = push_buffer_unique_name(app, scratch, buffer); - - Token_Array array = get_token_array_from_buffer(app, buffer); - if (array.tokens != 0) + Code_Index_File* file = code_index_get_file(buffer); + if (file != 0) { - Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer); - if (!lang) continue; - TSQuery* ts_query = lang->queries.ptr[Tree_Sitter_Language_Query_Tags]; - if (!ts_query) ts_query = lang->queries.ptr[Tree_Sitter_Language_Query_Highlights]; - Tree_Sitter_Query_Cursor query = tree_sitter_query_init(app, buffer, ts_query); - - TSQueryMatch query_match; - u32 capture_index; - bool reached_end = false; - while (tree_sitter_query_continue(&query, &query_match, &capture_index)) + for (i32 i = 0; i < file->note_array.count; i += 1) { - TSQueryCapture capture = query_match.captures[capture_index]; - - Range_i64 match_range; - match_range.start = (i64)ts_node_start_byte(capture.node); - match_range.end = (i64)ts_node_end_byte(capture.node); - - String_Const_u8 name; - u32 name_length; - name.str = (u8*)ts_query_capture_name_for_id(ts_query, capture.index, &name_length); - name.size = (u64)name_length; - - bool matches = false; - switch (query_kind) - { - case Tree_Sitter_Language_Query_Functions: - { - matches = string_match(name, SCu8("function")); - matches |= string_match(name, SCu8("definition.function")); - - // TODO(PS): ideally, these should be provided by the language - matches |= string_match(name, SCu8("definition.method")); - } break; - - case Tree_Sitter_Language_Query_Types: - { - matches = string_match(name, SCu8("type")); - matches |= string_match(name, SCu8("definition.type")); - - // TODO(PS): ideally, these should be provided by the language - matches |= string_match(name, SCu8("definition.class")); - matches |= string_match(name, SCu8("definition.interface")); - } break; - } - - if (matches) + Code_Index_Note *note = file->note_array.ptrs[i]; + if (note->note_kind == note_kind) { print_position( - app, &out, buffer, buffer_name, match_range, {}, scratch, true + app, &out, buffer, buffer_name, note->pos, {}, scratch, true ); } } - tree_sitter_query_end(&query); - - if (optional_target_buffer != 0) break; } + + if (optional_target_buffer != 0) break; } + code_index_unlock(); end_buffer_insertion(&out); @@ -653,7 +712,7 @@ CUSTOM_DOC("Creates a jump list of lines of the current buffer that appear to de { View_ID view = get_active_view(app, Access_ReadVisible); Buffer_ID buffer = view_get_buffer(app, view, Access_ReadVisible); - if (buffer != 0) tree_sitter_list_all_query_results(app, buffer, Tree_Sitter_Language_Query_Functions); + if (buffer != 0) tree_sitter_list_all_query_results(app, buffer, CodeIndexNote_Function); } CUSTOM_UI_COMMAND_SIG(tree_sitter_list_all_functions_current_buffer_lister) @@ -664,7 +723,7 @@ CUSTOM_DOC("Creates a lister of locations that look like function definitions an Buffer_ID buffer = view_get_buffer(app, view, Access_ReadVisible); if (buffer != 0) { - tree_sitter_list_all_query_results(app, buffer, Tree_Sitter_Language_Query_Functions); + tree_sitter_list_all_query_results(app, buffer, CodeIndexNote_Function); view = get_active_view(app, Access_Always); buffer = view_get_buffer(app, view, Access_Always); Marker_List *list = get_or_make_list_for_buffer(app, heap, buffer); @@ -679,14 +738,14 @@ CUSTOM_DOC("Creates a lister of locations that look like function definitions an CUSTOM_COMMAND_SIG(tree_sitter_list_all_functions_all_buffers) CUSTOM_DOC("Creates a jump list of lines from all buffers that appear to define or declare functions. Uses tree sitter") { - tree_sitter_list_all_query_results(app, 0, Tree_Sitter_Language_Query_Functions); + tree_sitter_list_all_query_results(app, 0, CodeIndexNote_Function); } CUSTOM_UI_COMMAND_SIG(tree_sitter_list_all_functions_all_buffers_lister) CUSTOM_DOC("Creates a lister of locations that look like function definitions and declarations all buffers. Uses tree sitter") { Heap *heap = &global_heap; - tree_sitter_list_all_query_results(app, 0, Tree_Sitter_Language_Query_Functions); + tree_sitter_list_all_query_results(app, 0, CodeIndexNote_Function); View_ID view = get_active_view(app, Access_Always); Buffer_ID buffer = view_get_buffer(app, view, Access_Always); Marker_List *list = get_or_make_list_for_buffer(app, heap, buffer); @@ -702,7 +761,7 @@ CUSTOM_DOC("Creates a jump list of lines of the current buffer that appear to de { View_ID view = get_active_view(app, Access_ReadVisible); Buffer_ID buffer = view_get_buffer(app, view, Access_ReadVisible); - if (buffer != 0) tree_sitter_list_all_query_results(app, buffer, Tree_Sitter_Language_Query_Types); + if (buffer != 0) tree_sitter_list_all_query_results(app, buffer, CodeIndexNote_Type); } CUSTOM_UI_COMMAND_SIG(tree_sitter_list_all_types_current_buffer_lister) @@ -713,7 +772,7 @@ CUSTOM_DOC("Creates a lister of locations that look like function definitions an Buffer_ID buffer = view_get_buffer(app, view, Access_ReadVisible); if (buffer != 0) { - tree_sitter_list_all_query_results(app, buffer, Tree_Sitter_Language_Query_Types); + tree_sitter_list_all_query_results(app, buffer, CodeIndexNote_Type); view = get_active_view(app, Access_Always); buffer = view_get_buffer(app, view, Access_Always); Marker_List *list = get_or_make_list_for_buffer(app, heap, buffer); @@ -728,14 +787,14 @@ CUSTOM_DOC("Creates a lister of locations that look like function definitions an CUSTOM_COMMAND_SIG(tree_sitter_list_all_types_all_buffers) CUSTOM_DOC("Creates a jump list of lines from all buffers that appear to define or declare types. Uses tree sitter") { - tree_sitter_list_all_query_results(app, 0, Tree_Sitter_Language_Query_Types); + tree_sitter_list_all_query_results(app, 0, CodeIndexNote_Type); } CUSTOM_UI_COMMAND_SIG(tree_sitter_list_all_types_all_buffers_lister) CUSTOM_DOC("Creates a lister of locations that look like type definitions and declarations all buffers. Uses tree sitter") { Heap *heap = &global_heap; - tree_sitter_list_all_query_results(app, 0, Tree_Sitter_Language_Query_Types); + tree_sitter_list_all_query_results(app, 0, CodeIndexNote_Type); View_ID view = get_active_view(app, Access_Always); Buffer_ID buffer = view_get_buffer(app, view, Access_Always); Marker_List *list = get_or_make_list_for_buffer(app, heap, buffer); diff --git a/code/custom/4coder_tree_sitter.h b/code/custom/4coder_tree_sitter.h index d2be4721..1b977c52 100644 --- a/code/custom/4coder_tree_sitter.h +++ b/code/custom/4coder_tree_sitter.h @@ -9,12 +9,6 @@ enum Tree_Sitter_Language_Query_Kind { Tree_Sitter_Language_Query_Highlights, Tree_Sitter_Language_Query_Tags, - - Tree_Sitter_Language_Query_Nests, - - Tree_Sitter_Language_Query_Functions, - Tree_Sitter_Language_Query_Types, - Tree_Sitter_Language_Query_Count, };