From 30ac43ce6625dcf2f28db4180a1c4af2497c36fd Mon Sep 17 00:00:00 2001 From: Peter Slattery Date: Sat, 2 Aug 2025 23:21:46 -0700 Subject: [PATCH] Prevent re-parsing the whole file by identifying the node containing the edit, or the node immediately before and after the edit, and parsing only those. Fall back to root for empty file case --- code/custom/4coder_custom_hooks.cpp | 21 +-- code/custom/4coder_tree_sitter.cpp | 281 +++++++++++++++++++--------- 2 files changed, 200 insertions(+), 102 deletions(-) diff --git a/code/custom/4coder_custom_hooks.cpp b/code/custom/4coder_custom_hooks.cpp index 74fe871f..e7877679 100644 --- a/code/custom/4coder_custom_hooks.cpp +++ b/code/custom/4coder_custom_hooks.cpp @@ -393,18 +393,15 @@ function void custom_render_buffer( if (use_tree_sitter_token_coloring) { Managed_Scope scope = buffer_get_managed_scope(app, buffer); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); - for (int i = 0; i < tree_data->last_update_node_range_count; i++) - { - Range_i64 last_update_range = tree_data->last_update_node_range[i]; - Rect_f32 range_min = text_layout_character_on_screen(app, text_layout_id, last_update_range.min); - Rect_f32 range_max = text_layout_character_on_screen(app, text_layout_id, last_update_range.max); - Rect_f32 range; - range.x0 = Min(range_min.x0, range_max.x0); - range.y0 = Min(range_min.y0, range_max.y0); - range.x1 = Max(range_min.x1, range_max.x1); - range.y1 = Max(range_min.y1, range_max.y1); - draw_rectangle(app, range, 0.f, 0x33FF00FF); - } + Range_i64 last_update_range = tree_data->last_update_node_range; + Rect_f32 range_min = text_layout_character_on_screen(app, text_layout_id, last_update_range.min); + Rect_f32 range_max = text_layout_character_on_screen(app, text_layout_id, last_update_range.max); + Rect_f32 range; + range.x0 = Min(range_min.x0, range_max.x0); + range.y0 = Min(range_min.y0, range_max.y0); + range.x1 = Max(range_min.x1, range_max.x1); + range.y1 = Max(range_min.y1, range_max.y1); + draw_rectangle(app, range, 0.f, 0x33FF00FF); } draw_set_clip(app, prev_clip); } diff --git a/code/custom/4coder_tree_sitter.cpp b/code/custom/4coder_tree_sitter.cpp index ec58d8ff..5b52fef6 100644 --- a/code/custom/4coder_tree_sitter.cpp +++ b/code/custom/4coder_tree_sitter.cpp @@ -206,6 +206,14 @@ tree_sitter_query_init(Application_Links* app, Buffer_ID buffer_id, TSQuery* que return result; } +function Tree_Sitter_Query_Cursor +tree_sitter_query_init(Application_Links* app, Buffer_ID buffer_id, TSQuery* query, TSNode first_node) +{ + Tree_Sitter_Query_Cursor result = tree_sitter_query_init(app, buffer_id, query); + result.first_node = first_node; + return result; +} + function bool tree_sitter_query_continue(Tree_Sitter_Query_Cursor* cursor, TSQueryMatch* match, u32* capture_index) { @@ -477,11 +485,6 @@ tree_sitter_code_index_update_state_create( } state.buffer_contents = push_whole_buffer(app, scratch, state.buffer_id); - state.query = tree_sitter_query_init( - app, - state.buffer_id, - state.language->queries.ptr[Tree_Sitter_Language_Query_Tags] - ); state.nest_stack_first = 0; state.nest_stack_last = 0; state.last_note = 0; @@ -509,6 +512,7 @@ tree_sitter_code_index_update_state_create( function void tree_sitter_code_index_update_process_query_match( Tree_Sitter_Code_Index_Update_State* state, + Tree_Sitter_Query_Cursor query, TSQueryMatch query_match, u32 capture_index, Arena* scratch @@ -519,7 +523,7 @@ tree_sitter_code_index_update_process_query_match( Range_i64 type_range = tree_sitter_node_to_range(type_node); u32 length; - const char* tmp = ts_query_capture_name_for_id(state->query.query, type_capture.index, &length); + const char* tmp = ts_query_capture_name_for_id(query.query, type_capture.index, &length); String_Const_u8 capture_name = SCu8((char*)tmp, length); if (string_match(capture_name, SCu8("scope_begin")) || string_match(capture_name, SCu8("scope_end"))) @@ -589,7 +593,6 @@ tree_sitter_code_index_update_complete( Arena* scratch, bool update_was_incremental ){ - tree_sitter_query_end(&state->query); if (state->ok) { Code_Index_Nest* free_nests = state->index->nest_list.first; @@ -699,18 +702,26 @@ tree_sitter_code_index_update_full_file(Application_Links* app, Buffer_ID buffer scratch ); + Tree_Sitter_Query_Cursor query = tree_sitter_query_init( + app, + state.buffer_id, + state.language->queries.ptr[Tree_Sitter_Language_Query_Tags] + ); + TSQueryMatch query_match; u32 capture_index; - while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index)) + while (state.ok && tree_sitter_query_continue(&query, &query_match, &capture_index)) { tree_sitter_code_index_update_process_query_match( &state, + query, query_match, capture_index, scratch ); } + tree_sitter_query_end(&query); tree_sitter_code_index_update_complete(app, &state, scratch, false); return state; } @@ -730,13 +741,18 @@ tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data) 0, scratch ); + Tree_Sitter_Query_Cursor query = tree_sitter_query_init( + app, + state.buffer_id, + state.language->queries.ptr[Tree_Sitter_Language_Query_Tags] + ); release_global_frame_mutex(app); TSQueryMatch query_match; u32 capture_index; - while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index)) + while (state.ok && tree_sitter_query_continue(&query, &query_match, &capture_index)) { - tree_sitter_code_index_update_process_query_match(&state, query_match, capture_index, scratch); + tree_sitter_code_index_update_process_query_match(&state, query, query_match, capture_index, scratch); if (async_check_canceled(actx)) { state.ok = false; @@ -744,6 +760,7 @@ tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data) } } + tree_sitter_query_end(&query); tree_sitter_code_index_update_complete(app, &state, scratch, false); } @@ -774,31 +791,111 @@ tree_sitter_code_index_update_tick(Application_Links* app) ); // Find the largest, non-root node for us to reparse in its entirety + TSNode root = ts_tree_root_node(tree_data->tree); TSNode first_node = ts_node_descendant_for_byte_range( - state.query.first_node, new_range.min, new_range.max + root, new_range.min, new_range.max ); + TSNode last_node = first_node; + bool would_reparse_entire_file = ts_node_eq(first_node, root); - printf("Initial Scope\n"); - print_tree_sitter_tree(first_node, 0, ""); + int query_count = 1; + Tree_Sitter_Query_Cursor queries[2]; - if (!ts_node_eq(first_node, state.query.first_node)) + if (would_reparse_entire_file) + { + TSNode prev_sibling = {0}; + TSNode next_sibling = {0}; + + u32 child_count = ts_node_child_count(root); + for (u32 i = 0; i < child_count; i++) + { + TSNode child = ts_node_child(root, i); + u32 start_byte = ts_node_start_byte(child); + u32 end_byte = ts_node_end_byte(child); + if (end_byte <= new_range.min) + { + prev_sibling = child; + } + else if (start_byte >= new_range.max) + { + next_sibling = child; + break; + } + } + + if (!ts_node_is_null(prev_sibling) && !ts_node_is_null(next_sibling)) + { + first_node = prev_sibling; + last_node = next_sibling; + query_count = 2; + queries[0] = tree_sitter_query_init( + app, + state.buffer_id, + state.language->queries.ptr[Tree_Sitter_Language_Query_Tags], + prev_sibling + ); + queries[1] = tree_sitter_query_init( + app, + state.buffer_id, + state.language->queries.ptr[Tree_Sitter_Language_Query_Tags], + next_sibling + ); + } + else if (!ts_node_is_null(prev_sibling)) + { + first_node = prev_sibling; + last_node = prev_sibling; + queries[0] = tree_sitter_query_init( + app, + state.buffer_id, + state.language->queries.ptr[Tree_Sitter_Language_Query_Tags], + prev_sibling + ); + } + else if (!ts_node_is_null(next_sibling)) + { + first_node = next_sibling; + last_node = next_sibling; + queries[0] = tree_sitter_query_init( + app, + state.buffer_id, + state.language->queries.ptr[Tree_Sitter_Language_Query_Tags], + next_sibling + ); + } + else + { + first_node = root; + last_node = root; + queries[0] = tree_sitter_query_init( + app, + state.buffer_id, + state.language->queries.ptr[Tree_Sitter_Language_Query_Tags], + root + ); + } + } + else { TSNode parent = ts_node_parent(first_node); - while (!ts_node_has_error(parent) && !ts_node_eq(parent, state.query.first_node)) + while (!ts_node_has_error(parent) && !ts_node_eq(parent, root)) { first_node = parent; parent = ts_node_parent(first_node); - printf("Ascending\n"); - print_tree_sitter_tree(first_node, 0, ""); } + last_node = first_node; + queries[0] = tree_sitter_query_init( + app, + state.buffer_id, + state.language->queries.ptr[Tree_Sitter_Language_Query_Tags], + first_node + ); } - state.query.first_node = first_node; Range_i64 edit_range; - edit_range.min = (i64)ts_node_start_byte(state.query.first_node); - edit_range.max = (i64)ts_node_end_byte(state.query.first_node); - tree_data->last_update_node_range[0] = edit_range; // TODO(PS): TEMP - remove me once debugging is done - tree_data->last_update_node_range_count = 1; + edit_range.min = (i64)ts_node_start_byte(first_node); + edit_range.max = (i64)ts_node_end_byte(last_node); + tree_data->last_update_node_range = edit_range; // TODO(PS): TEMP - remove me once debugging is done // Free Scope Delimiters & Notes that fall within old_range Code_Index_Scope_Delim* delim = state.index->scope_delim_list.first; @@ -866,82 +963,86 @@ tree_sitter_code_index_update_tick(Application_Links* app) if (state.index->note_list.count == 0) Assert(state.index->note_list.first == 0 && state.index->note_list.first == 0); Code_Index_Scope_Delim* delim_prev = before_range; - TSQueryMatch query_match; - u32 capture_index; - while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index)) + for (int i = 0; i < query_count; i++) { - // TODO(PS): @Collapse - the entire body of this while loop used to be in - // tree_sitter_code_index_update_process_query_match - // After we're done, see what we can do to collapse - - TSQueryCapture type_capture = query_match.captures[capture_index]; - - TSNode type_node = type_capture.node; - Range_i64 type_range = tree_sitter_node_to_range(type_node); - - u32 length; - const char* tmp = ts_query_capture_name_for_id(state.query.query, type_capture.index, &length); - String_Const_u8 capture_name = SCu8((char*)tmp, length); - - if (string_match(capture_name, SCu8("scope_begin")) || string_match(capture_name, SCu8("scope_end"))) + Tree_Sitter_Query_Cursor query = queries[i]; + TSQueryMatch query_match; + u32 capture_index; + while (state.ok && tree_sitter_query_continue(&query, &query_match, &capture_index)) { - Code_Index_Scope_Delim_Kind kind; - String_Const_u8 delim_char = string_substring(state.buffer_contents, type_range); - bool skip = false; - if (delim_char.str[0] == '{') kind = CodeIndexScopeDelim_ScopeOpen; - else if (delim_char.str[0] == '}') kind = CodeIndexScopeDelim_ScopeClose; - else if (delim_char.str[0] == '(') kind = CodeIndexScopeDelim_ParenOpen; - else if (delim_char.str[0] == ')') kind = CodeIndexScopeDelim_ParenClose; - else if (delim_char.str[0] == '[') kind = CodeIndexScopeDelim_BracketOpen; - else if (delim_char.str[0] == ']') kind = CodeIndexScopeDelim_BracketClose; + // TODO(PS): @Collapse - the entire body of this while loop used to be in + // tree_sitter_code_index_update_process_query_match + // After we're done, see what we can do to collapse + + TSQueryCapture type_capture = query_match.captures[capture_index]; + + TSNode type_node = type_capture.node; + Range_i64 type_range = tree_sitter_node_to_range(type_node); + + u32 length; + const char* tmp = ts_query_capture_name_for_id(query.query, type_capture.index, &length); + String_Const_u8 capture_name = SCu8((char*)tmp, length); + + if (string_match(capture_name, SCu8("scope_begin")) || string_match(capture_name, SCu8("scope_end"))) + { + Code_Index_Scope_Delim_Kind kind; + String_Const_u8 delim_char = string_substring(state.buffer_contents, type_range); + bool skip = false; + if (delim_char.str[0] == '{') kind = CodeIndexScopeDelim_ScopeOpen; + else if (delim_char.str[0] == '}') kind = CodeIndexScopeDelim_ScopeClose; + else if (delim_char.str[0] == '(') kind = CodeIndexScopeDelim_ParenOpen; + else if (delim_char.str[0] == ')') kind = CodeIndexScopeDelim_ParenClose; + else if (delim_char.str[0] == '[') kind = CodeIndexScopeDelim_BracketOpen; + else if (delim_char.str[0] == ']') kind = CodeIndexScopeDelim_BracketClose; else skip = true; - if (!skip) + if (!skip) + { + Code_Index_Scope_Delim* delim = code_index_new_scope_delim( + state.index, &state.index_arena, kind, type_range + ); + code_index_scope_delim_insert(&state.index->scope_delim_list, delim_prev, delim); + delim_prev = delim; + } + } + else if (string_match(capture_name, SCu8("definition.class"))) { - Code_Index_Scope_Delim* delim = code_index_new_scope_delim( - state.index, &state.index_arena, kind, type_range - ); - code_index_scope_delim_insert(&state.index->scope_delim_list, delim_prev, delim); - delim_prev = delim; - } - } - else if (string_match(capture_name, SCu8("definition.class"))) - { - Code_Index_Note* note = code_index_new_note(state.index, &state.index_arena, CodeIndexNote_Type, type_range, state.nest_stack_last); - state.last_note_match_id = query_match.id; - code_index_note_insert(&state.index->note_list, state.last_note, note); - state.last_note = note; - } - else if (string_match(capture_name, SCu8("definition.function"))) - { - Code_Index_Note* note = code_index_new_note(state.index, &state.index_arena, CodeIndexNote_Function, type_range, state.nest_stack_last); - state.last_note_match_id = query_match.id; - code_index_note_insert(&state.index->note_list, state.last_note, note); - state.last_note = note; - } - else if (string_match(capture_name, SCu8("definition.method"))) - { - Code_Index_Note* note = code_index_new_note(state.index, &state.index_arena, CodeIndexNote_Function, type_range, state.nest_stack_last);; - state.last_note_match_id = query_match.id; - code_index_note_insert(&state.index->note_list, state.last_note, note); - state.last_note = note; - } - else if (string_match(capture_name, SCu8("definition.type"))) - { - Code_Index_Note* note = code_index_new_note(state.index, &state.index_arena, CodeIndexNote_Type, type_range, state.nest_stack_last); - state.last_note_match_id = query_match.id; - code_index_note_insert(&state.index->note_list, state.last_note, note); - state.last_note = note; - } - else if (string_match(capture_name, SCu8("name"))) - { - if (state.last_note != 0 && state.last_note_match_id == query_match.id) - { - state.last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start); + Code_Index_Note* note = code_index_new_note(state.index, &state.index_arena, CodeIndexNote_Type, type_range, state.nest_stack_last); + state.last_note_match_id = query_match.id; + code_index_note_insert(&state.index->note_list, state.last_note, note); + state.last_note = note; + } + else if (string_match(capture_name, SCu8("definition.function"))) + { + Code_Index_Note* note = code_index_new_note(state.index, &state.index_arena, CodeIndexNote_Function, type_range, state.nest_stack_last); + state.last_note_match_id = query_match.id; + code_index_note_insert(&state.index->note_list, state.last_note, note); + state.last_note = note; + } + else if (string_match(capture_name, SCu8("definition.method"))) + { + Code_Index_Note* note = code_index_new_note(state.index, &state.index_arena, CodeIndexNote_Function, type_range, state.nest_stack_last);; + state.last_note_match_id = query_match.id; + code_index_note_insert(&state.index->note_list, state.last_note, note); + state.last_note = note; + } + else if (string_match(capture_name, SCu8("definition.type"))) + { + Code_Index_Note* note = code_index_new_note(state.index, &state.index_arena, CodeIndexNote_Type, type_range, state.nest_stack_last); + state.last_note_match_id = query_match.id; + code_index_note_insert(&state.index->note_list, state.last_note, note); + state.last_note = note; + } + else if (string_match(capture_name, SCu8("name"))) + { + if (state.last_note != 0 && state.last_note_match_id == query_match.id) + { + state.last_note->pos = Ii64_size(type_range.start, type_range.end - type_range.start); + } } } + tree_sitter_query_end(&query); } - tree_sitter_code_index_update_complete(app, &state, scratch, true); }