From 43fb4a757a7cb9292d4a95d02be12173f190939d Mon Sep 17 00:00:00 2001 From: Peter Slattery Date: Thu, 10 Jul 2025 08:53:54 -0700 Subject: [PATCH] Implement Tree_Sitter_Language_Definition, handle registering languages by extension, and looking up the appropriate language definition for a buffer. custom_begin_buffer uses new functions to identify which files to treat as code implement custom_render_buffer which uses tree sitter data to color tokens --- code/custom/4coder_custom_hooks.cpp | 329 ++++++++++++++++-------- code/custom/4coder_default_bindings.cpp | 6 +- code/custom/4coder_tree_sitter.cpp | 289 +++++++++++++++++++-- code/custom/4coder_tree_sitter.h | 56 ++-- 4 files changed, 519 insertions(+), 161 deletions(-) diff --git a/code/custom/4coder_custom_hooks.cpp b/code/custom/4coder_custom_hooks.cpp index 86261643..9be81138 100644 --- a/code/custom/4coder_custom_hooks.cpp +++ b/code/custom/4coder_custom_hooks.cpp @@ -3,125 +3,19 @@ // Begin Buffer /////////////////////////////////////////////////////////////////////////// -struct File_Language_Result -{ - File_Language_Kind kind; - bool treat_as_code; -}; - -function File_Language_Result -identify_file_language(Application_Links* app, Buffer_ID buffer_id) -{ - Scratch_Block scratch(app); - - File_Language_Result result; - block_zero_struct(&result); - String_Const_u8 file_name = push_buffer_file_name(app, scratch, buffer_id); - if (file_name.size <= 0) return result; - String_Const_u8 file_extension = string_file_extension(file_name); - result.kind = File_Language_Text; - - String_Const_u8 treat_as_code_string = def_get_config_string(scratch, vars_save_string_lit("treat_as_code")); - String_Const_u8_Array extensions_to_treat_as_code = parse_extension_line_to_extension_list(app, scratch, treat_as_code_string); - - for (i32 i = 0; i < extensions_to_treat_as_code.count; ++i) - { - if (string_match(file_extension, extensions_to_treat_as_code.strings[i])) - { - result.treat_as_code = true; - result.kind = File_Language_Unknown; - - if (string_match(file_extension, string_u8_litexpr("md"))) - { - print_message(app, SCu8("Language Detected as Markdown\n")); - result.kind = File_Language_Markdown; - } - else if (string_match(file_extension, string_u8_litexpr("c"))) - { - print_message(app, SCu8("Language Detected as C\n")); - result.kind = File_Language_CPP; - } - else if (string_match(file_extension, string_u8_litexpr("cpp")) || - string_match(file_extension, string_u8_litexpr("h")) || - string_match(file_extension, string_u8_litexpr("hpp")) || - string_match(file_extension, string_u8_litexpr("cc")) - ){ - print_message(app, SCu8("Language Detected as Cpp\n")); - result.kind = File_Language_CPP; - } - else if (string_match(file_extension, string_u8_litexpr("m"))){ - print_message(app, SCu8("Language Detected as ObjectiveC\n")); - result.kind = File_Language_ObjectiveC; - } - else if (string_match(file_extension, string_u8_litexpr("hlsl"))){ - print_message(app, SCu8("Language Detected as HLSL\n")); - result.kind = File_Language_HLSL; - } - else if (string_match(file_extension, string_u8_litexpr("glsl"))){ - print_message(app, SCu8("Language Detected as GLSL\n")); - result.kind = File_Language_GLSL; - } - else if (string_match(file_extension, string_u8_litexpr("jai"))){ - print_message(app, SCu8("Language Detected as Jai\n")); - result.kind = File_Language_Jai; - } - else if (string_match(file_extension, string_u8_litexpr("cs"))){ - print_message(app, SCu8("Language Detected as C#\n")); - result.kind = File_Language_CSharp; - } - else if (string_match(file_extension, string_u8_litexpr("swift"))){ - print_message(app, SCu8("Language Detected as Swift\n")); - result.kind = File_Language_Swift; - } - else if (string_match(file_extension, string_u8_litexpr("go"))){ - print_message(app, SCu8("Language Detected as Go\n")); - result.kind = File_Language_Go; - } - else if (string_match(file_extension, string_u8_litexpr("rs"))){ - print_message(app, SCu8("Language Detected as Rust\n")); - result.kind = File_Language_Rust; - } - else if (string_match(file_extension, string_u8_litexpr("js"))){ - print_message(app, SCu8("Language Detected as Javascript\n")); - result.kind = File_Language_Javascript; - } - else if (string_match(file_extension, string_u8_litexpr("ts"))){ - print_message(app, SCu8("Language Detected as Typescript\n")); - result.kind = File_Language_Typescript; - } - else if (string_match(file_extension, string_u8_litexpr("json"))){ - print_message(app, SCu8("Language Detected as JSON\n")); - result.kind = File_Language_JSON; - } - else if (string_match(file_extension, string_u8_litexpr("odin"))){ - print_message(app, SCu8("Language Detected as Odin\n")); - result.kind = File_Language_Odin; - } - else if (string_match(file_extension, string_u8_litexpr("zig"))){ - print_message(app, SCu8("Language Detected as Zig\n")); - result.kind = File_Language_Zig; - } - - if (result.kind != File_Language_Unknown) break; - } - } - - return result; -} - BUFFER_HOOK_SIG(custom_begin_buffer){ ProfileScope(app, "begin buffer"); Scratch_Block scratch(app); - File_Language_Result lang = identify_file_language(app, buffer_id); - bool begin_parse_task = false; - if (lang.treat_as_code) begin_parse_task = tree_sitter_begin_buffer(app, buffer_id, lang.kind); + Tree_Sitter_Language_Definition* language = tree_sitter_language_for_buffer(app, buffer_id); + bool treat_as_code = language != 0; + if (treat_as_code) tree_sitter_begin_buffer(app, buffer_id); String_ID file_map_id = vars_save_string_lit("keys_file"); String_ID code_map_id = vars_save_string_lit("keys_code"); - Command_Map_ID map_id = (lang.treat_as_code)?(code_map_id):(file_map_id); + Command_Map_ID map_id = (treat_as_code)?(code_map_id):(file_map_id); Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); Command_Map_ID *map_id_ptr = scope_attachment(app, scope, buffer_map_id, Command_Map_ID); *map_id_ptr = map_id; @@ -133,13 +27,13 @@ BUFFER_HOOK_SIG(custom_begin_buffer){ // NOTE(allen): Decide buffer settings b32 wrap_lines = true; b32 use_lexer = false; - if (lang.treat_as_code){ + if (treat_as_code){ wrap_lines = def_get_config_b32(vars_save_string_lit("enable_code_wrapping")); // TODO(PS): @Remove - consider removing the lexer for now? later, replace in favor of tree-sitter use_lexer = true; } - if (begin_parse_task) + if (treat_as_code) { Async_Task* parse_task = scope_attachment(app, scope, buffer_tree_sitter_parse_task_id, Async_Task); *parse_task = async_task_no_dep(&global_async_system, tree_sitter_parse_async, make_data_struct(&buffer_id)); @@ -165,7 +59,7 @@ BUFFER_HOOK_SIG(custom_begin_buffer){ buffer_set_layout(app, buffer_id, layout_virt_indent_index_generic); } else{ - if (lang.treat_as_code){ + if (treat_as_code){ buffer_set_layout(app, buffer_id, layout_virt_indent_literal_generic); } else{ @@ -181,7 +75,6 @@ BUFFER_HOOK_SIG(custom_begin_buffer){ // End Buffer /////////////////////////////////////////////////////////////////////////// - BUFFER_HOOK_SIG(custom_end_buffer){ Marker_List *list = get_marker_list_for_buffer(buffer_id); if (list != 0) delete_marker_list(list); @@ -190,3 +83,211 @@ BUFFER_HOOK_SIG(custom_end_buffer){ default_end_buffer(app, buffer_id); return(0); } + +/////////////////////////////////////////////////////////////////////////// +// Render Buffer +/////////////////////////////////////////////////////////////////////////// + +function void custom_render_buffer( + Application_Links *app, + View_ID view_id, + Face_ID face_id, + Buffer_ID buffer, + Text_Layout_ID text_layout_id, + Rect_f32 rect +){ + ProfileScope(app, "render buffer"); + Scratch_Block scratch(app); + + View_ID active_view = get_active_view(app, Access_Always); + b32 is_active_view = (active_view == view_id); + Rect_f32 prev_clip = draw_set_clip(app, rect); + + Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id); + + // NOTE(allen): Cursor shape + Face_Metrics metrics = get_face_metrics(app, face_id); + u64 cursor_roundness_100 = def_get_config_u64(app, vars_save_string_lit("cursor_roundness")); + f32 cursor_roundness = metrics.normal_advance*cursor_roundness_100*0.01f; + f32 mark_thickness = (f32)def_get_config_u64(app, vars_save_string_lit("mark_thickness")); + + // NOTE(allen): Token colorizing + Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer); + Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); + TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data); + + Token_Array token_array = get_token_array_from_buffer(app, buffer); + paint_text_color_fcolor(app, text_layout_id, visible_range, fcolor_id(defcolor_text_default)); // will get overridden by lang-specific token coloring below + if (token_array.tokens != 0 && tree) + { + if (use_tree_sitter_token_coloring) + { + draw_tree_sitter_node_colors(app, text_layout_id, buffer); + } + else + { + draw_cpp_token_colors(app, text_layout_id, &token_array); + } + } + + i64 cursor_pos = view_correct_cursor(app, view_id); + view_correct_mark(app, view_id); + + // NOTE(allen): Scope highlight + b32 use_scope_highlight = def_get_config_b32(vars_save_string_lit("use_scope_highlight")); + if (use_scope_highlight){ + Color_Array colors = finalize_color_array(defcolor_back_cycle); + draw_scope_highlight(app, buffer, text_layout_id, cursor_pos, colors.vals, colors.count); + } + + // NOTE(PS): QOL Column + if (qol_col_cursor.pos >= 0){ + Buffer_Seek seek = seek_line_col(qol_col_cursor.line, qol_col_cursor.col); + Buffer_Cursor cursor = buffer_compute_cursor(app, buffer, seek); + Rect_f32 col_rect = text_layout_character_on_screen(app, text_layout_id, cursor.pos); + if (col_rect.x1 > 0.f){ + col_rect.y0 = rect.y0; + col_rect.y1 = rect.y1; + draw_rectangle_fcolor(app, col_rect, 0.f, fcolor_id(defcolor_highlight_cursor_line)); + } + } + + b32 use_error_highlight = def_get_config_b32(vars_save_string_lit("use_error_highlight")); + b32 use_jump_highlight = def_get_config_b32(vars_save_string_lit("use_jump_highlight")); + if (use_error_highlight || use_jump_highlight){ + // NOTE(allen): Error highlight + String_Const_u8 name = string_u8_litexpr("*compilation*"); + Buffer_ID compilation_buffer = get_buffer_by_name(app, name, Access_Always); + if (use_error_highlight){ + draw_jump_highlights(app, buffer, text_layout_id, compilation_buffer, + fcolor_id(defcolor_highlight_junk)); + } + + // NOTE(allen): Search highlight + if (use_jump_highlight){ + Buffer_ID jump_buffer = get_locked_jump_buffer(app); + if (jump_buffer != compilation_buffer){ + draw_jump_highlights(app, buffer, text_layout_id, jump_buffer, + fcolor_id(defcolor_highlight_white)); + } + } + } + + // NOTE(allen): Color parens + b32 use_paren_helper = def_get_config_b32(vars_save_string_lit("use_paren_helper")); + if (use_paren_helper){ + Color_Array colors = finalize_color_array(defcolor_text_cycle); + draw_paren_highlight(app, buffer, text_layout_id, cursor_pos, colors.vals, colors.count); + } + + // NOTE(allen): Line highlight + b32 highlight_line_at_cursor = def_get_config_b32(vars_save_string_lit("highlight_line_at_cursor")); + if (highlight_line_at_cursor && is_active_view){ + i64 line_number = get_line_number_from_pos(app, buffer, cursor_pos); + draw_line_highlight(app, text_layout_id, line_number, fcolor_id(defcolor_highlight_cursor_line)); + } + + // NOTE(allen): Whitespace highlight + b64 show_whitespace = false; + view_get_setting(app, view_id, ViewSetting_ShowWhitespace, &show_whitespace); + if (show_whitespace){ + if (token_array.tokens == 0){ + draw_whitespace_highlight(app, buffer, text_layout_id, cursor_roundness); + } + else{ + draw_whitespace_highlight(app, text_layout_id, &token_array, cursor_roundness); + } + } + + // NOTE(allen): Cursor + switch (fcoder_mode){ + case FCoderMode_Original: + { + draw_original_4coder_style_cursor_mark_highlight(app, view_id, is_active_view, buffer, text_layout_id, cursor_roundness, mark_thickness); + }break; + case FCoderMode_NotepadLike: + { + draw_notepad_style_cursor_highlight(app, view_id, buffer, text_layout_id, cursor_roundness); + }break; + } + + // NOTE(allen): Fade ranges + paint_fade_ranges(app, text_layout_id, buffer); + + // NOTE(allen): put the actual text on the actual screen + draw_text_layout_default(app, text_layout_id); + + draw_set_clip(app, prev_clip); +} + +function void +custom_render_caller(Application_Links *app, Frame_Info frame_info, View_ID view_id){ + ProfileScope(app, "default render caller"); + View_ID active_view = get_active_view(app, Access_Always); + b32 is_active_view = (active_view == view_id); + + Rect_f32 region = draw_background_and_margin(app, view_id, is_active_view); + Rect_f32 prev_clip = draw_set_clip(app, region); + + Buffer_ID buffer = view_get_buffer(app, view_id, Access_Always); + Face_ID face_id = get_face_id(app, buffer); + Face_Metrics face_metrics = get_face_metrics(app, face_id); + f32 line_height = face_metrics.line_height; + f32 digit_advance = face_metrics.decimal_digit_advance; + + // NOTE(allen): file bar + b64 showing_file_bar = false; + if (view_get_setting(app, view_id, ViewSetting_ShowFileBar, &showing_file_bar) && showing_file_bar){ + Rect_f32_Pair pair = layout_file_bar_on_top(region, line_height); + draw_file_bar(app, view_id, buffer, face_id, pair.min); + region = pair.max; + } + + Buffer_Scroll scroll = view_get_buffer_scroll(app, view_id); + + Buffer_Point_Delta_Result delta = delta_apply(app, view_id, + frame_info.animation_dt, scroll); + if (!block_match_struct(&scroll.position, &delta.point)){ + block_copy_struct(&scroll.position, &delta.point); + view_set_buffer_scroll(app, view_id, scroll, SetBufferScroll_NoCursorChange); + } + if (delta.still_animating){ + animate_in_n_milliseconds(app, 0); + } + + // NOTE(allen): query bars + region = default_draw_query_bars(app, region, view_id, face_id); + + // NOTE(allen): FPS hud + if (show_fps_hud){ + Rect_f32_Pair pair = layout_fps_hud_on_bottom(region, line_height); + draw_fps_hud(app, frame_info, face_id, pair.max); + region = pair.min; + animate_in_n_milliseconds(app, 1000); + } + + // NOTE(allen): layout line numbers + b32 show_line_number_margins = def_get_config_b32(vars_save_string_lit("show_line_number_margins")); + Rect_f32 line_number_rect = {}; + if (show_line_number_margins){ + Rect_f32_Pair pair = layout_line_number_margin(app, buffer, region, digit_advance); + line_number_rect = pair.min; + region = pair.max; + } + + // NOTE(allen): begin buffer render + Buffer_Point buffer_point = scroll.position; + Text_Layout_ID text_layout_id = text_layout_create(app, buffer, region, buffer_point); + + // NOTE(allen): draw line numbers + if (show_line_number_margins){ + draw_line_number_margin(app, view_id, buffer, face_id, text_layout_id, line_number_rect); + } + + // NOTE(allen): draw the buffer + custom_render_buffer(app, view_id, face_id, buffer, text_layout_id, region); + loco_render_buffer(app, view_id, face_id, buffer, text_layout_id, region, frame_info); + + text_layout_free(app, text_layout_id); + draw_set_clip(app, prev_clip); +} diff --git a/code/custom/4coder_default_bindings.cpp b/code/custom/4coder_default_bindings.cpp index ce71a92d..c0c9bf03 100644 --- a/code/custom/4coder_default_bindings.cpp +++ b/code/custom/4coder_default_bindings.cpp @@ -527,12 +527,12 @@ custom_layer_init(Application_Links *app){ set_all_default_hooks(app); modal_init(3, tctx); - set_custom_hook(app, HookID_BeginBuffer, custom_begin_buffer); - set_custom_hook(app, HookID_EndBuffer, custom_end_buffer); + set_custom_hook(app, HookID_BeginBuffer, custom_begin_buffer); + set_custom_hook(app, HookID_EndBuffer, custom_end_buffer); + set_custom_hook(app, HookID_RenderCaller, custom_render_caller); custom_keyboard_bindings(); - #if 0 mapping_init(tctx, &framework_mapping); String_ID global_map_id = vars_save_string_lit("keys_global"); diff --git a/code/custom/4coder_tree_sitter.cpp b/code/custom/4coder_tree_sitter.cpp index ce51eee1..66c601f8 100644 --- a/code/custom/4coder_tree_sitter.cpp +++ b/code/custom/4coder_tree_sitter.cpp @@ -1,4 +1,108 @@ -function bool + +///////////////////////////////////////////// +// TEMP until I implement more generic language stuff +///////////////////////////////////////////// + +TSQuery* tree_sitter_cpp_index_query; +String_Const_u8 TS_CPP_INDEX_QUERY = string_u8_litexpr("(_ \"{\" @Start \"}\" @End ) @ScopeNest\n"); + +String_Const_u8 TS_CPP_HIGHLIGHT_QUERY = string_u8_litexpr("(call_expression function: [" + " (identifier) @defcolor_function" + " (field_expression field: (field_identifier) @defcolor_function)])" + "(function_declarator" + " declarator: [(identifier) (field_identifier)] @defcolor_function)" + + "(preproc_def" + " name: (identifier) @defcolor_macro)" + "(preproc_function_def" + " name: (identifier) @defcolor_macro)" + + "(type_identifier) @defcolor_type" + "(call_expression" + " function: (parenthesized_expression" + " (identifier) @defcolor_type))" + + "[(primitive_type) (type_qualifier) (storage_class_specifier)" + " (break_statement) (continue_statement) \"union\" \"return\" \"do\"" + " \"while\" \"for\" \"if\" \"class\" \"struct\" \"enum\" \"sizeof\"" + " \"else\" \"switch\" \"case\"] @defcolor_keyword" + + "[(number_literal) (string_literal)] @defcolor_str_constant" + "[(preproc_directive) \"#define\" \"#if\" \"#elif\" \"#else\" \"#endif\"" + " \"#include\"] @defcolor_preproc" + "[\"{\" \"}\" \";\" \":\" \",\"] @defcolor_text_default" + "(comment) @defcolor_comment"); + +//////////////////////////////////////////////////////////////////// +// Language Management +//////////////////////////////////////////////////////////////////// + +function void +tree_sitter_register_language(String_Const_u8 ext, TSLanguage* language, TSQuery* highlight_query) +{ + Tree_Sitter_Language_Definition* lang = 0; + u64 hash = table_hash_u8(ext.str, ext.size); + u64 slot = hash % ArrayCount(tree_sitter_languages.languages); + for (Tree_Sitter_Language_Definition* l = tree_sitter_languages.languages[slot]; l != 0; l = l->next) + { + if (l->extension_hash == hash && string_match(l->extension, ext)) + { + lang = l; break; + } + } + + if (lang == 0) + { + lang = push_array(&tree_sitter_languages.arena, Tree_Sitter_Language_Definition, 1); + lang->next = tree_sitter_languages.languages[slot]; + tree_sitter_languages.languages[slot] = lang; + + lang->extension_hash = hash; + lang->extension = push_string_copy(&tree_sitter_languages.arena, ext); + lang->language = language; + lang->highlight_query = highlight_query; + } +} + +function Tree_Sitter_Language_Definition* +tree_sitter_language_from_file_extension(String_Const_u8 ext) +{ + Tree_Sitter_Language_Definition* result = 0; + u64 ext_hash = table_hash_u8(ext.str, ext.size); + u64 slot = ext_hash % ArrayCount(tree_sitter_languages.languages); + for (Tree_Sitter_Language_Definition* l = tree_sitter_languages.languages[slot]; l != 0; l = l->next) + { + if (l->extension_hash == ext_hash && string_match(l->extension, ext)) + { + result = l; + break; + } + } + return result; +} + +function Tree_Sitter_Language_Definition* +tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id, Arena* arena) +{ + Tree_Sitter_Language_Definition* result = 0; + String_Const_u8 file_name = push_buffer_file_name(app, arena, buffer_id); + String_Const_u8 extension = string_file_extension(file_name); + result = tree_sitter_language_from_file_extension(extension); + return result; +} + +function Tree_Sitter_Language_Definition* +tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id) +{ + Scratch_Block scratch(app); + return tree_sitter_language_for_buffer(app, buffer_id, scratch); +} + +///////////////////////////////////////////// +// Tree Sitter Hook Internals +///////////////////////////////////////////// + +function void tree_sitter_init(Application_Links* app) { Buffer_ID buffer = create_buffer( @@ -8,30 +112,34 @@ tree_sitter_init(Application_Links* app) ); buffer_set_setting(app, buffer, BufferSetting_Unimportant, true); buffer_set_setting(app, buffer, BufferSetting_ReadOnly, true); + + tree_sitter_languages.arena = make_arena_system(KB(16)); + + u32 error_offset; + TSQueryError query_error; + + { // Register CPP + TSLanguage* language = tree_sitter_cpp(); + String_Const_u8 highlight_query_str = TS_CPP_HIGHLIGHT_QUERY; + TSQuery* highlight_query = ts_query_new( + language, + (const char *)TS_CPP_HIGHLIGHT_QUERY.str, + (u32)TS_CPP_HIGHLIGHT_QUERY.size, + &error_offset, &query_error + ); + tree_sitter_register_language(SCu8("cpp"), language, highlight_query); + tree_sitter_register_language(SCu8("h"), language, highlight_query); + tree_sitter_register_language(SCu8("hpp"), language, highlight_query); + tree_sitter_register_language(SCu8("cc"), language, highlight_query); + } } -function bool -tree_sitter_begin_buffer(Application_Links* app, Buffer_ID buffer_id, File_Language_Kind kind) +function void +tree_sitter_begin_buffer(Application_Links* app, Buffer_ID buffer_id) { Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); - switch (kind) - { - case File_Language_CPP: - { - tree_data->language = tree_sitter_cpp(); - } break; - - default: - tree_data->language = 0; - } - - if (tree_data->language != 0) - { - tree_data->tree_mutex = system_mutex_make(); - } - - return tree_data->language != 0; + tree_data->tree_mutex = system_mutex_make(); } function void @@ -39,7 +147,7 @@ tree_sitter_end_buffer(Application_Links* app, Buffer_ID buffer_id) { Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); - if (!tree_data || !tree_data->language) return; + if (!tree_data) return; Async_Task *tree_sitter_parse_task = scope_attachment(app, buffer_scope, buffer_tree_sitter_parse_task_id, Async_Task); if (async_task_is_running_or_pending(&global_async_system, *tree_sitter_parse_task)) @@ -74,11 +182,12 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id) ts_parser_set_timeout_micros(parser, 5000); acquire_global_frame_mutex(app); + Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id); String_Const_u8 src = push_whole_buffer(app, &arena, buffer_id); Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); TSTree *old_tree = tree_sitter_buffer_get_tree_copy(tree_data); - bool lang_set = ts_parser_set_language(parser, tree_data->language); + bool lang_set = ts_parser_set_language(parser, lang->language); release_global_frame_mutex(app); if (!lang_set) @@ -144,10 +253,146 @@ tree_sitter_parse_async(Async_Context* actx, String_Const_u8 data) tree_sitter_parse_async__inner(actx, buffer_id); } +function Range_i64 +tree_sitter_node_to_range(TSNode node) +{ + Range_i64 result; + result.start = ts_node_start_byte(node); + result.end = ts_node_end_byte(node); + return result; +} + function void tree_sitter_code_index_update_tick(Application_Links* app) { + Scratch_Block scratch(app); +#if 0 + // TODO(PS): this should be done when we register the language + if (!tree_sitter_cpp_index_query) + { + u32 error_offset; + TSQueryError query_error; + tree_sitter_cpp_index_query = ts_query_new( + tree_sitter_cpp(), TS_CPP_INDEX_QUERY, (u32)TS_CPP_INDEX_QUERY.size, &error_offset, &query_error + ); + if (!tree_sitter_cpp_index_query) + { + print_message(app, string_u8_litexpr("Failed to create cpp index query\n"); + } + } + + for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first; + modified_node != 0; + modified_node = modified_node->next + ){ + Temp_Memory_Block temp(scratch); + Buffer_ID buffer_id = modified_node->buffer; + + Arena arena = make_arena_system(KB(16)); + Code_Index_File* index = push_array_zero(&arena, Code_Index_File, 1); + index->buffer = buffer_id; + + Tree_Sitter_Code_Index_Nest_List nests = {}; + + Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id); + Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); + TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data); + if (tree) + { + TSQueryCursor* query_cursor = ts_query_cursor_new(); + ts_query_cursor_exec(query_cursor, tree_sitter_cpp_index_query, ts_tree_root_node(tree)); + + TSQueryMatch query_match; + while (ts_query_cursor_next_match(query_cursor, &match)) + { + TSQueryCapture type_capture = match.captures[0]; + TSNode type_node = type_capture.node; + Range_i64 type_range = tree_sitter_node_to_range(type_node); + + // TODO(PS): PICK UP HERE + } + } + ts_tree_delete(tree); + } +#endif +} + +//////////////////////////////////////////////////////////////////// +// Token Highlighting +//////////////////////////////////////////////////////////////////// + +function void +tree_sitter_highlight_node( + Application_Links* app, + TSQuery* query, + TSNode top_node, + TSQueryCursor* query_cursor, + Text_Layout_ID text_layout_id +){ + ts_query_cursor_exec(query_cursor, query, top_node); + + TSQueryMatch query_match; + u32 capture_index; + while (ts_query_cursor_next_capture(query_cursor, &query_match, &capture_index)) + { + TSQueryCapture capture = query_match.captures[capture_index]; + TSNode node = capture.node; + + u32 length; + const char* tmp = ts_query_capture_name_for_id(query, capture.index, &length); + String_Const_u8 capture_name = SCu8((char*)tmp, length); + + Range_i64 highlight_range = tree_sitter_node_to_range(node); + Managed_ID color_id = managed_id_get(app, SCu8("colors"), capture_name); + if (color_id != 0) + { + paint_text_color_fcolor(app, text_layout_id, highlight_range, fcolor_id(color_id)); + } + } +} + +function void +draw_tree_sitter_node_colors(Application_Links* app, Text_Layout_ID text_layout_id, Buffer_ID buffer_id) +{ + Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id); + TSQuery* query = lang->highlight_query; + + Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id); + Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id); + Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); + TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data); + if (tree) + { + TSNode root = ts_tree_root_node(tree); + // Get the smallest node that fully contains the visible range + TSNode visible_container = ts_node_descendant_for_byte_range(root, (u32)visible_range.start, (u32)visible_range.end); + TSQueryCursor* query_cursor = ts_query_cursor_new(); + if (ts_node_eq(root, visible_container)) + { + TSTreeCursor tree_cursor = ts_tree_cursor_new(visible_container); + if (ts_tree_cursor_goto_first_child_for_byte(&tree_cursor, (u32)visible_range.start) != -1) + { + do { + TSNode node = ts_tree_cursor_current_node(&tree_cursor); + Range_i64 child_range = tree_sitter_node_to_range(node); + if (child_range.start > visible_range.end) break; + tree_sitter_highlight_node(app, query, node, query_cursor, text_layout_id); + } while(ts_tree_cursor_goto_next_sibling(&tree_cursor)); + } + else + { + // Pathological case - just highligh the whole document. This is probably bad + tree_sitter_highlight_node(app, query, root, query_cursor, text_layout_id); + } + } + else + { + tree_sitter_highlight_node(app, query, visible_container, query_cursor, text_layout_id); + } + ts_query_cursor_delete(query_cursor); + } + ts_tree_delete(tree); } //////////////////////////////////////////////////////////////////// diff --git a/code/custom/4coder_tree_sitter.h b/code/custom/4coder_tree_sitter.h index 145cc9d4..7e9bc5d4 100644 --- a/code/custom/4coder_tree_sitter.h +++ b/code/custom/4coder_tree_sitter.h @@ -5,29 +5,25 @@ #include -enum File_Language_Kind +struct Tree_Sitter_Language_Definition { - File_Language_None, - File_Language_Unknown, - File_Language_Text, - File_Language_Markdown, - File_Language_C, - File_Language_CPP, - File_Language_ObjectiveC, - File_Language_HLSL, - File_Language_GLSL, - File_Language_Jai, - File_Language_CSharp, - File_Language_Swift, - File_Language_Go, - File_Language_Rust, - File_Language_Javascript, - File_Language_Typescript, - File_Language_JSON, - File_Language_Odin, - File_Language_Zig, + String_Const_u8 extension; + u64 extension_hash; + + TSLanguage* language; + TSQuery* highlight_query; + + Tree_Sitter_Language_Definition* next; }; +struct Tree_Sitter_Languages +{ + Arena arena; + Tree_Sitter_Language_Definition* languages[4096]; +}; + +global Tree_Sitter_Languages tree_sitter_languages; + extern "C" { TSLanguage *tree_sitter_cpp(); TSLanguage *tree_sitter_c(); @@ -38,13 +34,29 @@ CUSTOM_ID(attachment, buffer_tree_sitter_parse_task_id); struct Buffer_Tree_Sitter_Data { - TSLanguage* language; TSTree* tree; - System_Mutex tree_mutex; }; +struct Tree_Sitter_Code_Index_Nest_Node +{ + Tree_Sitter_Code_Index_Nest_Node* next; + Tree_Sitter_Code_Index_Nest_Node* prev; + Tree_Sitter_Code_Index_Nest_Node* parent; + Tree_Sitter_Code_Index_Nest_Node* child_first; + Tree_Sitter_Code_Index_Nest_Node* child_last; + + Code_Index_Nest* nest; +}; + +struct Tree_Sitter_Code_Index_Nest_List +{ + Tree_Sitter_Code_Index_Nest_Node* first; + Tree_Sitter_Code_Index_Nest_Node* last; +}; + b8 use_tree_sitter_code_indexing = true; +b8 use_tree_sitter_token_coloring = true; function void tree_sitter_code_index_update_tick(Application_Links *app); #endif //FCODER_TREE_SITTER_H