Implement Tree_Sitter_Language_Definition, handle registering languages by extension, and looking up the appropriate language definition for a buffer.

custom_begin_buffer uses new functions to identify which files to treat as code
implement custom_render_buffer which uses tree sitter data to color tokens
This commit is contained in:
Peter Slattery 2025-07-10 08:53:54 -07:00
parent 7caaed736b
commit 43fb4a757a
4 changed files with 519 additions and 161 deletions

View File

@ -3,125 +3,19 @@
// Begin Buffer // Begin Buffer
/////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////
struct File_Language_Result
{
File_Language_Kind kind;
bool treat_as_code;
};
function File_Language_Result
identify_file_language(Application_Links* app, Buffer_ID buffer_id)
{
Scratch_Block scratch(app);
File_Language_Result result;
block_zero_struct(&result);
String_Const_u8 file_name = push_buffer_file_name(app, scratch, buffer_id);
if (file_name.size <= 0) return result;
String_Const_u8 file_extension = string_file_extension(file_name);
result.kind = File_Language_Text;
String_Const_u8 treat_as_code_string = def_get_config_string(scratch, vars_save_string_lit("treat_as_code"));
String_Const_u8_Array extensions_to_treat_as_code = parse_extension_line_to_extension_list(app, scratch, treat_as_code_string);
for (i32 i = 0; i < extensions_to_treat_as_code.count; ++i)
{
if (string_match(file_extension, extensions_to_treat_as_code.strings[i]))
{
result.treat_as_code = true;
result.kind = File_Language_Unknown;
if (string_match(file_extension, string_u8_litexpr("md")))
{
print_message(app, SCu8("Language Detected as Markdown\n"));
result.kind = File_Language_Markdown;
}
else if (string_match(file_extension, string_u8_litexpr("c")))
{
print_message(app, SCu8("Language Detected as C\n"));
result.kind = File_Language_CPP;
}
else if (string_match(file_extension, string_u8_litexpr("cpp")) ||
string_match(file_extension, string_u8_litexpr("h")) ||
string_match(file_extension, string_u8_litexpr("hpp")) ||
string_match(file_extension, string_u8_litexpr("cc"))
){
print_message(app, SCu8("Language Detected as Cpp\n"));
result.kind = File_Language_CPP;
}
else if (string_match(file_extension, string_u8_litexpr("m"))){
print_message(app, SCu8("Language Detected as ObjectiveC\n"));
result.kind = File_Language_ObjectiveC;
}
else if (string_match(file_extension, string_u8_litexpr("hlsl"))){
print_message(app, SCu8("Language Detected as HLSL\n"));
result.kind = File_Language_HLSL;
}
else if (string_match(file_extension, string_u8_litexpr("glsl"))){
print_message(app, SCu8("Language Detected as GLSL\n"));
result.kind = File_Language_GLSL;
}
else if (string_match(file_extension, string_u8_litexpr("jai"))){
print_message(app, SCu8("Language Detected as Jai\n"));
result.kind = File_Language_Jai;
}
else if (string_match(file_extension, string_u8_litexpr("cs"))){
print_message(app, SCu8("Language Detected as C#\n"));
result.kind = File_Language_CSharp;
}
else if (string_match(file_extension, string_u8_litexpr("swift"))){
print_message(app, SCu8("Language Detected as Swift\n"));
result.kind = File_Language_Swift;
}
else if (string_match(file_extension, string_u8_litexpr("go"))){
print_message(app, SCu8("Language Detected as Go\n"));
result.kind = File_Language_Go;
}
else if (string_match(file_extension, string_u8_litexpr("rs"))){
print_message(app, SCu8("Language Detected as Rust\n"));
result.kind = File_Language_Rust;
}
else if (string_match(file_extension, string_u8_litexpr("js"))){
print_message(app, SCu8("Language Detected as Javascript\n"));
result.kind = File_Language_Javascript;
}
else if (string_match(file_extension, string_u8_litexpr("ts"))){
print_message(app, SCu8("Language Detected as Typescript\n"));
result.kind = File_Language_Typescript;
}
else if (string_match(file_extension, string_u8_litexpr("json"))){
print_message(app, SCu8("Language Detected as JSON\n"));
result.kind = File_Language_JSON;
}
else if (string_match(file_extension, string_u8_litexpr("odin"))){
print_message(app, SCu8("Language Detected as Odin\n"));
result.kind = File_Language_Odin;
}
else if (string_match(file_extension, string_u8_litexpr("zig"))){
print_message(app, SCu8("Language Detected as Zig\n"));
result.kind = File_Language_Zig;
}
if (result.kind != File_Language_Unknown) break;
}
}
return result;
}
BUFFER_HOOK_SIG(custom_begin_buffer){ BUFFER_HOOK_SIG(custom_begin_buffer){
ProfileScope(app, "begin buffer"); ProfileScope(app, "begin buffer");
Scratch_Block scratch(app); Scratch_Block scratch(app);
File_Language_Result lang = identify_file_language(app, buffer_id); Tree_Sitter_Language_Definition* language = tree_sitter_language_for_buffer(app, buffer_id);
bool begin_parse_task = false; bool treat_as_code = language != 0;
if (lang.treat_as_code) begin_parse_task = tree_sitter_begin_buffer(app, buffer_id, lang.kind); if (treat_as_code) tree_sitter_begin_buffer(app, buffer_id);
String_ID file_map_id = vars_save_string_lit("keys_file"); String_ID file_map_id = vars_save_string_lit("keys_file");
String_ID code_map_id = vars_save_string_lit("keys_code"); String_ID code_map_id = vars_save_string_lit("keys_code");
Command_Map_ID map_id = (lang.treat_as_code)?(code_map_id):(file_map_id); Command_Map_ID map_id = (treat_as_code)?(code_map_id):(file_map_id);
Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
Command_Map_ID *map_id_ptr = scope_attachment(app, scope, buffer_map_id, Command_Map_ID); Command_Map_ID *map_id_ptr = scope_attachment(app, scope, buffer_map_id, Command_Map_ID);
*map_id_ptr = map_id; *map_id_ptr = map_id;
@ -133,13 +27,13 @@ BUFFER_HOOK_SIG(custom_begin_buffer){
// NOTE(allen): Decide buffer settings // NOTE(allen): Decide buffer settings
b32 wrap_lines = true; b32 wrap_lines = true;
b32 use_lexer = false; b32 use_lexer = false;
if (lang.treat_as_code){ if (treat_as_code){
wrap_lines = def_get_config_b32(vars_save_string_lit("enable_code_wrapping")); wrap_lines = def_get_config_b32(vars_save_string_lit("enable_code_wrapping"));
// TODO(PS): @Remove - consider removing the lexer for now? later, replace in favor of tree-sitter // TODO(PS): @Remove - consider removing the lexer for now? later, replace in favor of tree-sitter
use_lexer = true; use_lexer = true;
} }
if (begin_parse_task) if (treat_as_code)
{ {
Async_Task* parse_task = scope_attachment(app, scope, buffer_tree_sitter_parse_task_id, Async_Task); Async_Task* parse_task = scope_attachment(app, scope, buffer_tree_sitter_parse_task_id, Async_Task);
*parse_task = async_task_no_dep(&global_async_system, tree_sitter_parse_async, make_data_struct(&buffer_id)); *parse_task = async_task_no_dep(&global_async_system, tree_sitter_parse_async, make_data_struct(&buffer_id));
@ -165,7 +59,7 @@ BUFFER_HOOK_SIG(custom_begin_buffer){
buffer_set_layout(app, buffer_id, layout_virt_indent_index_generic); buffer_set_layout(app, buffer_id, layout_virt_indent_index_generic);
} }
else{ else{
if (lang.treat_as_code){ if (treat_as_code){
buffer_set_layout(app, buffer_id, layout_virt_indent_literal_generic); buffer_set_layout(app, buffer_id, layout_virt_indent_literal_generic);
} }
else{ else{
@ -181,7 +75,6 @@ BUFFER_HOOK_SIG(custom_begin_buffer){
// End Buffer // End Buffer
/////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////
BUFFER_HOOK_SIG(custom_end_buffer){ BUFFER_HOOK_SIG(custom_end_buffer){
Marker_List *list = get_marker_list_for_buffer(buffer_id); Marker_List *list = get_marker_list_for_buffer(buffer_id);
if (list != 0) delete_marker_list(list); if (list != 0) delete_marker_list(list);
@ -190,3 +83,211 @@ BUFFER_HOOK_SIG(custom_end_buffer){
default_end_buffer(app, buffer_id); default_end_buffer(app, buffer_id);
return(0); return(0);
} }
///////////////////////////////////////////////////////////////////////////
// Render Buffer
///////////////////////////////////////////////////////////////////////////
function void custom_render_buffer(
Application_Links *app,
View_ID view_id,
Face_ID face_id,
Buffer_ID buffer,
Text_Layout_ID text_layout_id,
Rect_f32 rect
){
ProfileScope(app, "render buffer");
Scratch_Block scratch(app);
View_ID active_view = get_active_view(app, Access_Always);
b32 is_active_view = (active_view == view_id);
Rect_f32 prev_clip = draw_set_clip(app, rect);
Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id);
// NOTE(allen): Cursor shape
Face_Metrics metrics = get_face_metrics(app, face_id);
u64 cursor_roundness_100 = def_get_config_u64(app, vars_save_string_lit("cursor_roundness"));
f32 cursor_roundness = metrics.normal_advance*cursor_roundness_100*0.01f;
f32 mark_thickness = (f32)def_get_config_u64(app, vars_save_string_lit("mark_thickness"));
// NOTE(allen): Token colorizing
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data);
Token_Array token_array = get_token_array_from_buffer(app, buffer);
paint_text_color_fcolor(app, text_layout_id, visible_range, fcolor_id(defcolor_text_default)); // will get overridden by lang-specific token coloring below
if (token_array.tokens != 0 && tree)
{
if (use_tree_sitter_token_coloring)
{
draw_tree_sitter_node_colors(app, text_layout_id, buffer);
}
else
{
draw_cpp_token_colors(app, text_layout_id, &token_array);
}
}
i64 cursor_pos = view_correct_cursor(app, view_id);
view_correct_mark(app, view_id);
// NOTE(allen): Scope highlight
b32 use_scope_highlight = def_get_config_b32(vars_save_string_lit("use_scope_highlight"));
if (use_scope_highlight){
Color_Array colors = finalize_color_array(defcolor_back_cycle);
draw_scope_highlight(app, buffer, text_layout_id, cursor_pos, colors.vals, colors.count);
}
// NOTE(PS): QOL Column
if (qol_col_cursor.pos >= 0){
Buffer_Seek seek = seek_line_col(qol_col_cursor.line, qol_col_cursor.col);
Buffer_Cursor cursor = buffer_compute_cursor(app, buffer, seek);
Rect_f32 col_rect = text_layout_character_on_screen(app, text_layout_id, cursor.pos);
if (col_rect.x1 > 0.f){
col_rect.y0 = rect.y0;
col_rect.y1 = rect.y1;
draw_rectangle_fcolor(app, col_rect, 0.f, fcolor_id(defcolor_highlight_cursor_line));
}
}
b32 use_error_highlight = def_get_config_b32(vars_save_string_lit("use_error_highlight"));
b32 use_jump_highlight = def_get_config_b32(vars_save_string_lit("use_jump_highlight"));
if (use_error_highlight || use_jump_highlight){
// NOTE(allen): Error highlight
String_Const_u8 name = string_u8_litexpr("*compilation*");
Buffer_ID compilation_buffer = get_buffer_by_name(app, name, Access_Always);
if (use_error_highlight){
draw_jump_highlights(app, buffer, text_layout_id, compilation_buffer,
fcolor_id(defcolor_highlight_junk));
}
// NOTE(allen): Search highlight
if (use_jump_highlight){
Buffer_ID jump_buffer = get_locked_jump_buffer(app);
if (jump_buffer != compilation_buffer){
draw_jump_highlights(app, buffer, text_layout_id, jump_buffer,
fcolor_id(defcolor_highlight_white));
}
}
}
// NOTE(allen): Color parens
b32 use_paren_helper = def_get_config_b32(vars_save_string_lit("use_paren_helper"));
if (use_paren_helper){
Color_Array colors = finalize_color_array(defcolor_text_cycle);
draw_paren_highlight(app, buffer, text_layout_id, cursor_pos, colors.vals, colors.count);
}
// NOTE(allen): Line highlight
b32 highlight_line_at_cursor = def_get_config_b32(vars_save_string_lit("highlight_line_at_cursor"));
if (highlight_line_at_cursor && is_active_view){
i64 line_number = get_line_number_from_pos(app, buffer, cursor_pos);
draw_line_highlight(app, text_layout_id, line_number, fcolor_id(defcolor_highlight_cursor_line));
}
// NOTE(allen): Whitespace highlight
b64 show_whitespace = false;
view_get_setting(app, view_id, ViewSetting_ShowWhitespace, &show_whitespace);
if (show_whitespace){
if (token_array.tokens == 0){
draw_whitespace_highlight(app, buffer, text_layout_id, cursor_roundness);
}
else{
draw_whitespace_highlight(app, text_layout_id, &token_array, cursor_roundness);
}
}
// NOTE(allen): Cursor
switch (fcoder_mode){
case FCoderMode_Original:
{
draw_original_4coder_style_cursor_mark_highlight(app, view_id, is_active_view, buffer, text_layout_id, cursor_roundness, mark_thickness);
}break;
case FCoderMode_NotepadLike:
{
draw_notepad_style_cursor_highlight(app, view_id, buffer, text_layout_id, cursor_roundness);
}break;
}
// NOTE(allen): Fade ranges
paint_fade_ranges(app, text_layout_id, buffer);
// NOTE(allen): put the actual text on the actual screen
draw_text_layout_default(app, text_layout_id);
draw_set_clip(app, prev_clip);
}
function void
custom_render_caller(Application_Links *app, Frame_Info frame_info, View_ID view_id){
ProfileScope(app, "default render caller");
View_ID active_view = get_active_view(app, Access_Always);
b32 is_active_view = (active_view == view_id);
Rect_f32 region = draw_background_and_margin(app, view_id, is_active_view);
Rect_f32 prev_clip = draw_set_clip(app, region);
Buffer_ID buffer = view_get_buffer(app, view_id, Access_Always);
Face_ID face_id = get_face_id(app, buffer);
Face_Metrics face_metrics = get_face_metrics(app, face_id);
f32 line_height = face_metrics.line_height;
f32 digit_advance = face_metrics.decimal_digit_advance;
// NOTE(allen): file bar
b64 showing_file_bar = false;
if (view_get_setting(app, view_id, ViewSetting_ShowFileBar, &showing_file_bar) && showing_file_bar){
Rect_f32_Pair pair = layout_file_bar_on_top(region, line_height);
draw_file_bar(app, view_id, buffer, face_id, pair.min);
region = pair.max;
}
Buffer_Scroll scroll = view_get_buffer_scroll(app, view_id);
Buffer_Point_Delta_Result delta = delta_apply(app, view_id,
frame_info.animation_dt, scroll);
if (!block_match_struct(&scroll.position, &delta.point)){
block_copy_struct(&scroll.position, &delta.point);
view_set_buffer_scroll(app, view_id, scroll, SetBufferScroll_NoCursorChange);
}
if (delta.still_animating){
animate_in_n_milliseconds(app, 0);
}
// NOTE(allen): query bars
region = default_draw_query_bars(app, region, view_id, face_id);
// NOTE(allen): FPS hud
if (show_fps_hud){
Rect_f32_Pair pair = layout_fps_hud_on_bottom(region, line_height);
draw_fps_hud(app, frame_info, face_id, pair.max);
region = pair.min;
animate_in_n_milliseconds(app, 1000);
}
// NOTE(allen): layout line numbers
b32 show_line_number_margins = def_get_config_b32(vars_save_string_lit("show_line_number_margins"));
Rect_f32 line_number_rect = {};
if (show_line_number_margins){
Rect_f32_Pair pair = layout_line_number_margin(app, buffer, region, digit_advance);
line_number_rect = pair.min;
region = pair.max;
}
// NOTE(allen): begin buffer render
Buffer_Point buffer_point = scroll.position;
Text_Layout_ID text_layout_id = text_layout_create(app, buffer, region, buffer_point);
// NOTE(allen): draw line numbers
if (show_line_number_margins){
draw_line_number_margin(app, view_id, buffer, face_id, text_layout_id, line_number_rect);
}
// NOTE(allen): draw the buffer
custom_render_buffer(app, view_id, face_id, buffer, text_layout_id, region);
loco_render_buffer(app, view_id, face_id, buffer, text_layout_id, region, frame_info);
text_layout_free(app, text_layout_id);
draw_set_clip(app, prev_clip);
}

View File

@ -527,12 +527,12 @@ custom_layer_init(Application_Links *app){
set_all_default_hooks(app); set_all_default_hooks(app);
modal_init(3, tctx); modal_init(3, tctx);
set_custom_hook(app, HookID_BeginBuffer, custom_begin_buffer); set_custom_hook(app, HookID_BeginBuffer, custom_begin_buffer);
set_custom_hook(app, HookID_EndBuffer, custom_end_buffer); set_custom_hook(app, HookID_EndBuffer, custom_end_buffer);
set_custom_hook(app, HookID_RenderCaller, custom_render_caller);
custom_keyboard_bindings(); custom_keyboard_bindings();
#if 0 #if 0
mapping_init(tctx, &framework_mapping); mapping_init(tctx, &framework_mapping);
String_ID global_map_id = vars_save_string_lit("keys_global"); String_ID global_map_id = vars_save_string_lit("keys_global");

View File

@ -1,4 +1,108 @@
function bool
/////////////////////////////////////////////
// TEMP until I implement more generic language stuff
/////////////////////////////////////////////
TSQuery* tree_sitter_cpp_index_query;
String_Const_u8 TS_CPP_INDEX_QUERY = string_u8_litexpr("(_ \"{\" @Start \"}\" @End ) @ScopeNest\n");
String_Const_u8 TS_CPP_HIGHLIGHT_QUERY = string_u8_litexpr("(call_expression function: ["
" (identifier) @defcolor_function"
" (field_expression field: (field_identifier) @defcolor_function)])"
"(function_declarator"
" declarator: [(identifier) (field_identifier)] @defcolor_function)"
"(preproc_def"
" name: (identifier) @defcolor_macro)"
"(preproc_function_def"
" name: (identifier) @defcolor_macro)"
"(type_identifier) @defcolor_type"
"(call_expression"
" function: (parenthesized_expression"
" (identifier) @defcolor_type))"
"[(primitive_type) (type_qualifier) (storage_class_specifier)"
" (break_statement) (continue_statement) \"union\" \"return\" \"do\""
" \"while\" \"for\" \"if\" \"class\" \"struct\" \"enum\" \"sizeof\""
" \"else\" \"switch\" \"case\"] @defcolor_keyword"
"[(number_literal) (string_literal)] @defcolor_str_constant"
"[(preproc_directive) \"#define\" \"#if\" \"#elif\" \"#else\" \"#endif\""
" \"#include\"] @defcolor_preproc"
"[\"{\" \"}\" \";\" \":\" \",\"] @defcolor_text_default"
"(comment) @defcolor_comment");
////////////////////////////////////////////////////////////////////
// Language Management
////////////////////////////////////////////////////////////////////
function void
tree_sitter_register_language(String_Const_u8 ext, TSLanguage* language, TSQuery* highlight_query)
{
Tree_Sitter_Language_Definition* lang = 0;
u64 hash = table_hash_u8(ext.str, ext.size);
u64 slot = hash % ArrayCount(tree_sitter_languages.languages);
for (Tree_Sitter_Language_Definition* l = tree_sitter_languages.languages[slot]; l != 0; l = l->next)
{
if (l->extension_hash == hash && string_match(l->extension, ext))
{
lang = l; break;
}
}
if (lang == 0)
{
lang = push_array(&tree_sitter_languages.arena, Tree_Sitter_Language_Definition, 1);
lang->next = tree_sitter_languages.languages[slot];
tree_sitter_languages.languages[slot] = lang;
lang->extension_hash = hash;
lang->extension = push_string_copy(&tree_sitter_languages.arena, ext);
lang->language = language;
lang->highlight_query = highlight_query;
}
}
function Tree_Sitter_Language_Definition*
tree_sitter_language_from_file_extension(String_Const_u8 ext)
{
Tree_Sitter_Language_Definition* result = 0;
u64 ext_hash = table_hash_u8(ext.str, ext.size);
u64 slot = ext_hash % ArrayCount(tree_sitter_languages.languages);
for (Tree_Sitter_Language_Definition* l = tree_sitter_languages.languages[slot]; l != 0; l = l->next)
{
if (l->extension_hash == ext_hash && string_match(l->extension, ext))
{
result = l;
break;
}
}
return result;
}
function Tree_Sitter_Language_Definition*
tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id, Arena* arena)
{
Tree_Sitter_Language_Definition* result = 0;
String_Const_u8 file_name = push_buffer_file_name(app, arena, buffer_id);
String_Const_u8 extension = string_file_extension(file_name);
result = tree_sitter_language_from_file_extension(extension);
return result;
}
function Tree_Sitter_Language_Definition*
tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id)
{
Scratch_Block scratch(app);
return tree_sitter_language_for_buffer(app, buffer_id, scratch);
}
/////////////////////////////////////////////
// Tree Sitter Hook Internals
/////////////////////////////////////////////
function void
tree_sitter_init(Application_Links* app) tree_sitter_init(Application_Links* app)
{ {
Buffer_ID buffer = create_buffer( Buffer_ID buffer = create_buffer(
@ -8,30 +112,34 @@ tree_sitter_init(Application_Links* app)
); );
buffer_set_setting(app, buffer, BufferSetting_Unimportant, true); buffer_set_setting(app, buffer, BufferSetting_Unimportant, true);
buffer_set_setting(app, buffer, BufferSetting_ReadOnly, true); buffer_set_setting(app, buffer, BufferSetting_ReadOnly, true);
tree_sitter_languages.arena = make_arena_system(KB(16));
u32 error_offset;
TSQueryError query_error;
{ // Register CPP
TSLanguage* language = tree_sitter_cpp();
String_Const_u8 highlight_query_str = TS_CPP_HIGHLIGHT_QUERY;
TSQuery* highlight_query = ts_query_new(
language,
(const char *)TS_CPP_HIGHLIGHT_QUERY.str,
(u32)TS_CPP_HIGHLIGHT_QUERY.size,
&error_offset, &query_error
);
tree_sitter_register_language(SCu8("cpp"), language, highlight_query);
tree_sitter_register_language(SCu8("h"), language, highlight_query);
tree_sitter_register_language(SCu8("hpp"), language, highlight_query);
tree_sitter_register_language(SCu8("cc"), language, highlight_query);
}
} }
function bool function void
tree_sitter_begin_buffer(Application_Links* app, Buffer_ID buffer_id, File_Language_Kind kind) tree_sitter_begin_buffer(Application_Links* app, Buffer_ID buffer_id)
{ {
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id); Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
switch (kind) tree_data->tree_mutex = system_mutex_make();
{
case File_Language_CPP:
{
tree_data->language = tree_sitter_cpp();
} break;
default:
tree_data->language = 0;
}
if (tree_data->language != 0)
{
tree_data->tree_mutex = system_mutex_make();
}
return tree_data->language != 0;
} }
function void function void
@ -39,7 +147,7 @@ tree_sitter_end_buffer(Application_Links* app, Buffer_ID buffer_id)
{ {
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id); Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
if (!tree_data || !tree_data->language) return; if (!tree_data) return;
Async_Task *tree_sitter_parse_task = scope_attachment(app, buffer_scope, buffer_tree_sitter_parse_task_id, Async_Task); Async_Task *tree_sitter_parse_task = scope_attachment(app, buffer_scope, buffer_tree_sitter_parse_task_id, Async_Task);
if (async_task_is_running_or_pending(&global_async_system, *tree_sitter_parse_task)) if (async_task_is_running_or_pending(&global_async_system, *tree_sitter_parse_task))
@ -74,11 +182,12 @@ tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id)
ts_parser_set_timeout_micros(parser, 5000); ts_parser_set_timeout_micros(parser, 5000);
acquire_global_frame_mutex(app); acquire_global_frame_mutex(app);
Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
String_Const_u8 src = push_whole_buffer(app, &arena, buffer_id); String_Const_u8 src = push_whole_buffer(app, &arena, buffer_id);
Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data); Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
TSTree *old_tree = tree_sitter_buffer_get_tree_copy(tree_data); TSTree *old_tree = tree_sitter_buffer_get_tree_copy(tree_data);
bool lang_set = ts_parser_set_language(parser, tree_data->language); bool lang_set = ts_parser_set_language(parser, lang->language);
release_global_frame_mutex(app); release_global_frame_mutex(app);
if (!lang_set) if (!lang_set)
@ -144,10 +253,146 @@ tree_sitter_parse_async(Async_Context* actx, String_Const_u8 data)
tree_sitter_parse_async__inner(actx, buffer_id); tree_sitter_parse_async__inner(actx, buffer_id);
} }
function Range_i64
tree_sitter_node_to_range(TSNode node)
{
Range_i64 result;
result.start = ts_node_start_byte(node);
result.end = ts_node_end_byte(node);
return result;
}
function void function void
tree_sitter_code_index_update_tick(Application_Links* app) tree_sitter_code_index_update_tick(Application_Links* app)
{ {
Scratch_Block scratch(app);
#if 0
// TODO(PS): this should be done when we register the language
if (!tree_sitter_cpp_index_query)
{
u32 error_offset;
TSQueryError query_error;
tree_sitter_cpp_index_query = ts_query_new(
tree_sitter_cpp(), TS_CPP_INDEX_QUERY, (u32)TS_CPP_INDEX_QUERY.size, &error_offset, &query_error
);
if (!tree_sitter_cpp_index_query)
{
print_message(app, string_u8_litexpr("Failed to create cpp index query\n");
}
}
for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first;
modified_node != 0;
modified_node = modified_node->next
){
Temp_Memory_Block temp(scratch);
Buffer_ID buffer_id = modified_node->buffer;
Arena arena = make_arena_system(KB(16));
Code_Index_File* index = push_array_zero(&arena, Code_Index_File, 1);
index->buffer = buffer_id;
Tree_Sitter_Code_Index_Nest_List nests = {};
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data);
if (tree)
{
TSQueryCursor* query_cursor = ts_query_cursor_new();
ts_query_cursor_exec(query_cursor, tree_sitter_cpp_index_query, ts_tree_root_node(tree));
TSQueryMatch query_match;
while (ts_query_cursor_next_match(query_cursor, &match))
{
TSQueryCapture type_capture = match.captures[0];
TSNode type_node = type_capture.node;
Range_i64 type_range = tree_sitter_node_to_range(type_node);
// TODO(PS): PICK UP HERE
}
}
ts_tree_delete(tree);
}
#endif
}
////////////////////////////////////////////////////////////////////
// Token Highlighting
////////////////////////////////////////////////////////////////////
function void
tree_sitter_highlight_node(
Application_Links* app,
TSQuery* query,
TSNode top_node,
TSQueryCursor* query_cursor,
Text_Layout_ID text_layout_id
){
ts_query_cursor_exec(query_cursor, query, top_node);
TSQueryMatch query_match;
u32 capture_index;
while (ts_query_cursor_next_capture(query_cursor, &query_match, &capture_index))
{
TSQueryCapture capture = query_match.captures[capture_index];
TSNode node = capture.node;
u32 length;
const char* tmp = ts_query_capture_name_for_id(query, capture.index, &length);
String_Const_u8 capture_name = SCu8((char*)tmp, length);
Range_i64 highlight_range = tree_sitter_node_to_range(node);
Managed_ID color_id = managed_id_get(app, SCu8("colors"), capture_name);
if (color_id != 0)
{
paint_text_color_fcolor(app, text_layout_id, highlight_range, fcolor_id(color_id));
}
}
}
function void
draw_tree_sitter_node_colors(Application_Links* app, Text_Layout_ID text_layout_id, Buffer_ID buffer_id)
{
Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
TSQuery* query = lang->highlight_query;
Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id);
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data);
if (tree)
{
TSNode root = ts_tree_root_node(tree);
// Get the smallest node that fully contains the visible range
TSNode visible_container = ts_node_descendant_for_byte_range(root, (u32)visible_range.start, (u32)visible_range.end);
TSQueryCursor* query_cursor = ts_query_cursor_new();
if (ts_node_eq(root, visible_container))
{
TSTreeCursor tree_cursor = ts_tree_cursor_new(visible_container);
if (ts_tree_cursor_goto_first_child_for_byte(&tree_cursor, (u32)visible_range.start) != -1)
{
do {
TSNode node = ts_tree_cursor_current_node(&tree_cursor);
Range_i64 child_range = tree_sitter_node_to_range(node);
if (child_range.start > visible_range.end) break;
tree_sitter_highlight_node(app, query, node, query_cursor, text_layout_id);
} while(ts_tree_cursor_goto_next_sibling(&tree_cursor));
}
else
{
// Pathological case - just highligh the whole document. This is probably bad
tree_sitter_highlight_node(app, query, root, query_cursor, text_layout_id);
}
}
else
{
tree_sitter_highlight_node(app, query, visible_container, query_cursor, text_layout_id);
}
ts_query_cursor_delete(query_cursor);
}
ts_tree_delete(tree);
} }
//////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////

View File

@ -5,29 +5,25 @@
#include <tree_sitter/api.h> #include <tree_sitter/api.h>
enum File_Language_Kind struct Tree_Sitter_Language_Definition
{ {
File_Language_None, String_Const_u8 extension;
File_Language_Unknown, u64 extension_hash;
File_Language_Text,
File_Language_Markdown, TSLanguage* language;
File_Language_C, TSQuery* highlight_query;
File_Language_CPP,
File_Language_ObjectiveC, Tree_Sitter_Language_Definition* next;
File_Language_HLSL,
File_Language_GLSL,
File_Language_Jai,
File_Language_CSharp,
File_Language_Swift,
File_Language_Go,
File_Language_Rust,
File_Language_Javascript,
File_Language_Typescript,
File_Language_JSON,
File_Language_Odin,
File_Language_Zig,
}; };
struct Tree_Sitter_Languages
{
Arena arena;
Tree_Sitter_Language_Definition* languages[4096];
};
global Tree_Sitter_Languages tree_sitter_languages;
extern "C" { extern "C" {
TSLanguage *tree_sitter_cpp(); TSLanguage *tree_sitter_cpp();
TSLanguage *tree_sitter_c(); TSLanguage *tree_sitter_c();
@ -38,13 +34,29 @@ CUSTOM_ID(attachment, buffer_tree_sitter_parse_task_id);
struct Buffer_Tree_Sitter_Data struct Buffer_Tree_Sitter_Data
{ {
TSLanguage* language;
TSTree* tree; TSTree* tree;
System_Mutex tree_mutex; System_Mutex tree_mutex;
}; };
struct Tree_Sitter_Code_Index_Nest_Node
{
Tree_Sitter_Code_Index_Nest_Node* next;
Tree_Sitter_Code_Index_Nest_Node* prev;
Tree_Sitter_Code_Index_Nest_Node* parent;
Tree_Sitter_Code_Index_Nest_Node* child_first;
Tree_Sitter_Code_Index_Nest_Node* child_last;
Code_Index_Nest* nest;
};
struct Tree_Sitter_Code_Index_Nest_List
{
Tree_Sitter_Code_Index_Nest_Node* first;
Tree_Sitter_Code_Index_Nest_Node* last;
};
b8 use_tree_sitter_code_indexing = true; b8 use_tree_sitter_code_indexing = true;
b8 use_tree_sitter_token_coloring = true;
function void tree_sitter_code_index_update_tick(Application_Links *app); function void tree_sitter_code_index_update_tick(Application_Links *app);
#endif //FCODER_TREE_SITTER_H #endif //FCODER_TREE_SITTER_H