449 lines
17 KiB
C++
449 lines
17 KiB
C++
|
|
/////////////////////////////////////////////
|
|
// TEMP until I implement more generic language stuff
|
|
/////////////////////////////////////////////
|
|
|
|
TSQuery* tree_sitter_cpp_index_query;
|
|
String_Const_u8 TS_CPP_INDEX_QUERY = string_u8_litexpr("(_ \"{\" @Start \"}\" @End ) @ScopeNest\n");
|
|
|
|
String_Const_u8 TS_CPP_HIGHLIGHT_QUERY = string_u8_litexpr("(call_expression function: ["
|
|
" (identifier) @defcolor_function"
|
|
" (field_expression field: (field_identifier) @defcolor_function)])"
|
|
"(function_declarator"
|
|
" declarator: [(identifier) (field_identifier)] @defcolor_function)"
|
|
|
|
"(preproc_def"
|
|
" name: (identifier) @defcolor_macro)"
|
|
"(preproc_function_def"
|
|
" name: (identifier) @defcolor_macro)"
|
|
|
|
"(type_identifier) @defcolor_type"
|
|
"(call_expression"
|
|
" function: (parenthesized_expression"
|
|
" (identifier) @defcolor_type))"
|
|
|
|
"[(primitive_type) (type_qualifier) (storage_class_specifier)"
|
|
" (break_statement) (continue_statement) \"union\" \"return\" \"do\""
|
|
" \"while\" \"for\" \"if\" \"class\" \"struct\" \"enum\" \"sizeof\""
|
|
" \"else\" \"switch\" \"case\"] @defcolor_keyword"
|
|
|
|
"[(number_literal) (string_literal)] @defcolor_str_constant"
|
|
"[(preproc_directive) \"#define\" \"#if\" \"#elif\" \"#else\" \"#endif\""
|
|
" \"#include\"] @defcolor_preproc"
|
|
"[\"{\" \"}\" \";\" \":\" \",\"] @defcolor_text_default"
|
|
"(comment) @defcolor_comment");
|
|
|
|
////////////////////////////////////////////////////////////////////
|
|
// Language Management
|
|
////////////////////////////////////////////////////////////////////
|
|
|
|
function void
|
|
tree_sitter_register_language(String_Const_u8 ext, TSLanguage* language, TSQuery* highlight_query)
|
|
{
|
|
Tree_Sitter_Language_Definition* lang = 0;
|
|
u64 hash = table_hash_u8(ext.str, ext.size);
|
|
u64 slot = hash % ArrayCount(tree_sitter_languages.languages);
|
|
for (Tree_Sitter_Language_Definition* l = tree_sitter_languages.languages[slot]; l != 0; l = l->next)
|
|
{
|
|
if (l->extension_hash == hash && string_match(l->extension, ext))
|
|
{
|
|
lang = l; break;
|
|
}
|
|
}
|
|
|
|
if (lang == 0)
|
|
{
|
|
lang = push_array(&tree_sitter_languages.arena, Tree_Sitter_Language_Definition, 1);
|
|
lang->next = tree_sitter_languages.languages[slot];
|
|
tree_sitter_languages.languages[slot] = lang;
|
|
|
|
lang->extension_hash = hash;
|
|
lang->extension = push_string_copy(&tree_sitter_languages.arena, ext);
|
|
lang->language = language;
|
|
lang->highlight_query = highlight_query;
|
|
}
|
|
}
|
|
|
|
function Tree_Sitter_Language_Definition*
|
|
tree_sitter_language_from_file_extension(String_Const_u8 ext)
|
|
{
|
|
Tree_Sitter_Language_Definition* result = 0;
|
|
u64 ext_hash = table_hash_u8(ext.str, ext.size);
|
|
u64 slot = ext_hash % ArrayCount(tree_sitter_languages.languages);
|
|
for (Tree_Sitter_Language_Definition* l = tree_sitter_languages.languages[slot]; l != 0; l = l->next)
|
|
{
|
|
if (l->extension_hash == ext_hash && string_match(l->extension, ext))
|
|
{
|
|
result = l;
|
|
break;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function Tree_Sitter_Language_Definition*
|
|
tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id, Arena* arena)
|
|
{
|
|
Tree_Sitter_Language_Definition* result = 0;
|
|
String_Const_u8 file_name = push_buffer_file_name(app, arena, buffer_id);
|
|
String_Const_u8 extension = string_file_extension(file_name);
|
|
result = tree_sitter_language_from_file_extension(extension);
|
|
return result;
|
|
}
|
|
|
|
function Tree_Sitter_Language_Definition*
|
|
tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id)
|
|
{
|
|
Scratch_Block scratch(app);
|
|
return tree_sitter_language_for_buffer(app, buffer_id, scratch);
|
|
}
|
|
|
|
/////////////////////////////////////////////
|
|
// Tree Sitter Hook Internals
|
|
/////////////////////////////////////////////
|
|
|
|
function void
|
|
tree_sitter_init(Application_Links* app)
|
|
{
|
|
Buffer_ID buffer = create_buffer(
|
|
app,
|
|
string_u8_litexpr("*tree*"),
|
|
BufferCreate_NeverAttachToFile | BufferCreate_AlwaysNew
|
|
);
|
|
buffer_set_setting(app, buffer, BufferSetting_Unimportant, true);
|
|
buffer_set_setting(app, buffer, BufferSetting_ReadOnly, true);
|
|
|
|
tree_sitter_languages.arena = make_arena_system(KB(16));
|
|
|
|
u32 error_offset;
|
|
TSQueryError query_error;
|
|
|
|
{ // Register CPP
|
|
TSLanguage* language = tree_sitter_cpp();
|
|
String_Const_u8 highlight_query_str = TS_CPP_HIGHLIGHT_QUERY;
|
|
TSQuery* highlight_query = ts_query_new(
|
|
language,
|
|
(const char *)TS_CPP_HIGHLIGHT_QUERY.str,
|
|
(u32)TS_CPP_HIGHLIGHT_QUERY.size,
|
|
&error_offset, &query_error
|
|
);
|
|
tree_sitter_register_language(SCu8("cpp"), language, highlight_query);
|
|
tree_sitter_register_language(SCu8("h"), language, highlight_query);
|
|
tree_sitter_register_language(SCu8("hpp"), language, highlight_query);
|
|
tree_sitter_register_language(SCu8("cc"), language, highlight_query);
|
|
}
|
|
}
|
|
|
|
function void
|
|
tree_sitter_begin_buffer(Application_Links* app, Buffer_ID buffer_id)
|
|
{
|
|
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
|
|
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
|
|
tree_data->tree_mutex = system_mutex_make();
|
|
}
|
|
|
|
function void
|
|
tree_sitter_end_buffer(Application_Links* app, Buffer_ID buffer_id)
|
|
{
|
|
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
|
|
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
|
|
if (!tree_data) return;
|
|
|
|
Async_Task *tree_sitter_parse_task = scope_attachment(app, buffer_scope, buffer_tree_sitter_parse_task_id, Async_Task);
|
|
if (async_task_is_running_or_pending(&global_async_system, *tree_sitter_parse_task))
|
|
{
|
|
async_task_cancel(app, &global_async_system, *tree_sitter_parse_task);
|
|
}
|
|
|
|
system_mutex_acquire(tree_data->tree_mutex);
|
|
ts_tree_delete(tree_data->tree);
|
|
system_mutex_release(tree_data->tree_mutex);
|
|
system_mutex_free(tree_data->tree_mutex);
|
|
}
|
|
|
|
function TSTree*
|
|
tree_sitter_buffer_get_tree_copy(Buffer_Tree_Sitter_Data* tree_data)
|
|
{
|
|
TSTree* result = 0;
|
|
// system_mutex_acquire(tree_data->tree_mutex);
|
|
if (tree_data->tree) result = ts_tree_copy(tree_data->tree);
|
|
// system_mutex_release(tree_data->tree_mutex);
|
|
return result;
|
|
}
|
|
|
|
function void
|
|
tree_sitter_parse_async__inner(Async_Context* actx, Buffer_ID buffer_id)
|
|
{
|
|
Application_Links *app = actx->app;
|
|
|
|
Arena arena = make_arena_system(KB(16));
|
|
|
|
TSParser *parser = ts_parser_new();
|
|
ts_parser_set_timeout_micros(parser, 5000);
|
|
|
|
acquire_global_frame_mutex(app);
|
|
Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
|
|
String_Const_u8 src = push_whole_buffer(app, &arena, buffer_id);
|
|
Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
|
|
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
|
|
TSTree *old_tree = tree_sitter_buffer_get_tree_copy(tree_data);
|
|
bool lang_set = ts_parser_set_language(parser, lang->language);
|
|
release_global_frame_mutex(app);
|
|
|
|
if (!lang_set)
|
|
{
|
|
AssertMessageAlways("Failed to set the language for the parser."
|
|
"This probably means a language wasn't set"
|
|
"in the BeginBuffer hook.\n");
|
|
}
|
|
|
|
// Iterate until we get a tree or we find that we should cancel the parse
|
|
TSTree *new_tree = 0;
|
|
b32 canceled = false;
|
|
for (;;)
|
|
{
|
|
new_tree = ts_parser_parse_string(parser, old_tree, (char *)src.str, (u32)src.size);
|
|
if (async_check_canceled(actx))
|
|
{
|
|
canceled = true;
|
|
break;
|
|
}
|
|
if (new_tree) break;
|
|
}
|
|
|
|
if (!canceled && new_tree)
|
|
{
|
|
TSTree* old_buffer_tree;
|
|
acquire_global_frame_mutex(app);
|
|
{
|
|
// NOTE(jack): Copy the old pointer to delete it outside the mutex.
|
|
system_mutex_acquire(tree_data->tree_mutex);
|
|
old_buffer_tree = tree_data->tree;
|
|
tree_data->tree = new_tree;
|
|
system_mutex_acquire(tree_data->tree_mutex);
|
|
|
|
print_message(app, SCu8("Finished Parse\n"));
|
|
|
|
// TODO(PS): Just put the code index update call here
|
|
// NOTE(jack): This feels kinda hacky, this is here to trigger
|
|
// the code index update tick. The buffer is also makred by the
|
|
// async lexer so we will update the index too frequently. We
|
|
// should probably change the lexer to not mark as modified.
|
|
// TODO(jack): Should we instead trigger another async task here to
|
|
// update the code index once this is done?
|
|
buffer_mark_as_modified(buffer_id);
|
|
|
|
// Force a frame refresh by requesting another frame
|
|
animate_in_n_milliseconds(app, 0);
|
|
}
|
|
release_global_frame_mutex(app);
|
|
ts_tree_delete(old_buffer_tree);
|
|
}
|
|
|
|
ts_parser_delete(parser);
|
|
ts_tree_delete(old_tree);
|
|
linalloc_clear(&arena);
|
|
}
|
|
|
|
function void
|
|
tree_sitter_parse_async(Async_Context* actx, String_Const_u8 data)
|
|
{
|
|
if (data.size != sizeof(Buffer_ID)) return;
|
|
Buffer_ID buffer_id = *(Buffer_ID*)data.str;
|
|
tree_sitter_parse_async__inner(actx, buffer_id);
|
|
}
|
|
|
|
function Range_i64
|
|
tree_sitter_node_to_range(TSNode node)
|
|
{
|
|
Range_i64 result;
|
|
result.start = ts_node_start_byte(node);
|
|
result.end = ts_node_end_byte(node);
|
|
return result;
|
|
}
|
|
|
|
function void
|
|
tree_sitter_code_index_update_tick(Application_Links* app)
|
|
{
|
|
Scratch_Block scratch(app);
|
|
|
|
#if 0
|
|
// TODO(PS): this should be done when we register the language
|
|
if (!tree_sitter_cpp_index_query)
|
|
{
|
|
u32 error_offset;
|
|
TSQueryError query_error;
|
|
tree_sitter_cpp_index_query = ts_query_new(
|
|
tree_sitter_cpp(), TS_CPP_INDEX_QUERY, (u32)TS_CPP_INDEX_QUERY.size, &error_offset, &query_error
|
|
);
|
|
if (!tree_sitter_cpp_index_query)
|
|
{
|
|
print_message(app, string_u8_litexpr("Failed to create cpp index query\n");
|
|
}
|
|
}
|
|
|
|
for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first;
|
|
modified_node != 0;
|
|
modified_node = modified_node->next
|
|
){
|
|
Temp_Memory_Block temp(scratch);
|
|
Buffer_ID buffer_id = modified_node->buffer;
|
|
|
|
Arena arena = make_arena_system(KB(16));
|
|
Code_Index_File* index = push_array_zero(&arena, Code_Index_File, 1);
|
|
index->buffer = buffer_id;
|
|
|
|
Tree_Sitter_Code_Index_Nest_List nests = {};
|
|
|
|
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
|
|
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
|
|
TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data);
|
|
if (tree)
|
|
{
|
|
TSQueryCursor* query_cursor = ts_query_cursor_new();
|
|
ts_query_cursor_exec(query_cursor, tree_sitter_cpp_index_query, ts_tree_root_node(tree));
|
|
|
|
TSQueryMatch query_match;
|
|
while (ts_query_cursor_next_match(query_cursor, &match))
|
|
{
|
|
TSQueryCapture type_capture = match.captures[0];
|
|
TSNode type_node = type_capture.node;
|
|
Range_i64 type_range = tree_sitter_node_to_range(type_node);
|
|
|
|
// TODO(PS): PICK UP HERE
|
|
}
|
|
}
|
|
ts_tree_delete(tree);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////
|
|
// Token Highlighting
|
|
////////////////////////////////////////////////////////////////////
|
|
|
|
function void
|
|
tree_sitter_highlight_node(
|
|
Application_Links* app,
|
|
TSQuery* query,
|
|
TSNode top_node,
|
|
TSQueryCursor* query_cursor,
|
|
Text_Layout_ID text_layout_id
|
|
){
|
|
ts_query_cursor_exec(query_cursor, query, top_node);
|
|
|
|
TSQueryMatch query_match;
|
|
u32 capture_index;
|
|
while (ts_query_cursor_next_capture(query_cursor, &query_match, &capture_index))
|
|
{
|
|
TSQueryCapture capture = query_match.captures[capture_index];
|
|
TSNode node = capture.node;
|
|
|
|
u32 length;
|
|
const char* tmp = ts_query_capture_name_for_id(query, capture.index, &length);
|
|
String_Const_u8 capture_name = SCu8((char*)tmp, length);
|
|
|
|
Range_i64 highlight_range = tree_sitter_node_to_range(node);
|
|
Managed_ID color_id = managed_id_get(app, SCu8("colors"), capture_name);
|
|
if (color_id != 0)
|
|
{
|
|
paint_text_color_fcolor(app, text_layout_id, highlight_range, fcolor_id(color_id));
|
|
}
|
|
}
|
|
}
|
|
|
|
function void
|
|
draw_tree_sitter_node_colors(Application_Links* app, Text_Layout_ID text_layout_id, Buffer_ID buffer_id)
|
|
{
|
|
Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
|
|
TSQuery* query = lang->highlight_query;
|
|
|
|
Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id);
|
|
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
|
|
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
|
|
TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data);
|
|
if (tree)
|
|
{
|
|
TSNode root = ts_tree_root_node(tree);
|
|
// Get the smallest node that fully contains the visible range
|
|
TSNode visible_container = ts_node_descendant_for_byte_range(root, (u32)visible_range.start, (u32)visible_range.end);
|
|
TSQueryCursor* query_cursor = ts_query_cursor_new();
|
|
if (ts_node_eq(root, visible_container))
|
|
{
|
|
TSTreeCursor tree_cursor = ts_tree_cursor_new(visible_container);
|
|
if (ts_tree_cursor_goto_first_child_for_byte(&tree_cursor, (u32)visible_range.start) != -1)
|
|
{
|
|
do {
|
|
TSNode node = ts_tree_cursor_current_node(&tree_cursor);
|
|
Range_i64 child_range = tree_sitter_node_to_range(node);
|
|
if (child_range.start > visible_range.end) break;
|
|
tree_sitter_highlight_node(app, query, node, query_cursor, text_layout_id);
|
|
} while(ts_tree_cursor_goto_next_sibling(&tree_cursor));
|
|
}
|
|
else
|
|
{
|
|
// Pathological case - just highligh the whole document. This is probably bad
|
|
tree_sitter_highlight_node(app, query, root, query_cursor, text_layout_id);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
tree_sitter_highlight_node(app, query, visible_container, query_cursor, text_layout_id);
|
|
}
|
|
ts_query_cursor_delete(query_cursor);
|
|
}
|
|
ts_tree_delete(tree);
|
|
}
|
|
|
|
////////////////////////////////////////////////////////////////////
|
|
// DEBUG
|
|
////////////////////////////////////////////////////////////////////
|
|
|
|
char* prefix_buffer = " ";
|
|
|
|
function void
|
|
write_tree_sitter_tree_to_buffer__inner(Application_Links *app, Arena *arena, Buffer_ID buffer_id,
|
|
TSNode cur_node, i32 level = 0, const char *field="")
|
|
{
|
|
TSPoint start = ts_node_start_point(cur_node);
|
|
TSPoint end = ts_node_end_point(cur_node);
|
|
// + 1 on ts positions becuase the first line/column are zero in treesitter,
|
|
// but 4coder displays as 1 indexed in the filebar.
|
|
String_Const_u8 string = push_stringf(arena, "%.*s%s: %s [%d, %d] - [%d, %d]\n",
|
|
level*2, prefix_buffer, field, ts_node_type(cur_node),
|
|
start.row + 1, start.column + 1,
|
|
end.row + 1, end.column + 1);
|
|
|
|
buffer_replace_range(app, buffer_id, Ii64(buffer_get_size(app, buffer_id)), string);
|
|
|
|
u32 child_count = ts_node_child_count(cur_node);
|
|
for (u32 i = 0; i < child_count; ++i)
|
|
{
|
|
TSNode child = ts_node_child(cur_node, i);
|
|
if (ts_node_is_named(child))
|
|
{
|
|
field = ts_node_field_name_for_child(cur_node, i);
|
|
if (!field) field = "";
|
|
write_tree_sitter_tree_to_buffer__inner(app, arena, buffer_id, child, level + 1, field);
|
|
}
|
|
}
|
|
}
|
|
|
|
CUSTOM_COMMAND_SIG(tree_sitter_write_tree)
|
|
CUSTOM_DOC("Write the current buffer's tree sitter tree to *tree*")
|
|
{
|
|
Scratch_Block scratch(app);
|
|
Buffer_ID out_buffer = get_buffer_by_name(app, string_u8_litexpr("*tree*"), Access_Always);
|
|
|
|
View_ID view = get_active_view(app, Access_Always);
|
|
Buffer_ID buffer = view_get_buffer(app, view, Access_Visible);
|
|
|
|
Managed_Scope scope = buffer_get_managed_scope(app, buffer);
|
|
Buffer_Tree_Sitter_Data *tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
|
|
|
|
if (tree_data->tree)
|
|
{
|
|
TSNode root = ts_tree_root_node(tree_data->tree);
|
|
write_tree_sitter_tree_to_buffer__inner(app, scratch, out_buffer, root);
|
|
}
|
|
} |