ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
4coder/code/custom/4coder_tree_sitter.cpp

1399 lines
46 KiB
C++
Raw Blame History

/////////////////////////////////////////////
// TEMP until I implement more generic language stuff
/////////////////////////////////////////////
#include "languages/tree_sitter_cpp.h"
#include "languages/tree_sitter_jai.h"
#include "languages/tree_sitter_bash.h"
#include "languages/tree_sitter_ts.h"
function void
register_all_languages(Application_Links* app)
{
tree_sitter_register_cpp(app);
tree_sitter_register_jai(app);
tree_sitter_register_bash(app);
tree_sitter_register_ts(app);
}
////////////////////////////////////////////////////////////////////
// Language Management
////////////////////////////////////////////////////////////////////
function TSQuery*
tree_sitter_query_new(Application_Links* app, TSLanguage* language, String_Const_u8 query_string)
{
u32 error_offset;
TSQueryError query_error;
TSQuery* result = ts_query_new(
language, (const char *)query_string.str, (u32)query_string.size,
&error_offset, &query_error
);
if (!result)
{
print_message(app, SCu8("Error creating query\n"));
printf("%.*s\n", (int)Min(query_string.size-error_offset, 100), query_string.str + error_offset);
}
return result;
}
function void
tree_sitter_register_language(String_Const_u8 ext, TSLanguage* language, Tree_Sitter_Language_Queries queries, Tree_Sitter_Language_Flags flags)
{
Tree_Sitter_Language_Definition* lang = 0;
u64 hash = table_hash_u8(ext.str, ext.size);
u64 slot = hash % ArrayCount(tree_sitter_languages.languages);
for (Tree_Sitter_Language_Definition* l = tree_sitter_languages.languages[slot]; l != 0; l = l->next)
{
if (l->extension_hash == hash && string_match(l->extension, ext))
{
lang = l; break;
}
}
if (lang == 0)
{
lang = push_array(&tree_sitter_languages.arena, Tree_Sitter_Language_Definition, 1);
lang->next = tree_sitter_languages.languages[slot];
tree_sitter_languages.languages[slot] = lang;
lang->extension_hash = hash;
lang->extension = push_string_copy(&tree_sitter_languages.arena, ext);
lang->language = language;
lang->queries = queries;
lang->flags = flags;
}
}
function Tree_Sitter_Language_Definition*
tree_sitter_language_from_file_extension(String_Const_u8 ext)
{
Tree_Sitter_Language_Definition* result = 0;
u64 ext_hash = table_hash_u8(ext.str, ext.size);
u64 slot = ext_hash % ArrayCount(tree_sitter_languages.languages);
for (Tree_Sitter_Language_Definition* l = tree_sitter_languages.languages[slot]; l != 0; l = l->next)
{
if (l->extension_hash == ext_hash && string_match(l->extension, ext))
{
result = l;
break;
}
}
return result;
}
function Tree_Sitter_Language_Definition*
tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id, Arena* arena)
{
Tree_Sitter_Language_Definition* result = 0;
String_Const_u8 file_name = push_buffer_file_name(app, arena, buffer_id);
String_Const_u8 extension = string_file_extension(file_name);
result = tree_sitter_language_from_file_extension(extension);
return result;
}
function Tree_Sitter_Language_Definition*
tree_sitter_language_for_buffer(Application_Links* app, Buffer_ID buffer_id)
{
Scratch_Block scratch(app);
return tree_sitter_language_for_buffer(app, buffer_id, scratch);
}
////////////////////////////////////////////////////////////////////
// Async Management
////////////////////////////////////////////////////////////////////
function void
async_task_cancel_nowait(Application_Links* app, Async_System* async_system, Async_Task task)
{
system_mutex_acquire(async_system->mutex);
Async_Node *node = async_get_pending_node(async_system, task);
if (node != 0)
{
dll_remove(&node->node);
async_system->task_count -= 1;
async_free_node(async_system, node);
}
else
{
node = async_get_running_node(async_system, task);
if (node != 0)
{
b32 *cancel_signal = &node->thread->cancel_signal;
atomic_write_b32(cancel_signal, true);
// async_task_wait__inner(app, async_system, task);
}
}
system_mutex_release(async_system->mutex);
}
/////////////////////////////////////////////
// Tree Sitter Hook Internals
/////////////////////////////////////////////
function void
tree_sitter_init(Application_Links* app)
{
Buffer_ID buffer = create_buffer(
app,
string_u8_litexpr("*tree*"),
BufferCreate_NeverAttachToFile | BufferCreate_AlwaysNew
);
buffer_set_setting(app, buffer, BufferSetting_Unimportant, true);
buffer_set_setting(app, buffer, BufferSetting_ReadOnly, true);
tree_sitter_languages.arena = make_arena_system(KB(16));
register_all_languages(app);
}
function void
tree_sitter_begin_buffer(Application_Links* app, Buffer_ID buffer_id)
{
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
tree_data->tree_mutex = system_mutex_make();
}
function void
tree_sitter_end_buffer(Application_Links* app, Buffer_ID buffer_id)
{
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
if (!tree_data || !tree_data->tree) return;
system_mutex_acquire(tree_data->tree_mutex);
ts_tree_delete(tree_data->tree);
system_mutex_release(tree_data->tree_mutex);
system_mutex_free(tree_data->tree_mutex);
}
function TSTree*
tree_sitter_buffer_get_tree_copy(Buffer_Tree_Sitter_Data* tree_data)
{
TSTree* result = 0;
system_mutex_acquire(tree_data->tree_mutex);
if (tree_data->tree) result = ts_tree_copy(tree_data->tree);
system_mutex_release(tree_data->tree_mutex);
return result;
}
////////////////////////////////////////////////////////////////////
// Queries
////////////////////////////////////////////////////////////////////
function Tree_Sitter_Query_Cursor
tree_sitter_query_init(Application_Links* app, Buffer_ID buffer_id, TSQuery* query)
{
Tree_Sitter_Query_Cursor result = {};
result.buffer_id = buffer_id;
result.query = query;
result.ok = false;
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
result.tree = tree_sitter_buffer_get_tree_copy(tree_data);
if (result.tree) result.first_node = ts_tree_root_node(result.tree);
result.ok = (
result.query != 0 &&
result.tree != 0
);
return result;
}
function Tree_Sitter_Query_Cursor
tree_sitter_query_init(Application_Links* app, Buffer_ID buffer_id, TSQuery* query, TSNode first_node)
{
Tree_Sitter_Query_Cursor result = tree_sitter_query_init(app, buffer_id, query);
result.first_node = first_node;
return result;
}
function bool
tree_sitter_query_continue(Tree_Sitter_Query_Cursor* cursor, TSQueryMatch* match, u32* capture_index)
{
if (cursor->ok)
{
if (!cursor->query_cursor)
{
cursor->query_cursor = ts_query_cursor_new();
ts_query_cursor_exec(cursor->query_cursor, cursor->query, cursor->first_node);
}
cursor->ok = ts_query_cursor_next_capture(cursor->query_cursor, match, capture_index);
}
return cursor->ok;
}
function void
tree_sitter_query_end(Tree_Sitter_Query_Cursor* cursor)
{
if (cursor->query_cursor) ts_query_cursor_delete(cursor->query_cursor);
ts_tree_delete(cursor->tree);
}
////////////////////////////////////////////////////////////////////
// Tree Parsing
////////////////////////////////////////////////////////////////////
#define DO_CODE_INDEX_UPDATE 0
function Tree_Sitter_Parse_State
tree_sitter_parse_state_create(Application_Links* app, Buffer_ID buffer_id)
{
Tree_Sitter_Parse_State parse_state = {};
parse_state.parser = ts_parser_new();
ts_parser_set_timeout_micros(parse_state.parser, 5000);
Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
parse_state.tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
parse_state.old_tree = tree_sitter_buffer_get_tree_copy(parse_state.tree_data);
Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
bool lang_set = ts_parser_set_language(parse_state.parser, lang->language);
if (!lang_set)
{
AssertMessageAlways("Failed to set the language for the parser."
"This probably means a language wasn't set"
"in the BeginBuffer hook.\n");
}
return parse_state;
}
function void
tree_sitter_parse_state_destroy(Tree_Sitter_Parse_State* parse_state)
{
ts_parser_delete(parse_state->parser);
ts_tree_delete(parse_state->old_tree);
}
function TSTree*
tree_sitter_parse_complete(Tree_Sitter_Parse_State* parse_state, TSTree* new_tree)
{
// NOTE(jack): Copy the old pointer to delete it outside the mutex.
system_mutex_acquire(parse_state->tree_data->tree_mutex);
TSTree* old_buffer_tree = parse_state->tree_data->tree;
parse_state->tree_data->tree = new_tree;
system_mutex_release(parse_state->tree_data->tree_mutex);
return old_buffer_tree;
}
function void
tree_sitter_parse_full_file_async(Async_Context* actx, String_Const_u8 data)
{
if (data.size != sizeof(Buffer_ID)) return;
Buffer_ID buffer_id = *(Buffer_ID*)data.str;
Application_Links *app = actx->app;
Scratch_Block scratch(app);
acquire_global_frame_mutex(app);
String_Const_u8 buffer_contents = push_whole_buffer(app, scratch, buffer_id);
Tree_Sitter_Parse_State parse_state = tree_sitter_parse_state_create(
app, buffer_id
);
release_global_frame_mutex(app);
// Iterate until we get a tree or we find that we should cancel the parse
TSTree *new_tree = 0;
b32 canceled = false;
for (;;)
{
new_tree = ts_parser_parse_string(parse_state.parser, parse_state.old_tree, (char *)buffer_contents.str, (u32)buffer_contents.size);
if (async_check_canceled(actx))
{
canceled = true;
break;
}
if (new_tree) break;
}
if (!canceled && new_tree)
{
TSTree* old_buffer_tree;
acquire_global_frame_mutex(app);
{
old_buffer_tree = tree_sitter_parse_complete(&parse_state, new_tree);
animate_in_n_milliseconds(app, 0); // Force a frame refresh by requesting another frame
}
release_global_frame_mutex(app);
ts_tree_delete(old_buffer_tree);
}
tree_sitter_parse_state_destroy(&parse_state);
Managed_Scope scope = buffer_get_managed_scope(app, buffer_id);
Async_Task* code_index_update_task_ptr = scope_attachment(app, scope, buffer_update_code_index_task, Async_Task);
*code_index_update_task_ptr = async_task_no_dep(&global_async_system, tree_sitter_code_index_update_async, make_data_struct(&buffer_id));
}
function void
tree_sitter_parse_incremental(Application_Links* app, Buffer_ID buffer_id)
{
Scratch_Block scratch(app);
String_Const_u8 buffer_contents = push_whole_buffer(app, scratch, buffer_id);
Tree_Sitter_Parse_State parse_state = tree_sitter_parse_state_create(
app, buffer_id
);
TSTree *new_tree = 0;
while (!new_tree)
{
new_tree = ts_parser_parse_string(
parse_state.parser,
parse_state.old_tree,
(char *)buffer_contents.str,
(u32)buffer_contents.size
);
}
TSTree* old_buffer_tree = tree_sitter_parse_complete(&parse_state, new_tree);
ts_tree_delete(old_buffer_tree);
tree_sitter_parse_state_destroy(&parse_state);
}
////////////////////////////////////////////////////////////////////
// Code Index Updating
////////////////////////////////////////////////////////////////////
function Range_i64
tree_sitter_node_to_range(TSNode node)
{
Range_i64 result;
result.start = ts_node_start_byte(node);
result.end = ts_node_end_byte(node);
return result;
}
function Code_Index_Scope_Delim*
code_index_new_scope_delim(Code_Index_File* index, Arena* arena, Code_Index_Scope_Delim_Kind kind, Range_i64 range)
{
if (!index->scope_delim_free)
{
int count = 32;
Code_Index_Scope_Delim* arr = push_array_zero(arena, Code_Index_Scope_Delim, count);
for (int i = 0; i < count-1; i++) arr[i].next = &arr[i+1];
index->scope_delim_free = arr;
}
Code_Index_Scope_Delim *result = index->scope_delim_free;
index->scope_delim_free = result->next;
block_zero_struct(result);
result->kind = kind;
result->pos = range;
return result;
}
function void
code_index_scope_delim_insert(Code_Index_Scope_Delim_List* list, Code_Index_Scope_Delim* prev, Code_Index_Scope_Delim* delim)
{
if (!prev)
{
delim->next = list->first;
if (list->first) list->first->prev = delim;
else list->last = delim;
list->first = delim;
}
else
{
Code_Index_Scope_Delim* next = prev->next;
prev->next = delim;
delim->prev = prev;
delim->next = next;
if (next) next->prev = delim;
if (prev == list->last) list->last = delim;
}
list->count += 1;
}
function void
code_index_free_scope_delim(Code_Index_File* index, Code_Index_Scope_Delim* delim)
{
delim->next = index->scope_delim_free;
index->scope_delim_free = delim;
}
function Code_Index_Note*
code_index_new_note(Code_Index_File* index, Arena* arena, Code_Index_Note_Kind kind, Range_i64 range, Code_Index_Nest_Stack* parent)
{
if (!index->note_free)
{
int count = 32;
Code_Index_Note* arr = push_array_zero(arena, Code_Index_Note, count);
for (int i = 0; i < count-1; i++) arr[i].next = &arr[i+1];
index->note_free = arr;
}
Code_Index_Note *result = index->note_free;
index->note_free = result->next;
block_zero_struct(result);
result->file = index;
result->note_kind = kind;
result->pos = range;
if (parent != 0) result->parent = parent->nest;
return result;
}
function void
code_index_note_insert(Code_Index_Note_List* list, Code_Index_Note* prev, Code_Index_Note* note)
{
if (!prev)
{
note->next = list->first;
if (!list->last) list->last = note;
list->first = note;
}
else
{
Code_Index_Note* next = prev->next;
prev->next = note;
note->next = next;
if (prev == list->last) list->last = note;
}
list->count += 1;
}
function void
code_index_free_note(Code_Index_File* index, Code_Index_Note* note)
{
block_zero_struct(note);
note->next = index->note_free;
index->note_free = note;
}
function Tree_Sitter_Code_Index_Update_State
tree_sitter_code_index_update_state_create(
Application_Links* app,
Buffer_ID buffer_id,
Code_Index_File_Storage* code_index_storage,
Arena* scratch
){
Tree_Sitter_Code_Index_Update_State state = {};
state.ok = false;
state.buffer_id = buffer_id;
state.language = tree_sitter_language_for_buffer(app, state.buffer_id);
if (!state.language) return state;
if (code_index_storage != 0)
{
state.index_arena = code_index_storage->arena;
state.index = code_index_storage->file;
}
else
{
state.index_arena = make_arena_system(KB(16));
state.index = push_array_zero(&state.index_arena, Code_Index_File, 1);
string_pool_init(&state.index->string_pool);
state.index->buffer = buffer_id;
}
state.buffer_contents = push_whole_buffer(app, scratch, state.buffer_id);
state.nest_stack_first = 0;
state.nest_stack_last = 0;
state.last_note = 0;
state.last_note_match_id = max_u32;
code_index_note_list_hash_init(&state.new_notes);
state.ok = true;
return state;
}
function void
tree_sitter_code_index_update_process_query_match(
Tree_Sitter_Code_Index_Update_State* state,
Tree_Sitter_Query_Cursor query,
TSQueryMatch query_match,
u32 capture_index,
Arena* scratch
){
TSQueryCapture type_capture = query_match.captures[capture_index];
TSNode type_node = type_capture.node;
Range_i64 type_range = tree_sitter_node_to_range(type_node);
u32 length;
const char* tmp = ts_query_capture_name_for_id(query.query, type_capture.index, &length);
String_Const_u8 capture_name = SCu8((char*)tmp, length);
if (string_match(capture_name, SCu8("scope_begin")) || string_match(capture_name, SCu8("scope_end")))
{
Code_Index_Scope_Delim_Kind kind;
String_Const_u8 delim_char = string_substring(state->buffer_contents, type_range);
bool skip = false;
if (delim_char.str[0] == '{') kind = CodeIndexScopeDelim_ScopeOpen;
else if (delim_char.str[0] == '}') kind = CodeIndexScopeDelim_ScopeClose;
else if (delim_char.str[0] == '(') kind = CodeIndexScopeDelim_ParenOpen;
else if (delim_char.str[0] == ')') kind = CodeIndexScopeDelim_ParenClose;
else if (delim_char.str[0] == '[') kind = CodeIndexScopeDelim_BracketOpen;
else if (delim_char.str[0] == ']') kind = CodeIndexScopeDelim_BracketClose;
else skip = true;
if (!skip)
{
Code_Index_Scope_Delim* delim = code_index_new_scope_delim(
state->index, &state->index_arena, kind, type_range
);
code_index_scope_delim_insert(&state->index->scope_delim_list, state->last_delim, delim);
state->last_delim = delim;
}
}
else if (string_match(capture_name, SCu8("name")))
{
Range_i64 range = Ii64_size(type_range.start, type_range.end - type_range.start);
bool apply_name_to_last_note = state->new_notes.sentinel_last.prev_in_hash != 0;
apply_name_to_last_note &= state->last_note_match_id == query_match.id;
if (apply_name_to_last_note) state->new_notes.sentinel_last.prev_in_hash->pos = range;
}
else
{
Code_Index_Note_Kind kind;
if (string_match(capture_name, SCu8("definition.class"))) kind = CodeIndexNote_Type;
else if (string_match(capture_name, SCu8("definition.function"))) kind = CodeIndexNote_Function;
else if (string_match(capture_name, SCu8("definition.method"))) kind = CodeIndexNote_Function;
else if (string_match(capture_name, SCu8("definition.type"))) kind = CodeIndexNote_Type;
Code_Index_Note* note = code_index_new_note(state->index, &state->index_arena, kind, type_range, state->nest_stack_last);
code_index_note_list_hash_append(&state->new_notes, note);
state->last_note_match_id = query_match.id;
}
}
function void
tree_sitter_code_index_update_complete(
Application_Links* app,
Tree_Sitter_Code_Index_Update_State* state,
Arena* scratch,
bool update_was_incremental
){
ProfileScope(app, "Complete Code Index Update");
if (state->ok)
{
Code_Index_Nest* free_nests = state->index->nest_list.first;
block_zero_struct(&state->index->nest_list);
Code_Index_Scope_Delim* delim_at = state->index->scope_delim_list.first;
while (delim_at != 0)
{
bool is_open = (
delim_at->kind == CodeIndexScopeDelim_ScopeOpen ||
delim_at->kind == CodeIndexScopeDelim_ParenOpen ||
delim_at->kind == CodeIndexScopeDelim_BracketOpen
);
if (is_open)
{
if (!free_nests)
{
int count = 32;
free_nests = push_array_zero(&state->index_arena, Code_Index_Nest, count);
for (int i = 0; i < count-2; i++)
{
free_nests[i].next = &free_nests[i+1];
}
}
Code_Index_Nest* nest = free_nests;
if (nest->nest_list.first)
{
nest->nest_list.last->next = nest->next;
free_nests = nest->nest_list.first;
}
else
{
free_nests = nest->next;
}
block_zero_struct(nest);
nest->kind = CodeIndexNest_Scope;
nest->delim = delim_at->kind;
nest->open = delim_at->pos;
nest->close = Ii64(max_i64);
nest->file = state->index;
if (state->nest_stack_last != 0)
{
nest->parent = state->nest_stack_last->nest;
code_index_push_nest(&nest->parent->nest_list, nest);
}
else
{
code_index_push_nest(&state->index->nest_list, nest);
}
Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1);
stack->nest = nest;
stack->prev = state->nest_stack_last;
if (state->nest_stack_last != 0) state->nest_stack_last->next = stack;
else state->nest_stack_first = stack;
state->nest_stack_last = stack;
}
else
{
if (state->nest_stack_last != 0 && delims_are_open_close_pair(state->nest_stack_last->nest->delim, delim_at->kind))
{
Code_Index_Nest* nest = state->nest_stack_last->nest;
nest->close = delim_at->pos;
nest->is_closed = true;
state->nest_stack_last = state->nest_stack_last->prev;
if (state->nest_stack_last != 0) state->nest_stack_last->next = 0;
}
}
delim_at = delim_at->next;
}
{
ProfileScope(app, "Update Global Name Table");
Code_Index_Note* note = state->new_notes.sentinel_first.next_in_hash;
Code_Index_Note* next = 0;
for (int i = 0; i < state->new_notes.count; i++)
{
next = note->next_in_hash;
note->next = 0; note->next_in_hash = 0; note->prev_in_hash = 0;
String_Const_u8 source = string_substring(
state->buffer_contents,
note->pos
);
note->text = alloc_string_copy(
&state->index->string_pool,
source,
&state->index_arena
);
note->file = state->index;
Code_Index_Note_List_New* hash_list = code_index__list_from_string(note->text);
code_index_note_list_hash_append(hash_list, note);
code_index_note_insert(&state->index->note_list, state->last_note, note);
state->last_note = note;
note = next;
}
}
// TODO(PS): This should work similarly to how tree-sitter tree copies work
// - when you take the code_index, its ref count gets incremented
// - if you try to free it, decrement ref count. If ref count = 0, free
// - creating a new file means pushing the old files back one
// this way the memory still exists for existing processes to point to
// but the rest of the application can start using the new file
if (!update_was_incremental)
{
code_index_lock();
code_index_set_file(state->buffer_id, state->index_arena, state->index);
code_index_unlock();
}
buffer_clear_layout_cache(app, state->buffer_id);
}
else
{
linalloc_clear(&state->index_arena);
}
}
function Tree_Sitter_Code_Index_Update_State
tree_sitter_code_index_update_full_file(Application_Links* app, Buffer_ID buffer_id)
{
Scratch_Block scratch(app);
Tree_Sitter_Code_Index_Update_State state = tree_sitter_code_index_update_state_create(
app,
buffer_id,
0,
scratch
);
Tree_Sitter_Query_Cursor query = tree_sitter_query_init(
app,
state.buffer_id,
state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
);
TSQueryMatch query_match;
u32 capture_index;
while (state.ok && tree_sitter_query_continue(&query, &query_match, &capture_index))
{
tree_sitter_code_index_update_process_query_match(
&state,
query,
query_match,
capture_index,
scratch
);
}
tree_sitter_query_end(&query);
tree_sitter_code_index_update_complete(app, &state, scratch, false);
return state;
}
function void
tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data)
{
if (data.size != sizeof(Buffer_ID)) return;
Buffer_ID buffer_id = *(Buffer_ID*)data.str;
Application_Links* app = actx->app;
Scratch_Block scratch(app);
acquire_global_frame_mutex(app);
Tree_Sitter_Code_Index_Update_State state = tree_sitter_code_index_update_state_create(
app,
buffer_id,
0,
scratch
);
Tree_Sitter_Query_Cursor query = tree_sitter_query_init(
app,
state.buffer_id,
state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
);
release_global_frame_mutex(app);
TSQueryMatch query_match;
u32 capture_index;
while (state.ok && tree_sitter_query_continue(&query, &query_match, &capture_index))
{
tree_sitter_code_index_update_process_query_match(&state, query, query_match, capture_index, scratch);
if (async_check_canceled(actx))
{
state.ok = false;
break;
}
}
tree_sitter_query_end(&query);
tree_sitter_code_index_update_complete(app, &state, scratch, false);
}
function void
tree_sitter_code_index_update_tick(Application_Links* app)
{
Scratch_Block scratch(app);
for (Buffer_Modified_Node* modified_node = global_buffer_modified_set.first;
modified_node != 0;
modified_node = modified_node->next
){
ProfileScope(app, "Incremental Parse");
Buffer_ID buffer_id = modified_node->buffer;
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Async_Task* async_parse_task = scope_attachment(app, buffer_scope, buffer_parse_task, Async_Task);
Async_Task* async_code_index_update_task = scope_attachment(app, buffer_scope, buffer_update_code_index_task, Async_Task);
if (async_task_is_running_or_pending(&global_async_system, *async_parse_task)) {
async_task_wait(app, &global_async_system, *async_parse_task);
}
if (async_task_is_running_or_pending(&global_async_system, *async_code_index_update_task)) {
async_task_wait(app, &global_async_system, *async_code_index_update_task);
}
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
if (!tree_data || !tree_data->tree) continue;
Range_i64 old_range = tree_data->last_update_old_range;
Range_i64 new_range = tree_data->last_update_new_range;
if (old_range == new_range) continue;
Scratch_Block scratch(app);
Code_Index_File_Storage* storage = code_index_get_file_storage(buffer_id);
Tree_Sitter_Code_Index_Update_State state = tree_sitter_code_index_update_state_create(
app,
buffer_id,
storage,
scratch
);
// Find the largest, non-root node for us to reparse in its entirety
TSNode root = ts_tree_root_node(tree_data->tree);
TSNode first_node = ts_node_descendant_for_byte_range(
root, new_range.min, new_range.max
);
TSNode last_node = first_node;
bool would_reparse_entire_file = ts_node_eq(first_node, root);
int query_count = 0;
Tree_Sitter_Query_Cursor queries[2];
if (would_reparse_entire_file)
{
TSNode prev_sibling = {0};
TSNode next_sibling = {0};
u32 child_count = ts_node_child_count(root);
for (u32 i = 0; i < child_count; i++)
{
TSNode child = ts_node_child(root, i);
u32 start_byte = ts_node_start_byte(child);
u32 end_byte = ts_node_end_byte(child);
if (end_byte <= new_range.min)
{
prev_sibling = child;
}
else if (start_byte >= new_range.max)
{
next_sibling = child;
break;
}
}
if (ts_node_is_null(prev_sibling) && ts_node_is_null(next_sibling))
{
queries[query_count++] = tree_sitter_query_init(
app,
state.buffer_id,
state.language->queries.ptr[Tree_Sitter_Language_Query_Tags],
root
);
}
else
{
if (!ts_node_is_null(prev_sibling))
{
queries[query_count++] = tree_sitter_query_init(
app,
state.buffer_id,
state.language->queries.ptr[Tree_Sitter_Language_Query_Tags],
prev_sibling
);
}
if (!ts_node_is_null(next_sibling))
{
queries[query_count++] = tree_sitter_query_init(
app,
state.buffer_id,
state.language->queries.ptr[Tree_Sitter_Language_Query_Tags],
next_sibling
);
}
}
}
else
{
TSNode node_at = first_node;
TSNode parent = ts_node_parent(first_node);
while (!ts_node_has_error(parent) && !ts_node_eq(parent, root))
{
node_at = parent;
parent = ts_node_parent(node_at);
}
queries[query_count++] = tree_sitter_query_init(
app,
state.buffer_id,
state.language->queries.ptr[Tree_Sitter_Language_Query_Tags],
node_at
);
}
Range_i64 edit_range;
edit_range.min = (i64)ts_node_start_byte(queries[0].first_node);
edit_range.max = (i64)ts_node_end_byte(queries[query_count-1].first_node);
// Free Scope Delimiters that fall within old_range
{
ProfileScope(app, "Free Scope Delimiters");
Code_Index_Scope_Delim* delim = state.index->scope_delim_list.first;
Code_Index_Scope_Delim* delim_before_range = 0;
Code_Index_Scope_Delim* after_range = 0;
while (delim)
{
Code_Index_Scope_Delim* next = delim->next;
if (delim->pos.min < edit_range.min && delim->pos.max <= edit_range.min)
{
delim_before_range = delim;
}
if (range_overlap(delim->pos, edit_range))
{
state.index->scope_delim_list.count -= 1;
zdll_remove(
state.index->scope_delim_list.first,
state.index->scope_delim_list.last,
delim
);
code_index_free_scope_delim(state.index, delim);
}
if (delim->pos.min >= edit_range.max) break;
delim = next;
}
state.last_delim = delim_before_range;
}
// Free Scope Notes that fall within old_range
{
ProfileScope(app, "Free Notes");
Code_Index_Note* note = state.index->note_list.first;
Code_Index_Note* prev = 0;
Code_Index_Note* note_before_range = 0;
while (note)
{
Code_Index_Note* next = note->next;
if (note->pos.min < edit_range.min && note->pos.max <= edit_range.min)
{
note_before_range = note;
}
if (range_overlap(note->pos, edit_range))
{
Code_Index_Note* new_first = state.index->note_list.first;
Code_Index_Note* new_last = state.index->note_list.last;
Code_Index_Note* new_next = next;
if (note == state.index->note_list.first) new_first = next;
if (note == state.index->note_list.last)
{
if (prev) new_last = prev;
else new_last = 0;
}
if (new_first == 0) Assert(new_last == 0 && state.index->note_list.count == 1);
Code_Index_Note_List_New* list = code_index__list_from_string(note->text);
code_index_note_list_hash_remove(list, note);
free_string(&state.index->string_pool, note->text);
state.index->note_list.count -= 1;
state.index->note_list.first = new_first;
state.index->note_list.last = new_last;
if (prev) prev->next = new_next;
code_index_free_note(state.index, note);
}
else
{
prev = note;
}
if (note->pos.min >= edit_range.max) break;
note = next;
}
state.last_note = note_before_range;
if (state.index->note_list.count == 0) Assert(state.index->note_list.first == 0 && state.index->note_list.first == 0);
}
{
ProfileScope(app, "Perform Query");
for (int i = 0; i < query_count; i++)
{
Tree_Sitter_Query_Cursor query = queries[i];
TSQueryMatch query_match;
u32 capture_index;
while (state.ok && tree_sitter_query_continue(&query, &query_match, &capture_index))
{
tree_sitter_code_index_update_process_query_match(
&state, query, query_match, capture_index, scratch
);
}
tree_sitter_query_end(&query);
}
}
tree_sitter_code_index_update_complete(app, &state, scratch, true);
}
buffer_modified_set_clear();
}
////////////////////////////////////////////////////////////////////
// Token Highlighting
////////////////////////////////////////////////////////////////////
function bool
tree_sitter_highlight_node(
Application_Links* app,
TSQuery* query,
TSNode top_node,
TSQueryCursor* query_cursor,
Text_Layout_ID text_layout_id
){
bool highlighted_any_code = false;
ts_query_cursor_exec(query_cursor, query, top_node);
TSQueryMatch query_match;
u32 capture_index;
while (ts_query_cursor_next_capture(query_cursor, &query_match, &capture_index))
{
TSQueryCapture capture = query_match.captures[capture_index];
TSNode node = capture.node;
u32 length;
const char* tmp = ts_query_capture_name_for_id(query, capture.index, &length);
String_Const_u8 capture_name = SCu8((char*)tmp, length);
// TODO(PS): let each language provide a custom override
String_Const_u8 color_name = SCu8("defcolor_text_default");
if (string_match(capture_name, SCu8("function"))) {
color_name = SCu8("defcolor_function");
} else if (string_match(capture_name, SCu8("type"))) {
color_name = SCu8("defcolor_type");
} else if (string_match(capture_name, SCu8("constant")) || string_match(capture_name, SCu8("number"))) {
color_name = SCu8("defcolor_int_constant");
} else if (string_match(capture_name, SCu8("keyword"))) {
color_name = SCu8("defcolor_keyword");
} else if (string_match(capture_name, SCu8("string"))) {
color_name = SCu8("defcolor_str_constant");
} else if (string_match(capture_name, SCu8("comment"))) {
color_name = SCu8("defcolor_comment");
}
Range_i64 highlight_range = tree_sitter_node_to_range(node);
Managed_ID color_id = managed_id_get(app, SCu8("colors"), color_name);
if (color_id != 0)
{
paint_text_color_fcolor(app, text_layout_id, highlight_range, fcolor_id(color_id));
}
highlighted_any_code = true;
}
return highlighted_any_code;
}
function void
draw_tree_sitter_node_colors(Application_Links* app, Text_Layout_ID text_layout_id, Buffer_ID buffer_id)
{
Tree_Sitter_Language_Definition* lang = tree_sitter_language_for_buffer(app, buffer_id);
if (!lang) return;
TSQuery* query = lang->queries.ptr[Tree_Sitter_Language_Query_Highlights];
Managed_ID color_id = managed_id_get(app, SCu8("colors"), SCu8("defcolor_str_constant"));
Code_Index_File* file = code_index_get_file(buffer_id);
if (file != 0)
{
Code_Index_Scope_Delim* delim = file->scope_delim_list.first;
while (delim != 0)
{
paint_text_color_fcolor(app, text_layout_id, delim->pos, fcolor_id(color_id));
if (delim->next == delim) break;
delim = delim->next;
}
}
Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id);
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
TSTree* tree = tree_sitter_buffer_get_tree_copy(tree_data);
if (tree)
{
TSNode root = ts_tree_root_node(tree);
// Get the smallest node that fully contains the visible range
TSNode visible_container = ts_node_descendant_for_byte_range(root, (u32)visible_range.start, (u32)visible_range.end);
TSQueryCursor* query_cursor = ts_query_cursor_new();
if (ts_node_eq(root, visible_container))
{
TSTreeCursor tree_cursor = ts_tree_cursor_new(visible_container);
if (ts_tree_cursor_goto_first_child_for_byte(&tree_cursor, (u32)visible_range.start) != -1)
{
do {
TSNode node = ts_tree_cursor_current_node(&tree_cursor);
Range_i64 child_range = tree_sitter_node_to_range(node);
if (child_range.start > visible_range.end) break;
tree_sitter_highlight_node(app, query, node, query_cursor, text_layout_id);
} while(ts_tree_cursor_goto_next_sibling(&tree_cursor));
}
else
{
// Pathological case - just highligh the whole document. This is probably bad
tree_sitter_highlight_node(app, query, root, query_cursor, text_layout_id);
}
}
else
{
bool should_ascend = false;
do {
should_ascend = !tree_sitter_highlight_node(app, query, visible_container, query_cursor, text_layout_id);
if (should_ascend) visible_container = ts_node_parent(visible_container);
} while (should_ascend && !ts_node_is_null(visible_container));
}
ts_query_cursor_delete(query_cursor);
}
ts_tree_delete(tree);
}
////////////////////////////////////////////////////////////////////
// Lists
////////////////////////////////////////////////////////////////////
function String_Const_u8
convert_to_single_line_in_place(String_Const_u8 str)
{
String_Const_u8 result = str;
int dst = 0;
for (int src = 0; src < str.size; src++)
{
if (str.str[src] == '\n')
{
while (src < str.size && character_is_whitespace(str.str[src])) { src += 1; }
if (src >= str.size) break;
else {
result.str[dst] = ' ';
dst += 1;
}
}
result.str[dst] = str.str[src];
dst += 1;
}
result.size = dst;
return result;
}
function void
print_position(
Application_Links* app,
Buffer_Insertion* out,
Buffer_ID buffer,
String_Const_u8 buffer_name,
Range_i64 buffer_range,
String_Const_u8 prefix,
Arena* arena,
bool newlines_to_spaces
){
i64 line_number = get_line_number_from_pos(app, buffer, buffer_range.start);
insertf(out, "%.*s:%lld: ", string_expand(buffer_name), line_number);
if (prefix.size > 0) insertf(out, "%.*s ", string_expand(prefix));
Temp_Memory token_temp = begin_temp(arena);
String_Const_u8 line = push_buffer_range(app, arena, buffer, buffer_range);
if (newlines_to_spaces) line = convert_to_single_line_in_place(line);
insert_string(out, line);
end_temp(token_temp);
insertc(out, '\n');
}
function void
tree_sitter_list_all_query_results(
Application_Links *app,
Buffer_ID optional_target_buffer,
Code_Index_Note_Kind note_kind
){
String_Const_u8 decls_name = string_u8_litexpr("*decls*");
Buffer_ID decls_buffer = get_buffer_by_name(app, decls_name, Access_Always);
if (!buffer_exists(app, decls_buffer))
{
decls_buffer = create_buffer(app, decls_name, BufferCreate_AlwaysNew);
buffer_set_setting(app, decls_buffer, BufferSetting_Unimportant, true);
buffer_set_setting(app, decls_buffer, BufferSetting_ReadOnly, true);
}
else
{
clear_buffer(app, decls_buffer);
buffer_send_end_signal(app, decls_buffer);
}
Scratch_Block scratch(app);
Cursor insertion_cursor = make_cursor(push_array(scratch, u8, KB(256)), KB(256));
Buffer_Insertion out = begin_buffer_insertion_at_buffered(app, decls_buffer, 0, &insertion_cursor);
code_index_lock();
for (Buffer_ID buffer_it = get_buffer_next(app, 0, Access_Always);
buffer_it != 0;
buffer_it = get_buffer_next(app, buffer_it, Access_Always))
{
Buffer_ID buffer = buffer_it;
if (optional_target_buffer != 0) buffer = optional_target_buffer;
String_Const_u8 buffer_name = push_buffer_unique_name(app, scratch, buffer);
Code_Index_File* file = code_index_get_file(buffer);
if (file != 0)
{
Code_Index_Note *note = file->note_list.first;
for (i32 i = 0; i < file->note_list.count; i += 1, note = note->next)
{
if (note->note_kind == note_kind)
{
print_position(
app, &out, buffer, buffer_name, note->pos, {}, scratch, true
);
}
}
}
if (optional_target_buffer != 0) break;
}
code_index_unlock();
end_buffer_insertion(&out);
View_ID view = get_active_view(app, Access_Always);
view_set_buffer(app, view, decls_buffer, 0);
lock_jump_buffer(app, decls_name);
}
CUSTOM_COMMAND_SIG(tree_sitter_list_all_functions_current_buffer)
CUSTOM_DOC("Creates a jump list of lines of the current buffer that appear to define or declare functions. Uses tree sitter")
{
View_ID view = get_active_view(app, Access_ReadVisible);
Buffer_ID buffer = view_get_buffer(app, view, Access_ReadVisible);
if (buffer != 0) tree_sitter_list_all_query_results(app, buffer, CodeIndexNote_Function);
}
CUSTOM_UI_COMMAND_SIG(tree_sitter_list_all_functions_current_buffer_lister)
CUSTOM_DOC("Creates a lister of locations that look like function definitions and declarations in the buffer. Uses tree sitter")
{
Heap *heap = &global_heap;
View_ID view = get_active_view(app, Access_ReadVisible);
Buffer_ID buffer = view_get_buffer(app, view, Access_ReadVisible);
if (buffer != 0)
{
tree_sitter_list_all_query_results(app, buffer, CodeIndexNote_Function);
view = get_active_view(app, Access_Always);
buffer = view_get_buffer(app, view, Access_Always);
Marker_List *list = get_or_make_list_for_buffer(app, heap, buffer);
if (list != 0)
{
Jump_Lister_Result jump = get_jump_index_from_user(app, list, "Function:");
jump_to_jump_lister_result(app, view, list, &jump);
}
}
}
CUSTOM_COMMAND_SIG(tree_sitter_list_all_functions_all_buffers)
CUSTOM_DOC("Creates a jump list of lines from all buffers that appear to define or declare functions. Uses tree sitter")
{
tree_sitter_list_all_query_results(app, 0, CodeIndexNote_Function);
}
CUSTOM_UI_COMMAND_SIG(tree_sitter_list_all_functions_all_buffers_lister)
CUSTOM_DOC("Creates a lister of locations that look like function definitions and declarations all buffers. Uses tree sitter")
{
Heap *heap = &global_heap;
tree_sitter_list_all_query_results(app, 0, CodeIndexNote_Function);
View_ID view = get_active_view(app, Access_Always);
Buffer_ID buffer = view_get_buffer(app, view, Access_Always);
Marker_List *list = get_or_make_list_for_buffer(app, heap, buffer);
if (list != 0)
{
Jump_Lister_Result jump = get_jump_index_from_user(app, list, "Function:");
jump_to_jump_lister_result(app, view, list, &jump);
}
}
CUSTOM_COMMAND_SIG(tree_sitter_list_all_types_current_buffer)
CUSTOM_DOC("Creates a jump list of lines of the current buffer that appear to define or declare types. Uses tree sitter")
{
View_ID view = get_active_view(app, Access_ReadVisible);
Buffer_ID buffer = view_get_buffer(app, view, Access_ReadVisible);
if (buffer != 0) tree_sitter_list_all_query_results(app, buffer, CodeIndexNote_Type);
}
CUSTOM_UI_COMMAND_SIG(tree_sitter_list_all_types_current_buffer_lister)
CUSTOM_DOC("Creates a lister of locations that look like function definitions and declarations in the buffer. Uses tree sitter")
{
Heap *heap = &global_heap;
View_ID view = get_active_view(app, Access_ReadVisible);
Buffer_ID buffer = view_get_buffer(app, view, Access_ReadVisible);
if (buffer != 0)
{
tree_sitter_list_all_query_results(app, buffer, CodeIndexNote_Type);
view = get_active_view(app, Access_Always);
buffer = view_get_buffer(app, view, Access_Always);
Marker_List *list = get_or_make_list_for_buffer(app, heap, buffer);
if (list != 0)
{
Jump_Lister_Result jump = get_jump_index_from_user(app, list, "Type:");
jump_to_jump_lister_result(app, view, list, &jump);
}
}
}
CUSTOM_COMMAND_SIG(tree_sitter_list_all_types_all_buffers)
CUSTOM_DOC("Creates a jump list of lines from all buffers that appear to define or declare types. Uses tree sitter")
{
tree_sitter_list_all_query_results(app, 0, CodeIndexNote_Type);
}
CUSTOM_UI_COMMAND_SIG(tree_sitter_list_all_types_all_buffers_lister)
CUSTOM_DOC("Creates a lister of locations that look like type definitions and declarations all buffers. Uses tree sitter")
{
Heap *heap = &global_heap;
tree_sitter_list_all_query_results(app, 0, CodeIndexNote_Type);
View_ID view = get_active_view(app, Access_Always);
Buffer_ID buffer = view_get_buffer(app, view, Access_Always);
Marker_List *list = get_or_make_list_for_buffer(app, heap, buffer);
if (list != 0)
{
Jump_Lister_Result jump = get_jump_index_from_user(app, list, "Type:");
jump_to_jump_lister_result(app, view, list, &jump);
}
}
////////////////////////////////////////////////////////////////////
// DEBUG
////////////////////////////////////////////////////////////////////
char* prefix_buffer = " ";
function void
print_tree_sitter_tree(TSNode cur_node, i32 level, const char* field)
{
TSPoint start = ts_node_start_point(cur_node);
TSPoint end = ts_node_end_point(cur_node);
// + 1 on ts positions becuase the first line/column are zero in treesitter,
printf("%.*s%s: %s [%d, %d] - [%d, %d]\n",
level*2, prefix_buffer, field, ts_node_type(cur_node),
start.row + 1, start.column + 1,
end.row + 1, end.column + 1);
u32 child_count = ts_node_child_count(cur_node);
for (u32 i = 0; i < child_count; ++i)
{
TSNode child = ts_node_child(cur_node, i);
if (ts_node_is_named(child))
{
field = ts_node_field_name_for_child(cur_node, i);
if (!field) field = "";
print_tree_sitter_tree(child, level + 1, field);
}
}
}
function void
write_tree_sitter_tree_to_buffer__inner(Application_Links *app, Arena *arena, Buffer_ID buffer_id,
TSNode cur_node, i32 level = 0, const char *field="")
{
TSPoint start = ts_node_start_point(cur_node);
TSPoint end = ts_node_end_point(cur_node);
// + 1 on ts positions becuase the first line/column are zero in treesitter,
// but 4coder displays as 1 indexed in the filebar.
String_Const_u8 string = push_stringf(arena, "%.*s%s: %s [%d, %d] - [%d, %d]\n",
level*2, prefix_buffer, field, ts_node_type(cur_node),
start.row + 1, start.column + 1,
end.row + 1, end.column + 1);
buffer_replace_range(app, buffer_id, Ii64(buffer_get_size(app, buffer_id)), string);
u32 child_count = ts_node_child_count(cur_node);
for (u32 i = 0; i < child_count; ++i)
{
TSNode child = ts_node_child(cur_node, i);
if (ts_node_is_named(child))
{
field = ts_node_field_name_for_child(cur_node, i);
if (!field) field = "";
write_tree_sitter_tree_to_buffer__inner(app, arena, buffer_id, child, level + 1, field);
}
}
}
CUSTOM_COMMAND_SIG(tree_sitter_write_tree)
CUSTOM_DOC("Write the current buffer's tree sitter tree to *tree*")
{
Scratch_Block scratch(app);
Buffer_ID out_buffer = get_buffer_by_name(app, string_u8_litexpr("*tree*"), Access_Always);
View_ID view = get_active_view(app, Access_Always);
Buffer_ID buffer = view_get_buffer(app, view, Access_Visible);
Managed_Scope scope = buffer_get_managed_scope(app, buffer);
Buffer_Tree_Sitter_Data *tree_data = scope_attachment(app, scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
if (tree_data->tree)
{
TSNode root = ts_tree_root_node(tree_data->tree);
write_tree_sitter_tree_to_buffer__inner(app, scratch, out_buffer, root);
}
}
Reference in New Issue View Git Blame Copy Permalink