ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Parse delimiters before creating the nests tree

This commit is contained in:
Peter Slattery 2025-07-30 15:38:49 -07:00
parent ee082e42d1
commit 1dfc106e8c
4 changed files with 245 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
code/custom/4coder_code_index.cpp
View File

@ -6,6 +6,15 @@
global Code_Index global_code_index = {};
function bool
delims_are_open_close_pair(Code_Index_Scope_Delim_Kind a, Code_Index_Scope_Delim_Kind b)
{
if (a == CodeIndexScopeDelim_ScopeOpen) return b == CodeIndexScopeDelim_ScopeClose;
if (a == CodeIndexScopeDelim_ParenOpen) return b == CodeIndexScopeDelim_ParenClose;
if (a == CodeIndexScopeDelim_BracketOpen) return b == CodeIndexScopeDelim_BracketClose;
return false;
}
////////////////////////////////
// NOTE(allen): Lookups
@ -196,19 +205,26 @@ code_index_erase_file(Buffer_ID buffer){
}
}
function Code_Index_File*
code_index_get_file(Buffer_ID buffer){
Code_Index_File *result = 0;
function Code_Index_File_Storage*
code_index_get_file_storage(Buffer_ID buffer){
Code_Index_File_Storage *result = 0;
Table_Lookup lookup = table_lookup(&global_code_index.buffer_to_index_file, buffer);
if (lookup.found_match){
u64 val = 0;
table_read(&global_code_index.buffer_to_index_file, lookup, &val);
Code_Index_File_Storage *storage = (Code_Index_File_Storage*)IntAsPtr(val);
result = storage->file;
result = (Code_Index_File_Storage*)IntAsPtr(val);
}
return(result);
}
function Code_Index_File*
code_index_get_file(Buffer_ID buffer){
Code_Index_File *result = 0;
Code_Index_File_Storage *storage = code_index_get_file_storage(buffer);
if (storage) result = storage->file;
return result;
}
function void
index_shift(i64 *ptr, Range_i64 old_range, u64 new_size){
i64 i = *ptr;

34
code/custom/4coder_code_index.h
View File

@ -7,6 +7,35 @@
#if !defined(FCODER_CODE_INDEX_H)
#define FCODER_CODE_INDEX_H
typedef i32 Code_Index_Scope_Delim_Kind;
enum {
CodeIndexScopeDelim_ScopeOpen,
CodeIndexScopeDelim_ScopeClose,
CodeIndexScopeDelim_ParenOpen,
CodeIndexScopeDelim_ParenClose,
CodeIndexScopeDelim_BracketOpen,
CodeIndexScopeDelim_BracketClose
};
struct Code_Index_Scope_Delim {
Code_Index_Scope_Delim_Kind kind;
i32 depth;
Range_i64 pos;
Code_Index_Scope_Delim* next;
Code_Index_Scope_Delim* prev;
};
struct Code_Index_Scope_Delim_List {
Code_Index_Scope_Delim* first;
Code_Index_Scope_Delim* last;
};
struct Code_Index_Scope_Delim_Ptr_Array {
Code_Index_Scope_Delim** ptrs;
i32 count;
};
struct Code_Index_Nest_List{
struct Code_Index_Nest *first;
struct Code_Index_Nest *last;
@ -29,7 +58,10 @@ enum{
struct Code_Index_Nest{
Code_Index_Nest *next;
// TODO(PS): kind and delim are redundant, I don't want to break virtual indentation just yet
// so Im leaving them both here.
Code_Index_Nest_Kind kind;
Code_Index_Scope_Delim_Kind delim;
b32 is_closed;
Range_i64 open;
Range_i64 close;
@ -73,6 +105,8 @@ struct Code_Index_Note_Ptr_Array{
};
struct Code_Index_File{
Code_Index_Scope_Delim_List scope_delim_list;
Code_Index_Scope_Delim_Ptr_Array scope_delim_array;
Code_Index_Nest_List nest_list;
Code_Index_Nest_Ptr_Array nest_array;
Code_Index_Note_List note_list;

247
code/custom/4coder_tree_sitter.cpp
View File

@ -410,40 +410,31 @@ tree_sitter_code_index_update_process_query_match(
const char* tmp = ts_query_capture_name_for_id(state->query.query, type_capture.index, &length);
String_Const_u8 capture_name = SCu8((char*)tmp, length);
if (string_match(capture_name, SCu8("scope_begin")))
if (string_match(capture_name, SCu8("scope_begin")) || string_match(capture_name, SCu8("scope_end")))
{
Code_Index_Nest* nest = push_array_zero(&state->index_arena, Code_Index_Nest, 1);
nest->kind = CodeIndexNest_Scope;
nest->open = type_range;
nest->close = Ii64(max_i64);
nest->file = state->index;
if (state->nest_stack_last != 0)
Code_Index_Scope_Delim_Kind kind;
String_Const_u8 delim_char = string_substring(state->buffer_contents, type_range);
bool skip = false;
if (delim_char.str[0] == '{') kind = CodeIndexScopeDelim_ScopeOpen;
else if (delim_char.str[0] == '}') kind = CodeIndexScopeDelim_ScopeClose;
else if (delim_char.str[0] == '(') kind = CodeIndexScopeDelim_ParenOpen;
else if (delim_char.str[0] == ')') kind = CodeIndexScopeDelim_ParenClose;
else if (delim_char.str[0] == '[') kind = CodeIndexScopeDelim_BracketOpen;
else if (delim_char.str[0] == ']') kind = CodeIndexScopeDelim_BracketClose;
else skip = true;
if (!skip)
{
nest->parent = state->nest_stack_last->nest;
code_index_push_nest(&nest->parent->nest_list, nest);
Code_Index_Scope_Delim* delim = push_array_zero(&state->index_arena, Code_Index_Scope_Delim, 1);
delim->pos = type_range;
delim->kind = kind;
Code_Index_Scope_Delim_List* list = &state->index->scope_delim_list;
if (!list->first) list->first = delim;
if (list->last) list->last->next = delim;
delim->prev = list->last;
list->last = delim;
}
Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1);
stack->nest = nest;
stack->prev = state->nest_stack_last;
stack->match_id = query_match.id;
if (state->nest_stack_last != 0) state->nest_stack_last->next = stack;
else state->nest_stack_first = stack;
state->nest_stack_last = stack;
}
else if (string_match(capture_name, SCu8("scope_end")))
{
Assert(state->nest_stack_last != 0);
Assert(state->nest_stack_last->match_id == query_match.id);
Code_Index_Nest* nest = state->nest_stack_last->nest;
nest->close = type_range;
nest->is_closed = true;
nest->nest_array = code_index_nest_ptr_array_from_list(&state->index_arena, &nest->nest_list);
if (nest->parent == 0) code_index_push_nest(&state->index->nest_list, nest);
state->nest_stack_last = state->nest_stack_last->prev;
if (state->nest_stack_last != 0) state->nest_stack_last->next = 0;
}
else if (string_match(capture_name, SCu8("definition.class")))
{
@ -477,16 +468,89 @@ tree_sitter_code_index_update_process_query_match(
function void
tree_sitter_code_index_update_complete(
Application_Links* app,
Tree_Sitter_Code_Index_Update_State* state
Tree_Sitter_Code_Index_Update_State* state,
Arena* scratch
){
tree_sitter_query_end(&state->query);
if (state->ok)
{
Code_Index_Nest* free_nests = state->index->nest_list.first;
state->index->nest_list.first = 0;
state->index->nest_list.last = 0;
Code_Index_Scope_Delim* delim_at = state->index->scope_delim_list.first;
while (delim_at != 0)
{
bool is_open = (
delim_at->kind == CodeIndexScopeDelim_ScopeOpen ||
delim_at->kind == CodeIndexScopeDelim_ParenOpen ||
delim_at->kind == CodeIndexScopeDelim_BracketOpen
);
if (is_open)
{
if (!free_nests)
{
int count = 32;
free_nests = push_array_zero(&state->index_arena, Code_Index_Nest, count);
for (int i = 0; i < count-2; i++)
{
free_nests[i].next = &free_nests[i+1];
}
}
Code_Index_Nest* nest = free_nests;
if (nest->nest_list.first)
{
nest->nest_list.last->next = free_nests;
free_nests = nest->nest_list.first;
}
else
{
free_nests = nest->next;
}
nest->kind = CodeIndexNest_Scope;
nest->delim = delim_at->kind;
nest->open = delim_at->pos;
nest->close = Ii64(max_i64);
nest->file = state->index;
if (state->nest_stack_last != 0)
{
nest->parent = state->nest_stack_last->nest;
code_index_push_nest(&nest->parent->nest_list, nest);
}
else
{
code_index_push_nest(&state->index->nest_list, nest);
}
Code_Index_Nest_Stack* stack = push_array_zero(scratch, Code_Index_Nest_Stack, 1);
stack->nest = nest;
stack->prev = state->nest_stack_last;
if (state->nest_stack_last != 0) state->nest_stack_last->next = stack;
else state->nest_stack_first = stack;
state->nest_stack_last = stack;
}
else
{
if (state->nest_stack_last != 0 && delims_are_open_close_pair(state->nest_stack_last->nest->delim, delim_at->kind))
{
Code_Index_Nest* nest = state->nest_stack_last->nest;
nest->close = delim_at->pos;
nest->is_closed = true;
nest->nest_array = code_index_nest_ptr_array_from_list(&state->index_arena, &nest->nest_list);
state->nest_stack_last = state->nest_stack_last->prev;
if (state->nest_stack_last != 0) state->nest_stack_last->next = 0;
}
}
delim_at = delim_at->next;
}
while (state->nest_stack_last != 0)
{
Code_Index_Nest* nest = state->nest_stack_last->nest;
if (nest->parent != 0) code_index_push_nest(&nest->parent->nest_list, nest);
else code_index_push_nest(&state->index->nest_list, nest);
nest->nest_array = code_index_nest_ptr_array_from_list(&state->index_arena, &nest->nest_list);
state->nest_stack_last = state->nest_stack_last->prev;
}
@ -500,7 +564,6 @@ tree_sitter_code_index_update_complete(
state->index->nest_array = code_index_nest_ptr_array_from_list(&state->index_arena, &state->index->nest_list);
state->index->note_array = code_index_note_ptr_array_from_list(&state->index_arena, &state->index->note_list);
code_index_lock();
code_index_set_file(state->buffer_id, state->index_arena, state->index);
code_index_unlock();
@ -513,36 +576,6 @@ tree_sitter_code_index_update_complete(
}
}
function Tree_Sitter_Code_Index_Update_State
tree_sitter_code_index_update_full_file_async(Async_Context* actx, Buffer_ID buffer_id)
{
Application_Links* app = actx->app;
Scratch_Block scratch(app);
acquire_global_frame_mutex(app);
Tree_Sitter_Code_Index_Update_State state = tree_sitter_code_index_update_state_create(
app,
buffer_id,
scratch
);
release_global_frame_mutex(app);
TSQueryMatch query_match;
u32 capture_index;
while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index))
{
tree_sitter_code_index_update_process_query_match(&state, query_match, capture_index, scratch);
if (async_check_canceled(actx))
{
state.ok = false;
break;
}
}
tree_sitter_code_index_update_complete(app, &state);
return state;
}
function Tree_Sitter_Code_Index_Update_State
tree_sitter_code_index_update_full_file(Application_Links* app, Buffer_ID buffer_id)
{
@ -566,7 +599,7 @@ tree_sitter_code_index_update_full_file(Application_Links* app, Buffer_ID buffer
);
}
tree_sitter_code_index_update_complete(app, &state);
tree_sitter_code_index_update_complete(app, &state, scratch);
return state;
}
@ -575,9 +608,72 @@ tree_sitter_code_index_update_async(Async_Context* actx, String_Const_u8 data)
{
if (data.size != sizeof(Buffer_ID)) return;
Buffer_ID buffer_id = *(Buffer_ID*)data.str;
tree_sitter_code_index_update_full_file_async(actx, buffer_id);
Application_Links* app = actx->app;
Scratch_Block scratch(app);
acquire_global_frame_mutex(app);
Tree_Sitter_Code_Index_Update_State state = tree_sitter_code_index_update_state_create(
app,
buffer_id,
scratch
);
release_global_frame_mutex(app);
TSQueryMatch query_match;
u32 capture_index;
while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index))
{
tree_sitter_code_index_update_process_query_match(&state, query_match, capture_index, scratch);
if (async_check_canceled(actx))
{
state.ok = false;
break;
}
}
tree_sitter_code_index_update_complete(app, &state, scratch);
}
/*
function void
tree_sitter_code_index_update_incremental(Application_Links* app, Buffer_ID buffer_id)
{
// TODO(PS): @Collapse with tree_sitter_code_index_update_state_create
Tree_Sitter_Code_Index_Update_State state = {};
state.ok = false;
state.buffer_id = buffer_id;
state.language = tree_sitter_language_for_buffer(app, state.buffer_id);
if (!state.language) return state;
Code_Index_File_Storage* storage = code_index_get_file_storage(state.buffer_id);
state.index_arena = storage->arena;
state.index = storage->file;
state.buffer_contents = push_whole_buffer(app, scratch, state.buffer_id);
state.query = tree_sitter_query_init(
app, state.buffer_id, state.language->queries.ptr[Tree_Sitter_Language_Query_Tags]
);
state.nest_stack_first = 0;
state.nest_stack_last = 0;
state.last_note = 0;
state.last_note_match_id = max_u32;
state.ok = true;
TSQueryMatch query_match;
u32 capture_index;
while (state.ok && tree_sitter_query_continue(&state.query, &query_match, &capture_index))
{
tree_sitter_code_index_update_process_query_match(
&state,
query_match,
capture_index,
scratch
);
}
tree_sitter_code_index_update_complete(app, &state, scratch);
}
*/
function void
tree_sitter_code_index_update_tick(Application_Links* app)
{
@ -591,7 +687,11 @@ tree_sitter_code_index_update_tick(Application_Links* app)
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);
if (!tree_data || !tree_data->tree) continue;
#if 1
tree_sitter_code_index_update_full_file(app, buffer_id);
#else
tree_sitter_code_index_update_incremental(app, buffer_id);
#endif
}
buffer_modified_set_clear();
}
@ -658,6 +758,19 @@ draw_tree_sitter_node_colors(Application_Links* app, Text_Layout_ID text_layout_
if (!lang) return;
TSQuery* query = lang->queries.ptr[Tree_Sitter_Language_Query_Highlights];
Managed_ID color_id = managed_id_get(app, SCu8("colors"), SCu8("defcolor_str_constant"));
Code_Index_File* file = code_index_get_file(buffer_id);
if (file != 0)
{
Code_Index_Scope_Delim* delim = file->scope_delim_list.first;
while (delim != 0)
{
paint_text_color_fcolor(app, text_layout_id, delim->pos, fcolor_id(color_id));
if (delim->next == delim) break;
delim = delim->next;
}
}
Range_i64 visible_range = text_layout_get_visible_range(app, text_layout_id);
Managed_Scope buffer_scope = buffer_get_managed_scope(app, buffer_id);
Buffer_Tree_Sitter_Data* tree_data = scope_attachment(app, buffer_scope, buffer_tree_sitter_data_id, Buffer_Tree_Sitter_Data);

13
code/custom/languages/tree_sitter_cpp.h
View File

@ -21,9 +21,16 @@ String_Const_u8 TS_CPP_TAGS_QUERY_SCM = string_u8_litexpr(R"DONE(
(class_specifier name: (type_identifier) @name) @definition.class
(_ "{" @scope_begin "}" @scope_end )
(_ "(" @scope_begin ")" @scope_end )
(_ "[" @scope_begin "]" @scope_end )
"{" @scope_begin
"(" @scope_begin
"[" @scope_begin
"}" @scope_end
")" @scope_end
"]" @scope_end
; (_ "{" @scope_begin "}" @scope_end )
; (_ "(" @scope_begin ")" @scope_end )
; (_ "[" @scope_begin "]" @scope_end )
)DONE");