From 21a7a58871f86119e838896cc5d19d3fac1e0663 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 1 Feb 2019 14:50:33 -0800 Subject: [PATCH] begining rewrite of token iteration --- 4coder_function_list.cpp | 126 ++++++++++++++++++++++++++++ 4coder_generated/app_functions.h | 7 ++ 4coder_generated/command_metadata.h | 8 +- 4coder_helper.cpp | 107 +++++++++++++++++++++-- 4coder_helper.h | 18 +++- 4coder_scope_commands.h | 3 +- 4ed_api_implementation.cpp | 17 +++- 7 files changed, 270 insertions(+), 16 deletions(-) diff --git a/4coder_function_list.cpp b/4coder_function_list.cpp index 1ac2b6e0..686369bc 100644 --- a/4coder_function_list.cpp +++ b/4coder_function_list.cpp @@ -47,6 +47,131 @@ buffered_write_stream_write(Application_Links *app, Buffered_Write_Stream *strea } } +static Get_Positions_Results +get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t token_index, Function_Positions *positions_array, int32_t positions_max){ + Get_Positions_Results result = {}; + + Token_Range token_range = buffer_get_token_range(app, buffer->buffer_id); + if (token_range.first != 0){ + Token_Iterator token_it = make_token_iterator(token_range, token_index); + + + int32_t nest_level = 0; + int32_t paren_nest_level = 0; + + int32_t first_paren_index = 0; + int32_t first_paren_position = 0; + int32_t last_paren_index = 0; + + // Look for the next token at global scope that might need to be printed. + mode1: + Assert(nest_level == 0); + Assert(paren_nest_level == 0); + first_paren_index = 0; + first_paren_position = 0; + last_paren_index = 0; + for (Cpp_Token *token = token_iterator_current(&token_it); + token != 0; + token = token_iterator_goto_next(&token_it)){ + if (!(token->flags & CPP_TFLAG_PP_BODY)){ + switch (token->type){ + case CPP_TOKEN_BRACE_OPEN: + { + ++nest_level; + }break; + + case CPP_TOKEN_BRACE_CLOSE: + { + if (nest_level > 0){ + --nest_level; + } + }break; + + case CPP_TOKEN_PARENTHESE_OPEN: + { + if (nest_level == 0){ + first_paren_index = token_index; + first_paren_position = token->start; + goto paren_mode1; + } + }break; + } + } + } + goto end; + + // Look for a closing parenthese to mark the end of a function signature. + paren_mode1: + paren_nest_level = 0; + for (Cpp_Token *token = token_iterator_current(&token_it); + token != 0; + token = token_iterator_goto_next(&token_it)){ + if (!(token->flags & CPP_TFLAG_PP_BODY)){ + switch (token->type){ + case CPP_TOKEN_PARENTHESE_OPEN: + { + ++paren_nest_level; + }break; + + case CPP_TOKEN_PARENTHESE_CLOSE: + { + --paren_nest_level; + if (paren_nest_level == 0){ + last_paren_index = token_index; + goto paren_mode2; + } + }break; + } + } + } + goto end; + + // Look backwards from an open parenthese to find the start of a function signature. + paren_mode2: + { + Cpp_Token *restore_point = token_iterator_current(&token_it); + int32_t local_index = first_paren_index; + int32_t signature_start_index = 0; + + for (Cpp_Token *token = token_iterator_current(&token_it); + token != 0; + token = token_iterator_goto_prev(&token_it)){ + if ((token->flags & CPP_TFLAG_PP_BODY) || (token->flags & CPP_TFLAG_PP_DIRECTIVE) || token->type == CPP_TOKEN_BRACE_CLOSE || token->type == CPP_TOKEN_SEMICOLON || token->type == CPP_TOKEN_PARENTHESE_CLOSE){ + ++local_index; + signature_start_index = local_index; + goto paren_mode2_done; + } + } + + // When this loop ends by going all the way back to the beginning set the signature start to 0 and fall through to the printing phase. + signature_start_index = 0; + + paren_mode2_done:; + { + Function_Positions positions; + positions.sig_start_index = signature_start_index; + positions.sig_end_index = last_paren_index; + positions.open_paren_pos = first_paren_position; + positions_array[result.positions_count++] = positions; + } + + token_iterator_set(&token_it, restore_point); + if (result.positions_count >= positions_max){ + result.next_token_index = token_index; + result.still_looping = true; + goto end; + } + + goto mode1; + } + end:; + + } + + return(result); +} + +#if 0 static Get_Positions_Results get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t token_index, Function_Positions *positions_array, int32_t positions_max){ Get_Positions_Results result = {}; @@ -178,6 +303,7 @@ get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t t return(result); } +#endif static void print_positions_buffered(Application_Links *app, Buffer_Summary *buffer, Function_Positions *positions_array, int32_t positions_count, Buffered_Write_Stream *stream){ diff --git a/4coder_generated/app_functions.h b/4coder_generated/app_functions.h index f8dbe133..a1ba16ce 100644 --- a/4coder_generated/app_functions.h +++ b/4coder_generated/app_functions.h @@ -22,6 +22,7 @@ struct Application_Links; #define BUFFER_GET_MANAGED_SCOPE_SIG(n) Managed_Scope n(Application_Links *app, Buffer_ID buffer_id) #define BUFFER_TOKEN_COUNT_SIG(n) int32_t n(Application_Links *app, Buffer_Summary *buffer) #define BUFFER_READ_TOKENS_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out) +#define BUFFER_GET_TOKEN_RANGE_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out) #define BUFFER_GET_TOKEN_INDEX_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result) #define BUFFER_SEND_END_SIGNAL_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer) #define CREATE_BUFFER_SIG(n) Buffer_Summary n(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags) @@ -149,6 +150,7 @@ typedef BUFFER_SET_SETTING_SIG(Buffer_Set_Setting_Function); typedef BUFFER_GET_MANAGED_SCOPE_SIG(Buffer_Get_Managed_Scope_Function); typedef BUFFER_TOKEN_COUNT_SIG(Buffer_Token_Count_Function); typedef BUFFER_READ_TOKENS_SIG(Buffer_Read_Tokens_Function); +typedef BUFFER_GET_TOKEN_RANGE_SIG(Buffer_Get_Token_Range_Function); typedef BUFFER_GET_TOKEN_INDEX_SIG(Buffer_Get_Token_Index_Function); typedef BUFFER_SEND_END_SIGNAL_SIG(Buffer_Send_End_Signal_Function); typedef CREATE_BUFFER_SIG(Create_Buffer_Function); @@ -278,6 +280,7 @@ Buffer_Set_Setting_Function *buffer_set_setting; Buffer_Get_Managed_Scope_Function *buffer_get_managed_scope; Buffer_Token_Count_Function *buffer_token_count; Buffer_Read_Tokens_Function *buffer_read_tokens; +Buffer_Get_Token_Range_Function *buffer_get_token_range; Buffer_Get_Token_Index_Function *buffer_get_token_index; Buffer_Send_End_Signal_Function *buffer_send_end_signal; Create_Buffer_Function *create_buffer; @@ -406,6 +409,7 @@ Buffer_Set_Setting_Function *buffer_set_setting_; Buffer_Get_Managed_Scope_Function *buffer_get_managed_scope_; Buffer_Token_Count_Function *buffer_token_count_; Buffer_Read_Tokens_Function *buffer_read_tokens_; +Buffer_Get_Token_Range_Function *buffer_get_token_range_; Buffer_Get_Token_Index_Function *buffer_get_token_index_; Buffer_Send_End_Signal_Function *buffer_send_end_signal_; Create_Buffer_Function *create_buffer_; @@ -542,6 +546,7 @@ app_links->buffer_set_setting_ = Buffer_Set_Setting;\ app_links->buffer_get_managed_scope_ = Buffer_Get_Managed_Scope;\ app_links->buffer_token_count_ = Buffer_Token_Count;\ app_links->buffer_read_tokens_ = Buffer_Read_Tokens;\ +app_links->buffer_get_token_range_ = Buffer_Get_Token_Range;\ app_links->buffer_get_token_index_ = Buffer_Get_Token_Index;\ app_links->buffer_send_end_signal_ = Buffer_Send_End_Signal;\ app_links->create_buffer_ = Create_Buffer;\ @@ -670,6 +675,7 @@ static bool32 buffer_set_setting(Application_Links *app, Buffer_Summary *buffer, static Managed_Scope buffer_get_managed_scope(Application_Links *app, Buffer_ID buffer_id){return(app->buffer_get_managed_scope(app, buffer_id));} static int32_t buffer_token_count(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_token_count(app, buffer));} static bool32 buffer_read_tokens(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out){return(app->buffer_read_tokens(app, buffer, start_token, end_token, tokens_out));} +static bool32 buffer_get_token_range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out){return(app->buffer_get_token_range(app, buffer, first_token_out, one_past_last_token_out));} static bool32 buffer_get_token_index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result){return(app->buffer_get_token_index(app, buffer, pos, get_result));} static bool32 buffer_send_end_signal(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_send_end_signal(app, buffer));} static Buffer_Summary create_buffer(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags){return(app->create_buffer(app, filename, filename_len, flags));} @@ -798,6 +804,7 @@ static bool32 buffer_set_setting(Application_Links *app, Buffer_Summary *buffer, static Managed_Scope buffer_get_managed_scope(Application_Links *app, Buffer_ID buffer_id){return(app->buffer_get_managed_scope_(app, buffer_id));} static int32_t buffer_token_count(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_token_count_(app, buffer));} static bool32 buffer_read_tokens(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out){return(app->buffer_read_tokens_(app, buffer, start_token, end_token, tokens_out));} +static bool32 buffer_get_token_range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out){return(app->buffer_get_token_range_(app, buffer, first_token_out, one_past_last_token_out));} static bool32 buffer_get_token_index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result){return(app->buffer_get_token_index_(app, buffer, pos, get_result));} static bool32 buffer_send_end_signal(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_send_end_signal_(app, buffer));} static Buffer_Summary create_buffer(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags){return(app->create_buffer_(app, filename, filename_len, flags));} diff --git a/4coder_generated/command_metadata.h b/4coder_generated/command_metadata.h index 67d9a0b8..7b30ca62 100644 --- a/4coder_generated/command_metadata.h +++ b/4coder_generated/command_metadata.h @@ -312,10 +312,10 @@ static Command_Metadata fcoder_metacmd_table[220] = { { PROC_LINKS(interactive_switch_buffer, 0), "interactive_switch_buffer", 25, "Interactively switch to an open buffer.", 39, "w:\\4ed\\code\\4coder_lists.cpp", 28, 765 }, { PROC_LINKS(kill_buffer, 0), "kill_buffer", 11, "Kills the current buffer.", 25, "w:\\4ed\\code\\4coder_base_commands.cpp", 36, 1557 }, { PROC_LINKS(left_adjust_view, 0), "left_adjust_view", 16, "Sets the left size of the view near the x position of the cursor.", 65, "w:\\4ed\\code\\4coder_base_commands.cpp", 36, 133 }, -{ PROC_LINKS(list_all_functions_all_buffers, 0), "list_all_functions_all_buffers", 30, "Creates a jump list of lines from all buffers that appear to define or declare functions.", 89, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 343 }, -{ PROC_LINKS(list_all_functions_all_buffers_lister, 0), "list_all_functions_all_buffers_lister", 37, "Creates a lister of locations that look like function definitions and declarations all buffers.", 95, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 349 }, -{ PROC_LINKS(list_all_functions_current_buffer, 0), "list_all_functions_current_buffer", 33, "Creates a jump list of lines of the current buffer that appear to define or declare functions.", 94, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 320 }, -{ PROC_LINKS(list_all_functions_current_buffer_lister, 0), "list_all_functions_current_buffer_lister", 40, "Creates a lister of locations that look like function definitions and declarations in the buffer.", 97, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 330 }, +{ PROC_LINKS(list_all_functions_all_buffers, 0), "list_all_functions_all_buffers", 30, "Creates a jump list of lines from all buffers that appear to define or declare functions.", 89, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 469 }, +{ PROC_LINKS(list_all_functions_all_buffers_lister, 0), "list_all_functions_all_buffers_lister", 37, "Creates a lister of locations that look like function definitions and declarations all buffers.", 95, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 475 }, +{ PROC_LINKS(list_all_functions_current_buffer, 0), "list_all_functions_current_buffer", 33, "Creates a jump list of lines of the current buffer that appear to define or declare functions.", 94, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 446 }, +{ PROC_LINKS(list_all_functions_current_buffer_lister, 0), "list_all_functions_current_buffer_lister", 40, "Creates a lister of locations that look like function definitions and declarations in the buffer.", 97, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 456 }, { PROC_LINKS(list_all_locations, 0), "list_all_locations", 18, "Queries the user for a string and lists all exact case-sensitive matches found in all open buffers.", 99, "w:\\4ed\\code\\4coder_search.cpp", 29, 769 }, { PROC_LINKS(list_all_locations_case_insensitive, 0), "list_all_locations_case_insensitive", 35, "Queries the user for a string and lists all exact case-insensitive matches found in all open buffers.", 101, "w:\\4ed\\code\\4coder_search.cpp", 29, 783 }, { PROC_LINKS(list_all_locations_of_identifier, 0), "list_all_locations_of_identifier", 32, "Reads a token or word under the cursor and lists all exact case-sensitive mathces in all open buffers.", 102, "w:\\4ed\\code\\4coder_search.cpp", 29, 797 }, diff --git a/4coder_helper.cpp b/4coder_helper.cpp index e26aa74c..853d8de9 100644 --- a/4coder_helper.cpp +++ b/4coder_helper.cpp @@ -924,7 +924,7 @@ backward_stream_chunk(Stream_Chunk *chunk){ //////////////////////////////// static bool32 -init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary *buffer, +init_stream_tokens(Stream_Tokens_DEP *stream, Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Token *data, int32_t count){ bool32 result = false; @@ -955,13 +955,13 @@ init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary return(result); } -static Stream_Tokens -begin_temp_stream_token(Stream_Tokens *stream){ +static Stream_Tokens_DEP +begin_temp_stream_token(Stream_Tokens_DEP *stream){ return(*stream); } static void -end_temp_stream_token(Stream_Tokens *stream, Stream_Tokens temp){ +end_temp_stream_token(Stream_Tokens_DEP *stream, Stream_Tokens_DEP temp){ if (stream->start != temp.start || stream->end != temp.end){ Application_Links *app = stream->app; buffer_read_tokens(app, temp.buffer, temp.start, temp.end, temp.base_tokens); @@ -972,7 +972,7 @@ end_temp_stream_token(Stream_Tokens *stream, Stream_Tokens temp){ } static bool32 -forward_stream_tokens(Stream_Tokens *stream){ +forward_stream_tokens(Stream_Tokens_DEP *stream){ Application_Links *app = stream->app; Buffer_Summary *buffer = stream->buffer; bool32 result = false; @@ -997,7 +997,7 @@ forward_stream_tokens(Stream_Tokens *stream){ } static bool32 -backward_stream_tokens(Stream_Tokens *stream){ +backward_stream_tokens(Stream_Tokens_DEP *stream){ Application_Links *app = stream->app; Buffer_Summary *buffer = stream->buffer; bool32 result = false; @@ -1021,6 +1021,101 @@ backward_stream_tokens(Stream_Tokens *stream){ return(result); } +//////////////////////////////// + +static Token_Range +buffer_get_token_range(Application_Links *app, Buffer_ID buffer_id){ + Token_Range range = {}; + Buffer_Summary buffer = get_buffer(app, buffer_id, AccessAll); + buffer_get_token_range(app, &buffer, &range.first, &range.one_past_last); + return(range); +} + +static Token_Iterator +make_token_iterator(Token_Range range, Cpp_Token *token){ + Token_Iterator iterator = {}; + if (range.first != 0 && range.one_past_last != 0){ + if (token == 0 || token < range.first){ + token = range.first; + } + if (token > range.one_past_last){ + token = range.one_past_last; + } + iterator.token = token; + iterator.range = range; + } + return(iterator); +} + +static Token_Iterator +make_token_iterator(Token_Range range, int32_t index){ + return(make_token_iterator(range, range.first + index)); +} + +static void +token_iterator_set(Token_Iterator *iterator, Cpp_Token *token){ + *iterator = make_token_iterator(iterator->range, token); +} + +static Cpp_Token* +token_iterator_current(Token_Iterator *iterator){ + Cpp_Token *token = iterator->token; + if (token < iterator->range.first || iterator->range.one_past_last <= token){ + token = 0; + } + return(token); +} + +static int32_t +token_iterator_current_index(Token_Iterator *iterator){ + int32_t index = -1; + Cpp_Token *token = token_iterator_current(iterator); + if (token != 0 && iterator->range.first <= token && token <= iterator->range.one_past_last){ + index = (int32_t)(token - iterator->range.first); + } + return(index); +} + +static Cpp_Token* +token_iterator_goto_next(Token_Iterator *iterator){ + Cpp_Token *token = iterator->token; + Cpp_Token *one_past_last = iterator->range.one_past_last; + for (token += 1; token < one_past_last; token += 1){ + if (token->type != CPP_TOKEN_COMMENT){ + break; + } + } + *iterator = make_token_iterator(iterator->range, token); + return(token_iterator_current(iterator)); +} + +static Cpp_Token* +token_iterator_goto_next_raw(Token_Iterator *iterator){ + *iterator = make_token_iterator(iterator->range, iterator->token + 1); + return(token_iterator_current(iterator)); +} + +static Cpp_Token* +token_iterator_goto_prev(Token_Iterator *iterator){ + Cpp_Token *token = iterator->token; + Cpp_Token *first = iterator->range.first; + for (token -= 1; token > first; token -= 1){ + if (token->type != CPP_TOKEN_COMMENT){ + break; + } + } + *iterator = make_token_iterator(iterator->range, token); + return(token_iterator_current(iterator)); +} + +static Cpp_Token* +token_iterator_goto_prev_raw(Token_Iterator *iterator){ + *iterator = make_token_iterator(iterator->range, iterator->token - 1); + return(token_iterator_current(iterator)); +} + +//////////////////////////////// + static String get_query_string(Application_Links *app, char *query_str, char *string_space, int32_t space_size){ Query_Bar bar; diff --git a/4coder_helper.h b/4coder_helper.h index 70c31930..0efed944 100644 --- a/4coder_helper.h +++ b/4coder_helper.h @@ -115,14 +115,26 @@ struct Stream_Chunk{ char *data; }; -struct Stream_Tokens{ +struct Stream_Tokens_DEP{ Application_Links *app; Buffer_Summary *buffer; Cpp_Token *base_tokens; Cpp_Token *tokens; - int32_t start, end; - int32_t count, token_count; + int32_t start; + int32_t end; + int32_t count; + int32_t token_count; +}; + +struct Token_Range{ + Cpp_Token *first; + Cpp_Token *one_past_last; +}; + +struct Token_Iterator{ + Cpp_Token *token; + Token_Range range; }; //////////////////////////////// diff --git a/4coder_scope_commands.h b/4coder_scope_commands.h index d0db9ca0..3a27d605 100644 --- a/4coder_scope_commands.h +++ b/4coder_scope_commands.h @@ -16,8 +16,7 @@ enum{ }; struct Statement_Parser{ - Stream_Tokens stream; - int32_t token_index; + Token_Iterator token_iterator; Buffer_Summary *buffer; }; diff --git a/4ed_api_implementation.cpp b/4ed_api_implementation.cpp index 0afdbbd9..39d88208 100644 --- a/4ed_api_implementation.cpp +++ b/4ed_api_implementation.cpp @@ -1063,7 +1063,7 @@ The number of output tokens will be end_token - start_token.) Editing_File *file = imp_get_file(models, buffer); Cpp_Token_Array token_array = file->state.token_array; bool32 result = false; - if (file && token_array.tokens && file->state.tokens_complete){ + if (file != 0 && token_array.tokens != 0 && file->state.tokens_complete){ if (0 <= start_token && start_token <= end_token && end_token <= token_array.count){ result = true; memcpy(tokens_out, token_array.tokens + start_token, sizeof(Cpp_Token)*(end_token - start_token)); @@ -1072,6 +1072,21 @@ The number of output tokens will be end_token - start_token.) return(result); } +API_EXPORT bool32 +Buffer_Get_Token_Range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out) +{ + Models *models = (Models*)app->cmd_context; + Editing_File *file = imp_get_file(models, buffer); + Cpp_Token_Array token_array = file->state.token_array; + bool32 result = false; + if (file != 0 && token_array.tokens != 0 && file->state.tokens_complete){ + result = true; + *first_token_out = token_array.tokens; + *one_past_last_token_out = token_array.tokens + token_array.count; + } + return(result); +} + API_EXPORT bool32 Buffer_Get_Token_Index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result) /*