diff --git a/4coder_base_types.cpp b/4coder_base_types.cpp index cd861988..30a5f046 100644 --- a/4coder_base_types.cpp +++ b/4coder_base_types.cpp @@ -2751,6 +2751,23 @@ data_match(Data a, Data b){ //////////////////////////////// +internal b32 +character_is_basic_ascii(char c){ + return(' ' <= c && c <= '~'); +} +internal b32 +character_is_basic_ascii(u8 c){ + return(' ' <= c && c <= '~'); +} +internal b32 +character_is_basic_ascii(u16 c){ + return(' ' <= c && c <= '~'); +} +internal b32 +character_is_basic_ascii(u32 c){ + return(' ' <= c && c <= '~'); +} + internal b32 character_is_slash(char c){ return((c == '/') || (c == '\\')); diff --git a/4coder_default_hooks.cpp b/4coder_default_hooks.cpp index 31651cce..1d93e10d 100644 --- a/4coder_default_hooks.cpp +++ b/4coder_default_hooks.cpp @@ -4,10 +4,9 @@ // TOP -#include "languages/4coder_language_cpp.h" -#include "languages/4coder_language_rust.h" -#include "languages/4coder_language_cs.h" -#include "languages/4coder_language_java.h" +#include "languages/generated_lexer_cpp.h" + +#include "languages/generated_lexer_cpp.cpp" CUSTOM_COMMAND_SIG(set_bindings_choose); CUSTOM_COMMAND_SIG(set_bindings_default); diff --git a/4coder_default_include.cpp b/4coder_default_include.cpp index 18bd0ecf..0acfeb98 100644 --- a/4coder_default_include.cpp +++ b/4coder_default_include.cpp @@ -45,6 +45,7 @@ #include "4coder_string_match.cpp" #include "4coder_buffer_seek_constructors.cpp" +#include "4coder_token.cpp" #include "4coder_default_framework_variables.cpp" #include "4coder_helper.cpp" #include "4coder_seek.cpp" diff --git a/4coder_generated/command_metadata.h b/4coder_generated/command_metadata.h index 100127e4..5fee8981 100644 --- a/4coder_generated/command_metadata.h +++ b/4coder_generated/command_metadata.h @@ -2,24 +2,723 @@ #define command_id(c) (fcoder_metacmd_ID_##c) #define command_metadata(c) (&fcoder_metacmd_table[command_id(c)]) #define command_metadata_by_id(id) (&fcoder_metacmd_table[id]) -#define command_one_past_last_id 0 +#define command_one_past_last_id 233 #if defined(CUSTOM_COMMAND_SIG) #define PROC_LINKS(x,y) x #else #define PROC_LINKS(x,y) y #endif #if defined(CUSTOM_COMMAND_SIG) +CUSTOM_COMMAND_SIG(write_explicit_enum_flags); +CUSTOM_COMMAND_SIG(seek_beginning_of_textual_line); +CUSTOM_COMMAND_SIG(seek_end_of_textual_line); +CUSTOM_COMMAND_SIG(seek_beginning_of_line); +CUSTOM_COMMAND_SIG(seek_end_of_line); +CUSTOM_COMMAND_SIG(goto_beginning_of_file); +CUSTOM_COMMAND_SIG(goto_end_of_file); +CUSTOM_COMMAND_SIG(change_active_panel); +CUSTOM_COMMAND_SIG(change_active_panel_backwards); +CUSTOM_COMMAND_SIG(open_panel_vsplit); +CUSTOM_COMMAND_SIG(open_panel_hsplit); +CUSTOM_COMMAND_SIG(suppress_mouse); +CUSTOM_COMMAND_SIG(allow_mouse); +CUSTOM_COMMAND_SIG(toggle_mouse); +CUSTOM_COMMAND_SIG(set_mode_to_original); +CUSTOM_COMMAND_SIG(set_mode_to_notepad_like); +CUSTOM_COMMAND_SIG(toggle_highlight_line_at_cursor); +CUSTOM_COMMAND_SIG(toggle_highlight_enclosing_scopes); +CUSTOM_COMMAND_SIG(toggle_paren_matching_helper); +CUSTOM_COMMAND_SIG(toggle_fullscreen); +CUSTOM_COMMAND_SIG(remap_interactive); +CUSTOM_COMMAND_SIG(write_character); +CUSTOM_COMMAND_SIG(write_underscore); +CUSTOM_COMMAND_SIG(delete_char); +CUSTOM_COMMAND_SIG(backspace_char); +CUSTOM_COMMAND_SIG(set_mark); +CUSTOM_COMMAND_SIG(cursor_mark_swap); +CUSTOM_COMMAND_SIG(delete_range); +CUSTOM_COMMAND_SIG(backspace_alpha_numeric_boundary); +CUSTOM_COMMAND_SIG(delete_alpha_numeric_boundary); +CUSTOM_COMMAND_SIG(snipe_backward_whitespace_or_token_boundary); +CUSTOM_COMMAND_SIG(snipe_forward_whitespace_or_token_boundary); +CUSTOM_COMMAND_SIG(center_view); +CUSTOM_COMMAND_SIG(left_adjust_view); +CUSTOM_COMMAND_SIG(click_set_cursor_and_mark); +CUSTOM_COMMAND_SIG(click_set_cursor); +CUSTOM_COMMAND_SIG(click_set_cursor_if_lbutton); +CUSTOM_COMMAND_SIG(click_set_mark); +CUSTOM_COMMAND_SIG(mouse_wheel_scroll); +CUSTOM_COMMAND_SIG(move_up); +CUSTOM_COMMAND_SIG(move_down); +CUSTOM_COMMAND_SIG(move_up_10); +CUSTOM_COMMAND_SIG(move_down_10); +CUSTOM_COMMAND_SIG(move_down_textual); +CUSTOM_COMMAND_SIG(page_up); +CUSTOM_COMMAND_SIG(page_down); +CUSTOM_COMMAND_SIG(move_up_to_blank_line); +CUSTOM_COMMAND_SIG(move_down_to_blank_line); +CUSTOM_COMMAND_SIG(move_up_to_blank_line_skip_whitespace); +CUSTOM_COMMAND_SIG(move_down_to_blank_line_skip_whitespace); +CUSTOM_COMMAND_SIG(move_up_to_blank_line_end); +CUSTOM_COMMAND_SIG(move_down_to_blank_line_end); +CUSTOM_COMMAND_SIG(move_left); +CUSTOM_COMMAND_SIG(move_right); +CUSTOM_COMMAND_SIG(move_right_whitespace_boundary); +CUSTOM_COMMAND_SIG(move_left_whitespace_boundary); +CUSTOM_COMMAND_SIG(move_right_token_boundary); +CUSTOM_COMMAND_SIG(move_left_token_boundary); +CUSTOM_COMMAND_SIG(move_right_whitespace_or_token_boundary); +CUSTOM_COMMAND_SIG(move_left_whitespace_or_token_boundary); +CUSTOM_COMMAND_SIG(move_right_alpha_numeric_boundary); +CUSTOM_COMMAND_SIG(move_left_alpha_numeric_boundary); +CUSTOM_COMMAND_SIG(move_right_alpha_numeric_or_camel_boundary); +CUSTOM_COMMAND_SIG(move_left_alpha_numeric_or_camel_boundary); +CUSTOM_COMMAND_SIG(select_all); +CUSTOM_COMMAND_SIG(to_uppercase); +CUSTOM_COMMAND_SIG(to_lowercase); +CUSTOM_COMMAND_SIG(clean_all_lines); +CUSTOM_COMMAND_SIG(basic_change_active_panel); +CUSTOM_COMMAND_SIG(close_panel); +CUSTOM_COMMAND_SIG(show_scrollbar); +CUSTOM_COMMAND_SIG(hide_scrollbar); +CUSTOM_COMMAND_SIG(show_filebar); +CUSTOM_COMMAND_SIG(hide_filebar); +CUSTOM_COMMAND_SIG(toggle_filebar); +CUSTOM_COMMAND_SIG(toggle_line_wrap); +CUSTOM_COMMAND_SIG(toggle_fps_meter); +CUSTOM_COMMAND_SIG(increase_line_wrap); +CUSTOM_COMMAND_SIG(decrease_line_wrap); +CUSTOM_COMMAND_SIG(increase_face_size); +CUSTOM_COMMAND_SIG(decrease_face_size); +CUSTOM_COMMAND_SIG(mouse_wheel_change_face_size); +CUSTOM_COMMAND_SIG(toggle_virtual_whitespace); +CUSTOM_COMMAND_SIG(toggle_show_whitespace); +CUSTOM_COMMAND_SIG(toggle_line_numbers); +CUSTOM_COMMAND_SIG(eol_dosify); +CUSTOM_COMMAND_SIG(eol_nixify); +CUSTOM_COMMAND_SIG(exit_4coder); +CUSTOM_COMMAND_SIG(goto_line); +CUSTOM_COMMAND_SIG(search); +CUSTOM_COMMAND_SIG(reverse_search); +CUSTOM_COMMAND_SIG(search_identifier); +CUSTOM_COMMAND_SIG(reverse_search_identifier); +CUSTOM_COMMAND_SIG(replace_in_range); +CUSTOM_COMMAND_SIG(replace_in_buffer); +CUSTOM_COMMAND_SIG(replace_in_all_buffers); +CUSTOM_COMMAND_SIG(query_replace); +CUSTOM_COMMAND_SIG(query_replace_identifier); +CUSTOM_COMMAND_SIG(query_replace_selection); +CUSTOM_COMMAND_SIG(save_all_dirty_buffers); +CUSTOM_COMMAND_SIG(delete_file_query); +CUSTOM_COMMAND_SIG(save_to_query); +CUSTOM_COMMAND_SIG(rename_file_query); +CUSTOM_COMMAND_SIG(make_directory_query); +CUSTOM_COMMAND_SIG(move_line_up); +CUSTOM_COMMAND_SIG(move_line_down); +CUSTOM_COMMAND_SIG(duplicate_line); +CUSTOM_COMMAND_SIG(delete_line); +CUSTOM_COMMAND_SIG(open_file_in_quotes); +CUSTOM_COMMAND_SIG(open_matching_file_cpp); +CUSTOM_COMMAND_SIG(view_buffer_other_panel); +CUSTOM_COMMAND_SIG(swap_buffers_between_panels); +CUSTOM_COMMAND_SIG(kill_buffer); +CUSTOM_COMMAND_SIG(save); +CUSTOM_COMMAND_SIG(reopen); +CUSTOM_COMMAND_SIG(undo); +CUSTOM_COMMAND_SIG(redo); +CUSTOM_COMMAND_SIG(undo_all_buffers); +CUSTOM_COMMAND_SIG(redo_all_buffers); +CUSTOM_COMMAND_SIG(open_in_other); +CUSTOM_COMMAND_SIG(lister__quit); +CUSTOM_COMMAND_SIG(lister__activate); +CUSTOM_COMMAND_SIG(lister__write_character); +CUSTOM_COMMAND_SIG(lister__backspace_text_field); +CUSTOM_COMMAND_SIG(lister__move_up); +CUSTOM_COMMAND_SIG(lister__move_down); +CUSTOM_COMMAND_SIG(lister__wheel_scroll); +CUSTOM_COMMAND_SIG(lister__mouse_press); +CUSTOM_COMMAND_SIG(lister__mouse_release); +CUSTOM_COMMAND_SIG(lister__repaint); +CUSTOM_COMMAND_SIG(lister__write_character__default); +CUSTOM_COMMAND_SIG(lister__backspace_text_field__default); +CUSTOM_COMMAND_SIG(lister__move_up__default); +CUSTOM_COMMAND_SIG(lister__move_down__default); +CUSTOM_COMMAND_SIG(lister__write_character__file_path); +CUSTOM_COMMAND_SIG(lister__backspace_text_field__file_path); +CUSTOM_COMMAND_SIG(lister__write_character__fixed_list); +CUSTOM_COMMAND_SIG(interactive_switch_buffer); +CUSTOM_COMMAND_SIG(interactive_kill_buffer); +CUSTOM_COMMAND_SIG(interactive_open_or_new); +CUSTOM_COMMAND_SIG(interactive_new); +CUSTOM_COMMAND_SIG(interactive_open); +CUSTOM_COMMAND_SIG(command_lister); +CUSTOM_COMMAND_SIG(auto_tab_whole_file); +CUSTOM_COMMAND_SIG(auto_tab_line_at_cursor); +CUSTOM_COMMAND_SIG(auto_tab_range); +CUSTOM_COMMAND_SIG(write_and_auto_tab); +CUSTOM_COMMAND_SIG(list_all_locations); +CUSTOM_COMMAND_SIG(list_all_substring_locations); +CUSTOM_COMMAND_SIG(list_all_locations_case_insensitive); +CUSTOM_COMMAND_SIG(list_all_substring_locations_case_insensitive); +CUSTOM_COMMAND_SIG(list_all_locations_of_identifier); +CUSTOM_COMMAND_SIG(list_all_locations_of_identifier_case_insensitive); +CUSTOM_COMMAND_SIG(list_all_locations_of_selection); +CUSTOM_COMMAND_SIG(list_all_locations_of_selection_case_insensitive); +CUSTOM_COMMAND_SIG(list_all_locations_of_type_definition); +CUSTOM_COMMAND_SIG(list_all_locations_of_type_definition_of_identifier); +CUSTOM_COMMAND_SIG(word_complete); +CUSTOM_COMMAND_SIG(goto_jump_at_cursor); +CUSTOM_COMMAND_SIG(goto_jump_at_cursor_same_panel); +CUSTOM_COMMAND_SIG(goto_next_jump); +CUSTOM_COMMAND_SIG(goto_prev_jump); +CUSTOM_COMMAND_SIG(goto_next_jump_no_skips); +CUSTOM_COMMAND_SIG(goto_prev_jump_no_skips); +CUSTOM_COMMAND_SIG(goto_first_jump); +CUSTOM_COMMAND_SIG(goto_first_jump_same_panel_sticky); +CUSTOM_COMMAND_SIG(newline_or_goto_position); +CUSTOM_COMMAND_SIG(newline_or_goto_position_same_panel); +CUSTOM_COMMAND_SIG(view_jump_list_with_lister); +CUSTOM_COMMAND_SIG(log_graph__escape); +CUSTOM_COMMAND_SIG(log_graph__scroll_wheel); +CUSTOM_COMMAND_SIG(log_graph__page_up); +CUSTOM_COMMAND_SIG(log_graph__page_down); +CUSTOM_COMMAND_SIG(log_graph__click_select_event); +CUSTOM_COMMAND_SIG(log_graph__click_jump_to_event_source); +CUSTOM_COMMAND_SIG(show_the_log_graph); +CUSTOM_COMMAND_SIG(copy); +CUSTOM_COMMAND_SIG(cut); +CUSTOM_COMMAND_SIG(paste); +CUSTOM_COMMAND_SIG(paste_next); +CUSTOM_COMMAND_SIG(paste_and_indent); +CUSTOM_COMMAND_SIG(paste_next_and_indent); +CUSTOM_COMMAND_SIG(execute_previous_cli); +CUSTOM_COMMAND_SIG(execute_any_cli); +CUSTOM_COMMAND_SIG(build_search); +CUSTOM_COMMAND_SIG(build_in_build_panel); +CUSTOM_COMMAND_SIG(close_build_panel); +CUSTOM_COMMAND_SIG(change_to_build_panel); +CUSTOM_COMMAND_SIG(close_all_code); +CUSTOM_COMMAND_SIG(open_all_code); +CUSTOM_COMMAND_SIG(open_all_code_recursive); +CUSTOM_COMMAND_SIG(load_project); +CUSTOM_COMMAND_SIG(project_fkey_command); +CUSTOM_COMMAND_SIG(project_go_to_root_directory); +CUSTOM_COMMAND_SIG(setup_new_project); +CUSTOM_COMMAND_SIG(setup_build_bat); +CUSTOM_COMMAND_SIG(setup_build_sh); +CUSTOM_COMMAND_SIG(setup_build_bat_and_sh); +CUSTOM_COMMAND_SIG(project_command_lister); +CUSTOM_COMMAND_SIG(list_all_functions_current_buffer); +CUSTOM_COMMAND_SIG(list_all_functions_current_buffer_lister); +CUSTOM_COMMAND_SIG(list_all_functions_all_buffers); +CUSTOM_COMMAND_SIG(list_all_functions_all_buffers_lister); +CUSTOM_COMMAND_SIG(select_surrounding_scope); +CUSTOM_COMMAND_SIG(select_next_scope_absolute); +CUSTOM_COMMAND_SIG(select_prev_scope_absolute); +CUSTOM_COMMAND_SIG(place_in_scope); +CUSTOM_COMMAND_SIG(delete_current_scope); +CUSTOM_COMMAND_SIG(scope_absorb_down); +CUSTOM_COMMAND_SIG(open_long_braces); +CUSTOM_COMMAND_SIG(open_long_braces_semicolon); +CUSTOM_COMMAND_SIG(open_long_braces_break); +CUSTOM_COMMAND_SIG(if0_off); +CUSTOM_COMMAND_SIG(write_todo); +CUSTOM_COMMAND_SIG(write_hack); +CUSTOM_COMMAND_SIG(write_note); +CUSTOM_COMMAND_SIG(write_block); +CUSTOM_COMMAND_SIG(write_zero_struct); +CUSTOM_COMMAND_SIG(comment_line); +CUSTOM_COMMAND_SIG(uncomment_line); +CUSTOM_COMMAND_SIG(comment_line_toggle); +CUSTOM_COMMAND_SIG(snippet_lister); +CUSTOM_COMMAND_SIG(set_bindings_choose); +CUSTOM_COMMAND_SIG(set_bindings_default); +CUSTOM_COMMAND_SIG(set_bindings_mac_default); +CUSTOM_COMMAND_SIG(miblo_increment_basic); +CUSTOM_COMMAND_SIG(miblo_decrement_basic); +CUSTOM_COMMAND_SIG(miblo_increment_time_stamp); +CUSTOM_COMMAND_SIG(miblo_decrement_time_stamp); +CUSTOM_COMMAND_SIG(miblo_increment_time_stamp_minute); +CUSTOM_COMMAND_SIG(miblo_decrement_time_stamp_minute); +CUSTOM_COMMAND_SIG(rename_parameter); +CUSTOM_COMMAND_SIG(write_explicit_enum_values); #endif struct Command_Metadata{ PROC_LINKS(Custom_Command_Function, void) *proc; char *name; -i32 name_len; +int32_t name_len; char *description; -i32 description_len; +int32_t description_len; char *source_name; -i32 source_name_len; -i32 line_number; +int32_t source_name_len; +int32_t line_number; }; -static Command_Metadata fcoder_metacmd_table[0] = { +static Command_Metadata fcoder_metacmd_table[233] = { +{ PROC_LINKS(write_explicit_enum_flags, 0), "write_explicit_enum_flags", 25, "If the cursor is found to be on the '{' of an enum definition, the values of the enum will be filled in to give each a unique power of 2 value, starting from 1. Existing values are overwritten.", 194, "c:\\work\\4ed\\code\\4coder_experiments.cpp", 39, 502 }, +{ PROC_LINKS(seek_beginning_of_textual_line, 0), "seek_beginning_of_textual_line", 30, "Seeks the cursor to the beginning of the line across all text.", 62, "c:\\work\\4ed\\code\\4coder_seek.cpp", 32, 29 }, +{ PROC_LINKS(seek_end_of_textual_line, 0), "seek_end_of_textual_line", 24, "Seeks the cursor to the end of the line across all text.", 56, "c:\\work\\4ed\\code\\4coder_seek.cpp", 32, 35 }, +{ PROC_LINKS(seek_beginning_of_line, 0), "seek_beginning_of_line", 22, "Seeks the cursor to the beginning of the visual line.", 53, "c:\\work\\4ed\\code\\4coder_seek.cpp", 32, 41 }, +{ PROC_LINKS(seek_end_of_line, 0), "seek_end_of_line", 16, "Seeks the cursor to the end of the visual line.", 47, "c:\\work\\4ed\\code\\4coder_seek.cpp", 32, 47 }, +{ PROC_LINKS(goto_beginning_of_file, 0), "goto_beginning_of_file", 22, "Sets the cursor to the beginning of the file.", 45, "c:\\work\\4ed\\code\\4coder_seek.cpp", 32, 53 }, +{ PROC_LINKS(goto_end_of_file, 0), "goto_end_of_file", 16, "Sets the cursor to the end of the file.", 39, "c:\\work\\4ed\\code\\4coder_seek.cpp", 32, 61 }, +{ PROC_LINKS(change_active_panel, 0), "change_active_panel", 19, "Change the currently active panel, moving to the panel with the next highest view_id.", 85, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 201 }, +{ PROC_LINKS(change_active_panel_backwards, 0), "change_active_panel_backwards", 29, "Change the currently active panel, moving to the panel with the next lowest view_id.", 84, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 211 }, +{ PROC_LINKS(open_panel_vsplit, 0), "open_panel_vsplit", 17, "Create a new panel by vertically splitting the active panel.", 60, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 221 }, +{ PROC_LINKS(open_panel_hsplit, 0), "open_panel_hsplit", 17, "Create a new panel by horizontally splitting the active panel.", 62, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 231 }, +{ PROC_LINKS(suppress_mouse, 0), "suppress_mouse", 14, "Hides the mouse and causes all mosue input (clicks, position, wheel) to be ignored.", 83, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 294 }, +{ PROC_LINKS(allow_mouse, 0), "allow_mouse", 11, "Shows the mouse and causes all mouse input to be processed normally.", 68, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 300 }, +{ PROC_LINKS(toggle_mouse, 0), "toggle_mouse", 12, "Toggles the mouse suppression mode, see suppress_mouse and allow_mouse.", 71, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 306 }, +{ PROC_LINKS(set_mode_to_original, 0), "set_mode_to_original", 20, "Sets the edit mode to 4coder original.", 38, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 312 }, +{ PROC_LINKS(set_mode_to_notepad_like, 0), "set_mode_to_notepad_like", 24, "Sets the edit mode to Notepad like.", 35, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 318 }, +{ PROC_LINKS(toggle_highlight_line_at_cursor, 0), "toggle_highlight_line_at_cursor", 31, "Toggles the line highlight at the cursor.", 41, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 324 }, +{ PROC_LINKS(toggle_highlight_enclosing_scopes, 0), "toggle_highlight_enclosing_scopes", 33, "In code files scopes surrounding the cursor are highlighted with distinguishing colors.", 87, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 330 }, +{ PROC_LINKS(toggle_paren_matching_helper, 0), "toggle_paren_matching_helper", 28, "In code files matching parentheses pairs are colored with distinguishing colors.", 80, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 336 }, +{ PROC_LINKS(toggle_fullscreen, 0), "toggle_fullscreen", 17, "Toggle fullscreen mode on or off. The change(s) do not take effect until the next frame.", 89, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 342 }, +{ PROC_LINKS(remap_interactive, 0), "remap_interactive", 17, "Switch to a named key binding map.", 34, "c:\\work\\4ed\\code\\4coder_default_framework.cpp", 45, 350 }, +{ PROC_LINKS(write_character, 0), "write_character", 15, "Inserts whatever character was used to trigger this command.", 60, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 57 }, +{ PROC_LINKS(write_underscore, 0), "write_underscore", 16, "Inserts an underscore.", 22, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 66 }, +{ PROC_LINKS(delete_char, 0), "delete_char", 11, "Deletes the character to the right of the cursor.", 49, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 73 }, +{ PROC_LINKS(backspace_char, 0), "backspace_char", 14, "Deletes the character to the left of the cursor.", 48, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 90 }, +{ PROC_LINKS(set_mark, 0), "set_mark", 8, "Sets the mark to the current position of the cursor.", 52, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 109 }, +{ PROC_LINKS(cursor_mark_swap, 0), "cursor_mark_swap", 16, "Swaps the position of the cursor and the mark.", 46, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 118 }, +{ PROC_LINKS(delete_range, 0), "delete_range", 12, "Deletes the text in the range between the cursor and the mark.", 62, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 128 }, +{ PROC_LINKS(backspace_alpha_numeric_boundary, 0), "backspace_alpha_numeric_boundary", 32, "Delete characters between the cursor position and the first alphanumeric boundary to the left.", 94, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 148 }, +{ PROC_LINKS(delete_alpha_numeric_boundary, 0), "delete_alpha_numeric_boundary", 29, "Delete characters between the cursor position and the first alphanumeric boundary to the right.", 95, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 156 }, +{ PROC_LINKS(snipe_backward_whitespace_or_token_boundary, 0), "snipe_backward_whitespace_or_token_boundary", 43, "Delete a single, whole token on or to the left of the cursor and post it to the clipboard.", 90, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 176 }, +{ PROC_LINKS(snipe_forward_whitespace_or_token_boundary, 0), "snipe_forward_whitespace_or_token_boundary", 42, "Delete a single, whole token on or to the right of the cursor and post it to the clipboard.", 91, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 184 }, +{ PROC_LINKS(center_view, 0), "center_view", 11, "Centers the view vertically on the line on which the cursor sits.", 65, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 197 }, +{ PROC_LINKS(left_adjust_view, 0), "left_adjust_view", 16, "Sets the left size of the view near the x position of the cursor.", 65, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 211 }, +{ PROC_LINKS(click_set_cursor_and_mark, 0), "click_set_cursor_and_mark", 25, "Sets the cursor position and mark to the mouse position.", 56, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 223 }, +{ PROC_LINKS(click_set_cursor, 0), "click_set_cursor", 16, "Sets the cursor position to the mouse position.", 47, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 235 }, +{ PROC_LINKS(click_set_cursor_if_lbutton, 0), "click_set_cursor_if_lbutton", 27, "If the mouse left button is pressed, sets the cursor position to the mouse position.", 84, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 247 }, +{ PROC_LINKS(click_set_mark, 0), "click_set_mark", 14, "Sets the mark position to the mouse position.", 45, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 261 }, +{ PROC_LINKS(mouse_wheel_scroll, 0), "mouse_wheel_scroll", 18, "Reads the scroll wheel value from the mouse state and scrolls accordingly.", 74, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 273 }, +{ PROC_LINKS(move_up, 0), "move_up", 7, "Moves the cursor up one line.", 29, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 334 }, +{ PROC_LINKS(move_down, 0), "move_down", 9, "Moves the cursor down one line.", 31, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 340 }, +{ PROC_LINKS(move_up_10, 0), "move_up_10", 10, "Moves the cursor up ten lines.", 30, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 346 }, +{ PROC_LINKS(move_down_10, 0), "move_down_10", 12, "Moves the cursor down ten lines.", 32, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 352 }, +{ PROC_LINKS(move_down_textual, 0), "move_down_textual", 17, "Moves down to the next line of actual text, regardless of line wrapping.", 72, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 358 }, +{ PROC_LINKS(page_up, 0), "page_up", 7, "Scrolls the view up one view height and moves the cursor up one view height.", 76, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 368 }, +{ PROC_LINKS(page_down, 0), "page_down", 9, "Scrolls the view down one view height and moves the cursor down one view height.", 80, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 376 }, +{ PROC_LINKS(move_up_to_blank_line, 0), "move_up_to_blank_line", 21, "Seeks the cursor up to the next blank line.", 43, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 405 }, +{ PROC_LINKS(move_down_to_blank_line, 0), "move_down_to_blank_line", 23, "Seeks the cursor down to the next blank line.", 45, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 411 }, +{ PROC_LINKS(move_up_to_blank_line_skip_whitespace, 0), "move_up_to_blank_line_skip_whitespace", 37, "Seeks the cursor up to the next blank line and places it at the end of the line.", 80, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 417 }, +{ PROC_LINKS(move_down_to_blank_line_skip_whitespace, 0), "move_down_to_blank_line_skip_whitespace", 39, "Seeks the cursor down to the next blank line and places it at the end of the line.", 82, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 423 }, +{ PROC_LINKS(move_up_to_blank_line_end, 0), "move_up_to_blank_line_end", 25, "Seeks the cursor up to the next blank line and places it at the end of the line.", 80, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 429 }, +{ PROC_LINKS(move_down_to_blank_line_end, 0), "move_down_to_blank_line_end", 27, "Seeks the cursor down to the next blank line and places it at the end of the line.", 82, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 435 }, +{ PROC_LINKS(move_left, 0), "move_left", 9, "Moves the cursor one character to the left.", 43, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 441 }, +{ PROC_LINKS(move_right, 0), "move_right", 10, "Moves the cursor one character to the right.", 44, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 453 }, +{ PROC_LINKS(move_right_whitespace_boundary, 0), "move_right_whitespace_boundary", 30, "Seek right for the next boundary between whitespace and non-whitespace.", 71, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 475 }, +{ PROC_LINKS(move_left_whitespace_boundary, 0), "move_left_whitespace_boundary", 29, "Seek left for the next boundary between whitespace and non-whitespace.", 70, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 483 }, +{ PROC_LINKS(move_right_token_boundary, 0), "move_right_token_boundary", 25, "Seek right for the next end of a token.", 39, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 491 }, +{ PROC_LINKS(move_left_token_boundary, 0), "move_left_token_boundary", 24, "Seek left for the next beginning of a token.", 44, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 499 }, +{ PROC_LINKS(move_right_whitespace_or_token_boundary, 0), "move_right_whitespace_or_token_boundary", 39, "Seek right for the next end of a token or boundary between whitespace and non-whitespace.", 89, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 507 }, +{ PROC_LINKS(move_left_whitespace_or_token_boundary, 0), "move_left_whitespace_or_token_boundary", 38, "Seek left for the next end of a token or boundary between whitespace and non-whitespace.", 88, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 515 }, +{ PROC_LINKS(move_right_alpha_numeric_boundary, 0), "move_right_alpha_numeric_boundary", 33, "Seek right for boundary between alphanumeric characters and non-alphanumeric characters.", 88, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 523 }, +{ PROC_LINKS(move_left_alpha_numeric_boundary, 0), "move_left_alpha_numeric_boundary", 32, "Seek left for boundary between alphanumeric characters and non-alphanumeric characters.", 87, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 531 }, +{ PROC_LINKS(move_right_alpha_numeric_or_camel_boundary, 0), "move_right_alpha_numeric_or_camel_boundary", 42, "Seek right for boundary between alphanumeric characters or camel case word and non-alphanumeric characters.", 107, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 539 }, +{ PROC_LINKS(move_left_alpha_numeric_or_camel_boundary, 0), "move_left_alpha_numeric_or_camel_boundary", 41, "Seek left for boundary between alphanumeric characters or camel case word and non-alphanumeric characters.", 106, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 547 }, +{ PROC_LINKS(select_all, 0), "select_all", 10, "Puts the cursor at the top of the file, and the mark at the bottom of the file.", 79, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 568 }, +{ PROC_LINKS(to_uppercase, 0), "to_uppercase", 12, "Converts all ascii text in the range between the cursor and the mark to uppercase.", 82, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 581 }, +{ PROC_LINKS(to_lowercase, 0), "to_lowercase", 12, "Converts all ascii text in the range between the cursor and the mark to lowercase.", 82, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 594 }, +{ PROC_LINKS(clean_all_lines, 0), "clean_all_lines", 15, "Removes trailing whitespace from all lines in the current buffer.", 65, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 607 }, +{ PROC_LINKS(basic_change_active_panel, 0), "basic_change_active_panel", 25, "Change the currently active panel, moving to the panel with the next highest view_id. Will not skipe the build panel if it is open.", 132, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 641 }, +{ PROC_LINKS(close_panel, 0), "close_panel", 11, "Closes the currently active panel if it is not the only panel open.", 67, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 649 }, +{ PROC_LINKS(show_scrollbar, 0), "show_scrollbar", 14, "Sets the current view to show it's scrollbar.", 45, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 658 }, +{ PROC_LINKS(hide_scrollbar, 0), "hide_scrollbar", 14, "Sets the current view to hide it's scrollbar.", 45, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 665 }, +{ PROC_LINKS(show_filebar, 0), "show_filebar", 12, "Sets the current view to show it's filebar.", 43, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 672 }, +{ PROC_LINKS(hide_filebar, 0), "hide_filebar", 12, "Sets the current view to hide it's filebar.", 43, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 679 }, +{ PROC_LINKS(toggle_filebar, 0), "toggle_filebar", 14, "Toggles the visibility status of the current view's filebar.", 60, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 686 }, +{ PROC_LINKS(toggle_line_wrap, 0), "toggle_line_wrap", 16, "Toggles the current buffer's line wrapping status.", 50, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 695 }, +{ PROC_LINKS(toggle_fps_meter, 0), "toggle_fps_meter", 16, "Toggles the visibility of the FPS performance meter", 51, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 708 }, +{ PROC_LINKS(increase_line_wrap, 0), "increase_line_wrap", 18, "Increases the current buffer's width for line wrapping.", 55, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 714 }, +{ PROC_LINKS(decrease_line_wrap, 0), "decrease_line_wrap", 18, "Decrases the current buffer's width for line wrapping.", 54, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 727 }, +{ PROC_LINKS(increase_face_size, 0), "increase_face_size", 18, "Increase the size of the face used by the current buffer.", 57, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 740 }, +{ PROC_LINKS(decrease_face_size, 0), "decrease_face_size", 18, "Decrease the size of the face used by the current buffer.", 57, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 751 }, +{ PROC_LINKS(mouse_wheel_change_face_size, 0), "mouse_wheel_change_face_size", 28, "Reads the state of the mouse wheel and uses it to either increase or decrease the face size.", 92, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 762 }, +{ PROC_LINKS(toggle_virtual_whitespace, 0), "toggle_virtual_whitespace", 25, "Toggles the current buffer's virtual whitespace status.", 55, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 779 }, +{ PROC_LINKS(toggle_show_whitespace, 0), "toggle_show_whitespace", 22, "Toggles the current buffer's whitespace visibility status.", 58, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 789 }, +{ PROC_LINKS(toggle_line_numbers, 0), "toggle_line_numbers", 19, "Toggles the left margin line numbers.", 37, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 798 }, +{ PROC_LINKS(eol_dosify, 0), "eol_dosify", 10, "Puts the buffer in DOS line ending mode.", 40, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 804 }, +{ PROC_LINKS(eol_nixify, 0), "eol_nixify", 10, "Puts the buffer in NIX line ending mode.", 40, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 812 }, +{ PROC_LINKS(exit_4coder, 0), "exit_4coder", 11, "Attempts to close 4coder.", 25, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 820 }, +{ PROC_LINKS(goto_line, 0), "goto_line", 9, "Queries the user for a number, and jumps the cursor to the corresponding line.", 78, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 828 }, +{ PROC_LINKS(search, 0), "search", 6, "Begins an incremental search down through the current buffer for a user specified string.", 89, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1039 }, +{ PROC_LINKS(reverse_search, 0), "reverse_search", 14, "Begins an incremental search up through the current buffer for a user specified string.", 87, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1045 }, +{ PROC_LINKS(search_identifier, 0), "search_identifier", 17, "Begins an incremental search down through the current buffer for the word or token under the cursor.", 100, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1051 }, +{ PROC_LINKS(reverse_search_identifier, 0), "reverse_search_identifier", 25, "Begins an incremental search up through the current buffer for the word or token under the cursor.", 98, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1062 }, +{ PROC_LINKS(replace_in_range, 0), "replace_in_range", 16, "Queries the user for a needle and string. Replaces all occurences of needle with string in the range between cursor and the mark in the active buffer.", 150, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1113 }, +{ PROC_LINKS(replace_in_buffer, 0), "replace_in_buffer", 17, "Queries the user for a needle and string. Replaces all occurences of needle with string in the active buffer.", 109, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1122 }, +{ PROC_LINKS(replace_in_all_buffers, 0), "replace_in_all_buffers", 22, "Queries the user for a needle and string. Replaces all occurences of needle with string in all editable buffers.", 112, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1131 }, +{ PROC_LINKS(query_replace, 0), "query_replace", 13, "Queries the user for two strings, and incrementally replaces every occurence of the first string with the second string.", 120, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1219 }, +{ PROC_LINKS(query_replace_identifier, 0), "query_replace_identifier", 24, "Queries the user for a string, and incrementally replace every occurence of the word or token found at the cursor with the specified string.", 140, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1239 }, +{ PROC_LINKS(query_replace_selection, 0), "query_replace_selection", 23, "Queries the user for a string, and incrementally replace every occurence of the string found in the selected range with the specified string.", 141, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1255 }, +{ PROC_LINKS(save_all_dirty_buffers, 0), "save_all_dirty_buffers", 22, "Saves all buffers marked dirty (showing the '*' indicator).", 59, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1290 }, +{ PROC_LINKS(delete_file_query, 0), "delete_file_query", 17, "Deletes the file of the current buffer if 4coder has the appropriate access rights. Will ask the user for confirmation first.", 125, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1315 }, +{ PROC_LINKS(save_to_query, 0), "save_to_query", 13, "Queries the user for a file name and saves the contents of the current buffer, altering the buffer's name too.", 110, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1353 }, +{ PROC_LINKS(rename_file_query, 0), "rename_file_query", 17, "Queries the user for a new name and renames the file of the current buffer, altering the buffer's name too.", 107, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1388 }, +{ PROC_LINKS(make_directory_query, 0), "make_directory_query", 20, "Queries the user for a name and creates a new directory with the given name.", 76, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1428 }, +{ PROC_LINKS(move_line_up, 0), "move_line_up", 12, "Swaps the line under the cursor with the line above it, and moves the cursor up with it.", 88, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1461 }, +{ PROC_LINKS(move_line_down, 0), "move_line_down", 14, "Swaps the line under the cursor with the line below it, and moves the cursor down with it.", 90, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1467 }, +{ PROC_LINKS(duplicate_line, 0), "duplicate_line", 14, "Create a copy of the line on which the cursor sits.", 51, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1473 }, +{ PROC_LINKS(delete_line, 0), "delete_line", 11, "Delete the line the on which the cursor sits.", 45, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1487 }, +{ PROC_LINKS(open_file_in_quotes, 0), "open_file_in_quotes", 19, "Reads a filename from surrounding '\"' characters and attempts to open the corresponding file.", 94, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1552 }, +{ PROC_LINKS(open_matching_file_cpp, 0), "open_matching_file_cpp", 22, "If the current file is a *.cpp or *.h, attempts to open the corresponding *.h or *.cpp file in the other view.", 110, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1584 }, +{ PROC_LINKS(view_buffer_other_panel, 0), "view_buffer_other_panel", 23, "Set the other non-active panel to view the buffer that the active panel views, and switch to that panel.", 104, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1597 }, +{ PROC_LINKS(swap_buffers_between_panels, 0), "swap_buffers_between_panels", 27, "Set the other non-active panel to view the buffer that the active panel views, and switch to that panel.", 104, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1609 }, +{ PROC_LINKS(kill_buffer, 0), "kill_buffer", 11, "Kills the current buffer.", 25, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1645 }, +{ PROC_LINKS(save, 0), "save", 4, "Saves the current buffer.", 25, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1653 }, +{ PROC_LINKS(reopen, 0), "reopen", 6, "Reopen the current buffer from the hard drive.", 46, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1665 }, +{ PROC_LINKS(undo, 0), "undo", 4, "Advances backwards through the undo history of the current buffer.", 66, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1723 }, +{ PROC_LINKS(redo, 0), "redo", 4, "Advances forwards through the undo history of the current buffer.", 65, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1736 }, +{ PROC_LINKS(undo_all_buffers, 0), "undo_all_buffers", 16, "Advances backward through the undo history in the buffer containing the most recent regular edit.", 97, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1750 }, +{ PROC_LINKS(redo_all_buffers, 0), "redo_all_buffers", 16, "Advances forward through the undo history in the buffer containing the most recent regular edit.", 96, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1824 }, +{ PROC_LINKS(open_in_other, 0), "open_in_other", 13, "Interactively opens a file in the other panel.", 46, "c:\\work\\4ed\\code\\4coder_base_commands.cpp", 41, 1927 }, +{ PROC_LINKS(lister__quit, 0), "lister__quit", 12, "A lister mode command that quits the list without executing any actions.", 72, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 8 }, +{ PROC_LINKS(lister__activate, 0), "lister__activate", 16, "A lister mode command that activates the list's action on the highlighted item.", 79, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 15 }, +{ PROC_LINKS(lister__write_character, 0), "lister__write_character", 23, "A lister mode command that dispatches to the lister's write character handler.", 78, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 30 }, +{ PROC_LINKS(lister__backspace_text_field, 0), "lister__backspace_text_field", 28, "A lister mode command that dispatches to the lister's backspace text field handler.", 83, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 40 }, +{ PROC_LINKS(lister__move_up, 0), "lister__move_up", 15, "A lister mode command that dispatches to the lister's navigate up handler.", 74, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 50 }, +{ PROC_LINKS(lister__move_down, 0), "lister__move_down", 17, "A lister mode command that dispatches to the lister's navigate down handler.", 76, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 60 }, +{ PROC_LINKS(lister__wheel_scroll, 0), "lister__wheel_scroll", 20, "A lister mode command that scrolls the list in response to the mouse wheel.", 75, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 70 }, +{ PROC_LINKS(lister__mouse_press, 0), "lister__mouse_press", 19, "A lister mode command that beings a click interaction with a list item under the mouse.", 87, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 84 }, +{ PROC_LINKS(lister__mouse_release, 0), "lister__mouse_release", 21, "A lister mode command that ends a click interaction with a list item under the mouse, possibly activating it.", 109, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 95 }, +{ PROC_LINKS(lister__repaint, 0), "lister__repaint", 15, "A lister mode command that updates the lists UI data.", 53, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 110 }, +{ PROC_LINKS(lister__write_character__default, 0), "lister__write_character__default", 32, "A lister mode command that inserts a new character to the text field.", 69, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 120 }, +{ PROC_LINKS(lister__backspace_text_field__default, 0), "lister__backspace_text_field__default", 37, "A lister mode command that backspaces one character from the text field.", 72, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 139 }, +{ PROC_LINKS(lister__move_up__default, 0), "lister__move_up__default", 24, "A lister mode command that moves the highlighted item one up in the list.", 73, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 153 }, +{ PROC_LINKS(lister__move_down__default, 0), "lister__move_down__default", 26, "A lister mode command that moves the highlighted item one down in the list.", 75, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 168 }, +{ PROC_LINKS(lister__write_character__file_path, 0), "lister__write_character__file_path", 34, "A lister mode command that inserts a character into the text field of a file system list.", 89, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 183 }, +{ PROC_LINKS(lister__backspace_text_field__file_path, 0), "lister__backspace_text_field__file_path", 39, "A lister mode command that backspaces one character from the text field of a file system list.", 94, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 208 }, +{ PROC_LINKS(lister__write_character__fixed_list, 0), "lister__write_character__fixed_list", 35, "A lister mode command that handles input for the fixed sure to kill list.", 73, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 249 }, +{ PROC_LINKS(interactive_switch_buffer, 0), "interactive_switch_buffer", 25, "Interactively switch to an open buffer.", 39, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 723 }, +{ PROC_LINKS(interactive_kill_buffer, 0), "interactive_kill_buffer", 23, "Interactively kill an open buffer.", 34, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 742 }, +{ PROC_LINKS(interactive_open_or_new, 0), "interactive_open_or_new", 23, "Interactively open a file out of the file system.", 49, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 815 }, +{ PROC_LINKS(interactive_new, 0), "interactive_new", 15, "Interactively creates a new file.", 33, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 854 }, +{ PROC_LINKS(interactive_open, 0), "interactive_open", 16, "Interactively opens a file.", 27, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 887 }, +{ PROC_LINKS(command_lister, 0), "command_lister", 14, "Opens an interactive list of all registered commands.", 53, "c:\\work\\4ed\\code\\4coder_lists.cpp", 33, 969 }, +{ PROC_LINKS(auto_tab_whole_file, 0), "auto_tab_whole_file", 19, "Audo-indents the entire current buffer.", 39, "c:\\work\\4ed\\code\\4coder_auto_indent.cpp", 39, 526 }, +{ PROC_LINKS(auto_tab_line_at_cursor, 0), "auto_tab_line_at_cursor", 23, "Auto-indents the line on which the cursor sits.", 47, "c:\\work\\4ed\\code\\4coder_auto_indent.cpp", 39, 535 }, +{ PROC_LINKS(auto_tab_range, 0), "auto_tab_range", 14, "Auto-indents the range between the cursor and the mark.", 55, "c:\\work\\4ed\\code\\4coder_auto_indent.cpp", 39, 545 }, +{ PROC_LINKS(write_and_auto_tab, 0), "write_and_auto_tab", 18, "Inserts a character and auto-indents the line on which the cursor sits.", 71, "c:\\work\\4ed\\code\\4coder_auto_indent.cpp", 39, 555 }, +{ PROC_LINKS(list_all_locations, 0), "list_all_locations", 18, "Queries the user for a string and lists all exact case-sensitive matches found in all open buffers.", 99, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 166 }, +{ PROC_LINKS(list_all_substring_locations, 0), "list_all_substring_locations", 28, "Queries the user for a string and lists all case-sensitive substring matches found in all open buffers.", 103, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 172 }, +{ PROC_LINKS(list_all_locations_case_insensitive, 0), "list_all_locations_case_insensitive", 35, "Queries the user for a string and lists all exact case-insensitive matches found in all open buffers.", 101, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 178 }, +{ PROC_LINKS(list_all_substring_locations_case_insensitive, 0), "list_all_substring_locations_case_insensitive", 45, "Queries the user for a string and lists all case-insensitive substring matches found in all open buffers.", 105, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 184 }, +{ PROC_LINKS(list_all_locations_of_identifier, 0), "list_all_locations_of_identifier", 32, "Reads a token or word under the cursor and lists all exact case-sensitive mathces in all open buffers.", 102, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 190 }, +{ PROC_LINKS(list_all_locations_of_identifier_case_insensitive, 0), "list_all_locations_of_identifier_case_insensitive", 49, "Reads a token or word under the cursor and lists all exact case-insensitive mathces in all open buffers.", 104, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 196 }, +{ PROC_LINKS(list_all_locations_of_selection, 0), "list_all_locations_of_selection", 31, "Reads the string in the selected range and lists all exact case-sensitive mathces in all open buffers.", 102, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 202 }, +{ PROC_LINKS(list_all_locations_of_selection_case_insensitive, 0), "list_all_locations_of_selection_case_insensitive", 48, "Reads the string in the selected range and lists all exact case-insensitive mathces in all open buffers.", 104, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 208 }, +{ PROC_LINKS(list_all_locations_of_type_definition, 0), "list_all_locations_of_type_definition", 37, "Queries user for string, lists all locations of strings that appear to define a type whose name matches the input string.", 121, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 214 }, +{ PROC_LINKS(list_all_locations_of_type_definition_of_identifier, 0), "list_all_locations_of_type_definition_of_identifier", 51, "Reads a token or word under the cursor and lists all locations of strings that appear to define a type whose name matches it.", 125, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 222 }, +{ PROC_LINKS(word_complete, 0), "word_complete", 13, "Iteratively tries completing the word to the left of the cursor with other words in open buffers that have the same prefix string.", 130, "c:\\work\\4ed\\code\\4coder_search.cpp", 34, 376 }, +{ PROC_LINKS(goto_jump_at_cursor, 0), "goto_jump_at_cursor", 19, "If the cursor is found to be on a jump location, parses the jump location and brings up the file and position in another view and changes the active panel to the view containing the jump.", 187, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 353 }, +{ PROC_LINKS(goto_jump_at_cursor_same_panel, 0), "goto_jump_at_cursor_same_panel", 30, "If the cursor is found to be on a jump location, parses the jump location and brings up the file and position in this view, losing the compilation output or jump list.", 167, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 380 }, +{ PROC_LINKS(goto_next_jump, 0), "goto_next_jump", 14, "If a buffer containing jump locations has been locked in, goes to the next jump in the buffer, skipping sub jump locations.", 123, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 469 }, +{ PROC_LINKS(goto_prev_jump, 0), "goto_prev_jump", 14, "If a buffer containing jump locations has been locked in, goes to the previous jump in the buffer, skipping sub jump locations.", 127, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 486 }, +{ PROC_LINKS(goto_next_jump_no_skips, 0), "goto_next_jump_no_skips", 23, "If a buffer containing jump locations has been locked in, goes to the next jump in the buffer, and does not skip sub jump locations.", 132, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 499 }, +{ PROC_LINKS(goto_prev_jump_no_skips, 0), "goto_prev_jump_no_skips", 23, "If a buffer containing jump locations has been locked in, goes to the previous jump in the buffer, and does not skip sub jump locations.", 136, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 516 }, +{ PROC_LINKS(goto_first_jump, 0), "goto_first_jump", 15, "If a buffer containing jump locations has been locked in, goes to the first jump in the buffer.", 95, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 530 }, +{ PROC_LINKS(goto_first_jump_same_panel_sticky, 0), "goto_first_jump_same_panel_sticky", 33, "If a buffer containing jump locations has been locked in, goes to the first jump in the buffer and views the buffer in the panel where the jump list was.", 153, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 547 }, +{ PROC_LINKS(newline_or_goto_position, 0), "newline_or_goto_position", 24, "If the buffer in the active view is writable, inserts a character, otherwise performs goto_jump_at_cursor.", 106, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 569 }, +{ PROC_LINKS(newline_or_goto_position_same_panel, 0), "newline_or_goto_position_same_panel", 35, "If the buffer in the active view is writable, inserts a character, otherwise performs goto_jump_at_cursor_same_panel.", 117, "c:\\work\\4ed\\code\\4coder_jump_sticky.cpp", 39, 586 }, +{ PROC_LINKS(view_jump_list_with_lister, 0), "view_jump_list_with_lister", 26, "When executed on a buffer with jumps, creates a persistent lister for all the jumps", 83, "c:\\work\\4ed\\code\\4coder_jump_lister.cpp", 39, 104 }, +{ PROC_LINKS(log_graph__escape, 0), "log_graph__escape", 17, "Ends the log grapher", 20, "c:\\work\\4ed\\code\\4coder_log_parser.cpp", 38, 906 }, +{ PROC_LINKS(log_graph__scroll_wheel, 0), "log_graph__scroll_wheel", 23, "Scrolls the log graph", 21, "c:\\work\\4ed\\code\\4coder_log_parser.cpp", 38, 915 }, +{ PROC_LINKS(log_graph__page_up, 0), "log_graph__page_up", 18, "Scroll the log graph up one whole page", 38, "c:\\work\\4ed\\code\\4coder_log_parser.cpp", 38, 926 }, +{ PROC_LINKS(log_graph__page_down, 0), "log_graph__page_down", 20, "Scroll the log graph down one whole page", 40, "c:\\work\\4ed\\code\\4coder_log_parser.cpp", 38, 934 }, +{ PROC_LINKS(log_graph__click_select_event, 0), "log_graph__click_select_event", 29, "Select the event record at the mouse point in the log graph", 59, "c:\\work\\4ed\\code\\4coder_log_parser.cpp", 38, 968 }, +{ PROC_LINKS(log_graph__click_jump_to_event_source, 0), "log_graph__click_jump_to_event_source", 37, "Jump to the code that logged the event record at the mouse point in the log graph", 81, "c:\\work\\4ed\\code\\4coder_log_parser.cpp", 38, 987 }, +{ PROC_LINKS(show_the_log_graph, 0), "show_the_log_graph", 18, "Parser *log* and displays the 'log graph' UI", 44, "c:\\work\\4ed\\code\\4coder_log_parser.cpp", 38, 1035 }, +{ PROC_LINKS(copy, 0), "copy", 4, "Copy the text in the range from the cursor to the mark onto the clipboard.", 74, "c:\\work\\4ed\\code\\4coder_clipboard.cpp", 37, 19 }, +{ PROC_LINKS(cut, 0), "cut", 3, "Cut the text in the range from the cursor to the mark onto the clipboard.", 73, "c:\\work\\4ed\\code\\4coder_clipboard.cpp", 37, 28 }, +{ PROC_LINKS(paste, 0), "paste", 5, "At the cursor, insert the text at the top of the clipboard.", 59, "c:\\work\\4ed\\code\\4coder_clipboard.cpp", 37, 39 }, +{ PROC_LINKS(paste_next, 0), "paste_next", 10, "If the previous command was paste or paste_next, replaces the paste range with the next text down on the clipboard, otherwise operates as the paste command.", 156, "c:\\work\\4ed\\code\\4coder_clipboard.cpp", 37, 73 }, +{ PROC_LINKS(paste_and_indent, 0), "paste_and_indent", 16, "Paste from the top of clipboard and run auto-indent on the newly pasted text.", 77, "c:\\work\\4ed\\code\\4coder_clipboard.cpp", 37, 114 }, +{ PROC_LINKS(paste_next_and_indent, 0), "paste_next_and_indent", 21, "Paste the next item on the clipboard and run auto-indent on the newly pasted text.", 82, "c:\\work\\4ed\\code\\4coder_clipboard.cpp", 37, 121 }, +{ PROC_LINKS(execute_previous_cli, 0), "execute_previous_cli", 20, "If the command execute_any_cli has already been used, this will execute a CLI reusing the most recent buffer name and command.", 126, "c:\\work\\4ed\\code\\4coder_system_command.cpp", 42, 7 }, +{ PROC_LINKS(execute_any_cli, 0), "execute_any_cli", 15, "Queries for an output buffer name and system command, runs the system command as a CLI and prints the output to the specified buffer.", 133, "c:\\work\\4ed\\code\\4coder_system_command.cpp", 42, 22 }, +{ PROC_LINKS(build_search, 0), "build_search", 12, "Looks for a build.bat, build.sh, or makefile in the current and parent directories. Runs the first that it finds and prints the output to *compilation*.", 153, "c:\\work\\4ed\\code\\4coder_build_commands.cpp", 42, 128 }, +{ PROC_LINKS(build_in_build_panel, 0), "build_in_build_panel", 20, "Looks for a build.bat, build.sh, or makefile in the current and parent directories. Runs the first that it finds and prints the output to *compilation*. Puts the *compilation* buffer in a panel at the footer of the current view.", 230, "c:\\work\\4ed\\code\\4coder_build_commands.cpp", 42, 163 }, +{ PROC_LINKS(close_build_panel, 0), "close_build_panel", 17, "If the special build panel is open, closes it.", 46, "c:\\work\\4ed\\code\\4coder_build_commands.cpp", 42, 178 }, +{ PROC_LINKS(change_to_build_panel, 0), "change_to_build_panel", 21, "If the special build panel is open, makes the build panel the active panel.", 75, "c:\\work\\4ed\\code\\4coder_build_commands.cpp", 42, 184 }, +{ PROC_LINKS(close_all_code, 0), "close_all_code", 14, "Closes any buffer with a filename ending with an extension configured to be recognized as a code file type.", 107, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 921 }, +{ PROC_LINKS(open_all_code, 0), "open_all_code", 13, "Open all code in the current directory. File types are determined by extensions. An extension is considered code based on the extensions specified in 4coder.config.", 164, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 927 }, +{ PROC_LINKS(open_all_code_recursive, 0), "open_all_code_recursive", 23, "Works as open_all_code but also runs in all subdirectories.", 59, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 933 }, +{ PROC_LINKS(load_project, 0), "load_project", 12, "Looks for a project.4coder file in the current directory and tries to load it. Looks in parent directories until a project file is found or there are no more parents.", 167, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 941 }, +{ PROC_LINKS(project_fkey_command, 0), "project_fkey_command", 20, "Run an 'fkey command' configured in a project.4coder file. Determines the index of the 'fkey command' by which function key or numeric key was pressed to trigger the command.", 175, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 948 }, +{ PROC_LINKS(project_go_to_root_directory, 0), "project_go_to_root_directory", 28, "Changes 4coder's hot directory to the root directory of the currently loaded project. With no loaded project nothing hapepns.", 125, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 971 }, +{ PROC_LINKS(setup_new_project, 0), "setup_new_project", 17, "Queries the user for several configuration options and initializes a new 4coder project with build scripts for every OS.", 120, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 1306 }, +{ PROC_LINKS(setup_build_bat, 0), "setup_build_bat", 15, "Queries the user for several configuration options and initializes a new build batch script.", 92, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 1313 }, +{ PROC_LINKS(setup_build_sh, 0), "setup_build_sh", 14, "Queries the user for several configuration options and initializes a new build shell script.", 92, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 1319 }, +{ PROC_LINKS(setup_build_bat_and_sh, 0), "setup_build_bat_and_sh", 22, "Queries the user for several configuration options and initializes a new build batch script.", 92, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 1325 }, +{ PROC_LINKS(project_command_lister, 0), "project_command_lister", 22, "Open a lister of all commands in the currently loaded project.", 62, "c:\\work\\4ed\\code\\4coder_project_commands.cpp", 44, 1340 }, +{ PROC_LINKS(list_all_functions_current_buffer, 0), "list_all_functions_current_buffer", 33, "Creates a jump list of lines of the current buffer that appear to define or declare functions.", 94, "c:\\work\\4ed\\code\\4coder_function_list.cpp", 41, 276 }, +{ PROC_LINKS(list_all_functions_current_buffer_lister, 0), "list_all_functions_current_buffer_lister", 40, "Creates a lister of locations that look like function definitions and declarations in the buffer.", 97, "c:\\work\\4ed\\code\\4coder_function_list.cpp", 41, 286 }, +{ PROC_LINKS(list_all_functions_all_buffers, 0), "list_all_functions_all_buffers", 30, "Creates a jump list of lines from all buffers that appear to define or declare functions.", 89, "c:\\work\\4ed\\code\\4coder_function_list.cpp", 41, 298 }, +{ PROC_LINKS(list_all_functions_all_buffers_lister, 0), "list_all_functions_all_buffers_lister", 37, "Creates a lister of locations that look like function definitions and declarations all buffers.", 95, "c:\\work\\4ed\\code\\4coder_function_list.cpp", 41, 304 }, +{ PROC_LINKS(select_surrounding_scope, 0), "select_surrounding_scope", 24, "Finds the scope enclosed by '{' '}' surrounding the cursor and puts the cursor and mark on the '{' and '}'.", 107, "c:\\work\\4ed\\code\\4coder_scope_commands.cpp", 42, 352 }, +{ PROC_LINKS(select_next_scope_absolute, 0), "select_next_scope_absolute", 26, "Finds the first scope started by '{' after the cursor and puts the cursor and mark on the '{' and '}'.", 102, "c:\\work\\4ed\\code\\4coder_scope_commands.cpp", 42, 367 }, +{ PROC_LINKS(select_prev_scope_absolute, 0), "select_prev_scope_absolute", 26, "Finds the first scope started by '{' before the cursor and puts the cursor and mark on the '{' and '}'.", 103, "c:\\work\\4ed\\code\\4coder_scope_commands.cpp", 42, 386 }, +{ PROC_LINKS(place_in_scope, 0), "place_in_scope", 14, "Wraps the code contained in the range between cursor and mark with a new curly brace scope.", 91, "c:\\work\\4ed\\code\\4coder_scope_commands.cpp", 42, 460 }, +{ PROC_LINKS(delete_current_scope, 0), "delete_current_scope", 20, "Deletes the braces surrounding the currently selected scope. Leaves the contents within the scope.", 99, "c:\\work\\4ed\\code\\4coder_scope_commands.cpp", 42, 466 }, +{ PROC_LINKS(scope_absorb_down, 0), "scope_absorb_down", 17, "If a scope is currently selected, and a statement or block statement is present below the current scope, the statement is moved into the scope.", 143, "c:\\work\\4ed\\code\\4coder_scope_commands.cpp", 42, 698 }, +{ PROC_LINKS(open_long_braces, 0), "open_long_braces", 16, "At the cursor, insert a '{' and '}' separated by a blank line.", 62, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 46 }, +{ PROC_LINKS(open_long_braces_semicolon, 0), "open_long_braces_semicolon", 26, "At the cursor, insert a '{' and '};' separated by a blank line.", 63, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 54 }, +{ PROC_LINKS(open_long_braces_break, 0), "open_long_braces_break", 22, "At the cursor, insert a '{' and '}break;' separated by a blank line.", 68, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 62 }, +{ PROC_LINKS(if0_off, 0), "if0_off", 7, "Surround the range between the cursor and mark with an '#if 0' and an '#endif'", 78, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 70 }, +{ PROC_LINKS(write_todo, 0), "write_todo", 10, "At the cursor, insert a '// TODO' comment, includes user name if it was specified in config.4coder.", 99, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 76 }, +{ PROC_LINKS(write_hack, 0), "write_hack", 10, "At the cursor, insert a '// HACK' comment, includes user name if it was specified in config.4coder.", 99, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 82 }, +{ PROC_LINKS(write_note, 0), "write_note", 10, "At the cursor, insert a '// NOTE' comment, includes user name if it was specified in config.4coder.", 99, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 88 }, +{ PROC_LINKS(write_block, 0), "write_block", 11, "At the cursor, insert a block comment.", 38, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 94 }, +{ PROC_LINKS(write_zero_struct, 0), "write_zero_struct", 17, "At the cursor, insert a ' = {};'.", 33, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 100 }, +{ PROC_LINKS(comment_line, 0), "comment_line", 12, "Insert '//' at the beginning of the line after leading whitespace.", 66, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 125 }, +{ PROC_LINKS(uncomment_line, 0), "uncomment_line", 14, "If present, delete '//' at the beginning of the line after leading whitespace.", 78, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 137 }, +{ PROC_LINKS(comment_line_toggle, 0), "comment_line_toggle", 19, "Turns uncommented lines into commented lines and vice versa for comments starting with '//'.", 92, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 149 }, +{ PROC_LINKS(snippet_lister, 0), "snippet_lister", 14, "Opens a snippet lister for inserting whole pre-written snippets of text.", 72, "c:\\work\\4ed\\code\\4coder_combined_write_commands.cpp", 51, 235 }, +{ PROC_LINKS(set_bindings_choose, 0), "set_bindings_choose", 19, "Remap keybindings using the 'choose' mapping rule.", 50, "c:\\work\\4ed\\code\\4coder_remapping_commands.cpp", 46, 41 }, +{ PROC_LINKS(set_bindings_default, 0), "set_bindings_default", 20, "Remap keybindings using the 'default' mapping rule.", 51, "c:\\work\\4ed\\code\\4coder_remapping_commands.cpp", 46, 51 }, +{ PROC_LINKS(set_bindings_mac_default, 0), "set_bindings_mac_default", 24, "Remap keybindings using the 'mac-default' mapping rule.", 55, "c:\\work\\4ed\\code\\4coder_remapping_commands.cpp", 46, 66 }, +{ PROC_LINKS(miblo_increment_basic, 0), "miblo_increment_basic", 21, "Increment an integer under the cursor by one.", 45, "c:\\work\\4ed\\code\\4coder_miblo_numbers.cpp", 41, 29 }, +{ PROC_LINKS(miblo_decrement_basic, 0), "miblo_decrement_basic", 21, "Decrement an integer under the cursor by one.", 45, "c:\\work\\4ed\\code\\4coder_miblo_numbers.cpp", 41, 44 }, +{ PROC_LINKS(miblo_increment_time_stamp, 0), "miblo_increment_time_stamp", 26, "Increment a time stamp under the cursor by one second. (format [m]m:ss or h:mm:ss", 81, "c:\\work\\4ed\\code\\4coder_miblo_numbers.cpp", 41, 231 }, +{ PROC_LINKS(miblo_decrement_time_stamp, 0), "miblo_decrement_time_stamp", 26, "Decrement a time stamp under the cursor by one second. (format [m]m:ss or h:mm:ss", 81, "c:\\work\\4ed\\code\\4coder_miblo_numbers.cpp", 41, 237 }, +{ PROC_LINKS(miblo_increment_time_stamp_minute, 0), "miblo_increment_time_stamp_minute", 33, "Increment a time stamp under the cursor by one minute. (format [m]m:ss or h:mm:ss", 81, "c:\\work\\4ed\\code\\4coder_miblo_numbers.cpp", 41, 243 }, +{ PROC_LINKS(miblo_decrement_time_stamp_minute, 0), "miblo_decrement_time_stamp_minute", 33, "Decrement a time stamp under the cursor by one minute. (format [m]m:ss or h:mm:ss", 81, "c:\\work\\4ed\\code\\4coder_miblo_numbers.cpp", 41, 249 }, +{ PROC_LINKS(rename_parameter, 0), "rename_parameter", 16, "If the cursor is found to be on the name of a function parameter in the signature of a function definition, all occurences within the scope of the function will be replaced with a new provided string.", 200, "c:\\work\\4ed\\code\\4coder_experiments.cpp", 39, 187 }, +{ PROC_LINKS(write_explicit_enum_values, 0), "write_explicit_enum_values", 26, "If the cursor is found to be on the '{' of an enum definition, the values of the enum will be filled in sequentially starting from zero. Existing values are overwritten.", 170, "c:\\work\\4ed\\code\\4coder_experiments.cpp", 39, 496 }, }; +static int32_t fcoder_metacmd_ID_write_explicit_enum_flags = 0; +static int32_t fcoder_metacmd_ID_seek_beginning_of_textual_line = 1; +static int32_t fcoder_metacmd_ID_seek_end_of_textual_line = 2; +static int32_t fcoder_metacmd_ID_seek_beginning_of_line = 3; +static int32_t fcoder_metacmd_ID_seek_end_of_line = 4; +static int32_t fcoder_metacmd_ID_goto_beginning_of_file = 5; +static int32_t fcoder_metacmd_ID_goto_end_of_file = 6; +static int32_t fcoder_metacmd_ID_change_active_panel = 7; +static int32_t fcoder_metacmd_ID_change_active_panel_backwards = 8; +static int32_t fcoder_metacmd_ID_open_panel_vsplit = 9; +static int32_t fcoder_metacmd_ID_open_panel_hsplit = 10; +static int32_t fcoder_metacmd_ID_suppress_mouse = 11; +static int32_t fcoder_metacmd_ID_allow_mouse = 12; +static int32_t fcoder_metacmd_ID_toggle_mouse = 13; +static int32_t fcoder_metacmd_ID_set_mode_to_original = 14; +static int32_t fcoder_metacmd_ID_set_mode_to_notepad_like = 15; +static int32_t fcoder_metacmd_ID_toggle_highlight_line_at_cursor = 16; +static int32_t fcoder_metacmd_ID_toggle_highlight_enclosing_scopes = 17; +static int32_t fcoder_metacmd_ID_toggle_paren_matching_helper = 18; +static int32_t fcoder_metacmd_ID_toggle_fullscreen = 19; +static int32_t fcoder_metacmd_ID_remap_interactive = 20; +static int32_t fcoder_metacmd_ID_write_character = 21; +static int32_t fcoder_metacmd_ID_write_underscore = 22; +static int32_t fcoder_metacmd_ID_delete_char = 23; +static int32_t fcoder_metacmd_ID_backspace_char = 24; +static int32_t fcoder_metacmd_ID_set_mark = 25; +static int32_t fcoder_metacmd_ID_cursor_mark_swap = 26; +static int32_t fcoder_metacmd_ID_delete_range = 27; +static int32_t fcoder_metacmd_ID_backspace_alpha_numeric_boundary = 28; +static int32_t fcoder_metacmd_ID_delete_alpha_numeric_boundary = 29; +static int32_t fcoder_metacmd_ID_snipe_backward_whitespace_or_token_boundary = 30; +static int32_t fcoder_metacmd_ID_snipe_forward_whitespace_or_token_boundary = 31; +static int32_t fcoder_metacmd_ID_center_view = 32; +static int32_t fcoder_metacmd_ID_left_adjust_view = 33; +static int32_t fcoder_metacmd_ID_click_set_cursor_and_mark = 34; +static int32_t fcoder_metacmd_ID_click_set_cursor = 35; +static int32_t fcoder_metacmd_ID_click_set_cursor_if_lbutton = 36; +static int32_t fcoder_metacmd_ID_click_set_mark = 37; +static int32_t fcoder_metacmd_ID_mouse_wheel_scroll = 38; +static int32_t fcoder_metacmd_ID_move_up = 39; +static int32_t fcoder_metacmd_ID_move_down = 40; +static int32_t fcoder_metacmd_ID_move_up_10 = 41; +static int32_t fcoder_metacmd_ID_move_down_10 = 42; +static int32_t fcoder_metacmd_ID_move_down_textual = 43; +static int32_t fcoder_metacmd_ID_page_up = 44; +static int32_t fcoder_metacmd_ID_page_down = 45; +static int32_t fcoder_metacmd_ID_move_up_to_blank_line = 46; +static int32_t fcoder_metacmd_ID_move_down_to_blank_line = 47; +static int32_t fcoder_metacmd_ID_move_up_to_blank_line_skip_whitespace = 48; +static int32_t fcoder_metacmd_ID_move_down_to_blank_line_skip_whitespace = 49; +static int32_t fcoder_metacmd_ID_move_up_to_blank_line_end = 50; +static int32_t fcoder_metacmd_ID_move_down_to_blank_line_end = 51; +static int32_t fcoder_metacmd_ID_move_left = 52; +static int32_t fcoder_metacmd_ID_move_right = 53; +static int32_t fcoder_metacmd_ID_move_right_whitespace_boundary = 54; +static int32_t fcoder_metacmd_ID_move_left_whitespace_boundary = 55; +static int32_t fcoder_metacmd_ID_move_right_token_boundary = 56; +static int32_t fcoder_metacmd_ID_move_left_token_boundary = 57; +static int32_t fcoder_metacmd_ID_move_right_whitespace_or_token_boundary = 58; +static int32_t fcoder_metacmd_ID_move_left_whitespace_or_token_boundary = 59; +static int32_t fcoder_metacmd_ID_move_right_alpha_numeric_boundary = 60; +static int32_t fcoder_metacmd_ID_move_left_alpha_numeric_boundary = 61; +static int32_t fcoder_metacmd_ID_move_right_alpha_numeric_or_camel_boundary = 62; +static int32_t fcoder_metacmd_ID_move_left_alpha_numeric_or_camel_boundary = 63; +static int32_t fcoder_metacmd_ID_select_all = 64; +static int32_t fcoder_metacmd_ID_to_uppercase = 65; +static int32_t fcoder_metacmd_ID_to_lowercase = 66; +static int32_t fcoder_metacmd_ID_clean_all_lines = 67; +static int32_t fcoder_metacmd_ID_basic_change_active_panel = 68; +static int32_t fcoder_metacmd_ID_close_panel = 69; +static int32_t fcoder_metacmd_ID_show_scrollbar = 70; +static int32_t fcoder_metacmd_ID_hide_scrollbar = 71; +static int32_t fcoder_metacmd_ID_show_filebar = 72; +static int32_t fcoder_metacmd_ID_hide_filebar = 73; +static int32_t fcoder_metacmd_ID_toggle_filebar = 74; +static int32_t fcoder_metacmd_ID_toggle_line_wrap = 75; +static int32_t fcoder_metacmd_ID_toggle_fps_meter = 76; +static int32_t fcoder_metacmd_ID_increase_line_wrap = 77; +static int32_t fcoder_metacmd_ID_decrease_line_wrap = 78; +static int32_t fcoder_metacmd_ID_increase_face_size = 79; +static int32_t fcoder_metacmd_ID_decrease_face_size = 80; +static int32_t fcoder_metacmd_ID_mouse_wheel_change_face_size = 81; +static int32_t fcoder_metacmd_ID_toggle_virtual_whitespace = 82; +static int32_t fcoder_metacmd_ID_toggle_show_whitespace = 83; +static int32_t fcoder_metacmd_ID_toggle_line_numbers = 84; +static int32_t fcoder_metacmd_ID_eol_dosify = 85; +static int32_t fcoder_metacmd_ID_eol_nixify = 86; +static int32_t fcoder_metacmd_ID_exit_4coder = 87; +static int32_t fcoder_metacmd_ID_goto_line = 88; +static int32_t fcoder_metacmd_ID_search = 89; +static int32_t fcoder_metacmd_ID_reverse_search = 90; +static int32_t fcoder_metacmd_ID_search_identifier = 91; +static int32_t fcoder_metacmd_ID_reverse_search_identifier = 92; +static int32_t fcoder_metacmd_ID_replace_in_range = 93; +static int32_t fcoder_metacmd_ID_replace_in_buffer = 94; +static int32_t fcoder_metacmd_ID_replace_in_all_buffers = 95; +static int32_t fcoder_metacmd_ID_query_replace = 96; +static int32_t fcoder_metacmd_ID_query_replace_identifier = 97; +static int32_t fcoder_metacmd_ID_query_replace_selection = 98; +static int32_t fcoder_metacmd_ID_save_all_dirty_buffers = 99; +static int32_t fcoder_metacmd_ID_delete_file_query = 100; +static int32_t fcoder_metacmd_ID_save_to_query = 101; +static int32_t fcoder_metacmd_ID_rename_file_query = 102; +static int32_t fcoder_metacmd_ID_make_directory_query = 103; +static int32_t fcoder_metacmd_ID_move_line_up = 104; +static int32_t fcoder_metacmd_ID_move_line_down = 105; +static int32_t fcoder_metacmd_ID_duplicate_line = 106; +static int32_t fcoder_metacmd_ID_delete_line = 107; +static int32_t fcoder_metacmd_ID_open_file_in_quotes = 108; +static int32_t fcoder_metacmd_ID_open_matching_file_cpp = 109; +static int32_t fcoder_metacmd_ID_view_buffer_other_panel = 110; +static int32_t fcoder_metacmd_ID_swap_buffers_between_panels = 111; +static int32_t fcoder_metacmd_ID_kill_buffer = 112; +static int32_t fcoder_metacmd_ID_save = 113; +static int32_t fcoder_metacmd_ID_reopen = 114; +static int32_t fcoder_metacmd_ID_undo = 115; +static int32_t fcoder_metacmd_ID_redo = 116; +static int32_t fcoder_metacmd_ID_undo_all_buffers = 117; +static int32_t fcoder_metacmd_ID_redo_all_buffers = 118; +static int32_t fcoder_metacmd_ID_open_in_other = 119; +static int32_t fcoder_metacmd_ID_lister__quit = 120; +static int32_t fcoder_metacmd_ID_lister__activate = 121; +static int32_t fcoder_metacmd_ID_lister__write_character = 122; +static int32_t fcoder_metacmd_ID_lister__backspace_text_field = 123; +static int32_t fcoder_metacmd_ID_lister__move_up = 124; +static int32_t fcoder_metacmd_ID_lister__move_down = 125; +static int32_t fcoder_metacmd_ID_lister__wheel_scroll = 126; +static int32_t fcoder_metacmd_ID_lister__mouse_press = 127; +static int32_t fcoder_metacmd_ID_lister__mouse_release = 128; +static int32_t fcoder_metacmd_ID_lister__repaint = 129; +static int32_t fcoder_metacmd_ID_lister__write_character__default = 130; +static int32_t fcoder_metacmd_ID_lister__backspace_text_field__default = 131; +static int32_t fcoder_metacmd_ID_lister__move_up__default = 132; +static int32_t fcoder_metacmd_ID_lister__move_down__default = 133; +static int32_t fcoder_metacmd_ID_lister__write_character__file_path = 134; +static int32_t fcoder_metacmd_ID_lister__backspace_text_field__file_path = 135; +static int32_t fcoder_metacmd_ID_lister__write_character__fixed_list = 136; +static int32_t fcoder_metacmd_ID_interactive_switch_buffer = 137; +static int32_t fcoder_metacmd_ID_interactive_kill_buffer = 138; +static int32_t fcoder_metacmd_ID_interactive_open_or_new = 139; +static int32_t fcoder_metacmd_ID_interactive_new = 140; +static int32_t fcoder_metacmd_ID_interactive_open = 141; +static int32_t fcoder_metacmd_ID_command_lister = 142; +static int32_t fcoder_metacmd_ID_auto_tab_whole_file = 143; +static int32_t fcoder_metacmd_ID_auto_tab_line_at_cursor = 144; +static int32_t fcoder_metacmd_ID_auto_tab_range = 145; +static int32_t fcoder_metacmd_ID_write_and_auto_tab = 146; +static int32_t fcoder_metacmd_ID_list_all_locations = 147; +static int32_t fcoder_metacmd_ID_list_all_substring_locations = 148; +static int32_t fcoder_metacmd_ID_list_all_locations_case_insensitive = 149; +static int32_t fcoder_metacmd_ID_list_all_substring_locations_case_insensitive = 150; +static int32_t fcoder_metacmd_ID_list_all_locations_of_identifier = 151; +static int32_t fcoder_metacmd_ID_list_all_locations_of_identifier_case_insensitive = 152; +static int32_t fcoder_metacmd_ID_list_all_locations_of_selection = 153; +static int32_t fcoder_metacmd_ID_list_all_locations_of_selection_case_insensitive = 154; +static int32_t fcoder_metacmd_ID_list_all_locations_of_type_definition = 155; +static int32_t fcoder_metacmd_ID_list_all_locations_of_type_definition_of_identifier = 156; +static int32_t fcoder_metacmd_ID_word_complete = 157; +static int32_t fcoder_metacmd_ID_goto_jump_at_cursor = 158; +static int32_t fcoder_metacmd_ID_goto_jump_at_cursor_same_panel = 159; +static int32_t fcoder_metacmd_ID_goto_next_jump = 160; +static int32_t fcoder_metacmd_ID_goto_prev_jump = 161; +static int32_t fcoder_metacmd_ID_goto_next_jump_no_skips = 162; +static int32_t fcoder_metacmd_ID_goto_prev_jump_no_skips = 163; +static int32_t fcoder_metacmd_ID_goto_first_jump = 164; +static int32_t fcoder_metacmd_ID_goto_first_jump_same_panel_sticky = 165; +static int32_t fcoder_metacmd_ID_newline_or_goto_position = 166; +static int32_t fcoder_metacmd_ID_newline_or_goto_position_same_panel = 167; +static int32_t fcoder_metacmd_ID_view_jump_list_with_lister = 168; +static int32_t fcoder_metacmd_ID_log_graph__escape = 169; +static int32_t fcoder_metacmd_ID_log_graph__scroll_wheel = 170; +static int32_t fcoder_metacmd_ID_log_graph__page_up = 171; +static int32_t fcoder_metacmd_ID_log_graph__page_down = 172; +static int32_t fcoder_metacmd_ID_log_graph__click_select_event = 173; +static int32_t fcoder_metacmd_ID_log_graph__click_jump_to_event_source = 174; +static int32_t fcoder_metacmd_ID_show_the_log_graph = 175; +static int32_t fcoder_metacmd_ID_copy = 176; +static int32_t fcoder_metacmd_ID_cut = 177; +static int32_t fcoder_metacmd_ID_paste = 178; +static int32_t fcoder_metacmd_ID_paste_next = 179; +static int32_t fcoder_metacmd_ID_paste_and_indent = 180; +static int32_t fcoder_metacmd_ID_paste_next_and_indent = 181; +static int32_t fcoder_metacmd_ID_execute_previous_cli = 182; +static int32_t fcoder_metacmd_ID_execute_any_cli = 183; +static int32_t fcoder_metacmd_ID_build_search = 184; +static int32_t fcoder_metacmd_ID_build_in_build_panel = 185; +static int32_t fcoder_metacmd_ID_close_build_panel = 186; +static int32_t fcoder_metacmd_ID_change_to_build_panel = 187; +static int32_t fcoder_metacmd_ID_close_all_code = 188; +static int32_t fcoder_metacmd_ID_open_all_code = 189; +static int32_t fcoder_metacmd_ID_open_all_code_recursive = 190; +static int32_t fcoder_metacmd_ID_load_project = 191; +static int32_t fcoder_metacmd_ID_project_fkey_command = 192; +static int32_t fcoder_metacmd_ID_project_go_to_root_directory = 193; +static int32_t fcoder_metacmd_ID_setup_new_project = 194; +static int32_t fcoder_metacmd_ID_setup_build_bat = 195; +static int32_t fcoder_metacmd_ID_setup_build_sh = 196; +static int32_t fcoder_metacmd_ID_setup_build_bat_and_sh = 197; +static int32_t fcoder_metacmd_ID_project_command_lister = 198; +static int32_t fcoder_metacmd_ID_list_all_functions_current_buffer = 199; +static int32_t fcoder_metacmd_ID_list_all_functions_current_buffer_lister = 200; +static int32_t fcoder_metacmd_ID_list_all_functions_all_buffers = 201; +static int32_t fcoder_metacmd_ID_list_all_functions_all_buffers_lister = 202; +static int32_t fcoder_metacmd_ID_select_surrounding_scope = 203; +static int32_t fcoder_metacmd_ID_select_next_scope_absolute = 204; +static int32_t fcoder_metacmd_ID_select_prev_scope_absolute = 205; +static int32_t fcoder_metacmd_ID_place_in_scope = 206; +static int32_t fcoder_metacmd_ID_delete_current_scope = 207; +static int32_t fcoder_metacmd_ID_scope_absorb_down = 208; +static int32_t fcoder_metacmd_ID_open_long_braces = 209; +static int32_t fcoder_metacmd_ID_open_long_braces_semicolon = 210; +static int32_t fcoder_metacmd_ID_open_long_braces_break = 211; +static int32_t fcoder_metacmd_ID_if0_off = 212; +static int32_t fcoder_metacmd_ID_write_todo = 213; +static int32_t fcoder_metacmd_ID_write_hack = 214; +static int32_t fcoder_metacmd_ID_write_note = 215; +static int32_t fcoder_metacmd_ID_write_block = 216; +static int32_t fcoder_metacmd_ID_write_zero_struct = 217; +static int32_t fcoder_metacmd_ID_comment_line = 218; +static int32_t fcoder_metacmd_ID_uncomment_line = 219; +static int32_t fcoder_metacmd_ID_comment_line_toggle = 220; +static int32_t fcoder_metacmd_ID_snippet_lister = 221; +static int32_t fcoder_metacmd_ID_set_bindings_choose = 222; +static int32_t fcoder_metacmd_ID_set_bindings_default = 223; +static int32_t fcoder_metacmd_ID_set_bindings_mac_default = 224; +static int32_t fcoder_metacmd_ID_miblo_increment_basic = 225; +static int32_t fcoder_metacmd_ID_miblo_decrement_basic = 226; +static int32_t fcoder_metacmd_ID_miblo_increment_time_stamp = 227; +static int32_t fcoder_metacmd_ID_miblo_decrement_time_stamp = 228; +static int32_t fcoder_metacmd_ID_miblo_increment_time_stamp_minute = 229; +static int32_t fcoder_metacmd_ID_miblo_decrement_time_stamp_minute = 230; +static int32_t fcoder_metacmd_ID_rename_parameter = 231; +static int32_t fcoder_metacmd_ID_write_explicit_enum_values = 232; #endif diff --git a/4coder_lex_gen_cpp.cpp b/4coder_lex_gen_cpp.cpp new file mode 100644 index 00000000..bf575fe4 --- /dev/null +++ b/4coder_lex_gen_cpp.cpp @@ -0,0 +1,1105 @@ +/* +4coder_lex_gen_cpp.cpp - Model definition for a C++ lexer. +*/ + +// TOP + +#define LANG_NAME_LOWER cpp +#define LANG_NAME_CAMEL Cpp + +#include "lexer_generator/4coder_lex_gen_main.cpp" + +internal void +build_language_model(void){ + u8 utf8[129]; + smh_utf8_fill(utf8); + + smh_set_base_character_names(); + smh_typical_tokens(); + + // CPP Names + sm_char_name('!', "Not"); + sm_char_name('&', "And"); + sm_char_name('|', "Or"); + sm_char_name('%', "Mod"); + sm_char_name('^', "Xor"); + sm_char_name('?', "Ternary"); + sm_char_name('/', "Div"); + + // CPP Direct Toke Kinds + sm_select_base_kind(TokenBaseKind_Comment); + sm_direct_token_kind("BlockComment"); + sm_direct_token_kind("LineComment"); + + sm_select_base_kind(TokenBaseKind_Whitespace); + sm_direct_token_kind("Backslash"); + + sm_select_base_kind(TokenBaseKind_LiteralInteger); + sm_direct_token_kind("LiteralInteger"); + sm_direct_token_kind("LiteralIntegerU"); + sm_direct_token_kind("LiteralIntegerL"); + sm_direct_token_kind("LiteralIntegerUL"); + sm_direct_token_kind("LiteralIntegerLL"); + sm_direct_token_kind("LiteralIntegerULL"); + sm_direct_token_kind("LiteralIntegerHex"); + sm_direct_token_kind("LiteralIntegerHexU"); + sm_direct_token_kind("LiteralIntegerHexL"); + sm_direct_token_kind("LiteralIntegerHexUL"); + sm_direct_token_kind("LiteralIntegerHexLL"); + sm_direct_token_kind("LiteralIntegerHexULL"); + sm_direct_token_kind("LiteralIntegerOct"); + sm_direct_token_kind("LiteralIntegerOctU"); + sm_direct_token_kind("LiteralIntegerOctL"); + sm_direct_token_kind("LiteralIntegerOctUL"); + sm_direct_token_kind("LiteralIntegerOctLL"); + sm_direct_token_kind("LiteralIntegerOctULL"); + + sm_select_base_kind(TokenBaseKind_LiteralFloat); + sm_direct_token_kind("LiteralFloat32"); + sm_direct_token_kind("LiteralFloat64"); + + sm_select_base_kind(TokenBaseKind_LiteralString); + sm_direct_token_kind("LiteralString"); + sm_direct_token_kind("LiteralStringWide"); + sm_direct_token_kind("LiteralStringUTF8"); + sm_direct_token_kind("LiteralStringUTF16"); + sm_direct_token_kind("LiteralStringUTF32"); + sm_direct_token_kind("LiteralStringRaw"); + sm_direct_token_kind("LiteralStringWideRaw"); + sm_direct_token_kind("LiteralStringUTF8Raw"); + sm_direct_token_kind("LiteralStringUTF16Raw"); + sm_direct_token_kind("LiteralStringUTF32Raw"); + sm_direct_token_kind("LiteralCharacter"); + sm_direct_token_kind("LiteralCharacterWide"); + sm_direct_token_kind("LiteralCharacterUTF8"); + sm_direct_token_kind("LiteralCharacterUTF16"); + sm_direct_token_kind("LiteralCharacterUTF32"); + sm_direct_token_kind("PPIncludeFile"); + sm_direct_token_kind("PPErrorMessage"); + + sm_select_base_kind(TokenBaseKind_Keyword); + sm_direct_token_kind("KeywordGeneric"); + + // CPP Operators + Operator_Set *main_ops = sm_begin_op_set(); + + sm_select_base_kind(TokenBaseKind_ScopeOpen); + sm_op("{"); + sm_select_base_kind(TokenBaseKind_ScopeClose); + sm_op("}"); + sm_select_base_kind(TokenBaseKind_ParentheticalOpen); + sm_op("("); + sm_op("["); + sm_select_base_kind(TokenBaseKind_ParentheticalClose); + sm_op(")"); + sm_op("]"); + sm_select_base_kind(TokenBaseKind_Operator); + sm_op(";"); + sm_op(":"); + sm_op("..."); + + sm_op("::"); + sm_op("++"); + sm_op("--"); + sm_op("."); + sm_op("->", "Arrow"); + sm_op("+"); + sm_op("-"); + sm_op("!"); + sm_op("~"); + sm_op("*"); + sm_op("&"); + sm_op(".*"); + sm_op("->*", "ArrowStar"); + sm_op("/"); + sm_op("%"); + + sm_char_name('<', "Left"); + sm_char_name('>', "Right"); + sm_op("<<"); + sm_op(">>"); + + sm_op("<=>", "Compare"); + + sm_char_name('<', "Less"); + sm_char_name('>', "Grtr"); + sm_op("<"); + sm_op("<="); + sm_op(">"); + sm_op(">="); + sm_op("=="); + sm_op("!="); + + sm_op("^"); + sm_op("|"); + sm_op("&&"); + sm_op("||"); + sm_op("?"); + sm_op("="); + sm_op("+="); + sm_op("-="); + sm_op("*="); + sm_op("/="); + sm_op("%="); + + sm_char_name('<', "Left"); + sm_char_name('>', "Right"); + sm_op("<<="); + sm_op(">>="); + + sm_op(","); + + // CPP Preprocess Operators + Operator_Set *pp_ops = sm_begin_op_set(); + + sm_op("#", "PPStringify"); + sm_op("##", "PPConcat"); + + // CPP Keywords + Keyword_Set *main_keys = sm_begin_key_set("main_keys"); + + sm_select_base_kind(TokenBaseKind_Keyword); + sm_key("Void"); + sm_key("Bool"); + sm_key("Char"); + sm_key("Int"); + sm_key("Float"); + sm_key("Double"); + sm_key("Long"); + sm_key("Short"); + sm_key("Unsigned"); + sm_key("Signed"); + sm_key("Const"); + sm_key("Volatile"); + sm_key("Asm"); + sm_key("Break"); + sm_key("Case"); + sm_key("Catch"); + sm_key("Continue"); + sm_key("Default"); + sm_key("Do"); + sm_key("Else"); + sm_key("For"); + sm_key("Goto"); + sm_key("If"); + sm_key("Return"); + sm_key("Switch"); + sm_key("Try"); + sm_key("While"); + sm_key("StaticAssert", "static_assert"); + sm_key("ConstCast", "const_cast"); + sm_key("DynamicCast", "dynamic_cast"); + sm_key("ReinterpretCast", "reinterpret_cast"); + sm_key("StaticCast", "static_cast"); + sm_key("Class"); + sm_key("Enum"); + sm_key("Struct"); + sm_key("Typedef"); + sm_key("Union"); + sm_key("Template"); + sm_key("Typename"); + sm_key("Friend"); + sm_key("Namespace"); + sm_key("Private"); + sm_key("Protected"); + sm_key("Public"); + sm_key("Using"); + sm_key("Extern"); + sm_key("Export"); + sm_key("Inline"); + sm_key("Static"); + sm_key("Virtual"); + sm_key("AlignAs"); + sm_key("Explicit"); + sm_key("NoExcept"); + sm_key("NullPtr"); + sm_key("Operator"); + sm_key("Register"); + sm_key("This"); + sm_key("ThreadLocal", "thread_local"); + sm_key("SizeOf"); + sm_key("AlignOf"); + sm_key("DeclType"); + sm_key("TypeID"); + sm_key("New"); + sm_key("Delete"); + + sm_select_base_kind(TokenBaseKind_LiteralInteger); + sm_key("LiteralTrue", "true"); + sm_key("LiteralFalse", "false"); + + sm_select_base_kind(TokenBaseKind_Identifier); + sm_key_fallback("Identifier"); + + // CPP Preprocess Directives + Keyword_Set *pp_directive_set = sm_begin_key_set("pp_directives"); + + sm_select_base_kind(TokenBaseKind_Preprocessor); + sm_key("PPInclude", "#include"); + sm_key("PPVersion", "#version"); + sm_key("PPDefine", "#define"); + sm_key("PPUndef", "#undef"); + sm_key("PPIf", "#if"); + sm_key("PPIfDef", "#ifdef"); + sm_key("PPIfNDef", "#ifndef"); + sm_key("PPElse", "#else"); + sm_key("PPElIf", "#elif"); + sm_key("PPEndIf", "#endif"); + sm_key("PPError", "#error"); + sm_key("PPImport", "#import"); + sm_key("PPUsing", "#using"); + sm_key("PPLine", "#line"); + sm_key("PPPragma", "#pragma"); + + sm_select_base_kind(TokenBaseKind_LexError); + sm_key_fallback("PPUnknown"); + + // CPP Preprocess Keywords + Keyword_Set *pp_keys = sm_begin_key_set("pp_keys"); + + sm_select_base_kind(TokenBaseKind_Keyword); + sm_key("PPDefined", "defined"); + + // State Machine + State *root = sm_begin_state_machine(); + + Flag *is_hex = sm_add_flag(FlagResetRule_AutoZero); + Flag *is_oct = sm_add_flag(FlagResetRule_AutoZero); + Flag *is_wide = sm_add_flag(FlagResetRule_AutoZero); + Flag *is_utf8 = sm_add_flag(FlagResetRule_AutoZero); + Flag *is_utf16 = sm_add_flag(FlagResetRule_AutoZero); + Flag *is_utf32 = sm_add_flag(FlagResetRule_AutoZero); + Flag *is_char = sm_add_flag(FlagResetRule_AutoZero); + + Flag *is_pp_body = sm_add_flag(FlagResetRule_KeepState); + Flag *is_include_body = sm_add_flag(FlagResetRule_KeepState); + Flag *is_error_body = sm_add_flag(FlagResetRule_KeepState); + + sm_flag_bind(is_pp_body, TokenBaseFlag_PreprocessorBody); + +#define AddState(N) State *N = sm_add_state(#N) + + AddState(identifier); + AddState(whitespace); + AddState(whitespace_end_pp); + AddState(error_body); + AddState(backslash); + + AddState(operator_or_fnumber_dot); + AddState(operator_or_comment_slash); + + AddState(number); + AddState(znumber); + + AddState(fnumber_decimal); + AddState(fnumber_exponent); + AddState(fnumber_exponent_sign); + AddState(fnumber_exponent_digits); + + AddState(number_hex_first); + AddState(number_hex); + AddState(number_oct); + + AddState(U_number); + AddState(L_number); + AddState(UL_number); + AddState(LU_number); + AddState(l_number); + AddState(Ul_number); + AddState(lU_number); + AddState(LL_number); + AddState(ULL_number); + + AddState(pp_directive); + + AddState(include_pointy); + AddState(include_quotes); + + AddState(pre_L); + AddState(pre_u); + AddState(pre_U); + AddState(pre_u8); + AddState(pre_R); + + AddState(character); + AddState(string); + AddState(string_esc); + AddState(string_esc_oct2); + AddState(string_esc_oct1); + AddState(string_esc_hex); + AddState(string_esc_universal_8); + AddState(string_esc_universal_7); + AddState(string_esc_universal_6); + AddState(string_esc_universal_5); + AddState(string_esc_universal_4); + AddState(string_esc_universal_3); + AddState(string_esc_universal_2); + AddState(string_esc_universal_1); + + AddState(raw_string); + AddState(raw_string_get_delim); + AddState(raw_string_finish_delim); + AddState(raw_string_find_close); + AddState(raw_string_try_delim); + AddState(raw_string_try_quote); + + AddState(comment_block); + AddState(comment_block_try_close); + AddState(comment_block_newline); + AddState(comment_line); + + Operator_Set *main_ops_without_dot_or_slash = smo_copy_op_set(main_ops); + smo_remove_ops_with_prefix(main_ops_without_dot_or_slash, "."); + smo_remove_ops_with_prefix(main_ops_without_dot_or_slash, "/"); + + Operator_Set *main_ops_with_dot = smo_copy_op_set(main_ops); + smo_remove_ops_without_prefix(main_ops_with_dot, "."); + smo_ops_string_skip(main_ops_with_dot, 1); + + //// + + sm_select_state(root); + + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("EOF"); + sm_case_eof(emit); + } + + sm_case("abcdefghijklmnopqrstvwxyz" + "ABCDEFGHIJKMNOPQSTVWXYZ" + "_$", + identifier); + sm_case(utf8, identifier); + sm_case("L", pre_L); + sm_case("u", pre_u); + sm_case("U", pre_U); + sm_case("R", pre_R); + + sm_case_flagged(is_error_body, true, " \r\t\f\v", error_body); + sm_case_flagged(is_error_body, false, " \r\t\f\v", whitespace); + sm_case("\n", whitespace_end_pp); + sm_case("\\", backslash); + + sm_case(".", operator_or_fnumber_dot); + sm_case("/", operator_or_comment_slash); + { + Character_Set *char_set = smo_new_char_set(); + smo_char_set_union_ops_firsts(char_set, main_ops_without_dot_or_slash); + smo_char_set_remove(char_set, ".", emit); + } + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_fallback_peek(emit); + } + + //// + + sm_select_state(include_quotes); + sm_case("abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "_. /\\" + "0123456789", + include_quotes); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("PPIncludeFile"); + sm_case("\"", emit); + } + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_fallback_peek(emit); + } + + //// + + sm_select_state(pre_L); + sm_set_flag(is_wide, true); + sm_case("\"", string); + sm_case("R", pre_R); + sm_fallback_peek(identifier); + + //// + + sm_select_state(pre_u); + sm_set_flag(is_utf16, true); + sm_case("\"", string); + sm_case("8", pre_u8); + sm_case("R", pre_R); + sm_fallback_peek(identifier); + + //// + + sm_select_state(pre_U); + sm_set_flag(is_utf32, true); + sm_case("\"", string); + sm_case("R", pre_R); + sm_fallback_peek(identifier); + + //// + + sm_select_state(pre_u8); + sm_set_flag(is_utf8, true); + sm_case("\"", string); + sm_case("R", pre_R); + sm_fallback_peek(identifier); + + //// + + sm_select_state(pre_R); + sm_case("\"", raw_string); + sm_fallback_peek(identifier); + + //// + + sm_select_state(character); + sm_set_flag(is_char, true); + sm_fallback_peek(string); + + //// + + sm_select_state(string); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct(is_wide, "LiteralStringWide"); + sm_emit_handler_direct(is_utf8 , "LiteralStringUTF8"); + sm_emit_handler_direct(is_utf16, "LiteralStringUTF16"); + sm_emit_handler_direct(is_utf32, "LiteralStringUTF32"); + sm_emit_handler_direct("LiteralString"); + sm_case_flagged(is_char, false, "\"", emit); + } + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct(is_wide, "LiteralCharacterWide"); + sm_emit_handler_direct(is_utf8 , "LiteralCharacterUTF8"); + sm_emit_handler_direct(is_utf16, "LiteralCharacterUTF16"); + sm_emit_handler_direct(is_utf32, "LiteralCharacterUTF32"); + sm_emit_handler_direct("LiteralCharacter"); + sm_case_flagged(is_char, true, "\'", emit); + } + sm_case("\\", string_esc); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_case_peek("\n", emit); + } + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_case_eof_peek(emit); + } + sm_case_flagged(is_char, true, "\"", string); + sm_case_flagged(is_char, false, "\'", string); + sm_fallback(string); + + //// + + sm_select_state(string_esc); + sm_case("\n'\"?\\abfnrtv", string); + sm_case("01234567", string_esc_oct2); + sm_case("x", string_esc_hex); + sm_case("u", string_esc_universal_4); + sm_case("U", string_esc_universal_8); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_case_peek("\n", emit); + } + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_case_eof_peek(emit); + } + sm_fallback(string); + + //// + + sm_select_state(string_esc_oct2); + sm_case("01234567", string_esc_oct1); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_oct1); + sm_case("01234567", string); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_hex); + sm_case("0123456789abcdefABCDEF", string_esc_hex); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_universal_8); + sm_case("0123456789abcdefABCDEF", string_esc_universal_7); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_universal_7); + sm_case("0123456789abcdefABCDEF", string_esc_universal_6); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_universal_6); + sm_case("0123456789abcdefABCDEF", string_esc_universal_5); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_universal_5); + sm_case("0123456789abcdefABCDEF", string_esc_universal_4); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_universal_4); + sm_case("0123456789abcdefABCDEF", string_esc_universal_3); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_universal_3); + sm_case("0123456789abcdefABCDEF", string_esc_universal_2); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_universal_2); + sm_case("0123456789abcdefABCDEF", string_esc_universal_1); + sm_fallback_peek(string); + + //// + + sm_select_state(string_esc_universal_1); + sm_case("0123456789abcdefABCDEF", string); + sm_fallback_peek(string); + + //// + + sm_select_state(raw_string); + sm_delim_mark_first(); + sm_fallback_peek(raw_string_get_delim); + + //// + + sm_select_state(raw_string_get_delim); + sm_case_peek("(", raw_string_finish_delim); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_case(" \\)", emit); + } + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_case_eof_peek(emit); + } + sm_fallback(raw_string_get_delim); + + //// + + sm_select_state(raw_string_finish_delim); + sm_delim_mark_one_past_last(); + sm_fallback_peek(raw_string_find_close); + + //// + + sm_select_state(raw_string_find_close); + sm_case(")", raw_string_try_delim); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LexError"); + sm_case_eof_peek(emit); + } + sm_fallback(raw_string_find_close); + + //// + + sm_select_state(raw_string_try_delim); + sm_match_delim(raw_string_try_quote, raw_string_find_close); + + //// + + sm_select_state(raw_string_try_quote); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct(is_wide, "LiteralStringWideRaw"); + sm_emit_handler_direct(is_utf8 , "LiteralStringUTF8Raw"); + sm_emit_handler_direct(is_utf16, "LiteralStringUTF16Raw"); + sm_emit_handler_direct(is_utf32, "LiteralStringUTF32Raw"); + sm_emit_handler_direct("LiteralStringRaw"); + sm_case("\"", emit); + } + sm_fallback_peek(raw_string_find_close); + + //// + + sm_select_state(comment_block); + sm_case("*", comment_block_try_close); + sm_case("\n", comment_block_newline); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("BlockComment"); + sm_case_eof_peek(emit); + } + sm_fallback(comment_block); + + //// + + sm_select_state(comment_block_try_close); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("BlockComment"); + sm_case("/", emit); + } + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("BlockComment"); + sm_case_eof_peek(emit); + } + sm_case("*", comment_block_try_close); + sm_fallback(comment_block); + + //// + + sm_select_state(comment_block_newline); + sm_set_flag(is_pp_body, false); + sm_set_flag(is_include_body, false); + sm_fallback_peek(comment_block); + + //// + + sm_select_state(comment_line); + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LineComment"); + sm_case_peek("\n", emit); + } + { + Emit_Rule *emit = sm_emit_rule(); + sm_emit_handler_direct("LineComment"); + sm_case_eof_peek(emit); + } + sm_fallback(comment_line); + +} + +// BOTTOM + diff --git a/4coder_metadata_generator.cpp b/4coder_metadata_generator.cpp index 6b7f08f1..e6d1c2a0 100644 --- a/4coder_metadata_generator.cpp +++ b/4coder_metadata_generator.cpp @@ -8,13 +8,16 @@ #include "4coder_base_types.h" #include "4coder_token.h" +#include "languages/generated_lexer_cpp.h" #include "4coder_base_types.cpp" #include "4coder_stringf.cpp" #include "4coder_malloc_allocator.cpp" +#include "4coder_token.cpp" +#include "languages/generated_lexer_cpp.cpp" + #include "4coder_file.h" -#include "languages/4coder_language_cpp.h" #include #include @@ -412,7 +415,7 @@ require_define(Reader *reader, i64 *opt_pos_out){ b32 success = false; Token token = get_token(reader); - if (token.sub_kind == TokenCppKind_Define){ + if (token.sub_kind == TokenCppKind_PPDefine){ success = true; if (opt_pos_out != 0){ *opt_pos_out = token.pos; diff --git a/4coder_table.cpp b/4coder_table.cpp index 400da2a9..a946cee0 100644 --- a/4coder_table.cpp +++ b/4coder_table.cpp @@ -46,7 +46,8 @@ internal Table_Lookup table_lookup(Table_u64_u64 *table, u64 key){ Table_Lookup result = {}; - if (key != table_empty_key && key != table_erased_key){ + if (key != table_empty_key && key != table_erased_key && + table->slot_count > 0){ u64 *keys = table->keys; u32 slot_count = table->slot_count; @@ -205,7 +206,8 @@ internal Table_Lookup table_lookup(Table_u32_u16 *table, u32 key){ Table_Lookup result = {}; - if (key != table_empty_u32_key && key != table_erased_u32_key){ + if (key != table_empty_u32_key && key != table_erased_u32_key && + table->slot_count > 0){ u32 *keys = table->keys; u32 slot_count = table->slot_count; @@ -357,40 +359,43 @@ table_free(Table_Data_u64 *table){ internal Table_Lookup table_lookup(Table_Data_u64 *table, Data key){ - u64 *hashes = table->hashes; - u32 slot_count = table->slot_count; - - u64 hash = table_hash(key); - u32 first_index = hash % slot_count; - u32 index = first_index; Table_Lookup result = {}; - result.hash = hash; - for (;;){ - if (hash == hashes[index]){ - if (data_match(key, table->keys[index])){ + + if (table->slot_count > 0){ + u64 *hashes = table->hashes; + u32 slot_count = table->slot_count; + + u64 hash = table_hash(key); + u32 first_index = hash % slot_count; + u32 index = first_index; + result.hash = hash; + for (;;){ + if (hash == hashes[index]){ + if (data_match(key, table->keys[index])){ + result.index = index; + result.found_match = true; + result.found_empty_slot = false; + result.found_erased_slot = false; + break; + } + } + if (table_empty_slot == hashes[index]){ result.index = index; - result.found_match = true; - result.found_empty_slot = false; + result.found_empty_slot = true; result.found_erased_slot = false; break; } - } - if (table_empty_slot == hashes[index]){ - result.index = index; - result.found_empty_slot = true; - result.found_erased_slot = false; - break; - } - if (table_erased_slot == hashes[index] && !result.found_erased_slot){ - result.index = index; - result.found_erased_slot = true; - } - index += 1; - if (index >= slot_count){ - index = 0; - } - if (index == first_index){ - break; + if (table_erased_slot == hashes[index] && !result.found_erased_slot){ + result.index = index; + result.found_erased_slot = true; + } + index += 1; + if (index >= slot_count){ + index = 0; + } + if (index == first_index){ + break; + } } } @@ -514,7 +519,8 @@ internal Table_Lookup table_lookup(Table_u64_Data *table, u64 key){ Table_Lookup result = {}; - if (key != table_empty_key && key != table_erased_key){ + if (key != table_empty_key && key != table_erased_key && + table->slot_count > 0){ u64 *keys = table->keys; u32 slot_count = table->slot_count; @@ -622,9 +628,8 @@ table_insert(Table_u64_Data *table, u64 key, Data val){ } internal b32 -table_erase(Table_u64_Data *table, u64 key){ +table_erase(Table_u64_Data *table, Table_Lookup lookup){ b32 result = false; - Table_Lookup lookup = table_lookup(table, key); if (lookup.found_match){ table->keys[lookup.index] = 0; block_zero_struct(&table->vals[lookup.index]); @@ -634,6 +639,12 @@ table_erase(Table_u64_Data *table, u64 key){ return(result); } +internal b32 +table_erase(Table_u64_Data *table, u64 key){ + Table_Lookup lookup = table_lookup(table, key); + return(table_erase(table, lookup)); +} + internal void table_clear(Table_u64_Data *table){ block_zero_dynamic_array(table->keys, table->slot_count); @@ -667,40 +678,43 @@ table_free(Table_Data_Data *table){ internal Table_Lookup table_lookup(Table_Data_Data *table, Data key){ - u64 *hashes = table->hashes; - u32 slot_count = table->slot_count; - - u64 hash = table_hash(key); - u32 first_index = hash % slot_count; - u32 index = first_index; Table_Lookup result = {}; - result.hash = hash; - for (;;){ - if (hash == hashes[index]){ - if (data_match(key, table->keys[index])){ + + if (table->slot_count > 0){ + u64 *hashes = table->hashes; + u32 slot_count = table->slot_count; + + u64 hash = table_hash(key); + u32 first_index = hash % slot_count; + u32 index = first_index; + result.hash = hash; + for (;;){ + if (hash == hashes[index]){ + if (data_match(key, table->keys[index])){ + result.index = index; + result.found_match = true; + result.found_empty_slot = false; + result.found_erased_slot = false; + break; + } + } + if (table_empty_slot == hashes[index]){ result.index = index; - result.found_match = true; - result.found_empty_slot = false; + result.found_empty_slot = true; result.found_erased_slot = false; break; } - } - if (table_empty_slot == hashes[index]){ - result.index = index; - result.found_empty_slot = true; - result.found_erased_slot = false; - break; - } - if (table_erased_slot == hashes[index] && !result.found_erased_slot){ - result.index = index; - result.found_erased_slot = true; - } - index += 1; - if (index >= slot_count){ - index = 0; - } - if (index == first_index){ - break; + if (table_erased_slot == hashes[index] && !result.found_erased_slot){ + result.index = index; + result.found_erased_slot = true; + } + index += 1; + if (index >= slot_count){ + index = 0; + } + if (index == first_index){ + break; + } } } diff --git a/4coder_token.cpp b/4coder_token.cpp new file mode 100644 index 00000000..fea7a739 --- /dev/null +++ b/4coder_token.cpp @@ -0,0 +1,27 @@ +/* + * 4coder token types + */ + +// TOP + +internal void +token_list_push(Arena *arena, Token_List *list, Token *token){ + Token_Block *block = list->last; + if (block == 0 || block->count + 1 > block->max){ + block = push_array(arena, Token_Block, 1); + block->next = 0; + block->prev = 0; + u32 new_max = round_up_u32(1, KB(4)); + block->tokens = push_array(arena, Token, new_max); + block->count = 0; + block->max = new_max; + zdll_push_back(list->first, list->last, block); + list->node_count += 1; + } + block_copy_struct(&block->tokens[block->count], token); + block->count += 1; + list->total_count += 1; +} + +// BOTTOM + diff --git a/4coder_token.h b/4coder_token.h index a9f33ec9..b1aa966b 100644 --- a/4coder_token.h +++ b/4coder_token.h @@ -28,6 +28,24 @@ enum{ TokenBaseKind_COUNT, }; +char *token_base_kind_names[] ={ + "EOF", + "Whitespace", + "LexError", + "Comment", + "Keyword", + "Preprocessor", + "Identifier", + "Operator", + "LiteralInteger", + "LiteralFloat", + "LiteralString", + "ScopeOpen", + "ScopeClose", + "ParentheticalOpen", + "ParentheticalClose", +}; + typedef u16 Token_Base_Flag; enum{ TokenBaseFlag_PreprocessorBody = 1, @@ -48,6 +66,21 @@ struct Token_Array{ i64 max; }; +struct Token_Block{ + Token_Block *next; + Token_Block *prev; + Token *tokens; + i32 count; + i32 max; +}; + +struct Token_List{ + Token_Block *first; + Token_Block *last; + i32 node_count; + i32 total_count; +}; + #endif // BOTTOM diff --git a/4ed_api_implementation.cpp b/4ed_api_implementation.cpp index fc95fab9..e31d2812 100644 --- a/4ed_api_implementation.cpp +++ b/4ed_api_implementation.cpp @@ -1956,7 +1956,7 @@ Managed_Scope_Get_Attachment(Application_Links *app, Managed_Scope scope, Manage } else{ #define M \ - "ERROR: scope attachment already exists with a size smaller than the requested size; no attachment pointer can returned." + "ERROR: scope attachment already exists with a size smaller than the requested size; no attachment pointer can be returned." print_message(app, string_u8_litexpr(M)); } } diff --git a/build_generator.bat b/build_generator.bat new file mode 100644 index 00000000..3d8345e0 --- /dev/null +++ b/build_generator.bat @@ -0,0 +1,33 @@ +@echo off + +REM usage: build_generator [release] +REM src-file: a relative path to the target of a unity-build for a generator +REM binary-output-path: a relative path where the generator.exe will be written, assumed to be "." if unset + +set code_home=%~dp0 +if %code_home:~-1%==\ (set code_home=%code_home:~0,-1%) + +if NOT "%Platform%" == "X64" IF NOT "%Platform%" == "x64" (call "%code_home%\windows_scripts\setup_cl_x64.bat") + +set src=%1 +if "%src%" == "" (echo error: no input file & exit) + +set dst=%2 +if "%dst%" == "" (set dst=".") + +set opts=/W4 /wd4310 /wd4100 /wd4201 /wd4505 /wd4996 /wd4127 /wd4510 /wd4512 /wd4610 /wd4457 /wd4146 /WX +set opts=%opts% /GR- /nologo /FC +set debug=/Zi +set release=/O2 /Zi + +set mode=%debug% +if "%3" == "release" (set mode=%release%) + +set full_src="%cd%\%src%" + +pushd %dst% +call cl /I"%code_home%" %opts% %mode% %full_src% /Fegenerator +popd + + + diff --git a/buildsuper_x64.bat b/buildsuper_x64.bat index 496d96dc..aceb62a5 100644 --- a/buildsuper_x64.bat +++ b/buildsuper_x64.bat @@ -1,9 +1,5 @@ @echo off -REM This stores the path of the buildsuper.bat script -REM in CODE_HOME. This way you can always include the -REM default files no matter where you store your code. -REM And no matter how you call buildsuper.bat. set code_home=%~dp0 if %code_home:~-1%==\ (set code_home=%code_home:~0,-1%) diff --git a/languages/4coder_language_cpp.cpp b/languages/4coder_language_cpp.cpp deleted file mode 100644 index a69b3b26..00000000 --- a/languages/4coder_language_cpp.cpp +++ /dev/null @@ -1,24 +0,0 @@ -/* -4coder_language_cpp.cpp - C++ language parser. -*/ - -// TOP - -internal Token_Array -lex_cpp_initial(Base_Allocator *allocator, String_Const_u8 contents){ - Token_Array result = {}; - result.tokens = base_array(allocator, Token, 2); - result.count = 2; - result.max = 2; - result.tokens[0].pos = 0; - result.tokens[0].size = contents.size; - result.tokens[0].kind = TokenBaseKind_COUNT; - result.tokens[0].sub_kind = 0; - result.tokens[1].pos = contents.size; - result.tokens[1].size = 0; - result.tokens[1].kind = TokenBaseKind_EOF; - return(result); -} - -// BOTTOM - diff --git a/languages/4coder_language_cpp.h b/languages/4coder_language_cpp.h deleted file mode 100644 index 9a91b76c..00000000 --- a/languages/4coder_language_cpp.h +++ /dev/null @@ -1,19 +0,0 @@ -/* -4coder_language_cpp.h - C++ language extension for the token types. -*/ - -// TOP - -#if !defined(FCODER_LANGUAGE_CPP_H) -#define FCODER_LANGUAGE_CPP_H - -enum{ - -}; - -#endif - -// BOTTOM - - - diff --git a/languages/4coder_language_cs.h b/languages/4coder_language_cs.h deleted file mode 100644 index 4c96b063..00000000 --- a/languages/4coder_language_cs.h +++ /dev/null @@ -1,122 +0,0 @@ -/* -4coder_language_cs.h - Sets up the C# language context. -*/ - -// TOP - -#if !defined(FCODER_LANGUAGE_CS_H) -#define FCODER_LANGUAGE_CS_H - -#if 0 -static Parse_Context_ID parse_context_language_cs; - -#define PSAT(s, t) {s, sizeof(s)-1, t} -static void -init_language_cs(Application_Links *app){ - if (parse_context_language_cs != 0) return; - - Parser_String_And_Type kw[] = { - PSAT("abstract", CPP_TOKEN_KEY_OTHER), - PSAT("as", CPP_TOKEN_KEY_OTHER), - PSAT("base", CPP_TOKEN_KEY_OTHER), - PSAT("bool", CPP_TOKEN_KEY_OTHER), - PSAT("byte", CPP_TOKEN_KEY_OTHER), - PSAT("char", CPP_TOKEN_KEY_OTHER), - PSAT("checked", CPP_TOKEN_KEY_OTHER), - PSAT("class", CPP_TOKEN_KEY_OTHER), - PSAT("const", CPP_TOKEN_KEY_OTHER), - PSAT("decimal", CPP_TOKEN_KEY_OTHER), - PSAT("delegate", CPP_TOKEN_KEY_OTHER), - PSAT("double", CPP_TOKEN_KEY_OTHER), - PSAT("enum", CPP_TOKEN_KEY_OTHER), - PSAT("event", CPP_TOKEN_KEY_OTHER), - PSAT("explicit", CPP_TOKEN_KEY_OTHER), - PSAT("extern", CPP_TOKEN_KEY_OTHER), - PSAT("false", CPP_TOKEN_KEY_OTHER), - PSAT("fixed", CPP_TOKEN_KEY_OTHER), - PSAT("float", CPP_TOKEN_KEY_OTHER), - PSAT("implicit", CPP_TOKEN_KEY_OTHER), - PSAT("int", CPP_TOKEN_KEY_OTHER), - PSAT("interface", CPP_TOKEN_KEY_OTHER), - PSAT("internal", CPP_TOKEN_KEY_OTHER), - PSAT("is", CPP_TOKEN_KEY_OTHER), - PSAT("long", CPP_TOKEN_KEY_OTHER), - PSAT("namespace", CPP_TOKEN_KEY_OTHER), - PSAT("new", CPP_TOKEN_KEY_OTHER), - PSAT("null", CPP_TOKEN_KEY_OTHER), - PSAT("object", CPP_TOKEN_KEY_OTHER), - PSAT("operator", CPP_TOKEN_KEY_OTHER), - PSAT("out", CPP_TOKEN_KEY_OTHER), - PSAT("override", CPP_TOKEN_KEY_OTHER), - PSAT("params", CPP_TOKEN_KEY_OTHER), - PSAT("private", CPP_TOKEN_KEY_OTHER), - PSAT("protected", CPP_TOKEN_KEY_OTHER), - PSAT("public", CPP_TOKEN_KEY_OTHER), - PSAT("readonly", CPP_TOKEN_KEY_OTHER), - PSAT("ref", CPP_TOKEN_KEY_OTHER), - PSAT("sbyte", CPP_TOKEN_KEY_OTHER), - PSAT("sealed", CPP_TOKEN_KEY_OTHER), - PSAT("short", CPP_TOKEN_KEY_OTHER), - PSAT("sizeof", CPP_TOKEN_KEY_OTHER), - PSAT("stackalloc", CPP_TOKEN_KEY_OTHER), - PSAT("static", CPP_TOKEN_KEY_OTHER), - PSAT("string", CPP_TOKEN_KEY_OTHER), - PSAT("struct", CPP_TOKEN_KEY_OTHER), - PSAT("this", CPP_TOKEN_KEY_OTHER), - PSAT("true", CPP_TOKEN_KEY_OTHER), - PSAT("typeof", CPP_TOKEN_KEY_OTHER), - PSAT("uint", CPP_TOKEN_KEY_OTHER), - PSAT("ulong", CPP_TOKEN_KEY_OTHER), - PSAT("unchecked", CPP_TOKEN_KEY_OTHER), - PSAT("unsafe", CPP_TOKEN_KEY_OTHER), - PSAT("ushort", CPP_TOKEN_KEY_OTHER), - PSAT("using", CPP_TOKEN_KEY_OTHER), - PSAT("void", CPP_TOKEN_KEY_OTHER), - PSAT("volatile", CPP_TOKEN_KEY_OTHER), - - PSAT("if", CPP_TOKEN_IF), - PSAT("else", CPP_TOKEN_ELSE), - PSAT("switch", CPP_TOKEN_SWITCH), - PSAT("case", CPP_TOKEN_CASE), - PSAT("do", CPP_TOKEN_DO), - PSAT("for", CPP_TOKEN_FOR), - PSAT("foreach", CPP_TOKEN_FOR), - PSAT("in", CPP_TOKEN_KEY_OTHER), - PSAT("while", CPP_TOKEN_WHILE), - PSAT("break", CPP_TOKEN_BREAK), - PSAT("continue", CPP_TOKEN_CONTINUE), - PSAT("default", CPP_TOKEN_DEFAULT), - PSAT("goto", CPP_TOKEN_GOTO), - PSAT("return", CPP_TOKEN_RETURN), - PSAT("yield", CPP_TOKEN_KEY_OTHER), - PSAT("throw", CPP_TOKEN_THROW), - PSAT("try", CPP_TOKEN_TRY), - PSAT("catch", CPP_TOKEN_CATCH), - PSAT("finally", CPP_TOKEN_CATCH), - PSAT("lock", CPP_TOKEN_KEY_OTHER), - }; - - Parser_String_And_Type pp[] = { - PSAT("if", CPP_PP_IF), - PSAT("else", CPP_PP_ELSE), - PSAT("elif", CPP_PP_ELIF), - PSAT("endif", CPP_PP_ENDIF), - PSAT("define", CPP_PP_DEFINED), - PSAT("undef", CPP_PP_UNDEF), - PSAT("warning", CPP_PP_ERROR), - PSAT("error", CPP_PP_ERROR), - PSAT("line", CPP_PP_LINE), - PSAT("pragma", CPP_PP_PRAGMA), - PSAT("region", CPP_PP_UNKNOWN), - PSAT("endregion", CPP_PP_UNKNOWN), - }; - - parse_context_language_cs = create_parse_context(app, kw, ArrayCount(kw), pp, ArrayCount(pp)); -} -#undef PSAT -#endif - -#endif - -// BOTTOM - diff --git a/languages/4coder_language_java.h b/languages/4coder_language_java.h deleted file mode 100644 index 2d68e8d6..00000000 --- a/languages/4coder_language_java.h +++ /dev/null @@ -1,82 +0,0 @@ -/* -4coder_language_java.h - Sets up the Java language context. -*/ - -// TOP - -#if !defined(FCODER_LANGUAGE_JAVA_H) -#define FCODER_LANGUAGE_JAVA_H - -#if 0 -static Parse_Context_ID parse_context_language_java; - -#define PSAT(s, t) {s, sizeof(s)-1, t} -static void -init_language_java(Application_Links *app){ - if (parse_context_language_java != 0) return; - - Parser_String_And_Type kw[] = { - PSAT("abstract", CPP_TOKEN_KEY_OTHER), - PSAT("assert", CPP_TOKEN_KEY_OTHER), - PSAT("boolean", CPP_TOKEN_KEY_OTHER), - PSAT("char", CPP_TOKEN_KEY_OTHER), - PSAT("class", CPP_TOKEN_KEY_OTHER), - PSAT("const", CPP_TOKEN_KEY_OTHER), - PSAT("default", CPP_TOKEN_KEY_OTHER), - PSAT("double", CPP_TOKEN_KEY_OTHER), - PSAT("enum", CPP_TOKEN_KEY_OTHER), - PSAT("extends", CPP_TOKEN_KEY_OTHER), - PSAT("final", CPP_TOKEN_KEY_OTHER), - PSAT("float", CPP_TOKEN_KEY_OTHER), - PSAT("implements", CPP_TOKEN_KEY_OTHER), - PSAT("import", CPP_TOKEN_KEY_OTHER), - PSAT("instanceof", CPP_TOKEN_KEY_OTHER), - PSAT("int", CPP_TOKEN_KEY_OTHER), - PSAT("interface", CPP_TOKEN_KEY_OTHER), - PSAT("long", CPP_TOKEN_KEY_OTHER), - PSAT("native", CPP_TOKEN_KEY_OTHER), - PSAT("new", CPP_TOKEN_KEY_OTHER), - PSAT("package", CPP_TOKEN_KEY_OTHER), - PSAT("private", CPP_TOKEN_KEY_OTHER), - PSAT("protected", CPP_TOKEN_KEY_OTHER), - PSAT("public", CPP_TOKEN_KEY_OTHER), - PSAT("return", CPP_TOKEN_KEY_OTHER), - PSAT("short", CPP_TOKEN_KEY_OTHER), - PSAT("static", CPP_TOKEN_KEY_OTHER), - PSAT("strictfp", CPP_TOKEN_KEY_OTHER), - PSAT("super", CPP_TOKEN_KEY_OTHER), - PSAT("synchronized", CPP_TOKEN_KEY_OTHER), - PSAT("this", CPP_TOKEN_KEY_OTHER), - PSAT("transient", CPP_TOKEN_KEY_OTHER), - PSAT("void", CPP_TOKEN_KEY_OTHER), - PSAT("volatile", CPP_TOKEN_KEY_OTHER), - PSAT("true", CPP_TOKEN_KEY_OTHER), - PSAT("false", CPP_TOKEN_KEY_OTHER), - PSAT("null", CPP_TOKEN_KEY_OTHER), - - PSAT("if", CPP_TOKEN_IF), - PSAT("else", CPP_TOKEN_ELSE), - PSAT("switch", CPP_TOKEN_SWITCH), - PSAT("case", CPP_TOKEN_CASE), - PSAT("while", CPP_TOKEN_WHILE), - PSAT("do", CPP_TOKEN_DO), - PSAT("for", CPP_TOKEN_FOR), - PSAT("goto", CPP_TOKEN_GOTO), - PSAT("break", CPP_TOKEN_BREAK), - PSAT("continue", CPP_TOKEN_CONTINUE), - PSAT("throw", CPP_TOKEN_THROW), - PSAT("throws", CPP_TOKEN_THROW), - PSAT("try", CPP_TOKEN_TRY), - PSAT("catch", CPP_TOKEN_CATCH), - PSAT("finally", CPP_TOKEN_CATCH), - }; - - parse_context_language_java = create_parse_context(app, kw, ArrayCount(kw), 0, 0); -} -#undef PSAT -#endif - -#endif - -// BOTTOM - diff --git a/languages/4coder_language_rust.h b/languages/4coder_language_rust.h deleted file mode 100644 index 98747785..00000000 --- a/languages/4coder_language_rust.h +++ /dev/null @@ -1,81 +0,0 @@ -/* -4coder_language_rust.h - Sets up the Rust language context. -*/ - -// TOP - -#if !defined(FCODER_LANGUAGE_RUST_H) -#define FCODER_LANGUAGE_RUST_H - -#if 0 -static Parse_Context_ID parse_context_language_rust; - -#define PSAT(s, t) {s, sizeof(s)-1, t} -static void -init_language_rust(Application_Links *app){ - if (parse_context_language_rust != 0) return; - - Parser_String_And_Type kw[] = { - PSAT("abstract", CPP_TOKEN_KEY_OTHER), - PSAT("alignof", CPP_TOKEN_KEY_OTHER), - PSAT("as", CPP_TOKEN_KEY_OTHER), - PSAT("become", CPP_TOKEN_KEY_OTHER), - PSAT("box", CPP_TOKEN_KEY_OTHER), - PSAT("const", CPP_TOKEN_KEY_OTHER), - PSAT("crate", CPP_TOKEN_KEY_OTHER), - PSAT("enum", CPP_TOKEN_KEY_OTHER), - PSAT("extern", CPP_TOKEN_KEY_OTHER), - PSAT("false", CPP_TOKEN_KEY_OTHER), - PSAT("final", CPP_TOKEN_KEY_OTHER), - PSAT("fn", CPP_TOKEN_KEY_OTHER), - PSAT("impl", CPP_TOKEN_KEY_OTHER), - PSAT("in", CPP_TOKEN_KEY_OTHER), - PSAT("let", CPP_TOKEN_KEY_OTHER), - PSAT("loop", CPP_TOKEN_KEY_OTHER), - PSAT("macro", CPP_TOKEN_KEY_OTHER), - PSAT("match", CPP_TOKEN_KEY_OTHER), - PSAT("mod", CPP_TOKEN_KEY_OTHER), - PSAT("move", CPP_TOKEN_KEY_OTHER), - PSAT("mut", CPP_TOKEN_KEY_OTHER), - PSAT("offsetof", CPP_TOKEN_KEY_OTHER), - PSAT("override", CPP_TOKEN_KEY_OTHER), - PSAT("priv", CPP_TOKEN_KEY_OTHER), - PSAT("proc", CPP_TOKEN_KEY_OTHER), - PSAT("pub", CPP_TOKEN_KEY_OTHER), - PSAT("pure", CPP_TOKEN_KEY_OTHER), - PSAT("ref", CPP_TOKEN_KEY_OTHER), - PSAT("return", CPP_TOKEN_KEY_OTHER), - PSAT("Self", CPP_TOKEN_KEY_OTHER), - PSAT("self", CPP_TOKEN_KEY_OTHER), - PSAT("sizeof", CPP_TOKEN_KEY_OTHER), - PSAT("static", CPP_TOKEN_KEY_OTHER), - PSAT("struct", CPP_TOKEN_KEY_OTHER), - PSAT("super", CPP_TOKEN_KEY_OTHER), - PSAT("trait", CPP_TOKEN_KEY_OTHER), - PSAT("true", CPP_TOKEN_KEY_OTHER), - PSAT("type", CPP_TOKEN_KEY_OTHER), - PSAT("typeof", CPP_TOKEN_KEY_OTHER), - PSAT("unsafe", CPP_TOKEN_KEY_OTHER), - PSAT("unsized", CPP_TOKEN_KEY_OTHER), - PSAT("use", CPP_TOKEN_KEY_OTHER), - PSAT("virtual", CPP_TOKEN_KEY_OTHER), - PSAT("where", CPP_TOKEN_KEY_OTHER), - - PSAT("break", CPP_TOKEN_BREAK), - PSAT("continue", CPP_TOKEN_CONTINUE), - PSAT("do", CPP_TOKEN_DO), - PSAT("else", CPP_TOKEN_ELSE), - PSAT("for", CPP_TOKEN_FOR), - PSAT("if", CPP_TOKEN_IF), - PSAT("while", CPP_TOKEN_WHILE), - PSAT("yield", CPP_TOKEN_KEY_OTHER), - }; - - parse_context_language_rust = create_parse_context(app, kw, ArrayCount(kw), 0, 0); -} -#undef PSAT -#endif - -#endif - -// BOTTOM diff --git a/languages/4coder_lexer_cpp_test.cpp b/languages/4coder_lexer_cpp_test.cpp new file mode 100644 index 00000000..d50a5972 --- /dev/null +++ b/languages/4coder_lexer_cpp_test.cpp @@ -0,0 +1,489 @@ +/* +4coder_lexer_cpp_test.cpp - A tester for the C++ lexer. +*/ + +// TOP + +#if 0 + +0. +0.f +0.1 +0.1f +0.0 +0.0f +0.F +0.0F +.0 +.0f +.1 +.9 +.1f +.9f +0.e1 +0.e1f +0.e1F +0.E1f +0.E1F +0.e+1 +0.e+1f +0.e+1F +0.E+1f +0.E+1F +0.e-1 +0.e-1f +0.e-1F +0.E-1f +0.E-1F + +0.l +0.1 +0.1l +0.0l +0.L +0.0L +.0l +.1l +.9l +0.e1 +0.e1l +0.e1L +0.E1l +0.E1L +0.e+1 +0.e+1l +0.e+1L +0.E+1l +0.E+1L +0.e-1 +0.e-1l +0.e-1L +0.E-1l +0.E-1L + +0x0 +0x1 +0xa +0xf +0xg +0xA +0xF +0xG +0x01 +0x0a +0x0f +0x0g +0x0A +0x0F +0x0G + +0x0z +0x1z +0xaz +0xfz +0xAz +0xFz +0x01z +0x0az +0x0fz +0x0Az +0x0Fz + +0x0l +0x1l +0xal +0xfl +0xAl +0xFl +0x01l +0x0al +0x0fl +0x0Al +0x0Fl + +0x0u +0x1u +0xau +0xfu +0xAu +0xFu +0x01u +0x0au +0x0fu +0x0Au +0x0Fu + +0x0L +0x1L +0xaL +0xfL +0xAL +0xFL +0x01L +0x0aL +0x0fL +0x0AL +0x0FL + +0x0U +0x1U +0xaU +0xfU +0xAU +0xFU +0x01U +0x0aU +0x0fU +0x0AU +0x0FU + +0x0lu +0x1lu +0xalu +0xflu +0xAlu +0xFlu +0x01lu +0x0alu +0x0flu +0x0Alu +0x0Flu + +0x0Lu +0x1Lu +0xaLu +0xfLu +0xALu +0xFLu +0x01Lu +0x0aLu +0x0fLu +0x0ALu +0x0FLu + +0x0lU +0x1lU +0xalU +0xflU +0xAlU +0xFlU +0x01lU +0x0alU +0x0flU +0x0AlU +0x0FlU + +0x0LU +0x1LU +0xaLU +0xfLU +0xALU +0xFLU +0x01LU +0x0aLU +0x0fLU +0x0ALU +0x0FLU + +0x0ul +0x1ul +0xaul +0xful +0xAul +0xFul +0x01ul +0x0aul +0x0ful +0x0Aul +0x0Ful + +0x0uL +0x1uL +0xauL +0xfuL +0xAuL +0xFuL +0x01uL +0x0auL +0x0fuL +0x0AuL +0x0FuL + +0x0Ul +0x1Ul +0xaUl +0xfUl +0xAUl +0xFUl +0x01Ul +0x0aUl +0x0fUl +0x0AUl +0x0FUl + +0x0UL +0x1UL +0xaUL +0xfUL +0xAUL +0xFUL +0x01UL +0x0aUL +0x0fUL +0x0AUL +0x0FUL + +0x0llu +0x1llu +0xallu +0xfllu +0xAllu +0xFllu +0x01llu +0x0allu +0x0fllu +0x0Allu +0x0Fllu + +0x0LLu +0x1LLu +0xaLLu +0xfLLu +0xALLu +0xFLLu +0x01LLu +0x0aLLu +0x0fLLu +0x0ALLu +0x0FLLu + +0x0llU +0x1llU +0xallU +0xfllU +0xAllU +0xFllU +0x01llU +0x0allU +0x0fllU +0x0AllU +0x0FllU + +0x0LLU +0x1LLU +0xaLLU +0xfLLU +0xALLU +0xFLLU +0x01LLU +0x0aLLU +0x0fLLU +0x0ALLU +0x0FLLU + +0x0ull +0x1ull +0xaull +0xfull +0xAull +0xFull +0x01ull +0x0aull +0x0full +0x0Aull +0x0Full + +0x0uLL +0x1uLL +0xauLL +0xfuLL +0xAuLL +0xFuLL +0x01uLL +0x0auLL +0x0fuLL +0x0AuLL +0x0FuLL + +0x0Ull +0x1Ull +0xaUll +0xfUll +0xAUll +0xFUll +0x01Ull +0x0aUll +0x0fUll +0x0AUll +0x0FUll + +0x0ULL +0x1ULL +0xaULL +0xfULL +0xAULL +0xFULL +0x01ULL +0x0aULL +0x0fULL +0x0AULL +0x0FULL + +01 +07 +08 +09 +010 +011 +077 +078 +087 + +"foo" +"foo +foo" +"foo\ +bar" +L"foo" +L"foo +Lfoo" +L"foo\ +bar" +u"foo" +u"foo +ufoo" +u"foo\ +bar" +u8"foo" +u8"foo +u8foo" +u8"foo\ +bar" +U"foo" +U"foo +Ufoo" +U"foo\ +bar" +R"(foo)" +R"bar(foo)bar" +R"foo(foo)foo" +LR"bar(foo)bar" +uR"bar(foo)bar" +u8R"bar(foo)bar" +UR"bar(foo)bar" +LR"foo(foo)foo" +uR"foo(foo)foo" +u8R"foo(foo)foo" +UR"foo(foo)foo" +R"bar(foo + +\ + +)bar" +R"foo(foo + +\ + +)foo" +R"bar(foo + +" + +)" + +)b" + +)ba" + +)bar" + +R"bar(foo + +)bar + +)bar + +)bar + +)bar" + +#error ``` Foo bar is bad news bears! ``` + +#endif + +#include "4coder_base_types.h" +#include "4coder_token.h" +#include "generated_lexer_cpp.h" + +#include "4coder_base_types.cpp" +#include "4coder_token.cpp" +#include "generated_lexer_cpp.cpp" + +#include "4coder_stringf.cpp" +#include "4coder_malloc_allocator.cpp" + +#include +#include + +internal void +print_token_list(Token_List *list, String_Const_u8 text){ + for (Token_Block *block = list->first; + block != 0; + block = block->next){ + i32 count = block->count; + Token *token = block->tokens; + for (i32 i = 0; i < count; i += 1, token += 1){ + printf("[%5llu, %5llu) %20s / %20s : 0x%04x / 0x%04x\n", + token->pos, token->pos + token->size, + token_base_kind_names[token->kind], + token_cpp_kind_names[token->sub_kind], + token->flags, token->sub_flags); + printf("\t:%.*s:\n", token->size, text.str + token->pos); + } + } +} + +internal String_Const_u8 +file_read_all(Arena *arena, FILE *file){ + String_Const_u8 result = {}; + fseek(file, 0, SEEK_END); + result.size = ftell(file); + fseek(file, 0, SEEK_SET); + result.str = push_array(arena, u8, result.size + 1); + fread(result.str, result.size, 1, file); + result.str[result.size] = 0; + return(result); +} + +int main(void){ + Arena arena_ = make_arena_malloc(); + Arena *arena = &arena_; + + String_Const_u8 path_to_self = string_u8_litexpr(__FILE__); + path_to_self = string_remove_last_folder(path_to_self); + String_Const_u8 path_to_src = string_remove_last_folder(path_to_self); + + String_Const_u8 test_file_name = push_u8_stringf(arena, "%.*s/languages/4coder_lexer_cpp_test.cpp", + string_expand(path_to_src)); + + FILE *test_file = fopen((char*)test_file_name.str, "rb"); + if (test_file == 0){ + printf("error: count not open test file %s\n", test_file_name.str); + exit(1); + } + String_Const_u8 text = file_read_all(arena, test_file); + fclose(test_file); + + Token_List list = lex_full_input_cpp(arena, text); + print_token_list(&list, text); + + for (i32 i = 0; i < KB(4); i += 1){ + fprintf(stdout, "\n"); + } + fflush(stdout); + + return(0); +} + +// BOTTOM + diff --git a/languages/generated_lexer_cpp.cpp b/languages/generated_lexer_cpp.cpp new file mode 100644 index 00000000..79eb5fdf --- /dev/null +++ b/languages/generated_lexer_cpp.cpp @@ -0,0 +1,4612 @@ +#if !defined(FCODER_LEX_GEN_HAND_WRITTEN) +#define FCODER_LEX_GEN_HAND_WRITTEN + +internal u64 +lexeme_hash(u64 seed, u8 *ptr, umem size){ + u64 result = 0; + for (umem i = 0; i < size; i += 1, ptr += 1){ + result ^= ((*ptr) ^ result*59) + seed; + } + return(result); +} + +internal Lexeme_Table_Lookup +lexeme_table_lookup(u64 *hash_array, String_Const_u8 *key_array, + Lexeme_Table_Value *value_array, i32 slot_count, u64 seed, + u8 *ptr, umem size){ + Lexeme_Table_Lookup result = {}; + u64 hash = lexeme_hash(seed, ptr, size); + u64 comparison_hash = hash | 1; + i32 first_index = (hash % slot_count); + i32 index = first_index; + for (;;){ + if (hash_array[index] == comparison_hash){ + if (string_match(SCu8(ptr, size), key_array[index])){ + result.found_match = true; + result.base_kind = value_array[index].base_kind; + result.sub_kind = value_array[index].sub_kind; + break; + } + } + else if (hash_array[index] == 0){ + break; + } + index += 1; + if (index == slot_count){ + index = 0; + } + if (index == first_index){ + break; + } + } + return(result); +} + +#endif +u64 main_keys_hash_array[123] = { +0x0000000000000000,0x6f4a79edb9a5533f,0xe5852ac396610dc5,0x0000000000000000, +0x0000000000000000,0x0000000000000000,0xa2a5dfeb017ed5bd,0xcd6340d166644e7f, +0x8408b482fc52169f,0x0000000000000000,0xcedf49ae63560b69,0x0000000000000000, +0x0000000000000000,0x0000000000000000,0xe75cf7ee7286ec9d,0x0000000000000000, +0x8408b482c15279f5,0x0000000000000000,0xe5852ab101aa40f9,0x0000000000000000, +0x0000000000000000,0x0000000000000000,0x0000000000000000,0x0000000000000000, +0x0000000000000000,0x0000000000000000,0xcede45b28f565327,0x0000000000000000, +0x0000000000000000,0x3cb9f5ca3e080c89,0x0000000000000000,0x0000000000000000, +0xe5852ac24b661f2d,0xe5852ab17aec8db7,0x0000000000000000,0xcd6340d166659c89, +0x0000000000000000,0x0000000000000000,0x8408b482facd5b93,0x12cbed639027df6f, +0xcedeb002ce9cbaf7,0xcee7dcc940536a31,0x0000000000000000,0x3cb9f5ca3e080c1d, +0x0000000000000000,0xe5852ac3c0cbc80d,0xcede5b51ee94b7c1,0x9a20703d4bd687c5, +0x8408b4828b3f4029,0x0000000000000000,0x0000000000000000,0xcedf9451bda57031, +0xe5852ac394d6e7af,0x44af72658a8ddf6f,0xe5852ab45a7e4877,0x0000000000000000, +0x3cb9f115b38586fd,0xe5852aeaff3b438d,0x666eef765675122f,0x0000000000000000, +0x8408b482cd76ef71,0xcd6340d1662d7e9f,0x0000000000000000,0x0000000000000000, +0xa2a5dfeb017e0e17,0x0000000000000000,0x0000000000000000,0x0000000000000000, +0xcd6340d166330431,0x0000000000000000,0x0000000000000000,0xcede1b147919cf71, +0x0000000000000000,0xe5852ab14f69af13,0x0000000000000000,0x0000000000000000, +0x0000000000000000,0xbe87c88e9f25c76f,0xe5852ac371f40231,0x0000000000000000, +0x6fcb06cae6e806ff,0x0000000000000000,0x8408b482fa970537,0x3cb9f05f96b9d771, +0xe5852ab17aadebb5,0x23945d1e642d9f6f,0xe75cf7ee7286f2a7,0x0000000000000000, +0x0000000000000000,0x3cb9f6891369547f,0xcd6340d16634823f,0x0000000000000000, +0xe5852ab1ac65f55f,0x8408b482caf244cd,0xcd6340d1629b9d0f,0x0000000000000000, +0x0000000000000000,0x0000000000000000,0x8408b482fa83e8a9,0x3cb9cc3692b7c61b, +0xcede1b03b4c5ebb7,0x3cb9f5df428aa011,0xa2a5dfeb017fd991,0x0000000000000000, +0xcd6340d16794567f,0xcd6340d166781707,0x0000000000000000,0xcedf6504dde4ee5f, +0x8408b482fc53d937,0xe5852ab148ad9eb1,0x0000000000000000,0x0000000000000000, +0x0000000000000000,0x0000000000000000,0x0000000000000000,0xcd6340d16639a18b, +0xcd6340d1662d6e7f,0xcee7dfce15a7a951,0xa2a5dfeb017136ed,0x8408b482ca53786f, +0x0000000000000000,0xe5852ab148e5344f,0xa2a5dfeb01710e1d, +}; +u8 main_keys_key_array_1[] = {0x6e,0x61,0x6d,0x65,0x73,0x70,0x61,0x63,0x65,}; +u8 main_keys_key_array_2[] = {0x65,0x78,0x70,0x6f,0x72,0x74,}; +u8 main_keys_key_array_6[] = {0x74,0x72,0x79,}; +u8 main_keys_key_array_7[] = {0x74,0x72,0x75,0x65,}; +u8 main_keys_key_array_8[] = {0x66,0x61,0x6c,0x73,0x65,}; +u8 main_keys_key_array_10[] = {0x72,0x65,0x67,0x69,0x73,0x74,0x65,0x72,}; +u8 main_keys_key_array_14[] = {0x69,0x66,}; +u8 main_keys_key_array_16[] = {0x73,0x68,0x6f,0x72,0x74,}; +u8 main_keys_key_array_18[] = {0x70,0x75,0x62,0x6c,0x69,0x63,}; +u8 main_keys_key_array_26[] = {0x65,0x78,0x70,0x6c,0x69,0x63,0x69,0x74,}; +u8 main_keys_key_array_29[] = {0x61,0x6c,0x69,0x67,0x6e,0x61,0x73,}; +u8 main_keys_key_array_32[] = {0x64,0x6f,0x75,0x62,0x6c,0x65,}; +u8 main_keys_key_array_33[] = {0x73,0x69,0x67,0x6e,0x65,0x64,}; +u8 main_keys_key_array_35[] = {0x74,0x68,0x69,0x73,}; +u8 main_keys_key_array_38[] = {0x63,0x61,0x74,0x63,0x68,}; +u8 main_keys_key_array_39[] = {0x73,0x74,0x61,0x74,0x69,0x63,0x5f,0x63,0x61,0x73,0x74,}; +u8 main_keys_key_array_40[] = {0x63,0x6f,0x6e,0x74,0x69,0x6e,0x75,0x65,}; +u8 main_keys_key_array_41[] = {0x6f,0x70,0x65,0x72,0x61,0x74,0x6f,0x72,}; +u8 main_keys_key_array_43[] = {0x61,0x6c,0x69,0x67,0x6e,0x6f,0x66,}; +u8 main_keys_key_array_45[] = {0x66,0x72,0x69,0x65,0x6e,0x64,}; +u8 main_keys_key_array_46[] = {0x64,0x65,0x63,0x6c,0x74,0x79,0x70,0x65,}; +u8 main_keys_key_array_47[] = {0x73,0x74,0x61,0x74,0x69,0x63,0x5f,0x61,0x73,0x73,0x65,0x72,0x74,}; +u8 main_keys_key_array_48[] = {0x62,0x72,0x65,0x61,0x6b,}; +u8 main_keys_key_array_51[] = {0x74,0x65,0x6d,0x70,0x6c,0x61,0x74,0x65,}; +u8 main_keys_key_array_52[] = {0x65,0x78,0x74,0x65,0x72,0x6e,}; +u8 main_keys_key_array_53[] = {0x64,0x79,0x6e,0x61,0x6d,0x69,0x63,0x5f,0x63,0x61,0x73,0x74,}; +u8 main_keys_key_array_54[] = {0x74,0x79,0x70,0x65,0x69,0x64,}; +u8 main_keys_key_array_56[] = {0x74,0x79,0x70,0x65,0x64,0x65,0x66,}; +u8 main_keys_key_array_57[] = {0x69,0x6e,0x6c,0x69,0x6e,0x65,}; +u8 main_keys_key_array_58[] = {0x74,0x68,0x72,0x65,0x61,0x64,0x5f,0x6c,0x6f,0x63,0x61,0x6c,}; +u8 main_keys_key_array_60[] = {0x77,0x68,0x69,0x6c,0x65,}; +u8 main_keys_key_array_61[] = {0x65,0x6c,0x73,0x65,}; +u8 main_keys_key_array_64[] = {0x61,0x73,0x6d,}; +u8 main_keys_key_array_68[] = {0x63,0x68,0x61,0x72,}; +u8 main_keys_key_array_71[] = {0x76,0x6f,0x6c,0x61,0x74,0x69,0x6c,0x65,}; +u8 main_keys_key_array_73[] = {0x73,0x77,0x69,0x74,0x63,0x68,}; +u8 main_keys_key_array_77[] = {0x72,0x65,0x69,0x6e,0x74,0x65,0x72,0x70,0x72,0x65,0x74,0x5f,0x63,0x61,0x73,0x74,}; +u8 main_keys_key_array_78[] = {0x64,0x65,0x6c,0x65,0x74,0x65,}; +u8 main_keys_key_array_80[] = {0x70,0x72,0x6f,0x74,0x65,0x63,0x74,0x65,0x64,}; +u8 main_keys_key_array_82[] = {0x63,0x6f,0x6e,0x73,0x74,}; +u8 main_keys_key_array_83[] = {0x70,0x72,0x69,0x76,0x61,0x74,0x65,}; +u8 main_keys_key_array_84[] = {0x73,0x69,0x7a,0x65,0x6f,0x66,}; +u8 main_keys_key_array_85[] = {0x63,0x6f,0x6e,0x73,0x74,0x5f,0x63,0x61,0x73,0x74,}; +u8 main_keys_key_array_86[] = {0x64,0x6f,}; +u8 main_keys_key_array_89[] = {0x76,0x69,0x72,0x74,0x75,0x61,0x6c,}; +u8 main_keys_key_array_90[] = {0x63,0x61,0x73,0x65,}; +u8 main_keys_key_array_92[] = {0x72,0x65,0x74,0x75,0x72,0x6e,}; +u8 main_keys_key_array_93[] = {0x75,0x6e,0x69,0x6f,0x6e,}; +u8 main_keys_key_array_94[] = {0x6c,0x6f,0x6e,0x67,}; +u8 main_keys_key_array_98[] = {0x63,0x6c,0x61,0x73,0x73,}; +u8 main_keys_key_array_99[] = {0x6e,0x75,0x6c,0x6c,0x70,0x74,0x72,}; +u8 main_keys_key_array_100[] = {0x75,0x6e,0x73,0x69,0x67,0x6e,0x65,0x64,}; +u8 main_keys_key_array_101[] = {0x64,0x65,0x66,0x61,0x75,0x6c,0x74,}; +u8 main_keys_key_array_102[] = {0x66,0x6f,0x72,}; +u8 main_keys_key_array_104[] = {0x62,0x6f,0x6f,0x6c,}; +u8 main_keys_key_array_105[] = {0x76,0x6f,0x69,0x64,}; +u8 main_keys_key_array_107[] = {0x74,0x79,0x70,0x65,0x6e,0x61,0x6d,0x65,}; +u8 main_keys_key_array_108[] = {0x66,0x6c,0x6f,0x61,0x74,}; +u8 main_keys_key_array_109[] = {0x73,0x74,0x61,0x74,0x69,0x63,}; +u8 main_keys_key_array_115[] = {0x67,0x6f,0x74,0x6f,}; +u8 main_keys_key_array_116[] = {0x65,0x6e,0x75,0x6d,}; +u8 main_keys_key_array_117[] = {0x6e,0x6f,0x65,0x78,0x63,0x65,0x70,0x74,}; +u8 main_keys_key_array_118[] = {0x6e,0x65,0x77,}; +u8 main_keys_key_array_119[] = {0x75,0x73,0x69,0x6e,0x67,}; +u8 main_keys_key_array_121[] = {0x73,0x74,0x72,0x75,0x63,0x74,}; +u8 main_keys_key_array_122[] = {0x69,0x6e,0x74,}; +String_Const_u8 main_keys_key_array[123] = { +{0, 0}, +{main_keys_key_array_1, 9}, +{main_keys_key_array_2, 6}, +{0, 0}, +{0, 0}, +{0, 0}, +{main_keys_key_array_6, 3}, +{main_keys_key_array_7, 4}, +{main_keys_key_array_8, 5}, +{0, 0}, +{main_keys_key_array_10, 8}, +{0, 0}, +{0, 0}, +{0, 0}, +{main_keys_key_array_14, 2}, +{0, 0}, +{main_keys_key_array_16, 5}, +{0, 0}, +{main_keys_key_array_18, 6}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{main_keys_key_array_26, 8}, +{0, 0}, +{0, 0}, +{main_keys_key_array_29, 7}, +{0, 0}, +{0, 0}, +{main_keys_key_array_32, 6}, +{main_keys_key_array_33, 6}, +{0, 0}, +{main_keys_key_array_35, 4}, +{0, 0}, +{0, 0}, +{main_keys_key_array_38, 5}, +{main_keys_key_array_39, 11}, +{main_keys_key_array_40, 8}, +{main_keys_key_array_41, 8}, +{0, 0}, +{main_keys_key_array_43, 7}, +{0, 0}, +{main_keys_key_array_45, 6}, +{main_keys_key_array_46, 8}, +{main_keys_key_array_47, 13}, +{main_keys_key_array_48, 5}, +{0, 0}, +{0, 0}, +{main_keys_key_array_51, 8}, +{main_keys_key_array_52, 6}, +{main_keys_key_array_53, 12}, +{main_keys_key_array_54, 6}, +{0, 0}, +{main_keys_key_array_56, 7}, +{main_keys_key_array_57, 6}, +{main_keys_key_array_58, 12}, +{0, 0}, +{main_keys_key_array_60, 5}, +{main_keys_key_array_61, 4}, +{0, 0}, +{0, 0}, +{main_keys_key_array_64, 3}, +{0, 0}, +{0, 0}, +{0, 0}, +{main_keys_key_array_68, 4}, +{0, 0}, +{0, 0}, +{main_keys_key_array_71, 8}, +{0, 0}, +{main_keys_key_array_73, 6}, +{0, 0}, +{0, 0}, +{0, 0}, +{main_keys_key_array_77, 16}, +{main_keys_key_array_78, 6}, +{0, 0}, +{main_keys_key_array_80, 9}, +{0, 0}, +{main_keys_key_array_82, 5}, +{main_keys_key_array_83, 7}, +{main_keys_key_array_84, 6}, +{main_keys_key_array_85, 10}, +{main_keys_key_array_86, 2}, +{0, 0}, +{0, 0}, +{main_keys_key_array_89, 7}, +{main_keys_key_array_90, 4}, +{0, 0}, +{main_keys_key_array_92, 6}, +{main_keys_key_array_93, 5}, +{main_keys_key_array_94, 4}, +{0, 0}, +{0, 0}, +{0, 0}, +{main_keys_key_array_98, 5}, +{main_keys_key_array_99, 7}, +{main_keys_key_array_100, 8}, +{main_keys_key_array_101, 7}, +{main_keys_key_array_102, 3}, +{0, 0}, +{main_keys_key_array_104, 4}, +{main_keys_key_array_105, 4}, +{0, 0}, +{main_keys_key_array_107, 8}, +{main_keys_key_array_108, 5}, +{main_keys_key_array_109, 6}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{main_keys_key_array_115, 4}, +{main_keys_key_array_116, 4}, +{main_keys_key_array_117, 8}, +{main_keys_key_array_118, 3}, +{main_keys_key_array_119, 5}, +{0, 0}, +{main_keys_key_array_121, 6}, +{main_keys_key_array_122, 3}, +}; +Lexeme_Table_Value main_keys_value_array[123] = { +{0, 0}, +{4, TokenCppKind_Namespace}, +{4, TokenCppKind_Export}, +{0, 0}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Try}, +{8, TokenCppKind_LiteralTrue}, +{8, TokenCppKind_LiteralFalse}, +{0, 0}, +{4, TokenCppKind_Register}, +{0, 0}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_If}, +{0, 0}, +{4, TokenCppKind_Short}, +{0, 0}, +{4, TokenCppKind_Public}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Explicit}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_AlignAs}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Double}, +{4, TokenCppKind_Signed}, +{0, 0}, +{4, TokenCppKind_This}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Catch}, +{4, TokenCppKind_StaticCast}, +{4, TokenCppKind_Continue}, +{4, TokenCppKind_Operator}, +{0, 0}, +{4, TokenCppKind_AlignOf}, +{0, 0}, +{4, TokenCppKind_Friend}, +{4, TokenCppKind_DeclType}, +{4, TokenCppKind_StaticAssert}, +{4, TokenCppKind_Break}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Template}, +{4, TokenCppKind_Extern}, +{4, TokenCppKind_DynamicCast}, +{4, TokenCppKind_TypeID}, +{0, 0}, +{4, TokenCppKind_Typedef}, +{4, TokenCppKind_Inline}, +{4, TokenCppKind_ThreadLocal}, +{0, 0}, +{4, TokenCppKind_While}, +{4, TokenCppKind_Else}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Asm}, +{0, 0}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Char}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Volatile}, +{0, 0}, +{4, TokenCppKind_Switch}, +{0, 0}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_ReinterpretCast}, +{4, TokenCppKind_Delete}, +{0, 0}, +{4, TokenCppKind_Protected}, +{0, 0}, +{4, TokenCppKind_Const}, +{4, TokenCppKind_Private}, +{4, TokenCppKind_SizeOf}, +{4, TokenCppKind_ConstCast}, +{4, TokenCppKind_Do}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Virtual}, +{4, TokenCppKind_Case}, +{0, 0}, +{4, TokenCppKind_Return}, +{4, TokenCppKind_Union}, +{4, TokenCppKind_Long}, +{0, 0}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Class}, +{4, TokenCppKind_NullPtr}, +{4, TokenCppKind_Unsigned}, +{4, TokenCppKind_Default}, +{4, TokenCppKind_For}, +{0, 0}, +{4, TokenCppKind_Bool}, +{4, TokenCppKind_Void}, +{0, 0}, +{4, TokenCppKind_Typename}, +{4, TokenCppKind_Float}, +{4, TokenCppKind_Static}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{0, 0}, +{4, TokenCppKind_Goto}, +{4, TokenCppKind_Enum}, +{4, TokenCppKind_NoExcept}, +{4, TokenCppKind_New}, +{4, TokenCppKind_Using}, +{0, 0}, +{4, TokenCppKind_Struct}, +{4, TokenCppKind_Int}, +}; +i32 main_keys_slot_count = 123; +u64 main_keys_seed = 0xf38c04110edf5c20; +u64 pp_directives_hash_array[25] = { +0x6b2525b7e6f87877,0x0000000000000000,0xb6682cd11ed9fd9f,0x6b2525b40373aacb, +0xb6682cd11ea5bda7,0x0000000000000000,0x0000000000000000,0x0d56a51c5ba8244b, +0x0000000000000000,0x6b2525b7e6f864c5,0xb6682cd10cf54325,0x0dbf8cd197510b47, +0x83d780e616d1416b,0x0000000000000000,0x83d7ee6ef708249b,0x83d780e1399294e1, +0x0000000000000000,0xb6682cd10c14755f,0x6c3839b38ad2ee53,0x0000000000000000, +0x83d7ee6e1191055f,0xb6682d358a49de5f,0x0000000000000000,0x0000000000000000, +0x0000000000000000, +}; +u8 pp_directives_key_array_0[] = {0x23,0x65,0x6c,0x69,0x66,}; +u8 pp_directives_key_array_2[] = {0x23,0x65,0x72,0x72,0x6f,0x72,}; +u8 pp_directives_key_array_3[] = {0x23,0x6c,0x69,0x6e,0x65,}; +u8 pp_directives_key_array_4[] = {0x23,0x65,0x6e,0x64,0x69,0x66,}; +u8 pp_directives_key_array_7[] = {0x23,0x69,0x6e,0x63,0x6c,0x75,0x64,0x65,}; +u8 pp_directives_key_array_9[] = {0x23,0x65,0x6c,0x73,0x65,}; +u8 pp_directives_key_array_10[] = {0x23,0x75,0x73,0x69,0x6e,0x67,}; +u8 pp_directives_key_array_11[] = {0x23,0x76,0x65,0x72,0x73,0x69,0x6f,0x6e,}; +u8 pp_directives_key_array_12[] = {0x23,0x64,0x65,0x66,0x69,0x6e,0x65,}; +u8 pp_directives_key_array_14[] = {0x23,0x69,0x6d,0x70,0x6f,0x72,0x74,}; +u8 pp_directives_key_array_15[] = {0x23,0x70,0x72,0x61,0x67,0x6d,0x61,}; +u8 pp_directives_key_array_17[] = {0x23,0x75,0x6e,0x64,0x65,0x66,}; +u8 pp_directives_key_array_18[] = {0x23,0x69,0x66,}; +u8 pp_directives_key_array_20[] = {0x23,0x69,0x66,0x6e,0x64,0x65,0x66,}; +u8 pp_directives_key_array_21[] = {0x23,0x69,0x66,0x64,0x65,0x66,}; +String_Const_u8 pp_directives_key_array[25] = { +{pp_directives_key_array_0, 5}, +{0, 0}, +{pp_directives_key_array_2, 6}, +{pp_directives_key_array_3, 5}, +{pp_directives_key_array_4, 6}, +{0, 0}, +{0, 0}, +{pp_directives_key_array_7, 8}, +{0, 0}, +{pp_directives_key_array_9, 5}, +{pp_directives_key_array_10, 6}, +{pp_directives_key_array_11, 8}, +{pp_directives_key_array_12, 7}, +{0, 0}, +{pp_directives_key_array_14, 7}, +{pp_directives_key_array_15, 7}, +{0, 0}, +{pp_directives_key_array_17, 6}, +{pp_directives_key_array_18, 3}, +{0, 0}, +{pp_directives_key_array_20, 7}, +{pp_directives_key_array_21, 6}, +{0, 0}, +{0, 0}, +{0, 0}, +}; +Lexeme_Table_Value pp_directives_value_array[25] = { +{5, TokenCppKind_PPElIf}, +{0, 0}, +{5, TokenCppKind_PPError}, +{5, TokenCppKind_PPLine}, +{5, TokenCppKind_PPEndIf}, +{0, 0}, +{0, 0}, +{5, TokenCppKind_PPInclude}, +{0, 0}, +{5, TokenCppKind_PPElse}, +{5, TokenCppKind_PPUsing}, +{5, TokenCppKind_PPVersion}, +{5, TokenCppKind_PPDefine}, +{0, 0}, +{5, TokenCppKind_PPImport}, +{5, TokenCppKind_PPPragma}, +{0, 0}, +{5, TokenCppKind_PPUndef}, +{5, TokenCppKind_PPIf}, +{0, 0}, +{5, TokenCppKind_PPIfNDef}, +{5, TokenCppKind_PPIfDef}, +{0, 0}, +{0, 0}, +{0, 0}, +}; +i32 pp_directives_slot_count = 25; +u64 pp_directives_seed = 0x2bbd5804bccb85e6; +u64 pp_keys_hash_array[2] = { +0x0000000000000000,0x89659c00a323190b, +}; +u8 pp_keys_key_array_1[] = {0x64,0x65,0x66,0x69,0x6e,0x65,0x64,}; +String_Const_u8 pp_keys_key_array[2] = { +{0, 0}, +{pp_keys_key_array_1, 7}, +}; +Lexeme_Table_Value pp_keys_value_array[2] = { +{0, 0}, +{4, TokenCppKind_PPDefined}, +}; +i32 pp_keys_slot_count = 2; +u64 pp_keys_seed = 0xc87165e79dc2fa67; +internal Token_List +lex_full_input_cpp(Arena *arena, String_Const_u8 input){ +Token_List list = {}; +u32 flags_ZF0 = 0; +u32 flags_KF0 = 0; +u16 flags_KB0 = 0; +u8 *delim_first = input.str; +u8 *delim_one_past_last = input.str; +u8 *emit_ptr = input.str; +u8 *ptr = input.str; +u8 *opl_ptr = ptr + input.size; +{ +state_label_1: // root +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_EOF; +token.kind = 0; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto end; +} +} +switch (*ptr){ +case 0x00:case 0x01:case 0x02:case 0x03:case 0x04:case 0x05:case 0x06: +case 0x07:case 0x08:case 0x0e:case 0x0f:case 0x10:case 0x11:case 0x12: +case 0x13:case 0x14:case 0x15:case 0x16:case 0x17:case 0x18:case 0x19: +case 0x1a:case 0x1b:case 0x1c:case 0x1d:case 0x1e:case 0x1f:case 0x40: +case 0x60:case 0x7f: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x09:case 0x0b:case 0x0c:case 0x0d:case 0x20: +{ +if ((HasFlag(flags_KF0, 0x2))){ +ptr += 1; +goto state_label_4; // error_body +} +ptr += 1; +goto state_label_3; // whitespace +}break; +case 0x0a: +{ +ptr += 1; +flags_KB0 &= ~(0x1); +flags_KF0 &= ~(0x1); +flags_KF0 &= ~(0x2); +goto state_label_3; // whitespace +}break; +case 0x21: +{ +ptr += 1; +goto state_label_59; // op stage +}break; +case 0x22: +{ +if ((HasFlag(flags_KF0, 0x1))){ +ptr += 1; +goto state_label_25; // include_quotes +} +ptr += 1; +goto state_label_31; // string +}break; +case 0x23: +{ +if ((!HasFlag(flags_KB0, 0x1))){ +ptr += 1; +flags_KB0 |= 0x1; +goto state_label_23; // pp_directive +} +ptr += 1; +goto state_label_66; // op stage +}break; +default: +{ +ptr += 1; +goto state_label_2; // identifier +}break; +case 0x25: +{ +ptr += 1; +goto state_label_63; // op stage +}break; +case 0x26: +{ +ptr += 1; +goto state_label_60; // op stage +}break; +case 0x27: +{ +ptr += 1; +flags_ZF0 |= 0x40; +goto state_label_31; // string +}break; +case 0x28: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_ParenOp; +token.kind = 13; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x29: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_ParenCl; +token.kind = 14; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2a: +{ +ptr += 1; +goto state_label_62; // op stage +}break; +case 0x2b: +{ +ptr += 1; +goto state_label_52; // op stage +}break; +case 0x2c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Comma; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2d: +{ +ptr += 1; +goto state_label_53; // op stage +}break; +case 0x2e: +{ +ptr += 1; +goto state_label_6; // operator_or_fnumber_dot +}break; +case 0x2f: +{ +ptr += 1; +goto state_label_7; // operator_or_comment_slash +}break; +case 0x30: +{ +ptr += 1; +goto state_label_9; // znumber +}break; +case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36:case 0x37: +case 0x38:case 0x39: +{ +ptr += 1; +goto state_label_8; // number +}break; +case 0x3a: +{ +ptr += 1; +goto state_label_51; // op stage +}break; +case 0x3b: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Semicolon; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3c: +{ +if ((!HasFlag(flags_KF0, 0x1))){ +ptr += 1; +goto state_label_55; // op stage +} +ptr += 1; +goto state_label_24; // include_pointy +}break; +case 0x3d: +{ +ptr += 1; +goto state_label_58; // op stage +}break; +case 0x3e: +{ +ptr += 1; +goto state_label_56; // op stage +}break; +case 0x3f: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Ternary; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x4c: +{ +ptr += 1; +flags_ZF0 |= 0x4; +goto state_label_26; // pre_L +}break; +case 0x52: +{ +ptr += 1; +goto state_label_30; // pre_R +}break; +case 0x55: +{ +ptr += 1; +flags_ZF0 |= 0x20; +goto state_label_28; // pre_U +}break; +case 0x5b: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_BrackOp; +token.kind = 13; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x5c: +{ +ptr += 1; +goto state_label_5; // backslash +}break; +case 0x5d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_BrackCl; +token.kind = 14; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x5e: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Xor; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x75: +{ +ptr += 1; +flags_ZF0 |= 0x10; +goto state_label_27; // pre_u +}break; +case 0x7b: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_BraceOp; +token.kind = 11; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x7c: +{ +ptr += 1; +goto state_label_61; // op stage +}break; +case 0x7d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_BraceCl; +token.kind = 12; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x7e: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Tilde; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_2: // identifier +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_KB0, 0x1)){ +Lexeme_Table_Lookup lookup = lexeme_table_lookup(pp_keys_hash_array, pp_keys_key_array, pp_keys_value_array, pp_keys_slot_count, pp_keys_seed, emit_ptr, token.size); +if (lookup.found_match){ +token.kind = lookup.base_kind; +token.sub_kind = lookup.sub_kind; +break; +} +} +Lexeme_Table_Lookup lookup = lexeme_table_lookup(main_keys_hash_array, main_keys_key_array, main_keys_value_array, main_keys_slot_count, main_keys_seed, emit_ptr, token.size); +if (lookup.found_match){ +token.kind = lookup.base_kind; +token.sub_kind = lookup.sub_kind; +break; +} +token.sub_kind = TokenCppKind_Identifier; +token.kind = 6; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +case 0x00:case 0x01:case 0x02:case 0x03:case 0x04:case 0x05:case 0x06: +case 0x07:case 0x08:case 0x09:case 0x0a:case 0x0b:case 0x0c:case 0x0d: +case 0x0e:case 0x0f:case 0x10:case 0x11:case 0x12:case 0x13:case 0x14: +case 0x15:case 0x16:case 0x17:case 0x18:case 0x19:case 0x1a:case 0x1b: +case 0x1c:case 0x1d:case 0x1e:case 0x1f:case 0x20:case 0x21:case 0x22: +case 0x23:case 0x25:case 0x26:case 0x27:case 0x28:case 0x29:case 0x2a: +case 0x2b:case 0x2c:case 0x2d:case 0x2e:case 0x2f:case 0x3a:case 0x3b: +case 0x3c:case 0x3d:case 0x3e:case 0x3f:case 0x40:case 0x5b:case 0x5c: +case 0x5d:case 0x5e:case 0x60:case 0x7b:case 0x7c:case 0x7d:case 0x7e: +case 0x7f: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_KB0, 0x1)){ +Lexeme_Table_Lookup lookup = lexeme_table_lookup(pp_keys_hash_array, pp_keys_key_array, pp_keys_value_array, pp_keys_slot_count, pp_keys_seed, emit_ptr, token.size); +if (lookup.found_match){ +token.kind = lookup.base_kind; +token.sub_kind = lookup.sub_kind; +break; +} +} +Lexeme_Table_Lookup lookup = lexeme_table_lookup(main_keys_hash_array, main_keys_key_array, main_keys_value_array, main_keys_slot_count, main_keys_seed, emit_ptr, token.size); +if (lookup.found_match){ +token.kind = lookup.base_kind; +token.sub_kind = lookup.sub_kind; +break; +} +token.sub_kind = TokenCppKind_Identifier; +token.kind = 6; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +default: +{ +ptr += 1; +goto state_label_2; // identifier +}break; +} +} +{ +state_label_3: // whitespace +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Whitespace; +token.kind = 1; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Whitespace; +token.kind = 1; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x09:case 0x0b:case 0x0c:case 0x0d:case 0x20: +{ +ptr += 1; +goto state_label_3; // whitespace +}break; +case 0x0a: +{ +ptr += 1; +flags_KB0 &= ~(0x1); +flags_KF0 &= ~(0x1); +flags_KF0 &= ~(0x2); +goto state_label_3; // whitespace +}break; +} +} +{ +state_label_4: // error_body +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PPErrorMessage; +token.kind = 10; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +ptr += 1; +goto state_label_4; // error_body +}break; +case 0x0a: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PPErrorMessage; +token.kind = 10; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_5: // backslash +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Backslash; +token.kind = 1; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Backslash; +token.kind = 1; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x0a: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Backslash; +token.kind = 1; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_6: // operator_or_fnumber_dot +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Dot; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Dot; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2a: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_DotStar; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2e: +{ +ptr += 1; +goto state_label_67; // op stage +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39: +{ +ptr += 1; +goto state_label_10; // fnumber_decimal +}break; +} +} +{ +state_label_7: // operator_or_comment_slash +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Div; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Div; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2a: +{ +ptr += 1; +goto state_label_48; // comment_block +}break; +case 0x2f: +{ +ptr += 1; +goto state_label_50; // comment_line +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_DivEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_8: // number +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralInteger; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralInteger; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2e: +{ +ptr += 1; +goto state_label_10; // fnumber_decimal +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39: +{ +ptr += 1; +goto state_label_8; // number +}break; +case 0x45:case 0x65: +{ +ptr += 1; +goto state_label_11; // fnumber_exponent +}break; +case 0x4c: +{ +ptr += 1; +goto state_label_18; // L_number +}break; +case 0x55:case 0x75: +{ +ptr += 1; +goto state_label_17; // U_number +}break; +case 0x6c: +{ +ptr += 1; +goto state_label_20; // l_number +}break; +} +} +{ +state_label_9: // znumber +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralInteger; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralInteger; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2e: +{ +ptr += 1; +goto state_label_10; // fnumber_decimal +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37: +{ +ptr += 1; +flags_ZF0 |= 0x2; +goto state_label_16; // number_oct +}break; +case 0x45:case 0x65: +{ +ptr += 1; +goto state_label_11; // fnumber_exponent +}break; +case 0x4c: +{ +ptr += 1; +goto state_label_18; // L_number +}break; +case 0x55:case 0x75: +{ +ptr += 1; +goto state_label_17; // U_number +}break; +case 0x58:case 0x78: +{ +ptr += 1; +flags_ZF0 |= 0x1; +goto state_label_14; // number_hex_first +}break; +case 0x6c: +{ +ptr += 1; +goto state_label_20; // l_number +}break; +} +} +{ +state_label_10: // fnumber_decimal +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39: +{ +ptr += 1; +goto state_label_10; // fnumber_decimal +}break; +case 0x45:case 0x65: +{ +ptr += 1; +goto state_label_11; // fnumber_exponent +}break; +case 0x46:case 0x66: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat32; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x4c:case 0x6c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_11: // fnumber_exponent +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2b:case 0x2d: +{ +ptr += 1; +goto state_label_12; // fnumber_exponent_sign +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39: +{ +ptr += 1; +goto state_label_13; // fnumber_exponent_digits +}break; +case 0x46:case 0x66: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat32; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x4c:case 0x6c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_12: // fnumber_exponent_sign +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39: +{ +ptr += 1; +goto state_label_13; // fnumber_exponent_digits +}break; +case 0x46:case 0x66: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat32; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x4c:case 0x6c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_13: // fnumber_exponent_digits +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39: +{ +ptr += 1; +goto state_label_13; // fnumber_exponent_digits +}break; +case 0x46:case 0x66: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat32; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x4c:case 0x6c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralFloat64; +token.kind = 9; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_14: // number_hex_first +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_15; // number_hex +}break; +} +} +{ +state_label_15: // number_hex +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralIntegerHex; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralIntegerHex; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_15; // number_hex +}break; +case 0x4c: +{ +ptr += 1; +goto state_label_18; // L_number +}break; +case 0x55:case 0x75: +{ +ptr += 1; +goto state_label_17; // U_number +}break; +case 0x6c: +{ +ptr += 1; +goto state_label_20; // l_number +}break; +} +} +{ +state_label_16: // number_oct +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralIntegerOct; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LiteralIntegerOct; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37: +{ +ptr += 1; +flags_ZF0 |= 0x2; +goto state_label_16; // number_oct +}break; +case 0x4c: +{ +ptr += 1; +goto state_label_18; // L_number +}break; +case 0x55:case 0x75: +{ +ptr += 1; +goto state_label_17; // U_number +}break; +case 0x6c: +{ +ptr += 1; +goto state_label_20; // l_number +}break; +} +} +{ +state_label_17: // U_number +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexU; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctU; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerU; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexU; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctU; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerU; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x4c: +{ +ptr += 1; +goto state_label_19; // UL_number +}break; +case 0x6c: +{ +ptr += 1; +goto state_label_21; // Ul_number +}break; +} +} +{ +state_label_18: // L_number +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x4c: +{ +ptr += 1; +goto state_label_22; // LL_number +}break; +case 0x55:case 0x75: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexUL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctUL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerUL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_19: // UL_number +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexUL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctUL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerUL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexUL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctUL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerUL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x4c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexULL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctULL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerULL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_20: // l_number +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x55:case 0x75: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexUL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctUL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerUL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x6c: +{ +ptr += 1; +goto state_label_22; // LL_number +}break; +} +} +{ +state_label_21: // Ul_number +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexUL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctUL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerUL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexUL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctUL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerUL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x6c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexULL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctULL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerULL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_22: // LL_number +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexLL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctLL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerLL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexLL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctLL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerLL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x55:case 0x75: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x1)){ +token.sub_kind = TokenCppKind_LiteralIntegerHexULL; +token.kind = 8; +break; +} +if (HasFlag(flags_ZF0, 0x2)){ +token.sub_kind = TokenCppKind_LiteralIntegerOctULL; +token.kind = 8; +break; +} +token.sub_kind = TokenCppKind_LiteralIntegerULL; +token.kind = 8; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_23: // pp_directive +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +Lexeme_Table_Lookup lookup = lexeme_table_lookup(pp_directives_hash_array, pp_directives_key_array, pp_directives_value_array, pp_directives_slot_count, pp_directives_seed, emit_ptr, token.size); +if (lookup.found_match){ +token.kind = lookup.base_kind; +token.sub_kind = lookup.sub_kind; +break; +} +token.sub_kind = TokenCppKind_PPUnknown; +token.kind = 2; +}while(0); +switch (token.sub_kind){ +case TokenCppKind_PPInclude: +{ +flags_KF0 |= 0x1; +}break; +case TokenCppKind_PPError: +{ +flags_KF0 |= 0x2; +}break; +} +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +Lexeme_Table_Lookup lookup = lexeme_table_lookup(pp_directives_hash_array, pp_directives_key_array, pp_directives_value_array, pp_directives_slot_count, pp_directives_seed, emit_ptr, token.size); +if (lookup.found_match){ +token.kind = lookup.base_kind; +token.sub_kind = lookup.sub_kind; +break; +} +token.sub_kind = TokenCppKind_PPUnknown; +token.kind = 2; +}while(0); +switch (token.sub_kind){ +case TokenCppKind_PPInclude: +{ +flags_KF0 |= 0x1; +}break; +case TokenCppKind_PPError: +{ +flags_KF0 |= 0x2; +}break; +} +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x47:case 0x48:case 0x49:case 0x4a:case 0x4b: +case 0x4c:case 0x4d:case 0x4e:case 0x4f:case 0x50:case 0x51:case 0x52: +case 0x53:case 0x54:case 0x55:case 0x56:case 0x57:case 0x58:case 0x59: +case 0x5a:case 0x5f:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66:case 0x67:case 0x68:case 0x69:case 0x6a:case 0x6b:case 0x6c: +case 0x6d:case 0x6e:case 0x6f:case 0x70:case 0x71:case 0x72:case 0x73: +case 0x74:case 0x75:case 0x76:case 0x77:case 0x78:case 0x79:case 0x7a: +{ +ptr += 1; +flags_KB0 |= 0x1; +goto state_label_23; // pp_directive +}break; +} +} +{ +state_label_24: // include_pointy +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x20:case 0x2e:case 0x2f:case 0x30:case 0x31:case 0x32:case 0x33: +case 0x34:case 0x35:case 0x36:case 0x37:case 0x38:case 0x39:case 0x41: +case 0x42:case 0x43:case 0x44:case 0x45:case 0x46:case 0x47:case 0x48: +case 0x49:case 0x4a:case 0x4b:case 0x4c:case 0x4d:case 0x4e:case 0x4f: +case 0x50:case 0x51:case 0x52:case 0x53:case 0x54:case 0x55:case 0x56: +case 0x57:case 0x58:case 0x59:case 0x5a:case 0x5c:case 0x5f:case 0x61: +case 0x62:case 0x63:case 0x64:case 0x65:case 0x66:case 0x67:case 0x68: +case 0x69:case 0x6a:case 0x6b:case 0x6c:case 0x6d:case 0x6e:case 0x6f: +case 0x70:case 0x71:case 0x72:case 0x73:case 0x74:case 0x75:case 0x76: +case 0x77:case 0x78:case 0x79:case 0x7a: +{ +ptr += 1; +goto state_label_24; // include_pointy +}break; +case 0x3e: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PPIncludeFile; +token.kind = 10; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_25: // include_quotes +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x20:case 0x2e:case 0x2f:case 0x30:case 0x31:case 0x32:case 0x33: +case 0x34:case 0x35:case 0x36:case 0x37:case 0x38:case 0x39:case 0x41: +case 0x42:case 0x43:case 0x44:case 0x45:case 0x46:case 0x47:case 0x48: +case 0x49:case 0x4a:case 0x4b:case 0x4c:case 0x4d:case 0x4e:case 0x4f: +case 0x50:case 0x51:case 0x52:case 0x53:case 0x54:case 0x55:case 0x56: +case 0x57:case 0x58:case 0x59:case 0x5a:case 0x5c:case 0x5f:case 0x61: +case 0x62:case 0x63:case 0x64:case 0x65:case 0x66:case 0x67:case 0x68: +case 0x69:case 0x6a:case 0x6b:case 0x6c:case 0x6d:case 0x6e:case 0x6f: +case 0x70:case 0x71:case 0x72:case 0x73:case 0x74:case 0x75:case 0x76: +case 0x77:case 0x78:case 0x79:case 0x7a: +{ +ptr += 1; +goto state_label_25; // include_quotes +}break; +case 0x22: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PPIncludeFile; +token.kind = 10; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_26: // pre_L +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_2; // identifier +} +} +switch (*ptr){ +default: +{ +goto state_label_2; // identifier +}break; +case 0x22: +{ +ptr += 1; +goto state_label_31; // string +}break; +case 0x52: +{ +ptr += 1; +goto state_label_30; // pre_R +}break; +} +} +{ +state_label_27: // pre_u +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_2; // identifier +} +} +switch (*ptr){ +default: +{ +goto state_label_2; // identifier +}break; +case 0x22: +{ +ptr += 1; +goto state_label_31; // string +}break; +case 0x38: +{ +ptr += 1; +flags_ZF0 |= 0x8; +goto state_label_29; // pre_u8 +}break; +case 0x52: +{ +ptr += 1; +goto state_label_30; // pre_R +}break; +} +} +{ +state_label_28: // pre_U +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_2; // identifier +} +} +switch (*ptr){ +default: +{ +goto state_label_2; // identifier +}break; +case 0x22: +{ +ptr += 1; +goto state_label_31; // string +}break; +case 0x52: +{ +ptr += 1; +goto state_label_30; // pre_R +}break; +} +} +{ +state_label_29: // pre_u8 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_2; // identifier +} +} +switch (*ptr){ +default: +{ +goto state_label_2; // identifier +}break; +case 0x22: +{ +ptr += 1; +goto state_label_31; // string +}break; +case 0x52: +{ +ptr += 1; +goto state_label_30; // pre_R +}break; +} +} +{ +state_label_30: // pre_R +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_2; // identifier +} +} +switch (*ptr){ +default: +{ +goto state_label_2; // identifier +}break; +case 0x22: +{ +ptr += 1; +delim_first = ptr; +goto state_label_44; // raw_string_get_delim +}break; +} +} +{ +state_label_31: // string +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +ptr += 1; +goto state_label_31; // string +}break; +case 0x0a: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x22: +{ +if ((!HasFlag(flags_ZF0, 0x40))){ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x4)){ +token.sub_kind = TokenCppKind_LiteralStringWide; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x8)){ +token.sub_kind = TokenCppKind_LiteralStringUTF8; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x10)){ +token.sub_kind = TokenCppKind_LiteralStringUTF16; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x20)){ +token.sub_kind = TokenCppKind_LiteralStringUTF32; +token.kind = 10; +break; +} +token.sub_kind = TokenCppKind_LiteralString; +token.kind = 10; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +ptr += 1; +goto state_label_31; // string +}break; +case 0x27: +{ +if ((HasFlag(flags_ZF0, 0x40))){ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x4)){ +token.sub_kind = TokenCppKind_LiteralCharacterWide; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x8)){ +token.sub_kind = TokenCppKind_LiteralCharacterUTF8; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x10)){ +token.sub_kind = TokenCppKind_LiteralCharacterUTF16; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x20)){ +token.sub_kind = TokenCppKind_LiteralCharacterUTF32; +token.kind = 10; +break; +} +token.sub_kind = TokenCppKind_LiteralCharacter; +token.kind = 10; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +ptr += 1; +goto state_label_31; // string +}break; +case 0x5c: +{ +ptr += 1; +goto state_label_32; // string_esc +}break; +} +} +{ +state_label_32: // string_esc +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_EOF; +token.kind = 0; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto end; +} +} +switch (*ptr){ +default: +{ +ptr += 1; +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37: +{ +ptr += 1; +goto state_label_33; // string_esc_oct2 +}break; +case 0x55: +{ +ptr += 1; +goto state_label_36; // string_esc_universal_8 +}break; +case 0x75: +{ +ptr += 1; +goto state_label_40; // string_esc_universal_4 +}break; +case 0x78: +{ +ptr += 1; +goto state_label_35; // string_esc_hex +}break; +} +} +{ +state_label_33: // string_esc_oct2 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37: +{ +ptr += 1; +goto state_label_34; // string_esc_oct1 +}break; +} +} +{ +state_label_34: // string_esc_oct1 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37: +{ +ptr += 1; +goto state_label_31; // string +}break; +} +} +{ +state_label_35: // string_esc_hex +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_35; // string_esc_hex +}break; +} +} +{ +state_label_36: // string_esc_universal_8 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_37; // string_esc_universal_7 +}break; +} +} +{ +state_label_37: // string_esc_universal_7 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_38; // string_esc_universal_6 +}break; +} +} +{ +state_label_38: // string_esc_universal_6 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_39; // string_esc_universal_5 +}break; +} +} +{ +state_label_39: // string_esc_universal_5 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_40; // string_esc_universal_4 +}break; +} +} +{ +state_label_40: // string_esc_universal_4 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_41; // string_esc_universal_3 +}break; +} +} +{ +state_label_41: // string_esc_universal_3 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_42; // string_esc_universal_2 +}break; +} +} +{ +state_label_42: // string_esc_universal_2 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_43; // string_esc_universal_1 +}break; +} +} +{ +state_label_43: // string_esc_universal_1 +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_31; // string +} +} +switch (*ptr){ +default: +{ +goto state_label_31; // string +}break; +case 0x30:case 0x31:case 0x32:case 0x33:case 0x34:case 0x35:case 0x36: +case 0x37:case 0x38:case 0x39:case 0x41:case 0x42:case 0x43:case 0x44: +case 0x45:case 0x46:case 0x61:case 0x62:case 0x63:case 0x64:case 0x65: +case 0x66: +{ +ptr += 1; +goto state_label_31; // string +}break; +} +} +{ +state_label_44: // raw_string_get_delim +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_EOF; +token.kind = 0; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto end; +} +} +switch (*ptr){ +default: +{ +ptr += 1; +goto state_label_44; // raw_string_get_delim +}break; +case 0x20:case 0x29:case 0x5c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x28: +{ +delim_one_past_last = ptr; +ptr += 1; +goto state_label_45; // raw_string_find_close +}break; +} +} +{ +state_label_45: // raw_string_find_close +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_EOF; +token.kind = 0; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto end; +} +} +switch (*ptr){ +default: +{ +ptr += 1; +goto state_label_45; // raw_string_find_close +}break; +case 0x29: +{ +ptr += 1; +goto state_label_46; // raw_string_try_delim +}break; +} +} +{ +state_label_46: // raw_string_try_delim +umem delim_length = delim_one_past_last - delim_first; +umem parse_length = 0; +for (;;){ +if (parse_length == delim_length){ +goto state_label_47; // raw_string_try_quote +} +if (ptr == opl_ptr){ +goto state_label_47; // raw_string_try_quote +} +if (*ptr == delim_first[parse_length]){ +ptr += 1; +parse_length += 1; +} +else{ +goto state_label_45; // raw_string_find_close +} +} +} +{ +state_label_47: // raw_string_try_quote +if (ptr == opl_ptr){ +if ((true)){ +goto state_label_45; // raw_string_find_close +} +} +switch (*ptr){ +default: +{ +goto state_label_45; // raw_string_find_close +}break; +case 0x22: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +if (HasFlag(flags_ZF0, 0x4)){ +token.sub_kind = TokenCppKind_LiteralStringWideRaw; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x8)){ +token.sub_kind = TokenCppKind_LiteralStringUTF8Raw; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x10)){ +token.sub_kind = TokenCppKind_LiteralStringUTF16Raw; +token.kind = 10; +break; +} +if (HasFlag(flags_ZF0, 0x20)){ +token.sub_kind = TokenCppKind_LiteralStringUTF32Raw; +token.kind = 10; +break; +} +token.sub_kind = TokenCppKind_LiteralStringRaw; +token.kind = 10; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_48: // comment_block +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_BlockComment; +token.kind = 3; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_EOF; +token.kind = 0; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto end; +} +} +switch (*ptr){ +default: +{ +ptr += 1; +goto state_label_48; // comment_block +}break; +case 0x0a: +{ +ptr += 1; +flags_KB0 &= ~(0x1); +flags_KF0 &= ~(0x1); +goto state_label_48; // comment_block +}break; +case 0x2a: +{ +ptr += 1; +goto state_label_49; // comment_block_try_close +}break; +} +} +{ +state_label_49: // comment_block_try_close +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_BlockComment; +token.kind = 3; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_EOF; +token.kind = 0; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto end; +} +} +switch (*ptr){ +default: +{ +ptr += 1; +goto state_label_48; // comment_block +}break; +case 0x2a: +{ +ptr += 1; +goto state_label_49; // comment_block_try_close +}break; +case 0x2f: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_BlockComment; +token.kind = 3; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_50: // comment_line +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LineComment; +token.kind = 3; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +ptr += 1; +goto state_label_50; // comment_line +}break; +case 0x0a: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LineComment; +token.kind = 3; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_51: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Colon; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Colon; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3a: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_ColonColon; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_52: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Plus; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Plus; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2b: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PlusPlus; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PlusEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_53: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Minus; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Minus; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_MinusMinus; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_MinusEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3e: +{ +ptr += 1; +goto state_label_54; // op stage +}break; +} +} +{ +state_label_54: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Arrow; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Arrow; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2a: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_ArrowStar; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_55: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Less; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Less; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3c: +{ +ptr += 1; +goto state_label_64; // op stage +}break; +case 0x3d: +{ +ptr += 1; +goto state_label_57; // op stage +}break; +} +} +{ +state_label_56: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Grtr; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Grtr; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_GrtrEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3e: +{ +ptr += 1; +goto state_label_65; // op stage +}break; +} +} +{ +state_label_57: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LessEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LessEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3e: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Compare; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_58: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Eq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Eq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_EqEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_59: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Not; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Not; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_NotEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_60: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_And; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_And; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x26: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_AndAnd; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_61: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Or; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Or; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x7c: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_OrOr; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_62: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Star; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Star; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_StarEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_63: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Mod; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_Mod; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_ModEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_64: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LeftLeft; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LeftLeft; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LeftLeftEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_65: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_RightRight; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_RightRight; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x3d: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_RightRightEq; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_66: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PPStringify; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PPStringify; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x23: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_PPConcat; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +{ +state_label_67: // op stage +if (ptr == opl_ptr){ +if ((true)){ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +} +} +switch (*ptr){ +default: +{ +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_LexError; +token.kind = 2; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +case 0x2e: +{ +ptr += 1; +{ +Token token = {}; +token.pos = (i64)(emit_ptr - input.str); +token.size = (i64)(ptr - emit_ptr); +token.flags = flags_KB0; +do{ +token.sub_kind = TokenCppKind_DotDotDot; +token.kind = 7; +}while(0); +token_list_push(arena, &list, &token); +emit_ptr = ptr; +} +flags_ZF0 = 0; +goto state_label_1; // root +}break; +} +} +end:; +return(list); +} diff --git a/languages/generated_lexer_cpp.h b/languages/generated_lexer_cpp.h new file mode 100644 index 00000000..b3489c99 --- /dev/null +++ b/languages/generated_lexer_cpp.h @@ -0,0 +1,375 @@ +#if !defined(FCODER_LEX_GEN_HAND_WRITTEN_TYPES) +#define FCODER_LEX_GEN_HAND_WRITTEN_TYPES + +struct Lexeme_Table_Value{ + Token_Base_Kind base_kind; + u16 sub_kind; +}; + +struct Lexeme_Table_Lookup{ + b32 found_match; + Token_Base_Kind base_kind; + u16 sub_kind; +}; + +#endif +typedef u16 Token_Cpp_Kind; +enum{ +TokenCppKind_EOF = 0, +TokenCppKind_Whitespace = 1, +TokenCppKind_LexError = 2, +TokenCppKind_BlockComment = 3, +TokenCppKind_LineComment = 4, +TokenCppKind_Backslash = 5, +TokenCppKind_LiteralInteger = 6, +TokenCppKind_LiteralIntegerU = 7, +TokenCppKind_LiteralIntegerL = 8, +TokenCppKind_LiteralIntegerUL = 9, +TokenCppKind_LiteralIntegerLL = 10, +TokenCppKind_LiteralIntegerULL = 11, +TokenCppKind_LiteralIntegerHex = 12, +TokenCppKind_LiteralIntegerHexU = 13, +TokenCppKind_LiteralIntegerHexL = 14, +TokenCppKind_LiteralIntegerHexUL = 15, +TokenCppKind_LiteralIntegerHexLL = 16, +TokenCppKind_LiteralIntegerHexULL = 17, +TokenCppKind_LiteralIntegerOct = 18, +TokenCppKind_LiteralIntegerOctU = 19, +TokenCppKind_LiteralIntegerOctL = 20, +TokenCppKind_LiteralIntegerOctUL = 21, +TokenCppKind_LiteralIntegerOctLL = 22, +TokenCppKind_LiteralIntegerOctULL = 23, +TokenCppKind_LiteralFloat32 = 24, +TokenCppKind_LiteralFloat64 = 25, +TokenCppKind_LiteralString = 26, +TokenCppKind_LiteralStringWide = 27, +TokenCppKind_LiteralStringUTF8 = 28, +TokenCppKind_LiteralStringUTF16 = 29, +TokenCppKind_LiteralStringUTF32 = 30, +TokenCppKind_LiteralStringRaw = 31, +TokenCppKind_LiteralStringWideRaw = 32, +TokenCppKind_LiteralStringUTF8Raw = 33, +TokenCppKind_LiteralStringUTF16Raw = 34, +TokenCppKind_LiteralStringUTF32Raw = 35, +TokenCppKind_LiteralCharacter = 36, +TokenCppKind_LiteralCharacterWide = 37, +TokenCppKind_LiteralCharacterUTF8 = 38, +TokenCppKind_LiteralCharacterUTF16 = 39, +TokenCppKind_LiteralCharacterUTF32 = 40, +TokenCppKind_PPIncludeFile = 41, +TokenCppKind_PPErrorMessage = 42, +TokenCppKind_KeywordGeneric = 43, +TokenCppKind_BraceOp = 44, +TokenCppKind_BraceCl = 45, +TokenCppKind_ParenOp = 46, +TokenCppKind_BrackOp = 47, +TokenCppKind_ParenCl = 48, +TokenCppKind_BrackCl = 49, +TokenCppKind_Semicolon = 50, +TokenCppKind_Colon = 51, +TokenCppKind_DotDotDot = 52, +TokenCppKind_ColonColon = 53, +TokenCppKind_PlusPlus = 54, +TokenCppKind_MinusMinus = 55, +TokenCppKind_Dot = 56, +TokenCppKind_Arrow = 57, +TokenCppKind_Plus = 58, +TokenCppKind_Minus = 59, +TokenCppKind_Not = 60, +TokenCppKind_Tilde = 61, +TokenCppKind_Star = 62, +TokenCppKind_And = 63, +TokenCppKind_DotStar = 64, +TokenCppKind_ArrowStar = 65, +TokenCppKind_Div = 66, +TokenCppKind_Mod = 67, +TokenCppKind_LeftLeft = 68, +TokenCppKind_RightRight = 69, +TokenCppKind_Compare = 70, +TokenCppKind_Less = 71, +TokenCppKind_LessEq = 72, +TokenCppKind_Grtr = 73, +TokenCppKind_GrtrEq = 74, +TokenCppKind_EqEq = 75, +TokenCppKind_NotEq = 76, +TokenCppKind_Xor = 77, +TokenCppKind_Or = 78, +TokenCppKind_AndAnd = 79, +TokenCppKind_OrOr = 80, +TokenCppKind_Ternary = 81, +TokenCppKind_Eq = 82, +TokenCppKind_PlusEq = 83, +TokenCppKind_MinusEq = 84, +TokenCppKind_StarEq = 85, +TokenCppKind_DivEq = 86, +TokenCppKind_ModEq = 87, +TokenCppKind_LeftLeftEq = 88, +TokenCppKind_RightRightEq = 89, +TokenCppKind_Comma = 90, +TokenCppKind_PPStringify = 91, +TokenCppKind_PPConcat = 92, +TokenCppKind_Void = 93, +TokenCppKind_Bool = 94, +TokenCppKind_Char = 95, +TokenCppKind_Int = 96, +TokenCppKind_Float = 97, +TokenCppKind_Double = 98, +TokenCppKind_Long = 99, +TokenCppKind_Short = 100, +TokenCppKind_Unsigned = 101, +TokenCppKind_Signed = 102, +TokenCppKind_Const = 103, +TokenCppKind_Volatile = 104, +TokenCppKind_Asm = 105, +TokenCppKind_Break = 106, +TokenCppKind_Case = 107, +TokenCppKind_Catch = 108, +TokenCppKind_Continue = 109, +TokenCppKind_Default = 110, +TokenCppKind_Do = 111, +TokenCppKind_Else = 112, +TokenCppKind_For = 113, +TokenCppKind_Goto = 114, +TokenCppKind_If = 115, +TokenCppKind_Return = 116, +TokenCppKind_Switch = 117, +TokenCppKind_Try = 118, +TokenCppKind_While = 119, +TokenCppKind_StaticAssert = 120, +TokenCppKind_ConstCast = 121, +TokenCppKind_DynamicCast = 122, +TokenCppKind_ReinterpretCast = 123, +TokenCppKind_StaticCast = 124, +TokenCppKind_Class = 125, +TokenCppKind_Enum = 126, +TokenCppKind_Struct = 127, +TokenCppKind_Typedef = 128, +TokenCppKind_Union = 129, +TokenCppKind_Template = 130, +TokenCppKind_Typename = 131, +TokenCppKind_Friend = 132, +TokenCppKind_Namespace = 133, +TokenCppKind_Private = 134, +TokenCppKind_Protected = 135, +TokenCppKind_Public = 136, +TokenCppKind_Using = 137, +TokenCppKind_Extern = 138, +TokenCppKind_Export = 139, +TokenCppKind_Inline = 140, +TokenCppKind_Static = 141, +TokenCppKind_Virtual = 142, +TokenCppKind_AlignAs = 143, +TokenCppKind_Explicit = 144, +TokenCppKind_NoExcept = 145, +TokenCppKind_NullPtr = 146, +TokenCppKind_Operator = 147, +TokenCppKind_Register = 148, +TokenCppKind_This = 149, +TokenCppKind_ThreadLocal = 150, +TokenCppKind_SizeOf = 151, +TokenCppKind_AlignOf = 152, +TokenCppKind_DeclType = 153, +TokenCppKind_TypeID = 154, +TokenCppKind_New = 155, +TokenCppKind_Delete = 156, +TokenCppKind_LiteralTrue = 157, +TokenCppKind_LiteralFalse = 158, +TokenCppKind_Identifier = 159, +TokenCppKind_PPInclude = 160, +TokenCppKind_PPVersion = 161, +TokenCppKind_PPDefine = 162, +TokenCppKind_PPUndef = 163, +TokenCppKind_PPIf = 164, +TokenCppKind_PPIfDef = 165, +TokenCppKind_PPIfNDef = 166, +TokenCppKind_PPElse = 167, +TokenCppKind_PPElIf = 168, +TokenCppKind_PPEndIf = 169, +TokenCppKind_PPError = 170, +TokenCppKind_PPImport = 171, +TokenCppKind_PPUsing = 172, +TokenCppKind_PPLine = 173, +TokenCppKind_PPPragma = 174, +TokenCppKind_PPUnknown = 175, +TokenCppKind_PPDefined = 176, +TokenCppKind_COUNT = 177, +}; +char *token_cpp_kind_names[] = { +"EOF", +"Whitespace", +"LexError", +"BlockComment", +"LineComment", +"Backslash", +"LiteralInteger", +"LiteralIntegerU", +"LiteralIntegerL", +"LiteralIntegerUL", +"LiteralIntegerLL", +"LiteralIntegerULL", +"LiteralIntegerHex", +"LiteralIntegerHexU", +"LiteralIntegerHexL", +"LiteralIntegerHexUL", +"LiteralIntegerHexLL", +"LiteralIntegerHexULL", +"LiteralIntegerOct", +"LiteralIntegerOctU", +"LiteralIntegerOctL", +"LiteralIntegerOctUL", +"LiteralIntegerOctLL", +"LiteralIntegerOctULL", +"LiteralFloat32", +"LiteralFloat64", +"LiteralString", +"LiteralStringWide", +"LiteralStringUTF8", +"LiteralStringUTF16", +"LiteralStringUTF32", +"LiteralStringRaw", +"LiteralStringWideRaw", +"LiteralStringUTF8Raw", +"LiteralStringUTF16Raw", +"LiteralStringUTF32Raw", +"LiteralCharacter", +"LiteralCharacterWide", +"LiteralCharacterUTF8", +"LiteralCharacterUTF16", +"LiteralCharacterUTF32", +"PPIncludeFile", +"PPErrorMessage", +"KeywordGeneric", +"BraceOp", +"BraceCl", +"ParenOp", +"BrackOp", +"ParenCl", +"BrackCl", +"Semicolon", +"Colon", +"DotDotDot", +"ColonColon", +"PlusPlus", +"MinusMinus", +"Dot", +"Arrow", +"Plus", +"Minus", +"Not", +"Tilde", +"Star", +"And", +"DotStar", +"ArrowStar", +"Div", +"Mod", +"LeftLeft", +"RightRight", +"Compare", +"Less", +"LessEq", +"Grtr", +"GrtrEq", +"EqEq", +"NotEq", +"Xor", +"Or", +"AndAnd", +"OrOr", +"Ternary", +"Eq", +"PlusEq", +"MinusEq", +"StarEq", +"DivEq", +"ModEq", +"LeftLeftEq", +"RightRightEq", +"Comma", +"PPStringify", +"PPConcat", +"Void", +"Bool", +"Char", +"Int", +"Float", +"Double", +"Long", +"Short", +"Unsigned", +"Signed", +"Const", +"Volatile", +"Asm", +"Break", +"Case", +"Catch", +"Continue", +"Default", +"Do", +"Else", +"For", +"Goto", +"If", +"Return", +"Switch", +"Try", +"While", +"StaticAssert", +"ConstCast", +"DynamicCast", +"ReinterpretCast", +"StaticCast", +"Class", +"Enum", +"Struct", +"Typedef", +"Union", +"Template", +"Typename", +"Friend", +"Namespace", +"Private", +"Protected", +"Public", +"Using", +"Extern", +"Export", +"Inline", +"Static", +"Virtual", +"AlignAs", +"Explicit", +"NoExcept", +"NullPtr", +"Operator", +"Register", +"This", +"ThreadLocal", +"SizeOf", +"AlignOf", +"DeclType", +"TypeID", +"New", +"Delete", +"LiteralTrue", +"LiteralFalse", +"Identifier", +"PPInclude", +"PPVersion", +"PPDefine", +"PPUndef", +"PPIf", +"PPIfDef", +"PPIfNDef", +"PPElse", +"PPElIf", +"PPEndIf", +"PPError", +"PPImport", +"PPUsing", +"PPLine", +"PPPragma", +"PPUnknown", +"PPDefined", +}; diff --git a/lexer_generator/4coder_lex_gen_hand_written.cpp b/lexer_generator/4coder_lex_gen_hand_written.cpp new file mode 100644 index 00000000..0976b05e --- /dev/null +++ b/lexer_generator/4coder_lex_gen_hand_written.cpp @@ -0,0 +1,45 @@ +#if !defined(FCODER_LEX_GEN_HAND_WRITTEN) +#define FCODER_LEX_GEN_HAND_WRITTEN + +internal u64 +lexeme_hash(u64 seed, u8 *ptr, umem size){ + u64 result = 0; + for (umem i = 0; i < size; i += 1, ptr += 1){ + result ^= ((*ptr) ^ result*59) + seed; + } + return(result); +} + +internal Lexeme_Table_Lookup +lexeme_table_lookup(u64 *hash_array, String_Const_u8 *key_array, + Lexeme_Table_Value *value_array, i32 slot_count, u64 seed, + u8 *ptr, umem size){ + Lexeme_Table_Lookup result = {}; + u64 hash = lexeme_hash(seed, ptr, size); + u64 comparison_hash = hash | 1; + i32 first_index = (hash % slot_count); + i32 index = first_index; + for (;;){ + if (hash_array[index] == comparison_hash){ + if (string_match(SCu8(ptr, size), key_array[index])){ + result.found_match = true; + result.base_kind = value_array[index].base_kind; + result.sub_kind = value_array[index].sub_kind; + break; + } + } + else if (hash_array[index] == 0){ + break; + } + index += 1; + if (index == slot_count){ + index = 0; + } + if (index == first_index){ + break; + } + } + return(result); +} + +#endif \ No newline at end of file diff --git a/lexer_generator/4coder_lex_gen_hand_written.h b/lexer_generator/4coder_lex_gen_hand_written.h new file mode 100644 index 00000000..52397f30 --- /dev/null +++ b/lexer_generator/4coder_lex_gen_hand_written.h @@ -0,0 +1,15 @@ +#if !defined(FCODER_LEX_GEN_HAND_WRITTEN_TYPES) +#define FCODER_LEX_GEN_HAND_WRITTEN_TYPES + +struct Lexeme_Table_Value{ + Token_Base_Kind base_kind; + u16 sub_kind; +}; + +struct Lexeme_Table_Lookup{ + b32 found_match; + Token_Base_Kind base_kind; + u16 sub_kind; +}; + +#endif \ No newline at end of file diff --git a/lexer_generator/4coder_lex_gen_main.cpp b/lexer_generator/4coder_lex_gen_main.cpp index 616752c4..89528773 100644 --- a/lexer_generator/4coder_lex_gen_main.cpp +++ b/lexer_generator/4coder_lex_gen_main.cpp @@ -11,1092 +11,3947 @@ #include "4coder_base_types.h" #include "4coder_table.h" #include "4coder_token.h" +#include "pcg_basic.h" #include "4coder_base_types.cpp" #include "4coder_stringf.cpp" #include "4coder_malloc_allocator.cpp" +#include "4coder_hash_functions.cpp" #include "4coder_table.cpp" +#include "pcg_basic.c" + +#define LANG_NAME_LOWER_STR stringify(LANG_NAME_LOWER) +#define LANG_NAME_CAMEL_STR stringify(LANG_NAME_CAMEL) //////////////////////////////// +// NOTE(allen): PRIMARY MODEL + +struct Token_Kind_Node{ + Token_Kind_Node *next; + b32 optimized_in; + String_Const_u8 name; + Token_Base_Kind base_kind; +}; + +struct Token_Kind_Set{ + Token_Kind_Node *first; + Token_Kind_Node *last; + i32 count; + Table_Data_u64 name_to_ptr; +}; + struct Keyword{ Keyword *next; String_Const_u8 name; String_Const_u8 lexeme; - Token_Base_Kind base_kind; }; struct Keyword_Set{ + Keyword_Set *next; Keyword *first; Keyword *last; i32 count; + b32 has_fallback_token_kind; + String_Const_u8 fallback_name; + Table_Data_u64 name_to_ptr; + Table_Data_u64 lexeme_to_ptr; + String_Const_u8 pretty_name; }; -struct Character_Name_Set{ - Table_u64_Data char_to_name; +struct Keyword_Set_List{ + Keyword_Set *first; + Keyword_Set *last; + i32 count; }; +struct Keyword_Layout{ + u64 seed; + u64 error_score; + u64 max_single_error_score; + f32 iterations_per_lookup; + u64 *hashes; + u64 *contributed_error; + Keyword **slots; + i32 slot_count; +}; + +typedef i32 Flag_Reset_Rule; +enum{ + FlagResetRule_AutoZero, + FlagResetRule_KeepState, + FlagResetRule_COUNT, +}; + +struct Flag{ + Flag *next; + Flag_Reset_Rule reset_rule; + Token_Base_Flag emit_flags; + u16 emit_sub_flags; + + b32 optimized_in; + String_Const_u8 base_name; + i32 number; + i32 index; + u32 value; +}; + +struct Flag_Set{ + Flag *first; + Flag *last; + i32 count; +}; + +typedef i32 Emit_Handler_Kind; +enum{ + EmitHandlerKind_Direct, + EmitHandlerKind_Keywords, +}; + +struct Emit_Handler{ + Emit_Handler *next; + Emit_Handler_Kind kind; + Flag *flag_check; + union{ + String_Const_u8 token_name; + Keyword_Set *keywords; + }; +}; + +struct Emit_Check{ + Emit_Check *next; + String_Const_u8 emit_check; + Flag *flag; + b32 value; +}; + +struct Emit_Check_List{ + Emit_Check *first; + Emit_Check *last; + i32 count; +}; + +struct Emit_Rule{ + Emit_Check_List emit_checks; + Emit_Handler *first; + Emit_Handler *last; + i32 count; +}; + +typedef i32 Action_Kind; +enum{ + ActionKind_SetFlag, + ActionKind_ZeroFlags, + ActionKind_DelimMarkFirst, + ActionKind_DelimMarkOnePastLast, + ActionKind_Consume, + ActionKind_Emit, +}; + +struct Action{ + Action *next; + Action *prev; + Action_Kind kind; + union{ + struct{ + Flag *flag; + b32 value; + } set_flag; + Emit_Rule *emit_rule; + }; +}; + +struct Action_List{ + Action *first; + Action *last; + i32 count; +}; + +typedef i32 Action_Context; +enum{ + ActionContext_Normal, + ActionContext_EndOfFile, +}; + +typedef i32 Transition_Consume_Rule; +enum{ + Transition_Consume, + Transition_NoConsume, +}; + +global u16 smi_eof = 256; + +struct Field_Pin{ + Field_Pin *next; + + // This represents the set of flags with the particular /flag/ set to /flag/ + // exactly half of all flag state possibilities. + Flag *flag; + b32 value; +}; + +struct Field_Pin_List{ + Field_Pin_List *next; + + // This set is the intersection of the set represented by each pin. + // A list with nothing in it is _always_ the "full set". + Field_Pin *first; + Field_Pin *last; + i32 count; +}; + +struct Field_Set{ + // This set is the union of the set represented by each list. + Field_Pin_List *first; + Field_Pin_List *last; + i32 count; +}; + +struct Input_Set{ + u16 *inputs; + i32 count; +}; + +struct Condition_Node{ + Condition_Node *next; + Field_Set fields; + Input_Set inputs; +}; + +struct Condition_Set{ + Condition_Node *first; + Condition_Node *last; + i32 count; +}; + +typedef i32 Transition_Case_Kind; +enum{ + TransitionCaseKind_NONE, + + // intermediates only + TransitionCaseKind_CharaterArray, + TransitionCaseKind_EOF, + TransitionCaseKind_Fallback, + + // actually stored in Transition_Case "kind" field + TransitionCaseKind_DelimMatch, + TransitionCaseKind_DelimMatchFail, + TransitionCaseKind_ConditionSet, +}; + +struct Transition_Case{ + Transition_Case_Kind kind; + union{ + Condition_Set condition_set; + }; +}; + +struct Transition{ + Transition *next; + Transition *prev; + struct State *parent_state; + Transition_Case condition; + Action_List activation_actions; + struct State *dst_state; +}; + +struct Transition_List{ + Transition *first; + Transition *last; + i32 count; +}; + +struct Transition_Ptr_Node{ + Transition_Ptr_Node *next; + Transition *ptr; +}; + +struct Transition_Ptr_Set{ + Transition_Ptr_Node *first; + Transition_Ptr_Node *last; + i32 count; +}; + +struct State{ + State *next; + Transition_List transitions; + String_Const_u8 pretty_name; + + b32 optimized_in; + i32 number; + Transition_Ptr_Set back_references; + + Action_List on_entry_actions; +}; + +struct State_Set{ + State *first; + State *last; + i32 count; +}; + +struct Lexer_Model{ + State *root; + Flag_Set flags; + State_Set states; +}; + +struct Lexer_Primary_Context{ + Base_Allocator *allocator; + Arena arena; + Token_Kind_Set tokens; + Keyword_Set_List keywords; + Lexer_Model model; +}; + +//////////////////////////////// + +struct Flag_Ptr_Node{ + Flag_Ptr_Node *next; + Flag *flag; +}; + +struct Flag_Bucket{ + String_Const_u8 pretty_name; + Flag_Ptr_Node *first; + Flag_Ptr_Node *last; + i32 max_bits; + i32 count; + + i32 number_of_variables; +}; + +typedef i32 Flag_Bind_Property; +enum{ + FlagBindProperty_Free, + FlagBindProperty_Bound, + FlagBindProperty_COUNT, +}; + +struct Flag_Bucket_Set{ + Flag_Bucket buckets[FlagBindProperty_COUNT][FlagResetRule_COUNT]; +}; + +struct Partial_Transition{ + Partial_Transition *next; + Field_Set fields; + Action_List actions; + State *dst_state; +}; + +struct Partial_Transition_List{ + Partial_Transition *first; + Partial_Transition *last; + i32 count; +}; + +struct Grouped_Input_Handler{ + Grouped_Input_Handler *next; + + u8 inputs[256]; + i32 input_count; + b8 inputs_used[256]; + + Partial_Transition_List partial_transitions; +}; + +struct Grouped_Input_Handler_List{ + Grouped_Input_Handler *first; + Grouped_Input_Handler *last; + i32 count; + + Grouped_Input_Handler *group_with_biggest_input_set; +}; + +//////////////////////////////// + +// NOTE(allen): MODELING SYNTAX HELPERS + struct Operator{ Operator *next; - String_Const_u8 op; String_Const_u8 name; - Token_Base_Kind base_kind; + String_Const_u8 op; }; struct Operator_Set{ Operator *first; Operator *last; i32 count; + Table_Data_u64 lexeme_to_ptr; }; -struct Direct_Kind{ - Direct_Kind *next; - String_Const_u8 name; - Token_Base_Kind base_kind; +struct Lexer_Helper_Context{ + Lexer_Primary_Context primary_ctx; + Table_u64_Data char_to_name; + Token_Base_Kind selected_base_kind; + State *selected_state; + Operator_Set *selected_op_set; + Keyword_Set *selected_key_set; + Emit_Rule *selected_emit_rule; + Transition *selected_transition; + + // convenience pointer to primary's arena. + Arena *arena; }; +struct Character_Set{ + Table_u64_u64 table; +}; + +#include "4coder_lex_gen_hand_written.h" +#include "4coder_lex_gen_hand_written.cpp" + +//////////////////////////////// +//////////////////////////////// +//////////////////////////////// +//////////////////////////////// + +// NOTE(allen): INTERNAL CONSTRUCTORS + +internal void +smi_primary_init(Base_Allocator *allocator, Lexer_Primary_Context *ctx){ + ctx->allocator = allocator; + ctx->arena = make_arena(allocator); + ctx->model.root = 0; + ctx->tokens.name_to_ptr = make_table_Data_u64(allocator, 400); +} + +internal b32 +smi_try_add_token(Lexer_Primary_Context *ctx, String_Const_u8 name, Token_Base_Kind base_kind){ + b32 result = false; + Token_Kind_Set *set = &ctx->tokens; + Table_Lookup lookup = table_lookup(&set->name_to_ptr, make_data(name.str, name.size)); + if (!lookup.found_match){ + Token_Kind_Node *node = push_array_zero(&ctx->arena, Token_Kind_Node, 1); + node->name = push_string_copy(&ctx->arena, name); + node->base_kind = base_kind; + table_insert(&set->name_to_ptr, make_data(node->name.str, node->name.size), (u64)PtrAsInt(node)); + sll_queue_push(set->first, set->last, node); + set->count += 1; + result = true; + } + return(result); +} + +internal b32 +smi_key(Lexer_Primary_Context *ctx, Keyword_Set *set, String_Const_u8 name, String_Const_u8 lexeme, Token_Base_Kind base_kind){ + b32 result = false; + Table_Lookup lookup = table_lookup(&set->name_to_ptr, make_data(name.str, name.size)); + if (!lookup.found_match){ + lookup = table_lookup(&set->lexeme_to_ptr, make_data(lexeme.str, lexeme.size)); + if (!lookup.found_match){ + if (smi_try_add_token(ctx, name, base_kind)){ + Keyword *key = push_array_zero(&ctx->arena, Keyword, 1); + key->name = push_string_copy(&ctx->arena, name); + key->lexeme = push_string_copy(&ctx->arena, lexeme); + table_insert(&set->name_to_ptr, make_data(key->name.str, key->name.size), (u64)PtrAsInt(key)); + table_insert(&set->lexeme_to_ptr, make_data(key->lexeme.str, key->lexeme.size), (u64)PtrAsInt(key)); + sll_queue_push(set->first, set->last, key); + set->count += 1; + result = true; + } + } + } + return(result); +} + +internal b32 +smi_key_fallback(Lexer_Primary_Context *ctx, Keyword_Set *set, String_Const_u8 name, Token_Base_Kind base_kind){ + b32 result = false; + if (!set->has_fallback_token_kind){ + if (smi_try_add_token(ctx, name, base_kind)){ + set->has_fallback_token_kind = true; + set->fallback_name = push_string_copy(&ctx->arena, name); + result = true; + } + } + return(result); +} + +internal State* +smi_add_state(Lexer_Primary_Context *ctx, String_Const_u8 pretty_name){ + State_Set *set = &ctx->model.states; + State *state = push_array_zero(&ctx->arena, State, 1); + sll_queue_push(set->first, set->last, state); + set->count += 1; + state->pretty_name = push_string_copy(&ctx->arena, pretty_name); + return(state); +} + +internal Flag* +smi_add_flag(Lexer_Primary_Context *ctx, Flag_Reset_Rule rule){ + Flag_Set *set = &ctx->model.flags; + Flag *flag = push_array_zero(&ctx->arena, Flag, 1); + flag->reset_rule = rule; + sll_queue_push(set->first, set->last, flag); + set->count += 1; + return(flag); +} + +internal Emit_Rule* +smi_emit_rule(Arena *arena){ + return(push_array_zero(arena, Emit_Rule, 1)); +} + +internal Emit_Handler* +smi_emit_handler(Arena *arena, Emit_Rule *rule, String_Const_u8 name, Flag *flag_check){ + Emit_Handler *handler = push_array_zero(arena, Emit_Handler, 1); + handler->kind = EmitHandlerKind_Direct; + handler->flag_check = flag_check; + handler->token_name = name; + if (rule != 0){ + sll_queue_push(rule->first, rule->last, handler); + rule->count += 1; + } + return(handler); +} + +internal Emit_Handler* +smi_emit_handler(Arena *arena, Emit_Rule *rule, Keyword_Set *set, Flag *flag_check){ + Emit_Handler *handler = push_array_zero(arena, Emit_Handler, 1); + handler->kind = EmitHandlerKind_Keywords; + handler->flag_check = flag_check; + handler->keywords = set; + if (rule != 0){ + sll_queue_push(rule->first, rule->last, handler); + rule->count += 1; + } + return(handler); +} + +internal void +smi_append_set_flag(Arena *arena, Action_List *list, Flag *flag, b32 value){ + Action *action = push_array_zero(arena, Action, 1); + zdll_push_back(list->first, list->last, action); + list->count += 1; + action->kind = ActionKind_SetFlag; + action->set_flag.flag = flag; + action->set_flag.value = value; +} + +internal void +smi_append_zero_flags(Arena *arena, Action_List *list){ + Action *action = push_array_zero(arena, Action, 1); + zdll_push_back(list->first, list->last, action); + list->count += 1; + action->kind = ActionKind_ZeroFlags; +} + +internal void +smi_append_delim_mark_first(Arena *arena, Action_List *list){ + Action *action = push_array_zero(arena, Action, 1); + zdll_push_back(list->first, list->last, action); + list->count += 1; + action->kind = ActionKind_DelimMarkFirst; +} + +internal void +smi_append_delim_mark_one_past_last(Arena *arena, Action_List *list){ + Action *action = push_array_zero(arena, Action, 1); + zdll_push_back(list->first, list->last, action); + list->count += 1; + action->kind = ActionKind_DelimMarkOnePastLast; +} + +internal void +smi_append_consume(Arena *arena, Action_List *list){ + Action *action = push_array_zero(arena, Action, 1); + zdll_push_back(list->first, list->last, action); + list->count += 1; + action->kind = ActionKind_Consume; +} + +internal void +smi_append_emit(Arena *arena, Action_List *list, Emit_Rule *emit){ + Action *action = push_array_zero(arena, Action, 1); + zdll_push_back(list->first, list->last, action); + list->count += 1; + action->kind = ActionKind_Emit; + action->emit_rule = emit; +} + //////////////////////////////// -int main(void){ - // utf-8 bytes - u8 utf8[129]; +#if 0 +internal void +CHECK_PIN_LIST(Field_Pin_List *list){ + i32 counter = 0; + for (Field_Pin *pin = list->first; + pin != 0; + pin = pin->next){ + counter += 1; + } + Assert(counter == list->count); +} +#else +#define CHECK_PIN_LIST(x) +#endif + +internal Field_Pin* +smi_field_pin_copy(Arena *arena, Field_Pin *pin){ + Field_Pin *result = push_array_zero(arena, Field_Pin, 1); + result->flag = pin->flag; + result->value = pin->value; + return(result); +} + +internal Field_Pin_List* +smi_field_pin_list_copy(Arena *arena, Field_Pin_List list){ + CHECK_PIN_LIST(&list); + Field_Pin_List *new_list = push_array_zero(arena, Field_Pin_List, 1); + new_list->count = list.count; + for (Field_Pin *node = list.first; + node != 0; + node = node->next){ + Field_Pin *new_pin = smi_field_pin_copy(arena, node); + sll_queue_push(new_list->first, new_list->last, new_pin); + } + CHECK_PIN_LIST(new_list); + return(new_list); +} + +internal Field_Set +smi_field_set_copy(Arena *arena, Field_Set set){ + Field_Set result = {}; + result.count = set.count; + for (Field_Pin_List *pin_list = set.first; + pin_list != 0; + pin_list = pin_list->next){ + Field_Pin_List *new_list = smi_field_pin_list_copy(arena, *pin_list); + sll_queue_push(result.first, result.last, new_list); + } + return(result); +} + +internal void +smi_field_pin_sub__recursive(Arena *arena, Field_Pin_List a, Field_Pin_List *list, Field_Pin_List growing_list, Field_Set *result){ + if (list != 0){ + growing_list.count += 1; + Field_Pin_List *next_list = list->next; + for (Field_Pin *pin = list->first; + pin != 0; + pin = pin->next){ + Field_Pin local_pin = *pin; + local_pin.next = 0; + sll_queue_push(growing_list.first, growing_list.last, &local_pin); + smi_field_pin_sub__recursive(arena, a, next_list, growing_list, result); + } + } + else{ + b32 has_conflicts = false; + Temp_Memory restore_point = begin_temp(arena); + Field_Pin_List *new_list = smi_field_pin_list_copy(arena, a); + for (Field_Pin *pin = growing_list.first; + pin != 0; + pin = pin->next){ + b32 is_duplicate = false; + for (Field_Pin *a_pin = new_list->first; + a_pin != 0; + a_pin = a_pin->next){ + if (pin->flag == a_pin->flag){ + if (pin->value == a_pin->value){ + end_temp(restore_point); + has_conflicts = true; + goto double_break; + } + is_duplicate = true; + break; + } + } + if (!is_duplicate){ + Field_Pin *new_pin = smi_field_pin_copy(arena, pin); + new_pin->value = !new_pin->value; + sll_queue_push(new_list->first, new_list->last, new_pin); + new_list->count += 1; + } + } + double_break:; + + if (!has_conflicts){ + CHECK_PIN_LIST(new_list); + sll_queue_push(result->first, result->last, new_list); + result->count += 1; + } + } +} + +internal Field_Set +smi_field_pin_sub(Arena *arena, Field_Pin_List a, Field_Set b){ + Field_Set result = {}; + Field_Pin_List *list = b.first; + Field_Pin_List growing_list = {}; + smi_field_pin_sub__recursive(arena, a, list, growing_list, &result); + return(result); +} + +internal Field_Set +smi_field_set_subtract(Arena *arena, Field_Set a, Field_Set b){ + Field_Set result = {}; + for (Field_Pin_List *list = a.first; + list != 0; + list = list->next){ + Field_Set partial = smi_field_pin_sub(arena, *list, b); + if (result.first == 0){ + result = partial; + } + else{ + if (partial.first != 0){ + result.last->next = partial.first; + result.last = partial.last; + result.count += partial.count; + } + } + } + return(result); +} + +internal Field_Set +smi_field_set_intersect(Arena *arena, Field_Set a, Field_Set b){ + Field_Set result = {}; + for (Field_Pin_List *a_list = a.first; + a_list != 0; + a_list = a_list->next){ + for (Field_Pin_List *b_list = b.first; + b_list != 0; + b_list = b_list->next){ + b32 has_conflicts = false; + Temp_Memory restore_point = begin_temp(arena); + Field_Pin_List *new_list = smi_field_pin_list_copy(arena, *a_list); + for (Field_Pin *b_pin = b_list->first; + b_pin != 0; + b_pin = b_pin->next){ + b32 is_duplicate = false; + for (Field_Pin *pin = new_list->first; + pin != 0; + pin = pin->next){ + if (pin->flag == pin->flag){ + if (pin->value != pin->value){ + end_temp(restore_point); + has_conflicts = true; + goto double_break; + } + is_duplicate = true; + break; + } + } + if (!is_duplicate){ + Field_Pin *new_pin = smi_field_pin_copy(arena, b_pin); + sll_queue_push(new_list->first, new_list->last, new_pin); + new_list->count += 1; + } + } + double_break:; + + if (!has_conflicts){ + sll_queue_push(result.first, result.last, new_list); + result.count += 1; + } + } + } + return(result); +} + +internal b32 +smi_field_set_match(Arena *scratch, Field_Set a, Field_Set b){ + Temp_Memory temp = begin_temp(scratch); + b32 result = false; + Field_Set sub = smi_field_set_subtract(scratch, a, b); + if (sub.count == 0){ + sub = smi_field_set_subtract(scratch, b, a); + if (sub.count == 0){ + result = true; + } + } + end_temp(temp); + return(result); +} + +internal Field_Set +smi_field_set_union(Arena *arena, Field_Set a, Field_Set b){ + Field_Set result = {}; + if (a.first != 0){ + if (b.first != 0){ + a = smi_field_set_copy(arena, a); + // TODO(allen): simplify these lists by seeing if they union + // cleanly with the lists in a! + b = smi_field_set_copy(arena, b); + result.first = a.first; + a.last->next = b.first; + result.last = b.last; + result.count = a.count + b.count; + } + else{ + result = smi_field_set_copy(arena, a); + } + } + else{ + if (b.first != 0){ + result = smi_field_set_copy(arena, b); + } + } + return(result); +} + +internal Field_Set +smi_field_set_construct(Arena *arena){ + Field_Set result = {}; + Field_Pin_List *list = push_array_zero(arena, Field_Pin_List, 1); + sll_queue_push(result.first, result.last, list); + result.count += 1; + return(result); +} + +internal Field_Set +smi_field_set_construct(Arena *arena, Flag *flag, b32 value){ + Field_Set result = {}; + if (flag != 0){ + Field_Pin_List *list = push_array_zero(arena, Field_Pin_List, 1); + sll_queue_push(result.first, result.last, list); + result.count += 1; + Field_Pin *pin = push_array_zero(arena, Field_Pin, 1); + sll_queue_push(list->first, list->last, pin); + list->count += 1; + pin->flag = flag; + pin->value = value; + } + else{ + result = smi_field_set_construct(arena); + } + return(result); +} + +internal Input_Set +smi_input_set_copy(Arena *arena, Input_Set set){ + Input_Set result = {}; + result.inputs = push_array_write(arena, u16, set.count, set.inputs); + result.count = set.count; + return(result); +} + +internal Input_Set +smi_input_set_subtract(Arena *arena, Input_Set a, Input_Set b){ + Input_Set result = {}; + if (a.count > 0){ + Temp_Memory restore_point = begin_temp(arena); + result = smi_input_set_copy(arena, a); + for (i32 i = 0; i < result.count; i += 1){ + b32 is_subtracted = false; + for (i32 j = 0; j < b.count; j += 1){ + if (result.inputs[i] == b.inputs[j]){ + is_subtracted = true; + break; + } + } + if (is_subtracted){ + result.count -= 1; + result.inputs[i] = result.inputs[result.count]; + i -= 1; + } + } + if (a.count == 0){ + end_temp(restore_point); + block_zero_struct(&result); + } + } + return(result); +} + +internal Input_Set +smi_input_set_intersect(Arena *arena, Input_Set a, Input_Set b){ + Input_Set result = {}; + if (a.count > 0 && b.count > 0){ + Temp_Memory restore_point = begin_temp(arena); + result = smi_input_set_copy(arena, a); + for (i32 i = 0; i < result.count; i += 1){ + b32 is_shared = false; + for (i32 j = 0; j < b.count; j += 1){ + if (result.inputs[i] == b.inputs[j]){ + is_shared = true; + break; + } + } + if (!is_shared){ + result.count -= 1; + result.inputs[i] = result.inputs[result.count]; + i -= 1; + } + } + if (result.count == 0){ + end_temp(restore_point); + block_zero_struct(&result); + } + } + return(result); +} + +internal Input_Set +smi_input_set_union(Arena *arena, Input_Set a, Input_Set b){ + Input_Set result = {}; + if (a.count > 0 || b.count > 0){ + result.inputs = push_array_zero(arena, u16, a.count + b.count); + block_copy_dynamic_array(result.inputs, a.inputs, a.count); + result.count = a.count; + for (i32 i = 0; i < b.count; i += 1){ + b32 is_duplicate = false; + for (i32 j = 0; j < result.count; j += 1){ + if (result.inputs[j] == b.inputs[i]){ + is_duplicate = true; + break; + } + } + if (!is_duplicate){ + result.inputs[result.count] = b.inputs[i]; + result.count += 1; + } + } + } + return(result); +} + +internal Input_Set +smi_input_set_construct(Arena *arena, String_Const_u8 characters){ + Input_Set result = {}; + result.count = (i32)characters.size; + result.inputs = push_array_zero(arena, u16, result.count); + for (umem i = 0; i < characters.size; i += 1){ + result.inputs[i] = (u16)characters.str[i]; + } + return(result); +} + +internal Input_Set +smi_input_set_construct_eof(Arena *arena){ + Input_Set result = {}; + result.count = 1; + result.inputs = push_array_zero(arena, u16, result.count); + result.inputs[0] = smi_eof; + return(result); +} + +internal Input_Set +smi_input_set_construct_fallback(Arena *arena){ + Input_Set result = {}; + result.count = 257; + result.inputs = push_array_zero(arena, u16, result.count); + for (u16 i = 0; i < 257; i += 1){ + result.inputs[i] = i; + } + return(result); +} + +internal Condition_Node* +smi_condition_node_copy(Arena *arena, Condition_Node *node){ + Condition_Node *result = push_array_zero(arena, Condition_Node, 1); + result->fields = smi_field_set_copy(arena, node->fields); + result->inputs = smi_input_set_copy(arena, node->inputs); + return(result); +} + +internal Condition_Set +smi_condition_set_copy(Arena *arena, Condition_Set set){ + Condition_Set result = {}; + for (Condition_Node *node = set.first; + node != 0; + node = node->next){ + Condition_Node *new_node = smi_condition_node_copy(arena, node); + sll_queue_push(result.first, result.last, new_node); + result.count += 1; + } + return(result); +} + +internal Condition_Set +smi_condition_node_sub(Arena *arena, Condition_Node a, Condition_Node b){ + Condition_Set result = {}; + Input_Set a_minus_b_input = smi_input_set_subtract(arena, a.inputs, b.inputs); + if (a_minus_b_input.count == 0){ + Field_Set a_minus_b_fields = smi_field_set_subtract(arena, a.fields, b.fields); + if (a_minus_b_fields.count > 0){ + Condition_Node *new_node = push_array_zero(arena, Condition_Node, 1); + new_node->inputs = smi_input_set_copy(arena, a.inputs); + new_node->fields = a_minus_b_fields; + sll_queue_push(result.first, result.last, new_node); + result.count += 1; + } + } + else{ + if (a_minus_b_input.count == a.inputs.count){ + Condition_Node *new_node = push_array_zero(arena, Condition_Node, 1); + new_node->inputs = a_minus_b_input; + new_node->fields = smi_field_set_copy(arena, a.fields); + sll_queue_push(result.first, result.last, new_node); + result.count += 1; + } + else{ + Field_Set a_minus_b_fields = smi_field_set_subtract(arena, a.fields, b.fields); + if (a_minus_b_fields.count == 0){ + Condition_Node *new_node = push_array_zero(arena, Condition_Node, 1); + new_node->inputs = a_minus_b_input; + new_node->fields = smi_field_set_copy(arena, a.fields); + sll_queue_push(result.first, result.last, new_node); + result.count += 1; + } + else{ + Input_Set a_int_b_input = smi_input_set_intersect(arena, a.inputs, b.inputs); + Condition_Node *node_1 = push_array_zero(arena, Condition_Node, 1); + node_1->inputs = a_int_b_input; + node_1->fields = a_minus_b_fields; + sll_queue_push(result.first, result.last, node_1); + Condition_Node *node_2 = push_array_zero(arena, Condition_Node, 1); + node_2->inputs = a_minus_b_input; + node_2->fields = smi_field_set_copy(arena, a.fields); + sll_queue_push(result.first, result.last, node_2); + result.count += 2; + } + } + } + return(result); +} + +internal Condition_Node* +smi_condition_node_int(Arena *arena, Condition_Node a, Condition_Node b){ + Condition_Node *result = push_array_zero(arena, Condition_Node, 1); + result->inputs = smi_input_set_intersect(arena, a.inputs, b.inputs); + result->fields = smi_field_set_intersect(arena, a.fields, b.fields); + return(result); +} + +internal Condition_Set +smi_condition_set_subtract_node(Arena *arena, Condition_Set a, Condition_Node *b){ + Condition_Set result = {}; + for (Condition_Node *node = a.first; + node != 0; + node = node->next){ + Condition_Set partial = smi_condition_node_sub(arena, *node, *b); + if (result.first == 0){ + result = partial; + } + else{ + if (partial.first != 0){ + result.last->next = partial.first; + result.last = partial.last; + result.count += partial.count; + } + } + } + return(result); +} + +internal Condition_Set +smi_condition_set_subtract(Arena *arena, Condition_Set a, Condition_Set b){ + Condition_Set result = a; + for (Condition_Node *node = b.first; + node != 0; + node = node->next){ + result = smi_condition_set_subtract_node(arena, result, node); + } + return(result); +} + +internal Condition_Set +smi_condition_set_intersect(Arena *arena, Condition_Set a, Condition_Set b){ + Condition_Set result = {}; + for (Condition_Node *a_node = a.first; + a_node != 0; + a_node = a_node->next){ + for (Condition_Node *b_node = b.first; + b_node != 0; + b_node = b_node->next){ + Condition_Node *node = smi_condition_node_int(arena, *a_node, *b_node); + if (node->inputs.count > 0 && node->fields.count > 0){ + sll_queue_push(result.first, result.last, node); + result.count += 1; + } + } + } + return(result); +} + +internal Condition_Set +smi_condition_set_union(Arena *arena, Condition_Set a, Condition_Set b){ + Condition_Set result = {}; + if (a.count != 0){ + if (b.count != 0){ + a = smi_condition_set_copy(arena, a); + // TODO(allen): simplify these down! + b = smi_condition_set_copy(arena, b); + result.first = a.first; + a.last->next = b.first; + result.last = b.last; + result.count = a.count + b.count; + } + else{ + result = smi_condition_set_copy(arena, a); + } + } + else{ + if (b.count != 0){ + result = smi_condition_set_copy(arena, b); + } + } + return(result); +} + +internal Condition_Node* +smi_condition_node(Arena *arena, Input_Set inputs, Field_Set fields){ + Condition_Node *node = push_array_zero(arena, Condition_Node, 1); + node->fields = fields; + node->inputs = inputs; + return(node); +} + +internal Condition_Set +smi_condition(Arena *arena, Input_Set inputs, Field_Set fields){ + Condition_Set result = {}; + Condition_Node *node = smi_condition_node(arena, inputs, fields); + sll_queue_push(result.first, result.last, node); + result.count += 1; + return(result); +} + +//////////////////////////////// + +internal Transition* +smi_case(Lexer_Primary_Context *ctx, State *state, + Transition_Case_Kind kind, String_Const_u8 characters, Flag *flag_check,b32 flag_check_value, + State *dst, Transition_Consume_Rule consume_rule, Emit_Rule *emit){ + Transition *transition = push_array_zero(&ctx->arena, Transition, 1); + transition->parent_state = state; + + switch (kind){ + default: + { + transition->condition.kind = kind; + }break; + + case TransitionCaseKind_CharaterArray: + { + transition->condition.kind = TransitionCaseKind_ConditionSet; + Input_Set inputs = smi_input_set_construct(&ctx->arena, characters); + Field_Set fields = smi_field_set_construct(&ctx->arena, + flag_check, flag_check_value); + transition->condition.condition_set = smi_condition(&ctx->arena, inputs, fields); + }break; + + case TransitionCaseKind_EOF: + { + transition->condition.kind = TransitionCaseKind_ConditionSet; + Input_Set inputs = smi_input_set_construct_eof(&ctx->arena); + Field_Set fields = smi_field_set_construct(&ctx->arena, + flag_check, flag_check_value); + transition->condition.condition_set = smi_condition(&ctx->arena, inputs, fields); + }break; + + case TransitionCaseKind_Fallback: + { + transition->condition.kind = TransitionCaseKind_ConditionSet; + Input_Set inputs = smi_input_set_construct_fallback(&ctx->arena); + Field_Set fields = smi_field_set_construct(&ctx->arena, + flag_check, flag_check_value); + transition->condition.condition_set = smi_condition(&ctx->arena, inputs, fields); + }break; + } + + transition->dst_state = dst; + + if (consume_rule == Transition_Consume){ + smi_append_consume(&ctx->arena, &transition->activation_actions); + } + + if (emit != 0){ + smi_append_emit(&ctx->arena, &transition->activation_actions, emit); + } + + zdll_push_back(state->transitions.first, state->transitions.last, transition); + state->transitions.count += 1; + return(transition); +} + +//////////////////////////////// + +// NOTE(allen): CONSTRUCTORS + +global Lexer_Helper_Context helper_ctx = {}; + +internal void +sm_helper_init(Base_Allocator *allocator){ + smi_primary_init(allocator, &helper_ctx.primary_ctx); + helper_ctx.char_to_name = make_table_u64_Data(allocator, 100); + + helper_ctx.arena = &helper_ctx.primary_ctx.arena; +} + +internal void +sm_char_name(u8 c, char *str){ + Table_Lookup lookup = table_lookup(&helper_ctx.char_to_name, c); + if (lookup.found_match){ + table_erase(&helper_ctx.char_to_name, lookup); + } + String_Const_u8 string = push_string_copy(helper_ctx.arena, SCu8(str)); + table_insert(&helper_ctx.char_to_name, c, make_data(string.str, string.size)); +} + +internal void +sm_select_base_kind(Token_Base_Kind kind){ + helper_ctx.selected_base_kind = kind; +} + +internal void +sm_select_state(State *state){ + helper_ctx.selected_state = state; +} + +internal void +sm_select_op_set(Operator_Set *set){ + helper_ctx.selected_op_set = set; +} + +internal void +sm_select_key_set(Keyword_Set *set){ + helper_ctx.selected_key_set = set; +} + +internal void +sm_select_emit(Emit_Rule *emit){ + helper_ctx.selected_emit_rule = emit; +} + +internal void +sm_select_transition(Transition *transition){ + helper_ctx.selected_transition = transition; +} + +internal b32 +sm_direct_token_kind(char *str){ + return(smi_try_add_token(&helper_ctx.primary_ctx, SCu8(str), helper_ctx.selected_base_kind)); +} + +internal Operator_Set* +sm_begin_op_set(void){ + Operator_Set *set = push_array_zero(helper_ctx.arena, Operator_Set, 1); + set->lexeme_to_ptr = make_table_Data_u64(helper_ctx.primary_ctx.allocator, 100); + helper_ctx.selected_op_set = set; + return(set); +} + +internal b32 +sm_op(String_Const_u8 lexeme, String_Const_u8 name){ + b32 result = false; + Operator_Set *set = helper_ctx.selected_op_set; + Table_Lookup lookup = table_lookup(&set->lexeme_to_ptr, make_data(lexeme.str, lexeme.size)); + if (!lookup.found_match){ + if (smi_try_add_token(&helper_ctx.primary_ctx, name, helper_ctx.selected_base_kind)){ + Operator *op = push_array_zero(helper_ctx.arena, Operator, 1); + op->name = push_string_copy(helper_ctx.arena, name); + op->op = push_string_copy(helper_ctx.arena, lexeme); + table_insert(&set->lexeme_to_ptr, make_data(op->op.str, op->op.size), (u64)PtrAsInt(op)); + sll_queue_push(set->first, set->last, op); + set->count += 1; + result = true; + } + } + return(result); +} + +internal b32 +sm_op(char *lexeme, char *name){ + return(sm_op(SCu8(lexeme), SCu8(name))); +} + +internal b32 +sm_op(char *lexeme){ + String_Const_u8 l = SCu8(lexeme); + List_String_Const_u8 name_list = {}; + for (umem i = 0; i < l.size; i += 1){ + Table_Lookup lookup = table_lookup(&helper_ctx.char_to_name, l.str[i]); + // If this fails first check that all the characters in the lexeme are named! + Assert(lookup.found_match); + Data name_data = {}; + table_read(&helper_ctx.char_to_name, lookup, &name_data); + string_list_push(helper_ctx.arena, &name_list, SCu8(name_data.data, name_data.size)); + } + String_Const_u8 name = string_list_flatten(helper_ctx.arena, name_list); + return(sm_op(l, name)); +} + +internal Keyword_Set* +sm_begin_key_set(String_Const_u8 pretty_name){ + Keyword_Set *set = push_array_zero(helper_ctx.arena, Keyword_Set, 1); + set->name_to_ptr = make_table_Data_u64(helper_ctx.primary_ctx.allocator, 100); + set->lexeme_to_ptr = make_table_Data_u64(helper_ctx.primary_ctx.allocator, 100); + set->pretty_name = push_string_copy(helper_ctx.arena, pretty_name); + sll_queue_push(helper_ctx.primary_ctx.keywords.first, + helper_ctx.primary_ctx.keywords.last, set); + helper_ctx.primary_ctx.keywords.count += 1; + helper_ctx.selected_key_set = set; + return(set); +} + +internal Keyword_Set* +sm_begin_key_set(char *pretty_name){ + return(sm_begin_key_set(SCu8(pretty_name))); +} + +internal b32 +sm_key(String_Const_u8 name, String_Const_u8 lexeme){ + return(smi_key(&helper_ctx.primary_ctx, helper_ctx.selected_key_set, name, lexeme, helper_ctx.selected_base_kind)); +} + +internal b32 +sm_key(char *str, char *lexeme){ + return(sm_key(SCu8(str), SCu8(lexeme))); +} + +internal b32 +sm_key(char *str){ + String_Const_u8 name = SCu8(str); + String_Const_u8 lexeme = push_string_copy(helper_ctx.arena, name); + lexeme = string_mod_lower(lexeme); + return(sm_key(name, lexeme)); +} + +internal b32 +sm_key_fallback(String_Const_u8 name){ + return(smi_key_fallback(&helper_ctx.primary_ctx, helper_ctx.selected_key_set, name, helper_ctx.selected_base_kind)); +} + +internal b32 +sm_key_fallback(char *str){ + return(sm_key_fallback(SCu8(str))); +} + +internal State* +sm_add_state(String_Const_u8 pretty_name){ + return(smi_add_state(&helper_ctx.primary_ctx, pretty_name)); +} + +internal State* +sm_add_state(char *pretty_name){ + return(smi_add_state(&helper_ctx.primary_ctx, SCu8(pretty_name))); +} + +internal State* +sm_begin_state_machine(void){ + State *state = sm_add_state("root"); + // If this fails first check sm_begin_state_machine is only called once + Assert(helper_ctx.primary_ctx.model.root == 0); + helper_ctx.primary_ctx.model.root = state; + return(state); +} + +internal Flag* +sm_add_flag(Flag_Reset_Rule rule){ + return(smi_add_flag(&helper_ctx.primary_ctx, rule)); +} + +internal void +sm_flag_bind(Flag *flag, Token_Base_Kind emit_flags){ + flag->emit_flags = emit_flags; +} + +internal void +sm_sub_flag_bind(Flag *flag, u16 emit_sub_flags){ + flag->emit_sub_flags = emit_sub_flags; +} + +internal Emit_Rule* +sm_emit_rule(void){ + Emit_Rule *rule = smi_emit_rule(helper_ctx.arena); + helper_ctx.selected_emit_rule = rule; + return(rule); +} + +internal void +sm_emit_handler_direct(Flag *flag_check, String_Const_u8 name){ + Emit_Rule *rule = helper_ctx.selected_emit_rule; + smi_emit_handler(helper_ctx.arena, rule, name, flag_check); +} + +internal void +sm_emit_handler_direct(char *name){ + sm_emit_handler_direct(0, SCu8(name)); +} + +internal void +sm_emit_handler_direct(Flag *flag_check, char *name){ + sm_emit_handler_direct(flag_check, SCu8(name)); +} + +internal void +sm_emit_handler_keys(Flag *flag_check, Keyword_Set *set){ + Emit_Rule *rule = helper_ctx.selected_emit_rule; + smi_emit_handler(helper_ctx.arena, rule, set, flag_check); +} + +internal void +sm_emit_handler_keys(Keyword_Set *set){ + sm_emit_handler_keys(0, set); +} + +internal Transition* +sm_case(String_Const_u8 str, Flag *flag_check, b32 flag_check_value, State *dst, Transition_Consume_Rule consume_rule, Emit_Rule *emit){ + Transition *transition = smi_case(&helper_ctx.primary_ctx, helper_ctx.selected_state, TransitionCaseKind_CharaterArray, str, + flag_check, flag_check_value, dst, consume_rule, emit); + helper_ctx.selected_transition = transition; + return(transition); +} +internal Transition* +sm_case(Transition_Case_Kind kind, Flag *flag_check, b32 flag_check_value, State *dst, Transition_Consume_Rule consume_rule, Emit_Rule *emit){ + Assert(kind != TransitionCaseKind_CharaterArray); + String_Const_u8 str = {}; + Transition *transition = smi_case(&helper_ctx.primary_ctx, helper_ctx.selected_state, kind, str, + flag_check, flag_check_value, dst, consume_rule, emit); + helper_ctx.selected_transition = transition; + return(transition); +} + +internal Transition* +sm_case(char *str, State *dst){ + return(sm_case(SCu8(str), 0, 0, dst, Transition_Consume, 0)); +} +internal Transition* +sm_case(u8 *str, State *dst){ + return(sm_case(SCu8(str), 0, 0, dst, Transition_Consume, 0)); +} +internal Transition* +sm_case_peek(char *str, State *dst){ + return(sm_case(SCu8(str), 0, 0, dst, Transition_NoConsume, 0)); +} +internal Transition* +sm_case_peek(u8 *str, State *dst){ + return(sm_case(SCu8(str), 0, 0, dst, Transition_NoConsume, 0)); +} +internal Transition* +sm_case_flagged(Flag *flag_check, b32 flag_check_value, char *str, State *dst){ + return(sm_case(SCu8(str), flag_check, flag_check_value, dst, Transition_Consume, 0)); +} +internal Transition* +sm_case_flagged(Flag *flag_check, b32 flag_check_value, u8 *str, State *dst){ + return(sm_case(SCu8(str), flag_check, flag_check_value, dst, Transition_Consume, 0)); +} +internal Transition* +sm_case_peek_flagged(Flag *flag_check, b32 flag_check_value, char *str, State *dst){ + return(sm_case(SCu8(str), flag_check, flag_check_value, dst, Transition_NoConsume, 0)); +} +internal Transition* +sm_case_peek_flagged(Flag *flag_check, b32 flag_check_value, u8 *str, State *dst){ + return(sm_case(SCu8(str), flag_check, flag_check_value, dst, Transition_NoConsume, 0)); +} +internal Transition* +sm_case(char *str, Emit_Rule *emit){ + return(sm_case(SCu8(str), 0, 0, helper_ctx.primary_ctx.model.root, Transition_Consume, emit)); +} +internal Transition* +sm_case(u8 *str, Emit_Rule *emit){ + return(sm_case(SCu8(str), 0, 0, helper_ctx.primary_ctx.model.root, Transition_Consume, emit)); +} +internal Transition* +sm_case_peek(char *str, Emit_Rule *emit){ + return(sm_case(SCu8(str), 0, 0, helper_ctx.primary_ctx.model.root, Transition_NoConsume, emit)); +} +internal Transition* +sm_case_peek(u8 *str, Emit_Rule *emit){ + return(sm_case(SCu8(str), 0, 0, helper_ctx.primary_ctx.model.root, Transition_NoConsume, emit)); +} +internal Transition* +sm_case_flagged(Flag *flag_check, b32 flag_check_value, char *str, Emit_Rule *emit){ + return(sm_case(SCu8(str), flag_check, flag_check_value, helper_ctx.primary_ctx.model.root, Transition_Consume, emit)); +} +internal Transition* +sm_case_flagged(Flag *flag_check, b32 flag_check_value, u8 *str, Emit_Rule *emit){ + return(sm_case(SCu8(str), flag_check, flag_check_value, helper_ctx.primary_ctx.model.root, Transition_Consume, emit)); +} +internal Transition* +sm_case_peek_flagged(Flag *flag_check, b32 flag_check_value, char *str, Emit_Rule *emit){ + return(sm_case(SCu8(str), flag_check, flag_check_value, helper_ctx.primary_ctx.model.root, Transition_NoConsume, emit)); +} +internal Transition* +sm_case_peek_flagged(Flag *flag_check, b32 flag_check_value, u8 *str, Emit_Rule *emit){ + return(sm_case(SCu8(str), flag_check, flag_check_value, helper_ctx.primary_ctx.model.root, Transition_NoConsume, emit)); +} + +internal Transition* +sm_case_eof(State *dst){ + return(sm_case(TransitionCaseKind_EOF, 0, 0, dst, Transition_Consume, 0)); +} +internal Transition* +sm_case_eof_peek(State *dst){ + return(sm_case(TransitionCaseKind_EOF, 0, 0, dst, Transition_NoConsume, 0)); +} +internal Transition* +sm_case_eof_flagged(Flag *flag_check, b32 flag_check_value, State *dst){ + return(sm_case(TransitionCaseKind_EOF, flag_check, flag_check_value, dst, Transition_Consume, 0)); +} +internal Transition* +sm_case_eof_peek_flagged(Flag *flag_check, b32 flag_check_value, State *dst){ + return(sm_case(TransitionCaseKind_EOF, flag_check, flag_check_value, dst, Transition_NoConsume, 0)); +} +internal Transition* +sm_case_eof(Emit_Rule *emit){ + return(sm_case(TransitionCaseKind_EOF, 0, 0, helper_ctx.primary_ctx.model.root, Transition_Consume, emit)); +} +internal Transition* +sm_case_eof_peek(Emit_Rule *emit){ + return(sm_case(TransitionCaseKind_EOF, 0, 0, helper_ctx.primary_ctx.model.root, Transition_NoConsume, emit)); +} +internal Transition* +sm_case_eof_flagged(Flag *flag_check, b32 flag_check_value, Emit_Rule *emit){ + return(sm_case(TransitionCaseKind_EOF, flag_check, flag_check_value, helper_ctx.primary_ctx.model.root, Transition_Consume, emit)); +} +internal Transition* +sm_case_eof_peek_flagged(Flag *flag_check, b32 flag_check_value, Emit_Rule *emit){ + return(sm_case(TransitionCaseKind_EOF, flag_check, flag_check_value, helper_ctx.primary_ctx.model.root, Transition_NoConsume, emit)); +} + +internal Transition* +sm_fallback(State *dst){ + return(sm_case(TransitionCaseKind_Fallback, 0, 0, dst, Transition_Consume, 0)); +} +internal Transition* +sm_fallback_peek(State *dst){ + return(sm_case(TransitionCaseKind_Fallback, 0, 0, dst, Transition_NoConsume, 0)); +} +internal Transition* +sm_fallback_flagged(Flag *flag_check, b32 flag_check_value, State *dst){ + return(sm_case(TransitionCaseKind_Fallback, flag_check, flag_check_value, dst, Transition_Consume, 0)); +} +internal Transition* +sm_fallback_peek_flagged(Flag *flag_check, b32 flag_check_value, State *dst){ + return(sm_case(TransitionCaseKind_Fallback, flag_check, flag_check_value, dst, Transition_NoConsume, 0)); +} +internal Transition* +sm_fallback(Emit_Rule *emit){ + return(sm_case(TransitionCaseKind_Fallback, 0, 0, helper_ctx.primary_ctx.model.root, Transition_Consume, emit)); +} +internal Transition* +sm_fallback_peek(Emit_Rule *emit){ + return(sm_case(TransitionCaseKind_Fallback, 0, 0, helper_ctx.primary_ctx.model.root, Transition_NoConsume, emit)); +} +internal Transition* +sm_fallback_flagged(Flag *flag_check, b32 flag_check_value, Emit_Rule *emit){ + return(sm_case(TransitionCaseKind_Fallback, flag_check, flag_check_value, helper_ctx.primary_ctx.model.root, Transition_Consume, emit)); +} +internal Transition* +sm_fallback_peek_flagged(Flag *flag_check, b32 flag_check_value, Emit_Rule *emit){ + return(sm_case(TransitionCaseKind_Fallback, flag_check, flag_check_value, helper_ctx.primary_ctx.model.root, Transition_NoConsume, emit)); +} + +internal void +sm_match_delim(State *dst, State *fail_dst){ + sm_case(TransitionCaseKind_DelimMatch, 0, 0, dst, Transition_NoConsume, 0); + sm_case(TransitionCaseKind_DelimMatchFail, 0, 0, fail_dst, Transition_NoConsume, 0); +} + +internal void +sm_on_transition_set_flag(Flag *flag, b32 value){ + Transition *transition = helper_ctx.selected_transition; + smi_append_set_flag(helper_ctx.arena, &transition->activation_actions, flag, value); +} + +internal void +sm_emit_check_set_flag(String_Const_u8 emit_check, Flag *flag, b32 value){ + Emit_Rule *rule = helper_ctx.selected_emit_rule; + Emit_Check *new_check = push_array_zero(helper_ctx.arena, Emit_Check, 1); + sll_queue_push(rule->emit_checks.first, rule->emit_checks.last, new_check); + rule->emit_checks.count += 1; + new_check->emit_check = push_string_copy(helper_ctx.arena, emit_check); + new_check->flag = flag; + new_check->value = value; +} + +internal void +sm_emit_check_set_flag(char *emit_check, Flag *flag, b32 value){ + sm_emit_check_set_flag(SCu8(emit_check), flag, value); +} + +internal void +sm_set_flag(Flag *flag, b32 value){ + State *state = helper_ctx.selected_state; + smi_append_set_flag(helper_ctx.arena, &state->on_entry_actions, flag, value); +} + +internal void +sm_delim_mark_first(void){ + State *state = helper_ctx.selected_state; + smi_append_delim_mark_first(helper_ctx.arena, &state->on_entry_actions); +} + +internal void +sm_delim_mark_one_past_last(void){ + State *state = helper_ctx.selected_state; + smi_append_delim_mark_one_past_last(helper_ctx.arena, &state->on_entry_actions); +} + +//////////////////////////////// + +// NOTE(allen): OPERATORS FOR COMPOSING MODEL COMPONENTS AS EXPRESSIONS + +internal Operator_Set* +smo_copy_op_set(Operator_Set *set){ + Operator_Set *new_set = push_array_zero(helper_ctx.arena, Operator_Set, 1); + new_set->lexeme_to_ptr = make_table_Data_u64(helper_ctx.primary_ctx.allocator, set->count*2); + for (Operator *node = set->first; + node != 0; + node = node->next){ + Operator *new_node = push_array_zero(helper_ctx.arena, Operator, 1); + sll_queue_push(new_set->first, new_set->last, new_node); + new_set->count += 1; + new_node->name = node->name; + new_node->op = node->op; + table_insert(&new_set->lexeme_to_ptr, make_data(new_node->op.str, new_node->op.size), (u64)PtrAsInt(new_node)); + } + return(new_set); +} + +internal void +smo_remove_ops_with_prefix(Operator_Set *set, String_Const_u8 prefix){ + Operator *first = 0; + Operator *last = 0; + i32 count = 0; + + for (Operator *node = set->first, *next = 0; + node != 0; + node = next){ + next = node->next; + if (string_match(prefix, string_prefix(node->op, prefix.size))){ + table_erase(&set->lexeme_to_ptr, make_data(node->op.str, node->op.size)); + } + else{ + sll_queue_push(first, last, node); + count += 1; + } + } + + set->first = first; + set->last = last; + set->count = count; +} + +internal void +smo_remove_ops_with_prefix(Operator_Set *set, char *prefix){ + smo_remove_ops_with_prefix(set, SCu8(prefix)); +} + +internal void +smo_remove_ops_without_prefix(Operator_Set *set, String_Const_u8 prefix){ + Operator *first = 0; + Operator *last = 0; + i32 count = 0; + + for (Operator *node = set->first, *next = 0; + node != 0; + node = next){ + next = node->next; + if (!string_match(prefix, string_prefix(node->op, prefix.size))){ + table_erase(&set->lexeme_to_ptr, make_data(node->op.str, node->op.size)); + } + else{ + sll_queue_push(first, last, node); + count += 1; + } + } + + set->first = first; + set->last = last; + set->count = count; +} + +internal void +smo_remove_ops_without_prefix(Operator_Set *set, char *prefix){ + smo_remove_ops_without_prefix(set, SCu8(prefix)); +} + +internal void +smo_ops_string_skip(Operator_Set *set, umem size){ + Operator_Set new_set = {}; + new_set.lexeme_to_ptr = make_table_Data_u64(helper_ctx.primary_ctx.allocator, set->count*2); + + for (Operator *node = set->first, *next = 0; + node != 0; + node = next){ + next = node->next; + if (node->op.size > size){ + String_Const_u8 new_op = string_skip(node->op, size); + if (table_insert(&new_set.lexeme_to_ptr, make_data(new_op.str, new_op.size), (u64)PtrAsInt(node))){ + node->op = new_op; + sll_queue_push(new_set.first, new_set.last, node); + new_set.count += 1; + } + } + } + + table_free(&set->lexeme_to_ptr); + *set = new_set; +} + +internal Character_Set* +smo_new_char_set(void){ + Character_Set *set = push_array_zero(helper_ctx.arena, Character_Set, 1); + set->table = make_table_u64_u64(helper_ctx.primary_ctx.allocator, 100); + return(set); +} + +internal void +smo_char_set_union_ops_firsts(Character_Set *chars, Operator_Set *ops){ + for (Operator *node = ops->first; + node != 0; + node = node->next){ + String_Const_u8 lexeme = node->op; + u64 c = lexeme.str[0]; + table_insert(&chars->table, c, c); + } +} + +internal void +smo_char_set_remove(Character_Set *set, char *str){ + for (char *ptr = str; *ptr != 0; ptr += 1){ + table_erase(&set->table, (u64)(*ptr)); + } +} + +internal char* +smo_char_set_get_array(Character_Set *set){ + u32 count = set->table.used_count; + char *result = push_array_zero(helper_ctx.arena, char, count + 1); + u32 index = 0; + u32 slot_count = set->table.slot_count; + for (u32 i = 0; i < slot_count; i += 1){ + u64 c = set->table.keys[i]; + if (c != table_empty_key && c != table_erased_key){ + result[index] = (u8)(c); + index += 1; + } + } + result[count] = 0; + return(result); +} + +internal State* +smo_op_set_lexer_root(Operator_Set *set, State *machine_root, String_Const_u8 fallback_token_name){ + Base_Allocator *allocator = helper_ctx.primary_ctx.allocator; + Table_Data_u64 string_to_state = make_table_Data_u64(allocator, set->count*8); + + State *root = sm_add_state("op root"); + + for (Operator *node = set->first; + node != 0; + node = node->next){ + String_Const_u8 lexeme = node->op; + for (umem i = 1; i < lexeme.size; i += 1){ + String_Const_u8 prefix = string_prefix(lexeme, i); + Table_Lookup lookup = table_lookup(&string_to_state, make_data(prefix.str, prefix.size)); + if (!lookup.found_match){ + State *state = sm_add_state("op stage"); + State *parent = 0; + if (prefix.size == 1){ + parent = root; + } + else{ + Table_Lookup lookup = table_lookup(&string_to_state, make_data(prefix.str, prefix.size - 1)); + Assert(lookup.found_match); + u64 val = 0; + table_read(&string_to_state, lookup, &val); + parent = (State*)IntAsPtr(val); + } + u8 space[1]; + space[0] = prefix.str[prefix.size - 1]; + String_Const_u8 string = {space, 1}; + smi_case(&helper_ctx.primary_ctx, parent, TransitionCaseKind_CharaterArray, string, 0, 0, state, Transition_Consume, 0); + table_insert(&string_to_state, make_data(prefix.str, prefix.size), (u64)PtrAsInt(state)); + } + } + } + + for (Operator *node = set->first; + node != 0; + node = node->next){ + String_Const_u8 lexeme = node->op; + Table_Lookup lookup = table_lookup(&string_to_state, make_data(lexeme.str, lexeme.size)); + if (!lookup.found_match){ + State *parent = 0; + if (lexeme.size == 1){ + parent = root; + } + else{ + Table_Lookup lookup = table_lookup(&string_to_state, make_data(lexeme.str, lexeme.size - 1)); + Assert(lookup.found_match); + u64 val = 0; + table_read(&string_to_state, lookup, &val); + parent = (State*)IntAsPtr(val); + } + u8 space[1]; + space[0] = lexeme.str[lexeme.size - 1]; + String_Const_u8 string = {space, 1}; + Emit_Rule *emit = smi_emit_rule(helper_ctx.arena); + smi_emit_handler(helper_ctx.arena, emit, node->name, 0); + smi_case(&helper_ctx.primary_ctx, parent, TransitionCaseKind_CharaterArray, string, 0, 0, machine_root, Transition_Consume, emit); + } + } + + for (Operator *node = set->first; + node != 0; + node = node->next){ + String_Const_u8 lexeme = node->op; + Table_Lookup lookup = table_lookup(&string_to_state, make_data(lexeme.str, lexeme.size)); + if (lookup.found_match){ + u64 val = 0; + table_read(&string_to_state, lookup, &val); + State *state = (State*)IntAsPtr(val); + String_Const_u8 string = {}; + Emit_Rule *emit = smi_emit_rule(helper_ctx.arena); + smi_emit_handler(helper_ctx.arena, emit, node->name, 0); + smi_case(&helper_ctx.primary_ctx, state, TransitionCaseKind_Fallback, string, 0, 0, machine_root, Transition_NoConsume, emit); + } + } + + { + String_Const_u8 zero_string = {}; + Emit_Rule *emit = smi_emit_rule(helper_ctx.arena); + smi_emit_handler(helper_ctx.arena, emit, fallback_token_name, 0); + smi_case(&helper_ctx.primary_ctx, root, TransitionCaseKind_Fallback, zero_string, 0, 0, machine_root, Transition_NoConsume, emit); + } + for (Operator *node = set->first; + node != 0; + node = node->next){ + String_Const_u8 lexeme = node->op; + for (umem i = 1; i < lexeme.size; i += 1){ + String_Const_u8 prefix = string_prefix(lexeme, i); + Table_Lookup lookup = table_lookup(&string_to_state, make_data(prefix.str, prefix.size)); + Assert(lookup.found_match); + u64 val = 0; + table_read(&string_to_state, lookup, &val); + State *state = (State*)IntAsPtr(val); + String_Const_u8 string = {}; + Emit_Rule *emit = smi_emit_rule(helper_ctx.arena); + smi_emit_handler(helper_ctx.arena, emit, fallback_token_name, 0); + smi_case(&helper_ctx.primary_ctx, state, TransitionCaseKind_Fallback, string, 0, 0, machine_root, Transition_NoConsume, emit); + } + } + + table_free(&string_to_state); + + return(root); +} + +internal State* +smo_op_set_lexer_root(Operator_Set *set, State *machine_root, char *fallback_token_name){ + return(smo_op_set_lexer_root(set, machine_root, SCu8(fallback_token_name))); +} + +//////////////////////////////// + +// NOTE(allen): HELPERS + +// NOTE(allen): utf8 should be an u8 array with 129 slots. +// This will fill it out to represent all characters above the ASCII range. +internal void +smh_utf8_fill(u8 *utf8){ for (u16 i = 0; i < 128; i += 1){ - utf8[i] = i + 128; + utf8[i] = (u8)(i + 128); } utf8[128] = 0; +} + +internal void +smh_set_base_character_names(void){ + sm_char_name('{', "BraceOp"); + sm_char_name('}', "BraceCl"); + sm_char_name('(', "ParenOp"); + sm_char_name(')', "ParenCl"); + sm_char_name('[', "BrackOp"); + sm_char_name(']', "BrackCl"); + sm_char_name('-', "Minus"); + sm_char_name('+', "Plus"); + sm_char_name('.', "Dot"); + sm_char_name('!', "Bang"); + sm_char_name('*', "Star"); + sm_char_name(',', "Comma"); + sm_char_name(':', "Colon"); + sm_char_name(';', "Semicolon"); + sm_char_name('@', "At"); + sm_char_name('#', "Pound"); + sm_char_name('$', "Dollar"); + sm_char_name('%', "Percent"); + sm_char_name('^', "Carrot"); + sm_char_name('&', "Amp"); + sm_char_name('=', "Eq"); + sm_char_name('<', "Less"); + sm_char_name('>', "Grtr"); + sm_char_name('~', "Tilde"); + sm_char_name('/', "Slash"); + sm_char_name('?', "Question"); + sm_char_name('|', "Pipe"); +} + +internal void +smh_typical_tokens(void){ + sm_select_base_kind(TokenBaseKind_EOF); + sm_direct_token_kind("EOF"); - // Base Names - char_name('{', "BraceOp"); - char_name('}', "BraceCl"); - char_name('(', "ParenOp"); - char_name(')', "ParenCl"); - char_name('[', "BrackOp"); - char_name(']', "BrackCl"); - char_name('-', "Minus"); - char_name('+', "Plus"); - char_name('.', "Dot"); - char_name('!', "Bang"); - char_name('*', "Star"); - char_name(',', "Comma"); - char_name(':', "Colon"); - char_name(';', "Semicolon"); - char_name('@', "At"); - char_name('#', "Pound"); - char_name('$', "Dollar"); - char_name('%', "Percent"); - char_name('^', "Carrot"); - char_name('&', "And"); - char_name('=', "Eq"); - char_name('<', "Less"); - char_name('>', "Grtr"); - char_name('~', "Tilde"); - char_name('/', "Slash"); - char_name('?', "Question"); - char_name('|', "Pipe"); + sm_select_base_kind(TokenBaseKind_Whitespace); + sm_direct_token_kind("Whitespace"); - // CPP Names - char_name('!', "Not"); - char_name('|', "Or"); - char_name('%', "Mod"); - char_name('^', "Xor"); - char_name('?', "Ternary"); + sm_select_base_kind(TokenBaseKind_LexError); + sm_direct_token_kind("LexError"); +} + +//////////////////////////////// +//////////////////////////////// +//////////////////////////////// +//////////////////////////////// +//////////////////////////////// + +// NOTE(allen): OPTIMIZER + +internal String_Const_u8 +string_char_subtract(String_Const_u8 a, String_Const_u8 b){ + for (umem i = 0; i < b.size; i += 1){ + u8 c = b.str[i]; + for (umem j = 0; j < a.size;){ + if (a.str[j] == c){ + a.str[j] = a.str[a.size - 1]; + a.size -= 1; + } + else{ + j += 1; + } + } + } + return(a); +} + +internal Action_List +opt_copy_action_list(Arena *arena, Action_List actions){ + Action_List result = {}; + for (Action *node = actions.first; + node != 0; + node = node->next){ + Action *new_node = push_array_write(arena, Action, 1, node); + zdll_push_back(result.first, result.last, new_node); + result.count += 1; + } + return(result); +} + +internal Flag* +opt_flag_fixup(Flag *old_flag, Table_u64_u64 old_to_new){ + Flag *result = 0; + if (old_flag != 0){ + Table_Lookup lookup = table_lookup(&old_to_new, (u64)PtrAsInt(old_flag)); + Assert(lookup.found_match); + u64 val = 0; + table_read(&old_to_new, lookup, &val); + result = (Flag*)IntAsPtr(val); + } + return(result); +} + +internal Transition_Case +opt_copy_condition(Arena *arena, Transition_Case condition, Table_u64_u64 old_to_new){ + Transition_Case result = condition; + if (result.kind == TransitionCaseKind_ConditionSet){ + result.condition_set = smi_condition_set_copy(arena, condition.condition_set); + for (Condition_Node *node = result.condition_set.first; + node != 0; + node = node->next){ + Field_Set fields = node->fields; + for (Field_Pin_List *pin_list = fields.first; + pin_list != 0; + pin_list = pin_list->next){ + for (Field_Pin *pin = pin_list->first; + pin != 0; + pin = pin->next){ + pin->flag = opt_flag_fixup(pin->flag, old_to_new); + } + } + } + } + return(result); +} + +internal Emit_Rule* +opt_copy_emit_rule(Arena *arena, Emit_Rule *emit, Table_u64_u64 old_to_new){ + Emit_Rule *new_emit = push_array_write(arena, Emit_Rule, 1, emit); + block_zero_struct(&new_emit->emit_checks); + for (Emit_Check *emit_check = emit->emit_checks.first; + emit_check != 0; + emit_check = emit_check->next){ + Emit_Check *new_emit_check = push_array_write(arena, Emit_Check, 1, emit_check); + sll_queue_push(new_emit->emit_checks.first, new_emit->emit_checks.last, new_emit_check); + new_emit->emit_checks.count += 1; + new_emit_check->flag = opt_flag_fixup(new_emit_check->flag, old_to_new); + } + new_emit->first = 0; + new_emit->last = 0; + for (Emit_Handler *handler = emit->first; + handler != 0; + handler = handler->next){ + Emit_Handler *new_handler = push_array_write(arena, Emit_Handler, 1, handler); + sll_queue_push(new_emit->first, new_emit->last, new_handler); + new_handler->flag_check = opt_flag_fixup(handler->flag_check, old_to_new); + } + return(new_emit); +} + +internal Lexer_Model +opt_copy_model(Arena *arena, Lexer_Model model){ + Lexer_Model result = {}; - // Typical Base Token Kinds - select_base_kind(TokenBaseKind_EOF); - direct_token_kind("EOF"); + i32 pointer_count = model.states.count + model.flags.count; + Table_u64_u64 old_to_new = make_table_u64_u64(arena->base_allocator, pointer_count*2); + Table_u64_u64 new_to_old = make_table_u64_u64(arena->base_allocator, pointer_count*2); - select_base_kind(TokenBaseKind_Whitespace); - direct_token_kind("Whitespace"); - - select_base_kind(TokenBaseKind_LexError); - direct_token_kind("LexError"); - - // CPP Direct Toke Kinds - select_base_kind(TokenBaseKind_Comment); - direct_token_kind("BlockComment"); - direct_token_kind("LineComment"); - - select_base_kind(TokenBaseKind_Whitespace); - direct_token_kind("Backslash"); - - select_base_kind(TokenBaseKind_LiteralInteger); - direct_token_kind("LiteralInteger"): - direct_token_kind("LiteralIntegerU"): - direct_token_kind("LiteralIntegerL"): - direct_token_kind("LiteralIntegerUL"): - direct_token_kind("LiteralIntegerLL"): - direct_token_kind("LiteralIntegerULL"): - direct_token_kind("LiteralIntegerHex"): - direct_token_kind("LiteralIntegerHexU"): - direct_token_kind("LiteralIntegerHexL"): - direct_token_kind("LiteralIntegerHexUL"): - direct_token_kind("LiteralIntegerHexLL"): - direct_token_kind("LiteralIntegerHexULL"): - direct_token_kind("LiteralIntegerOct"): - direct_token_kind("LiteralIntegerOctU"): - direct_token_kind("LiteralIntegerOctL"): - direct_token_kind("LiteralIntegerOctUL"): - direct_token_kind("LiteralIntegerOctLL"): - direct_token_kind("LiteralIntegerOctULL"): - - select_base_kind(TokenBaseKind_LiteralFloat); - direct_token_kind("LiteralFloat32"): - direct_token_kind("LiteralFloat64"): - - select_base_kind(TokenBaseKind_LiteralString); - direct_token_kind("LiteralString"): - direct_token_kind("LiteralStringWide"): - direct_token_kind("LiteralStringUTF8"): - direct_token_kind("LiteralStringUTF16"): - direct_token_kind("LiteralStringUTF32"): - direct_token_kind("LiteralStringRaw"): - direct_token_kind("LiteralStringWideRaw"): - direct_token_kind("LiteralStringUTF8Raw"): - direct_token_kind("LiteralStringUTF16Raw"): - direct_token_kind("LiteralStringUTF32Raw"): - direct_token_kind("LiteralCharacter"): - direct_token_kind("LiteralCharacterWide"): - direct_token_kind("LiteralCharacterUTF8"): - direct_token_kind("LiteralCharacterUTF16"): - direct_token_kind("LiteralCharacterUTF32"): - direct_token_kind("PPIncludeFile"); - - // CPP Operators - Operator_Set *main_ops = begin_op_set(); - - select_base_kind(TokenBaseKind_ScopeOpen); - op("{"); - select_base_kind(TokenBaseKind_ScopeClose); - op("}"); - select_base_kind(TokenBaseKind_ParentheticalOpen); - op("("); - op("["); - select_base_kind(TokenBaseKind_ParentheticalClose); - op(")"); - op("]"); - select_base_kind(TokenBaseKind_Operator); - op(":"); - op("..."); - - op("::"); - op("++"); - op("--"); - op("."); - op("->", "Arrow"); - op("+"); - op("-"); - op("!"); - op("~"); - op("*"); - op("&"); - op(".*"); - op("->*", "ArrowStar"); - op("/"); - op("%"); - - char_name('<', "Left"); - char_name('>', "Right"); - op("<<"); - op(">>"); - - op("<=>", "Compare"); - - char_name('<', "Less"); - char_name('>', "Grtr"); - op("<"); - op("<="); - op(">"); - op(">="); - op("=="); - op("!="); - - op("^"); - op("|"); - op("&&"); - op("||"); - op("?"); - op("="); - op("+="); - op("-="); - op("*="); - op("/="); - op("%="); - - char_name('<', "Left"); - char_name('>', "Right"); - op("<<="); - op(">>="); - - op(","); - - // CPP Preprocess Operators - Operator_Set *pp_ops = begin_op_set(); - - op("#", "PPStringify"); - op("##", "PPConcat"); - - // CPP Keywords - Keyword_Set *main_keys = begin_key_set(); - - select_base_kind(TokenBaseKind_Keyword); - key("Void"); - key("Bool"); - key("Char"); - key("Int"); - key("Float"); - key("Double"); - key("Long"); - key("Short"); - key("Unsigned"); - key("Signed"); - key("Const"); - key("Volatile"); - key("Asm"); - key("Break"); - key("Case"); - key("Catch"); - key("Continue"); - key("Default"); - key("Do"); - key("Else"); - key("For"); - key("Goto"); - key("If"); - key("Return"); - key("Switch"); - key("Try"); - key("While"); - key("StaticAssert", "static_assert"); - key("ConstCast", "const_cast"); - key("DynamicCast", "dynamic_cast"); - key("ReinterpretCast", "reinterpret_cast"); - key("StaticCast", "static_cast"); - key("Class"); - key("Enum"); - key("Struct"); - key("Typedef"); - key("Union"); - key("Template"); - key("Typename"); - key("Friend"); - key("Namespace"); - key("Private"); - key("Protected"); - key("Public"); - key("Using"); - key("Extern"); - key("Export"); - key("Inline"); - key("Static"); - key("Virtual"); - key("AlignAs"); - key("Explicit"); - key("NoExcept"); - key("NullPtr"); - key("Operator"); - key("Register"); - key("This"); - key("ThreadLocal", "thread_local"); - key("SizeOf"); - key("AlignOf"); - key("DeclType"); - key("TypeID"); - key("New"); - key("Delete"); - key_unmatchable("KeywordGeneric"); - - select_base_kind(TokenBaseKind_LiteralInteger); - key("LiteralTrue", "true"); - key("LiteralFalse", "false"); - - select_base_kind(TokenBaseKind_Identifier); - key_fallback("Identifier"); - - // CPP Preprocess Directives - Keyword_Set *pp_directive_set = begin_key_set(); - - select_base_kind(TokenBaseKind_Preprocessor); - key("PPInclude", "#include"); - key("PPVersion", "#version"); - key("PPDefine", "#define"); - key("PPUndef", "#undef"); - key("PPIf", "#if"); - key("PPIfDef", "#ifdef"); - key("PPIfNDef", "#ifndef"); - key("PPElse", "#else"); - key("PPElIf", "#elif"); - key("PPEndIf", "#endif"); - key("PPError", "#error"); - key("PPImport", "#import"); - key("PPUsing", "#using"); - key("PPLine", "#line"); - key("PPPragma", "#pragma"); - key_fallback("PPUnknown"); - - // CPP Preprocess Keywords - Keyword_Set *pp_keys = begin_key_set(); - - select_base_kind(TokenBaseKind_Keyword); - key("PPDefined", "defined"); - - // State Machine - State *root = begin_state_machine(); - - Flag *is_hex = add_flag(AutoZero); - Flag *is_oct = add_flag(AutoZero); - Flag *is_pp_body = add_flag(KeepState); - Flag *is_include_body = add_flag(KeepState); - Flag *is_wide = add_flag(AutoZero); - Flag *is_utf8 = add_flag(AutoZero); - Flag *is_utf16 = add_flag(AutoZero); - Flag *is_utf32 = add_flag(AutoZero); - Flag *is_char = add_flag(AutoZero); - - flag_bind(is_pp_body, TokenBaseFlag_PreprocessorBody); - - State *identifier = add_state(); - State *whitespace = add_state(); - State *whitespace_end_pp = add_state(); - State *backslash = add_state(); - - State *operator_or_fnumber_dot = add_state(); - State *operator_or_comment_slash = add_state(); - - State *number = add_state(); - State *znumber = add_state(); - - State *fnumber_decimal = add_state(); - State *fnumber_exponent = add_state(); - State *fnumber_exponent_sign = add_state(); - State *fnumber_exponent_digits = add_state(); - - State *number_hex = add_state(); - State *number_oct = add_state(); - - State *U_number = add_state(); - State *L_number = add_state(); - State *UL_number = add_state(); - State *LU_number = add_state(); - State *l_number = add_state(); - State *Ul_number = add_state(); - State *lU_number = add_state(); - State *LL_number = add_state(); - State *ULL_number = add_state(); - - State *pp_directive = add_state(); - - State *include_pointy = add_state(); - State *include_quotes = add_state(); - - State *pre_L = add_state(); - State *pre_u = add_state(); - State *pre_U = add_state(); - State *pre_u8 = add_state(); - State *pre_R = add_state(); - - State *character = add_state(); - State *string = add_state(); - State *string_esc = add_state(); - State *string_esc_oc2 = add_state(); - State *string_esc_oc1 = add_state(); - State *string_esc_hex = add_state(); - State *string_esc_universal_8 = add_state(); - State *string_esc_universal_7 = add_state(); - State *string_esc_universal_6 = add_state(); - State *string_esc_universal_5 = add_state(); - State *string_esc_universal_4 = add_state(); - State *string_esc_universal_3 = add_state(); - State *string_esc_universal_2 = add_state(); - State *string_esc_universal_1 = add_state(); - - State *raw_string = add_state(); - State *raw_string_get_delim = add_state(); - State *raw_string_finish_delim = add_state(); - State *raw_string_find_close = add_state(); - State *raw_string_try_delim = add_state(); - State *raw_string_try_quote = add_state(); - - State *comment_block = add_state(); - State *comment_block_try_close = add_state(); - State *comment_block_newline = add_state(); - State *comment_line = add_state(); - - Operator_Set *main_ops_without_dot_or_slash = copy_op_set(main_ops); - remove_ops_with_prefix(main_ops_without_dot, "."); - remove_ops_with_prefix(main_ops_without_dot, "/"); - - Operator_Set *main_ops_with_dot = copy_op_set(main_ops); - remove_ops_without_prefix(main_ops_with_dot, "."); - ops_string_skip(main_ops_with_dot, 1); - - //// - - select_state(root); - sm_case("abcdefghijklmnopqrstvwxyz" - "ABCDEFGHIJKMNOPQSTVWXYZ" - "_$", - identifier); - sm_case(utf8, identifier); - sm_case("L", pre_L); - sm_case("u", pre_u); - sm_case("U", pre_U); - sm_case("R", pre_R); - - sm_case(" \r\t\f\v", whitespace); - sm_case("\n", whitespace_end_pp); - sm_case("\\", backslash); - - sm_case(".", operator_or_fnumber_dot); - sm_case("/", operator_or_comment_slash); - { - Character_Set *char_set = new_char_set(); - char_set_union_ops(char_set, main_ops_without_dot_or_slash); - char_set_remove(char_set, ".next){ + Flag *new_flag = push_array_zero(arena, Flag, 1); + sll_queue_push(result.flags.first, result.flags.last, new_flag); + result.flags.count += 1; + new_flag->reset_rule = flag->reset_rule; + new_flag->emit_flags = flag->emit_flags; + new_flag->emit_sub_flags = flag->emit_sub_flags; + table_insert(&old_to_new, (u64)PtrAsInt(flag), (u64)PtrAsInt(new_flag)); + table_insert(&new_to_old, (u64)PtrAsInt(new_flag), (u64)PtrAsInt(flag)); } - sm_case_flagged(is_include_body, true, "<", include_pointy); - sm_case_flagged(is_include_body, true, "\"", include_quotes); - - sm_case("123456789", number); - sm_case("0", znumber); - - sm_case_flagged(is_include_body, false, "\"", string); - sm_case("\'", character); - sm_case("#", pp_directive); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LexError"); - sm_fallback(emit); + for (State *state = model.states.first; + state != 0; + state = state->next){ + State *new_state = push_array_zero(arena, State, 1); + sll_queue_push(result.states.first, result.states.last, new_state); + result.states.count += 1; + table_insert(&old_to_new, (u64)PtrAsInt(state), (u64)PtrAsInt(new_state)); + table_insert(&new_to_old, (u64)PtrAsInt(new_state), (u64)PtrAsInt(state)); + new_state->pretty_name = push_string_copy(arena, state->pretty_name); } - //// - - select_state(identifier); - sm_case("abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "_$" - "0123456789", - identifier); - sm_case(utf8); - { - Emit_Rule *emit = add_emit_rule(); - handler_keys(emit, main_keys); - sm_fallback_peek(emit); + for (State *new_state = result.states.first; + new_state != 0; + new_state = new_state->next){ + Table_Lookup lookup = table_lookup(&new_to_old, (u64)PtrAsInt(new_state)); + Assert(lookup.found_match); + State *state = 0; + u64 val = 0; + table_read(&new_to_old, lookup, &val); + state = (State*)(IntAsPtr(val)); + + for (Transition *trans = state->transitions.first; + trans != 0; + trans = trans->next){ + Transition *new_trans = push_array_zero(arena, Transition, 1); + zdll_push_back(new_state->transitions.first, new_state->transitions.last, new_trans); + new_state->transitions.count += 1; + new_trans->parent_state = new_state; + new_trans->condition = opt_copy_condition(arena, trans->condition, old_to_new); + new_trans->activation_actions = opt_copy_action_list(arena, trans->activation_actions); + for (Action *action = new_trans->activation_actions.first; + action != 0; + action = action->next){ + switch (action->kind){ + case ActionKind_SetFlag: + { + action->set_flag.flag = opt_flag_fixup(action->set_flag.flag, old_to_new); + }break; + + case ActionKind_Emit: + { + action->emit_rule = opt_copy_emit_rule(arena, action->emit_rule, old_to_new); + }break; + } + } + + lookup = table_lookup(&old_to_new, (u64)PtrAsInt(trans->dst_state)); + Assert(lookup.found_match); + + State *new_dst_state = 0; + table_read(&old_to_new, lookup, &val); + new_dst_state = (State*)(IntAsPtr(val)); + + new_trans->dst_state = new_dst_state; + } } - //// + table_free(&old_to_new); + table_free(&new_to_old); - select_state(whitespace); - sm_case(" \t\r\f\v", whitespace); - sm_case("\n", whitespace_end_pp); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("Whitespace"); - sm_fallback_peek(emit); + for (State *state = model.states.first, *new_state = result.states.first; + state != 0 && new_state != 0; + state = state->next, new_state = new_state->next){ + if (model.root == state){ + result.root = new_state; + break; + } + } + Assert(result.root); + return(result); +} + +internal void +opt_simplify_transitions(Lexer_Primary_Context *ctx){ + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + Transition_List *transitions = &state->transitions; + + b32 is_delim_match = false; + if (transitions->first->condition.kind == TransitionCaseKind_DelimMatch){ + is_delim_match = true; + } + + if (!is_delim_match){ + Transition *first = 0; + Transition *last = 0; + i32 count = 0; + + for (Transition *trans = transitions->first, *next = 0; + trans != 0; + trans = next){ + next = trans->next; + Transition_Case condition = trans->condition; + Assert(condition.kind == TransitionCaseKind_ConditionSet); + Condition_Set condition_set = condition.condition_set; + for (Transition *prev_trans = first; + prev_trans != 0; + prev_trans = prev_trans->next){ + Transition_Case prev_condition = prev_trans->condition; + condition_set = smi_condition_set_subtract(&ctx->arena, + condition_set, + prev_condition.condition_set); + if (condition_set.count == 0){ + break; + } + } + if (condition_set.count != 0){ + trans->condition.condition_set = condition_set; + zdll_push_back(first, last, trans); + count += 1; + } + } + + transitions->first = first; + transitions->last = last; + transitions->count = count; + } + } +} + +internal void +opt_mark_all_states_excluded(Lexer_Primary_Context *ctx){ + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + state->optimized_in = false; + } +} + +internal void +opt_mark_all_states_included(Lexer_Primary_Context *ctx){ + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + state->optimized_in = true; + } +} + +internal void +opt_discard_all_excluded_states(Lexer_Primary_Context *ctx){ + State *first = 0; + State *last = 0; + i32 count = 0; + for (State *state = ctx->model.states.first, *next = 0; + state != 0; + state = next){ + next = state->next; + if (state->optimized_in){ + state->optimized_in = false; + sll_queue_push(first, last, state); + count += 1; + } + } + ctx->model.states.first = first; + ctx->model.states.last = last; + ctx->model.states.count = count; +} + +internal void +opt_include_reachable_states(State *state){ + if (!state->optimized_in){ + state->optimized_in = true; + for (Transition *trans = state->transitions.first; + trans != 0; + trans = trans->next){ + opt_include_reachable_states(trans->dst_state); + } + } +} + +internal void +opt_update_state_back_references(Lexer_Primary_Context *ctx){ + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + block_zero_struct(&state->back_references); } - //// - - select_state(whitespace_end_pp); - sm_set_flag(is_pp_body, false); - sm_set_flag(is_include_body, false); - sm_fallback_peek(whitespace); - - //// - - select_state(backslash); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("Backslash"); - sm_case("\n", emit); + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + for (Transition *trans = state->transitions.first; + trans != 0; + trans = trans->next){ + State *dst = trans->dst_state; + Transition_Ptr_Node *new_ptr_node = push_array_zero(&ctx->arena, Transition_Ptr_Node, 1); + new_ptr_node->ptr = trans; + sll_queue_push(dst->back_references.first, + dst->back_references.last, + new_ptr_node); + dst->back_references.count += 1; + } } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("Backslash"); - sm_fallback_peek(emit); +} + +internal void +opt_set_auto_zero_flags_on_root(Lexer_Primary_Context *ctx){ + State *root = ctx->model.root; + smi_append_zero_flags(&ctx->arena, &root->on_entry_actions); +} + +internal void +opt_transfer_state_actions_to_transitions(Lexer_Primary_Context *ctx){ + opt_update_state_back_references(ctx); + + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + Action_List actions = state->on_entry_actions; + if (actions.count > 0){ + for (Transition_Ptr_Node *node = state->back_references.first; + node != 0; + node = node->next){ + Transition *trans = node->ptr; + Action_List actions_copy = opt_copy_action_list(&ctx->arena, actions); + if (trans->activation_actions.first == 0){ + trans->activation_actions = actions_copy; + } + else{ + trans->activation_actions.last->next = actions_copy.first; + actions_copy.first->prev = trans->activation_actions.last; + trans->activation_actions.last = actions_copy.last; + trans->activation_actions.count += actions_copy.count; + } + } + block_zero_struct(&state->on_entry_actions); + } + } +} + +internal void +opt_flags_set_numbers(Lexer_Model model){ + i32 number = 0; + for (Flag *flag = model.flags.first; + flag != 0; + flag = flag->next){ + flag->number = number; + number += 1; + } +} + +internal void +opt_states_set_numbers(Lexer_Model model){ + i32 number = 1; + for (State *state = model.states.first; + state != 0; + state = state->next){ + state->number = number; + number += 1; + } +} + +internal void +opt_transition_pull_actions_backward(Lexer_Primary_Context *ctx, Transition *a, Transition *b){ + if (b->activation_actions.count > 0){ + Action_List b_actions = opt_copy_action_list(&ctx->arena, b->activation_actions); + if (a->activation_actions.first == 0){ + a->activation_actions = b_actions; + } + else{ + if (b_actions.first != 0){ + a->activation_actions.last->next = b_actions.first; + a->activation_actions.last = b_actions.last; + a->activation_actions.count += b_actions.count; + } + } + } + a->dst_state = b->dst_state; +} + +internal void +opt_transition_push_actions_forward(Lexer_Primary_Context *ctx, Transition *a, Transition *b){ + if (b->activation_actions.count > 0){ + Action_List a_actions = opt_copy_action_list(&ctx->arena, a->activation_actions); + if (b->activation_actions.first == 0){ + b->activation_actions = a_actions; + } + else{ + if (a_actions.first != 0){ + a_actions.last->next = b->activation_actions.first; + b->activation_actions.first = a_actions.first; + b->activation_actions.count += a_actions.count; + } + } + } +} + +internal b32 +opt_action_list_contains_consume(Action_List list){ + b32 result = false; + for (Action *act = list.first; + act != 0; + act = act->next){ + if (act->kind == ActionKind_Consume){ + result = true; + break; + } + } + return(result); +} + +internal void +opt_skip_past_thunk_states(Lexer_Primary_Context *ctx){ + opt_mark_all_states_included(ctx); + + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + // TODO(allen): A more complete thunk state test would check if all transitions + // have the same effect. If they do, then it is a thunk state. Only having + // one transition is just a special case of this more general rule. + if (state->transitions.count == 1){ + Transition *trans = state->transitions.first; + // TODO(allen): Consumes could be pulled forward into the transition actions + // for these types of "thunk states" as well, but only if we add a new concept + // for representing "action blocks" separately from actions contained in a + // transition handler, so that a handler can have multiple blocks. Then we would + // need to be able to identify thunk cycles, and add an entire extra concept to + // the state machine generated code, that it can sometimes get into a "stateless" + // thunk loop that can never be exited, but continues to consume one input at + // a time doing each action block. + b32 contains_consume = opt_action_list_contains_consume(trans->activation_actions); + if (!contains_consume){ + state->optimized_in = false; + } + } } - //// - - select_state(operator_or_comment_slash); - sm_case("*", comment_block); - sm_case("/", comment_line); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("SlashEq"); - sm_case("=", emit); + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + if (state->optimized_in){ + Transition_List *transitions = &state->transitions; + for (Transition *trans = transitions->first; + trans != 0; + trans = trans->next){ + for (;!trans->dst_state->optimized_in;){ + Transition *dst_trans = trans->dst_state->transitions.first; + opt_transition_pull_actions_backward(ctx, trans, dst_trans); + } + } + } } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("Slash"); - sm_fallback_peek(emit); +} + +internal b32 +opt_emit_rule_match(Emit_Rule *rule_a, Emit_Rule *rule_b){ + b32 result = true; + if (rule_a->emit_checks.count != rule_b->emit_checks.count){ + result = false; + goto end; + } + for (Emit_Check *check_a = rule_a->emit_checks.first, *check_b = rule_b->emit_checks.first; + check_a != 0 && check_b != 0; + check_a = check_a->next, check_b = check_b->next){ + if (check_a->flag != check_b->flag || + !string_match(check_a->emit_check, check_b->emit_check) || + check_a->value != check_b->value){ + result = false; + goto end; + } } - //// - - select_state(operator_or_fnumber_dot); - sm_case("012345679", fnumber_decimal); - { - Character_Set *char_set = new_char_set(); - char_set_union_ops(main_ops_with_dot); - char *char_set_array = char_set_get_array(char_set); - State *operator_state = op_set_lexer_root(main_ops_with_dot); - sm_case(char_set_array, operator_state); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("Dot"); - sm_fallback_peek(emit); + if (rule_a->count != rule_b->count){ + result = false; + goto end; } - //// - - select_state(number); - sm_case("012345679", number); - sm_case(".", fnumber_decimal); - sm_case("Ee", fnumber_exponent); - sm_case("Uu", U_number); - sm_case("L", L_number); - sm_case("l", l_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralInteger"); - sm_fallback_peek(emit); + for (Emit_Handler *handler_a = rule_a->first, *handler_b = rule_b->first; + handler_a != 0 && handler_b != 0; + handler_a = handler_a->next, handler_b = handler_b->next){ + if (handler_a->kind != handler_b->kind || + handler_a->flag_check != handler_b->flag_check){ + result = false; + goto end; + } + switch (handler_a->kind){ + case EmitHandlerKind_Direct: + { + if (!string_match(handler_a->token_name, handler_b->token_name)){ + result = false; + goto end; + } + }break; + case EmitHandlerKind_Keywords: + { + if (handler_a->keywords != handler_b->keywords){ + result = false; + goto end; + } + }break; + } } - //// + end:; + return(result); +} + +internal b32 +opt_action_lists_match(Action_List a, Action_List b){ + b32 result = false; + if (a.count == b.count){ + result = true; + for (Action *node_a = a.first, *node_b = b.first; + node_a != 0 && node_b != 0; + node_a = node_a->next, node_b = node_b->next){ + if (node_a->kind != node_b->kind){ + result = false; + goto double_break; + } + + switch (node_a->kind){ + case ActionKind_SetFlag: + { + if (node_a->set_flag.flag != node_b->set_flag.flag || + node_a->set_flag.value != node_b->set_flag.value){ + result = false; + goto double_break; + } + }break; + + case ActionKind_Emit: + { + if (!opt_emit_rule_match(node_a->emit_rule, node_b->emit_rule)){ + result = false; + goto double_break; + } + }break; + } + } + } + double_break:; + return(result); +} + +internal void +opt_merge_redundant_transitions_in_each_state(Lexer_Primary_Context *ctx){ + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + Transition_List *transitions = &state->transitions; + + Transition *first = 0; + Transition *last = 0; + i32 count = 0; + + for (Transition *trans = transitions->first, *next = 0; + trans != 0; + trans = next){ + next = trans->next; + + Transition *merge_trans = 0; + for (Transition *comp_trans = trans->next; + comp_trans != 0; + comp_trans = comp_trans->next){ + if (opt_action_lists_match(trans->activation_actions, comp_trans->activation_actions) && + trans->dst_state == comp_trans->dst_state){ + merge_trans = comp_trans; + break; + } + } + + if (merge_trans != 0){ + Assert(trans->condition.kind == TransitionCaseKind_ConditionSet); + Assert(merge_trans->condition.kind == TransitionCaseKind_ConditionSet); + merge_trans->condition.condition_set = + smi_condition_set_union(&ctx->arena, + trans->condition.condition_set, + merge_trans->condition.condition_set); + } + else{ + zdll_push_back(first, last, trans); + count += 1; + } + } + + transitions->first = first; + transitions->last = last; + transitions->count = count; + } +} + +internal b32 +opt_condition_set_is_subset(Arena *scratch, Condition_Set sub, Condition_Set super){ + Temp_Memory temp = begin_temp(scratch); + Condition_Set left_over = smi_condition_set_subtract(scratch, sub, super); + b32 result = (left_over.count == 0); + end_temp(temp); + return(result); +} + +internal void +opt_remove_peeks_without_creating_transition_splits(Lexer_Primary_Context *ctx){ + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + Transition_List *transitions = &state->transitions; + if (transitions->first->condition.kind != TransitionCaseKind_ConditionSet){ + continue; + } + + for (Transition *trans = transitions->first; + trans != 0; + trans = trans->next){ + i32 step_counter = 0; + for (;!opt_action_list_contains_consume(trans->activation_actions); + step_counter += 1){ + // NOTE(allen): Hitting this (most likely) indicates a peek cycle + // that wasn't caught by type checking. + Assert(step_counter < ctx->model.states.count); + + b32 found_action_extension = false; + State *dst_state = trans->dst_state; + Transition_List *dst_transitions = &dst_state->transitions; + if (dst_transitions->first->condition.kind != TransitionCaseKind_ConditionSet){ + break; + } + + for (Transition *dst_trans = dst_transitions->first; + dst_trans != 0; + dst_trans = dst_trans->next){ + if (opt_condition_set_is_subset(&ctx->arena, + trans->condition.condition_set, + dst_trans->condition.condition_set)){ + opt_transition_pull_actions_backward(ctx, trans, dst_trans); + found_action_extension = true; + break; + } + } + if (!found_action_extension){ + break; + } + } + } + } +} + +internal void +opt_remove_peeks_into_single_entry_point_states(Lexer_Primary_Context *ctx){ + opt_update_state_back_references(ctx); + opt_mark_all_states_included(ctx); - select_state(znumber); - sm_case(".", fnumber_decimal); - sm_case("Ee", fnumber_exponent); - sm_case("Uu", U_number); - sm_case("L", L_number); - sm_case("l", l_number); - sm_case("Xx", number_hex); - sm_case("01234567", number_oct); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralInteger"); - sm_fallback_peek(emit); + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + if (state->transitions.first->condition.kind != TransitionCaseKind_ConditionSet){ + continue; + } + + if (state->back_references.count == 1){ + Transition *src_trans = state->back_references.first->ptr; + if (src_trans->condition.kind != TransitionCaseKind_ConditionSet){ + continue; + } + + if (!opt_action_list_contains_consume(src_trans->activation_actions)){ + State *src_state = src_trans->parent_state; + + state->optimized_in = false; + + Transition *first = 0; + Transition *last = 0; + i32 count = 0; + + for (Transition *trans = state->transitions.first, *next = 0; + trans != 0; + trans = next){ + next = trans->next; + trans->condition.condition_set = + smi_condition_set_intersect(&ctx->arena, + trans->condition.condition_set, + src_trans->condition.condition_set); + if (trans->condition.condition_set.count > 0){ + trans->parent_state = src_state; + opt_transition_push_actions_forward(ctx, src_trans, trans); + zdll_push_back(first, last, trans); + count += 1; + } + } + + Assert(count != 0); + if (src_trans->prev != 0){ + src_trans->prev->next = first; + } + if (src_trans->next != 0){ + src_trans->next->prev = last; + } + first->prev = src_trans->prev; + last->next = src_trans->next; + src_state->transitions.count += count; + } + } + } +} + +internal b32 +opt_condition_is_eof_only(Transition_Case condition){ + b32 result = false; + if (condition.kind == TransitionCaseKind_ConditionSet){ + result = true; + for (Condition_Node *node = condition.condition_set.first; + node != 0; + node = node->next){ + Input_Set inputs = node->inputs; + if (inputs.count > 1 || inputs.inputs[0] != smi_eof){ + result = false; + break; + } + } + } + return(result); +} + +internal Keyword_Layout +opt_key_layout(Arena *arena, Keyword_Set keywords, i32 slot_count, u64 seed){ + Keyword_Layout layout = {}; + slot_count = clamp_bot(keywords.count + 1, slot_count); + layout.seed = seed; + layout.hashes = push_array_zero(arena, u64, slot_count); + layout.contributed_error = push_array_zero(arena, u64, slot_count); + layout.slots = push_array_zero(arena, Keyword*, slot_count); + layout.slot_count = slot_count; + for (Keyword *keyword = keywords.first; + keyword != 0; + keyword = keyword->next){ + u64 hash = lexeme_hash(seed, keyword->lexeme.str, keyword->lexeme.size); + i32 first_index = (hash%slot_count); + i32 index = first_index; + + Keyword *keyword_insert = keyword; + u64 contributed_error = 0; + + for (;;){ + if (layout.slots[index] == 0){ + layout.hashes[index] = hash; + layout.contributed_error[index] = contributed_error; + layout.slots[index] = keyword_insert; + break; + } + else{ + if (contributed_error > layout.contributed_error[index]){ + Swap(u64, hash, layout.hashes[index]); + Swap(Keyword*, keyword_insert, layout.slots[index]); + Swap(u64, contributed_error, layout.contributed_error[index]); + } + } + index += 1; + contributed_error += 1; + if (index >= slot_count){ + index = 0; + } + if (index == first_index){ + InvalidPath; + } + } + } + i32 max_run_length = 0; + i32 run_length = 0; + for (i32 i = 0; i < slot_count; i += 1){ + if (layout.slots[i] == 0){ + run_length = 0; + } + else{ + run_length += 1; + layout.error_score += run_length; + max_run_length = max(max_run_length, run_length); + } + } + i32 total_run_length = run_length; + for (i32 i = 0; i < slot_count; i += 1){ + if (layout.slots[i] == 0){ + break; + } + else{ + layout.error_score += run_length; + total_run_length += 1; + max_run_length = max(max_run_length, total_run_length); + } + } + layout.max_single_error_score = max_run_length; + layout.iterations_per_lookup = (f32)layout.error_score/(f32)layout.slot_count; + return(layout); +} + +internal u64 +random_u64_dirty(void){ + u64 a = pcg32_random(); + u64 b = pcg32_random(); + return((b << 32) | a); +} + +#if 0 +internal Keyword_Layout +opt_key_layout(Arena *arena, Keyword_Set keywords){ + i32 slot_count = keywords.count*2; + u64 seed = random_u64_dirty(); + return(opt_key_layout(arena, keywords, slot_count, seed)); +} +#endif + +internal Keyword_Layout +opt_key_layout(Arena *arena, Keyword_Set keywords){ + i32 init_slot_count = keywords.count + 1; + if (keywords.count == 1){ + init_slot_count = 1; } - //// +#if 0 + // heavy optimization effort + f32 acceptable_error_threshold = 2.f; + f32 accumulated_error_threshold = 8000.f; + i32 acceptable_max_single_error = 4; + i32 accumulated_max_single_error_threshold = Thousand(800); +#else + // light optimization effort + f32 acceptable_error_threshold = 1.1f; + f32 accumulated_error_threshold = 200.f; + i32 acceptable_max_single_error = 5; + i32 accumulated_max_single_error_threshold = Thousand(40); +#endif - select_state(fnumber_decimal); - sm_case("012345679", fnumber_decimal); - sm_case("Ee", fnumber_exponent); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat32"); - sm_case("Ff", emit); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat64"); - sm_case("Ll", emit); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat64"); - sm_fallback_peek(emit); + Keyword_Layout best_layout = {}; + best_layout.iterations_per_lookup = max_f32; + i32 slot_count = init_slot_count; + for (;; slot_count += 1){ + f32 accumulated_error = 0; + for (;;){ + u64 seed = random_u64_dirty(); + Temp_Memory restore_point = begin_temp(arena); + Keyword_Layout layout = opt_key_layout(arena, keywords, slot_count, seed); + accumulated_error += layout.iterations_per_lookup; + + if (layout.iterations_per_lookup < best_layout.iterations_per_lookup){ + best_layout = layout; + if (layout.iterations_per_lookup <= acceptable_error_threshold){ + goto optimize_max_single_error; + } + } + else{ + end_temp(restore_point); + } + if (accumulated_error >= accumulated_error_threshold){ + break; + } + } } - //// - - select_state(fnumber_exponent); - sm_case("+-", fnumber_exponent_sign); - sm_case("0123456789", fnumber_exponent_digits); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat32"); - sm_case("Ff", emit); + optimize_max_single_error: + if (best_layout.max_single_error_score <= acceptable_max_single_error){ + goto finished; } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat64"); - sm_case("Ll", emit); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat64"); - sm_fallback_peek(emit); + for (;; slot_count += 1){ + u64 accumulated_error = 0; + for (;;){ + u64 seed = random_u64_dirty(); + Temp_Memory restore_point = begin_temp(arena); + Keyword_Layout layout = opt_key_layout(arena, keywords, slot_count, seed); + + u64 adjusted_error_score = (layout.max_single_error_score + acceptable_max_single_error - 1)/acceptable_max_single_error; + adjusted_error_score *= adjusted_error_score; + adjusted_error_score *= acceptable_max_single_error; + + accumulated_error += adjusted_error_score; + + if (layout.max_single_error_score < best_layout.max_single_error_score && + layout.iterations_per_lookup <= best_layout.iterations_per_lookup){ + best_layout = layout; + if (layout.max_single_error_score <= acceptable_max_single_error){ + goto finished; + } + } + else{ + end_temp(restore_point); + } + if (accumulated_error >= accumulated_max_single_error_threshold){ + break; + } + } } - //// - select_state(fnumber_exponent_sign); - sm_case("0123456789", fnumber_exponent_digits); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat32"); - sm_case("Ff", emit); + finished:; + return(best_layout); +} + +//////////////////////////////// + +internal b32 +opt__input_set_contains(Input_Set set, u16 x){ + b32 result = false; + for (i32 i = 0; i < set.count; i += 1){ + if (set.inputs[i] == x){ + result = true; + break; + } } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat64"); - sm_case("Ll", emit); + return(result); +} + +internal b32 +opt__partial_transition_match(Arena *scratch, Partial_Transition *a, Partial_Transition *b){ + b32 result = false; + if (smi_field_set_match(scratch, a->fields, b->fields)){ + if (opt_action_lists_match(a->actions, b->actions)){ + if (a->dst_state == b->dst_state){ + result = true; + } + } } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat64"); - sm_fallback_peek(emit); + return(result); +} + +internal void +opt__push_partial_transition(Arena *arena, Partial_Transition_List *list, Field_Set fields, Transition *trans){ + Partial_Transition partial = {}; + partial.fields = fields; + partial.actions = trans->activation_actions; + partial.dst_state = trans->dst_state; + + b32 is_duplicate = false; + for (Partial_Transition *node = list->first; + node != 0; + node = node->next){ + if (opt__partial_transition_match(arena, node, &partial)){ + is_duplicate = true; + break; + } } - //// + if (!is_duplicate){ + Partial_Transition *result = push_array_write(arena, Partial_Transition, 1, &partial); + sll_queue_push(list->first, list->last, result); + list->count += 1; + } +} + +internal b32 +opt__partial_transition_list_match(Arena *scratch, Partial_Transition_List *a, Partial_Transition_List *b){ + b32 result = false; + if (a->count == b->count){ + result = true; + for (Partial_Transition *node_a = a->first; + node_a != 0; + node_a = node_a->next){ + b32 has_match = false; + for (Partial_Transition *node_b = b->first; + node_b != 0; + node_b = node_b->next){ + if (opt__partial_transition_match(scratch, node_a, node_b)){ + has_match = true; + break; + } + } + if (!has_match){ + result = false; + } + } + } + return(result); +} + +internal void +opt__insert_input_into_group(Grouped_Input_Handler *group, u8 x){ + if (!group->inputs_used[x]){ + group->inputs_used[x] = true; + group->inputs[group->input_count] = x; + group->input_count += 1; + } +} + +internal Grouped_Input_Handler_List +opt_grouped_input_handlers(Arena *arena, Transition *first_trans){ + Grouped_Input_Handler_List result = {}; - select_state(fnumber_exponent_digits); - sm_case("0123456789", fnumber_exponent_digits); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat32"); - sm_case("Ff", emit); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat64"); - sm_case("Ll", emit); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralFloat64"); - sm_fallback_peek(emit); + Assert(first_trans->condition.kind == TransitionCaseKind_ConditionSet); + + Grouped_Input_Handler *biggest_group = 0; + i32 size_of_biggest = 0; + + for (u16 i = 0; i <= 255; i += 1){ + Temp_Memory restore_point = begin_temp(arena); + Partial_Transition_List list = {}; + for (Transition *trans = first_trans; + trans != 0; + trans = trans->next){ + Assert(trans->condition.kind == TransitionCaseKind_ConditionSet); + Condition_Set condition_set = trans->condition.condition_set; + for (Condition_Node *node = condition_set.first; + node != 0; + node = node->next){ + if (opt__input_set_contains(node->inputs, i)){ + opt__push_partial_transition(arena, &list, node->fields, trans); + } + } + } + + Grouped_Input_Handler *matching_group = 0; + for (Grouped_Input_Handler *group = result.first; + group != 0; + group = group->next){ + if (opt__partial_transition_list_match(arena, &group->partial_transitions, &list)){ + matching_group = group; + break; + } + } + + if (matching_group != 0){ + end_temp(restore_point); + } + else{ + matching_group = push_array_zero(arena, Grouped_Input_Handler, 1); + sll_queue_push(result.first, result.last, matching_group); + result.count += 1; + matching_group->partial_transitions = list; + } + opt__insert_input_into_group(matching_group, (u8)i); + + if (matching_group->input_count > size_of_biggest){ + size_of_biggest = matching_group->input_count; + biggest_group = matching_group; + } } - //// + result.group_with_biggest_input_set = biggest_group; + return(result); +} + +//////////////////////////////// + +internal void +debug_print_states(Lexer_Primary_Context *ctx){ + printf("Number of States: %d\n", ctx->model.states.count); + i32 transition_count = 0; + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + Transition_List *transitions = &state->transitions; + transition_count += transitions->count; + } + printf("Number of Transitions: %d\n", transition_count); + for (State *state = ctx->model.states.first; + state != 0; + state = state->next){ + printf("State: %.*s\n", string_expand(state->pretty_name)); + } +} + +internal void +debug_print_transitions(Arena *scratch, Lexer_Model model){ + Temp_Memory temp = begin_temp(scratch); - select_state(number_hex); - sm_set_flag(is_hex, true); - sm_case("012345679abcdefABCDEF", number_hex); - sm_case("Uu", U_number); - sm_case("L", L_number); - sm_case("l", l_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralIntegerHex"); - sm_fallback_peek(emit); + i32 field_bit_width = model.flags.count; + char *field_memory = push_array(scratch, char, field_bit_width); + + printf("Number of States: %d\n", model.states.count); + i32 transition_count = 0; + for (State *state = model.states.first; + state != 0; + state = state->next){ + Transition_List *transitions = &state->transitions; + transition_count += transitions->count; + } + printf("Number of Transitions: %d\n", transition_count); + + for (State *state = model.states.first; + state != 0; + state = state->next){ + printf("State: %.*s\n", string_expand(state->pretty_name)); + + Transition_List *transitions = &state->transitions; + for (Transition *trans = transitions->first; + trans != 0; + trans = trans->next){ +#define transition_on "Transition on " + if (trans->condition.kind == TransitionCaseKind_DelimMatch){ + printf("\t" transition_on "\n"); + } + else{ + printf("\t" transition_on ""); + for (Condition_Node *node = trans->condition.condition_set.first; + node != 0; + node = node->next){ + printf("([%3d]", node->inputs.count); + if (node->inputs.count < 10){ + b32 all_printable = true; + char ascii[30]; + i32 j = 0; + for (i32 i = 0; i < node->inputs.count; i += 1){ + b32 is_ascii = character_is_basic_ascii(node->inputs.inputs[i]); + b32 is_eof = (node->inputs.inputs[i] == smi_eof); + if (!(is_ascii || is_eof)){ + all_printable = false; + break; + } + if (is_ascii){ + ascii[j] = (char)(node->inputs.inputs[i]); + j += 1; + } + else if (is_eof){ + ascii[j] = 'E'; + j += 1; + ascii[j] = 'O'; + j += 1; + ascii[j] = 'F'; + j += 1; + } + } + if (all_printable){ + printf(" = {%.*s}", j, ascii); + } + } + + printf(" x "); + + printf("("); + for (Field_Pin_List *pins = node->fields.first; + pins != 0; + pins = pins->next){ + block_fill_u8(field_memory, field_bit_width, '*'); + for (Field_Pin *pin = pins->first; + pin != 0; + pin = pin->next){ + i32 flag_number = pin->flag->number; + field_memory[flag_number] = pin->value?'1':'0'; + } + printf("%.*s", field_bit_width, field_memory); + if (pins->next != 0){ + printf(", "); + } + } + printf("))"); + if (node->next != 0){ + printf(" union\n\t%.*s", sizeof(transition_on) - 1, + " "); + } + } + printf(":\n"); + } + + for (Action *act = trans->activation_actions.first; + act != 0; + act = act->next){ + switch (act->kind){ + case ActionKind_SetFlag: + { + printf("\t\tSet Flag\n"); + }break; + + case ActionKind_ZeroFlags: + { + printf("\t\tZero Flags\n"); + }break; + + case ActionKind_DelimMarkFirst: + { + printf("\t\tDelim Mark First\n"); + }break; + + case ActionKind_DelimMarkOnePastLast: + { + printf("\t\tDelim Mark One Past Last\n"); + }break; + + case ActionKind_Consume: + { + printf("\t\tConsume\n"); + }break; + + case ActionKind_Emit: + { + printf("\t\tEmit\n"); + }break; + } + } + printf("\t\tGo to %.*s;\n", string_expand(trans->dst_state->pretty_name)); + } } - //// + end_temp(temp); +} + +internal void +debug_print_transitions(Lexer_Primary_Context *ctx){ + debug_print_transitions(&ctx->arena, ctx->model); +} + +internal void +debug_print_keyword_table_metrics(Keyword_Layout key_layout, i32 keyword_count){ + printf("used count: %d\n", keyword_count); + printf("slot count: %d\n", key_layout.slot_count); + printf("table load factor: %f\n", (f32)keyword_count/(f32)key_layout.slot_count); + printf("error score: %llu\n", key_layout.error_score); + printf("error per lookup: %f\n", key_layout.iterations_per_lookup); + printf("max single error score: %llu\n", key_layout.max_single_error_score); + for (i32 i = 0; i < key_layout.slot_count; i += 1){ + Keyword *keyword = key_layout.slots[i]; + if (keyword == 0){ + printf("[%d] -> \n", i); + } + else{ + printf("[%d] -> \"%.*s\"\n", i, string_expand(keyword->lexeme)); + } + } +} + +//////////////////////////////// + +internal char* +gen_token_full_name(Arena *arena, String_Const_u8 base_name){ + String_Const_u8 string = push_u8_stringf(arena, + "Token" LANG_NAME_CAMEL_STR "Kind_%.*s", + string_expand(base_name)); + return((char*)(string.str)); +} + +internal void +gen_tokens(Arena *scratch, Token_Kind_Set tokens, FILE *out){ + Temp_Memory temp = begin_temp(scratch); + i32 counter = 0; + fprintf(out, "typedef u16 Token_" LANG_NAME_CAMEL_STR "_Kind;\n"); + fprintf(out, "enum{\n"); + for (Token_Kind_Node *node = tokens.first; + node != 0; + node = node->next){ + char *full_name = gen_token_full_name(scratch, node->name); + fprintf(out, "%s = %d,\n", full_name, counter); + counter += 1; + } + char *full_name = gen_token_full_name(scratch, SCu8("COUNT")); + fprintf(out, "%s = %d,\n", full_name, counter); + fprintf(out, "};\n"); + fprintf(out, "char *token_" LANG_NAME_LOWER_STR "_kind_names[] = {\n"); + for (Token_Kind_Node *node = tokens.first; + node != 0; + node = node->next){ + fprintf(out, "\"%.*s\",\n", string_expand(node->name)); + } + fprintf(out, "};\n"); + end_temp(temp); +} + +internal void +gen_keyword_table(Arena *scratch, Token_Kind_Set tokens, Keyword_Set keywords, FILE *out){ + Temp_Memory temp = begin_temp(scratch); + Keyword_Layout key_layout = opt_key_layout(scratch, keywords); - select_state(number_oct); - sm_set_flag(is_oct, true); - sm_case("01234567", number_oct); - sm_case("Uu", U_number); - sm_case("L", L_number); - sm_case("l", l_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LiteralIntegerOct"); - sm_fallback_peek(emit); + fprintf(out, "u64 %.*s_hash_array[%d] = {\n", + string_expand(keywords.pretty_name), key_layout.slot_count); + for (i32 i = 0; i < key_layout.slot_count; i += 1){ + if (key_layout.slots[i] == 0){ + fprintf(out, "0x%016x,", 0); + } + else{ + fprintf(out, "0x%016llx,", (u64)((u64)(key_layout.hashes[i]) | 1)); + } + if (i % 4 == 3 || i + 1 == key_layout.slot_count){ + fprintf(out, "\n"); + } + } + fprintf(out, "};\n"); + + for (i32 i = 0; i < key_layout.slot_count; i += 1){ + if (key_layout.slots[i] != 0){ + fprintf(out, "u8 %.*s_key_array_%d[] = {", + string_expand(keywords.pretty_name), i); + String_Const_u8 lexeme = key_layout.slots[i]->lexeme; + for (umem i = 0; i < lexeme.size; i += 1){ + fprintf(out, "0x%02x,", lexeme.str[i]); + } + fprintf(out, "};\n"); + } } - //// + fprintf(out, "String_Const_u8 %.*s_key_array[%d] = {\n", + string_expand(keywords.pretty_name), key_layout.slot_count); + for (i32 i = 0; i < key_layout.slot_count; i += 1){ + if (key_layout.slots[i] == 0){ + fprintf(out, "{0, 0},\n"); + } + else{ + fprintf(out, "{%.*s_key_array_%d, %llu},\n", + string_expand(keywords.pretty_name), i, key_layout.slots[i]->lexeme.size); + } + } + fprintf(out, "};\n"); - select_state(U_number); - sm_case("L", UL_number); - sm_case("l", Ul_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexU"); - handler_token_kind(is_oct, "LiteralIntegerOctU"); - handler_token_kind("LiteralIntegerU"); - sm_fallback_peek(emit); + fprintf(out, "Lexeme_Table_Value %.*s_value_array[%d] = {\n", + string_expand(keywords.pretty_name), key_layout.slot_count); + for (i32 i = 0; i < key_layout.slot_count; i += 1){ + if (key_layout.slots[i] == 0){ + fprintf(out, "{0, 0},\n"); + } + else{ + Temp_Memory temp2 = begin_temp(scratch); + Keyword *keyword = key_layout.slots[i]; + String_Const_u8 name = keyword->name; + + char *full_token_name = gen_token_full_name(scratch, name); + Table_Lookup lookup = table_lookup(&tokens.name_to_ptr, make_data(name.str, name.size)); + Assert(lookup.found_match); + u64 val = 0; + table_read(&tokens.name_to_ptr, lookup, &val); + Token_Kind_Node *token_node = (Token_Kind_Node*)IntAsPtr(val); + + fprintf(out, "{%u, %s},\n", token_node->base_kind, full_token_name); + end_temp(temp2); + } + } + fprintf(out, "};\n"); + + fprintf(out, "i32 %.*s_slot_count = %d;\n", + string_expand(keywords.pretty_name), key_layout.slot_count); + fprintf(out, "u64 %.*s_seed = 0x%016llx;\n", + string_expand(keywords.pretty_name), key_layout.seed); + + end_temp(temp); +} + +internal void +gen_flag_check__cont_flow(Flag *flag, b32 value, FILE *out){ + if (value == 0){ + fprintf(out, "!"); + } + fprintf(out, "HasFlag(%.*s%d, 0x%x)", string_expand(flag->base_name), flag->index, flag->value); +} + +internal void +gen_SLOW_field_set_check__cont_flow(Field_Set fields, FILE *out){ + for (Field_Pin_List *pin_list = fields.first; + pin_list != 0; + pin_list = pin_list->next){ + fprintf(out, "("); + if (pin_list->count > 0){ + for (Field_Pin *pin = pin_list->first; + pin != 0; + pin = pin->next){ + gen_flag_check__cont_flow(pin->flag, pin->value, out); + if (pin->next != 0){ + fprintf(out, " && "); + } + } + } + else{ + fprintf(out, "true"); + } + fprintf(out, ")"); + + if (pin_list->next != 0){ + fprintf(out, " || "); + } + } +} + +internal void +gen_goto_state__cont_flow(State *state, Action_Context context, FILE *out){ + switch (context){ + case ActionContext_Normal: + { + fprintf(out, "goto state_label_%d; // %.*s\n", + state->number, string_expand(state->pretty_name)); + }break; + case ActionContext_EndOfFile: + { + fprintf(out, "goto end;\n"); + }break; + } +} + +internal void +gen_goto_dst_state__cont_flow(Transition *trans, Action_Context context, FILE *out){ + gen_goto_state__cont_flow(trans->dst_state, context, out); +} + +internal void +gen_action__set_flag(Flag *flag, b32 value, FILE *out){ + if (flag != 0){ + if (value == 0){ + fprintf(out, "%.*s%d &= ~(0x%x);\n", + string_expand(flag->base_name), flag->index, flag->value); + } + else{ + fprintf(out, "%.*s%d |= 0x%x;\n", + string_expand(flag->base_name), flag->index, flag->value); + } + } +} + +internal void +gen_emit__fill_token_flags(Flag_Set flags, Flag_Bucket_Set bucket_set, FILE *out){ + if (bucket_set.buckets[FlagBindProperty_Bound][FlagResetRule_AutoZero].count > 0){ + if (bucket_set.buckets[FlagBindProperty_Bound][FlagResetRule_KeepState].count > 0){ + fprintf(out, "token.flags = flag_ZB0 | flags_KB0;\n"); + } + else{ + fprintf(out, "token.flags = flags_ZB0;\n"); + } + } + else{ + if (bucket_set.buckets[FlagBindProperty_Bound][FlagResetRule_KeepState].count > 0){ + fprintf(out, "token.flags = flags_KB0;\n"); + } + } + for (Flag *flag = flags.first; + flag != 0; + flag = flag->next){ + if (flag->emit_sub_flags != 0){ + fprintf(out, "if ("); + gen_flag_check__cont_flow(flag, true, out); + fprintf(out, "){\n"); + fprintf(out, "token.sub_flags |= 0x%x;\n", flag->emit_sub_flags); + fprintf(out, "}\n"); + } + } +} + +internal void +gen_emit__fill_token_base_kind(Token_Kind_Set tokens, String_Const_u8 name, FILE *out){ + Table_Lookup lookup = table_lookup(&tokens.name_to_ptr, make_data(name.str, name.size)); + Assert(lookup.found_match); + u64 val = 0; + table_read(&tokens.name_to_ptr, lookup, &val); + Token_Kind_Node *node = (Token_Kind_Node*)IntAsPtr(val); + Token_Base_Kind base_kind = node->base_kind; + // TODO(allen): pretty names for token base kinds? + fprintf(out, "token.kind = %u;\n", base_kind); +} + +internal void +gen_emit__direct(Arena *scratch, Token_Kind_Set tokens, String_Const_u8 base_name, FILE *out){ + Temp_Memory temp = begin_temp(scratch); + char *token_full_name = gen_token_full_name(scratch, base_name); + fprintf(out, "token.sub_kind = %s;\n", token_full_name); + gen_emit__fill_token_base_kind(tokens, base_name, out); + end_temp(temp); +} + +internal Action_Context +gen_SLOW_action_list__cont_flow(Arena *scratch, Token_Kind_Set tokens, Flag_Set flags, + Flag_Bucket_Set bucket_set, Action_List action_list, + Action_Context context, FILE *out){ + Action_Context result_context = ActionContext_Normal; + for (Action *action = action_list.first; + action != 0; + action = action->next){ + switch (action->kind){ + case ActionKind_SetFlag: + { + gen_action__set_flag(action->set_flag.flag, action->set_flag.value, out); + }break; + + case ActionKind_ZeroFlags: + { + for (i32 i = 0; i < FlagBindProperty_COUNT; i += 1){ + Flag_Bucket *bucket = &bucket_set.buckets[i][FlagResetRule_AutoZero]; + for (i32 j = 0; j < bucket->number_of_variables; j += 1){ + fprintf(out, "%.*s%d = 0;\n", string_expand(bucket->pretty_name), j); + } + } + }break; + + case ActionKind_DelimMarkFirst: + { + fprintf(out, "delim_first = ptr;\n"); + }break; + + case ActionKind_DelimMarkOnePastLast: + { + fprintf(out, "delim_one_past_last = ptr;\n"); + }break; + + case ActionKind_Consume: + { + if (context != ActionContext_EndOfFile){ + fprintf(out, "ptr += 1;\n"); + } + else{ + result_context = ActionContext_EndOfFile; + } + }break; + + case ActionKind_Emit: + { + Emit_Rule *emit = action->emit_rule; + + fprintf(out, "{\n"); + fprintf(out, "Token token = {};\n"); + + fprintf(out, "token.pos = (i64)(emit_ptr - input.str);\n"); + fprintf(out, "token.size = (i64)(ptr - emit_ptr);\n"); + + gen_emit__fill_token_flags(flags, bucket_set, out); + + fprintf(out, "do{\n"); + b32 keep_looping = true; + for (Emit_Handler *handler = emit->first; + handler != 0 && keep_looping; + handler = handler->next){ + if (handler->flag_check != 0){ + fprintf(out, "if ("); + gen_flag_check__cont_flow(handler->flag_check, true, out); + fprintf(out, "){\n"); + } + + switch (handler->kind){ + case EmitHandlerKind_Direct: + { + gen_emit__direct(scratch, tokens, handler->token_name, out); + if (handler->flag_check != 0){ + fprintf(out, "break;\n"); + } + keep_looping = false; + }break; + + case EmitHandlerKind_Keywords: + { + Keyword_Set *keywords = handler->keywords; + fprintf(out, "Lexeme_Table_Lookup lookup = " + "lexeme_table_lookup(%.*s_hash_array, %.*s_key_array, " + "%.*s_value_array, %.*s_slot_count, %.*s_seed, " + "emit_ptr, token.size);\n", + string_expand(keywords->pretty_name), + string_expand(keywords->pretty_name), + string_expand(keywords->pretty_name), + string_expand(keywords->pretty_name), + string_expand(keywords->pretty_name)); + fprintf(out, "if (lookup.found_match){\n"); + fprintf(out, "token.kind = lookup.base_kind;\n"); + fprintf(out, "token.sub_kind = lookup.sub_kind;\n"); + fprintf(out, "break;\n"); + fprintf(out, "}\n"); + if (handler->keywords->has_fallback_token_kind){ + gen_emit__direct(scratch, tokens, + keywords->fallback_name, out); + keep_looping = false; + } + }break; + } + + if (handler->flag_check != 0){ + fprintf(out, "}\n"); + keep_looping = true; + } + } + fprintf(out, "}while(0);\n"); + + if (emit->emit_checks.count > 0){ + fprintf(out, "switch (token.sub_kind){\n"); + for (Emit_Check *emit_check = emit->emit_checks.first; + emit_check != 0; + emit_check = emit_check->next){ + Temp_Memory temp = begin_temp(scratch); + char *emit_check_full_name = gen_token_full_name(scratch, emit_check->emit_check); + fprintf(out, "case %s:\n", emit_check_full_name); + fprintf(out, "{\n"); + gen_action__set_flag(emit_check->flag, emit_check->value, out); + fprintf(out, "}break;\n"); + end_temp(temp); + } + fprintf(out, "}\n"); + } + + fprintf(out, "token_list_push(arena, &list, &token);\n"); + fprintf(out, "emit_ptr = ptr;\n"); + fprintf(out, "}\n"); + }break; + } + } + return(result_context); +} + +internal void +gen_flag_declarations__cont_flow(Flag_Bucket *bucket, FILE *out){ + i32 max_bits = bucket->max_bits; + i32 number_of_flag_variables = (bucket->count + max_bits - 1)/max_bits; + String_Const_u8 pretty_name = bucket->pretty_name; + for (i32 i = 0; i < number_of_flag_variables; i += 1){ + fprintf(out, "u%d %.*s%d = 0;\n", max_bits, string_expand(pretty_name), i); + } + bucket->number_of_variables = number_of_flag_variables; +} + +internal void +gen_bound_flag_fill_lookup__cont_flow(Flag_Bucket *bucket){ + i32 counter = 0; + for (Flag_Ptr_Node *node = bucket->first; + node != 0; + node = node->next, counter += 1){ + Flag *flag = node->flag; + flag->base_name = bucket->pretty_name; + flag->number = counter; + flag->index = 0; + flag->value = flag->emit_flags; + } +} + +internal void +gen_flag_fill_lookup__cont_flow(Flag_Bucket *bucket){ + i32 max_bits = bucket->max_bits; + i32 counter = 0; + for (Flag_Ptr_Node *node = bucket->first; + node != 0; + node = node->next, counter += 1){ + Flag *flag = node->flag; + flag->base_name = bucket->pretty_name; + flag->number = counter; + flag->index = counter/max_bits; + flag->value = (1 << (counter % max_bits)); + } +} + +internal void +gen_contiguous_control_flow_lexer(Arena *scratch, Token_Kind_Set tokens, Lexer_Model model, FILE *out){ + Temp_Memory temp = begin_temp(scratch); + + model = opt_copy_model(scratch, model); + + opt_flags_set_numbers(model); + opt_states_set_numbers(model); + + Input_Set cut_inputs = smi_input_set_construct_eof(scratch); + Field_Set cut_fields = smi_field_set_construct(scratch); + Condition_Set cut_set = smi_condition(scratch, cut_inputs, cut_fields); + + // Split EOFs and insert at beginning + for (State *state = model.states.first; + state != 0; + state = state->next){ + Transition_List *transitions = &state->transitions; + if (transitions->first->condition.kind == TransitionCaseKind_ConditionSet){ + Transition *first = 0; + Transition *last = 0; + i32 count = 0; + + for (Transition *trans = transitions->first, *next = 0; + trans != 0; + trans = next){ + next = trans->next; + + Assert(trans->condition.kind == TransitionCaseKind_ConditionSet); + Condition_Set original = trans->condition.condition_set; + Condition_Set condition_int = smi_condition_set_intersect(scratch, original, cut_set); + if (condition_int.count == 0){ + zdll_push_back(first, last, trans); + count += 1; + } + else{ + trans->condition.condition_set = condition_int; + zdll_push_front(first, last, trans); + count += 1; + + Condition_Set condition_sub = smi_condition_set_subtract(scratch, original, cut_set); + if (condition_sub.count > 0){ + Transition *new_trans = push_array(scratch, Transition, 1); + zdll_push_back(first, last, new_trans); + count += 1; + new_trans->parent_state = state; + new_trans->condition.kind = TransitionCaseKind_ConditionSet; + new_trans->condition.condition_set = condition_sub; + new_trans->activation_actions = opt_copy_action_list(scratch, trans->activation_actions); + new_trans->dst_state = trans->dst_state; + } + } + } + + state->transitions.first = first; + state->transitions.last = last; + state->transitions.count = count; + } } - //// + Flag_Bucket_Set bucket_set = {}; + bucket_set.buckets[FlagBindProperty_Free][FlagResetRule_AutoZero].pretty_name = string_u8_litexpr("flags_ZF"); + bucket_set.buckets[FlagBindProperty_Free][FlagResetRule_AutoZero].max_bits = 32; + bucket_set.buckets[FlagBindProperty_Free][FlagResetRule_KeepState].pretty_name = string_u8_litexpr("flags_KF"); + bucket_set.buckets[FlagBindProperty_Free][FlagResetRule_KeepState].max_bits = 32; + bucket_set.buckets[FlagBindProperty_Bound][FlagResetRule_AutoZero].pretty_name = string_u8_litexpr("flags_ZB"); + bucket_set.buckets[FlagBindProperty_Bound][FlagResetRule_AutoZero].max_bits = 16; + bucket_set.buckets[FlagBindProperty_Bound][FlagResetRule_KeepState].pretty_name = string_u8_litexpr("flags_KB"); + bucket_set.buckets[FlagBindProperty_Bound][FlagResetRule_KeepState].max_bits = 16; - select_state(L_number); - sm_case("L", LL_number); - sm_case("Uu", LU_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexL"); - handler_token_kind(is_oct, "LiteralIntegerOctL"); - handler_token_kind("LiteralIntegerL"); - sm_fallback_peek(emit); + for (Flag *flag = model.flags.first; + flag != 0; + flag = flag->next){ + Flag_Reset_Rule reset_rule = flag->reset_rule; + Flag_Bind_Property bind_property = + (flag->emit_flags != 0)?FlagBindProperty_Bound:FlagBindProperty_Free; + + Flag_Bucket *bucket = &bucket_set.buckets[bind_property][reset_rule]; + Flag_Ptr_Node *node = push_array(scratch, Flag_Ptr_Node, 1); + sll_queue_push(bucket->first, bucket->last, node); + bucket->count += 1; + node->flag = flag; } - //// - - select_state(l_number); - sm_case("l", LL_number); - sm_case("Uu", lU_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexL"); - handler_token_kind(is_oct, "LiteralIntegerOctL"); - handler_token_kind("LiteralIntegerL"); - sm_fallback_peek(emit); + for (i32 i = 0; i < FlagBindProperty_COUNT; i += 1){ + for (i32 j = 0; j < FlagResetRule_COUNT; j += 1){ + if (i == FlagBindProperty_Bound){ + gen_bound_flag_fill_lookup__cont_flow(&bucket_set.buckets[i][j]); + } + else{ + gen_flag_fill_lookup__cont_flow(&bucket_set.buckets[i][j]); + } + } } - //// + fprintf(out, "internal Token_List\n"); + fprintf(out, "lex_full_input_" LANG_NAME_LOWER_STR "(Arena *arena, String_Const_u8 input){\n"); + fprintf(out, "Token_List list = {};\n"); - select_state(LL_number); - sm_case("Uu", ULL_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexLL"); - handler_token_kind(is_oct, "LiteralIntegerOctLL"); - handler_token_kind("LiteralIntegerLL"); - sm_fallback_peek(emit); + for (i32 i = 0; i < FlagBindProperty_COUNT; i += 1){ + for (i32 j = 0; j < FlagResetRule_COUNT; j += 1){ + gen_flag_declarations__cont_flow(&bucket_set.buckets[i][j], out); + } } - //// + fprintf(out, "u8 *delim_first = input.str;\n"); + fprintf(out, "u8 *delim_one_past_last = input.str;\n"); - select_state(UL_number); - sm_case("L", ULL_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexUL"); - handler_token_kind(is_oct, "LiteralIntegerOctUL"); - handler_token_kind("LiteralIntegerUL"); - sm_fallback_peek(emit); + fprintf(out, "u8 *emit_ptr = input.str;\n"); + + fprintf(out, "u8 *ptr = input.str;\n"); + fprintf(out, "u8 *opl_ptr = ptr + input.size;\n"); + + for (State *state = model.states.first; + state != 0; + state = state->next){ + fprintf(out, "{\n"); + fprintf(out, "state_label_%d: // %.*s\n", + state->number, string_expand(state->pretty_name)); + + Transition_List *transitions = &state->transitions; + Transition *trans = transitions->first; + + Transition_Case_Kind state_trans_kind = trans->condition.kind; + + switch (state_trans_kind){ + default: + { + InvalidPath; + }break; + + case TransitionCaseKind_DelimMatch: + { + Transition *success_trans = trans; + Transition *failure_trans = trans->next; + Assert(failure_trans->condition.kind == TransitionCaseKind_DelimMatchFail); + + fprintf(out, "umem delim_length = delim_one_past_last - delim_first;\n"); + fprintf(out, "umem parse_length = 0;\n"); + fprintf(out, "for (;;){\n"); + { + fprintf(out, "if (parse_length == delim_length){\n"); + { + gen_SLOW_action_list__cont_flow(scratch, tokens, model.flags, bucket_set, + success_trans->activation_actions, + ActionContext_Normal, out); + gen_goto_dst_state__cont_flow(success_trans, ActionContext_Normal, out); + } + fprintf(out, "}\n"); + fprintf(out, "if (ptr == opl_ptr){\n"); + { + gen_SLOW_action_list__cont_flow(scratch, tokens, model.flags, bucket_set, + failure_trans->activation_actions, + ActionContext_Normal, out); + gen_goto_dst_state__cont_flow(success_trans, ActionContext_Normal, out); + } + fprintf(out, "}\n"); + + fprintf(out, "if (*ptr == delim_first[parse_length]){\n"); + fprintf(out, "ptr += 1;\n"); + fprintf(out, "parse_length += 1;\n"); + fprintf(out, "}\n"); + fprintf(out, "else{\n"); + { + gen_SLOW_action_list__cont_flow(scratch, tokens, model.flags, bucket_set, + failure_trans->activation_actions, + ActionContext_Normal, out); + gen_goto_dst_state__cont_flow(failure_trans, ActionContext_Normal, out); + } + fprintf(out, "}\n"); + } + fprintf(out, "}\n"); + }break; + + case TransitionCaseKind_ConditionSet: + { + { + fprintf(out, "if (ptr == opl_ptr){\n"); + for (; + trans != 0; + trans = trans->next){ + if (opt_condition_is_eof_only(trans->condition)){ + Assert(trans->condition.condition_set.count == 1); + Condition_Node *node = trans->condition.condition_set.first; + fprintf(out, "if ("); + gen_SLOW_field_set_check__cont_flow(node->fields, out); + fprintf(out, "){\n"); + Action_Context action_ctx = ActionContext_EndOfFile; + action_ctx = gen_SLOW_action_list__cont_flow(scratch, tokens, model.flags, + bucket_set, + trans->activation_actions, + action_ctx, out); + gen_goto_dst_state__cont_flow(trans, action_ctx, out); + fprintf(out, "}\n"); + } + else{ + break; + } + } + fprintf(out, "}\n"); + } + + Grouped_Input_Handler_List group_list = opt_grouped_input_handlers(scratch, trans); + + fprintf(out, "switch (*ptr){\n"); + for (Grouped_Input_Handler *group = group_list.first; + group != 0; + group = group->next){ + + if (group == group_list.group_with_biggest_input_set){ + fprintf(out, "default:\n"); + } + else{ + i32 input_count = group->input_count; + u8 *inputs = group->inputs; + for (i32 i = 0; i < input_count; i += 1){ + fprintf(out, "case 0x%02x:", inputs[i]); + if ((i % 7) == 6 || i + 1 == input_count){ + fprintf(out, "\n"); + } + } + } + + fprintf(out, "{\n"); + for (Partial_Transition *partial = group->partial_transitions.first; + partial != 0; + partial = partial->next){ + if (partial->next != 0){ + fprintf(out, "if ("); + gen_SLOW_field_set_check__cont_flow(partial->fields, out); + fprintf(out, "){\n"); + } + + { + gen_SLOW_action_list__cont_flow(scratch, tokens, model.flags, bucket_set, + partial->actions, ActionContext_Normal, + out); + gen_goto_state__cont_flow(partial->dst_state, ActionContext_Normal, out); + } + + if (partial->next != 0){ + fprintf(out, "}\n"); + } + } + fprintf(out, "}break;\n"); + } + fprintf(out, "}\n"); + }break; + } + + fprintf(out, "}\n"); } - //// + fprintf(out, "end:;\n"); + fprintf(out, "return(list);\n"); + fprintf(out, "}\n"); - select_state(Ul_number); - sm_case("l", ULL_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexUL"); - handler_token_kind(is_oct, "LiteralIntegerOctUL"); - handler_token_kind("LiteralIntegerUL"); - sm_fallback_peek(emit); + end_temp(temp); +} + +//////////////////////////////// + +#include +#include + +internal void +build_language_model(void); + +internal String_Const_u8 +file_read_all(Arena *arena, FILE *file){ + String_Const_u8 result = {}; + fseek(file, 0, SEEK_END); + result.size = ftell(file); + fseek(file, 0, SEEK_SET); + result.str = push_array(arena, u8, result.size + 1); + fread(result.str, result.size, 1, file); + result.str[result.size] = 0; + return(result); +} + +int main(void){ + pcg32_srandom(time(0), time(0)); + + Base_Allocator *allocator = get_allocator_malloc(); + sm_helper_init(allocator); + + build_language_model(); + + Lexer_Primary_Context *ctx = &helper_ctx.primary_ctx; + + // NOTE(allen): Type checking + // DelimMatch only with a single catch-all fallback, no peeks. + // Remove the declaration of states and flags? + // Flag bindings are one to one + + //////////////////////////////// + + // NOTE(allen): High level reorganization of state machine + + opt_set_auto_zero_flags_on_root(ctx); + opt_transfer_state_actions_to_transitions(ctx); + + //////////////////////////////// + + // NOTE(allen): High level optimization + + opt_simplify_transitions(ctx); + + opt_mark_all_states_excluded(ctx); + opt_include_reachable_states(ctx->model.root); + opt_discard_all_excluded_states(ctx); + + opt_merge_redundant_transitions_in_each_state(ctx); + + opt_skip_past_thunk_states(ctx); + + opt_mark_all_states_excluded(ctx); + opt_include_reachable_states(ctx->model.root); + opt_discard_all_excluded_states(ctx); + + opt_remove_peeks_without_creating_transition_splits(ctx); + + opt_mark_all_states_excluded(ctx); + opt_include_reachable_states(ctx->model.root); + opt_discard_all_excluded_states(ctx); + + opt_remove_peeks_into_single_entry_point_states(ctx); + + opt_discard_all_excluded_states(ctx); + + opt_states_set_numbers(ctx->model); + + //////////////////////////////// + + // NOTE(allen): Debug inspection of model + +#if 0 + opt_flags_set_numbers(ctx->model); + debug_print_transitions(ctx); +#endif + + //////////////////////////////// + + // NOTE(allen): Arrange input files and output files + + String_Const_u8 path_to_self = string_u8_litexpr(__FILE__); + path_to_self = string_remove_last_folder(path_to_self); + + String_Const_u8 hand_written_h_name = push_u8_stringf(&ctx->arena, + "%.*s4coder_lex_gen_hand_written.h", + string_expand(path_to_self)); + String_Const_u8 hand_written_name = push_u8_stringf(&ctx->arena, + "%.*s4coder_lex_gen_hand_written.cpp", + string_expand(path_to_self)); + + + FILE *hand_written_h_file = fopen((char*)hand_written_h_name.str, "rb"); + if (hand_written_h_file == 0){ + printf("error: could not open 4coder_lex_gen_hand_written.h\n"); + exit(1); } - //// + String_Const_u8 hand_written_h = file_read_all(&ctx->arena, hand_written_h_file); + fclose(hand_written_h_file); - select_state(LU_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexUL"); - handler_token_kind(is_oct, "LiteralIntegerOctUL"); - handler_token_kind("LiteralIntegerUL"); - sm_fallback_peek(emit); + FILE *hand_written_file = fopen((char*)hand_written_name.str , "rb"); + if (hand_written_file == 0){ + printf("error: could not open 4coder_lex_gen_hand_written.cpp\n"); + exit(1); } - //// + String_Const_u8 hand_written = file_read_all(&ctx->arena, hand_written_file); + fclose(hand_written_file); - select_state(lU_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexUL"); - handler_token_kind(is_oct, "LiteralIntegerOctUL"); - handler_token_kind("LiteralIntegerUL"); - sm_fallback_peek(emit); + String_Const_u8 path_to_src = string_remove_last_folder(path_to_self); + + String_Const_u8 out_h_name = push_u8_stringf(&ctx->arena, "%.*slanguages/generated_lexer_" LANG_NAME_LOWER_STR ".h", + string_expand(path_to_src)); + String_Const_u8 out_cpp_name = push_u8_stringf(&ctx->arena, "%.*slanguages/generated_lexer_" LANG_NAME_LOWER_STR ".cpp", + string_expand(path_to_src)); + + FILE *out_h_file = fopen((char*)out_h_name.str, "wb"); + if (out_h_file == 0){ + printf("error: could not open output file %.*s\n", string_expand(out_h_name)); + exit(1); } - //// - - select_state(ULL_number); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_hex, "LiteralIntegerHexULL"); - handler_token_kind(is_oct, "LiteralIntegerOctULL"); - handler_token_kind("LiteralIntegerULL"); - sm_fallback_peek(emit); + FILE *out_cpp_file = fopen((char*)out_cpp_name.str, "wb"); + if (out_cpp_file == 0){ + printf("error: could not open output file %.*s\n", string_expand(out_cpp_name)); + exit(1); } - //// + //////////////////////////////// - select_state(pp_directive); - sm_set_flag(is_pp_body, true); - sm_case("abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "_" - "0123456789", - pp_directive); - { - Emit_Rule *emit = add_emit_rule(); - handler_keys(emit, pp_directive_set); - handler_key_flag_on_result(emit, "PPInclude", is_include_body, true); - sm_fallback_peek(emit); + // NOTE(allen): Code generation + + fprintf(out_h_file, "%s\n", hand_written_h.str); + gen_tokens(&ctx->arena, ctx->tokens, out_h_file); + + fprintf(out_cpp_file, "%s\n", hand_written.str); + for (Keyword_Set *set = ctx->keywords.first; + set != 0; + set = set->next){ + gen_keyword_table(&ctx->arena, ctx->tokens, *set, out_cpp_file); } + gen_contiguous_control_flow_lexer(&ctx->arena, ctx->tokens, ctx->model, out_cpp_file); - //// + fclose(out_h_file); + fclose(out_cpp_file); - select_state(include_pointy); - sm_case("abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "_. /\\" - "0123456789", - include_pointy); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("PPIncludeFile"); - sm_case(">", emit); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LexError"); - sm_fallback(emit); - } + printf("%.*s:1:\n", string_expand(out_h_name)); + printf("%.*s:1:\n", string_expand(out_cpp_name)); - //// - - select_state(include_quotes); - sm_case("abcdefghijklmnopqrstuvwxyz" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "_. /\\" - "0123456789", - include_pointy); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("PPIncludeFile"); - sm_case("\"", emit); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LexError"); - sm_fallback(emit); - } - - //// - - select_state(pre_L); - sm_set_flag(is_wide, true); - sm_case("\"", string); - sm_case("R", pre_R); - sm_fallback_peek(identifier); - - //// - - select_state(pre_u); - sm_set_flag(is_utf16, true); - sm_case("\"", string); - sm_case("8", pre_u8); - sm_case("R", pre_R); - sm_fallback_peek(identifier); - - //// - - select_state(pre_U); - sm_set_flag(is_utf32, true); - sm_case("\"", string); - sm_case("R", pre_R); - sm_fallback_peek(identifier); - - //// - - select_state(pre_u8); - sm_set_flag(is_utf8, true); - sm_case("\"", string); - sm_case("R", pre_R); - sm_fallback_peek(identifier); - - //// - - select_state(pre_R); - sm_case("\"", raw_string); - sm_fallback_peek(identifier); - - //// - - select_state(character); - sm_set_flag(is_char, true); - sm_fallback_peek(string); - - //// - - select_state(string); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_wide, "LiteralStringWide"); - handler_token_kind(is_utf8 , "LiteralStringUTF8"); - handler_token_kind(is_utf16, "LiteralStringUTF16"); - handler_token_kind(is_utf32, "LiteralStringUTF32"); - handler_token_kind("LiteralString"); - sm_case_flagged(is_char, false, "\"", emit); - } - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_wide, "LiteralCharacterWide"); - handler_token_kind(is_utf8 , "LiteralCharacterUTF8"); - handler_token_kind(is_utf16, "LiteralCharacterUTF16"); - handler_token_kind(is_utf32, "LiteralCharacterUTF32"); - handler_token_kind("LiteralCharacter"); - sm_case_flagged(is_char, true, "\'", emit); - } - sm_case("\\", string_esc); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LexError"); - sm_case_peek("\n", emit); - } - sm_case_flagged(is_char, true, "\"", string); - sm_case_flagged(is_char, false, "\'", string); - sm_fallback(string); - - //// - - select_state(string_esc); - sm_case("'\"?\\abfnrtv", string); - sm_case("01234567", string_esc_oct2); - sm_case("x", string_esc_hex); - sm_case("u", string_esc_universal_4); - sm_case("U", string_esc_universal_8); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LexError"); - sm_case_peek("\n", emit); - } - sm_fallback(string); - - //// - - select_state(string_esc_oct2); - sm_case("01234567", string_esc_oct1); - sm_fallback_peek(string); - - //// - - select_state(string_esc_oct1); - sm_case("01234567", string); - sm_fallback_peek(string); - - //// - - select_state(string_esc_hex); - sm_case("0123456789abcdefABCDEF", string_esc_hex); - sm_fallback_peek(string); - - //// - - select_state(string_esc_universal_8); - sm_case("0123456789abcdefABCDEF", string_esc_universal_7); - sm_fallback_peek(string); - - //// - - select_state(string_esc_universal_7); - sm_case("0123456789abcdefABCDEF", string_esc_universal_6); - sm_fallback_peek(string); - - //// - - select_state(string_esc_universal_6); - sm_case("0123456789abcdefABCDEF", string_esc_universal_5); - sm_fallback_peek(string); - - //// - - select_state(string_esc_universal_5); - sm_case("0123456789abcdefABCDEF", string_esc_universal_4); - sm_fallback_peek(string); - - //// - - select_state(string_esc_universal_4); - sm_case("0123456789abcdefABCDEF", string_esc_universal_3); - sm_fallback_peek(string); - - //// - - select_state(string_esc_universal_3); - sm_case("0123456789abcdefABCDEF", string_esc_universal_2); - sm_fallback_peek(string); - - //// - - select_state(string_esc_universal_2); - sm_case("0123456789abcdefABCDEF", string_esc_universal_1); - sm_fallback_peek(string); - - //// - - select_state(string_esc_universal_1); - sm_case("0123456789abcdefABCDEF", string); - sm_fallback_peek(string); - - //// - - select_state(raw_string); - sm_delim_mark_first(); - sm_fallback_peek(raw_string_get_delim); - - //// - - select_state(raw_string_get_delim); - sm_case_peek("(", raw_string_finish_delim); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LexError"); - sm_case(" \\)", emit); - } - sm_fallback(raw_string_get_delim); - - //// - - select_state(raw_string_finish_delim); - sm_delim_mark_one_past_last(); - sm_fallback(raw_string_find_close); - - //// - - select_state(raw_string_find_close); - sm_case(")", raw_string_try_delim); - sm_fallback(raw_string_find_close); - - //// - - select_state(raw_string_try_delim); - sm_match_delim(raw_string_try_quote); - sm_fallback_peek(raw_string_find_close); - - //// - - select_state(raw_string_try_quote); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind(is_wide, "LiteralStringWideRaw"); - handler_token_kind(is_utf8 , "LiteralStringUTF8Raw"); - handler_token_kind(is_utf16, "LiteralStringUTF16Raw"); - handler_token_kind(is_utf32, "LiteralStringUTF32Raw"); - handler_token_kind("LiteralStringRaw"); - sm_case("\"", emit); - } - sm_fallback_peek(raw_string_find_close); - - //// - - select_state(comment_block); - sm_case("*", comment_block_try_close); - sm_case("\n", comment_block_newline); - sm_fallback(comment_block); - - //// - - select_state(comment_block_try_close); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("BlockComment"); - sm_case("/", emit); - } - sm_case("*", comment_block_try_close); - sm_fallback(comment_block); - - //// - - select_state(comment_block_newline); - sm_set_flag(is_pp_body, false); - sm_set_flag(is_include_body, false); - sm_fallback_peek(comment_block); - - //// - - select_state(comment_line); - { - Emit_Rule *emit = add_emit_rule(); - handler_token_kind("LineComment"); - sm_case_peek("\n", emit); - } - sm_fallback(comment_line); + // NOTE(allen): Simplifying the state machine + // Isolate the state machine's parts into small L.U.T. then generate tables? + // If using L.U.T: Optimize all action lists that don't contain a "consume" action + // NOTE(allen): State machine generation + // Implementation: Control Flow + // Feature: Fully Contiguous input + // + // Implementation: L.U.T. Accelerated + // + // Feature: Spatially chunked input + // Feature: Temporally chunked input + return(0); } // BOTTOM diff --git a/lexer_generator/pcg_basic.c b/lexer_generator/pcg_basic.c new file mode 100644 index 00000000..a4e11029 --- /dev/null +++ b/lexer_generator/pcg_basic.c @@ -0,0 +1,116 @@ +/* + * PCG Random Number Generation for C. + * + * Copyright 2014 Melissa O'Neill + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For additional information about the PCG random number generation scheme, + * including its license and other licensing options, visit + * + * http://www.pcg-random.org + */ + +/* + * This code is derived from the full C implementation, which is in turn + * derived from the canonical C++ PCG implementation. The C++ version + * has many additional features and is preferable if you can use C++ in + * your project. + */ + +#include "pcg_basic.h" + +// state for global RNGs + +static pcg32_random_t pcg32_global = PCG32_INITIALIZER; + +// pcg32_srandom(initstate, initseq) +// pcg32_srandom_r(rng, initstate, initseq): +// Seed the rng. Specified in two parts, state initializer and a +// sequence selection constant (a.k.a. stream id) + +void pcg32_srandom_r(pcg32_random_t* rng, uint64_t initstate, uint64_t initseq) +{ + rng->state = 0U; + rng->inc = (initseq << 1u) | 1u; + pcg32_random_r(rng); + rng->state += initstate; + pcg32_random_r(rng); +} + +void pcg32_srandom(uint64_t seed, uint64_t seq) +{ + pcg32_srandom_r(&pcg32_global, seed, seq); +} + +// pcg32_random() +// pcg32_random_r(rng) +// Generate a uniformly distributed 32-bit random number + +uint32_t pcg32_random_r(pcg32_random_t* rng) +{ + uint64_t oldstate = rng->state; + rng->state = oldstate * 6364136223846793005ULL + rng->inc; + uint32_t xorshifted = (uint32_t)(((oldstate >> 18u) ^ oldstate) >> 27u); + uint32_t rot = oldstate >> 59u; + return (xorshifted >> rot) | (xorshifted << ((-rot) & 31)); +} + +uint32_t pcg32_random() +{ + return pcg32_random_r(&pcg32_global); +} + + +// pcg32_boundedrand(bound): +// pcg32_boundedrand_r(rng, bound): +// Generate a uniformly distributed number, r, where 0 <= r < bound + +uint32_t pcg32_boundedrand_r(pcg32_random_t* rng, uint32_t bound) +{ + // To avoid bias, we need to make the range of the RNG a multiple of + // bound, which we do by dropping output less than a threshold. + // A naive scheme to calculate the threshold would be to do + // + // uint32_t threshold = 0x100000000ull % bound; + // + // but 64-bit div/mod is slower than 32-bit div/mod (especially on + // 32-bit platforms). In essence, we do + // + // uint32_t threshold = (0x100000000ull-bound) % bound; + // + // because this version will calculate the same modulus, but the LHS + // value is less than 2^32. + + uint32_t threshold = -bound % bound; + + // Uniformity guarantees that this loop will terminate. In practice, it + // should usually terminate quickly; on average (assuming all bounds are + // equally likely), 82.25% of the time, we can expect it to require just + // one iteration. In the worst case, someone passes a bound of 2^31 + 1 + // (i.e., 2147483649), which invalidates almost 50% of the range. In + // practice, bounds are typically small and only a tiny amount of the range + // is eliminated. + for (;;) { + uint32_t r = pcg32_random_r(rng); + if (r >= threshold) + return r % bound; + } +} + + +uint32_t pcg32_boundedrand(uint32_t bound) +{ + return pcg32_boundedrand_r(&pcg32_global, bound); +} + diff --git a/lexer_generator/pcg_basic.h b/lexer_generator/pcg_basic.h new file mode 100644 index 00000000..e2b526aa --- /dev/null +++ b/lexer_generator/pcg_basic.h @@ -0,0 +1,78 @@ +/* + * PCG Random Number Generation for C. + * + * Copyright 2014 Melissa O'Neill + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * For additional information about the PCG random number generation scheme, + * including its license and other licensing options, visit + * + * http://www.pcg-random.org + */ + +/* + * This code is derived from the full C implementation, which is in turn + * derived from the canonical C++ PCG implementation. The C++ version + * has many additional features and is preferable if you can use C++ in + * your project. + */ + +#ifndef PCG_BASIC_H_INCLUDED +#define PCG_BASIC_H_INCLUDED 1 + +#include + +#if __cplusplus +extern "C" { +#endif + +struct pcg_state_setseq_64 { // Internals are *Private*. + uint64_t state; // RNG state. All values are possible. + uint64_t inc; // Controls which RNG sequence (stream) is + // selected. Must *always* be odd. +}; +typedef struct pcg_state_setseq_64 pcg32_random_t; + +// If you *must* statically initialize it, here's one. + +#define PCG32_INITIALIZER { 0x853c49e6748fea9bULL, 0xda3e39cb94b95bdbULL } + +// pcg32_srandom(initstate, initseq) +// pcg32_srandom_r(rng, initstate, initseq): +// Seed the rng. Specified in two parts, state initializer and a +// sequence selection constant (a.k.a. stream id) + +void pcg32_srandom(uint64_t initstate, uint64_t initseq); +void pcg32_srandom_r(pcg32_random_t* rng, uint64_t initstate, + uint64_t initseq); + +// pcg32_random() +// pcg32_random_r(rng) +// Generate a uniformly distributed 32-bit random number + +uint32_t pcg32_random(void); +uint32_t pcg32_random_r(pcg32_random_t* rng); + +// pcg32_boundedrand(bound): +// pcg32_boundedrand_r(rng, bound): +// Generate a uniformly distributed number, r, where 0 <= r < bound + +uint32_t pcg32_boundedrand(uint32_t bound); +uint32_t pcg32_boundedrand_r(pcg32_random_t* rng, uint32_t bound); + +#if __cplusplus +} +#endif + +#endif // PCG_BASIC_H_INCLUDED diff --git a/meta/4ed_build.cpp b/meta/4ed_build.cpp index 3e5df5c9..d24632f8 100644 --- a/meta/4ed_build.cpp +++ b/meta/4ed_build.cpp @@ -467,11 +467,6 @@ build_and_run(Arena *arena, char *cdir, char *filename, char *name, u32 flags){ } } -internal void -fsm_generator(Arena *arena, char *cdir){ - build_and_run(arena, cdir, "meta/4ed_fsm_table_generator.cpp", "fsmgen", OPTS | DEBUG_INFO); -} - internal void string_build(Arena *arena, char *cdir){ char *dir = fm_str(arena, BUILD_DIR); @@ -574,8 +569,6 @@ build_main(Arena *arena, char *cdir, b32 update_local_theme, u32 flags, u32 arch internal void standard_build(Arena *arena, char *cdir, u32 flags, u32 arch){ - //fsm_generator(arena, cdir); - //do_buildsuper(arena, cdir, fm_str(arena, custom_files[Custom_Default]), arch); do_buildsuper(arena, cdir, fm_str(arena, custom_files[Custom_Experiments]), arch); //do_buildsuper(arena, cdir, fm_str(arena, custom_files[Custom_Casey]), arch); @@ -602,8 +595,6 @@ get_4coder_dist_name(Arena *arena, u32 platform, char *tier, u32 arch){ internal void package(Arena *arena, char *cdir){ // NOTE(allen): meta - fsm_generator(arena, cdir); - char *build_dir = fm_str(arena, BUILD_DIR); char *pack_dir = fm_str(arena, PACK_DIR); char *fonts_source_dir = fm_str(arena, "../4coder-non-source/dist_files/fonts"); diff --git a/meta/4ed_fsm_table_generator.cpp b/meta/4ed_fsm_table_generator.cpp deleted file mode 100644 index 4573bebb..00000000 --- a/meta/4ed_fsm_table_generator.cpp +++ /dev/null @@ -1,988 +0,0 @@ -/* - * FSM table generator: - * Generate FSM tables as ".c" files from FSM functions. - * - * 23.03.2016 (dd.mm.yyyy) - */ - -// TOP - -#include "../4coder_base_types.h" -#include "../4coder_base_types.cpp" - -#include -#include -#include - -#define LEXER_TABLE_FILE "4coder_lib/4cpp_lexer_tables.c" - -#include "../4coder_lib/4cpp_lexer_types.h" -#include "../4ed_mem.cpp" - -struct Whitespace_FSM{ - unsigned char pp_state; - unsigned char white_done; -}; - -Whitespace_FSM -whitespace_skip_fsm(Whitespace_FSM wfsm, char c){ - if (wfsm.pp_state != LSPP_default){ - if (c == '\n') wfsm.pp_state = LSPP_default; - } - if (!(c == ' ' || c == '\n' || c == '\t' || c == '\r' || c == '\f' || c == '\v')){ - wfsm.white_done = 1; - } - return(wfsm); -} - -#define FSM_SIG(n) Cpp_Lex_FSM n(Cpp_Lex_FSM fsm, char c, b32 get_flags) -typedef FSM_SIG(FSM_Function); - -FSM_SIG(int_fsm){ - switch (fsm.state){ - case LSINT_default: - { - switch (c){ - case 'u': case 'U': fsm.state = LSINT_u; break; - case 'l': fsm.state = LSINT_l; break; - case 'L': fsm.state = LSINT_L; break; - default: fsm.emit_token = true; break; - } - }break; - - case LSINT_u: - { - switch (c){ - case 'l': fsm.state = LSINT_ul; break; - case 'L': fsm.state = LSINT_uL; break; - default: fsm.emit_token = true; break; - } - }break; - - case LSINT_l: - { - switch (c){ - case 'l': fsm.state = LSINT_ll; break; - case 'U': case 'u': fsm.state = LSINT_extra; break; - default: fsm.emit_token = true; break; - } - }break; - - case LSINT_L: - { - switch (c){ - case 'L': fsm.state = LSINT_ll; break; - case 'U': case 'u': fsm.state = LSINT_extra; break; - default: fsm.emit_token = true; break; - } - }break; - - case LSINT_ul: - { - switch (c){ - case 'l': fsm.state = LSINT_extra; break; - default: fsm.emit_token = true; break; - } - }break; - - case LSINT_uL: - { - switch (c){ - case 'L': fsm.state = LSINT_extra; break; - default: fsm.emit_token = true; break; - } - }break; - - case LSINT_ll: - { - switch (c){ - case 'u': case 'U': fsm.state = LSINT_extra; break; - default: fsm.emit_token = true; break; - } - }break; - - case LSINT_extra: - fsm.emit_token = true; - break; - } - return(fsm); -} - -FSM_SIG(normal_str_fsm){ - - if (!get_flags){ - switch (fsm.state){ - case LSSTR_default: - case LSSTR_multiline: - { - switch (c){ - case '\n': case 0: fsm.state = LSSTR_error; fsm.emit_token = true; break; - case '\\': fsm.state = LSSTR_escape; break; - case '"': fsm.emit_token = true; break; - default: break; - } - }break; - - case LSSTR_escape: - { - switch (c){ - case '\n': fsm.state = LSSTR_multiline; break; - default: fsm.state = LSSTR_default; break; - } - }break; - } - } - else{ - switch (fsm.state){ - case LSSTR_multiline: - { - fsm.flags = 1; - }break; - } - } - - return(fsm); -} - -FSM_SIG(normal_char_fsm){ - - if (!get_flags){ - switch (fsm.state){ - case LSSTR_default: - case LSSTR_multiline: - { - switch (c){ - case '\n': case 0: fsm.state = LSSTR_error; fsm.emit_token = true; break; - case '\\': fsm.state = LSSTR_escape; break; - case '\'': fsm.emit_token = true; break; - default: break; - } - }break; - - case LSSTR_escape: - { - switch (c){ - case '\n': fsm.state = LSSTR_multiline; break; - default: fsm.state = LSSTR_default; break; - } - }break; - } - } - else{ - switch (fsm.state){ - case LSSTR_multiline: - { - fsm.flags = 1; - }break; - } - } - - return(fsm); -} - -FSM_SIG(raw_str_fsm){ - - if (!get_flags){ - switch (fsm.state){ - case LSSTR_default: - { - switch (c){ - case 0: case ')': case '\\': case ' ': case '\n': fsm.emit_token = true; break; - case '(': fsm.state = LSSTR_get_delim; fsm.emit_token = true; break; - default: break; - } - }break; - - case LSSTR_get_delim: - case LSSTR_multiline: - { - switch (c){ - case '\n': fsm.state = LSSTR_multiline; break; - case 0: case '"': fsm.state = LSSTR_check_delim; fsm.emit_token = true; break; - default: break; - } - }break; - } - } - else{ - switch (fsm.state){ - case LSSTR_multiline: - { - fsm.flags = 1; - }break; - } - } - - return(fsm); -} - -FSM_SIG(include_str_fsm){ - switch (fsm.state){ - case LSSTR_default: - { - switch (c){ - case '\n': case 0: fsm.state = LSSTR_error; fsm.emit_token = true; break; - case '>': fsm.emit_token = true; break; - default: break; - } - }break; - } - return(fsm); -} - -b32 -is_identifier_char(u8 c, b32 ignore_string_delims){ - b32 result = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$' || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"')); - return(result); -} - -b32 -is_identifier_char_restricted(u8 c, b32 ignore_string_delims){ - b32 result = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"')); - return(result); -} - -b32 -is_identifier_char_non_numeric(u8 c, b32 ignore_string_delims){ - b32 result = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$' || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"')); - return(result); -} - -Cpp_Lex_FSM -main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, b32 ignore_string_delims){ - if (c == 0){ - switch (fsm.state){ - case LS_string_R: - case LS_string_LUu8: - case LS_string_u: - { - fsm.state = LS_identifier; - }break; - } - fsm.emit_token = true; - } - else{ - switch (pp_state){ - case LSPP_error: - { - fsm.state = LS_error_message; - if (c == '\n'){ - fsm.emit_token = true; - } - }break; - - default: - switch (fsm.state){ - case LS_default: - if (!ignore_string_delims && c == 'R'){ - fsm.state = LS_string_R; - } - else if (!ignore_string_delims && (c == 'U' || c == 'L')){ - fsm.state = LS_string_LUu8; - } - else if (!ignore_string_delims && c == 'u'){ - fsm.state = LS_string_u; - } - else if (is_identifier_char_non_numeric(c, ignore_string_delims)){ - fsm.state = LS_identifier; - } - else if (c >= '1' && c <= '9'){ - fsm.state = LS_number; - } - else if (c == '0'){ - fsm.state = LS_number0; - } - else{ - switch (c){ - case '\'': - { - if (ignore_string_delims){ - fsm.state = LS_identifier; - } - else{ - fsm.state = LS_char; - fsm.emit_token = true; - } - }break; - - case '"': - { - if (ignore_string_delims){ - fsm.state = LS_identifier; - } - else{ - fsm.state = LS_string_normal; - fsm.emit_token = true; - } - }break; - - case '/': fsm.state = LS_comment_pre; break; - - case '.': fsm.state = LS_dot; break; - - case '<': - if (pp_state == LSPP_include && !ignore_string_delims){ - fsm.state = LS_string_include; - fsm.emit_token = true; - } - else{ - fsm.state = LS_less; - } - break; - - case '>': fsm.state = LS_more; break; - - case '-': fsm.state = LS_minus; break; - - case '&': fsm.state = LS_and; break; - case '|': fsm.state = LS_or; break; - - case '+': fsm.state = LS_plus; break; - - case ':': fsm.state = LS_colon; break; - - case '*': case '%': case '^': case '=': - case '!': fsm.state = LS_single_op; break; - - case '#': - if (pp_state == LSPP_default){ - fsm.state = LS_pp; - } - else{ - fsm.state = LS_pound; - } - break; - -#define OperCase(op,type) case op: fsm.emit_token = true; break; - OperCase('{', CPP_TOKEN_BRACE_OPEN); - OperCase('}', CPP_TOKEN_BRACE_CLOSE); - - OperCase('[', CPP_TOKEN_BRACKET_OPEN); - OperCase(']', CPP_TOKEN_BRACKET_CLOSE); - - OperCase('(', CPP_TOKEN_PARENTHESE_OPEN); - OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE); - - OperCase('~', CPP_TOKEN_TILDE); - OperCase(',', CPP_TOKEN_COMMA); - OperCase(';', CPP_TOKEN_SEMICOLON); - OperCase('?', CPP_TOKEN_TERNARY_QMARK); - - OperCase('@', CPP_TOKEN_JUNK); - OperCase('\\', CPP_TOKEN_JUNK); -#undef OperCase - } - } - break; - - case LS_identifier: - { - b32 is_ident = is_identifier_char(c, ignore_string_delims); - if (!is_ident){ - fsm.emit_token = true; - } - } - break; - - case LS_pound: - { - fsm.emit_token = true; - }break; - - case LS_pp: - { - if (c == ' ' || c == '\r' || c == '\v' || c == '\f'){ - // NOTE(allen): do nothing - } - else if (is_identifier_char_restricted(c, ignore_string_delims)){ - fsm.state = LS_ppdef; - } - else{ - fsm.emit_token = true; - } - }break; - - case LS_ppdef: - { - int is_ident = is_identifier_char_restricted(c, ignore_string_delims); - if (!is_ident){ - fsm.emit_token = true; - } - }break; - - case LS_string_R: - { - if (ignore_string_delims){ - fsm.state = LS_count; - fsm.emit_token = true; - } - else{ - switch (c){ - case '"': fsm.state = LS_string_raw; fsm.emit_token = true; break; - default: - { - fsm.state = LS_identifier; - b32 is_ident = is_identifier_char(c, ignore_string_delims); - if (!is_ident){ - fsm.emit_token = true; - } - }break; - } - } - }break; - - case LS_string_LUu8: - { - if (ignore_string_delims){ - fsm.state = LS_count; - fsm.emit_token = true; - } - else{ - switch (c){ - case '"': fsm.state = LS_string_normal; fsm.emit_token = true; break; - case '\'': fsm.state = LS_char; fsm.emit_token = true; break; - case 'R': fsm.state = LS_string_R; break; - default: - { - fsm.state = LS_identifier; - b32 is_ident = is_identifier_char(c, ignore_string_delims); - if (!is_ident){ - fsm.emit_token = true; - } - }break; - } - } - }break; - - case LS_string_u: - { - if (ignore_string_delims){ - fsm.state = LS_count; - fsm.emit_token = true; - } - else{ - switch (c){ - case '"': fsm.state = LS_string_normal; fsm.emit_token = true; break; - case '\'': fsm.state = LS_char; fsm.emit_token = true; break; - case '8': fsm.state = LS_string_LUu8; break; - case 'R': fsm.state = LS_string_R; break; - default: - { - fsm.state = LS_identifier; - b32 is_ident = is_identifier_char(c, ignore_string_delims); - if (!is_ident){ - fsm.emit_token = true; - } - }break; - } - } - }break; - - case LS_number: - if (c >= '0' && c <= '9'){ - fsm.state = LS_number; - } - else{ - switch (c){ - case '.': fsm.state = LS_float; break; - default: fsm.emit_token = true; break; - } - } - break; - - case LS_number0: - if (c >= '0' && c <= '9'){ - fsm.state = LS_number; - } - else if (c == 'x'){ - fsm.state = LS_hex; - } - else if (c == '.'){ - fsm.state = LS_float; - } - else{ - fsm.emit_token = true; - } - break; - - case LS_float: - if (!(c >= '0' && c <= '9')){ - switch (c){ - case 'e': fsm.state = LS_crazy_float0; break; - default: fsm.emit_token = true; break; - } - } - break; - - case LS_crazy_float0: - { - if (('0' <= c && c <= '9') || c == '-' || c == '+'){ - fsm.state = LS_crazy_float1; - } - else{ - fsm.emit_token = true; - } - } - break; - - case LS_crazy_float1: - { - if (!(c >= '0' && c <= '9')){ - fsm.emit_token = true; - } - } - break; - - case LS_hex: - { - int is_hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || c >= 128; - if (!is_hex){ - fsm.emit_token = true; - } - }break; - - case LS_dot: - { - if (c >= '0' && c <= '9'){ - fsm.state = LS_float; - } - else{ - switch (c){ - case '.': fsm.state = LS_ellipsis; break; - case '*': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - } - }break; - - case LS_ellipsis: fsm.emit_token = true; break; - - case LS_less: - { - switch (c){ - case '<': fsm.state = LS_single_op; break; - case '=': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - - case LS_more: - { - switch (c){ - case '>': fsm.state = LS_single_op; break; - case '=': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - - case LS_comment_pre: - { - switch (c){ - case '/': fsm.state = LS_comment; break; - case '*': fsm.state = LS_comment_block; break; - case '=': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - - case LS_comment: - { - switch (c){ - case '\\': fsm.state = LS_comment_slashed; break; - case '\n': fsm.emit_token = true; break; - } - }break; - - case LS_comment_slashed: - { - switch (c){ - case '\r': case '\f': case '\v': break; - default: fsm.state = LS_comment; break; - } - }break; - - case LS_comment_block: - { - switch (c){ - case '*': fsm.state = LS_comment_block_ending; break; - } - }break; - - case LS_comment_block_ending: - { - switch (c){ - case '*': fsm.state = LS_comment_block_ending; break; - case '/': fsm.emit_token = true; break; - default: fsm.state = LS_comment_block; break; - } - }break; - - case LS_minus: - { - switch (c){ - case '>': fsm.state = LS_arrow; break; - case '-': fsm.emit_token = true; break; - case '=': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - - case LS_arrow: - { - switch (c){ - case '*': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - - case LS_and: - switch (c){ - case '&': fsm.emit_token = true; break; - case '=': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - break; - - case LS_or: - { - switch (c){ - case '|': fsm.emit_token = true; break; - case '=': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - - case LS_plus: - { - switch (c){ - case '+': fsm.emit_token = true; break; - case '=': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - - case LS_colon: - { - switch (c){ - case ':': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - - case LS_single_op: - { - switch (c){ - case '=': fsm.emit_token = true; break; - default: fsm.emit_token = true; break; - } - }break; - } - break; - } - } - return(fsm); -} - -static void -begin_table(FILE *file, char *type, char *group_name, char *table_name){ - fprintf(file, "%s %s_%s[] = {\n", type, group_name, table_name); -} - -static void -begin_table(FILE *file, char *type, char *table_name){ - fprintf(file, "%s %s[] = {\n", type, table_name); -} - -static void -begin_ptr_table(FILE *file, char *type, char *table_name){ - fprintf(file, "%s * %s[] = {\n", type, table_name); -} - -static void -do_table_item(FILE *file, uint16_t item){ - fprintf(file, "%3d,", (int32_t)item); -} - -static void -do_table_item_binary(FILE *file, uint16_t item){ - if (item == 0){ - fprintf(file, "0x00,"); - } - else{ - fprintf(file, "%#04x,", item); - } -} - -static void -do_table_item_direct(FILE *file, char *item, char *tail){ - fprintf(file, "%s%s,", item, tail); -} - -static void -end_row(FILE *file){ - fprintf(file, "\n"); -} - -static void -end_table(FILE *file){ - fprintf(file, "};\n\n"); -} - -struct FSM_Tables{ - u8 *full_transition_table; - u8 *marks; - u8 *eq_class; - u8 *eq_class_rep; - u8 *reduced_transition_table; - u8 *flags; - - u8 eq_class_counter; - u16 state_count; -}; - -static void -allocate_full_tables(FSM_Tables *table, uint8_t state_count){ - table->full_transition_table = (uint8_t*)malloc(state_count * 256); - table->marks = (uint8_t*)malloc(state_count * 256); - table->eq_class = (uint8_t*)malloc(state_count * 256); - table->eq_class_rep = (uint8_t*)malloc(state_count * 256); - table->state_count = state_count; - block_zero(table->marks, 256); -} - -static void -do_table_reduction(FSM_Tables *table, uint16_t state_count){ - { - table->eq_class_counter = 0; - uint8_t *c_line = table->full_transition_table; - for (uint16_t c = 0; c < 256; ++c){ - if (table->marks[c] == 0){ - table->eq_class[c] = table->eq_class_counter; - table->eq_class_rep[table->eq_class_counter] = (uint8_t)c; - uint8_t *c2_line = c_line + state_count; - for (uint16_t c2 = c + 1; c2 < 256; ++c2){ - if (block_compare(c_line, c2_line, state_count) == 0){ - table->marks[c2] = 1; - table->eq_class[c2] = table->eq_class_counter; - } - c2_line += state_count; - } - ++table->eq_class_counter; - } - c_line += state_count; - } - } - - table->reduced_transition_table = (uint8_t*)malloc(state_count * table->eq_class_counter); - { - uint8_t *r_line = table->reduced_transition_table; - for (uint16_t eq = 0; eq < table->eq_class_counter; ++eq){ - uint8_t *u_line = table->full_transition_table + state_count * table->eq_class_rep[eq]; - block_copy(r_line, u_line, state_count); - r_line += state_count; - } - } -} - -static FSM_Tables -generate_whitespace_skip_table(){ - uint8_t state_count = LSPP_count; - FSM_Tables table = {}; - allocate_full_tables(&table, state_count); - - int32_t i = 0; - Whitespace_FSM wfsm = {}; - Whitespace_FSM new_wfsm; - for (uint16_t c = 0; c < 256; ++c){ - for (uint8_t state = 0; state < state_count; ++state){ - wfsm.pp_state = state; - wfsm.white_done = 0; - new_wfsm = whitespace_skip_fsm(wfsm, (uint8_t)c); - table.full_transition_table[i++] = new_wfsm.pp_state + state_count*new_wfsm.white_done; - } - } - - do_table_reduction(&table, state_count); - - return(table); -} - -static FSM_Tables -generate_table(u8 state_count, FSM_Function *fsm_call){ - FSM_Tables table = {}; - allocate_full_tables(&table, state_count); - - i32 i = 0; - Cpp_Lex_FSM fsm = {}; - Cpp_Lex_FSM new_fsm = {}; - for (uint16_t c = 0; c < 256; ++c){ - for (u8 state = 0; state < state_count; ++state){ - fsm.state = state; - fsm.emit_token = false; - new_fsm = fsm_call(fsm, (u8)c, false); - table.full_transition_table[i++] = new_fsm.state + state_count*new_fsm.emit_token; - } - } - - for (u8 state = 0; state < state_count; ++state){ - fsm.state = state; - fsm.emit_token = false; - fsm.flags = 0; - new_fsm = fsm_call(fsm, 0, true); - if (new_fsm.flags != 0){ - if (table.flags == 0){ - table.flags = (u8*)malloc(state_count); - memset(table.flags, 0, state_count); - } - table.flags[state] = new_fsm.flags; - } - } - - do_table_reduction(&table, state_count); - - return(table); -} - -static FSM_Tables -generate_fsm_table(uint8_t pp_state, b32 ignore_string_delims){ - uint8_t state_count = LS_count; - FSM_Tables table = {}; - allocate_full_tables(&table, state_count); - - int32_t i = 0; - Cpp_Lex_FSM fsm = {}; - Cpp_Lex_FSM new_fsm = {}; - for (uint16_t c = 0; c < 256; ++c){ - for (uint8_t state = 0; state < state_count; ++state){ - fsm.state = state; - fsm.emit_token = false; - new_fsm = main_fsm(fsm, pp_state, (uint8_t)c, ignore_string_delims); - table.full_transition_table[i++] = new_fsm.state + state_count*new_fsm.emit_token; - } - } - - do_table_reduction(&table, state_count); - - return(table); -} - -static void -render_fsm_table(FILE *file, FSM_Tables tables, char *group_name){ - begin_table(file, "u16", group_name, "eq_classes"); - for (u16 c = 0; c < 256; ++c){ - if ((c % 16) == 0 && c > 0){ - end_row(file); - } - do_table_item(file, tables.eq_class[c]*tables.state_count); - } - end_row(file); - end_table(file); - - fprintf(file, "const i32 num_%s_eq_classes = %d;\n\n", group_name, tables.eq_class_counter); - - i32 i = 0; - begin_table(file, "u8", group_name, "table"); - for (u16 c = 0; c < tables.eq_class_counter; ++c){ - for (u8 state = 0; state < tables.state_count; ++state){ - do_table_item(file, tables.reduced_transition_table[i++]); - } - end_row(file); - } - end_table(file); - - if (tables.flags != 0){ - begin_table(file, "u8", group_name, "flags"); - for (u8 state = 0; state < tables.state_count; ++state){ - if ((state % 4) == 0 && state > 0){ - end_row(file); - } - do_table_item_binary(file, tables.flags[state]); - } - end_row(file); - end_table(file); - } -} - -static void -render_variable(FILE *file, char *type, char *variable, uint32_t x){ - fprintf(file, "%s %s = %d;\n\n", type, variable, x); -} - -static void -render_comment(FILE *file, char *comment){ - fprintf(file, "/*\n%s*/\n", comment); -} - -struct PP_Names{ - uint8_t pp_state; - char *name; - b32 ignore_string_delims; -}; - -static PP_Names pp_names[] = { - {LSPP_default, "main_fsm", false}, - {LSPP_include, "pp_include_fsm", false}, - {LSPP_macro_identifier, "pp_macro_fsm", false}, - {LSPP_identifier, "pp_identifier_fsm", false}, - {LSPP_body_if, "pp_body_if_fsm", false}, - {LSPP_body, "pp_body_fsm", false}, - {LSPP_number, "pp_number_fsm", false}, - {LSPP_error, "pp_error_fsm", false}, - {LSPP_junk, "pp_junk_fsm", false}, - {LSPP_default, "no_string_fsm", true }, -}; - -int -main(){ - FILE *file = fopen(LEXER_TABLE_FILE, "wb"); - - FSM_Tables wtables = generate_whitespace_skip_table(); - render_fsm_table(file, wtables, "whitespace_fsm"); - - FSM_Tables itables = generate_table(LSINT_count, int_fsm); - render_fsm_table(file, itables, "int_fsm"); - - { - struct{ - char *name; - uint8_t count; - FSM_Function *fsm_call; - } static tables[] = { - {"raw_str", LSSTR_count, raw_str_fsm }, - {"normal_str", LSSTR_count, normal_str_fsm }, - {"include_str", LSSTR_include_count, include_str_fsm }, - {"normal_char", LSSTR_count, normal_char_fsm } - }; - - for (u32 i = 0; i < ArrayCount(tables); ++i){ - FSM_Tables str_tables = generate_table(tables[i].count, tables[i].fsm_call); - render_fsm_table(file, str_tables, tables[i].name); - } - } - - for (int32_t i = 0; i < ArrayCount(pp_names); ++i){ - FSM_Tables tables = generate_fsm_table(pp_names[i].pp_state, pp_names[i].ignore_string_delims); - render_fsm_table(file, tables, pp_names[i].name); - } - - begin_ptr_table(file, "uint16_t", "get_eq_classes"); - for (int32_t i = 0; i < ArrayCount(pp_names); ++i){ - do_table_item_direct(file, pp_names[i].name, "_eq_classes"); - end_row(file); - } - end_table(file); - - begin_ptr_table(file, "uint8_t", "get_table"); - for (int32_t i = 0; i < ArrayCount(pp_names); ++i){ - do_table_item_direct(file, pp_names[i].name, "_table"); - end_row(file); - } - end_table(file); - - fclose(file); - return(0); -} - -// BOTTOM - - diff --git a/project.4coder b/project.4coder index a381c9c7..c4b292fe 100644 --- a/project.4coder +++ b/project.4coder @@ -72,13 +72,28 @@ command_list = { { .name = "run pinky one time", .out = "*run*", .footer_panel = false, .save_dirty_files = false, .cmd = { { "pushd ..\\..\\pinky\\build & one_time", .os = "win" }, }, }, + + { .name = "run generator", + .out = "*run*", .footer_panel = false, .save_dirty_files = true, + .cmd = { { "..\\build\\generator", .os = "win" }, }, }, + { .name = "build C++ lexer generator", + .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, + .cmd = { { "build_generator 4coder_lex_gen_cpp.cpp ..\\build", .os = "win" }, }, }, + + { .name = "build token tester", + .out = "*compilation*", .footer_panel = true, .save_dirty_files = true, + .cmd = { { "build_generator languages\\4coder_lexer_cpp_test.cpp ..\\build", .os = "win" }, }, } }; -fkey_command[1] = "build x64"; -fkey_command[3] = "build x86"; -fkey_command[4] = "build metadata"; -fkey_command[5] = "build config parser generator"; -fkey_command[6] = "generate config parser"; +//fkey_command[1] = "build x64"; +//fkey_command[3] = "build x86"; +//fkey_command[4] = "build metadata"; + +fkey_command[1] = "build C++ lexer generator"; +fkey_command[2] = "build token tester"; +fkey_command[3] = "run generator"; + + fkey_command[9] = "build one time"; fkey_command[10] = "run one time"; fkey_command[12] = "package";