From 27dcead16fb68b66f10541bc2cc1e2bd6072f863 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Tue, 2 May 2017 17:35:48 -0400 Subject: [PATCH] finished up the lexer fixes --- 4cpp/4cpp_lexer_tables.c | 20 ++++++++++---------- meta/fsm_table_generator.cpp | 1 + 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/4cpp/4cpp_lexer_tables.c b/4cpp/4cpp_lexer_tables.c index 4741e8b6..fb48dd3a 100644 --- a/4cpp/4cpp_lexer_tables.c +++ b/4cpp/4cpp_lexer_tables.c @@ -202,7 +202,7 @@ u16_4tech main_fsm_eq_classes[] = { const i32_4tech num_main_fsm_eq_classes = 32; u8_4tech main_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, @@ -258,7 +258,7 @@ u16_4tech pp_include_fsm_eq_classes[] = { const i32_4tech num_pp_include_fsm_eq_classes = 32; u8_4tech pp_include_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, @@ -314,7 +314,7 @@ u16_4tech pp_macro_fsm_eq_classes[] = { const i32_4tech num_pp_macro_fsm_eq_classes = 32; u8_4tech pp_macro_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, @@ -370,7 +370,7 @@ u16_4tech pp_identifier_fsm_eq_classes[] = { const i32_4tech num_pp_identifier_fsm_eq_classes = 32; u8_4tech pp_identifier_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, @@ -426,7 +426,7 @@ u16_4tech pp_body_if_fsm_eq_classes[] = { const i32_4tech num_pp_body_if_fsm_eq_classes = 32; u8_4tech pp_body_if_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, @@ -482,7 +482,7 @@ u16_4tech pp_body_fsm_eq_classes[] = { const i32_4tech num_pp_body_fsm_eq_classes = 32; u8_4tech pp_body_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, @@ -538,7 +538,7 @@ u16_4tech pp_number_fsm_eq_classes[] = { const i32_4tech num_pp_number_fsm_eq_classes = 32; u8_4tech pp_number_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, @@ -594,7 +594,7 @@ u16_4tech pp_error_fsm_eq_classes[] = { const i32_4tech num_pp_error_fsm_eq_classes = 3; u8_4tech pp_error_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, 61, }; @@ -621,7 +621,7 @@ u16_4tech pp_junk_fsm_eq_classes[] = { const i32_4tech num_pp_junk_fsm_eq_classes = 32; u8_4tech pp_junk_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, @@ -677,7 +677,7 @@ u16_4tech no_string_fsm_eq_classes[] = { const i32_4tech num_no_string_fsm_eq_classes = 29; u8_4tech no_string_fsm_table[] = { - 31, 32, 33, 34, 35, 32, 32, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, + 31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 46, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, 0, 32, 33, 3, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 16, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30, diff --git a/meta/fsm_table_generator.cpp b/meta/fsm_table_generator.cpp index ef322002..19587a7d 100644 --- a/meta/fsm_table_generator.cpp +++ b/meta/fsm_table_generator.cpp @@ -285,6 +285,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli switch (fsm.state){ case LS_string_R: case LS_string_LUu8: + case LS_string_u: { fsm.state = LS_identifier; }break;