API parser

4ed_api_definition.cpp

@@ -1,7 +1,7 @@
 /*
  * Mr. 4th Dimention - Allen Webster
  *
- * 02.10.2019
+ * 03.10.2019
  *
  * System API definition program.
  *
@@ -9,44 +9,6 @@
 
 // TOP
 
-#include "4coder_base_types.h"
-
-#include "4coder_base_types.cpp"
-#include "4coder_stringf.cpp"
-
-#include "4coder_malloc_allocator.cpp"
-
-////////////////////////////////
-
-struct API_Param{
-    API_Param *next;
-    String_Const_u8 type_name;
-    String_Const_u8 name;
-};
-
-struct API_Param_List{
-    API_Param *first;
-    API_Param *last;
-    i32 count;
-};
-
-struct API_Call{
-    API_Call *next;
-    String_Const_u8 name;
-    String_Const_u8 return_type;
-    API_Param_List params;
-};
-
-struct API_Definition{
-    API_Call *first;
-    API_Call *last;
-    i32 count;
-    
-    String_Const_u8 name;
-};
-
-////////////////////////////////
-
 function API_Definition*
 begin_api(Arena *arena, char *name){
     API_Definition *api = push_array_zero(arena, API_Definition, 1);
@@ -55,15 +17,20 @@ begin_api(Arena *arena, char *name){
 }
 
 function API_Call*
-api_call(Arena *arena, API_Definition *api, char *name, char *return_type){
+api_call(Arena *arena, API_Definition *api, String_Const_u8 name, String_Const_u8 return_type){
     API_Call *call = push_array_zero(arena, API_Call, 1);
     sll_queue_push(api->first, api->last, call);
     api->count += 1;
-    call->name = SCu8(name);
+    call->name = name;
-    call->return_type = SCu8(return_type);
+    call->return_type = return_type;
     return(call);
 }
 
+function API_Call*
+api_call(Arena *arena, API_Definition *api, char *name, char *return_type){
+    return(api_call(arena, api, SCu8(name), SCu8(return_type)));
+}
+
 function API_Param*
 api_param(Arena *arena, API_Call *call, char *type_name, char *name){
     API_Param *param = push_array_zero(arena, API_Param, 1);
@@ -74,8 +41,64 @@ api_param(Arena *arena, API_Call *call, char *type_name, char *name){
     return(param);
 }
 
+function void
+api_set_param_list(API_Call *call, API_Param_List list){
+    call->params = list;
+}
+
+function API_Definition*
+api_get_api(Arena *arena, API_Definition_List *list, String_Const_u8 name){
+    API_Definition *result = 0;
+    for (API_Definition *node = list->first;
+         node != 0;
+         node = node->next){
+        if (string_match(name, node->name)){
+            result = node;
+            break;
+        }
+    }
+    if (result == 0){
+        result = push_array_zero(arena, API_Definition, 1);
+        sll_queue_push(list->first, list->last, result);
+        list->count += 1;
+        result->name = name;
+    }
+    return(result);
+}
+
 ////////////////////////////////
 
+#if !defined(SKIP_STDIO)
+#include <stdio.h>
+
+function void
+generate_api_master_list(Arena *scratch, API_Definition *api, FILE *out){
+    fprintf(out, "// %.*s\n", string_expand(api->name));
+    for (API_Call *call = api->first;
+         call != 0;
+         call = call->next){
+        fprintf(out, "%.*s %.*s(",
+                string_expand(call->return_type),
+                string_expand(call->name));
+        if (call->params.count == 0){
+            fprintf(out, "void");
+        }
+        else{
+            for (API_Param *param = call->params.first;
+                 param != 0;
+                 param = param->next){
+                fprintf(out, "%.*s %.*s",
+                        string_expand(param->type_name),
+                        string_expand(param->name));
+                if (param->next != 0){
+                    fprintf(out, ", ");
+                }
+            }
+        }
+        fprintf(out, ");\n");
+    }
+}
+
 function void
 generate_header(Arena *scratch, API_Definition *api, FILE *out){
     for (API_Call *call = api->first;
@@ -217,60 +240,7 @@ generate_cpp(Arena *scratch, API_Definition *api, FILE *out){
     fprintf(out, "#endif\n");
 }
 
-////////////////////////////////
+#endif
-
-function API_Definition*
-define_api(Arena *arena);
-
-int
-main(void){
-    Arena arena = make_arena_malloc();
-    API_Definition *api = define_api(&arena);
-    
-    ////////////////////////////////
-    
-    // NOTE(allen): Arrange output files
-    
-    String_Const_u8 path_to_self = string_u8_litexpr(__FILE__);
-    path_to_self = string_remove_last_folder(path_to_self);
-    
-    String_Const_u8 fname_h = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.h",
-                                              string_expand(path_to_self),
-                                              string_expand(api->name));
-    
-    String_Const_u8 fname_cpp = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.cpp",
-                                                string_expand(path_to_self),
-                                                string_expand(api->name));
-    
-    FILE *out_file_h = fopen((char*)fname_h.str, "wb");
-    if (out_file_h == 0){
-        printf("could not open output file: '%s'\n", fname_h.str);
-        exit(1);
-    }
-    
-    FILE *out_file_cpp = fopen((char*)fname_cpp.str, "wb");
-    if (out_file_cpp == 0){
-        printf("could not open output file: '%s'\n", fname_cpp.str);
-        exit(1);
-    }
-    
-    printf("%s:1:\n", fname_h.str);
-    printf("%s:1:\n", fname_cpp.str);
-    
-    ////////////////////////////////
-    
-    // NOTE(allen): Generate output
-    
-    generate_header(&arena, api, out_file_h);
-    generate_cpp(&arena, api, out_file_cpp);
-    
-    ////////////////////////////////
-    
-    fclose(out_file_h);
-    fclose(out_file_cpp);
-    
-    return(0);
-}
 
 // BOTTOM
 

4ed_api_definition.h

@@ -0,0 +1,53 @@
+/*
+ * Mr. 4th Dimention - Allen Webster
+ *
+ * 03.10.2019
+ *
+ * System API definition types.
+ *
+ */
+
+// TOP
+
+#if !defined(FRED_API_DEFINITION_H)
+#define FRED_API_DEFINITION_H
+
+struct API_Param{
+    API_Param *next;
+    String_Const_u8 type_name;
+    String_Const_u8 name;
+};
+
+struct API_Param_List{
+    API_Param *first;
+    API_Param *last;
+    i32 count;
+};
+
+struct API_Call{
+    API_Call *next;
+    String_Const_u8 name;
+    String_Const_u8 return_type;
+    API_Param_List params;
+};
+
+struct API_Definition{
+    API_Definition *next;
+    
+    API_Call *first;
+    API_Call *last;
+    i32 count;
+    
+    String_Const_u8 name;
+};
+
+struct API_Definition_List{
+    API_Definition *first;
+    API_Definition *last;
+    i32 count;
+};
+
+#endif
+
+// BOTTOM
+

4ed_api_definition_main.cpp

@@ -0,0 +1,78 @@
+/*
+ * Mr. 4th Dimention - Allen Webster
+ *
+ * 02.10.2019
+ *
+ * System API definition program.
+ *
+ */
+
+// TOP
+
+#include "4coder_base_types.h"
+#include "4ed_api_definition.h"
+
+#include "4coder_base_types.cpp"
+#include "4ed_api_definition.cpp"
+#include "4coder_stringf.cpp"
+#include "4coder_malloc_allocator.cpp"
+
+#include <stdio.h>
+
+////////////////////////////////
+
+function API_Definition*
+define_api(Arena *arena);
+
+int
+main(void){
+    Arena arena = make_arena_malloc();
+    API_Definition *api = define_api(&arena);
+    
+    ////////////////////////////////
+    
+    // NOTE(allen): Arrange output files
+    
+    String_Const_u8 path_to_self = string_u8_litexpr(__FILE__);
+    path_to_self = string_remove_last_folder(path_to_self);
+    
+    String_Const_u8 fname_h = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.h",
+                                              string_expand(path_to_self),
+                                              string_expand(api->name));
+    
+    String_Const_u8 fname_cpp = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.cpp",
+                                                string_expand(path_to_self),
+                                                string_expand(api->name));
+    
+    FILE *out_file_h = fopen((char*)fname_h.str, "wb");
+    if (out_file_h == 0){
+        printf("could not open output file: '%s'\n", fname_h.str);
+        exit(1);
+    }
+    
+    FILE *out_file_cpp = fopen((char*)fname_cpp.str, "wb");
+    if (out_file_cpp == 0){
+        printf("could not open output file: '%s'\n", fname_cpp.str);
+        exit(1);
+    }
+    
+    printf("%s:1:\n", fname_h.str);
+    printf("%s:1:\n", fname_cpp.str);
+    
+    ////////////////////////////////
+    
+    // NOTE(allen): Generate output
+    
+    generate_header(&arena, api, out_file_h);
+    generate_cpp(&arena, api, out_file_cpp);
+    
+    ////////////////////////////////
+    
+    fclose(out_file_h);
+    fclose(out_file_cpp);
+    
+    return(0);
+}
+
+// BOTTOM
+

4ed_api_parser.cpp

@@ -0,0 +1,252 @@
+/*
+ * Mr. 4th Dimention - Allen Webster
+ *
+ * 03.10.2019
+ *
+ * Parser that extracts an API from C++ source code.
+ *
+ */
+
+// TOP
+
+#include "4coder_base_types.h"
+#include "4ed_api_definition.h"
+#include "4coder_token.h"
+#include "generated/lexer_cpp.h"
+
+#include "4coder_base_types.cpp"
+#include "4ed_api_definition.cpp"
+#include "4coder_stringf.cpp"
+#include "4coder_malloc_allocator.cpp"
+#include "4coder_token.cpp"
+#include "generated/lexer_cpp.cpp"
+#include "4coder_file.cpp"
+
+////////////////////////////////
+
+/*
+function:
+api ( <identifier> ) function <identifier> {*} <identifier> ( <param_list> )
+
+param_list:
+void |
+<identifier> {*} <identifier> [, <identifier> {*} <identifier>]
+
+anything_else:
+***
+
+api_source:
+{function|anything_else} EOF
+*/
+
+function b32
+api_parse__match(Token_Iterator *it, Token_Cpp_Kind sub_kind){
+    b32 match = false;
+    Token *token = token_it_read(it);
+    if (token->sub_kind == sub_kind){
+        if (token_it_inc(it)){
+            match = true;
+        }
+    }
+    return(match);
+}
+
+function b32
+api_parse__match_identifier(Token_Iterator *it, String_Const_u8 source, String_Const_u8 *lexeme){
+    b32 match = false;
+    Token *token = token_it_read(it);
+    if (token->kind == TokenBaseKind_Identifier ||
+        token->kind == TokenBaseKind_Keyword){
+        if (token_it_inc(it)){
+            *lexeme = string_substring(source, Ii64(token));
+            match = true;
+        }
+    }
+    return(match);
+}
+
+function b32
+api_parse__match_identifier(Token_Iterator *it, String_Const_u8 source, char *lexeme){
+    b32 match = false;
+    Token *token = token_it_read(it);
+    if ((token->kind == TokenBaseKind_Identifier ||
+         token->kind == TokenBaseKind_Keyword) &&
+        string_match(SCu8(lexeme), string_substring(source, Ii64(token)))){
+        if (token_it_inc(it)){
+            match = true;
+        }
+    }
+    return(match);
+}
+
+function String_Const_u8
+api_parse__type_name_with_stars(Arena *arena, String_Const_u8 type, i32 star_counter){
+    if (star_counter > 0){
+        i32 type_full_size = type.size + star_counter;
+        u8 *type_buffer = push_array(arena, u8, type_full_size + 1);
+        block_copy(type_buffer, type.str, type.size);
+        block_fill_u8(type_buffer + type.size, star_counter, (u8)'*');
+        type_buffer[type_full_size] = 0;
+        type = SCu8(type_buffer, type_full_size);
+    }
+    return(type);
+}
+
+function void
+api_parse_add_param(Arena *arena, API_Param_List *list, String_Const_u8 type, i32 star_counter, String_Const_u8 name){
+    type = api_parse__type_name_with_stars(arena, type, star_counter);
+    API_Param *param = push_array(arena, API_Param, 1);
+    sll_queue_push(list->first, list->last, param);
+    list->count += 1;
+    param->type_name = type;
+    param->name = name;
+}
+
+function void
+api_parse_add_function(Arena *arena, API_Definition_List *list, String_Const_u8 api_name, String_Const_u8 func_name, String_Const_u8 type, i32 star_counter, API_Param_List param_list){
+    API_Definition *api = api_get_api(arena, list, api_name);
+    type = api_parse__type_name_with_stars(arena, type, star_counter);
+    API_Call *call = api_call(arena, api, func_name, type);
+    api_set_param_list(call, param_list);
+}
+
+function void
+api_parse_source_add_to_list(Arena *arena, String_Const_u8 source, API_Definition_List *list){
+    Token_List token_list = lex_full_input_cpp(arena, source);
+    Token_Iterator token_it = token_iterator(token_iterator(0, &token_list));
+    
+    for (;;){
+        Token *token = token_it_read(&token_it);
+        if (token->sub_kind == TokenCppKind_EOF){
+            break;
+        }
+        
+        if (api_parse__match_identifier(&token_it, source, "api")){
+            String_Const_u8 api_name = {};
+            String_Const_u8 ret_type = {};
+            i32 ret_type_star_counter = 0;
+            String_Const_u8 func_name = {};
+            API_Param_List param_list = {};
+            
+            b32 success = false;
+            if (api_parse__match(&token_it, TokenCppKind_ParenOp)){
+                if (api_parse__match_identifier(&token_it, source, &api_name)){
+                    if (api_parse__match(&token_it, TokenCppKind_ParenCl)){
+                        if (api_parse__match_identifier(&token_it, source, "function")){
+                            if (api_parse__match_identifier(&token_it, source, &ret_type)){
+                                for (;api_parse__match(&token_it, TokenCppKind_Star);){
+                                    ret_type_star_counter += 1;
+                                }
+                                if (api_parse__match_identifier(&token_it, source, &func_name)){
+                                    if (api_parse__match(&token_it, TokenCppKind_ParenOp)){
+                                        b32 param_list_success = false;
+                                        if (api_parse__match_identifier(&token_it, source, "void")){
+                                            param_list_success = true;
+                                        }
+                                        else{
+                                            for (;;){
+                                                String_Const_u8 type = {};
+                                                i32 star_counter = 0;
+                                                String_Const_u8 name = {};
+                                                if (api_parse__match_identifier(&token_it, source, &type)){
+                                                    for (;api_parse__match(&token_it, TokenCppKind_Star);){
+                                                        star_counter += 1;
+                                                    }
+                                                    if (api_parse__match_identifier(&token_it, source, &name)){
+                                                        param_list_success = true;
+                                                    }
+                                                    else{
+                                                        break;
+                                                    }
+                                                }
+                                                else{
+                                                    break;
+                                                }
+                                                if (param_list_success){
+                                                    api_parse_add_param(arena, &param_list, type, star_counter, name);
+                                                }
+                                                if (api_parse__match(&token_it, TokenCppKind_Comma)){
+                                                    param_list_success = false;
+                                                }
+                                                else{
+                                                    break;
+                                                }
+                                            }
+                                        }
+                                        if (param_list_success){
+                                            if (api_parse__match(&token_it, TokenCppKind_ParenCl)){
+                                                success = true;
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            
+            if (success){
+                api_parse_add_function(arena, list, api_name, func_name, ret_type, ret_type_star_counter, param_list);
+            }
+        }
+        else{
+            if (!token_it_inc(&token_it)){
+                break;
+            }
+        }
+    }
+    
+}
+
+function API_Definition_List
+api_parse_source(Arena *arena, String_Const_u8 source){
+    API_Definition_List list = {};
+    api_parse_source_add_to_list(arena, source, &list);
+    return(list);
+}
+
+////////////////////////////////
+
+int
+main(int argc, char **argv){
+    Arena arena = make_arena_malloc();
+    
+    char *hack[] = {
+        "nothing",
+        "../code/4ed_api_implementation.cpp"
+    };
+    argv = hack;
+    argc = 2;
+    
+    if (argc < 2){
+        printf("usage: <script> <source> {<source>}\n"
+               " source : file to load and parse into the output list\n");
+        exit(1);
+    }
+    
+    API_Definition_List list = {};
+    for (i32 i = 1; i < argc; i += 1){
+        FILE *file = fopen(argv[i], "rb");
+        if (file == 0){
+            printf("error: could not open input file: '%s'\n", argv[i]);
+            continue;
+        }
+        
+        String_Const_u8 text = file_load_all(&arena, file);
+        fclose(file);
+        
+        if (text.size > 0){
+            api_parse_source_add_to_list(&arena, text, &list);
+        }
+    }
+    
+    for (API_Definition *node = list.first;
+         node != 0;
+         node = node->next){
+        generate_api_master_list(&arena, node, stdout);
+    }
+}
+
+// BOTTOM
+

4ed_font_api.cpp

@@ -9,7 +9,7 @@
 
 // TOP
 
-#include "4ed_api_definition.cpp"
+#include "4ed_api_definition_main.cpp"
 
 function API_Definition*
 define_api(Arena *arena){

4ed_graphics_api.cpp

@@ -9,7 +9,7 @@
 
 // TOP
 
-#include "4ed_api_definition.cpp"
+#include "4ed_api_definition_main.cpp"
 
 function API_Definition*
 define_api(Arena *arena){

4ed_system_api.cpp

@@ -9,7 +9,7 @@
 
 // TOP
 
-#include "4ed_api_definition.cpp"
+#include "4ed_api_definition_main.cpp"
 
 function API_Definition*
 define_api(Arena *arena){

custom/4coder_file.cpp

@@ -0,0 +1,26 @@
+/*
+ * Mr. 4th Dimention - Allen Webster
+ *
+ * 03.10.2019
+ *
+ * Basic helpers for C std file handling.
+ *
+ */
+
+// TOP
+
+#include <stdio.h>
+
+function String_Const_u8
+file_load_all(Arena *arena, FILE *file){
+    fseek(file, 0, SEEK_END);
+    umem size = ftell(file);
+    fseek(file, 0, SEEK_SET);
+    u8 *buffer = push_array(arena, u8, size + 1);
+    fread(buffer, 1, size, file);
+    buffer[size] = 0;
+    return(SCu8(buffer, size));
+}
+
+// BOTTOM
+

custom/4coder_token.cpp

@@ -4,6 +4,11 @@
 
 // TOP
 
+function Interval_i64
+Ii64(Token *token){
+    return(Ii64_size(token->pos, token->size));
+}
+
 internal void
 token_list_push(Arena *arena, Token_List *list, Token *token){
     Token_Block *block = list->last;

custom/generated/command_metadata.h

@@ -387,10 +387,10 @@ static Command_Metadata fcoder_metacmd_table[226] = {
 { PROC_LINKS(interactive_new, 0), "interactive_new", 15,  "Interactively creates a new file.", 33, "w:\\4ed\\code\\custom\\4coder_lists.cpp", 35, 847 },
 { PROC_LINKS(interactive_open, 0), "interactive_open", 16,  "Interactively opens a file.", 27, "w:\\4ed\\code\\custom\\4coder_lists.cpp", 35, 880 },
 { PROC_LINKS(command_lister, 0), "command_lister", 14,  "Opens an interactive list of all registered commands.", 53, "w:\\4ed\\code\\custom\\4coder_lists.cpp", 35, 960 },
-{ PROC_LINKS(auto_tab_whole_file, 0), "auto_tab_whole_file", 19,  "Audo-indents the entire current buffer.", 39, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 655 },
+{ PROC_LINKS(auto_tab_whole_file, 0), "auto_tab_whole_file", 19,  "Audo-indents the entire current buffer.", 39, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 309 },
-{ PROC_LINKS(auto_tab_line_at_cursor, 0), "auto_tab_line_at_cursor", 23,  "Auto-indents the line on which the cursor sits.", 47, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 664 },
+{ PROC_LINKS(auto_tab_line_at_cursor, 0), "auto_tab_line_at_cursor", 23,  "Auto-indents the line on which the cursor sits.", 47, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 318 },
-{ PROC_LINKS(auto_tab_range, 0), "auto_tab_range", 14,  "Auto-indents the range between the cursor and the mark.", 55, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 674 },
+{ PROC_LINKS(auto_tab_range, 0), "auto_tab_range", 14,  "Auto-indents the range between the cursor and the mark.", 55, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 328 },
-{ PROC_LINKS(write_and_auto_tab, 0), "write_and_auto_tab", 18,  "Inserts a character and auto-indents the line on which the cursor sits.", 71, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 684 },
+{ PROC_LINKS(write_and_auto_tab, 0), "write_and_auto_tab", 18,  "Inserts a character and auto-indents the line on which the cursor sits.", 71, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 338 },
 { PROC_LINKS(list_all_locations, 0), "list_all_locations", 18,  "Queries the user for a string and lists all exact case-sensitive matches found in all open buffers.", 99, "w:\\4ed\\code\\custom\\4coder_search.cpp", 36, 166 },
 { PROC_LINKS(list_all_substring_locations, 0), "list_all_substring_locations", 28,  "Queries the user for a string and lists all case-sensitive substring matches found in all open buffers.", 103, "w:\\4ed\\code\\custom\\4coder_search.cpp", 36, 172 },
 { PROC_LINKS(list_all_locations_case_insensitive, 0), "list_all_locations_case_insensitive", 35,  "Queries the user for a string and lists all exact case-insensitive matches found in all open buffers.", 101, "w:\\4ed\\code\\custom\\4coder_search.cpp", 36, 178 },

project.4coder

@@ -56,10 +56,14 @@ command_list = {
  { .name = "build graphics api",
    .out = "*compilation*", .footer_panel = true, .save_dirty_files = true,
    .cmd = { { "custom\\bin\\build_one_time 4ed_graphics_api.cpp ..\\build", .os = "win" }, }, },
+
+ { .name = "build api parser",
+   .out = "*compilation*", .footer_panel = true, .save_dirty_files = true,
+   .cmd = { { "custom\\bin\\build_one_time 4ed_api_parser.cpp ..\\build", .os = "win" }, }, },
 };
 
 fkey_command[1] = "build x64";
-fkey_command[2] = "build graphics api";
+fkey_command[2] = "build api parser";
 fkey_command[4] = "run one time";
 fkey_command[5] = "build C++ lexer generator";
 fkey_command[6] = "build token tester";