ps
/
4coder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

API parser

This commit is contained in:
Allen Webster 2019-10-03 16:00:31 -07:00
parent 5bb89e2460
commit 7a507ee6e4
11 changed files with 492 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
4ed_api_definition.cpp
View File

@ -1,7 +1,7 @@
/*
* Mr. 4th Dimention - Allen Webster
*
* 02.10.2019
* 03.10.2019
*
* System API definition program.
*
@ -9,44 +9,6 @@
// TOP
#include "4coder_base_types.h"
#include "4coder_base_types.cpp"
#include "4coder_stringf.cpp"
#include "4coder_malloc_allocator.cpp"
////////////////////////////////
struct API_Param{
API_Param *next;
String_Const_u8 type_name;
String_Const_u8 name;
};
struct API_Param_List{
API_Param *first;
API_Param *last;
i32 count;
};
struct API_Call{
API_Call *next;
String_Const_u8 name;
String_Const_u8 return_type;
API_Param_List params;
};
struct API_Definition{
API_Call *first;
API_Call *last;
i32 count;
String_Const_u8 name;
};
////////////////////////////////
function API_Definition*
begin_api(Arena *arena, char *name){
API_Definition *api = push_array_zero(arena, API_Definition, 1);
@ -55,15 +17,20 @@ begin_api(Arena *arena, char *name){
}
function API_Call*
api_call(Arena *arena, API_Definition *api, char *name, char *return_type){
api_call(Arena *arena, API_Definition *api, String_Const_u8 name, String_Const_u8 return_type){
API_Call *call = push_array_zero(arena, API_Call, 1);
sll_queue_push(api->first, api->last, call);
api->count += 1;
call->name = SCu8(name);
call->return_type = SCu8(return_type);
call->name = name;
call->return_type = return_type;
return(call);
}
function API_Call*
api_call(Arena *arena, API_Definition *api, char *name, char *return_type){
return(api_call(arena, api, SCu8(name), SCu8(return_type)));
}
function API_Param*
api_param(Arena *arena, API_Call *call, char *type_name, char *name){
API_Param *param = push_array_zero(arena, API_Param, 1);
@ -74,8 +41,64 @@ api_param(Arena *arena, API_Call *call, char *type_name, char *name){
return(param);
}
function void
api_set_param_list(API_Call *call, API_Param_List list){
call->params = list;
}
function API_Definition*
api_get_api(Arena *arena, API_Definition_List *list, String_Const_u8 name){
API_Definition *result = 0;
for (API_Definition *node = list->first;
node != 0;
node = node->next){
if (string_match(name, node->name)){
result = node;
break;
}
}
if (result == 0){
result = push_array_zero(arena, API_Definition, 1);
sll_queue_push(list->first, list->last, result);
list->count += 1;
result->name = name;
}
return(result);
}
////////////////////////////////
#if !defined(SKIP_STDIO)
#include <stdio.h>
function void
generate_api_master_list(Arena *scratch, API_Definition *api, FILE *out){
fprintf(out, "// %.*s\n", string_expand(api->name));
for (API_Call *call = api->first;
call != 0;
call = call->next){
fprintf(out, "%.*s %.*s(",
string_expand(call->return_type),
string_expand(call->name));
if (call->params.count == 0){
fprintf(out, "void");
}
else{
for (API_Param *param = call->params.first;
param != 0;
param = param->next){
fprintf(out, "%.*s %.*s",
string_expand(param->type_name),
string_expand(param->name));
if (param->next != 0){
fprintf(out, ", ");
}
}
}
fprintf(out, ");\n");
}
}
function void
generate_header(Arena *scratch, API_Definition *api, FILE *out){
for (API_Call *call = api->first;
@ -217,60 +240,7 @@ generate_cpp(Arena *scratch, API_Definition *api, FILE *out){
fprintf(out, "#endif\n");
}
////////////////////////////////
function API_Definition*
define_api(Arena *arena);
int
main(void){
Arena arena = make_arena_malloc();
API_Definition *api = define_api(&arena);
////////////////////////////////
// NOTE(allen): Arrange output files
String_Const_u8 path_to_self = string_u8_litexpr(__FILE__);
path_to_self = string_remove_last_folder(path_to_self);
String_Const_u8 fname_h = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.h",
string_expand(path_to_self),
string_expand(api->name));
String_Const_u8 fname_cpp = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.cpp",
string_expand(path_to_self),
string_expand(api->name));
FILE *out_file_h = fopen((char*)fname_h.str, "wb");
if (out_file_h == 0){
printf("could not open output file: '%s'\n", fname_h.str);
exit(1);
}
FILE *out_file_cpp = fopen((char*)fname_cpp.str, "wb");
if (out_file_cpp == 0){
printf("could not open output file: '%s'\n", fname_cpp.str);
exit(1);
}
printf("%s:1:\n", fname_h.str);
printf("%s:1:\n", fname_cpp.str);
////////////////////////////////
// NOTE(allen): Generate output
generate_header(&arena, api, out_file_h);
generate_cpp(&arena, api, out_file_cpp);
////////////////////////////////
fclose(out_file_h);
fclose(out_file_cpp);
return(0);
}
#endif
// BOTTOM

53
4ed_api_definition.h Normal file
View File

@ -0,0 +1,53 @@
/*
* Mr. 4th Dimention - Allen Webster
*
* 03.10.2019
*
* System API definition types.
*
*/
// TOP
#if !defined(FRED_API_DEFINITION_H)
#define FRED_API_DEFINITION_H
struct API_Param{
API_Param *next;
String_Const_u8 type_name;
String_Const_u8 name;
};
struct API_Param_List{
API_Param *first;
API_Param *last;
i32 count;
};
struct API_Call{
API_Call *next;
String_Const_u8 name;
String_Const_u8 return_type;
API_Param_List params;
};
struct API_Definition{
API_Definition *next;
API_Call *first;
API_Call *last;
i32 count;
String_Const_u8 name;
};
struct API_Definition_List{
API_Definition *first;
API_Definition *last;
i32 count;
};
#endif
// BOTTOM

78
4ed_api_definition_main.cpp Normal file
View File

@ -0,0 +1,78 @@
/*
* Mr. 4th Dimention - Allen Webster
*
* 02.10.2019
*
* System API definition program.
*
*/
// TOP
#include "4coder_base_types.h"
#include "4ed_api_definition.h"
#include "4coder_base_types.cpp"
#include "4ed_api_definition.cpp"
#include "4coder_stringf.cpp"
#include "4coder_malloc_allocator.cpp"
#include <stdio.h>
////////////////////////////////
function API_Definition*
define_api(Arena *arena);
int
main(void){
Arena arena = make_arena_malloc();
API_Definition *api = define_api(&arena);
////////////////////////////////
// NOTE(allen): Arrange output files
String_Const_u8 path_to_self = string_u8_litexpr(__FILE__);
path_to_self = string_remove_last_folder(path_to_self);
String_Const_u8 fname_h = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.h",
string_expand(path_to_self),
string_expand(api->name));
String_Const_u8 fname_cpp = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.cpp",
string_expand(path_to_self),
string_expand(api->name));
FILE *out_file_h = fopen((char*)fname_h.str, "wb");
if (out_file_h == 0){
printf("could not open output file: '%s'\n", fname_h.str);
exit(1);
}
FILE *out_file_cpp = fopen((char*)fname_cpp.str, "wb");
if (out_file_cpp == 0){
printf("could not open output file: '%s'\n", fname_cpp.str);
exit(1);
}
printf("%s:1:\n", fname_h.str);
printf("%s:1:\n", fname_cpp.str);
////////////////////////////////
// NOTE(allen): Generate output
generate_header(&arena, api, out_file_h);
generate_cpp(&arena, api, out_file_cpp);
////////////////////////////////
fclose(out_file_h);
fclose(out_file_cpp);
return(0);
}
// BOTTOM

252
4ed_api_parser.cpp Normal file
View File

@ -0,0 +1,252 @@
/*
* Mr. 4th Dimention - Allen Webster
*
* 03.10.2019
*
* Parser that extracts an API from C++ source code.
*
*/
// TOP
#include "4coder_base_types.h"
#include "4ed_api_definition.h"
#include "4coder_token.h"
#include "generated/lexer_cpp.h"
#include "4coder_base_types.cpp"
#include "4ed_api_definition.cpp"
#include "4coder_stringf.cpp"
#include "4coder_malloc_allocator.cpp"
#include "4coder_token.cpp"
#include "generated/lexer_cpp.cpp"
#include "4coder_file.cpp"
////////////////////////////////
/*
function:
api ( <identifier> ) function <identifier> {*} <identifier> ( <param_list> )
param_list:
void |
<identifier> {*} <identifier> [, <identifier> {*} <identifier>]
anything_else:
***
api_source:
{function|anything_else} EOF
*/
function b32
api_parse__match(Token_Iterator *it, Token_Cpp_Kind sub_kind){
b32 match = false;
Token *token = token_it_read(it);
if (token->sub_kind == sub_kind){
if (token_it_inc(it)){
match = true;
}
}
return(match);
}
function b32
api_parse__match_identifier(Token_Iterator *it, String_Const_u8 source, String_Const_u8 *lexeme){
b32 match = false;
Token *token = token_it_read(it);
if (token->kind == TokenBaseKind_Identifier ||
token->kind == TokenBaseKind_Keyword){
if (token_it_inc(it)){
*lexeme = string_substring(source, Ii64(token));
match = true;
}
}
return(match);
}
function b32
api_parse__match_identifier(Token_Iterator *it, String_Const_u8 source, char *lexeme){
b32 match = false;
Token *token = token_it_read(it);
if ((token->kind == TokenBaseKind_Identifier ||
token->kind == TokenBaseKind_Keyword) &&
string_match(SCu8(lexeme), string_substring(source, Ii64(token)))){
if (token_it_inc(it)){
match = true;
}
}
return(match);
}
function String_Const_u8
api_parse__type_name_with_stars(Arena *arena, String_Const_u8 type, i32 star_counter){
if (star_counter > 0){
i32 type_full_size = type.size + star_counter;
u8 *type_buffer = push_array(arena, u8, type_full_size + 1);
block_copy(type_buffer, type.str, type.size);
block_fill_u8(type_buffer + type.size, star_counter, (u8)'*');
type_buffer[type_full_size] = 0;
type = SCu8(type_buffer, type_full_size);
}
return(type);
}
function void
api_parse_add_param(Arena *arena, API_Param_List *list, String_Const_u8 type, i32 star_counter, String_Const_u8 name){
type = api_parse__type_name_with_stars(arena, type, star_counter);
API_Param *param = push_array(arena, API_Param, 1);
sll_queue_push(list->first, list->last, param);
list->count += 1;
param->type_name = type;
param->name = name;
}
function void
api_parse_add_function(Arena *arena, API_Definition_List *list, String_Const_u8 api_name, String_Const_u8 func_name, String_Const_u8 type, i32 star_counter, API_Param_List param_list){
API_Definition *api = api_get_api(arena, list, api_name);
type = api_parse__type_name_with_stars(arena, type, star_counter);
API_Call *call = api_call(arena, api, func_name, type);
api_set_param_list(call, param_list);
}
function void
api_parse_source_add_to_list(Arena *arena, String_Const_u8 source, API_Definition_List *list){
Token_List token_list = lex_full_input_cpp(arena, source);
Token_Iterator token_it = token_iterator(token_iterator(0, &token_list));
for (;;){
Token *token = token_it_read(&token_it);
if (token->sub_kind == TokenCppKind_EOF){
break;
}
if (api_parse__match_identifier(&token_it, source, "api")){
String_Const_u8 api_name = {};
String_Const_u8 ret_type = {};
i32 ret_type_star_counter = 0;
String_Const_u8 func_name = {};
API_Param_List param_list = {};
b32 success = false;
if (api_parse__match(&token_it, TokenCppKind_ParenOp)){
if (api_parse__match_identifier(&token_it, source, &api_name)){
if (api_parse__match(&token_it, TokenCppKind_ParenCl)){
if (api_parse__match_identifier(&token_it, source, "function")){
if (api_parse__match_identifier(&token_it, source, &ret_type)){
for (;api_parse__match(&token_it, TokenCppKind_Star);){
ret_type_star_counter += 1;
}
if (api_parse__match_identifier(&token_it, source, &func_name)){
if (api_parse__match(&token_it, TokenCppKind_ParenOp)){
b32 param_list_success = false;
if (api_parse__match_identifier(&token_it, source, "void")){
param_list_success = true;
}
else{
for (;;){
String_Const_u8 type = {};
i32 star_counter = 0;
String_Const_u8 name = {};
if (api_parse__match_identifier(&token_it, source, &type)){
for (;api_parse__match(&token_it, TokenCppKind_Star);){
star_counter += 1;
}
if (api_parse__match_identifier(&token_it, source, &name)){
param_list_success = true;
}
else{
break;
}
}
else{
break;
}
if (param_list_success){
api_parse_add_param(arena, &param_list, type, star_counter, name);
}
if (api_parse__match(&token_it, TokenCppKind_Comma)){
param_list_success = false;
}
else{
break;
}
}
}
if (param_list_success){
if (api_parse__match(&token_it, TokenCppKind_ParenCl)){
success = true;
}
}
}
}
}
}
}
}
}
if (success){
api_parse_add_function(arena, list, api_name, func_name, ret_type, ret_type_star_counter, param_list);
}
}
else{
if (!token_it_inc(&token_it)){
break;
}
}
}
}
function API_Definition_List
api_parse_source(Arena *arena, String_Const_u8 source){
API_Definition_List list = {};
api_parse_source_add_to_list(arena, source, &list);
return(list);
}
////////////////////////////////
int
main(int argc, char **argv){
Arena arena = make_arena_malloc();
char *hack[] = {
"nothing",
"../code/4ed_api_implementation.cpp"
};
argv = hack;
argc = 2;
if (argc < 2){
printf("usage: <script> <source> {<source>}\n"
" source : file to load and parse into the output list\n");
exit(1);
}
API_Definition_List list = {};
for (i32 i = 1; i < argc; i += 1){
FILE *file = fopen(argv[i], "rb");
if (file == 0){
printf("error: could not open input file: '%s'\n", argv[i]);
continue;
}
String_Const_u8 text = file_load_all(&arena, file);
fclose(file);
if (text.size > 0){
api_parse_source_add_to_list(&arena, text, &list);
}
}
for (API_Definition *node = list.first;
node != 0;
node = node->next){
generate_api_master_list(&arena, node, stdout);
}
}
// BOTTOM

2
4ed_font_api.cpp
View File

@ -9,7 +9,7 @@
// TOP
#include "4ed_api_definition.cpp"
#include "4ed_api_definition_main.cpp"
function API_Definition*
define_api(Arena *arena){

2
4ed_graphics_api.cpp
View File

@ -9,7 +9,7 @@
// TOP
#include "4ed_api_definition.cpp"
#include "4ed_api_definition_main.cpp"
function API_Definition*
define_api(Arena *arena){

2
4ed_system_api.cpp
View File

@ -9,7 +9,7 @@
// TOP
#include "4ed_api_definition.cpp"
#include "4ed_api_definition_main.cpp"
function API_Definition*
define_api(Arena *arena){

26
custom/4coder_file.cpp Normal file
View File

@ -0,0 +1,26 @@
/*
* Mr. 4th Dimention - Allen Webster
*
* 03.10.2019
*
* Basic helpers for C std file handling.
*
*/
// TOP
#include <stdio.h>
function String_Const_u8
file_load_all(Arena *arena, FILE *file){
fseek(file, 0, SEEK_END);
umem size = ftell(file);
fseek(file, 0, SEEK_SET);
u8 *buffer = push_array(arena, u8, size + 1);
fread(buffer, 1, size, file);
buffer[size] = 0;
return(SCu8(buffer, size));
}
// BOTTOM

5
custom/4coder_token.cpp
View File

@ -4,6 +4,11 @@
// TOP
function Interval_i64
Ii64(Token *token){
return(Ii64_size(token->pos, token->size));
}
internal void
token_list_push(Arena *arena, Token_List *list, Token *token){
Token_Block *block = list->last;

8
custom/generated/command_metadata.h
View File

@ -387,10 +387,10 @@ static Command_Metadata fcoder_metacmd_table[226] = {
{ PROC_LINKS(interactive_new, 0), "interactive_new", 15, "Interactively creates a new file.", 33, "w:\\4ed\\code\\custom\\4coder_lists.cpp", 35, 847 },
{ PROC_LINKS(interactive_open, 0), "interactive_open", 16, "Interactively opens a file.", 27, "w:\\4ed\\code\\custom\\4coder_lists.cpp", 35, 880 },
{ PROC_LINKS(command_lister, 0), "command_lister", 14, "Opens an interactive list of all registered commands.", 53, "w:\\4ed\\code\\custom\\4coder_lists.cpp", 35, 960 },
{ PROC_LINKS(auto_tab_whole_file, 0), "auto_tab_whole_file", 19, "Audo-indents the entire current buffer.", 39, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 655 },
{ PROC_LINKS(auto_tab_line_at_cursor, 0), "auto_tab_line_at_cursor", 23, "Auto-indents the line on which the cursor sits.", 47, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 664 },
{ PROC_LINKS(auto_tab_range, 0), "auto_tab_range", 14, "Auto-indents the range between the cursor and the mark.", 55, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 674 },
{ PROC_LINKS(write_and_auto_tab, 0), "write_and_auto_tab", 18, "Inserts a character and auto-indents the line on which the cursor sits.", 71, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 684 },
{ PROC_LINKS(auto_tab_whole_file, 0), "auto_tab_whole_file", 19, "Audo-indents the entire current buffer.", 39, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 309 },
{ PROC_LINKS(auto_tab_line_at_cursor, 0), "auto_tab_line_at_cursor", 23, "Auto-indents the line on which the cursor sits.", 47, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 318 },
{ PROC_LINKS(auto_tab_range, 0), "auto_tab_range", 14, "Auto-indents the range between the cursor and the mark.", 55, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 328 },
{ PROC_LINKS(write_and_auto_tab, 0), "write_and_auto_tab", 18, "Inserts a character and auto-indents the line on which the cursor sits.", 71, "w:\\4ed\\code\\custom\\4coder_auto_indent.cpp", 41, 338 },
{ PROC_LINKS(list_all_locations, 0), "list_all_locations", 18, "Queries the user for a string and lists all exact case-sensitive matches found in all open buffers.", 99, "w:\\4ed\\code\\custom\\4coder_search.cpp", 36, 166 },
{ PROC_LINKS(list_all_substring_locations, 0), "list_all_substring_locations", 28, "Queries the user for a string and lists all case-sensitive substring matches found in all open buffers.", 103, "w:\\4ed\\code\\custom\\4coder_search.cpp", 36, 172 },
{ PROC_LINKS(list_all_locations_case_insensitive, 0), "list_all_locations_case_insensitive", 35, "Queries the user for a string and lists all exact case-insensitive matches found in all open buffers.", 101, "w:\\4ed\\code\\custom\\4coder_search.cpp", 36, 178 },

6
project.4coder
View File

@ -56,10 +56,14 @@ command_list = {
{ .name = "build graphics api",
.out = "*compilation*", .footer_panel = true, .save_dirty_files = true,
.cmd = { { "custom\\bin\\build_one_time 4ed_graphics_api.cpp ..\\build", .os = "win" }, }, },
{ .name = "build api parser",
.out = "*compilation*", .footer_panel = true, .save_dirty_files = true,
.cmd = { { "custom\\bin\\build_one_time 4ed_api_parser.cpp ..\\build", .os = "win" }, }, },
};
fkey_command[1] = "build x64";
fkey_command[2] = "build graphics api";
fkey_command[2] = "build api parser";
fkey_command[4] = "run one time";
fkey_command[5] = "build C++ lexer generator";
fkey_command[6] = "build token tester";