Implemented tree_sitter_code_index_update_tick - populates Code_Index_Nests and Code_Index_Notes from each languages Tags query

Switched tree_sitter_list_all_query_results to list Code_Index_Note's matching a Code_Index_Note_Kind in each file
- remove old query types NESTS, FUNCTIONS, TYPES

build_new/scripts/build-libs.sh

@@ -15,6 +15,42 @@ source "$HELPERS_DIR/print-routines.sh"
 # Build Tree Sitter
 # =============================================================================
 
+build_tree_sitter_language() {
+  LANG_DIR=$1
+  LANG_NAME=$2
+
+  PARSER_SRC="${CUSTOM_ROOT}/lang/$LANG_DIR/parser.c"
+  PARSER_OUT="$TEMP_OUT_DIR/${LANG_DIR}_parser.o"
+
+  SCANNER_SRC="${CUSTOM_ROOT}/lang/$LANG_DIR/scanner.cc"
+  SCANNER_OUT="$TEMP_OUT_DIR/${LANG_DIR}_scanner.o"
+
+  print_step "Building tree-sitter $LANG_NAME Language Lib ($LANG_DIR)"
+  BUILT_ANYTHING=0
+
+  if [ -f $PARSER_SRC ]; then
+    echo "  Building Parser..."
+    clang $CLANG_OPTS "${INCLUDES[@]}" "$PARSER_SRC"  -o "$PARSER_OUT"
+    if [ $? == 0 ]; then
+      BUILT_ANYTHING=1
+    fi
+  fi
+
+  if [ -f $SCANNER_SRC ]; then
+    echo "  Building Scanner..."
+    clang $CLANG_OPTS "${INCLUDES[@]}" "$SCANNER_SRC" -o "$SCANNER_OUT"
+    if [ $? == 0 ]; then
+      BUILT_ANYTHING=1
+    fi
+  fi
+
+  if [ $BUILT_ANYTHING == 1 ]; then
+    print_success "Complete"
+  else
+    print_warning "Failed to build anything."
+  fi
+}
+
 build_tree_sitter() {
   BIN_NAME="custom_4coder"
 
@@ -32,23 +68,17 @@ build_tree_sitter() {
 
   TEMP_OUT_DIR=$BUILD_TEMP_DIR/tree-sitter
   mkdir -p $TEMP_OUT_DIR
+  rm $TEMP_OUT_DIR/*.o
+  rm "$BUILD_TEMP_DIR/tree-sitter.a"
 
   # Build tree-sitter.lib/.a
   print_step "Building tree-sitter lib"
   clang $CLANG_OPTS "${INCLUDES[@]}" "$CUSTOM_ROOT/lib/src/lib.c"             -o $TEMP_OUT_DIR/tree-sitter.o
   print_success "Complete"
 
-  # Lang: C++ (This needs to be two calls to clang so that you can specify the obj file names)
+  build_tree_sitter_language "cpp" "C++"
-  print_step "Building tree-sitter C++ Language Lib"
+  build_tree_sitter_language "jai" "Jai"
-  clang $CLANG_OPTS "${INCLUDES[@]}" "${CUSTOM_ROOT}/lang/cpp/parser.c"   -o $TEMP_OUT_DIR/cpp_parser.o
+  build_tree_sitter_language "bash" "Bash"
-  clang $CLANG_OPTS "${INCLUDES[@]}" "${CUSTOM_ROOT}/lang/cpp/scanner.cc" -o $TEMP_OUT_DIR/cpp_scanner.o
-  print_success "Complete"
-
-  # Lang: Jai
-  print_step "Building tree-sitter Jai Language Lib"
-  clang $CLANG_OPTS "${INCLUDES[@]}" "${CUSTOM_ROOT}/lang/jai/parser.c"   -o $TEMP_OUT_DIR/jai_parser.o
-  clang $CLANG_OPTS "${INCLUDES[@]}" "${CUSTOM_ROOT}/lang/jai/scanner.cc" -o $TEMP_OUT_DIR/jai_scanner.o
-  print_success "Complete"
 
   # Link tree-sitter lib and parser obj files into a static library to link into main custom dll
   print_step "Linking tree-sitter static library"

code/4ed_api_definition.cpp

@@ -505,10 +505,12 @@ api_definition_generate_api_includes(Arena *arena, API_Definition *api, Generate
         return(false);
     }
 
+    #if 0
     printf("%s:1:\n", fname_ml.str);
     printf("%s:1:\n", fname_h.str);
     printf("%s:1:\n", fname_cpp.str);
     printf("%s:1:\n", fname_con.str);
+    #endif
 
     ////////////////////////////////
 

code/4ed_generate_keycodes.cpp

@@ -70,7 +70,7 @@ function Event_Code*
 add_code(Arena *arena, Event_Code_List *list, String_Const_u8 name){
     Event_Code *code = push_array(arena, Event_Code, 1);
     sll_queue_push(list->first, list->last, code);
-    list->count;
+    list->count += 1;
     code->name = push_string_copy(arena, name);
     return(code);
 }

code/custom/4coder_tree_sitter.h

@@ -8,9 +8,7 @@
 enum Tree_Sitter_Language_Query_Kind
 {
   Tree_Sitter_Language_Query_Highlights,
-  Tree_Sitter_Language_Query_Functions,
+  Tree_Sitter_Language_Query_Tags,
-  Tree_Sitter_Language_Query_Types,
-
   Tree_Sitter_Language_Query_Count,
 };
 
@@ -38,10 +36,6 @@ struct Tree_Sitter_Languages
 
 global Tree_Sitter_Languages tree_sitter_languages;
 
-extern "C" {
-  TSLanguage *tree_sitter_cpp();
-  TSLanguage *tree_sitter_jai();
-}
 
 CUSTOM_ID(attachment, buffer_tree_sitter_data_id);
 CUSTOM_ID(attachment, buffer_tree_sitter_parse_task_id);
@@ -69,6 +63,9 @@ struct Tree_Sitter_Code_Index_Nest_List
   Tree_Sitter_Code_Index_Nest_Node* last;
 };
 
+function TSQuery* tree_sitter_query_new(Application_Links* app, TSLanguage* language, String_Const_u8 query_string);
+function void tree_sitter_register_language(String_Const_u8 ext, TSLanguage* language, Tree_Sitter_Language_Queries queries);
+
 b8 use_tree_sitter_code_indexing = true;
 b8 use_tree_sitter_token_coloring = true;
 function void tree_sitter_code_index_update_tick(Application_Links *app);

code/custom/bin/build_one_time.sh

@@ -35,8 +35,8 @@ debug=-g
 
 opts="-Wno-macro-redefined -Wno-write-strings -Wno-null-dereference -Wno-comment -Wno-switch -Wno-missing-declarations -Wno-logical-op-parentheses -g -DOS_LINUX=1 -DOS_WINDOWS=0 -DOS_MAC=0"
 
-pushd $dst
+pushd $dst > /dev/null
 g++ -I"$CUSTOM_ROOT" $opts $full_target -o one_time
-popd
+popd > /dev/null
 
 

code/custom/languages/tree_sitter_bash.h

@@ -0,0 +1,80 @@
+/* date = July 11th 2025 6:02 pm */
+
+#ifndef TREE_SITTER_BASH_H
+#define TREE_SITTER_BASH_H
+
+// Source: https://github.com/tree-sitter/tree-sitter-bash/blob/master/queries/highlights.scm
+String_Const_u8 TS_BASH_HIGHLIGHT_QUERY = string_u8_litexpr(R"DONE(
+[
+  (string)
+  (raw_string)
+  (heredoc_body)
+  (heredoc_start)
+] @string
+
+(command_name) @function.call
+
+(variable_name) @property
+
+[
+  "case"
+  "do"
+  "done"
+  "elif"
+  "else"
+  "esac"
+  "export"
+  "fi"
+  "for"
+  "function"
+  "if"
+  "in"
+  "select"
+  "then"
+  "unset"
+  "until"
+  "while"
+] @keyword
+
+(comment) @comment
+
+(function_definition name: (word) @function)
+
+(file_descriptor) @number
+
+[
+  (command_substitution)
+  (process_substitution)
+  (expansion)
+] @embedded
+
+[
+  "$"
+  "&&"
+  ">"
+  ">>"
+  "<"
+  "|"
+] @operator
+
+(
+  (command (_) @constant)
+  (#match? @constant "^-")
+)
+
+)DONE");
+
+extern "C" {
+  TSLanguage* tree_sitter_bash();
+}
+
+void
+tree_sitter_register_bash(Application_Links* app)
+{
+  TSLanguage* language = tree_sitter_bash();
+  Tree_Sitter_Language_Queries queries = {};
+  queries.ptr[Tree_Sitter_Language_Query_Highlights] = tree_sitter_query_new(app, language, TS_BASH_HIGHLIGHT_QUERY);
+  tree_sitter_register_language(SCu8("sh"), language, queries);
+}
+
+#endif //TREE_SITTER_BASH_H

code/custom/languages/tree_sitter_cpp.h

@@ -0,0 +1,211 @@
+/* date = July 11th 2025 6:04 pm */
+
+#ifndef TREE_SITTER_CPP_H
+#define TREE_SITTER_CPP_H
+
+// Source: https://github.com/tree-sitter/tree-sitter-cpp/blob/master/queries/tags.scm
+String_Const_u8 TS_CPP_TAGS_QUERY_SCM = string_u8_litexpr(R"DONE(
+(struct_specifier name: (type_identifier) @name body:(_)) @definition.class
+
+(declaration type: (union_specifier name: (type_identifier) @name)) @definition.class
+
+(function_declarator declarator: (identifier) @name) @definition.function
+
+(function_declarator declarator: (field_identifier) @name) @definition.function
+
+(function_declarator declarator: (qualified_identifier scope: (namespace_identifier) @local.scope name: (identifier) @name)) @definition.method
+
+(type_definition declarator: (type_identifier) @name) @definition.type
+
+(enum_specifier name: (type_identifier) @name) @definition.type
+
+(class_specifier name: (type_identifier) @name) @definition.class
+
+(_ "{" @scope_begin "}" @scope_end )
+(_ "(" @scope_begin ")" @scope_end )
+(_ "[" @scope_begin "]" @scope_end )
+
+)DONE");
+
+// Source: https://github.com/tree-sitter/tree-sitter-cpp/blob/master/queries/highlights.scm
+String_Const_u8 TS_CPP_HIGHLIGHT_QUERY_SCM = string_u8_litexpr(R"DONE(
+(identifier) @variable
+
+((identifier) @constant
+ (#match? @constant "^[A-Z][A-Z\\d_]*$"))
+
+"break" @keyword
+"case" @keyword
+"const" @keyword
+"continue" @keyword
+"default" @keyword
+"do" @keyword
+"else" @keyword
+"enum" @keyword
+"extern" @keyword
+"for" @keyword
+"if" @keyword
+"inline" @keyword
+"return" @keyword
+"sizeof" @keyword
+"static" @keyword
+"struct" @keyword
+"switch" @keyword
+"typedef" @keyword
+"union" @keyword
+"volatile" @keyword
+"while" @keyword
+
+"#define" @keyword
+"#elif" @keyword
+"#else" @keyword
+"#endif" @keyword
+"#if" @keyword
+"#ifdef" @keyword
+"#ifndef" @keyword
+"#include" @keyword
+(preproc_directive) @keyword
+
+"--" @operator
+"-" @operator
+"-=" @operator
+"->" @operator
+"=" @operator
+"!=" @operator
+"*" @operator
+"&" @operator
+"&&" @operator
+"+" @operator
+"++" @operator
+"+=" @operator
+"<" @operator
+"==" @operator
+">" @operator
+"||" @operator
+
+"." @delimiter
+";" @delimiter
+
+(string_literal) @string
+(system_lib_string) @string
+
+(null) @constant
+(number_literal) @number
+(char_literal) @number
+
+(field_identifier) @property
+(statement_identifier) @label
+(type_identifier) @type
+(primitive_type) @type
+(sized_type_specifier) @type
+
+(call_expression
+  function: (identifier) @function)
+(call_expression
+  function: (field_expression
+    field: (field_identifier) @function))
+(function_declarator
+  declarator: (identifier) @function)
+(preproc_function_def
+  name: (identifier) @function.special)
+
+(comment) @comment
+
+; Functions
+
+(call_expression
+  function: (qualified_identifier
+    name: (identifier) @function))
+
+(template_function
+  name: (identifier) @function)
+
+(template_method
+  name: (field_identifier) @function)
+
+(template_function
+  name: (identifier) @function)
+
+(function_declarator
+  declarator: (qualified_identifier
+    name: (identifier) @function))
+
+(function_declarator
+  declarator: (field_identifier) @function)
+
+; Types
+
+((namespace_identifier) @type
+ (#match? @type "^[A-Z]"))
+
+(auto) @type
+
+; Constants
+
+(this) @variable.builtin
+(null "nullptr" @constant)
+
+; Modules
+(module_name
+  (identifier) @module)
+
+; Keywords
+
+[
+ "catch"
+ "class"
+ "co_await"
+ "co_return"
+ "co_yield"
+ "constexpr"
+ "constinit"
+ "consteval"
+ "delete"
+ "explicit"
+ "final"
+ "friend"
+ "mutable"
+ "namespace"
+ "noexcept"
+ "new"
+ "override"
+ "private"
+ "protected"
+ "public"
+ "template"
+ "throw"
+ "try"
+ "typename"
+ "using"
+ "concept"
+ "requires"
+ "virtual"
+ "import"
+ "export"
+ "module"
+] @keyword
+
+; Strings
+
+(raw_string_literal) @string
+)DONE");
+
+extern "C" {
+  TSLanguage *tree_sitter_cpp();
+}
+
+void
+tree_sitter_register_cpp(Application_Links* app)
+{
+  TSLanguage* language = tree_sitter_cpp();
+  Tree_Sitter_Language_Queries queries = {};
+  queries.ptr[Tree_Sitter_Language_Query_Highlights] = tree_sitter_query_new(app, language, TS_CPP_HIGHLIGHT_QUERY_SCM);
+  queries.ptr[Tree_Sitter_Language_Query_Tags]       = tree_sitter_query_new(app, language, TS_CPP_TAGS_QUERY_SCM);
+  tree_sitter_register_language(SCu8("c"),   language, queries);
+  tree_sitter_register_language(SCu8("cpp"), language, queries);
+  tree_sitter_register_language(SCu8("h"),   language, queries);
+  tree_sitter_register_language(SCu8("hpp"), language, queries);
+  tree_sitter_register_language(SCu8("cc"),  language, queries);
+}
+
+#endif //TREE_SITTER_CPP_H

code/custom/languages/tree_sitter_jai.h

@@ -0,0 +1,298 @@
+/* date = July 11th 2025 6:03 pm */
+
+#ifndef TREE_SITTER_JAI_H
+#define TREE_SITTER_JAI_H
+
+String_Const_u8 TS_JAI_TAGS_QUERY = string_u8_litexpr(R"DONE(
+
+(procedure_declaration
+  name: (identifier) @definition.function
+)
+
+(struct_declaration
+  name: (identifier) @definition.type
+)
+
+(enum_declaration
+  name: (identifier) @definition.type
+)
+
+)DONE");
+
+// NOTE(PS): source: https://github.com/St0wy/tree-sitter-jai/blob/main/queries/highlights.scm
+String_Const_u8 TS_JAI_HIGHLIGHT_QUERY = string_u8_litexpr(R"DONE(
+; Includes
+
+[
+  (import)
+  (load)
+] @include
+
+; Keywords
+[
+  ; from modules/Jai_Lexer
+  "if"
+  "xx"
+
+  "ifx"
+  "for"
+
+  "then"
+  "else"
+  "null"
+  "case"
+  "enum"
+  "true"
+  "cast"
+
+  "while"
+  "break"
+  "using"
+  "defer"
+  "false"
+  "union"
+
+  "return"
+  "struct"
+  "inline"
+  "remove"
+
+  ; "size_of"
+  "type_of"
+  ; "code_of"
+  ; "context"
+
+  "continue"
+  "operator"
+
+  ; "type_info"
+  "no_inline"
+  "interface"
+
+  "enum_flags"
+
+  ; "is_constant"
+
+  "push_context"
+
+  ; "initializer_of"
+] @keyword
+
+[
+  "return"
+] @keyword.return
+
+[
+  "if"
+  "else"
+  "case"
+  "break"
+] @keyword.conditional
+
+((if_expression
+  [
+    "then"
+    "ifx"
+    "else"
+  ] @keyword.conditional.ternary)
+  (#set! "priority" 105))
+
+; Repeats
+
+[
+  "for"
+  "while"
+  "continue"
+] @keyword.repeat
+
+; Variables
+
+; (identifier) @variable
+name: (identifier) @variable
+argument: (identifier) @variable
+named_argument: (identifier) @variable
+(member_expression (identifier) @variable)
+(parenthesized_expression (identifier) @variable)
+
+((identifier) @variable.builtin
+  (#any-of? @variable.builtin "context"))
+
+; Namespaces
+
+(import (identifier) @namespace)
+
+; Parameters
+
+(parameter (identifier) @parameter ":" "="? (identifier)? @constant)
+
+; (call_expression argument: (identifier) @parameter "=")
+
+; Functions
+
+; (procedure_declaration (identifier) @function (procedure (block)))
+(procedure_declaration (identifier) @function (block))
+
+(call_expression function: (identifier) @function.call)
+
+; Types
+
+type: (types) @type
+type: (identifier) @type
+((types) @type)
+
+modifier: (identifier) @keyword
+keyword: (identifier) @keyword
+
+((types (identifier) @type.builtin)
+  (#any-of? @type.builtin
+    "bool" "int" "string"
+    "s8" "s16" "s32" "s64"
+    "u8" "u16" "u32" "u64"
+    "Type" "Any"))
+
+(struct_declaration (identifier) @type ":" ":")
+
+(enum_declaration (identifier) @type ":" ":")
+
+; (const_declaration (identifier) @type ":" ":" [(array_type) (pointer_type)])
+
+; ; I don't like this
+; ((identifier) @type
+;   (#lua-match? @type "^[A-Z][a-zA-Z0-9]*$")
+;   (#not-has-parent? @type parameter procedure_declaration call_expression))
+
+; Fields
+
+(member_expression "." (identifier) @field)
+
+(assignment_statement (identifier) @field "="?)
+(update_statement (identifier) @field)
+
+; Constants
+
+((identifier) @constant
+  (#lua-match? @constant "^_*[A-Z][A-Z0-9_]*$")
+  (#not-has-parent? @constant type parameter))
+
+(member_expression . "." (identifier) @constant)
+
+(enum_declaration "{" (identifier) @constant)
+
+; Literals
+
+(integer) @number
+(float) @number
+
+(string) @string
+
+;(character) @character
+
+(string (escape_sequence) @string.escape)
+
+(boolean) @boolean
+
+[
+  (uninitialized)
+  (null)
+] @constant.builtin
+
+; Operators
+
+[
+  ":"
+  "="
+  "+"
+  "-"
+  "*"
+  "/"
+  "%"
+  ">"
+  ">="
+  "<"
+  "<="
+  "=="
+  "!="
+  "|"
+  "~"
+  "&"
+  "&~"
+  "<<"
+  ">>"
+  "<<<"
+  ">>>"
+  "||"
+  "&&"
+  "!"
+  ".."
+  "+="
+  "-="
+  "*="
+  "/="
+  "%="
+  "&="
+  "|="
+  "^="
+  "<<="
+  ">>="
+  "<<<="
+  ">>>="
+  "||="
+  "&&="
+] @operator
+
+; Punctuation
+
+[ "{" "}" ] @punctuation.bracket
+
+[ "(" ")" ] @punctuation.bracket
+
+[ "[" "]" ] @punctuation.bracket
+
+[
+  "`"
+  "->"
+  "."
+  ","
+  ":"
+  ";"
+] @punctuation.delimiter
+
+; Comments
+
+[
+  (block_comment)
+  (comment)
+] @comment @spell
+
+; Errors
+
+(ERROR) @error
+
+(block_comment) @comment
+
+directive: ("#") @keyword ; #if
+type: ("type_of") @type
+
+(compiler_directive) @keyword
+(heredoc_start) @none
+(heredoc_end) @none
+(heredoc_body) @string
+(note) @string
+
+)DONE");
+
+extern "C" {
+  TSLanguage *tree_sitter_jai();
+}
+
+void
+tree_sitter_register_jai (Application_Links* app)
+{
+  TSLanguage* language = tree_sitter_jai();
+  Tree_Sitter_Language_Queries queries = {};
+  queries.ptr[Tree_Sitter_Language_Query_Highlights] = tree_sitter_query_new(app, language, TS_JAI_HIGHLIGHT_QUERY);
+  queries.ptr[Tree_Sitter_Language_Query_Tags]       = tree_sitter_query_new(app, language, TS_JAI_TAGS_QUERY);
+  tree_sitter_register_language(SCu8("jai"), language, queries);
+}
+
+#endif //TREE_SITTER_JAI_H

code/custom/lexer_generator/4coder_lex_gen_main.cpp

@@ -732,8 +732,8 @@ smi_field_set_intersect(Arena *arena, Field_Set a, Field_Set b){
                 for (Field_Pin *pin = new_list->first;
                      pin != 0;
                      pin = pin->next){
-                    if (pin->flag == pin->flag){
+                    if (pin->flag == b_pin->flag){
-                        if (pin->value != pin->value){
+                        if (pin->value != b_pin->value){
                             end_temp(restore_point);
                             has_conflicts = true;
                             goto double_break;
@@ -4036,8 +4036,8 @@ int main(void){
     fclose(out_h_file);
     fclose(out_cpp_file);
 
-    printf("%.*s:1:\n", string_expand(out_h_name));
+    //printf("%.*s:1:\n", string_expand(out_h_name));
-    printf("%.*s:1:\n", string_expand(out_cpp_name));
+    //printf("%.*s:1:\n", string_expand(out_cpp_name));
 
     // NOTE(allen): Simplifying the state machine
     // Isolate the state machine's parts into small L.U.T. then generate tables?

project.4coder

@@ -12,6 +12,7 @@ patterns = {
 };
 blacklist_patterns = {
   ".*",
+  "non-source",
 };
 load_paths_base = {
   { ".", .relative = true, .recursive = true, },