Add memory-aware concurrent indexing for large repositories

internal/cbm/arena.c

@@ -9,6 +9,7 @@ void cbm_arena_init(CBMArena *a) {
     a->block_size = CBM_ARENA_DEFAULT_BLOCK_SIZE;
     a->blocks[0] = (char *)malloc(a->block_size);
     if (a->blocks[0]) {
+        a->block_sizes[0] = a->block_size;
         a->nblocks = SKIP_ONE;
     }
 }
@@ -26,6 +27,7 @@ static int arena_grow(CBMArena *a, size_t min_size) {
         return 0;
     }
     a->blocks[a->nblocks] = block;
+    a->block_sizes[a->nblocks] = new_size;
     a->nblocks++;
     a->block_size = new_size;
     a->used = 0;
@@ -39,6 +41,16 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) {
     // 8-byte alignment
     n = (n + 7) & ~(size_t)7;
 
+    if (n <= 64) {
+        a->alloc_le_64 += n;
+    } else if (n <= 256) {
+        a->alloc_le_256 += n;
+    } else if (n <= 4096) {
+        a->alloc_le_4096 += n;
+    } else {
+        a->alloc_gt_4096 += n;
+    }
+
     if (a->nblocks == 0) {
         return NULL;
     }
@@ -51,6 +63,7 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) {
 
     char *ptr = a->blocks[a->nblocks - SKIP_ONE] + a->used;
     a->used += n;
+    a->total_alloc += n;
     return ptr;
 }
 
@@ -60,6 +73,7 @@ char *cbm_arena_strdup(CBMArena *a, const char *s) {
     size_t len = strlen(s);
     char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE);
     if (dst) {
+        a->strdup_alloc += len + SKIP_ONE;
         memcpy(dst, s, len + SKIP_ONE);
     }
     return dst;
@@ -70,6 +84,7 @@ char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len) {
         return NULL;
     char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE);
     if (dst) {
+        a->strdup_alloc += len + SKIP_ONE;
         memcpy(dst, s, len);
         dst[len] = '\0';
     }
@@ -91,6 +106,7 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) {
     if (!dst) {
         return NULL;
     }
+    a->sprintf_alloc += (size_t)needed + SKIP_ONE;
 
     va_start(args, fmt);
     vsnprintf(dst, (size_t)needed + SKIP_ONE, fmt, args);

internal/cbm/arena.h

@@ -1,41 +1,13 @@
-#ifndef CBM_ARENA_H
-#define CBM_ARENA_H
-
-#include <stddef.h>
-
-// CBMArena is a simple bump allocator that allocates from fixed-size blocks.
-// All memory is freed at once via cbm_arena_destroy(). Individual frees are not
-// supported — this is by design for per-file extraction where all data has the
-// same lifetime.
-#define CBM_ARENA_MAX_BLOCKS 256
-#define CBM_ARENA_DEFAULT_BLOCK_SIZE (64 * 1024) // 64KB initial
-
-typedef struct {
-    char *blocks[CBM_ARENA_MAX_BLOCKS];
-    size_t block_sizes[CBM_ARENA_MAX_BLOCKS]; // per-block sizes (for stats)
-    int nblocks;
-    size_t block_size;
-    size_t used;        // bytes used in current block
-    size_t total_alloc; // cumulative bytes allocated (for stats)
-} CBMArena;
-
-// Initialize an arena with the default block size.
-void cbm_arena_init(CBMArena *a);
-
-// Allocate n bytes from the arena. Returns NULL on OOM or block exhaustion.
-// All returned pointers are 8-byte aligned.
-void *cbm_arena_alloc(CBMArena *a, size_t n);
-
-// Duplicate a string into arena memory. Returns arena-owned copy.
-char *cbm_arena_strdup(CBMArena *a, const char *s);
-
-// Duplicate a string of known length into arena memory. NUL-terminates.
-char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len);
-
-// sprintf into arena memory. Returns arena-owned string.
-char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
-
-// Free all blocks. Arena is invalid after this call.
-void cbm_arena_destroy(CBMArena *a);
-
-#endif // CBM_ARENA_H
+/*
+ * Extraction compatibility include.
+ *
+ * CBMArena used to be duplicated here and in src/foundation/arena.h. The
+ * production binary links the foundation implementation, so any field drift
+ * between those definitions is an ABI violation. Keep one canonical layout.
+ */
+#ifndef CBM_EXTRACTION_ARENA_COMPAT_H
+#define CBM_EXTRACTION_ARENA_COMPAT_H
+
+#include "../../src/foundation/arena.h"
+
+#endif /* CBM_EXTRACTION_ARENA_COMPAT_H */

internal/cbm/cbm.c

@@ -103,19 +103,18 @@ void cbm_reset_profile(void) {
 
 // --- Growable array push functions ---
 
-#define GROW_ARRAY(arr, arena)                                                                   \
-    do {                                                                                         \
-        if ((arr)->count >= (arr)->cap) {                                                        \
-            int new_cap = (arr)->cap == 0 ? CBM_SZ_32 : (arr)->cap * PAIR_LEN;                   \
-            void *new_items = cbm_arena_alloc((arena), (size_t)new_cap * sizeof(*(arr)->items)); \
-            if (!new_items)                                                                      \
-                return;                                                                          \
-            if ((arr)->items && (arr)->count > 0) {                                              \
-                memcpy(new_items, (arr)->items, (size_t)(arr)->count * sizeof(*(arr)->items));   \
-            }                                                                                    \
-            (arr)->items = new_items;                                                            \
-            (arr)->cap = new_cap;                                                                \
-        }                                                                                        \
+#define GROW_ARRAY(arr, arena)                                                                     \
+    do {                                                                                           \
+        (void)(arena);                                                                             \
+        if ((arr)->count >= (arr)->cap) {                                                          \
+            int new_cap = (arr)->cap == 0 ? CBM_SZ_32 : (arr)->cap * PAIR_LEN;                     \
+            void *new_items =                                                                      \
+                cbm_arena_realloc((arena), (arr)->items, (size_t)new_cap * sizeof(*(arr)->items)); \
+            if (!new_items)                                                                        \
+                return;                                                                            \
+            (arr)->items = new_items;                                                              \
+            (arr)->cap = new_cap;                                                                  \
+        }                                                                                          \
     } while (0)
 
 void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def) {

internal/cbm/extract_node_stack.h

@@ -4,10 +4,12 @@
  * Replaces fixed-size TSNode stack[] arrays that silently drop AST subtrees
  * when the stack overflows (GitHub issue #199).
  *
- * Uses the arena allocator for zero-fragmentation growth: old blocks are
- * abandoned (freed when the arena is destroyed at end of file extraction).
- * Initial capacity matches the previous fixed caps so small files allocate
- * no extra memory.
+ * The common capacity lives inline in the traversal's own stack frame. Only an
+ * unusually broad AST spills into the result arena. Call-site capacities
+ * historically mirrored fixed stack limits (usually 512, sometimes 4096), so
+ * allocating every traversal there made temporary work survive with durable
+ * extraction results across the whole repository. Inline storage preserves the
+ * same traversal order and geometric growth without that lifetime inversion.
  */
 #ifndef CBM_EXTRACT_NODE_STACK_H
 #define CBM_EXTRACT_NODE_STACK_H
@@ -20,19 +22,24 @@ typedef struct {
     TSNode *items;
     int count;
     int cap;
+    TSNode inline_items[128];
 } TSNodeStack;
 
-/* Initialize a stack with the given initial capacity, arena-allocated. */
+enum { TS_NSTACK_INLINE_CAP = 128 };
+
+/* Initialize with inline storage; arena is used only if the stack spills. */
 static inline void ts_nstack_init(TSNodeStack *s, CBMArena *arena, int initial_cap) {
-    s->items = (TSNode *)cbm_arena_alloc(arena, (size_t)initial_cap * sizeof(TSNode));
+    (void)arena;
+    (void)initial_cap;
+    s->items = s->inline_items;
     s->count = 0;
-    s->cap = s->items ? initial_cap : 0;
+    s->cap = TS_NSTACK_INLINE_CAP;
 }
 
 /* Push a node onto the stack, growing 2x if needed. */
 static inline void ts_nstack_push(TSNodeStack *s, CBMArena *arena, TSNode node) {
     if (s->count >= s->cap) {
-        int new_cap = s->cap ? s->cap * 2 : 512;
+        int new_cap = s->cap ? s->cap * 2 : TS_NSTACK_INLINE_CAP;
         TSNode *new_items = (TSNode *)cbm_arena_alloc(arena, (size_t)new_cap * sizeof(TSNode));
         if (!new_items)
             return; /* OOM: best-effort, stop growing */

internal/cbm/lsp/kotlin_lsp.c

@@ -412,28 +412,13 @@ static void kt_emit_resolved_kind(KotlinLSPContext *ctx, const char *callee_qn,
         return;
     }
 
-    CBMResolvedCallArray *arr = ctx->resolved_calls;
-    if (arr->count >= arr->cap) {
-        int new_cap = arr->cap == 0 ? 16 : arr->cap * 2;
-        CBMResolvedCall *new_items = (CBMResolvedCall *)cbm_arena_alloc(
-            ctx->arena, (size_t)new_cap * sizeof(CBMResolvedCall));
-        if (!new_items) {
-            return;
-        }
-        if (arr->items && arr->count > 0) {
-            memcpy(new_items, arr->items, (size_t)arr->count * sizeof(CBMResolvedCall));
-        }
-        arr->items = new_items;
-        arr->cap = new_cap;
-    }
-    CBMResolvedCall *rc = &arr->items[arr->count];
-    memset(rc, 0, sizeof(CBMResolvedCall));
-    rc->caller_qn = ctx->enclosing_func_qn;
-    rc->callee_qn = cbm_arena_strdup(ctx->arena, callee_qn);
-    rc->strategy = strategy;
-    rc->confidence = confidence;
-    rc->kind = kind;
-    arr->count++;
+    CBMResolvedCall rc = {0};
+    rc.caller_qn = ctx->enclosing_func_qn;
+    rc.callee_qn = cbm_arena_strdup(ctx->arena, callee_qn);
+    rc.strategy = strategy;
+    rc.confidence = confidence;
+    rc.kind = kind;
+    cbm_resolvedcall_push(ctx->resolved_calls, ctx->arena, rc);
 }
 
 static void kt_stamp_resolved_site(KotlinLSPContext *ctx, int first, TSNode site) {
@@ -3177,8 +3162,7 @@ static const CBMType *kt_eval_navigation_expression_type_at(KotlinLSPContext *ct
             if (is_member) {
                 /* A member call on an `object`/`companion object` singleton is a
                  * static dispatch; on a regular class instance it is a method. */
-                const CBMRegisteredType *recv_rt =
-                    cbm_registry_lookup_type(ctx->registry, recv_qn);
+                const CBMRegisteredType *recv_rt = cbm_registry_lookup_type(ctx->registry, recv_qn);
                 strat = (recv_rt && recv_rt->is_object) ? "lsp_kt_static" : "lsp_kt_method";
             }
             /* A call through the lambda implicit parameter `it` (e.g. inside
@@ -5255,8 +5239,7 @@ static void kt_register_cross_def(CBMTypeRegistry *reg, CBMArena *arena, const C
         rt.is_interface = (strcmp(d->label, "Interface") == 0) || d->is_interface;
         if (field_map) {
             const KtCrossFieldList *fl =
-                (const KtCrossFieldList *)cbm_ht_get((CBMHashTable *)field_map,
-                                                     d->qualified_name);
+                (const KtCrossFieldList *)cbm_ht_get((CBMHashTable *)field_map, d->qualified_name);
             if (fl && fl->count > 0) {
                 rt.field_names = fl->names;
                 rt.field_types = fl->types;