diff --git a/internal/cbm/arena.c b/internal/cbm/arena.c index cb8690963..c09aac6f4 100644 --- a/internal/cbm/arena.c +++ b/internal/cbm/arena.c @@ -9,6 +9,7 @@ void cbm_arena_init(CBMArena *a) { a->block_size = CBM_ARENA_DEFAULT_BLOCK_SIZE; a->blocks[0] = (char *)malloc(a->block_size); if (a->blocks[0]) { + a->block_sizes[0] = a->block_size; a->nblocks = SKIP_ONE; } } @@ -26,6 +27,7 @@ static int arena_grow(CBMArena *a, size_t min_size) { return 0; } a->blocks[a->nblocks] = block; + a->block_sizes[a->nblocks] = new_size; a->nblocks++; a->block_size = new_size; a->used = 0; @@ -39,6 +41,16 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) { // 8-byte alignment n = (n + 7) & ~(size_t)7; + if (n <= 64) { + a->alloc_le_64 += n; + } else if (n <= 256) { + a->alloc_le_256 += n; + } else if (n <= 4096) { + a->alloc_le_4096 += n; + } else { + a->alloc_gt_4096 += n; + } + if (a->nblocks == 0) { return NULL; } @@ -51,6 +63,7 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) { char *ptr = a->blocks[a->nblocks - SKIP_ONE] + a->used; a->used += n; + a->total_alloc += n; return ptr; } @@ -60,6 +73,7 @@ char *cbm_arena_strdup(CBMArena *a, const char *s) { size_t len = strlen(s); char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE); if (dst) { + a->strdup_alloc += len + SKIP_ONE; memcpy(dst, s, len + SKIP_ONE); } return dst; @@ -70,6 +84,7 @@ char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len) { return NULL; char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE); if (dst) { + a->strdup_alloc += len + SKIP_ONE; memcpy(dst, s, len); dst[len] = '\0'; } @@ -91,6 +106,7 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) { if (!dst) { return NULL; } + a->sprintf_alloc += (size_t)needed + SKIP_ONE; va_start(args, fmt); vsnprintf(dst, (size_t)needed + SKIP_ONE, fmt, args); diff --git a/internal/cbm/arena.h b/internal/cbm/arena.h index 5c6bef9f0..d6f135060 100644 --- a/internal/cbm/arena.h +++ b/internal/cbm/arena.h @@ -1,41 +1,13 @@ -#ifndef CBM_ARENA_H -#define CBM_ARENA_H - -#include - -// CBMArena is a simple bump allocator that allocates from fixed-size blocks. -// All memory is freed at once via cbm_arena_destroy(). Individual frees are not -// supported — this is by design for per-file extraction where all data has the -// same lifetime. -#define CBM_ARENA_MAX_BLOCKS 256 -#define CBM_ARENA_DEFAULT_BLOCK_SIZE (64 * 1024) // 64KB initial - -typedef struct { - char *blocks[CBM_ARENA_MAX_BLOCKS]; - size_t block_sizes[CBM_ARENA_MAX_BLOCKS]; // per-block sizes (for stats) - int nblocks; - size_t block_size; - size_t used; // bytes used in current block - size_t total_alloc; // cumulative bytes allocated (for stats) -} CBMArena; - -// Initialize an arena with the default block size. -void cbm_arena_init(CBMArena *a); - -// Allocate n bytes from the arena. Returns NULL on OOM or block exhaustion. -// All returned pointers are 8-byte aligned. -void *cbm_arena_alloc(CBMArena *a, size_t n); - -// Duplicate a string into arena memory. Returns arena-owned copy. -char *cbm_arena_strdup(CBMArena *a, const char *s); - -// Duplicate a string of known length into arena memory. NUL-terminates. -char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len); - -// sprintf into arena memory. Returns arena-owned string. -char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) __attribute__((format(printf, 2, 3))); - -// Free all blocks. Arena is invalid after this call. -void cbm_arena_destroy(CBMArena *a); - -#endif // CBM_ARENA_H +/* + * Extraction compatibility include. + * + * CBMArena used to be duplicated here and in src/foundation/arena.h. The + * production binary links the foundation implementation, so any field drift + * between those definitions is an ABI violation. Keep one canonical layout. + */ +#ifndef CBM_EXTRACTION_ARENA_COMPAT_H +#define CBM_EXTRACTION_ARENA_COMPAT_H + +#include "../../src/foundation/arena.h" + +#endif /* CBM_EXTRACTION_ARENA_COMPAT_H */ diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index da6e720a2..565f24d34 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -103,19 +103,18 @@ void cbm_reset_profile(void) { // --- Growable array push functions --- -#define GROW_ARRAY(arr, arena) \ - do { \ - if ((arr)->count >= (arr)->cap) { \ - int new_cap = (arr)->cap == 0 ? CBM_SZ_32 : (arr)->cap * PAIR_LEN; \ - void *new_items = cbm_arena_alloc((arena), (size_t)new_cap * sizeof(*(arr)->items)); \ - if (!new_items) \ - return; \ - if ((arr)->items && (arr)->count > 0) { \ - memcpy(new_items, (arr)->items, (size_t)(arr)->count * sizeof(*(arr)->items)); \ - } \ - (arr)->items = new_items; \ - (arr)->cap = new_cap; \ - } \ +#define GROW_ARRAY(arr, arena) \ + do { \ + (void)(arena); \ + if ((arr)->count >= (arr)->cap) { \ + int new_cap = (arr)->cap == 0 ? CBM_SZ_32 : (arr)->cap * PAIR_LEN; \ + void *new_items = \ + cbm_arena_realloc((arena), (arr)->items, (size_t)new_cap * sizeof(*(arr)->items)); \ + if (!new_items) \ + return; \ + (arr)->items = new_items; \ + (arr)->cap = new_cap; \ + } \ } while (0) void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def) { diff --git a/internal/cbm/extract_node_stack.h b/internal/cbm/extract_node_stack.h index d112ccc2f..d7b4c268e 100644 --- a/internal/cbm/extract_node_stack.h +++ b/internal/cbm/extract_node_stack.h @@ -4,10 +4,12 @@ * Replaces fixed-size TSNode stack[] arrays that silently drop AST subtrees * when the stack overflows (GitHub issue #199). * - * Uses the arena allocator for zero-fragmentation growth: old blocks are - * abandoned (freed when the arena is destroyed at end of file extraction). - * Initial capacity matches the previous fixed caps so small files allocate - * no extra memory. + * The common capacity lives inline in the traversal's own stack frame. Only an + * unusually broad AST spills into the result arena. Call-site capacities + * historically mirrored fixed stack limits (usually 512, sometimes 4096), so + * allocating every traversal there made temporary work survive with durable + * extraction results across the whole repository. Inline storage preserves the + * same traversal order and geometric growth without that lifetime inversion. */ #ifndef CBM_EXTRACT_NODE_STACK_H #define CBM_EXTRACT_NODE_STACK_H @@ -20,19 +22,24 @@ typedef struct { TSNode *items; int count; int cap; + TSNode inline_items[128]; } TSNodeStack; -/* Initialize a stack with the given initial capacity, arena-allocated. */ +enum { TS_NSTACK_INLINE_CAP = 128 }; + +/* Initialize with inline storage; arena is used only if the stack spills. */ static inline void ts_nstack_init(TSNodeStack *s, CBMArena *arena, int initial_cap) { - s->items = (TSNode *)cbm_arena_alloc(arena, (size_t)initial_cap * sizeof(TSNode)); + (void)arena; + (void)initial_cap; + s->items = s->inline_items; s->count = 0; - s->cap = s->items ? initial_cap : 0; + s->cap = TS_NSTACK_INLINE_CAP; } /* Push a node onto the stack, growing 2x if needed. */ static inline void ts_nstack_push(TSNodeStack *s, CBMArena *arena, TSNode node) { if (s->count >= s->cap) { - int new_cap = s->cap ? s->cap * 2 : 512; + int new_cap = s->cap ? s->cap * 2 : TS_NSTACK_INLINE_CAP; TSNode *new_items = (TSNode *)cbm_arena_alloc(arena, (size_t)new_cap * sizeof(TSNode)); if (!new_items) return; /* OOM: best-effort, stop growing */ diff --git a/internal/cbm/lsp/kotlin_lsp.c b/internal/cbm/lsp/kotlin_lsp.c index 0ca49afd5..35b06a293 100644 --- a/internal/cbm/lsp/kotlin_lsp.c +++ b/internal/cbm/lsp/kotlin_lsp.c @@ -412,28 +412,13 @@ static void kt_emit_resolved_kind(KotlinLSPContext *ctx, const char *callee_qn, return; } - CBMResolvedCallArray *arr = ctx->resolved_calls; - if (arr->count >= arr->cap) { - int new_cap = arr->cap == 0 ? 16 : arr->cap * 2; - CBMResolvedCall *new_items = (CBMResolvedCall *)cbm_arena_alloc( - ctx->arena, (size_t)new_cap * sizeof(CBMResolvedCall)); - if (!new_items) { - return; - } - if (arr->items && arr->count > 0) { - memcpy(new_items, arr->items, (size_t)arr->count * sizeof(CBMResolvedCall)); - } - arr->items = new_items; - arr->cap = new_cap; - } - CBMResolvedCall *rc = &arr->items[arr->count]; - memset(rc, 0, sizeof(CBMResolvedCall)); - rc->caller_qn = ctx->enclosing_func_qn; - rc->callee_qn = cbm_arena_strdup(ctx->arena, callee_qn); - rc->strategy = strategy; - rc->confidence = confidence; - rc->kind = kind; - arr->count++; + CBMResolvedCall rc = {0}; + rc.caller_qn = ctx->enclosing_func_qn; + rc.callee_qn = cbm_arena_strdup(ctx->arena, callee_qn); + rc.strategy = strategy; + rc.confidence = confidence; + rc.kind = kind; + cbm_resolvedcall_push(ctx->resolved_calls, ctx->arena, rc); } static void kt_stamp_resolved_site(KotlinLSPContext *ctx, int first, TSNode site) { @@ -3177,8 +3162,7 @@ static const CBMType *kt_eval_navigation_expression_type_at(KotlinLSPContext *ct if (is_member) { /* A member call on an `object`/`companion object` singleton is a * static dispatch; on a regular class instance it is a method. */ - const CBMRegisteredType *recv_rt = - cbm_registry_lookup_type(ctx->registry, recv_qn); + const CBMRegisteredType *recv_rt = cbm_registry_lookup_type(ctx->registry, recv_qn); strat = (recv_rt && recv_rt->is_object) ? "lsp_kt_static" : "lsp_kt_method"; } /* A call through the lambda implicit parameter `it` (e.g. inside @@ -5255,8 +5239,7 @@ static void kt_register_cross_def(CBMTypeRegistry *reg, CBMArena *arena, const C rt.is_interface = (strcmp(d->label, "Interface") == 0) || d->is_interface; if (field_map) { const KtCrossFieldList *fl = - (const KtCrossFieldList *)cbm_ht_get((CBMHashTable *)field_map, - d->qualified_name); + (const KtCrossFieldList *)cbm_ht_get((CBMHashTable *)field_map, d->qualified_name); if (fl && fl->count > 0) { rt.field_names = fl->names; rt.field_types = fl->types; diff --git a/src/daemon/application.c b/src/daemon/application.c index 12543808e..e57a4202b 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -5,6 +5,7 @@ #include "daemon/application_internal.h" #include "cli/cli.h" +#include "discover/discover.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/compat_thread.h" @@ -25,6 +26,7 @@ #include #include +#include #include #include #include @@ -62,8 +64,16 @@ enum { APPLICATION_BACKGROUND_REAP_MS = 10000, APPLICATION_UPDATE_VERSION_CAP = 128, APPLICATION_UPDATE_NOTICE_CAP = 1024, + APPLICATION_LARGE_FILE_THRESHOLD = 10000, + APPLICATION_RESOURCE_CLASSIFY_TIMEOUT_MS = 5000, }; +static const size_t APPLICATION_LARGE_SOURCE_BYTES = (size_t)512 * (size_t)1024 * (size_t)1024; +static const size_t APPLICATION_DAILY_WORKER_RESERVATION_BYTES = + (size_t)384 * (size_t)1024 * (size_t)1024; +static const size_t APPLICATION_LARGE_WORKER_BUDGET_BYTES = + (size_t)3072 * (size_t)1024 * (size_t)1024; + /* There is deliberately NO production update-check provider. The daemon used to * spawn `curl` against the GitHub releases API on the first eligible session of * every run, purely to print "a newer version exists". That put a release URL @@ -83,6 +93,8 @@ typedef struct cbm_daemon_application_watch cbm_daemon_application_watch_t; typedef struct cbm_daemon_application_session cbm_daemon_application_session_t; typedef struct cbm_daemon_application_job cbm_daemon_application_job_t; typedef struct cbm_daemon_application_mutation cbm_daemon_application_mutation_t; +typedef struct cbm_daemon_application_admission_waiter cbm_daemon_application_admission_waiter_t; +typedef struct cbm_daemon_application_resource_history cbm_daemon_application_resource_history_t; typedef struct cbm_daemon_application_watch_job_subscription cbm_daemon_application_watch_job_subscription_t; @@ -140,6 +152,7 @@ struct cbm_daemon_application_job { cbm_thread_t thread; size_t subscribers; size_t watcher_waiters; + size_t observed_rss_bytes; bool thread_started; bool thread_done; bool terminal; @@ -147,6 +160,7 @@ struct cbm_daemon_application_job { bool cancelled; bool cancel_requested; bool supervision_failed; + bool large_repository; cbm_daemon_application_job_t *next; }; @@ -168,6 +182,17 @@ struct cbm_daemon_application_mutation { cbm_daemon_application_mutation_t *next; }; +struct cbm_daemon_application_admission_waiter { + uint64_t ticket; + cbm_daemon_application_admission_waiter_t *next; +}; + +struct cbm_daemon_application_resource_history { + char *project_key; + size_t peak_rss_bytes; + cbm_daemon_application_resource_history_t *next; +}; + struct cbm_daemon_application { cbm_mutex_t mutex; struct cbm_watcher *watcher; @@ -177,11 +202,17 @@ struct cbm_daemon_application { cbm_daemon_application_job_t *jobs; cbm_daemon_application_watch_job_subscription_t *watch_job_subscriptions; cbm_daemon_application_mutation_t *mutations; + cbm_daemon_application_admission_waiter_t *admission_head; + cbm_daemon_application_admission_waiter_t *admission_tail; + cbm_daemon_application_resource_history_t *resource_history; + uint64_t next_admission_ticket; cbm_daemon_application_worker_ops_t worker_ops; cbm_daemon_application_update_ops_t update_ops; cbm_project_lock_manager_t *project_locks; size_t physical_job_limit; + size_t aggregate_memory_budget_bytes; size_t worker_memory_budget_bytes; + size_t large_worker_memory_budget_bytes; size_t active_mutations; size_t update_owners; cbm_daemon_application_update_worker_t update_worker; @@ -211,7 +242,13 @@ static void *application_job_thread(void *opaque); static char *application_auto_index_args(const char *root_path); static cbm_daemon_application_job_t *application_job_subscribe_locked( cbm_daemon_application_t *application, const char *project_key, const char *root_path, - const char *args_json, application_job_subscribe_status_t *status_out); + const char *args_json, bool large_repository, bool queue_head, + application_job_subscribe_status_t *status_out); +static cbm_daemon_application_resource_history_t *application_resource_history_find_locked( + cbm_daemon_application_t *application, const char *project_key); +static void application_resource_history_record_locked(cbm_daemon_application_t *application, + const char *project_key, + size_t peak_rss_bytes); static atomic_bool g_application_fail_next_job_thread_start_for_test = ATOMIC_VAR_INIT(false); @@ -324,6 +361,13 @@ static bool application_worker_cancel_default(void *context, return cbm_index_worker_request_cancel((cbm_index_worker_handle_t *)worker); } +static bool application_worker_observed_rss_default(void *context, + cbm_daemon_application_worker_t worker, + size_t *bytes_out) { + (void)context; + return cbm_index_worker_observed_tree_rss((cbm_index_worker_handle_t *)worker, bytes_out); +} + static const char *application_worker_log_path_default(void *context, cbm_daemon_application_worker_t worker) { (void)context; @@ -384,6 +428,27 @@ static cbm_daemon_application_watch_t *application_find_watch_locked( return NULL; } +static cbm_daemon_application_resource_history_t *application_resource_history_ensure_locked( + cbm_daemon_application_t *application, const char *project_key) { + cbm_daemon_application_resource_history_t *history = + application_resource_history_find_locked(application, project_key); + if (history) { + return history; + } + history = calloc(1, sizeof(*history)); + if (!history) { + return NULL; + } + history->project_key = strdup(project_key); + if (!history->project_key) { + free(history); + return NULL; + } + history->next = application->resource_history; + application->resource_history = history; + return history; +} + static void application_remove_watch_entry_locked(cbm_daemon_application_t *application, cbm_daemon_application_watch_t *watch, bool unregister_physical_watch) { @@ -1086,7 +1151,9 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli cbm_daemon_application_worker_t worker = NULL; application_tmp_lock(); int start_result = application->worker_ops.start( - application->worker_ops.context, job->args_json, application->worker_memory_budget_bytes, + application->worker_ops.context, job->args_json, + job->large_repository ? application->large_worker_memory_budget_bytes + : application->worker_memory_budget_bytes, marker_path, quarantine_path, &worker); application_tmp_unlock(); if (start_result != 0 || !worker) { @@ -1108,6 +1175,17 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli for (;;) { cbm_index_worker_poll_t state = application->worker_ops.poll(application->worker_ops.context, worker, &borrowed); + if (application->worker_ops.observed_rss) { + size_t observed = 0; + if (application->worker_ops.observed_rss(application->worker_ops.context, worker, + &observed)) { + cbm_mutex_lock(&application->mutex); + if (job->worker == worker && observed > job->observed_rss_bytes) { + job->observed_rss_bytes = observed; + } + cbm_mutex_unlock(&application->mutex); + } + } if (state == CBM_INDEX_WORKER_POLL_TERMINAL) { break; } @@ -1431,7 +1509,7 @@ static void application_auto_index_retry_pending_locked(cbm_daemon_application_t } application_job_subscribe_status_t subscribe_status = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; cbm_daemon_application_job_t *retry = application_job_subscribe_locked( - application, project, root_path, args, &subscribe_status); + application, project, root_path, args, false, false, &subscribe_status); free(args); if (retry) { session->auto_index_job = retry; @@ -1462,6 +1540,8 @@ static void application_job_publish(cbm_daemon_application_job_t *job, job->supervision_failed = execution->supervision_failed || (execution->unsafe_terminal && execution->have_last_result && !execution->last_result.cancellation_requested); + application_resource_history_record_locked(application, job->project_key, + job->observed_rss_bytes); job->terminal = true; job->thread_done = true; for (cbm_daemon_application_session_t *session = application->sessions; session; @@ -1549,6 +1629,140 @@ static char *application_index_project_key(const char *root_path, const char *ar return key; } +/* Git can enumerate the relevant tracked/untracked set without paying a full + * filesystem walk through very large ignored dependency trees. The output is + * NUL-delimited, so unusual filenames remain unambiguous. This is a fast + * classification hint only: the pipeline's normal discovery remains the + * authority for the actual index. */ +static cbm_discover_status_t application_git_measure_bounded(const char *root_path, int max_files, + size_t max_total_bytes, + uint64_t deadline_ms, int *count_out, + size_t *total_bytes_out) { + char output_path[APPLICATION_PATH_CAP]; + if (!application_unique_recovery_file(output_path, "classify")) { + return CBM_DISCOVER_ERROR; + } + const char *argv[] = {"git", "-C", root_path, "ls-files", + "-z", "--cached", "--others", "--exclude-standard", + NULL}; + cbm_proc_opts_t options = { + .bin = "git", + .argv = argv, + .log_file = output_path, + .quiet_timeout_ms = APPLICATION_RESOURCE_CLASSIFY_TIMEOUT_MS, + .delete_log_on_exit = false, + }; + cbm_proc_result_t result; + int run = cbm_subprocess_run(&options, &result); + if (run != 0 || result.outcome != CBM_PROC_CLEAN || !result.tree_quiesced) { + (void)cbm_unlink(output_path); + return CBM_DISCOVER_ERROR; + } + FILE *file = cbm_fopen(output_path, "rb"); + if (!file) { + (void)cbm_unlink(output_path); + return CBM_DISCOVER_ERROR; + } + char relative[APPLICATION_PATH_CAP]; + size_t length = 0; + int count = 0; + size_t total = 0; + cbm_discover_status_t status = CBM_DISCOVER_OK; + for (;;) { + int byte = fgetc(file); + if (byte == EOF) { + if (length != 0 || ferror(file)) { + status = CBM_DISCOVER_ERROR; + } + break; + } + if (cbm_now_ms() >= deadline_ms || length + 1 >= sizeof(relative)) { + status = CBM_DISCOVER_ERROR; + break; + } + if (byte != '\0') { + relative[length++] = (char)byte; + continue; + } + relative[length] = '\0'; + length = 0; + const char *basename = strrchr(relative, '/'); + basename = basename ? basename + 1 : relative; + if (cbm_language_for_filename(basename) == CBM_LANG_COUNT) { + continue; + } + char absolute[APPLICATION_PATH_CAP]; + int written = snprintf(absolute, sizeof(absolute), "%s/%s", root_path, relative); + struct stat file_status; + if (written <= 0 || (size_t)written >= sizeof(absolute) || + stat(absolute, &file_status) != 0 || !S_ISREG(file_status.st_mode)) { + status = CBM_DISCOVER_ERROR; + break; + } + size_t measured = file_status.st_size > 0 ? (size_t)file_status.st_size : 0; + if (count >= max_files || total > max_total_bytes || measured > max_total_bytes - total) { + status = CBM_DISCOVER_LIMIT_EXCEEDED; + break; + } + count++; + total += measured; + } + (void)fclose(file); + (void)cbm_unlink(output_path); + *count_out = status == CBM_DISCOVER_ERROR ? -1 : count; + *total_bytes_out = total; + return status; +} + +static bool application_index_is_large_repository(const char *root_path) { + char mode[CBM_SZ_32]; + const char *configured = cbm_safe_getenv("CBM_INDEX_RESOURCE_MODE", mode, sizeof(mode), NULL); + if (configured && strcmp(mode, "daily") == 0) { + return false; + } + if (configured && strcmp(mode, "large") == 0) { + return true; + } + if (configured && strcmp(mode, "auto") != 0) { + /* The worker will reject the invalid mode. Reserve the safer large + * lane until that contained failure is published. */ + return true; + } + cbm_discover_opts_t options = { + .mode = CBM_MODE_FULL, + .ignore_file = NULL, + .max_file_size = 0, + }; + int files = -1; + size_t bytes = 0; + uint64_t deadline = cbm_now_ms() + APPLICATION_RESOURCE_CLASSIFY_TIMEOUT_MS; + cbm_discover_status_t status = application_git_measure_bounded( + root_path, APPLICATION_LARGE_FILE_THRESHOLD - 1, APPLICATION_LARGE_SOURCE_BYTES - 1, + deadline, &files, &bytes); + const char *measurement = "git_files"; + if (status == CBM_DISCOVER_ERROR) { + files = -1; + bytes = 0; + deadline = cbm_now_ms() + APPLICATION_RESOURCE_CLASSIFY_TIMEOUT_MS; + status = cbm_discover_measure_bounded( + root_path, &options, APPLICATION_LARGE_FILE_THRESHOLD - 1, + APPLICATION_LARGE_SOURCE_BYTES - 1, deadline, &files, &bytes); + measurement = "filesystem_fallback"; + } + bool large = status != CBM_DISCOVER_OK; + char files_text[CBM_SZ_32]; + char bytes_text[CBM_SZ_32]; + (void)snprintf(files_text, sizeof(files_text), "%d", files); + (void)snprintf(bytes_text, sizeof(bytes_text), "%zu", bytes); + cbm_log_info( + "daemon.index.resource_mode", "selected", large ? "large" : "daily", "reason", + status == CBM_DISCOVER_LIMIT_EXCEEDED + ? "threshold" + : (status == CBM_DISCOVER_OK ? "within_daily_limits" : "classification_failed_closed"), + "files", files_text, "source_bytes", bytes_text, "measurement", measurement); + return large; +} + static size_t application_active_job_count_locked(cbm_daemon_application_t *application) { size_t count = 0; for (cbm_daemon_application_job_t *job = application->jobs; job; job = job->next) { @@ -1559,6 +1773,102 @@ static size_t application_active_job_count_locked(cbm_daemon_application_t *appl return count; } +static size_t application_active_observed_rss_locked(cbm_daemon_application_t *application) { + size_t total = 0; + for (cbm_daemon_application_job_t *job = application->jobs; job; job = job->next) { + if (!job->terminal) { + total = job->observed_rss_bytes > SIZE_MAX - total ? SIZE_MAX + : total + job->observed_rss_bytes; + } + } + return total; +} + +static cbm_daemon_application_resource_history_t *application_resource_history_find_locked( + cbm_daemon_application_t *application, const char *project_key) { + for (cbm_daemon_application_resource_history_t *history = application->resource_history; + history; history = history->next) { + if (strcmp(history->project_key, project_key) == 0) { + return history; + } + } + return NULL; +} + +static void application_resource_history_record_locked(cbm_daemon_application_t *application, + const char *project_key, + size_t peak_rss_bytes) { + if (!project_key || peak_rss_bytes == 0) { + return; + } + cbm_daemon_application_resource_history_t *history = + application_resource_history_ensure_locked(application, project_key); + if (!history) { + return; + } + if (peak_rss_bytes > history->peak_rss_bytes) { + history->peak_rss_bytes = peak_rss_bytes; + } +} + +/* Explicit user requests own FIFO admission tickets. Background auto-index + * and watcher work may coalesce with an already-running project, but cannot + * consume a newly freed physical slot while an explicit request is queued. */ +static void application_admission_enqueue_locked( + cbm_daemon_application_t *application, cbm_daemon_application_admission_waiter_t *waiter) { + waiter->ticket = ++application->next_admission_ticket; + waiter->next = NULL; + if (application->admission_tail) { + application->admission_tail->next = waiter; + } else { + application->admission_head = waiter; + } + application->admission_tail = waiter; +} + +static void application_admission_remove_locked(cbm_daemon_application_t *application, + cbm_daemon_application_admission_waiter_t *waiter) { + cbm_daemon_application_admission_waiter_t **cursor = &application->admission_head; + while (*cursor && *cursor != waiter) { + cursor = &(*cursor)->next; + } + if (*cursor != waiter) { + return; + } + *cursor = waiter->next; + if (application->admission_tail == waiter) { + application->admission_tail = NULL; + for (cbm_daemon_application_admission_waiter_t *tail = application->admission_head; tail; + tail = tail->next) { + application->admission_tail = tail; + } + } + waiter->next = NULL; +} + +static size_t application_admission_position_locked( + const cbm_daemon_application_t *application, + const cbm_daemon_application_admission_waiter_t *waiter) { + size_t position = 1; + for (const cbm_daemon_application_admission_waiter_t *entry = application->admission_head; + entry; entry = entry->next, position++) { + if (entry == waiter) { + return position; + } + } + return 0; +} + +static size_t application_active_large_job_count_locked(cbm_daemon_application_t *application) { + size_t count = 0; + for (cbm_daemon_application_job_t *job = application->jobs; job; job = job->next) { + if (!job->terminal && job->large_repository) { + count++; + } + } + return count; +} + /* Compare the effective index request, not its JSON spelling. yyjson's deep * equality treats object member order as insignificant, while the small * normalization below removes values that the index handler interprets as @@ -1607,7 +1917,8 @@ static bool application_index_args_equal(const char *left, const char *right) { * this admission in the same critical section closes the unwatch race. */ static cbm_daemon_application_job_t *application_job_subscribe_locked( cbm_daemon_application_t *application, const char *project_key, const char *root_path, - const char *args_json, application_job_subscribe_status_t *status_out) { + const char *args_json, bool large_repository, bool queue_head, + application_job_subscribe_status_t *status_out) { *status_out = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; if (application->stopping) { return NULL; @@ -1628,13 +1939,38 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( return job; } - if (application_active_job_count_locked(application) >= application->physical_job_limit) { + size_t active_jobs = application_active_job_count_locked(application); + size_t active_large_jobs = application_active_large_job_count_locked(application); + size_t observed_rss = application_active_observed_rss_locked(application); + size_t reservation = application->worker_memory_budget_bytes; + cbm_daemon_application_resource_history_t *history = + application_resource_history_find_locked(application, project_key); + if (history && history->peak_rss_bytes > reservation) { + reservation = history->peak_rss_bytes; + } + bool daily_memory_busy = + !large_repository && application->aggregate_memory_budget_bytes > 0 && + (observed_rss > application->aggregate_memory_budget_bytes || + reservation > application->aggregate_memory_budget_bytes - observed_rss); + if ((application->admission_head && !queue_head) || + active_jobs >= application->physical_job_limit || (large_repository && active_jobs > 0) || + (!large_repository && active_large_jobs > 0)) { char limit[32]; (void)snprintf(limit, sizeof(limit), "%zu", application->physical_job_limit); cbm_log_warn("daemon.index.admission_busy", "limit", limit, "project", project_key); *status_out = APPLICATION_JOB_SUBSCRIBE_BUSY; return NULL; } + if (daily_memory_busy) { + char observed[CBM_SZ_32]; + char reserved[CBM_SZ_32]; + (void)snprintf(observed, sizeof(observed), "%zu", observed_rss); + (void)snprintf(reserved, sizeof(reserved), "%zu", reservation); + cbm_log_warn("daemon.index.admission_memory_busy", "project", project_key, "observed_rss", + observed, "reservation", reserved); + *status_out = APPLICATION_JOB_SUBSCRIBE_BUSY; + return NULL; + } job = calloc(1, sizeof(*job)); if (job) { @@ -1648,6 +1984,7 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( return NULL; } job->application = application; + job->large_repository = large_repository; job->subscribers = 1; job->next = application->jobs; application->jobs = job; @@ -1668,17 +2005,6 @@ static cbm_daemon_application_job_t *application_job_subscribe_locked( return job; } -static cbm_daemon_application_job_t *application_job_subscribe( - cbm_daemon_application_t *application, const char *project_key, const char *root_path, - const char *args_json, application_job_subscribe_status_t *status_out) { - application_jobs_reap_completed(application); - cbm_mutex_lock(&application->mutex); - cbm_daemon_application_job_t *job = application_job_subscribe_locked( - application, project_key, root_path, args_json, status_out); - cbm_mutex_unlock(&application->mutex); - return job; -} - static void application_job_unsubscribe_locked(cbm_daemon_application_job_t *job) { if (!job || job->subscribers == 0) { return; @@ -2000,7 +2326,7 @@ static void application_background_initialize_impl(cbm_daemon_application_sessio if (attempt_auto_index && args) { application_job_subscribe_status_t subscribe_status = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; cbm_daemon_application_job_t *job = application_job_subscribe_locked( - application, project, root_path, args, &subscribe_status); + application, project, root_path, args, false, false, &subscribe_status); if (job) { session->auto_index_job = job; session->auto_index_subscribed = true; @@ -2214,13 +2540,42 @@ static char *application_index_execute(void *context, const char *root_path, } application_job_subscribe_status_t subscribe_status = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; cbm_daemon_application_job_t *job = NULL; + bool large_repository = application_index_is_large_repository(root_path); + cbm_daemon_application_admission_waiter_t waiter = {0}; + cbm_mutex_lock(&session->application->mutex); + application_admission_enqueue_locked(session->application, &waiter); + size_t initial_position = application_admission_position_locked(session->application, &waiter); + cbm_mutex_unlock(&session->application->mutex); + char ticket_text[CBM_SZ_32]; + char position_text[CBM_SZ_32]; + (void)snprintf(ticket_text, sizeof(ticket_text), "%" PRIu64, waiter.ticket); + (void)snprintf(position_text, sizeof(position_text), "%zu", initial_position); + cbm_log_info("daemon.index.queued", "project", project_key, "ticket", ticket_text, "position", + position_text, "resource_mode", large_repository ? "large" : "daily"); for (;;) { - job = application_job_subscribe(session->application, project_key, root_path, args_json, - &subscribe_status); + application_jobs_reap_completed(session->application); + cbm_mutex_lock(&session->application->mutex); + bool queued_cancelled = application_request_cancelled_locked(session); + bool queue_head = session->application->admission_head == &waiter; + if (queued_cancelled || session->application->stopping) { + application_admission_remove_locked(session->application, &waiter); + cbm_mutex_unlock(&session->application->mutex); + free(project_key); + return cbm_mcp_text_result(queued_cancelled + ? "index operation cancelled for this session" + : "daemon index coordinator is stopping or unavailable", + true); + } + job = application_job_subscribe_locked(session->application, project_key, root_path, + args_json, large_repository, queue_head, + &subscribe_status); if (job || (subscribe_status != APPLICATION_JOB_SUBSCRIBE_BUSY && subscribe_status != APPLICATION_JOB_SUBSCRIBE_CANCELLING)) { + application_admission_remove_locked(session->application, &waiter); + cbm_mutex_unlock(&session->application->mutex); break; } + cbm_mutex_unlock(&session->application->mutex); /* Physical job limit reached (or a same-project cancel still * draining): QUEUE instead of surfacing a raw busy error. The * request thread blocks for the whole index anyway, so waiting for @@ -2229,13 +2584,6 @@ static char *application_index_execute(void *context, const char *root_path, * exits this loop with the error below). */ atomic_fetch_add_explicit(&g_application_busy_queue_waits_for_test, 1, memory_order_release); - cbm_mutex_lock(&session->application->mutex); - bool queued_cancelled = application_request_cancelled_locked(session); - cbm_mutex_unlock(&session->application->mutex); - if (queued_cancelled) { - free(project_key); - return cbm_mcp_text_result("index operation cancelled for this session", true); - } cbm_usleep(APPLICATION_JOB_POLL_US); } free(project_key); @@ -2879,6 +3227,7 @@ cbm_daemon_application_t *cbm_daemon_application_new( } cbm_mutex_init(&application->mutex); application->physical_job_limit = APPLICATION_DEFAULT_PHYSICAL_JOB_LIMIT; + application->large_worker_memory_budget_bytes = APPLICATION_LARGE_WORKER_BUDGET_BYTES; size_t aggregate_memory_budget_bytes = cbm_mem_budget(); if (config) { if ((config->ui_readiness_secret != NULL || config->ui_readiness_secret_length != 0) && @@ -2912,15 +3261,30 @@ cbm_daemon_application_t *cbm_daemon_application_new( application->ui_readiness_secret_set = true; } } - /* Equal fixed slices keep admission deterministic: starting fewer jobs does + /* Resource tokens adapt daily indexing to one through four workers. A + * normal 1.5 GiB daemon budget admits four 384 MiB reservations; tighter + * budgets reduce concurrency instead of overcommitting. An explicit + * physical limit remains a hard upper bound. Equal fixed slices keep + * admission deterministic: starting fewer jobs does * not let an early worker claim memory reserved for later concurrent jobs. * The absurd sub-byte-per-slot case is made safe by reducing effective * capacity before division; normal daemon budgets are many orders larger. */ + if (aggregate_memory_budget_bytes > 0) { + size_t token_limit = + aggregate_memory_budget_bytes / APPLICATION_DAILY_WORKER_RESERVATION_BYTES; + if (token_limit == 0) { + token_limit = 1; + } + if (application->physical_job_limit > token_limit) { + application->physical_job_limit = token_limit; + } + } if (aggregate_memory_budget_bytes > 0 && application->physical_job_limit > aggregate_memory_budget_bytes) { application->physical_job_limit = aggregate_memory_budget_bytes; } if (aggregate_memory_budget_bytes > 0 && application->physical_job_limit > 0) { + application->aggregate_memory_budget_bytes = aggregate_memory_budget_bytes; application->worker_memory_budget_bytes = aggregate_memory_budget_bytes / application->physical_job_limit; } @@ -2930,6 +3294,7 @@ cbm_daemon_application_t *cbm_daemon_application_new( .start = application_worker_start_default, .poll = application_worker_poll_default, .cancel = application_worker_cancel_default, + .observed_rss = application_worker_observed_rss_default, .log_path = application_worker_log_path_default, .destroy = application_worker_destroy_default, }; @@ -3048,6 +3413,8 @@ bool cbm_daemon_application_free_with_timeout(cbm_daemon_application_t *applicat application->watch_job_subscriptions = NULL; cbm_daemon_application_mutation_t *mutations = application->mutations; application->mutations = NULL; + cbm_daemon_application_resource_history_t *resource_history = application->resource_history; + application->resource_history = NULL; cbm_mutex_unlock(&application->mutex); while (sessions) { cbm_daemon_application_session_t *next = sessions->next; @@ -3084,6 +3451,12 @@ bool cbm_daemon_application_free_with_timeout(cbm_daemon_application_t *applicat free(mutations); mutations = next; } + while (resource_history) { + cbm_daemon_application_resource_history_t *next = resource_history->next; + free(resource_history->project_key); + free(resource_history); + resource_history = next; + } cbm_mutex_destroy(&application->mutex); cbm_secure_zero(application->ui_readiness_secret, sizeof(application->ui_readiness_secret)); free(application); @@ -3401,6 +3774,7 @@ static int application_background_index(cbm_daemon_application_t *application, } application_job_subscribe_status_t subscribe_status = APPLICATION_JOB_SUBSCRIBE_UNAVAILABLE; + bool large_repository = application_index_is_large_repository(canonical_root); application_jobs_reap_completed(application); cbm_mutex_lock(&application->mutex); cbm_daemon_application_watch_t *watch = @@ -3410,9 +3784,10 @@ static int application_background_index(cbm_daemon_application_t *application, size_t watch_owner_count = 0; bool watch_subscriptions_ok = true; cbm_daemon_application_job_t *job = - watch_live ? application_job_subscribe_locked(application, project_key, canonical_root, - args, &subscribe_status) - : NULL; + watch_live + ? application_job_subscribe_locked(application, project_key, canonical_root, args, + large_repository, false, &subscribe_status) + : NULL; if (job && require_live_watch) { watch_subscriptions_ok = application_watch_job_subscribe_sessions_locked( application, watch, job, &watch_owner_count); @@ -3528,6 +3903,16 @@ size_t cbm_daemon_application_worker_memory_budget_bytes(cbm_daemon_application_ return budget; } +size_t cbm_daemon_application_observed_index_rss_for_test(cbm_daemon_application_t *application) { + if (!application) { + return 0; + } + cbm_mutex_lock(&application->mutex); + size_t observed = application_active_observed_rss_locked(application); + cbm_mutex_unlock(&application->mutex); + return observed; +} + bool cbm_daemon_application_session_retains_store_for_test( const cbm_daemon_runtime_application_session_t *opaque_session) { const cbm_daemon_application_session_t *session = diff --git a/src/daemon/application.h b/src/daemon/application.h index 933186aa7..bc758eb2b 100644 --- a/src/daemon/application.h +++ b/src/daemon/application.h @@ -36,6 +36,7 @@ typedef struct { cbm_index_worker_poll_t (*poll)(void *context, cbm_daemon_application_worker_t worker, const cbm_index_worker_result_t **result_out); bool (*cancel)(void *context, cbm_daemon_application_worker_t worker); + bool (*observed_rss)(void *context, cbm_daemon_application_worker_t worker, size_t *bytes_out); const char *(*log_path)(void *context, cbm_daemon_application_worker_t worker); void (*destroy)(void *context, cbm_daemon_application_worker_t worker); } cbm_daemon_application_worker_ops_t; diff --git a/src/daemon/application_internal.h b/src/daemon/application_internal.h index 04f24421a..fd6093b30 100644 --- a/src/daemon/application_internal.h +++ b/src/daemon/application_internal.h @@ -37,6 +37,9 @@ int cbm_daemon_application_background_initializes_for_test(void); * that a request QUEUED rather than erroring or starting. */ int cbm_daemon_application_busy_queue_waits_for_test(void); +/* Sum of sampled process-group RSS high-water marks for active index jobs. */ +size_t cbm_daemon_application_observed_index_rss_for_test(cbm_daemon_application_t *application); + /* Build the JSON-RPC error substituted for a reply too large to frame (#1375). * Exposed because the alternative — driving a real >10 MiB reply — needs a * ~20k-node index, a fixture cost the unit suite should not carry. The diff --git a/src/discover/discover.c b/src/discover/discover.c index b89db8a3f..96b41501e 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -413,6 +413,8 @@ typedef struct { int count; int capacity; int max_files; + size_t max_total_bytes; + size_t total_bytes; uint64_t deadline_ms; bool count_only; bool collect_excluded; @@ -504,8 +506,15 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, fl->limit_exceeded = true; return; } + size_t measured_size = size > 0 ? (size_t)size : 0; + if (fl->count_only && (fl->total_bytes > fl->max_total_bytes || + measured_size > fl->max_total_bytes - fl->total_bytes)) { + fl->limit_exceeded = true; + return; + } if (fl->count_only) { fl->count++; + fl->total_bytes += measured_size; return; } if (fl->count >= fl->capacity) { @@ -1117,7 +1126,8 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc int *excluded_count_out, cbm_ignored_file_t **ignored_out, int *ignored_count_out, int *ignored_total_out, bool count_only, int max_files, - uint64_t deadline_ms) { + size_t max_total_bytes, uint64_t deadline_ms, + size_t *total_bytes_out) { if (excluded_out) { *excluded_out = NULL; } @@ -1133,6 +1143,9 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc if (ignored_total_out) { *ignored_total_out = 0; } + if (total_bytes_out) { + *total_bytes_out = 0; + } if (!repo_path || !out || !count || (count_only && max_files < 0)) { return CBM_DISCOVER_ERROR; } @@ -1206,6 +1219,7 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc /* Walk */ file_list_t fl = { .max_files = count_only ? max_files : -1, + .max_total_bytes = count_only ? max_total_bytes : SIZE_MAX, .deadline_ms = count_only ? deadline_ms : 0, .count_only = count_only, .collect_excluded = !count_only && excluded_out != NULL, @@ -1224,6 +1238,9 @@ static cbm_discover_status_t discover_impl(const char *repo_path, const cbm_disc cbm_discover_free_excluded(fl.excluded, fl.excluded_count); cbm_discover_free_ignored(fl.ignored, fl.ignored_count); *count = fl.count; + if (total_bytes_out) { + *total_bytes_out = fl.total_bytes; + } if (fl.failed) { return CBM_DISCOVER_ERROR; } @@ -1269,7 +1286,7 @@ int cbm_discover_ex2(const char *repo_path, const cbm_discover_opts_t *opts, cbm cbm_ignored_file_t **ignored_out, int *ignored_count_out, int *ignored_total_out) { return discover_impl(repo_path, opts, out, count, excluded_out, excluded_count_out, ignored_out, - ignored_count_out, ignored_total_out, false, 0, 0); + ignored_count_out, ignored_total_out, false, 0, SIZE_MAX, 0, NULL); } cbm_discover_status_t cbm_discover_count_bounded(const char *repo_path, @@ -1283,10 +1300,36 @@ cbm_discover_status_t cbm_discover_count_bounded(const char *repo_path, } cbm_file_info_t *files = NULL; int count = 0; - cbm_discover_status_t status = discover_impl(repo_path, opts, &files, &count, NULL, NULL, NULL, - NULL, NULL, true, max_files, deadline_ms); + cbm_discover_status_t status = + discover_impl(repo_path, opts, &files, &count, NULL, NULL, NULL, NULL, NULL, true, + max_files, SIZE_MAX, deadline_ms, NULL); + cbm_discover_free(files, count); + *count_out = status == CBM_DISCOVER_ERROR ? -1 : count; + return status; +} + +cbm_discover_status_t cbm_discover_measure_bounded(const char *repo_path, + const cbm_discover_opts_t *opts, int max_files, + size_t max_total_bytes, uint64_t deadline_ms, + int *count_out, size_t *total_bytes_out) { + if (count_out) { + *count_out = -1; + } + if (total_bytes_out) { + *total_bytes_out = 0; + } + if (!repo_path || !count_out || !total_bytes_out || max_files < 0) { + return CBM_DISCOVER_ERROR; + } + cbm_file_info_t *files = NULL; + int count = 0; + size_t total_bytes = 0; + cbm_discover_status_t status = + discover_impl(repo_path, opts, &files, &count, NULL, NULL, NULL, NULL, NULL, true, + max_files, max_total_bytes, deadline_ms, &total_bytes); cbm_discover_free(files, count); *count_out = status == CBM_DISCOVER_ERROR ? -1 : count; + *total_bytes_out = total_bytes; return status; } diff --git a/src/discover/discover.h b/src/discover/discover.h index d49c0acc2..b1d03230b 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -164,6 +164,15 @@ cbm_discover_status_t cbm_discover_count_bounded(const char *repo_path, const cbm_discover_opts_t *opts, int max_files, uint64_t deadline_ms, int *count_out); +/* Count indexable files and their aggregate source bytes without retaining a + * file array. Stops when either limit would be exceeded. This is the daemon's + * pre-spawn resource classifier, so a large job is identified before it can + * consume a daily-worker slot. */ +cbm_discover_status_t cbm_discover_measure_bounded(const char *repo_path, + const cbm_discover_opts_t *opts, int max_files, + size_t max_total_bytes, uint64_t deadline_ms, + int *count_out, size_t *total_bytes_out); + /* Like cbm_discover(), but also reports the directory subtrees that were * skipped during the walk (hardcoded ALWAYS_SKIP/FAST_SKIP dirs + gitignore * matches), so callers can surface which subtrees were dropped (#411). diff --git a/src/foundation/arena.c b/src/foundation/arena.c index 3ae0b9a83..168d3fbff 100644 --- a/src/foundation/arena.c +++ b/src/foundation/arena.c @@ -18,6 +18,22 @@ enum { ARENA_ALIGN = 7, ARENA_GROW_OK = 1 }; #include #include +typedef struct cbm_arena_heap_allocation { + void *ptr; + struct cbm_arena_heap_allocation *next; +} cbm_arena_heap_allocation_t; + +static void arena_free_heap_allocations(CBMArena *a) { + cbm_arena_heap_allocation_t *allocation = (cbm_arena_heap_allocation_t *)a->heap_allocations; + while (allocation) { + cbm_arena_heap_allocation_t *next = allocation->next; + free(allocation->ptr); + free(allocation); + allocation = next; + } + a->heap_allocations = NULL; +} + void cbm_arena_init(CBMArena *a) { cbm_arena_init_sized(a, CBM_ARENA_DEFAULT_BLOCK_SIZE); } @@ -61,6 +77,15 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) { } /* 8-byte alignment */ n = (n + ARENA_ALIGN) & ~(size_t)ARENA_ALIGN; + if (n <= 64) { + a->alloc_le_64 += n; + } else if (n <= 256) { + a->alloc_le_256 += n; + } else if (n <= 4096) { + a->alloc_le_4096 += n; + } else { + a->alloc_gt_4096 += n; + } if (a->nblocks == 0) { return NULL; } @@ -75,6 +100,41 @@ void *cbm_arena_alloc(CBMArena *a, size_t n) { return ptr; } +void *cbm_arena_realloc(CBMArena *a, void *ptr, size_t n) { + if (!a || n == 0) { + return NULL; + } + cbm_arena_heap_allocation_t *allocation = (cbm_arena_heap_allocation_t *)a->heap_allocations; + if (ptr) { + while (allocation && allocation->ptr != ptr) { + allocation = allocation->next; + } + if (!allocation) { + return NULL; + } + void *grown = realloc(ptr, n); + if (!grown) { + return NULL; + } + allocation->ptr = grown; + return grown; + } + + void *created = malloc(n); + if (!created) { + return NULL; + } + allocation = (cbm_arena_heap_allocation_t *)malloc(sizeof(*allocation)); + if (!allocation) { + free(created); + return NULL; + } + allocation->ptr = created; + allocation->next = (cbm_arena_heap_allocation_t *)a->heap_allocations; + a->heap_allocations = allocation; + return created; +} + void *cbm_arena_calloc(CBMArena *a, size_t n) { void *p = cbm_arena_alloc(a, n); if (p) { @@ -90,6 +150,7 @@ char *cbm_arena_strdup(CBMArena *a, const char *s) { size_t len = strlen(s); char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE); if (dst) { + a->strdup_alloc += len + SKIP_ONE; memcpy(dst, s, len + SKIP_ONE); } return dst; @@ -101,6 +162,7 @@ char *cbm_arena_strndup(CBMArena *a, const char *s, size_t len) { } char *dst = (char *)cbm_arena_alloc(a, len + SKIP_ONE); if (dst) { + a->strdup_alloc += len + SKIP_ONE; memcpy(dst, s, len); dst[len] = '\0'; } @@ -120,6 +182,7 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) { if (!dst) { return NULL; } + a->sprintf_alloc += (size_t)needed + SKIP_ONE; va_start(args, fmt); vsnprintf(dst, (size_t)needed + SKIP_ONE, fmt, args); @@ -128,6 +191,7 @@ char *cbm_arena_sprintf(CBMArena *a, const char *fmt, ...) { } void cbm_arena_reset(CBMArena *a) { + arena_free_heap_allocations(a); /* Keep first block, free the rest */ for (int i = SKIP_ONE; i < a->nblocks; i++) { free(a->blocks[i]); @@ -147,6 +211,7 @@ void cbm_arena_reset(CBMArena *a) { } void cbm_arena_destroy(CBMArena *a) { + arena_free_heap_allocations(a); for (int i = 0; i < a->nblocks; i++) { free(a->blocks[i]); } diff --git a/src/foundation/arena.h b/src/foundation/arena.h index 663f07459..83d9964bf 100644 --- a/src/foundation/arena.h +++ b/src/foundation/arena.h @@ -21,9 +21,16 @@ typedef struct { char *blocks[CBM_ARENA_MAX_BLOCKS]; size_t block_sizes[CBM_ARENA_MAX_BLOCKS]; /* per-block sizes (for stats) */ int nblocks; - size_t block_size; /* current block capacity */ - size_t used; /* bytes used in current block */ - size_t total_alloc; /* cumulative bytes allocated (for stats) */ + size_t block_size; /* current block capacity */ + size_t used; /* bytes used in current block */ + size_t total_alloc; /* cumulative bytes allocated (for stats) */ + size_t strdup_alloc; /* requested bytes returned by strdup/strndup helpers */ + size_t sprintf_alloc; /* requested bytes returned by sprintf helper */ + size_t alloc_le_64; + size_t alloc_le_256; + size_t alloc_le_4096; + size_t alloc_gt_4096; + void *heap_allocations; /* private list used by cbm_arena_realloc() */ } CBMArena; /* Initialize arena with default block size. */ @@ -35,6 +42,11 @@ void cbm_arena_init_sized(CBMArena *a, size_t block_size); /* Allocate n bytes (8-byte aligned). Returns NULL on OOM. */ void *cbm_arena_alloc(CBMArena *a, size_t n); +/* Grow a heap allocation whose lifetime is owned by this arena. Passing NULL + * creates a tracked allocation; subsequent calls must use the same arena. + * Tracked allocations are released by reset/destroy alongside bump blocks. */ +void *cbm_arena_realloc(CBMArena *a, void *ptr, size_t n); + /* Allocate n bytes, zero-initialized. */ void *cbm_arena_calloc(CBMArena *a, size_t n); diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 9174391f4..1f5e54421 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -22,11 +22,17 @@ * non-ASCII repo path survives CreateProcess (#423/#20) */ #include /* free */ #else +#ifndef __APPLE__ +#include +#endif #include #include #include #ifdef __APPLE__ +#include #include +#include +#include extern char **environ; #endif #include @@ -423,6 +429,7 @@ struct cbm_subprocess { bool root_reaped; uint64_t force_started_ms; bool containment_failed; + size_t observed_tree_rss; cbm_proc_result_t result; #ifdef _WIN32 @@ -436,6 +443,116 @@ struct cbm_subprocess { #endif }; +static bool cbm_subprocess_tree_rss_sample(cbm_subprocess_t *process, size_t *bytes_out) { + *bytes_out = 0; +#ifdef _WIN32 + JOBOBJECT_EXTENDED_LIMIT_INFORMATION information; + memset(&information, 0, sizeof(information)); + if (!process->job || !QueryInformationJobObject(process->job, JobObjectExtendedLimitInformation, + &information, sizeof(information), NULL)) { + return false; + } + *bytes_out = (size_t)information.PeakJobMemoryUsed; + return true; +#elif defined(__APPLE__) + int query[4] = {CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0}; + size_t length = 0; + if (sysctl(query, 3, NULL, &length, NULL, 0) != 0 || length == 0) { + return false; + } + struct kinfo_proc *processes = malloc(length); + if (!processes || sysctl(query, 3, processes, &length, NULL, 0) != 0) { + free(processes); + return false; + } + size_t total = 0; + size_t count = length / sizeof(*processes); + for (size_t i = 0; i < count; i++) { + if (processes[i].kp_eproc.e_pgid != process->pgid) { + continue; + } + struct rusage_info_v2 usage; + memset(&usage, 0, sizeof(usage)); + if (proc_pid_rusage(processes[i].kp_proc.p_pid, RUSAGE_INFO_V2, (rusage_info_t *)&usage) == + 0) { + size_t resident = (size_t)usage.ri_resident_size; + total = resident > SIZE_MAX - total ? SIZE_MAX : total + resident; + } + } + free(processes); + *bytes_out = total; + return true; +#else + DIR *directory = opendir("/proc"); + if (!directory) { + return false; + } + long page_size = sysconf(_SC_PAGESIZE); + size_t total = 0; + struct dirent *entry = NULL; + while (page_size > 0 && (entry = readdir(directory)) != NULL) { + char *end = NULL; + long pid = strtol(entry->d_name, &end, 10); + if (pid <= 0 || !end || *end != '\0') { + continue; + } + char path[128]; + (void)snprintf(path, sizeof(path), "/proc/%ld/stat", pid); + FILE *stat_file = fopen(path, "r"); + char stat_line[4096]; + if (!stat_file || !fgets(stat_line, sizeof(stat_line), stat_file)) { + if (stat_file) { + (void)fclose(stat_file); + } + continue; + } + (void)fclose(stat_file); + char *command_end = strrchr(stat_line, ')'); + char state = 0; + long parent = 0; + long group = 0; + if (!command_end || sscanf(command_end + 1, " %c %ld %ld", &state, &parent, &group) != 3 || + group != process->pgid) { + continue; + } + (void)snprintf(path, sizeof(path), "/proc/%ld/statm", pid); + FILE *memory_file = fopen(path, "r"); + unsigned long pages = 0; + unsigned long resident_pages = 0; + bool read = memory_file && fscanf(memory_file, "%lu %lu", &pages, &resident_pages) == 2; + if (memory_file) { + (void)fclose(memory_file); + } + if (read) { + size_t resident = resident_pages > SIZE_MAX / (size_t)page_size + ? SIZE_MAX + : (size_t)resident_pages * (size_t)page_size; + total = resident > SIZE_MAX - total ? SIZE_MAX : total + resident; + } + } + (void)closedir(directory); + *bytes_out = total; + return true; +#endif +} + +bool cbm_subprocess_observed_tree_rss(cbm_subprocess_t *process, size_t *bytes_out) { + if (!bytes_out) { + return false; + } + *bytes_out = 0; + if (!process) { + return false; + } + size_t sample = 0; + bool sampled = cbm_subprocess_tree_rss_sample(process, &sample); + if (sampled && sample > process->observed_tree_rss) { + process->observed_tree_rss = sample; + } + *bytes_out = process->observed_tree_rss; + return sampled || process->observed_tree_rss > 0; +} + static void cbm_subprocess_result_init(cbm_proc_result_t *result) { result->outcome = CBM_PROC_SPAWN_FAILED; result->exit_code = -1; diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 62592c592..67e2ca501 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -125,6 +125,12 @@ cbm_proc_poll_t cbm_subprocess_poll(cbm_subprocess_t *process, cbm_proc_result_t * performs signal delivery/escalation. */ bool cbm_subprocess_request_cancel(cbm_subprocess_t *process); +/* Return the largest observed resident byte total for the complete contained + * process tree. The sample is refreshed on each call while the tree is live; + * after exit the cached high-water mark remains available until destroy. + * false means the platform sample was unavailable and leaves *bytes_out zero. */ +bool cbm_subprocess_observed_tree_rss(cbm_subprocess_t *process, size_t *bytes_out); + /* Release a terminal handle. This never waits or implicitly cancels; passing a * still-running handle violates the API contract. NULL is a no-op. */ void cbm_subprocess_destroy(cbm_subprocess_t *process); diff --git a/src/main.c b/src/main.c index 1304c2093..dad5d05a2 100644 --- a/src/main.c +++ b/src/main.c @@ -1449,6 +1449,23 @@ static bool main_set_client_context(cbm_daemon_runtime_client_t *client, const c CBM_DAEMON_RUNTIME_APPLICATION_OK; } +/* A daemon worker inherits the daemon process environment, whose + * CBM_ALLOWED_ROOT belongs to whichever frontend started that daemon. The + * daemon has already authorized and canonicalized repo_path before spawning + * this single-request worker, so replace that stale process boundary with the + * exact request root. This is worker-local and fail-closed: it never mutates + * the concurrent daemon and cannot broaden the job beyond the one canonical + * repository encoded in the immutable worker argv. */ +static bool main_install_index_worker_request_scope(const char *args_json) { + char *repo_path = cbm_mcp_get_string_arg(args_json, "repo_path"); + char canonical[MAIN_PATH_CAP]; + bool installed = repo_path && repo_path[0] && + cbm_canonical_path(repo_path, canonical, sizeof(canonical)) && + cbm_setenv("CBM_ALLOWED_ROOT", canonical, 1) == 0; + free(repo_path); + return installed; +} + /* Parse a strict MAJOR.MINOR.PATCH triple; false for anything else (dev * builds and prereleases never participate in auto-drain decisions). */ static bool main_semver_triple(const char *text, long out[3]) { @@ -2710,6 +2727,11 @@ int main(int argc, char **argv) { char *worker_repo_path = cbm_mcp_get_string_arg(invocation.args_json, "repo_path"); cbm_index_worker_log_begin(invocation.args_json, worker_repo_path); free(worker_repo_path); + if (!main_install_index_worker_request_scope(invocation.args_json)) { + (void)fprintf(stderr, + "CBM index worker could not start: request workspace scope invalid\n"); + return EXIT_FAILURE; + } cbm_daemon_ipc_endpoint_t *worker_endpoint = cbm_daemon_bootstrap_endpoint_new(NULL); cbm_project_lock_manager_t *worker_project_locks = worker_endpoint ? cbm_project_lock_manager_new(worker_endpoint) : NULL; diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index b5257dfc1..d2fadff2b 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -779,6 +779,10 @@ bool cbm_index_worker_request_cancel(cbm_index_worker_handle_t *handle) { cbm_subprocess_request_cancel(handle->process); } +bool cbm_index_worker_observed_tree_rss(cbm_index_worker_handle_t *handle, size_t *bytes_out) { + return handle && cbm_subprocess_observed_tree_rss(handle->process, bytes_out); +} + const char *cbm_index_worker_response_path(const cbm_index_worker_handle_t *handle) { return handle ? handle->response_path : NULL; } diff --git a/src/mcp/index_supervisor.h b/src/mcp/index_supervisor.h index 31259df90..9483c66ae 100644 --- a/src/mcp/index_supervisor.h +++ b/src/mcp/index_supervisor.h @@ -179,6 +179,7 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, * bounded containment failure is explicitly surfaced in the result). The * owner must stop concurrent cancellation producers before destroy. */ bool cbm_index_worker_request_cancel(cbm_index_worker_handle_t *handle); +bool cbm_index_worker_observed_tree_rss(cbm_index_worker_handle_t *handle, size_t *bytes_out); /* Borrowed diagnostic paths, stable until destroy. Every start uses securely * created unique files, so concurrent jobs in one daemon cannot collide. The diff --git a/src/pipeline/pass_complexity.c b/src/pipeline/pass_complexity.c index 4a3ebf85e..737b18422 100644 --- a/src/pipeline/pass_complexity.c +++ b/src/pipeline/pass_complexity.c @@ -28,6 +28,28 @@ enum { CBM_TLD_MAX_DEPTH = 256 }; /* recursion-depth cap (cycle/stack guard) */ +typedef struct { + int64_t id; + const char *qualified_name; +} tld_target_t; + +static int compare_tld_target(const void *a, const void *b) { + const tld_target_t *ta = a; + const tld_target_t *tb = b; + const char *aq = ta->qualified_name ? ta->qualified_name : ""; + const char *bq = tb->qualified_name ? tb->qualified_name : ""; + int by_name = strcmp(aq, bq); + return by_name != 0 ? by_name : (ta->id > tb->id) - (ta->id < tb->id); +} + +static int compare_node_qn(const void *a, const void *b) { + const cbm_gbuf_node_t *na = *(cbm_gbuf_node_t *const *)a; + const cbm_gbuf_node_t *nb = *(cbm_gbuf_node_t *const *)b; + const char *aq = na && na->qualified_name ? na->qualified_name : ""; + const char *bq = nb && nb->qualified_name ? nb->qualified_name : ""; + return strcmp(aq, bq); +} + /* Int → string for structured logging (thread-safe ring buffer). */ static const char *itoa_cx(int val) { enum { RING = 2, MASK = 1 }; @@ -121,8 +143,36 @@ static int tld_dfs(const cbm_gbuf_t *gb, int64_t id, const int *loop_depth, int const cbm_gbuf_edge_t **edges = NULL; int ne = 0; cbm_gbuf_find_edges_by_source_type(gb, id, "CALLS", &edges, &ne); + tld_target_t *targets = ne > 0 ? malloc((size_t)ne * sizeof(*targets)) : NULL; + if (ne > 0 && !targets) { + /* Preserve the pass's prior best-effort behavior under allocation + * pressure; determinism is guaranteed whenever the tiny sort buffer + * can be allocated. */ + for (int i = 0; i < ne; i++) { + int64_t c = edges[i]->target_id; + if (c == id) { + recursive[id] = true; + continue; + } + int ct = tld_dfs(gb, c, loop_depth, tld, state, recursive, maxid, depth + 1); + if (ct > best) { + best = ct; + } + } + tld[id] = loop_depth[id] + best; + state[id] = 2; + return tld[id]; + } + int nt = 0; for (int i = 0; i < ne; i++) { - int64_t c = edges[i]->target_id; + const cbm_gbuf_node_t *callee = cbm_gbuf_find_by_id(gb, edges[i]->target_id); + if (callee) { + targets[nt++] = (tld_target_t){edges[i]->target_id, callee->qualified_name}; + } + } + qsort(targets, (size_t)nt, sizeof(*targets), compare_tld_target); + for (int i = 0; i < nt; i++) { + int64_t c = targets[i].id; if (c == id) { recursive[id] = true; /* direct self-recursion */ continue; @@ -132,6 +182,7 @@ static int tld_dfs(const cbm_gbuf_t *gb, int64_t id, const int *loop_depth, int best = ct; } } + free(targets); tld[id] = loop_depth[id] + best; state[id] = 2; return tld[id]; @@ -185,16 +236,28 @@ void cbm_pipeline_pass_complexity(cbm_pipeline_ctx_t *ctx) { seed_loop_depths(gb, "Function", loop_depth, recursive, nptr, maxid); seed_loop_depths(gb, "Method", loop_depth, recursive, nptr, maxid); + cbm_gbuf_node_t **ordered = malloc(sz * sizeof(*ordered)); + if (!ordered) { + free(loop_depth); + free(tld); + free(state); + free(recursive); + free(nptr); + return; + } int updated = 0; for (int64_t id = 1; id <= maxid; id++) { - if (!nptr[id]) { - continue; /* only Function/Method nodes */ + if (nptr[id]) { + ordered[updated++] = nptr[id]; } + } + qsort(ordered, (size_t)updated, sizeof(*ordered), compare_node_qn); + for (int i = 0; i < updated; i++) { + int64_t id = ordered[i]->id; if (state[id] != 2) { tld_dfs(gb, id, loop_depth, tld, state, recursive, maxid, 0); } append_complexity_props(nptr[id], tld[id], recursive[id]); - updated++; } cbm_log_info("pass.complexity", "functions", itoa_cx(updated)); @@ -204,4 +267,5 @@ void cbm_pipeline_pass_complexity(cbm_pipeline_ctx_t *ctx) { free(state); free(recursive); free(nptr); + free(ordered); } diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 1eeb55f83..45298d501 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -210,6 +210,7 @@ static cbm_parallel_extract_opts_t cbm_parallel_extract_resolve_opts( if (opts->retain_per_file_max_bytes > 0) { resolved.retain_per_file_max_bytes = opts->retain_per_file_max_bytes; } + resolved.backpressure_futile = opts->backpressure_futile; } /* Correctness invariant: a single file can never exceed the total budget. */ @@ -666,7 +667,7 @@ typedef struct { * in-flight transients, holds the memory, so napping cannot reclaim it. * While set, pulls skip the nap (the designed soft overshoot); the cheap * over-budget probe re-arms the gate once RSS drains under budget. */ - _Atomic int bp_futile; + _Atomic int *bp_futile; const CBMMacroTable *macro_table; /* ObjectScript $$$macros (NULL if none) */ const CBMReturnTypeTable *return_type_table; /* ObjectScript return types (NULL if none) */ @@ -760,7 +761,7 @@ static void extract_worker(int worker_id, void *ctx_ptr) { * the gate as soon as RSS drains under budget. */ if (cbm_mem_budget() > 0) { bool over = cbm_mem_over_budget(); - bool futile = atomic_load_explicit(&ec->bp_futile, memory_order_relaxed) != 0; + bool futile = atomic_load_explicit(ec->bp_futile, memory_order_relaxed) != 0; if (over && !futile) { cbm_mem_collect(); atomic_fetch_add_explicit(&g_bp_nap_cycles, SKIP_ONE, memory_order_relaxed); @@ -775,12 +776,12 @@ static void extract_worker(int worker_id, void *ctx_ptr) { /* Log only the 0→1 transition: all workers race into the * gate before anyone latches, so a plain store would WARN * once per worker (12 lines per latch event). */ - if (atomic_exchange_explicit(&ec->bp_futile, 1, memory_order_relaxed) == 0) { + if (atomic_exchange_explicit(ec->bp_futile, 1, memory_order_relaxed) == 0) { cbm_log_warn("mem.backpressure.futile", "action", "soft_overshoot"); } } } else if (!over && futile) { - atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed); + atomic_store_explicit(ec->bp_futile, 0, memory_order_relaxed); } } @@ -1019,6 +1020,110 @@ static void log_extract_mem_stats(int worker_count) { } } +static void log_extract_cache_mem(const char *phase, const cbm_file_info_t *files, + CBMFileResult **result_cache, int file_count, + int64_t retained_bytes) { + size_t arena_requested = 0; + size_t arena_capacity = 0; + size_t arena_blocks = 0; + size_t live_array_capacity = 0; + size_t strdup_requested = 0; + size_t sprintf_requested = 0; + size_t alloc_le_64 = 0; + size_t alloc_le_256 = 0; + size_t alloc_le_4096 = 0; + size_t alloc_gt_4096 = 0; + size_t max_requested = 0; + const char *max_requested_path = ""; + int over_1m = 0; + int over_8m = 0; + int over_32m = 0; + int over_128m = 0; + int result_count = 0; + for (int i = 0; i < file_count; i++) { + CBMFileResult *result = result_cache[i]; + if (!result) { + continue; + } + result_count++; + arena_requested += result->arena.total_alloc; + strdup_requested += result->arena.strdup_alloc; + sprintf_requested += result->arena.sprintf_alloc; + alloc_le_64 += result->arena.alloc_le_64; + alloc_le_256 += result->arena.alloc_le_256; + alloc_le_4096 += result->arena.alloc_le_4096; + alloc_gt_4096 += result->arena.alloc_gt_4096; + if (result->arena.total_alloc > max_requested) { + max_requested = result->arena.total_alloc; + max_requested_path = files[i].rel_path ? files[i].rel_path : ""; + } + over_1m += result->arena.total_alloc >= (size_t)1024 * 1024; + over_8m += result->arena.total_alloc >= (size_t)8 * 1024 * 1024; + over_32m += result->arena.total_alloc >= (size_t)32 * 1024 * 1024; + over_128m += result->arena.total_alloc >= (size_t)128 * 1024 * 1024; +#define ADD_ARRAY_CAPACITY(field) \ + live_array_capacity += (size_t)result->field.cap * sizeof(*result->field.items) + ADD_ARRAY_CAPACITY(defs); + ADD_ARRAY_CAPACITY(calls); + ADD_ARRAY_CAPACITY(imports); + ADD_ARRAY_CAPACITY(usages); + ADD_ARRAY_CAPACITY(throws); + ADD_ARRAY_CAPACITY(rw); + ADD_ARRAY_CAPACITY(type_refs); + ADD_ARRAY_CAPACITY(env_accesses); + ADD_ARRAY_CAPACITY(type_assigns); + ADD_ARRAY_CAPACITY(impl_traits); + ADD_ARRAY_CAPACITY(resolved_calls); + ADD_ARRAY_CAPACITY(string_refs); + ADD_ARRAY_CAPACITY(infra_bindings); + ADD_ARRAY_CAPACITY(channels); +#undef ADD_ARRAY_CAPACITY + arena_blocks += (size_t)result->arena.nblocks; + for (int block = 0; block < result->arena.nblocks; block++) { + arena_capacity += result->arena.block_sizes[block]; + } + } + enum { BYTES_PER_MIB = 1024 * 1024 }; + char results_text[CBM_SZ_32]; + char blocks_text[CBM_SZ_32]; + char requested_text[CBM_SZ_32]; + char capacity_text[CBM_SZ_32]; + char retained_text[CBM_SZ_32]; + char rss_text[CBM_SZ_32]; + char live_array_text[CBM_SZ_32]; + char strdup_text[CBM_SZ_32]; + char sprintf_text[CBM_SZ_32]; + char max_requested_text[CBM_SZ_32]; + char distribution_text[CBM_SZ_128]; + char size_distribution_text[CBM_SZ_128]; + (void)snprintf(results_text, sizeof(results_text), "%d", result_count); + (void)snprintf(blocks_text, sizeof(blocks_text), "%zu", arena_blocks); + (void)snprintf(requested_text, sizeof(requested_text), "%zu", arena_requested / BYTES_PER_MIB); + (void)snprintf(capacity_text, sizeof(capacity_text), "%zu", arena_capacity / BYTES_PER_MIB); + (void)snprintf(retained_text, sizeof(retained_text), "%lld", + (long long)(retained_bytes / BYTES_PER_MIB)); + (void)snprintf(rss_text, sizeof(rss_text), "%zu", cbm_mem_rss() / BYTES_PER_MIB); + (void)snprintf(live_array_text, sizeof(live_array_text), "%zu", + live_array_capacity / BYTES_PER_MIB); + (void)snprintf(strdup_text, sizeof(strdup_text), "%zu", strdup_requested / BYTES_PER_MIB); + (void)snprintf(sprintf_text, sizeof(sprintf_text), "%zu", sprintf_requested / BYTES_PER_MIB); + (void)snprintf(max_requested_text, sizeof(max_requested_text), "%zu", + max_requested / BYTES_PER_MIB); + (void)snprintf(distribution_text, sizeof(distribution_text), + "ge1m=%d,ge8m=%d,ge32m=%d,ge128m=%d", over_1m, over_8m, over_32m, over_128m); + (void)snprintf(size_distribution_text, sizeof(size_distribution_text), + "le64=%zu,le256=%zu,le4096=%zu,gt4096=%zu", alloc_le_64 / BYTES_PER_MIB, + alloc_le_256 / BYTES_PER_MIB, alloc_le_4096 / BYTES_PER_MIB, + alloc_gt_4096 / BYTES_PER_MIB); + cbm_log_info("parallel.extract.cache_mem", "phase", phase, "results", results_text, + "arena_blocks", blocks_text, "arena_requested_mb", requested_text, + "arena_capacity_mb", capacity_text, "live_array_capacity_mb", live_array_text, + "strdup_requested_mb", strdup_text, "sprintf_requested_mb", sprintf_text, + "max_requested_mb", max_requested_text, "max_requested_path", max_requested_path, + "requested_distribution", distribution_text, "retained_source_mb", retained_text, + "allocation_size_mib", size_distribution_text, "rss_mb", rss_text); +} + /* Forward declaration: macro table builder lives in pipeline.c (shared path). */ CBMMacroTable *cbm_build_macro_table_from_files(const cbm_file_info_t *files, int count, const char *repo_path); @@ -1027,6 +1132,10 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file CBMFileResult **result_cache, _Atomic int64_t *shared_ids, int worker_count, const cbm_parallel_extract_opts_t *opts) { cbm_parallel_extract_opts_t resolved_opts = cbm_parallel_extract_resolve_opts(opts); + _Atomic int local_bp_futile; + atomic_init(&local_bp_futile, 0); + _Atomic int *bp_futile = + resolved_opts.backpressure_futile ? resolved_opts.backpressure_futile : &local_bp_futile; if (file_count == 0) { return 0; @@ -1121,13 +1230,13 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file .retain_per_file_max_bytes = resolved_opts.retain_per_file_max_bytes, .macro_table = pp_macro_table, .return_type_table = ctx->return_type_table, + .bp_futile = bp_futile, }; atomic_init(&ec.next_worker_id, 0); atomic_init(&ec.next_file_idx, 0); atomic_init(&ec.retained_bytes, 0); atomic_init(&ec.retain_cap_warned, 0); atomic_init(&ec.oversized_warned, 0); - atomic_init(&ec.bp_futile, 0); /* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */ CBM_PROF_START(t_dispatch); @@ -1136,6 +1245,8 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file cbm_parallel_for(worker_count, extract_worker, &ec, parallel_opts); cbm_scale_end(&ec.scale); CBM_PROF_END_N("parallel_extract", "3_dispatch_workers_parallel", t_dispatch, file_count); + log_extract_cache_mem("post_dispatch", files, result_cache, file_count, + atomic_load_explicit(&ec.retained_bytes, memory_order_relaxed)); /* Sub-phase: Merge all local gbufs into main gbuf (SEQUENTIAL, gbuf not thread-safe) */ CBM_PROF_START(t_merge); @@ -1150,6 +1261,8 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file } } CBM_PROF_END_N("parallel_extract", "4_merge_gbufs_seq", t_merge, total_nodes); + log_extract_cache_mem("post_graph_merge", files, result_cache, file_count, + atomic_load_explicit(&ec.retained_bytes, memory_order_relaxed)); /* Merge per-worker skip lists into the pipeline (SEQUENTIAL — no lock). * Runs unconditionally (not gated on local_gbuf) so a worker whose files all @@ -1299,55 +1412,73 @@ static void create_channel_edges(cbm_pipeline_ctx_t *ctx, const CBMFileResult *r } } -int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, - int file_count, CBMFileResult **result_cache) { - cbm_log_info("parallel.registry.start", "files", itoa_log(file_count)); - +int cbm_register_definitions_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count, CBMFileResult **result_cache) { int reg_entries = 0; int defines_edges = 0; - int imports_edges = 0; - - /* Namespace/package → File-QN map for namespace imports (C# `using`, - * Java/Kotlin `import`, PHP `use`). Built from the full result cache so - * every declaring file is visible regardless of loop order. */ - const char **rels = (const char **)calloc((size_t)file_count, sizeof(char *)); - if (rels) { - for (int i = 0; i < file_count; i++) { - rels[i] = files[i].rel_path; - } - } - CBMHashTable *namespace_map = - cbm_pipeline_namespace_map_build(ctx->project_name, result_cache, rels, file_count); - free(rels); - for (int i = 0; i < file_count; i++) { if (cbm_pipeline_check_cancel(ctx)) { - cbm_pipeline_namespace_map_free(namespace_map); return CBM_NOT_FOUND; } - CBMFileResult *result = result_cache[i]; if (!result) { continue; } - const char *rel = files[i].rel_path; - - /* Register callable symbols + DEFINES/DEFINES_METHOD edges */ for (int d = 0; d < result->defs.count; d++) { defines_edges += register_and_link_def(ctx, &result->defs.items[d], rel, ®_entries); } + } + cbm_log_info("parallel.registry.definitions", "entries", itoa_log(reg_entries), "defines", + itoa_log(defines_edges)); + return 0; +} +int cbm_create_relationship_carriers_from_cache(cbm_pipeline_ctx_t *ctx, + const cbm_file_info_t *files, int file_count, + CBMFileResult **result_cache, + CBMHashTable *namespace_map) { + int imports_edges = 0; + for (int i = 0; i < file_count; i++) { + if (cbm_pipeline_check_cancel(ctx)) { + return CBM_NOT_FOUND; + } + CBMFileResult *result = result_cache[i]; + if (!result) { + continue; + } + const char *rel = files[i].rel_path; imports_edges += create_imports_edges(ctx, result, rel, namespace_map); create_channel_edges(ctx, result, rel); cbm_pipeline_create_env_configures_for_file(ctx, result, rel); } + cbm_log_info("parallel.registry.relationship_carriers", "imports", itoa_log(imports_edges)); + return 0; +} - cbm_pipeline_namespace_map_free(namespace_map); +int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count, CBMFileResult **result_cache) { + cbm_log_info("parallel.registry.start", "files", itoa_log(file_count)); - cbm_log_info("parallel.registry.done", "entries", itoa_log(reg_entries), "defines", - itoa_log(defines_edges), "imports", itoa_log(imports_edges)); - return 0; + const char **rels = (const char **)calloc((size_t)file_count, sizeof(char *)); + if (rels) { + for (int i = 0; i < file_count; i++) { + rels[i] = files[i].rel_path; + } + } + CBMHashTable *namespace_map = + cbm_pipeline_namespace_map_build(ctx->project_name, result_cache, rels, file_count); + free(rels); + + int rc = cbm_register_definitions_from_cache(ctx, files, file_count, result_cache); + if (rc == 0) { + rc = cbm_create_relationship_carriers_from_cache(ctx, files, file_count, result_cache, + namespace_map); + } + + cbm_pipeline_namespace_map_free(namespace_map); + cbm_log_info("parallel.registry.done", "status", rc == 0 ? "ok" : "failed"); + return rc; } /* ── Phase 4: Parallel Resolution ────────────────────────────────── */ @@ -3210,11 +3341,11 @@ static void resolve_worker(int worker_id, void *ctx_ptr) { cbm_service_pattern_cache_end(); } -int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, - CBMFileResult **result_cache, _Atomic int64_t *shared_ids, - int worker_count, CBMLSPDef *all_defs, int def_count, - char *const *def_modules, struct CBMModuleDefIndex *module_def_index, - void *cross_registries_v) { +int cbm_parallel_resolve_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, + CBMFileResult **result_cache, _Atomic int64_t *shared_ids, + int worker_count, CBMLSPDef *all_defs, int def_count, + char *const *def_modules, struct CBMModuleDefIndex *module_def_index, + void *cross_registries_v, bool finalize_graph) { /* See header: typed as void* across the TU boundary; cast back here. */ CBMCrossLspRegistries *cross_registries = (CBMCrossLspRegistries *)cross_registries_v; if (file_count == 0) { @@ -3318,12 +3449,13 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, cbm_aligned_free(workers); - /* Go-style implicit interface satisfaction (needs full graph, serial) */ - int go_impl = cbm_pipeline_implements_go(ctx); - - /* Explicit-language override detection (same serial full-graph tail the - * sequential pipeline runs — the two venues must emit identical graphs). */ - total_lsp_overrides += cbm_pipeline_override_explicit(ctx); + int go_impl = 0; + if (finalize_graph) { + /* These scans require the complete graph and therefore run once after + * the final batch in the bounded large-repository path. */ + go_impl = cbm_pipeline_implements_go(ctx); + total_lsp_overrides += cbm_pipeline_override_explicit(ctx); + } if (atomic_load(ctx->cancelled)) { return CBM_NOT_FOUND; @@ -3491,3 +3623,21 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, "per_lookup", tp_buf, "fallback_rows", fb_buf); return 0; } + +int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, + CBMFileResult **result_cache, _Atomic int64_t *shared_ids, + int worker_count, CBMLSPDef *all_defs, int def_count, + char *const *def_modules, struct CBMModuleDefIndex *module_def_index, + void *cross_registries_v) { + return cbm_parallel_resolve_ex(ctx, files, file_count, result_cache, shared_ids, worker_count, + all_defs, def_count, def_modules, module_def_index, + cross_registries_v, true); +} + +int cbm_parallel_resolve_finalize(cbm_pipeline_ctx_t *ctx) { + int go_impl = cbm_pipeline_implements_go(ctx); + int overrides = cbm_pipeline_override_explicit(ctx); + cbm_log_info("parallel.resolve.finalize", "go_implements", itoa_log(go_impl), "overrides", + itoa_log(overrides)); + return cbm_pipeline_check_cancel(ctx) ? CBM_NOT_FOUND : 0; +} diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index a44864659..1c81cf889 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -2066,13 +2066,13 @@ const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t /* ── Namespace map ───────────────────────────────────────────────── */ -CBMHashTable *cbm_pipeline_namespace_map_build(const char *project_name, - CBMFileResult *const *results, - const char *const *rels, int count) { +CBMHashTable *cbm_pipeline_namespace_map_build_names(const char *project_name, + const char *const *namespace_names, + const char *const *rels, int count) { CBMHashTable *map = NULL; for (int i = 0; i < count; i++) { - const CBMFileResult *r = results[i]; - if (!r || !r->namespace_name || !r->namespace_name[0] || !rels[i]) { + const char *namespace_name = namespace_names ? namespace_names[i] : NULL; + if (!namespace_name || !namespace_name[0] || !rels[i]) { continue; } if (!map) { @@ -2088,7 +2088,7 @@ CBMHashTable *cbm_pipeline_namespace_map_build(const char *project_name, /* Normalize the namespace key to dot-separated form so it matches the * dot-normalized lookups in cbm_pipeline_resolve_import_node (PHP uses * '\\', some grammars '::' or '/'). */ - char *key = strdup(r->namespace_name); + char *key = strdup(namespace_name); if (!key) { free(file_qn); continue; @@ -2128,6 +2128,22 @@ CBMHashTable *cbm_pipeline_namespace_map_build(const char *project_name, return map; } +CBMHashTable *cbm_pipeline_namespace_map_build(const char *project_name, + CBMFileResult *const *results, + const char *const *rels, int count) { + const char **namespace_names = calloc((size_t)count, sizeof(*namespace_names)); + if (!namespace_names && count > 0) { + return NULL; + } + for (int i = 0; i < count; i++) { + namespace_names[i] = results[i] ? results[i]->namespace_name : NULL; + } + CBMHashTable *map = + cbm_pipeline_namespace_map_build_names(project_name, namespace_names, rels, count); + free(namespace_names); + return map; +} + static void ns_map_free_entry(const char *key, void *value, void *ud) { (void)ud; free((void *)key); /* strdup'd in cbm_pipeline_namespace_map_build */ diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 938437549..b52239826 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -890,6 +890,67 @@ static bool route_sr_denied(const CBMStringRef *sr) { return is_upstream_config_key(sr->key_path); } +static CBMHashTable *cbm_pipeline_collect_infra_route_denials(const cbm_file_info_t *files, + CBMFileResult **result_cache, + int file_count, + CBMHashTable *denied) { + if (!denied) { + denied = cbm_ht_create(16); + } + if (!denied) { + return NULL; + } + for (int i = 0; i < file_count; i++) { + if (!result_cache[i] || !is_infra_file(files[i].rel_path) || + is_ci_tooling_config(files[i].rel_path)) { + continue; + } + for (int si = 0; si < result_cache[i]->string_refs.count; si++) { + const CBMStringRef *sr = &result_cache[i]->string_refs.items[si]; + if (sr->kind != CBM_STRREF_URL || !sr->value || !strstr(sr->value, "://") || + !route_sr_denied(sr) || cbm_ht_has(denied, sr->value)) { + continue; + } + char *owned = strdup(sr->value); + if (owned) { + cbm_ht_set(denied, owned, owned); + } + } + } + return denied; +} + +static void cbm_pipeline_emit_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_info_t *files, + CBMFileResult **result_cache, int file_count, + CBMHashTable *denied) { + for (int i = 0; i < file_count; i++) { + if (!result_cache[i] || !is_infra_file(files[i].rel_path) || + is_ci_tooling_config(files[i].rel_path)) { + continue; + } + for (int si = 0; si < result_cache[i]->string_refs.count; si++) { + const CBMStringRef *sr = &result_cache[i]->string_refs.items[si]; + if (sr->kind == CBM_STRREF_URL && sr->value && strstr(sr->value, "://") && + (!denied || !cbm_ht_has(denied, sr->value))) { + try_upsert_infra_route(gbuf, sr, files[i].rel_path); + } + } + } +} + +static void free_owned_string_entry(const char *key, void *value, void *userdata) { + (void)value; + (void)userdata; + free((void *)key); +} + +static void cbm_pipeline_free_infra_route_denials(CBMHashTable *denied) { + if (denied) { + cbm_ht_foreach(denied, free_owned_string_entry, NULL); + cbm_ht_free(denied); + } +} + static void cbm_pipeline_extract_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_info_t *files, CBMFileResult **result_cache, int file_count) { /* DENY-WINS-BY-VALUE: the same URL is often extracted as several string_refs @@ -898,29 +959,10 @@ static void cbm_pipeline_extract_infra_routes(cbm_gbuf_t *gbuf, const cbm_file_i * per-ref guard — e.g. a denied full path `registries.terraform-registry.url` * is defeated by a sibling leaf `url`. So pass 1 collects every URL value * denied under ANY of its refs; pass 2 mints only values never denied. (#521) */ - CBMHashTable *denied = cbm_ht_create(16); - for (int pass = 0; pass < 2; pass++) { - for (int i = 0; i < file_count; i++) { - if (!result_cache[i] || !is_infra_file(files[i].rel_path) || - is_ci_tooling_config(files[i].rel_path)) { - continue; - } - for (int si = 0; si < result_cache[i]->string_refs.count; si++) { - const CBMStringRef *sr = &result_cache[i]->string_refs.items[si]; - if (sr->kind != CBM_STRREF_URL || !sr->value || !strstr(sr->value, "://")) { - continue; - } - if (pass == 0) { - if (denied && route_sr_denied(sr)) { - cbm_ht_set(denied, sr->value, (void *)1); - } - } else if (!denied || !cbm_ht_has(denied, sr->value)) { - try_upsert_infra_route(gbuf, sr, files[i].rel_path); - } - } - } - } - cbm_ht_free(denied); + CBMHashTable *denied = + cbm_pipeline_collect_infra_route_denials(files, result_cache, file_count, NULL); + cbm_pipeline_emit_infra_routes(gbuf, files, result_cache, file_count, denied); + cbm_pipeline_free_infra_route_denials(denied); } /* Run decorator_tags, configlink, and route matching passes. */ @@ -1167,10 +1209,359 @@ static int run_sequential_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, return rc; } +enum { + CBM_LARGE_REPOSITORY_FILE_THRESHOLD = 10000, + CBM_LARGE_REPOSITORY_BATCH_FILES = 512, +}; + +static const size_t CBM_LARGE_REPOSITORY_SOURCE_BYTES = (size_t)512 * (size_t)1024 * (size_t)1024; + +int cbm_pipeline_streaming_batch_size(const cbm_file_info_t *files, int file_count) { + char value[CBM_SZ_32]; + if (cbm_safe_getenv("CBM_STREAMING_BATCH_FILES", value, sizeof(value), NULL) != NULL) { + char *end = NULL; + long parsed = strtol(value, &end, 10); + if (!end || *end != '\0' || parsed <= 0 || parsed > 4096) { + cbm_log_error("pipeline.streaming.invalid_batch", "value", value); + return -1; + } + return parsed > file_count ? file_count : (int)parsed; + } + + char mode[CBM_SZ_32]; + const char *configured = cbm_safe_getenv("CBM_INDEX_RESOURCE_MODE", mode, sizeof(mode), NULL); + if (configured && strcmp(mode, "daily") != 0 && strcmp(mode, "large") != 0 && + strcmp(mode, "auto") != 0) { + cbm_log_error("pipeline.resource_mode.invalid", "value", mode); + return -1; + } + if (configured && strcmp(mode, "daily") == 0) { + cbm_log_info("pipeline.resource_mode", "selected", "daily", "reason", "manual"); + return 0; + } + + size_t source_bytes = 0; + for (int i = 0; files && i < file_count; i++) { + size_t file_size = files[i].size > 0 ? (size_t)files[i].size : 0; + source_bytes = file_size > SIZE_MAX - source_bytes ? SIZE_MAX : source_bytes + file_size; + } + bool large = (configured && strcmp(mode, "large") == 0) || + file_count >= CBM_LARGE_REPOSITORY_FILE_THRESHOLD || + source_bytes >= CBM_LARGE_REPOSITORY_SOURCE_BYTES; + if (!large) { + cbm_log_info("pipeline.resource_mode", "selected", "daily", "reason", "auto"); + return 0; + } + cbm_log_info( + "pipeline.resource_mode", "selected", "large", "reason", + configured && strcmp(mode, "large") == 0 + ? "manual" + : (file_count >= CBM_LARGE_REPOSITORY_FILE_THRESHOLD ? "file_count" : "source_bytes")); + return file_count < CBM_LARGE_REPOSITORY_BATCH_FILES ? file_count + : CBM_LARGE_REPOSITORY_BATCH_FILES; +} + +static void free_result_cache(CBMFileResult **cache, int count) { + if (!cache) { + return; + } + for (int i = 0; i < count; i++) { + cbm_free_result(cache[i]); + } + free(cache); +} + +static int append_surface_rows(cbm_lsp_surface_row_t **all_rows, int *all_count, int *all_cap, + cbm_lsp_surface_row_t *rows, int count) { + if (count <= 0) { + free(rows); + return 0; + } + if (*all_count > INT32_MAX - count) { + cbm_store_free_lsp_surfaces(rows, count); + return -1; + } + int needed = *all_count + count; + if (needed > *all_cap) { + int new_cap = *all_cap > 0 ? *all_cap : 256; + while (new_cap < needed && new_cap <= INT32_MAX / 2) { + new_cap *= 2; + } + if (new_cap < needed) { + new_cap = needed; + } + cbm_lsp_surface_row_t *grown = realloc(*all_rows, (size_t)new_cap * sizeof(**all_rows)); + if (!grown) { + cbm_store_free_lsp_surfaces(rows, count); + return -1; + } + *all_rows = grown; + *all_cap = new_cap; + } + memcpy(*all_rows + *all_count, rows, (size_t)count * sizeof(*rows)); + *all_count = needed; + free(rows); /* element ownership transferred */ + return 0; +} + +static int rehydrate_surface_defs(CBMArena *arena, const cbm_lsp_surface_row_t *rows, int row_count, + CBMLSPDef **out_defs, int *out_count) { + CBMLSPDef *defs = NULL; + int count = 0; + int cap = 0; + for (int i = 0; i < row_count; i++) { + CBMLSPDef *file_defs = NULL; + int file_count = cbm_lsp_surface_defs_from_json(arena, rows[i].defs_json, &file_defs); + if (file_count < 0 || count > INT32_MAX - file_count) { + free(defs); + return -1; + } + int needed = count + file_count; + if (needed > cap) { + int new_cap = cap > 0 ? cap : 1024; + while (new_cap < needed && new_cap <= INT32_MAX / 2) { + new_cap *= 2; + } + if (new_cap < needed) { + new_cap = needed; + } + CBMLSPDef *grown = realloc(defs, (size_t)new_cap * sizeof(*defs)); + if (!grown) { + free(defs); + return -1; + } + defs = grown; + cap = new_cap; + } + if (file_count > 0) { + memcpy(defs + count, file_defs, (size_t)file_count * sizeof(*defs)); + count += file_count; + } + } + *out_defs = defs; + *out_count = count; + return 0; +} + +static int run_parallel_streaming_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, + const cbm_file_info_t *files, int file_count, + int worker_count, int batch_size, struct timespec *t) { + cbm_log_info("pipeline.mode", "mode", "parallel_streaming", "workers", itoa_buf(worker_count), + "files", itoa_buf(file_count), "batch_files", itoa_buf(batch_size)); + int rc = CBM_NOT_FOUND; + _Atomic int64_t shared_ids; + atomic_init(&shared_ids, cbm_gbuf_next_id(p->gbuf)); + char **namespace_names = calloc((size_t)file_count, sizeof(*namespace_names)); + const char **rels = calloc((size_t)file_count, sizeof(*rels)); + char **def_modules = calloc((size_t)file_count, sizeof(*def_modules)); + cbm_lsp_surface_row_t *surface_rows = NULL; + int surface_count = 0; + int surface_cap = 0; + CBMHashTable *infra_denied = NULL; + CBMHashTable *namespace_map = NULL; + CBMLSPDef *all_defs = NULL; + int def_count = 0; + CBMArena all_defs_arena; + bool all_defs_arena_live = false; + CBMModuleDefIndex *module_def_index = NULL; + CBMArena cross_lsp_arena; + bool cross_lsp_arena_live = false; + CBMCrossLspRegistries cross_registries = {0}; + CBMFileResult **cache = NULL; + int cache_count = 0; + if (!namespace_names || !rels || !def_modules) { + goto cleanup; + } + for (int i = 0; i < file_count; i++) { + rels[i] = files[i].rel_path; + } + + cbm_parallel_extract_opts_t extract_opts = { + .retain_sources = false, + .retain_sources_set = true, + }; + _Atomic int streaming_bp_futile; + atomic_init(&streaming_bp_futile, 0); + extract_opts.backpressure_futile = &streaming_bp_futile; + + cbm_clock_gettime(CLOCK_MONOTONIC, t); + for (int offset = 0; offset < file_count; offset += batch_size) { + cache_count = file_count - offset < batch_size ? file_count - offset : batch_size; + cache = calloc((size_t)cache_count, sizeof(*cache)); + if (!cache) { + goto cleanup; + } + rc = cbm_parallel_extract_ex(ctx, files + offset, cache_count, cache, &shared_ids, + worker_count, &extract_opts); + if (rc != 0 || check_cancel(p)) { + goto cleanup; + } + cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids)); + rc = cbm_register_definitions_from_cache(ctx, files + offset, cache_count, cache); + if (rc != 0) { + goto cleanup; + } + /* Registry linking inserts serial edges into the main graph. Carry its + * new watermark into the next extraction batch so worker-local IDs + * cannot collide with those edges. */ + atomic_store(&shared_ids, cbm_gbuf_next_id(p->gbuf)); + for (int i = 0; i < cache_count; i++) { + if (cache[i] && cache[i]->namespace_name) { + namespace_names[offset + i] = strdup(cache[i]->namespace_name); + if (!namespace_names[offset + i]) { + goto cleanup; + } + } + } + infra_denied = cbm_pipeline_collect_infra_route_denials(files + offset, cache, cache_count, + infra_denied); + + int *def_starts = calloc((size_t)cache_count + 1, sizeof(*def_starts)); + int batch_def_count = 0; + CBMLSPDef *batch_defs = + def_starts ? cbm_pxc_collect_all_defs(ctx, cache, files + offset, cache_count, + ctx->project_name, def_modules + offset, + &batch_def_count, def_starts) + : NULL; + cbm_lsp_surface_row_t *batch_rows = NULL; + int batch_row_count = 0; + if (!def_starts || + cbm_lsp_surface_build_rows(ctx->project_name, cache, files + offset, cache_count, + batch_defs, def_starts, &batch_rows, + &batch_row_count) != 0 || + append_surface_rows(&surface_rows, &surface_count, &surface_cap, batch_rows, + batch_row_count) != 0) { + free(batch_defs); + free(def_starts); + goto cleanup; + } + free(batch_defs); + free(def_starts); + free_result_cache(cache, cache_count); + cache = NULL; + cache_count = 0; + cbm_mem_collect(); + cbm_log_info("pipeline.streaming.pass_a_batch", "completed", + itoa_buf(offset + batch_size > file_count ? file_count : offset + batch_size), + "total", itoa_buf(file_count), "rss_mb", + itoa_buf((int)(cbm_mem_rss() / (1024 * 1024)))); + } + cbm_log_info("pass.timing", "pass", "streaming_pass_a", "elapsed_ms", + itoa_buf((int)elapsed_ms(*t))); + log_phase_mem("streaming_pass_a"); + + namespace_map = cbm_pipeline_namespace_map_build_names( + ctx->project_name, (const char *const *)namespace_names, rels, file_count); + cbm_arena_init(&all_defs_arena); + all_defs_arena_live = true; + if (rehydrate_surface_defs(&all_defs_arena, surface_rows, surface_count, &all_defs, + &def_count) != 0) { + goto cleanup; + } + module_def_index = all_defs ? cbm_pxc_build_module_def_index(all_defs, def_count) : NULL; + cbm_arena_init(&cross_lsp_arena); + cross_lsp_arena_live = true; + if (all_defs) { + cross_registries.go = cbm_go_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.python = + cbm_py_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.c = cbm_c_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.cs = cbm_cs_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + cross_registries.ts = cbm_ts_build_cross_registry(&cross_lsp_arena, all_defs, def_count); + } + + cbm_clock_gettime(CLOCK_MONOTONIC, t); + for (int offset = 0; offset < file_count; offset += batch_size) { + cache_count = file_count - offset < batch_size ? file_count - offset : batch_size; + cache = calloc((size_t)cache_count, sizeof(*cache)); + if (!cache) { + goto cleanup; + } + rc = cbm_parallel_extract_ex(ctx, files + offset, cache_count, cache, &shared_ids, + worker_count, &extract_opts); + if (rc != 0 || check_cancel(p)) { + goto cleanup; + } + cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids)); + rc = cbm_create_relationship_carriers_from_cache(ctx, files + offset, cache_count, cache, + namespace_map); + if (rc != 0) { + goto cleanup; + } + cbm_pipeline_emit_infra_routes(p->gbuf, files + offset, cache, cache_count, infra_denied); + cbm_pipeline_process_infra_bindings(p->gbuf, files + offset, cache, cache_count); + /* Carrier/infra creation materializes serial nodes and edges between + * the two parallel phases. This mirrors the full-cache pipeline's + * registry watermark handoff. */ + atomic_store(&shared_ids, cbm_gbuf_next_id(p->gbuf)); + rc = cbm_parallel_resolve_ex(ctx, files + offset, cache_count, cache, &shared_ids, + worker_count, all_defs, def_count, def_modules + offset, + module_def_index, &cross_registries, false); + if (rc != 0) { + goto cleanup; + } + cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids)); + free_result_cache(cache, cache_count); + cache = NULL; + cache_count = 0; + cbm_mem_collect(); + cbm_log_info("pipeline.streaming.pass_b_batch", "completed", + itoa_buf(offset + batch_size > file_count ? file_count : offset + batch_size), + "total", itoa_buf(file_count), "rss_mb", + itoa_buf((int)(cbm_mem_rss() / (1024 * 1024)))); + } + rc = cbm_parallel_resolve_finalize(ctx); + if (rc != 0) { + goto cleanup; + } + cbm_gbuf_set_next_id(p->gbuf, atomic_load(&shared_ids)); + cbm_log_info("pass.timing", "pass", "streaming_pass_b", "elapsed_ms", + itoa_buf((int)elapsed_ms(*t))); + log_phase_mem("streaming_pass_b"); + cbm_pipeline_set_lsp_surfaces(ctx->pipeline, surface_rows, surface_count); + surface_rows = NULL; + surface_count = 0; + + cbm_clock_gettime(CLOCK_MONOTONIC, t); + cbm_pipeline_pass_k8s(ctx, files, file_count); + cbm_log_info("pass.timing", "pass", "k8s", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); + rc = check_cancel(p) ? CBM_NOT_FOUND : 0; + +cleanup: + free_result_cache(cache, cache_count); + cbm_pipeline_namespace_map_free(namespace_map); + cbm_pipeline_free_infra_route_denials(infra_denied); + cbm_pxc_free_module_def_index(module_def_index); + if (cross_lsp_arena_live) { + cbm_arena_destroy(&cross_lsp_arena); + } + free(all_defs); + if (all_defs_arena_live) { + cbm_arena_destroy(&all_defs_arena); + } + cbm_store_free_lsp_surfaces(surface_rows, surface_count); + for (int i = 0; i < file_count; i++) { + free(namespace_names ? namespace_names[i] : NULL); + free(def_modules ? def_modules[i] : NULL); + } + free(namespace_names); + free(rels); + free(def_modules); + return rc; +} + /* Run the parallel pipeline path: extract, registry, resolve, infra, k8s. */ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, int worker_count, struct timespec *t) { + int batch_size = cbm_pipeline_streaming_batch_size(files, file_count); + if (batch_size < 0) { + return CBM_NOT_FOUND; + } + if (batch_size > 0 && batch_size < file_count) { + return run_parallel_streaming_pipeline(p, ctx, files, file_count, worker_count, batch_size, + t); + } cbm_log_info("pipeline.mode", "mode", "parallel", "workers", itoa_buf(worker_count), "files", itoa_buf(file_count)); _Atomic int64_t shared_ids; @@ -1184,6 +1575,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, int rc = cbm_parallel_extract(ctx, files, file_count, cache, &shared_ids, worker_count); cbm_log_info("pass.timing", "pass", "parallel_extract", "elapsed_ms", itoa_buf((int)elapsed_ms(*t))); + log_phase_mem("parallel_extract"); if (rc != 0 || check_cancel(p)) { for (int i = 0; i < file_count; i++) { cbm_free_result(cache[i]); @@ -1200,6 +1592,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, cbm_mem_collect(); cbm_log_info("mem.collect", "phase", "post_extract", "rss_mb", itoa_buf((int)(cbm_mem_rss() / (1024 * 1024)))); + log_phase_mem("post_extract_collect"); cbm_clock_gettime(CLOCK_MONOTONIC, t); rc = cbm_build_registry_from_cache(ctx, files, file_count, cache); cbm_log_info("pass.timing", "pass", "registry_build", "elapsed_ms", @@ -1353,6 +1746,7 @@ static int run_parallel_pipeline(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, } } free(cache); + log_phase_mem("post_resolve_release"); if (rc != 0) { return rc; } @@ -2126,10 +2520,12 @@ static int run_post_extraction(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, if (rc != 0) { return rc; } + log_phase_mem("tests_and_history"); CBM_PROF_START(t_predump); run_predump_passes(p, ctx); CBM_PROF_END("pipeline", "3_predump_passes_total", t_predump); + log_phase_mem("predump"); #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API if (cbm_pipeline_persist_test_take_cancel_after_predump()) { @@ -2145,6 +2541,7 @@ static int run_post_extraction(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, CBM_PROF_START(t_dump); rc = dump_and_persist_hashes(p, baseline_manifest, baseline_count, &t); CBM_PROF_END("pipeline", "4_dump_and_persist", t_dump); + log_phase_mem("dump_and_persist"); return rc; } @@ -2159,6 +2556,7 @@ static int run_extraction_phase(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx, pass_structure(p, files, file_count); CBM_PROF_END_N("pipeline", "pass_structure", t_struct, file_count); cbm_log_info("pass.timing", "pass", "structure", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); + log_phase_mem("structure"); if (check_cancel(p)) { return CBM_NOT_FOUND; } @@ -2243,6 +2641,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { CBM_PROF_END_N("pipeline", "1_discover", t_discover, file_count); cbm_log_info("pipeline.discover", "files", itoa_buf(file_count), "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); + log_phase_mem("discover"); if (rc != 0 || check_cancel(p)) { rc = CBM_NOT_FOUND; goto cleanup; @@ -2262,6 +2661,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { rc = CBM_PIPELINE_ABORT_PRESERVE_DB; goto cleanup; } + log_phase_mem("semantic_manifest"); /* Check for existing DB → try incremental or delete for reindex */ rc = try_incremental_or_delete_db(p, files, file_count, baseline_manifest, baseline_count, @@ -2347,6 +2747,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { CBM_PROF_END("pipeline", "TOTAL", t_pipeline_total); cleanup: + log_phase_mem("pre_cleanup"); cbm_pkgmap_free(cbm_pipeline_get_pkgmap()); cbm_pipeline_set_pkgmap(NULL); cbm_discover_free(files, file_count); @@ -2369,6 +2770,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_set_user_lang_config(NULL); cbm_userconfig_free(p->userconfig); p->userconfig = NULL; + log_phase_mem("post_cleanup"); return rc; } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index e686ac3b7..0fed4c93c 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -205,6 +205,9 @@ const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t CBMHashTable *cbm_pipeline_namespace_map_build(const char *project_name, CBMFileResult *const *results, const char *const *rels, int count); +CBMHashTable *cbm_pipeline_namespace_map_build_names(const char *project_name, + const char *const *namespace_names, + const char *const *rels, int count); void cbm_pipeline_namespace_map_free(CBMHashTable *map); /* Parse a manifest file and collect pkg entries. Returns true if basename matched. */ @@ -512,6 +515,9 @@ typedef struct { bool retain_sources_set; /* false keeps the default retain_sources policy */ size_t retain_total_budget_bytes; size_t retain_per_file_max_bytes; + /* Optional run-scoped latch shared by repeated streaming batches. When + * NULL, cbm_parallel_extract_ex owns a fresh latch for this invocation. */ + _Atomic int *backpressure_futile; } cbm_parallel_extract_opts_t; int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, @@ -526,6 +532,12 @@ int cbm_parallel_extract(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, * Registers callable symbols (Function/Method/Class) in ctx->registry. */ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, CBMFileResult **result_cache); +int cbm_register_definitions_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count, CBMFileResult **result_cache); +int cbm_create_relationship_carriers_from_cache(cbm_pipeline_ctx_t *ctx, + const cbm_file_info_t *files, int file_count, + CBMFileResult **result_cache, + CBMHashTable *namespace_map); /* Phase 4: Parallel call/usage/semantic resolution. * Each worker resolves calls, usages, throws, rw, inherits, decorates, @@ -559,6 +571,17 @@ int cbm_parallel_resolve(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, * Typed as void* here to dodge the typedef/tag ordering * problem — pass_parallel.c casts back to CBMCrossLspRegistries*. */ void *cross_registries); +int cbm_parallel_resolve_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, + CBMFileResult **result_cache, _Atomic int64_t *shared_ids, + int worker_count, CBMLSPDef *all_defs, int def_count, + char *const *def_modules, struct CBMModuleDefIndex *module_def_index, + void *cross_registries, bool finalize_graph); +int cbm_parallel_resolve_finalize(cbm_pipeline_ctx_t *ctx); + +/* Stage 5C resource-mode selector. Returns zero for the daily full-cache path, + * a positive bounded batch size for the large-repository path, or -1 for an + * invalid explicit override. Exposed for deterministic policy tests. */ +int cbm_pipeline_streaming_batch_size(const cbm_file_info_t *files, int file_count); /* Post-merge: create Route nodes for HTTP_CALLS/ASYNC_CALLS edges that * have url_path in properties but point to library functions instead of routes. diff --git a/tests/test_arena.c b/tests/test_arena.c index 788c1eb9c..dc2beae66 100644 --- a/tests/test_arena.c +++ b/tests/test_arena.c @@ -45,6 +45,38 @@ TEST(arena_alloc_zero) { PASS(); } +TEST(arena_realloc_tracks_lifetime) { + CBMArena a; + cbm_arena_init(&a); + unsigned char *p = (unsigned char *)cbm_arena_realloc(&a, NULL, 16); + ASSERT_NOT_NULL(p); + for (int i = 0; i < 16; i++) { + p[i] = (unsigned char)i; + } + p = (unsigned char *)cbm_arena_realloc(&a, p, 64); + ASSERT_NOT_NULL(p); + for (int i = 0; i < 16; i++) { + ASSERT_EQ(p[i], (unsigned char)i); + } + ASSERT_NOT_NULL(a.heap_allocations); + cbm_arena_reset(&a); + ASSERT_NULL(a.heap_allocations); + p = (unsigned char *)cbm_arena_realloc(&a, NULL, 32); + ASSERT_NOT_NULL(p); + cbm_arena_destroy(&a); + ASSERT_NULL(a.heap_allocations); + PASS(); +} + +TEST(arena_realloc_rejects_foreign_pointer) { + CBMArena a; + cbm_arena_init(&a); + unsigned char foreign = 0; + ASSERT_NULL(cbm_arena_realloc(&a, &foreign, 16)); + cbm_arena_destroy(&a); + PASS(); +} + TEST(arena_alloc_null_arena) { void *p = cbm_arena_alloc(NULL, 16); ASSERT_NULL(p); @@ -432,6 +464,8 @@ SUITE(arena) { RUN_TEST(arena_init_sized); RUN_TEST(arena_alloc_basic); RUN_TEST(arena_alloc_zero); + RUN_TEST(arena_realloc_tracks_lifetime); + RUN_TEST(arena_realloc_rejects_foreign_pointer); RUN_TEST(arena_alloc_null_arena); RUN_TEST(arena_alloc_alignment); RUN_TEST(arena_alloc_grows_blocks); diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index e0ef6b459..392ff6268 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1319,6 +1319,7 @@ typedef struct { atomic_bool scripted; atomic_int hold_destroy_attempt; atomic_bool release_destroy; + atomic_size_t observed_rss; atomic_int project_lock_attempts; atomic_int project_lock_acquisitions; cbm_project_lock_manager_t *project_locks; @@ -1329,6 +1330,7 @@ typedef struct { char marker_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; char quarantine_seen[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; + char repo_paths[APP_FAKE_MAX_ATTEMPTS][APP_TEST_PATH_CAP]; size_t memory_budgets[APP_FAKE_MAX_ATTEMPTS]; } app_fake_worker_context_t; @@ -1351,6 +1353,7 @@ static void app_fake_worker_context_init(app_fake_worker_context_t *context) { atomic_init(&context->scripted, false); atomic_init(&context->hold_destroy_attempt, -1); atomic_init(&context->release_destroy, false); + atomic_init(&context->observed_rss, 0); atomic_init(&context->project_lock_attempts, 0); atomic_init(&context->project_lock_acquisitions, 0); } @@ -1393,6 +1396,12 @@ static int app_fake_worker_start(void *opaque, const char *args_json, size_t mem atomic_init(&worker->cancelled, false); worker->result.exit_code = -1; if (worker->attempt < APP_FAKE_MAX_ATTEMPTS) { + char *repo_path = cbm_mcp_get_string_arg(args_json, "repo_path"); + if (repo_path) { + (void)snprintf(context->repo_paths[worker->attempt], APP_TEST_PATH_CAP, "%s", + repo_path); + } + free(repo_path); context->memory_budgets[worker->attempt] = memory_budget_bytes; if (marker_file) { (void)snprintf(context->marker_paths[worker->attempt], APP_TEST_PATH_CAP, "%s", @@ -1476,6 +1485,17 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, return CBM_INDEX_WORKER_POLL_RUNNING; } +static bool app_fake_worker_observed_rss(void *opaque, cbm_daemon_application_worker_t worker, + size_t *bytes_out) { + (void)worker; + app_fake_worker_context_t *context = opaque; + if (!context || !bytes_out) { + return false; + } + *bytes_out = atomic_load(&context->observed_rss); + return true; +} + static bool app_fake_worker_cancel(void *opaque, cbm_daemon_application_worker_t handle) { app_fake_worker_context_t *context = opaque; app_fake_worker_t *worker = handle; @@ -2953,6 +2973,95 @@ TEST(daemon_application_coalesces_semantically_identical_index_requests) { PASS(); } +/* Regression contract: a shared daemon application must retain the workspace + * boundary on the owning session. A second live client with a different root + * must neither inherit the first client's boundary nor require a common + * ancestor grant. */ +TEST(daemon_application_keeps_allowed_root_session_scoped) { + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.allow_completion, true); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = {.worker_ops = &worker_ops}; + cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *sessions[2] = {app_test_open(&callbacks, 31), + app_test_open(&callbacks, 32)}; + + char parent[APP_TEST_PATH_CAP]; + char roots[2][APP_TEST_PATH_CAP]; + (void)snprintf(parent, sizeof(parent), "%s/cbm-app-session-roots-XXXXXX", cbm_tmpdir()); + bool paths_ok = cbm_mkdtemp(parent) != NULL; + for (size_t i = 0; paths_ok && i < 2; i++) { + int written = + snprintf(roots[i], sizeof(roots[i]), "%s/%s", parent, i == 0 ? "alpha" : "beta"); + paths_ok = written > 0 && written < (int)sizeof(roots[i]) && cbm_mkdir_p(roots[i], 0700); + } + + uint8_t *contexts[2] = {NULL, NULL}; + uint32_t context_lengths[2] = {0, 0}; + uint8_t *tools[2] = {NULL, NULL}; + uint32_t tool_lengths[2] = {0, 0}; + char args[2][APP_TEST_PATH_CAP + 64]; + bool encoded = paths_ok && sessions[0] && sessions[1]; + for (size_t i = 0; encoded && i < 2; i++) { + int written = snprintf(args[i], sizeof(args[i]), "{\"repo_path\":\"%s\",\"mode\":\"fast\"}", + roots[i]); + encoded = written > 0 && written < (int)sizeof(args[i]) && + app_test_context_request(roots[i], roots[i], &contexts[i], &context_lengths[i]) && + app_test_tool_request("index_repository", args[i], &tools[i], &tool_lengths[i]); + } + + bool requests_ok = encoded; + uint8_t *responses[2] = {NULL, NULL}; + uint32_t response_lengths[2] = {0, 0}; + for (size_t i = 0; requests_ok && i < 2; i++) { + uint8_t *empty = NULL; + uint32_t empty_length = 0; + requests_ok = + app_test_request(&callbacks, sessions[i], contexts[i], context_lengths[i], &empty, + &empty_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK && + !empty && empty_length == 0; + free(empty); + } + for (size_t i = 0; requests_ok && i < 2; i++) { + requests_ok = + app_test_request(&callbacks, sessions[i], tools[i], tool_lengths[i], &responses[i], + &response_lengths[i]) == CBM_DAEMON_RUNTIME_APPLICATION_OK && + responses[i] && strstr((char *)responses[i], "indexed") && + !strstr((char *)responses[i], "outside the allowed root"); + } + + for (size_t i = 0; i < 2; i++) { + if (sessions[i]) { + callbacks.session_close(callbacks.context, sessions[i]); + } + } + (void)cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + + ASSERT_TRUE(requests_ok); + ASSERT_EQ(atomic_load(&fake.starts), 2); + ASSERT_EQ(atomic_load(&fake.destroys), 2); + + for (size_t i = 0; i < 2; i++) { + free(contexts[i]); + free(tools[i]); + free(responses[i]); + (void)cbm_rmdir(roots[i]); + } + (void)cbm_rmdir(parent); + PASS(); +} + TEST(daemon_application_fresh_request_does_not_reuse_terminal_subscribed_job) { enum { PRIOR_SUBSCRIBERS = 16 }; static const char stale_response[] = @@ -5138,9 +5247,345 @@ TEST(daemon_application_queues_explicit_index_behind_physical_job_limit) { PASS(); } +TEST(daemon_application_explicit_index_queue_is_fifo) { + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .worker_ops = &worker_ops, + .physical_job_limit = 1, + }; + cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + char roots[3][APP_TEST_PATH_CAP]; + bool roots_ok = true; + for (int i = 0; i < 3; i++) { + (void)snprintf(roots[i], sizeof(roots[i]), "%s/cbm-app-fifo-%d-XXXXXX", cbm_tmpdir(), i); + roots_ok = roots_ok && cbm_mkdtemp(roots[i]) != NULL; + } + app_index_thread_t first = { + .application = application, + .project = "fifo-first", + .root = roots[0], + .result = -1, + }; + cbm_thread_t first_thread; + bool first_started = application && roots_ok && + cbm_thread_create(&first_thread, 0, app_index_thread, &first) == 0; + bool first_admitted = first_started && app_wait_for_atomic_int(&fake.starts, 1); + + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *sessions[2] = { + app_test_open(&callbacks, 61), + app_test_open(&callbacks, 62), + }; + uint8_t *contexts[2] = {NULL, NULL}; + uint32_t context_lengths[2] = {0, 0}; + uint8_t *tools[2] = {NULL, NULL}; + uint32_t tool_lengths[2] = {0, 0}; + bool sessions_ok = first_admitted && sessions[0] && sessions[1]; + for (int i = 0; i < 2 && sessions_ok; i++) { + char args[APP_TEST_PATH_CAP + 32]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", roots[i + 1]); + sessions_ok = app_test_context_request(roots[i + 1], roots[i + 1], &contexts[i], + &context_lengths[i]) && + app_test_tool_request("index_repository", args, &tools[i], &tool_lengths[i]); + uint8_t *response = NULL; + uint32_t response_length = 0; + sessions_ok = + sessions_ok && + app_test_request(&callbacks, sessions[i], contexts[i], context_lengths[i], &response, + &response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(response); + } + + app_session_request_thread_t requests[2] = { + {.callbacks = &callbacks, + .session = sessions[0], + .payload = tools[0], + .payload_length = tool_lengths[0], + .status = -1}, + {.callbacks = &callbacks, + .session = sessions[1], + .payload = tools[1], + .payload_length = tool_lengths[1], + .status = -1}, + }; + cbm_thread_t request_threads[2]; + bool request_started[2] = {false, false}; + bool queued_in_order = sessions_ok; + for (int i = 0; i < 2 && queued_in_order; i++) { + int baseline = cbm_daemon_application_busy_queue_waits_for_test(); + request_started[i] = cbm_thread_create(&request_threads[i], 0, app_session_request_thread, + &requests[i]) == 0; + uint64_t deadline = cbm_now_ms() + APP_TEST_TIMEOUT_MS; + while (request_started[i] && cbm_now_ms() < deadline && + cbm_daemon_application_busy_queue_waits_for_test() <= baseline) { + cbm_usleep(1000); + } + queued_in_order = + request_started[i] && cbm_daemon_application_busy_queue_waits_for_test() > baseline; + } + + atomic_store(&fake.allow_completion, true); + bool joined = true; + for (int i = 0; i < 2; i++) { + if (request_started[i]) { + joined = cbm_thread_join(&request_threads[i]) == 0 && joined; + } + } + if (first_started) { + joined = cbm_thread_join(&first_thread) == 0 && joined; + } + bool responses_ok = joined; + for (int i = 0; i < 2; i++) { + responses_ok = responses_ok && requests[i].status == CBM_DAEMON_RUNTIME_APPLICATION_OK && + requests[i].response && + strstr((char *)requests[i].response, "indexed") != NULL; + } + const char *second_name = strrchr(roots[1], '/'); + const char *third_name = strrchr(roots[2], '/'); + const char *second_started_name = strrchr(fake.repo_paths[1], '/'); + const char *third_started_name = strrchr(fake.repo_paths[2], '/'); + bool fifo = atomic_load(&fake.starts) == 3 && second_name && third_name && + second_started_name && third_started_name && + strcmp(second_started_name, second_name) == 0 && + strcmp(third_started_name, third_name) == 0; + + for (int i = 0; i < 2; i++) { + if (sessions[i]) { + callbacks.session_close(callbacks.context, sessions[i]); + } + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + for (int i = 0; i < 2; i++) { + free(contexts[i]); + free(tools[i]); + free(requests[i].response); + } + for (int i = 0; i < 3; i++) { + (void)cbm_rmdir(roots[i]); + } + + ASSERT_TRUE(roots_ok); + ASSERT_TRUE(first_started); + ASSERT_TRUE(first_admitted); + ASSERT_TRUE(sessions_ok); + ASSERT_TRUE(queued_in_order); + ASSERT_TRUE(responses_ok); + ASSERT_TRUE(fifo); + ASSERT_TRUE(stopped); + PASS(); +} + +TEST(daemon_application_cancelled_queue_head_is_removed) { + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = {.worker_ops = &worker_ops, .physical_job_limit = 1}; + cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + cbm_daemon_application_set_permanent(application, true); + char roots[3][APP_TEST_PATH_CAP]; + bool roots_ok = true; + for (int i = 0; i < 3; i++) { + (void)snprintf(roots[i], sizeof(roots[i]), "%s/cbm-app-queue-cancel-%d-XXXXXX", + cbm_tmpdir(), i); + roots_ok = roots_ok && cbm_mkdtemp(roots[i]) != NULL; + } + app_index_thread_t first = { + .application = application, + .project = "queue-cancel-first", + .root = roots[0], + .result = -1, + }; + cbm_thread_t first_thread; + bool first_started = application && roots_ok && + cbm_thread_create(&first_thread, 0, app_index_thread, &first) == 0; + bool first_admitted = first_started && app_wait_for_atomic_int(&fake.starts, 1); + + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 63); + uint8_t *context = NULL; + uint32_t context_length = 0; + uint8_t *tool = NULL; + uint32_t tool_length = 0; + char args[APP_TEST_PATH_CAP + 32]; + (void)snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", roots[1]); + bool session_ok = first_admitted && session && + app_test_context_request(roots[1], roots[1], &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *context_response = NULL; + uint32_t context_response_length = 0; + session_ok = session_ok && + app_test_request(&callbacks, session, context, context_length, &context_response, + &context_response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(context_response); + int baseline = cbm_daemon_application_busy_queue_waits_for_test(); + app_session_request_thread_t queued = { + .callbacks = &callbacks, + .session = session, + .payload = tool, + .payload_length = tool_length, + .status = -1, + }; + cbm_thread_t request_thread; + bool request_started = + session_ok && + cbm_thread_create(&request_thread, 0, app_session_request_thread, &queued) == 0; + bool parked = false; + if (request_started) { + uint64_t deadline = cbm_now_ms() + APP_TEST_TIMEOUT_MS; + while (cbm_now_ms() < deadline) { + if (cbm_daemon_application_busy_queue_waits_for_test() > baseline) { + parked = true; + break; + } + cbm_usleep(1000); + } + } + if (parked) { + callbacks.session_cancel(callbacks.context, session); + } + bool request_joined = request_started && cbm_thread_join(&request_thread) == 0; + bool cancelled_without_start = request_joined && atomic_load(&fake.starts) == 1; + atomic_store(&fake.allow_completion, true); + if (first_started) { + (void)cbm_thread_join(&first_thread); + } + bool first_completed = app_wait_for_atomic_int(&fake.destroys, 1); + int next = cancelled_without_start && first_completed + ? cbm_daemon_application_index(application, "queue-cancel-next", roots[2]) + : -1; + int final_starts = atomic_load(&fake.starts); + + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + free(context); + free(tool); + free(queued.response); + for (int i = 0; i < 3; i++) { + (void)cbm_rmdir(roots[i]); + } + + ASSERT_TRUE(roots_ok); + ASSERT_TRUE(first_started); + ASSERT_TRUE(first_admitted); + ASSERT_TRUE(session_ok); + ASSERT_TRUE(request_started); + ASSERT_TRUE(parked); + ASSERT_TRUE(cancelled_without_start); + ASSERT_TRUE(first_completed); + ASSERT_EQ(next, 0); + ASSERT_EQ(final_starts, 2); + ASSERT_TRUE(stopped); + PASS(); +} + +TEST(daemon_application_daily_capacity_adapts_to_memory_tokens) { + cbm_daemon_application_config_t config = { + .physical_job_limit = 4, + .aggregate_memory_budget_bytes = (size_t)768 * (size_t)1024 * (size_t)1024, + }; + cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + size_t limit = cbm_daemon_application_physical_job_limit(application); + size_t worker_budget = cbm_daemon_application_worker_memory_budget_bytes(application); + cbm_daemon_application_free(application); + + ASSERT_EQ(limit, 2); + ASSERT_EQ(worker_budget, (size_t)384 * (size_t)1024 * (size_t)1024); + PASS(); +} + +TEST(daemon_application_observed_rss_blocks_unsafe_daily_admission) { + const size_t observed_rss = (size_t)500 * (size_t)1024 * (size_t)1024; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.observed_rss, observed_rss); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .observed_rss = app_fake_worker_observed_rss, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = { + .worker_ops = &worker_ops, + .physical_job_limit = 4, + .aggregate_memory_budget_bytes = (size_t)768 * (size_t)1024 * (size_t)1024, + }; + cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + char roots[2][APP_TEST_PATH_CAP]; + bool roots_ok = true; + for (int i = 0; i < 2; i++) { + (void)snprintf(roots[i], sizeof(roots[i]), "%s/cbm-app-rss-gate-%d-XXXXXX", cbm_tmpdir(), + i); + roots_ok = roots_ok && cbm_mkdtemp(roots[i]) != NULL; + } + app_index_thread_t first = { + .application = application, + .project = "rss-first", + .root = roots[0], + .result = -1, + }; + cbm_thread_t first_thread; + bool started = application && roots_ok && + cbm_thread_create(&first_thread, 0, app_index_thread, &first) == 0; + bool sampled = false; + if (started && app_wait_for_atomic_int(&fake.starts, 1)) { + uint64_t deadline = cbm_now_ms() + APP_TEST_TIMEOUT_MS; + while (cbm_now_ms() < deadline) { + if (cbm_daemon_application_observed_index_rss_for_test(application) >= observed_rss) { + sampled = true; + break; + } + cbm_usleep(1000); + } + } + int second = sampled ? cbm_daemon_application_index(application, "rss-second", roots[1]) : -1; + bool no_unsafe_start = atomic_load(&fake.starts) == 1; + atomic_store(&fake.allow_completion, true); + if (started) { + (void)cbm_thread_join(&first_thread); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + for (int i = 0; i < 2; i++) { + (void)cbm_rmdir(roots[i]); + } + + ASSERT_TRUE(roots_ok); + ASSERT_TRUE(started); + ASSERT_TRUE(sampled); + ASSERT_EQ(second, 1); + ASSERT_TRUE(no_unsafe_start); + ASSERT_TRUE(stopped); + PASS(); +} + TEST(daemon_application_default_limit_admits_four_and_rejects_fifth) { enum { DEFAULT_CAP_RUNNING = 4, DEFAULT_CAP_TOTAL = 5 }; - const size_t aggregate_budget = 4099; + const size_t aggregate_budget = (size_t)1536 * (size_t)1024 * (size_t)1024 + (size_t)3; app_fake_worker_context_t fake; app_fake_worker_context_init(&fake); cbm_daemon_application_worker_ops_t worker_ops = { @@ -5215,6 +5660,67 @@ TEST(daemon_application_default_limit_admits_four_and_rejects_fifth) { PASS(); } +TEST(daemon_application_large_repository_uses_single_slot_and_large_budget) { + app_env_backup_t mode_environment; + bool mode_saved = app_env_backup_capture(&mode_environment, "CBM_INDEX_RESOURCE_MODE"); + bool mode_set = mode_saved && cbm_setenv("CBM_INDEX_RESOURCE_MODE", "large", 1) == 0; + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = {.worker_ops = &worker_ops}; + cbm_daemon_application_t *application = mode_set ? cbm_daemon_application_new(&config) : NULL; + char first_root[APP_TEST_PATH_CAP]; + char second_root[APP_TEST_PATH_CAP]; + snprintf(first_root, sizeof(first_root), "%s/cbm-app-large-first-XXXXXX", cbm_tmpdir()); + snprintf(second_root, sizeof(second_root), "%s/cbm-app-large-second-XXXXXX", cbm_tmpdir()); + bool roots_ok = cbm_mkdtemp(first_root) != NULL && cbm_mkdtemp(second_root) != NULL; + app_index_thread_t first = { + .application = application, + .project = "large-first", + .root = first_root, + .result = -1, + }; + cbm_thread_t thread; + bool started = + application && roots_ok && cbm_thread_create(&thread, 0, app_index_thread, &first) == 0; + bool admitted = started && app_wait_for_atomic_int(&fake.starts, 1); + bool daily_set = admitted && cbm_setenv("CBM_INDEX_RESOURCE_MODE", "daily", 1) == 0; + int second = + daily_set ? cbm_daemon_application_index(application, "large-second", second_root) : -1; + bool one_worker = atomic_load(&fake.starts) == 1; + size_t assigned_budget = fake.memory_budgets[0]; + atomic_store(&fake.allow_completion, true); + if (started) { + (void)cbm_thread_join(&thread); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + bool mode_restored = app_env_backup_restore(&mode_environment); + (void)cbm_rmdir(first_root); + (void)cbm_rmdir(second_root); + + ASSERT_TRUE(mode_saved); + ASSERT_TRUE(mode_set); + ASSERT_TRUE(roots_ok); + ASSERT_TRUE(started); + ASSERT_TRUE(admitted); + ASSERT_TRUE(daily_set); + ASSERT_EQ(second, 1); + ASSERT_TRUE(one_worker); + ASSERT_EQ(assigned_budget, (size_t)3072 * 1024 * 1024); + ASSERT_EQ(first.result, 0); + ASSERT_TRUE(stopped); + ASSERT_TRUE(mode_restored); + PASS(); +} + /* RED on the former void destruction contract: free could retain an * application with live owners, but the daemon host had no way to observe that * refusal and freed the application's borrowed dependencies anyway. */ @@ -5369,6 +5875,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_update_generation_retries_cancelled_check); RUN_TEST(daemon_application_final_disconnect_cancels_and_joins_update_generation); RUN_TEST(daemon_application_coalesces_semantically_identical_index_requests); + RUN_TEST(daemon_application_keeps_allowed_root_session_scoped); RUN_TEST(daemon_application_fresh_request_does_not_reuse_terminal_subscribed_job); RUN_TEST(daemon_application_request_cancel_detaches_only_one_coalesced_subscriber); RUN_TEST(daemon_application_cancels_physical_job_only_after_final_session); @@ -5392,7 +5899,12 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_cancellation_between_recovery_attempts_stops_retry); RUN_TEST(daemon_application_thread_start_failure_rolls_back_job_reservation); RUN_TEST(daemon_application_queues_explicit_index_behind_physical_job_limit); + RUN_TEST(daemon_application_explicit_index_queue_is_fifo); + RUN_TEST(daemon_application_cancelled_queue_head_is_removed); + RUN_TEST(daemon_application_daily_capacity_adapts_to_memory_tokens); + RUN_TEST(daemon_application_observed_rss_blocks_unsafe_daily_admission); RUN_TEST(daemon_application_default_limit_admits_four_and_rejects_fifth); + RUN_TEST(daemon_application_large_repository_uses_single_slot_and_large_budget); RUN_TEST(daemon_application_free_reports_retained_live_ownership); RUN_TEST(daemon_application_rejects_clean_exit_when_process_tree_is_not_contained); } diff --git a/tests/test_discover.c b/tests/test_discover.c index 7e0007b11..c60c46712 100644 --- a/tests/test_discover.c +++ b/tests/test_discover.c @@ -475,6 +475,32 @@ TEST(discover_bounded_count_matches_shebang_discovery) { PASS(); } +TEST(discover_bounded_measure_stops_on_source_bytes) { + char *base = th_mktempdir("cbm_disc_measure"); + ASSERT(base != NULL); + th_write_file(TH_PATH(base, "first.c"), "1234567890"); + th_write_file(TH_PATH(base, "second.py"), "1234567890"); + + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + int limited_count = -1; + size_t limited_bytes = 0; + cbm_discover_status_t limited = cbm_discover_measure_bounded( + base, &opts, 100, 15, cbm_now_ms() + 2000, &limited_count, &limited_bytes); + int exact_count = -1; + size_t exact_bytes = 0; + cbm_discover_status_t exact = cbm_discover_measure_bounded( + base, &opts, 100, 20, cbm_now_ms() + 2000, &exact_count, &exact_bytes); + + th_cleanup(base); + ASSERT_EQ(limited, CBM_DISCOVER_LIMIT_EXCEEDED); + ASSERT_EQ(limited_count, 1); + ASSERT_EQ(limited_bytes, 10); + ASSERT_EQ(exact, CBM_DISCOVER_OK); + ASSERT_EQ(exact_count, 2); + ASSERT_EQ(exact_bytes, 20); + PASS(); +} + TEST(discover_skips_git_dir) { char *base = th_mktempdir("cbm_disc_git"); ASSERT(base != NULL); @@ -1767,6 +1793,7 @@ SUITE(discover) { RUN_TEST(discover_bounded_count_is_allocation_free_and_limit_exact); RUN_TEST(discover_bounded_count_fails_closed_after_deadline); RUN_TEST(discover_bounded_count_matches_shebang_discovery); + RUN_TEST(discover_bounded_measure_stops_on_source_bytes); RUN_TEST(discover_skips_git_dir); RUN_TEST(discover_with_gitignore); RUN_TEST(discover_with_global_xdg_ignore); diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 5b7e6a8f7..2aff472a1 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1225,19 +1225,18 @@ TEST(form_procedure) { /* --- Oracle PL/SQL --- */ TEST(plsql_package_and_call) { - const char *src = - "CREATE OR REPLACE PACKAGE BODY emp_pkg AS\n" - " FUNCTION hire(p_name VARCHAR2) RETURN NUMBER IS\n" - " v_sal NUMBER;\n" - " BEGIN\n" - " v_sal := util_pkg.calc_salary(p_name);\n" - " IF v_sal > 0 THEN\n" - " RETURN v_sal;\n" - " END IF;\n" - " RAISE no_data_found;\n" - " END;\n" - "END emp_pkg;\n" - "/\n"; + const char *src = "CREATE OR REPLACE PACKAGE BODY emp_pkg AS\n" + " FUNCTION hire(p_name VARCHAR2) RETURN NUMBER IS\n" + " v_sal NUMBER;\n" + " BEGIN\n" + " v_sal := util_pkg.calc_salary(p_name);\n" + " IF v_sal > 0 THEN\n" + " RETURN v_sal;\n" + " END IF;\n" + " RAISE no_data_found;\n" + " END;\n" + "END emp_pkg;\n" + "/\n"; CBMFileResult *r = extract(src, CBM_LANG_PLSQL, "t", "emp_pkg.pkb"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); @@ -2202,9 +2201,10 @@ TEST(dbt_source_and_two_arg_ref) { /* Both dbt builtins name the relation in their LAST string argument: * source('group','table') -> table, and the two-argument * ref('package','model') form -> model. */ - CBMFileResult *r = extract("SELECT * FROM {{ source('raw', 'customers') }}\n" - "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", - CBM_LANG_SQL, "t", "models/stg_customers.sql"); + CBMFileResult *r = + extract("SELECT * FROM {{ source('raw', 'customers') }}\n" + "UNION ALL SELECT * FROM {{ ref('analytics', 'legacy_customers') }}\n", + CBM_LANG_SQL, "t", "models/stg_customers.sql"); ASSERT_NOT_NULL(r); ASSERT(has_def(r, "Model", "stg_customers")); ASSERT(has_usage(r, "customers")); @@ -3223,8 +3223,8 @@ TEST(vue_embedded_structure_negative_controls_issue1410) { } TEST(vue_embedded_structure_host_controls_issue1410) { - CBMFileResult *plain = extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT, - "t", "plain.ts"); + CBMFileResult *plain = + extract("function plainTs(): void { target(); }\n", CBM_LANG_TYPESCRIPT, "t", "plain.ts"); ASSERT_NOT_NULL(plain); ASSERT_FALSE(plain->has_error); ASSERT_EQ(count_defs_named(plain, "Function", "plainTs"), 1); @@ -3682,11 +3682,11 @@ TEST(extract_java_method_annotations_issue382) { /* ── ArkTS (HarmonyOS .ets) ─────────────────────────────────────── */ TEST(arkts_component_struct) { - CBMFileResult *r = extract( - "@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n" - " build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n" - " .width('100%')\n }\n}\n", - CBM_LANG_ARKTS, "t", "Index.ets"); + CBMFileResult *r = + extract("@Entry\n@Component\nstruct Index {\n @State message: string = 'Hello'\n\n" + " build() {\n Column() {\n Text(this.message).fontSize(20)\n }\n" + " .width('100%')\n }\n}\n", + CBM_LANG_ARKTS, "t", "Index.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_def(r, "Struct", "Index")); @@ -3717,12 +3717,12 @@ TEST(arkts_exported_struct_decorators) { } TEST(arkts_member_decorators) { - CBMFileResult *r = extract( - "@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n" - " @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n" - " @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n" - " build() {\n }\n}\n", - CBM_LANG_ARKTS, "t", "S.ets"); + CBMFileResult *r = + extract("@Component\nstruct S {\n @State a: number = 0\n @Prop b: string\n" + " @Link c: boolean\n @Provide('k') d: string = ''\n @Consume('k') e: string\n" + " @StorageLink('s') f: number = 1\n @State @Watch('onW') g: boolean = false\n\n" + " build() {\n }\n}\n", + CBM_LANG_ARKTS, "t", "S.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(decorators_contain(find_def_by_name(r, "a"), "State")); @@ -3762,11 +3762,11 @@ TEST(arkts_no_phantom_builtin_defs) { } TEST(arkts_builder_extend_styles) { - CBMFileResult *r = extract( - "@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n" - "@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n" - "@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n", - CBM_LANG_ARKTS, "t", "b.ets"); + CBMFileResult *r = + extract("@Builder\nfunction card(t: string) {\n Column() {\n Text(t)\n }\n}\n\n" + "@Extend(Text)\nfunction fancy(size: number) {\n .fontSize(size)\n}\n\n" + "@Styles\nfunction pressed() {\n .backgroundColor('#eee')\n}\n", + CBM_LANG_ARKTS, "t", "b.ets"); ASSERT_NOT_NULL(r); ASSERT_FALSE(r->has_error); ASSERT(has_def(r, "Function", "card")); @@ -5179,8 +5179,8 @@ TEST(extract_c_test_dir_marks_is_test_issue1294) { * not be (#1294). */ TEST(extract_python_method_test_dir_marks_is_test_issue1294) { const char *src = "class Foo:\n" - " def helper(self):\n" - " pass\n"; + " def helper(self):\n" + " pass\n"; /* Python's LSP layer injects synthetic builtin stub Methods (str.upper, * dict.get, ...) into defs.items alongside real ones (py_builtins.c), so @@ -6434,10 +6434,46 @@ TEST(non_config_language_module_has_no_promoted_description_issue519) { PASS(); } +static int pointer_is_in_result_arena(const CBMFileResult *result, const void *pointer) { + uintptr_t value = (uintptr_t)pointer; + for (int i = 0; i < result->arena.nblocks; i++) { + uintptr_t start = (uintptr_t)result->arena.blocks[i]; + uintptr_t end = start + result->arena.block_sizes[i]; + if (value >= start && value < end) { + return 1; + } + } + return 0; +} + +TEST(extraction_result_arrays_have_independent_ownership) { + char source[16384]; + size_t used = 0; + for (int i = 0; i < 96; i++) { + int written = snprintf(source + used, sizeof(source) - used, + "export function f%d() { return g%d(); }\n", i, i); + ASSERT(written > 0); + used += (size_t)written; + ASSERT(used < sizeof(source)); + } + CBMFileResult *result = extract(source, CBM_LANG_TYPESCRIPT, "ownership", "many.ts"); + ASSERT_NOT_NULL(result); + ASSERT(result->defs.count >= 96); + ASSERT(result->calls.count >= 96); + ASSERT_NOT_NULL(result->defs.items); + ASSERT_NOT_NULL(result->calls.items); + ASSERT(!pointer_is_in_result_arena(result, result->defs.items)); + ASSERT(!pointer_is_in_result_arena(result, result->calls.items)); + cbm_free_result(result); + PASS(); +} + SUITE(extraction) { /* Initialize extraction library */ cbm_init(); + RUN_TEST(extraction_result_arrays_have_independent_ownership); + /* Wide-flat-file linearity (ms-typescript hang) */ RUN_TEST(extract_wide_flat_file_is_linear); #if defined(CBM_CALL_REFERENCE_LOOKUP_TEST_API) && CBM_CALL_REFERENCE_LOOKUP_TEST_API diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index a45541e57..7e7ef0cde 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -1052,14 +1052,13 @@ static NamedEdgePropertyObservation observe_named_edge_callee_property( } observation.database_opened = true; - static const char sql[] = - "SELECT e.properties, json_valid(e.properties), " - "CASE WHEN json_valid(e.properties) " - "THEN json_extract(e.properties, '$.callee') END " - "FROM edges e " - "JOIN nodes src ON src.id=e.source_id AND src.project=e.project " - "JOIN nodes tgt ON tgt.id=e.target_id AND tgt.project=e.project " - "WHERE e.project=?1 AND e.type=?2 AND src.name=?3 AND tgt.name=?4;"; + static const char sql[] = "SELECT e.properties, json_valid(e.properties), " + "CASE WHEN json_valid(e.properties) " + "THEN json_extract(e.properties, '$.callee') END " + "FROM edges e " + "JOIN nodes src ON src.id=e.source_id AND src.project=e.project " + "JOIN nodes tgt ON tgt.id=e.target_id AND tgt.project=e.project " + "WHERE e.project=?1 AND e.type=?2 AND src.name=?3 AND tgt.name=?4;"; sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK || sqlite3_bind_text(stmt, 1, project, -1, SQLITE_TRANSIENT) != SQLITE_OK || @@ -1747,13 +1746,13 @@ TEST(pipeline_call_reference_sequential_parallel_edge_set_parity) { long_reference_name[0] = 'l'; long_reference_name[LONG_REFERENCE_NAME_LEN] = '\0'; char long_reference_source[1024]; - int long_reference_source_len = snprintf( - long_reference_source, sizeof(long_reference_source), - "package parity\n" - "func %s() {}\n" - "func longPropertiesReferenceAccept(callback func()) {}\n" - "func longPropertiesReferenceSite() { longPropertiesReferenceAccept(%s) }\n", - long_reference_name, long_reference_name); + int long_reference_source_len = + snprintf(long_reference_source, sizeof(long_reference_source), + "package parity\n" + "func %s() {}\n" + "func longPropertiesReferenceAccept(callback func()) {}\n" + "func longPropertiesReferenceSite() { longPropertiesReferenceAccept(%s) }\n", + long_reference_name, long_reference_name); if (long_reference_source_len <= 0 || (size_t)long_reference_source_len >= sizeof(long_reference_source)) { th_rmtree(tmp); @@ -1941,18 +1940,18 @@ TEST(pipeline_call_reference_sequential_parallel_edge_set_parity) { named_edge_count(sequential_store, sequential_project, "CALLS", shadow_controls[i].source_name, shadow_controls[i].target_name); } - sequential_long_reference = named_edge_count( - sequential_store, sequential_project, "CALL_REFERENCE", "longPropertiesReferenceSite", - long_reference_name); - sequential_long_usage = named_edge_count(sequential_store, sequential_project, "USAGE", - "longPropertiesReferenceSite", - long_reference_name); - sequential_long_calls = named_edge_count(sequential_store, sequential_project, "CALLS", - "longPropertiesReferenceSite", - long_reference_name); + sequential_long_reference = + named_edge_count(sequential_store, sequential_project, "CALL_REFERENCE", + "longPropertiesReferenceSite", long_reference_name); + sequential_long_usage = + named_edge_count(sequential_store, sequential_project, "USAGE", + "longPropertiesReferenceSite", long_reference_name); + sequential_long_calls = + named_edge_count(sequential_store, sequential_project, "CALLS", + "longPropertiesReferenceSite", long_reference_name); sequential_long_property = observe_named_edge_callee_property( - sequential_db_path, sequential_project, "CALL_REFERENCE", - "longPropertiesReferenceSite", long_reference_name, long_reference_name); + sequential_db_path, sequential_project, "CALL_REFERENCE", "longPropertiesReferenceSite", + long_reference_name, long_reference_name); cbm_store_close(sequential_store); } cbm_pipeline_free(sequential); @@ -1990,9 +1989,9 @@ TEST(pipeline_call_reference_sequential_parallel_edge_set_parity) { named_edge_count(parallel_store, parallel_project, "CALLS", shadow_controls[i].source_name, shadow_controls[i].target_name); } - parallel_long_reference = named_edge_count( - parallel_store, parallel_project, "CALL_REFERENCE", "longPropertiesReferenceSite", - long_reference_name); + parallel_long_reference = + named_edge_count(parallel_store, parallel_project, "CALL_REFERENCE", + "longPropertiesReferenceSite", long_reference_name); parallel_long_usage = named_edge_count(parallel_store, parallel_project, "USAGE", "longPropertiesReferenceSite", long_reference_name); parallel_long_calls = named_edge_count(parallel_store, parallel_project, "CALLS", @@ -2472,8 +2471,8 @@ static void closure_probe_repo(const char *tmp) { } /* Fresh full reference build of the same tree into its own DB. */ -static void closure_fresh_full(const char *tmp, const char *db_path, int *out_nodes, - int *out_edges, int *out_ref_edges, const char *project_hint) { +static void closure_fresh_full(const char *tmp, const char *db_path, int *out_nodes, int *out_edges, + int *out_ref_edges, const char *project_hint) { *out_nodes = -1; *out_edges = -2; *out_ref_edges = -3; @@ -2490,9 +2489,9 @@ static void closure_fresh_full(const char *tmp, const char *db_path, int *out_no if (store) { *out_nodes = cbm_store_count_nodes(store, project); *out_edges = cbm_store_count_edges(store, project); - *out_ref_edges = named_edge_to_file_count(store, project, "CALL_REFERENCE", - "closureProbeCaller", "closureProbeHelper", - "lib.ts"); + *out_ref_edges = + named_edge_to_file_count(store, project, "CALL_REFERENCE", "closureProbeCaller", + "closureProbeHelper", "lib.ts"); cbm_store_close(store); } } @@ -2577,8 +2576,8 @@ TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full) { ASSERT_NOT_NULL(store); repaired_nodes = cbm_store_count_nodes(store, project); repaired_edges = cbm_store_count_edges(store, project); - repaired_refs = named_edge_to_file_count(store, project, "CALL_REFERENCE", - "closureProbeCaller", "closureProbeHelper", "lib.ts"); + repaired_refs = named_edge_to_file_count(store, project, "CALL_REFERENCE", "closureProbeCaller", + "closureProbeHelper", "lib.ts"); cbm_store_close(store); char full_db[512]; @@ -2634,8 +2633,8 @@ TEST(pipeline_closure_repair_removed_def_drops_dependent_edge) { int repaired_refs = -1; cbm_store_t *store = cbm_store_open_path(db); ASSERT_NOT_NULL(store); - repaired_refs = named_edge_to_file_count(store, project, "CALL_REFERENCE", - "closureProbeCaller", "closureProbeHelper", "lib.ts"); + repaired_refs = named_edge_to_file_count(store, project, "CALL_REFERENCE", "closureProbeCaller", + "closureProbeHelper", "lib.ts"); int repaired_nodes = cbm_store_count_nodes(store, project); int repaired_edges = cbm_store_count_edges(store, project); cbm_store_close(store); @@ -2878,8 +2877,7 @@ TEST(pipeline_incremental_tsconfig_alias_change_matches_fresh_full) { * target_a.ts to target_b.ts. Since alias-config governance landed this * runs as a closure repair, and the convergence assertions below now * prove that route rather than being satisfied by a full rebuild. */ - ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), - CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); const char *incremental_project = cbm_pipeline_project_name(incremental); cbm_store_t *incremental_store = cbm_store_open_path(incremental_db); ASSERT_NOT_NULL(incremental_store); @@ -3067,8 +3065,8 @@ TEST(pipeline_publication_never_uses_a_predictable_staging_path) { static const char canary[] = "canary-must-survive\n"; char canary_path[PREDICTABLE_CANARIES][640]; for (int i = 0; i < PREDICTABLE_CANARIES; i++) { - snprintf(canary_path[i], sizeof(canary_path[i]), "%s.stage.%ld.%d", db_path, - (long)getpid(), i + 1); + snprintf(canary_path[i], sizeof(canary_path[i]), "%s.stage.%ld.%d", db_path, (long)getpid(), + i + 1); ASSERT_EQ(th_write_file(canary_path[i], canary), 0); } @@ -6249,7 +6247,7 @@ TEST(pipeline_swift_cross_package_import) { cbm_edge_t *edges = NULL; int ec = 0; ASSERT_EQ(cbm_store_find_edges_by_source_type(s, importer.id, "IMPORTS", &edges, &ec), - CBM_STORE_OK); + CBM_STORE_OK); bool found_exact_edge = false; for (int i = 0; i < ec; i++) { @@ -6327,19 +6325,18 @@ TEST(pipeline_python_cross_module_call) { * unique_name (candidates==1) is #1572 and is not this claim. */ TEST(pipeline_cross_language_same_name_does_not_share_calls_issue725) { const char *files[] = {"store.py", "app.py", "web/src/pages/Editor.js"}; - const char *contents[] = { - "class Store:\n" - " def commit(self):\n" - " return True\n", + const char *contents[] = {"class Store:\n" + " def commit(self):\n" + " return True\n", - "from store import Store\n" - "\n" - "def save():\n" - " return Store().commit()\n", + "from store import Store\n" + "\n" + "def save():\n" + " return Store().commit()\n", - "export function commit() {\n" - " return 1;\n" - "}\n"}; + "export function commit() {\n" + " return 1;\n" + "}\n"}; if (setup_lang_repo(files, contents, 3) != 0) FAIL("tmpdir"); @@ -9686,17 +9683,16 @@ static const char *pkg_entries_entry_for(const cbm_pkg_entries_t *e, const char * above for the full end-to-end proof. */ TEST(pkgmap_swift_targets_registers_module) { - static const char src[] = - "// swift-tools-version:5.9\n" - "import PackageDescription\n" - "let package = Package(\n" - " name: \"Core\",\n" - " targets: [.target(name: \"Core\", dependencies: [])]\n" - ")\n"; + static const char src[] = "// swift-tools-version:5.9\n" + "import PackageDescription\n" + "let package = Package(\n" + " name: \"Core\",\n" + " targets: [.target(name: \"Core\", dependencies: [])]\n" + ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "Core")); ASSERT_STR_EQ(pkg_entries_entry_for(&entries, "Core"), "Core/Sources/Core"); @@ -9717,8 +9713,8 @@ TEST(pkgmap_swift_products_do_not_register_alias) { ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_FALSE(pkg_entries_has_name(&entries, "CoreKit")); ASSERT_TRUE(pkg_entries_has_name(&entries, "CoreImpl")); @@ -9737,15 +9733,14 @@ TEST(pkgmap_swift_products_do_not_register_alias) { * fixture in this file happens to follow `name:` with `dependencies:` or a * comma, so this specific shape was previously untested and unnoticed. */ TEST(pkgmap_swift_target_name_immediately_before_close_paren) { - static const char src[] = - "let package = Package(\n" - " name: \"Core\",\n" - " targets: [.target(name: \"Core\")]\n" - ")\n"; + static const char src[] = "let package = Package(\n" + " name: \"Core\",\n" + " targets: [.target(name: \"Core\")]\n" + ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "Core")); ASSERT_STR_EQ(pkg_entries_entry_for(&entries, "Core"), "Core/Sources/Core"); @@ -9763,8 +9758,8 @@ TEST(pkgmap_swift_target_honors_literal_path) { ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "Core")); ASSERT_STR_EQ(pkg_entries_entry_for(&entries, "Core"), "Core/Vendor/CoreLegacy"); @@ -9778,16 +9773,15 @@ TEST(pkgmap_swift_target_honors_literal_path) { * target entirely (fail closed), even though its `name:` is a valid * literal. */ TEST(pkgmap_swift_target_computed_path_fails_closed) { - static const char src[] = - "let customPath = computePath()\n" - "let package = Package(\n" - " name: \"Core\",\n" - " targets: [.target(name: \"Core\", path: customPath)]\n" - ")\n"; + static const char src[] = "let customPath = computePath()\n" + "let package = Package(\n" + " name: \"Core\",\n" + " targets: [.target(name: \"Core\", path: customPath)]\n" + ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_EQ(entries.count, 0); cbm_pkg_entries_free(&entries); @@ -9809,8 +9803,8 @@ TEST(pkgmap_swift_target_in_comment_or_string_not_registered) { ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, - (int)strlen(src), &entries); + bool ok = + cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, (int)strlen(src), &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "App")); ASSERT_FALSE(pkg_entries_has_name(&entries, "Decoy")); @@ -9839,8 +9833,8 @@ TEST(pkgmap_swift_dependencies_do_not_leak_entries) { ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, - (int)strlen(src), &entries); + bool ok = + cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, (int)strlen(src), &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "App")); ASSERT_FALSE(pkg_entries_has_name(&entries, "Core")); @@ -9855,18 +9849,17 @@ TEST(pkgmap_swift_dependencies_do_not_leak_entries) { * (Utils/UtilsPkg) name OTHER modules, not this manifest's own * products/targets, so neither mints an entry. */ TEST(pkgmap_swift_target_name_dependency_does_not_leak_entry) { - static const char src[] = - "let package = Package(\n" - " name: \"App\",\n" - " targets: [.target(name: \"App\", dependencies: [\n" - " \"Core\",\n" - " .product(name: \"Utils\", package: \"UtilsPkg\")\n" - " ])]\n" - ")\n"; + static const char src[] = "let package = Package(\n" + " name: \"App\",\n" + " targets: [.target(name: \"App\", dependencies: [\n" + " \"Core\",\n" + " .product(name: \"Utils\", package: \"UtilsPkg\")\n" + " ])]\n" + ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, - (int)strlen(src), &entries); + bool ok = + cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, (int)strlen(src), &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "App")); ASSERT_FALSE(pkg_entries_has_name(&entries, "Core")); @@ -12084,6 +12077,49 @@ TEST(pipeline_backpressure_futile_nap_disengages) { PASS(); } +TEST(pipeline_resource_mode_selection) { + const char *old_mode = getenv("CBM_INDEX_RESOURCE_MODE"); + const char *old_batch = getenv("CBM_STREAMING_BATCH_FILES"); + char *saved_mode = old_mode ? strdup(old_mode) : NULL; + char *saved_batch = old_batch ? strdup(old_batch) : NULL; + ASSERT_TRUE(!old_mode || saved_mode != NULL); + ASSERT_TRUE(!old_batch || saved_batch != NULL); + ASSERT_EQ(cbm_unsetenv("CBM_INDEX_RESOURCE_MODE"), 0); + ASSERT_EQ(cbm_unsetenv("CBM_STREAMING_BATCH_FILES"), 0); + + cbm_file_info_t small[1] = {{0}}; + small[0].size = 1024; + ASSERT_EQ(cbm_pipeline_streaming_batch_size(small, 1), 0); + + cbm_file_info_t *many = calloc(10000, sizeof(*many)); + ASSERT_NOT_NULL(many); + ASSERT_EQ(cbm_pipeline_streaming_batch_size(many, 10000), 512); + free(many); + + small[0].size = (int64_t)512 * 1024 * 1024; + ASSERT_EQ(cbm_pipeline_streaming_batch_size(small, 1), 1); + + cbm_file_info_t manual[600] = {{0}}; + ASSERT_EQ(cbm_setenv("CBM_INDEX_RESOURCE_MODE", "large", 1), 0); + ASSERT_EQ(cbm_pipeline_streaming_batch_size(manual, 600), 512); + ASSERT_EQ(cbm_setenv("CBM_INDEX_RESOURCE_MODE", "daily", 1), 0); + ASSERT_EQ(cbm_pipeline_streaming_batch_size(manual, 600), 0); + ASSERT_EQ(cbm_setenv("CBM_INDEX_RESOURCE_MODE", "invalid", 1), 0); + ASSERT_EQ(cbm_pipeline_streaming_batch_size(manual, 600), -1); + ASSERT_EQ(cbm_setenv("CBM_STREAMING_BATCH_FILES", "37", 1), 0); + ASSERT_EQ(cbm_pipeline_streaming_batch_size(manual, 600), 37); + + int restore_mode = saved_mode ? cbm_setenv("CBM_INDEX_RESOURCE_MODE", saved_mode, 1) + : cbm_unsetenv("CBM_INDEX_RESOURCE_MODE"); + int restore_batch = saved_batch ? cbm_setenv("CBM_STREAMING_BATCH_FILES", saved_batch, 1) + : cbm_unsetenv("CBM_STREAMING_BATCH_FILES"); + free(saved_mode); + free(saved_batch); + ASSERT_EQ(restore_mode, 0); + ASSERT_EQ(restore_batch, 0); + PASS(); +} + /* TS cross-registry test hooks (ts_lsp.c) — extern to avoid pulling the * tree-sitter-typed ts_lsp.h into this store-level test. */ extern long cbm_ts_full_registry_builds(void); @@ -12686,7 +12722,6 @@ TEST(pipeline_delta_patch_indexes_docstring_into_fts_body) { PASS(); } - /* End-to-end for #518/#519: source → docstring → properties JSON → nodes_fts * `body` → findable. Each layer has its own test; this one proves they connect. * It is also the guard on the size budget: build_def_props drops an oversized @@ -12767,6 +12802,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_run_null); /* Extraction back-pressure */ RUN_TEST(pipeline_backpressure_futile_nap_disengages); + RUN_TEST(pipeline_resource_mode_selection); /* Sequential cross-LSP shared registry (ms-typescript quadratic) */ RUN_TEST(pipeline_seq_ts_cross_uses_shared_registry); /* File persistence */ diff --git a/tests/test_subprocess.c b/tests/test_subprocess.c index c9932bf2d..cb9699243 100644 --- a/tests/test_subprocess.c +++ b/tests/test_subprocess.c @@ -476,6 +476,43 @@ TEST(subprocess_natural_completion_is_cached_across_polls) { #endif } +TEST(subprocess_observes_contained_tree_rss) { +#ifdef _WIN32 + SKIP_PLATFORM("POSIX shell process-group RSS probe; Windows Job telemetry is compile-covered"); +#else + const char *argv[] = {"/bin/sh", "-c", "sleep 5 & wait", NULL}; + cbm_proc_opts_t opts = {0}; + opts.bin = "/bin/sh"; + opts.argv = argv; + opts.cancel_grace_ms = 100; + cbm_subprocess_t *process = NULL; + ASSERT_EQ(cbm_subprocess_spawn(&opts, &process), 0); + ASSERT_NOT_NULL(process); + + size_t observed = 0; + uint64_t deadline = cbm_now_ms() + 2000; + while (cbm_now_ms() < deadline && observed == 0) { + (void)cbm_subprocess_observed_tree_rss(process, &observed); + cbm_usleep(1000); + } + bool cancelled = cbm_subprocess_request_cancel(process); + cbm_proc_result_t result; + bool terminal = poll_until_terminal(process, 2000, &result); + size_t cached = 0; + bool cached_available = cbm_subprocess_observed_tree_rss(process, &cached); + if (terminal) { + cbm_subprocess_destroy(process); + } + + ASSERT_TRUE(observed > 0); + ASSERT_TRUE(cancelled); + ASSERT_TRUE(terminal); + ASSERT_TRUE(cached_available); + ASSERT_TRUE(cached >= observed); + PASS(); +#endif +} + TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree) { #ifdef _WIN32 SKIP_PLATFORM("POSIX process-group probe; native Windows Job Object tree probe pending"); @@ -1177,6 +1214,7 @@ SUITE(subprocess) { RUN_TEST(subprocess_run_null_bin_rejected); RUN_TEST(subprocess_spawn_returns_while_child_is_running); RUN_TEST(subprocess_natural_completion_is_cached_across_polls); + RUN_TEST(subprocess_observes_contained_tree_rss); RUN_TEST(subprocess_cancel_is_idempotent_and_kills_ignoring_tree); RUN_TEST(subprocess_quiet_timeout_kills_ignoring_tree); RUN_TEST(subprocess_windows_job_object_cancellation_quiesces_descendant_tree); diff --git a/tests/test_worker_error_response.sh b/tests/test_worker_error_response.sh index 33e2db115..5fa490c81 100755 --- a/tests/test_worker_error_response.sh +++ b/tests/test_worker_error_response.sh @@ -46,11 +46,26 @@ cleanup() { } trap cleanup EXIT -missing="${tmpdir}/repository-does-not-exist" +repository="${tmpdir}/source-repository" +mkdir -p "${repository}" +# Use enough source files to select the parallel pipeline, then request its +# deterministic invalid-resource-mode error. The repository must exist so the +# worker's request-scoped workspace boundary remains fail-closed on every OS. +i=1 +while [[ ${i} -le 64 ]]; do + printf 'def f%s():\n pass\n' "${i}" >"${repository}/f${i}.py" + i=$((i + 1)) +done +repository_arg="${repository}" +if command -v cygpath >/dev/null 2>&1; then + # JSON arguments are opaque to MSYS2 argv conversion. Use a forward-slash + # Windows path so the native worker can canonicalize the request scope. + repository_arg="$(cygpath -m "${repository}")" +fi response="${tmpdir}/worker.response" -args="{\"repo_path\":\"${missing}\",\"mode\":\"fast\"}" +args="{\"repo_path\":\"${repository_arg}\",\"mode\":\"fast\"}" -if ! CBM_CACHE_DIR="${tmpdir}/cache-worker" \ +if ! CBM_INDEX_RESOURCE_MODE=invalid CBM_CACHE_DIR="${tmpdir}/cache-worker" \ "${BINARY}" cli --index-worker \ --index-worker-build "${BUILD_FINGERPRINT}" \ index_repository "${args}" \ @@ -67,8 +82,8 @@ if [[ ! -s "${response}" ]] || ! grep -q 'Pipeline failed' "${response}"; then fi set +e -CBM_CACHE_DIR="${tmpdir}/cache-supervisor" \ - "${BINARY}" cli index_repository --repo-path "${missing}" --mode fast \ +CBM_INDEX_RESOURCE_MODE=invalid CBM_CACHE_DIR="${tmpdir}/cache-supervisor" \ + "${BINARY}" cli index_repository --repo-path "${repository_arg}" --mode fast \ >"${tmpdir}/supervisor.out" 2>"${tmpdir}/supervisor.err" cli_rc=$? set -e