From f0fac62b7140f2cc3ccc011c74e798be7681ec61 Mon Sep 17 00:00:00 2001 From: News Date: Mon, 21 Sep 2026 20:51:34 +0900 Subject: [PATCH] fix: keep indexing stable with Windows and source settings --- Makefile.cbm | 1 + src/foundation/compat.c | 53 +++++--- src/foundation/compat.h | 8 +- src/foundation/limits.h | 1 + src/foundation/mem_override_win.c | 52 ++++++- src/foundation/source_encoding.c | 128 ++++++++++++++++++ src/foundation/source_encoding.h | 21 +++ src/mcp/index_supervisor.c | 6 + src/mcp/mcp.c | 6 +- src/pipeline/pass_calls.c | 23 +++- src/pipeline/pass_compile_commands.c | 194 +++++++++++++++++++++++++-- src/pipeline/pass_definitions.c | 25 +++- src/pipeline/pass_parallel.c | 26 +++- src/pipeline/pass_semantic.c | 30 ++++- src/pipeline/pass_usages.c | 22 ++- src/pipeline/pipeline.c | 4 + src/pipeline/pipeline_incremental.c | 50 +++++++ src/pipeline/pipeline_internal.h | 9 ++ tests/test_pipeline.c | 55 ++++++++ tests/test_platform.c | 57 ++++++++ 20 files changed, 725 insertions(+), 46 deletions(-) create mode 100644 src/foundation/source_encoding.c create mode 100644 src/foundation/source_encoding.h diff --git a/Makefile.cbm b/Makefile.cbm index 305cb6937..b28b6ac8d 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -329,6 +329,7 @@ FOUNDATION_SRCS = \ src/foundation/subprocess.c \ src/foundation/sha256.c \ src/foundation/secure_random.c \ + src/foundation/source_encoding.c \ src/foundation/macos_acl.c \ src/foundation/private_file_lock.c \ src/foundation/lock_registry.c diff --git a/src/foundation/compat.c b/src/foundation/compat.c index 94e7020bd..22d013788 100644 --- a/src/foundation/compat.c +++ b/src/foundation/compat.c @@ -269,36 +269,55 @@ int cbm_mkstemp(char *tmpl) { errno = ENAMETOOLONG; return CBM_NOT_FOUND; } - /* Wide-API expansion and open: worker staging files land inside - * CBM_CACHE_DIR, which users may place at non-ASCII paths; the ANSI CRT - * (_mktemp/_open) mangles those bytes in the local codepage. */ + /* Keep the six-character mkstemp contract, but do not use _wmktemp: + * that CRT helper has a tiny name space on Windows and retained worker + * logs can exhaust it during recovery. The exclusive open closes races + * with other processes; collisions simply draw another random name. */ wchar_t *wide_template = cbm_utf8_to_wide(buf); - if (!wide_template || !_wmktemp(wide_template)) { - free(wide_template); + if (!wide_template) { + errno = EINVAL; return CBM_NOT_FOUND; } - char *expanded_for_open = cbm_wide_to_utf8(wide_template); - wchar_t *wide_open = expanded_for_open ? cbm_path_to_wide(expanded_for_open) : NULL; - free(expanded_for_open); - if (!wide_open) { + size_t wide_len = wcslen(wide_template); + if (wide_len < 6 || wcscmp(wide_template + wide_len - 6, L"XXXXXX") != 0) { free(wide_template); + errno = EINVAL; return CBM_NOT_FOUND; } - int fd = _wopen(wide_open, _O_CREAT | _O_EXCL | _O_RDWR | _O_BINARY, _S_IREAD | _S_IWRITE); - free(wide_open); - if (fd >= 0) { + static const wchar_t hex[] = L"0123456789abcdef"; + for (int attempt = 0; attempt < 128; attempt++) { + unsigned int random_bits = 0; + if (!cbm_secure_random(&random_bits, sizeof(random_bits))) { + errno = EIO; + break; + } + for (int digit = 0; digit < 6; digit++) { + wide_template[wide_len - 6 + digit] = hex[(random_bits >> (digit * 4)) & 0xf]; + } char *expanded = cbm_wide_to_utf8(wide_template); - if (!expanded || strlen(expanded) >= sizeof(buf)) { + wchar_t *wide_open = expanded ? cbm_path_to_wide(expanded) : NULL; + if (!expanded || !wide_open || strlen(expanded) >= sizeof(buf)) { + free(expanded); + free(wide_open); + errno = ENAMETOOLONG; + break; + } + int fd = _wopen(wide_open, _O_CREAT | _O_EXCL | _O_RDWR | _O_BINARY, + _S_IREAD | _S_IWRITE); + free(wide_open); + if (fd >= 0) { + strcpy(tmpl, expanded); free(expanded); free(wide_template); - (void)_close(fd); - return CBM_NOT_FOUND; + return fd; } - strcpy(tmpl, expanded); free(expanded); + if (errno != EEXIST) { + break; + } } free(wide_template); - return fd; + return CBM_NOT_FOUND; } #endif diff --git a/src/foundation/compat.h b/src/foundation/compat.h index 821d901f0..7acb96c95 100644 --- a/src/foundation/compat.h +++ b/src/foundation/compat.h @@ -166,11 +166,9 @@ static inline int cbm_setenv(const char *name, const char *value, int overwrite) free(wide_value); return EINVAL; } - /* Keep the CRT's narrow environment useful for legacy getenv callers, - * then repair the process-wide Windows environment with the actual UTF-16 - * value. _putenv_s alone routes UTF-8 path bytes through the active ANSI - * code page, which corrupts non-ASCII cache roots inherited by children. */ - int status = _putenv_s(name, value); + /* Update the CRT through its wide environment. _putenv_s rejects valid + * UTF-8 values when the active ANSI code page cannot represent them. */ + int status = _wputenv_s(wide_name, wide_value); if (status == 0 && !SetEnvironmentVariableW(wide_name, wide_value)) { status = EINVAL; } diff --git a/src/foundation/limits.h b/src/foundation/limits.h index b914c3f79..54f21f6ca 100644 --- a/src/foundation/limits.h +++ b/src/foundation/limits.h @@ -17,6 +17,7 @@ typedef enum { CBM_READ_EMPTY, /* zero/negative size — benign, nothing to index */ CBM_READ_OVERSIZED, /* size exceeds cbm_max_file_bytes() */ CBM_READ_OOM, /* buffer allocation failed */ + CBM_READ_ENCODING, /* source bytes are neither UTF-8 nor CP949 */ } cbm_read_status_t; /* Maximum size (bytes) of a single source file the indexer will read into diff --git a/src/foundation/mem_override_win.c b/src/foundation/mem_override_win.c index e18b09d11..d06c01505 100644 --- a/src/foundation/mem_override_win.c +++ b/src/foundation/mem_override_win.c @@ -56,6 +56,7 @@ void *__real_malloc(size_t size); void *__real_calloc(size_t count, size_t size); void *__real_realloc(void *block, size_t size); void __real_free(void *block); +void __real__aligned_free(void *block); char *__real_strdup(const char *text); size_t __real__msize(void *block); /* Wrappers must call __real_* for anything that is itself wrapped: a plain @@ -155,9 +156,58 @@ void __wrap__aligned_free(void *block) { mi_free(block); return; } - __real_free(block); + /* A CRT aligned block has its own bookkeeping header. Plain free is not + * its matching deallocator. */ + __real__aligned_free(block); +} + +#if defined(CBM_MEM_GLOBAL_OVERRIDE) && CBM_MEM_GLOBAL_OVERRIDE +/* LLVM-MinGW's static C++ runtime calls _aligned_malloc through the linker's + * wrapper but its aligned operator delete can call the CRT import pointer + * __imp__aligned_free directly. --wrap=_aligned_free does not rewrite that + * import-pointer reference. Provide the six C++ ABI delete entry points so + * every aligned new/delete pair reaches the same owner-aware deallocator. */ +void cbm_cxx_aligned_delete(void *block) __asm__("_ZdlPvSt11align_val_t"); +void cbm_cxx_aligned_delete(void *block) { + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_delete_nothrow(void *block, const void *tag) + __asm__("_ZdlPvSt11align_val_tRKSt9nothrow_t"); +void cbm_cxx_aligned_delete_nothrow(void *block, const void *tag) { + (void)tag; + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_delete_sized(void *block, size_t size, size_t alignment) + __asm__("_ZdlPvySt11align_val_t"); +void cbm_cxx_aligned_delete_sized(void *block, size_t size, size_t alignment) { + (void)size; + (void)alignment; + __wrap__aligned_free(block); } +void cbm_cxx_aligned_array_delete(void *block) __asm__("_ZdaPvSt11align_val_t"); +void cbm_cxx_aligned_array_delete(void *block) { + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_array_delete_nothrow(void *block, const void *tag) + __asm__("_ZdaPvSt11align_val_tRKSt9nothrow_t"); +void cbm_cxx_aligned_array_delete_nothrow(void *block, const void *tag) { + (void)tag; + __wrap__aligned_free(block); +} + +void cbm_cxx_aligned_array_delete_sized(void *block, size_t size, size_t alignment) + __asm__("_ZdaPvySt11align_val_t"); +void cbm_cxx_aligned_array_delete_sized(void *block, size_t size, size_t alignment) { + (void)size; + (void)alignment; + __wrap__aligned_free(block); +} +#endif + void *__wrap_realloc(void *block, size_t size) { if (!block) { void *fresh = mi_malloc(size); diff --git a/src/foundation/source_encoding.c b/src/foundation/source_encoding.c new file mode 100644 index 000000000..8710eaff7 --- /dev/null +++ b/src/foundation/source_encoding.c @@ -0,0 +1,128 @@ +#include "foundation/source_encoding.h" + +#include +#include +#include + +#ifdef _WIN32 +#include +#else +#include +#include +#endif + +static int source_valid_utf8(const unsigned char *bytes, size_t length) { + for (size_t i = 0; i < length;) { + unsigned char first = bytes[i]; + if (first < 0x80) { + if (first == 0) return 0; + i++; + continue; + } + unsigned int count; + uint32_t cp; + if (first >= 0xc2 && first <= 0xdf) { + count = 2; + cp = first & 0x1f; + } else if (first >= 0xe0 && first <= 0xef) { + count = 3; + cp = first & 0x0f; + } else if (first >= 0xf0 && first <= 0xf4) { + count = 4; + cp = first & 0x07; + } else { + return 0; + } + if (count > length - i) return 0; + for (unsigned int j = 1; j < count; j++) { + unsigned char next = bytes[i + j]; + if ((next & 0xc0) != 0x80) return 0; + cp = (cp << 6) | (next & 0x3f); + } + if ((count == 2 && cp < 0x80) || (count == 3 && cp < 0x800) || + (count == 4 && cp < 0x10000) || cp > 0x10ffff || + (cp >= 0xd800 && cp <= 0xdfff)) return 0; + i += count; + } + return 1; +} + +char *cbm_source_transcode_utf8(const char *input, size_t length, size_t *out_length, + cbm_source_encoding_t *encoding) { + if (!input || !out_length || !encoding) return NULL; + *out_length = length; + *encoding = CBM_SOURCE_INVALID; + size_t bom = length >= 3 && (unsigned char)input[0] == 0xef && + (unsigned char)input[1] == 0xbb && + (unsigned char)input[2] == 0xbf ? 3 : 0; + const char *requested = getenv("CBM_SOURCE_ENCODING"); + int force_cp949 = requested && + (strcmp(requested, "cp949") == 0 || strcmp(requested, "euc-kr") == 0); + int force_utf8 = requested && strcmp(requested, "utf8") == 0; + int valid_utf8 = source_valid_utf8((const unsigned char *)input + bom, length - bom); + if (bom && !valid_utf8) return NULL; + if (valid_utf8 && (!force_cp949 || bom)) { + *encoding = bom ? CBM_SOURCE_UTF8_BOM : CBM_SOURCE_UTF8; + if (!bom) return NULL; + char *copy = malloc(length - bom + 17); + if (!copy) { + *encoding = CBM_SOURCE_INVALID; + return NULL; + } + memcpy(copy, input + bom, length - bom); + memset(copy + length - bom, 0, 17); + *out_length = length - bom; + return copy; + } + if (force_utf8) return NULL; + if (length == 0 || length > (size_t)INT32_MAX) return NULL; + /* A NUL in a source buffer is binary data, not a legacy text encoding. */ + if (memchr(input, 0, length)) return NULL; +#ifdef _WIN32 + int wide_length = MultiByteToWideChar(949, MB_ERR_INVALID_CHARS, input, (int)length, NULL, 0); + if (wide_length <= 0) return NULL; + wchar_t *wide = malloc((size_t)wide_length * sizeof(*wide)); + if (!wide) return NULL; + if (MultiByteToWideChar(949, MB_ERR_INVALID_CHARS, input, (int)length, wide, + wide_length) != wide_length) { + free(wide); + return NULL; + } + int needed = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wide, wide_length, NULL, 0, + NULL, NULL); + char *converted = needed > 0 ? malloc((size_t)needed + 17) : NULL; + if (!converted || WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wide, wide_length, + converted, needed, NULL, NULL) != needed) { + free(converted); + free(wide); + return NULL; + } + free(wide); + memset(converted + needed, 0, 17); + *out_length = (size_t)needed; +#else + if (length > (SIZE_MAX - 17) / 4) return NULL; + iconv_t converter = iconv_open("UTF-8", "CP949"); + if (converter == (iconv_t)-1) return NULL; + size_t capacity = length * 4 + 17; + char *converted = malloc(capacity); + if (!converted) { + iconv_close(converter); + return NULL; + } + char *read_ptr = (char *)input; + char *write_ptr = converted; + size_t remaining = length; + size_t available = capacity - 17; + size_t result = iconv(converter, &read_ptr, &remaining, &write_ptr, &available); + iconv_close(converter); + if (result == (size_t)-1 || remaining != 0) { + free(converted); + return NULL; + } + *out_length = (size_t)(write_ptr - converted); + memset(write_ptr, 0, 17); +#endif + *encoding = CBM_SOURCE_CP949; + return converted; +} diff --git a/src/foundation/source_encoding.h b/src/foundation/source_encoding.h new file mode 100644 index 000000000..416abf559 --- /dev/null +++ b/src/foundation/source_encoding.h @@ -0,0 +1,21 @@ +#ifndef CBM_SOURCE_ENCODING_H +#define CBM_SOURCE_ENCODING_H + +#include + +typedef enum { + CBM_SOURCE_UTF8, + CBM_SOURCE_UTF8_BOM, + CBM_SOURCE_CP949, + CBM_SOURCE_INVALID +} cbm_source_encoding_t; + +/* Auto-detect UTF-8 and CP949/EUC-KR, or honor CBM_SOURCE_ENCODING=utf8, + * cp949, or euc-kr for a repository with a known encoding. + * Return a malloc-owned UTF-8 replacement when conversion is needed. + * A NULL result with CBM_SOURCE_UTF8 means the input is already UTF-8; + * a NULL result with CBM_SOURCE_INVALID means conversion failed. */ +char *cbm_source_transcode_utf8(const char *input, size_t length, size_t *out_length, + cbm_source_encoding_t *encoding); + +#endif diff --git a/src/mcp/index_supervisor.c b/src/mcp/index_supervisor.c index 4cd69f642..5e15a7eed 100644 --- a/src/mcp/index_supervisor.c +++ b/src/mcp/index_supervisor.c @@ -13,6 +13,7 @@ #include "ui/http_server.h" /* cbm_http_server_resolve_binary_path */ #include +#include #include #include #include @@ -705,6 +706,11 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_ worker_result_init(&handle->result); if (!worker_unique_file(handle->response_path, sizeof(handle->response_path), "response") || !worker_unique_file(handle->log_path, sizeof(handle->log_path), "log")) { + int saved_errno = errno; + char error_text[CBM_SZ_32]; + (void)snprintf(error_text, sizeof(error_text), "%d", saved_errno); + cbm_log_error("index.supervisor.artifact_create_failed", "artifact", + handle->response_path[0] ? "log" : "response", "errno", error_text); (void)cbm_unlink(handle->response_path); (void)cbm_unlink(handle->log_path); free(handle); diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index 7bfca8d7a..577658070 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -10776,7 +10776,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) { cbm_mcp_supervised_result_disposition_t recovery_disposition = cbm_mcp_supervised_result_disposition(rc2, &wr2); if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_FALLBACK) { - last_outcome = wr2.outcome; + /* A recovery setup failure must not replace the worker failure + * that made us enter recovery in the first place. */ + cbm_log_error("index.supervisor.recovery_start_failed", "original_outcome", + cbm_proc_outcome_str(last_outcome), "recovery_outcome", + cbm_proc_outcome_str(wr2.outcome)); cbm_index_worker_result_free(&wr2); break; /* spawn failed mid-recovery — give up */ } diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index c9744dda2..ba0232bcf 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -10,6 +10,7 @@ * Depends on: pass_definitions having populated the registry and graph buffer */ #include "foundation/constants.h" +#include "foundation/source_encoding.h" enum { PC_RING = 4, PC_RING_MASK = 3, PC_SIG_SCAN = 15, PC_REGEX_GRP = 2 }; /* Confidence for a service-pattern HTTP/ASYNC edge emitted when registry @@ -73,6 +74,18 @@ static char *read_file(const char *path, int *out_len) { if (nread > (size_t)size) { nread = (size_t)size; } + size_t decoded_len = 0; + cbm_source_encoding_t encoding; + char *decoded = cbm_source_transcode_utf8(buf, nread, &decoded_len, &encoding); + if (encoding == CBM_SOURCE_INVALID) { + free(buf); + return NULL; + } + if (decoded) { + free(buf); + buf = decoded; + nread = decoded_len; + } memset(buf + nread, 0, CBM_TS_LOOKAHEAD_PAD); *out_len = (int)nread; return buf; @@ -761,9 +774,13 @@ static CBMFileResult *calls_get_or_extract(cbm_pipeline_ctx_t *ctx, int idx, if (!src) { return NULL; } - CBMFileResult *r = cbm_extract_file_ex(src, slen, fi->language, ctx->project_name, fi->rel_path, - CBM_EXTRACT_BUDGET, NULL, NULL, ctx->macro_table, - ctx->return_type_table); + const cbm_compile_flags_t *flags = + cbm_compile_commands_find(ctx->compile_commands, fi->rel_path); + CBMFileResult *r = cbm_extract_file_ex( + src, slen, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, + flags ? (const char **)flags->defines : NULL, + flags ? (const char **)flags->include_paths : NULL, ctx->macro_table, + ctx->return_type_table); free(src); if (r) { *owned = true; diff --git a/src/pipeline/pass_compile_commands.c b/src/pipeline/pass_compile_commands.c index 174983977..1dbdd7a4a 100644 --- a/src/pipeline/pass_compile_commands.c +++ b/src/pipeline/pass_compile_commands.c @@ -11,7 +11,12 @@ enum { CC_FLAG_IDX = 1, CC_FLAG_SKIP = 2 }; #define SLEN(s) (sizeof(s) - 1) #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/platform.h" +#include "foundation/log.h" +#include #include #include #include @@ -63,7 +68,8 @@ static char *resolve_path(const char *path, const char *directory) { } /* Absolute path */ - if (path[0] == '/') { + if (path[0] == '/' || + (isalpha((unsigned char)path[0]) && path[1] == ':')) { return strdup(path); } @@ -81,7 +87,7 @@ static char *resolve_path(const char *path, const char *directory) { static bool try_include_flag(cbm_compile_flags_t *f, const char **args, int argc, int *i, const char *directory) { const char *arg = args[*i]; - if (arg[0] == '-' && arg[CC_FLAG_IDX] == 'I') { + if ((arg[0] == '-' || arg[0] == '/') && arg[CC_FLAG_IDX] == 'I') { const char *path = arg + CC_FLAG_SKIP; if (*path == '\0' && *i + SKIP_ONE < argc) { (*i)++; @@ -103,7 +109,7 @@ static bool try_include_flag(cbm_compile_flags_t *f, const char **args, int argc /* Try to consume a -D define flag. Returns true if consumed. */ static bool try_define_flag(cbm_compile_flags_t *f, const char **args, int argc, int *i) { const char *arg = args[*i]; - if (arg[0] != '-' || arg[CC_FLAG_IDX] != 'D') { + if ((arg[0] != '-' && arg[0] != '/') || arg[CC_FLAG_IDX] != 'D') { return false; } const char *define = arg + CC_FLAG_SKIP; @@ -122,8 +128,12 @@ cbm_compile_flags_t *cbm_extract_flags(const char **args, int argc, const char * if (!f) { return NULL; } - f->include_paths = calloc(argc, sizeof(char *)); - f->defines = calloc(argc, sizeof(char *)); + f->include_paths = calloc((size_t)argc + 1, sizeof(char *)); + f->defines = calloc((size_t)argc + 1, sizeof(char *)); + if (!f->include_paths || !f->defines) { + cbm_compile_flags_free(f); + return NULL; + } for (int i = 0; i < argc; i++) { if (try_include_flag(f, args, argc, &i, directory)) { @@ -205,7 +215,14 @@ static int process_compile_entry(yyjson_val *entry, const char *repo_path, char return 0; } - cbm_compile_flags_t *f = cbm_extract_flags(flag_args, flag_argc, directory); + char resolved_dir[CBM_SZ_4K]; + if (directory[0] && directory[0] != '/' && + !(isalpha((unsigned char)directory[0]) && directory[1] == ':')) { + snprintf(resolved_dir, sizeof(resolved_dir), "%s/%s", repo_path, directory); + } else { + snprintf(resolved_dir, sizeof(resolved_dir), "%s", directory); + } + cbm_compile_flags_t *f = cbm_extract_flags(flag_args, flag_argc, resolved_dir); if (cmd_val && yyjson_is_str(cmd_val)) { for (int j = 0; j < flag_argc; j++) { @@ -218,14 +235,30 @@ static int process_compile_entry(yyjson_val *entry, const char *repo_path, char } char abs_path[CBM_SZ_4K]; - if (file_path[0] != '/' && directory && directory[0]) { - snprintf(abs_path, sizeof(abs_path), "%s/%s", directory, file_path); + if (file_path[0] != '/' && + !(isalpha((unsigned char)file_path[0]) && file_path[1] == ':')) { + snprintf(abs_path, sizeof(abs_path), "%s/%s", resolved_dir[0] ? resolved_dir : repo_path, + file_path); } else { snprintf(abs_path, sizeof(abs_path), "%s", file_path); } - - size_t repo_len = strlen(repo_path); - if (strncmp(abs_path, repo_path, repo_len) != 0 || abs_path[repo_len] != '/') { + char canonical_file[CBM_SZ_4K]; + char canonical_repo[CBM_SZ_4K]; + if (cbm_canonical_path(abs_path, canonical_file, sizeof(canonical_file))) { + snprintf(abs_path, sizeof(abs_path), "%s", canonical_file); + } + if (!cbm_canonical_path(repo_path, canonical_repo, sizeof(canonical_repo))) { + snprintf(canonical_repo, sizeof(canonical_repo), "%s", repo_path); + } + cbm_normalize_path_sep(abs_path); + cbm_normalize_path_sep(canonical_repo); + size_t repo_len = strlen(canonical_repo); +#ifdef _WIN32 + int prefix_matches = _strnicmp(abs_path, canonical_repo, repo_len) == 0; +#else + int prefix_matches = strncmp(abs_path, canonical_repo, repo_len) == 0; +#endif + if (!prefix_matches || abs_path[repo_len] != '/') { cbm_compile_flags_free(f); return 0; } @@ -262,6 +295,12 @@ int cbm_parse_compile_commands(const char *json_data, const char *repo_path, cha char **paths = calloc(arr_len, sizeof(char *)); cbm_compile_flags_t **flags = calloc(arr_len, sizeof(cbm_compile_flags_t *)); + if (!paths || !flags) { + free(paths); + free(flags); + yyjson_doc_free(doc); + return CBM_NOT_FOUND; + } int count = 0; yyjson_val *entry; @@ -283,3 +322,136 @@ int cbm_parse_compile_commands(const char *json_data, const char *repo_path, cha *out_flags = flags; return count; } + +typedef struct { + char *path; + cbm_compile_flags_t *flags; +} cbm_compile_command_entry_t; + +struct cbm_compile_commands { + cbm_compile_command_entry_t *entries; + int count; +}; + +static int command_path_compare(const char *a, const char *b) { +#ifdef _WIN32 + return _stricmp(a, b); +#else + return strcmp(a, b); +#endif +} + +static int command_entry_compare(const void *a, const void *b) { + const cbm_compile_command_entry_t *left = a; + const cbm_compile_command_entry_t *right = b; + return command_path_compare(left->path, right->path); +} + +const cbm_compile_flags_t *cbm_compile_commands_find(const cbm_compile_commands_t *commands, + const char *relative_path) { + if (!commands || !relative_path) return NULL; + int lo = 0, hi = commands->count; + while (lo < hi) { + int mid = lo + (hi - lo) / 2; + int comparison = command_path_compare(commands->entries[mid].path, relative_path); + if (comparison < 0) lo = mid + 1; + else hi = mid; + } + return lo < commands->count && + command_path_compare(commands->entries[lo].path, relative_path) == 0 + ? commands->entries[lo].flags + : NULL; +} + +void cbm_compile_commands_free(cbm_compile_commands_t *commands) { + if (!commands) return; + for (int i = 0; i < commands->count; i++) { + free(commands->entries[i].path); + cbm_compile_flags_free(commands->entries[i].flags); + } + free(commands->entries); + free(commands); +} + +cbm_compile_commands_t *cbm_compile_commands_load(const char *repo_path) { + if (!repo_path) return NULL; + char configured[CBM_SZ_4K] = ""; + (void)cbm_safe_getenv("CBM_COMPILE_COMMANDS_PATH", configured, sizeof(configured), NULL); + char paths[4][CBM_SZ_4K]; + if (configured[0] && configured[0] != '/' && + !(isalpha((unsigned char)configured[0]) && configured[1] == ':')) { + snprintf(paths[0], sizeof(paths[0]), "%s/%s", repo_path, configured); + } else { + snprintf(paths[0], sizeof(paths[0]), "%s", configured); + } + snprintf(paths[1], sizeof(paths[1]), "%s/compile_commands.json", repo_path); + snprintf(paths[2], sizeof(paths[2]), "%s/build/compile_commands.json", repo_path); + snprintf(paths[3], sizeof(paths[3]), "%s/out/compile_commands.json", repo_path); + for (size_t candidate = 0; candidate < 4; candidate++) { + if (!paths[candidate][0]) continue; + FILE *file = cbm_fopen(paths[candidate], "rb"); + if (!file) continue; + if (fseek(file, 0, SEEK_END) != 0) { + fclose(file); + continue; + } + long size = ftell(file); + if (size <= 0 || size > 128L * 1024L * 1024L || fseek(file, 0, SEEK_SET) != 0) { + fclose(file); + continue; + } + char *json = malloc((size_t)size + 1); + if (!json) { + fclose(file); + return NULL; + } + size_t got = fread(json, 1, (size_t)size, file); + fclose(file); + json[got] = '\0'; + if (got != (size_t)size) { + free(json); + continue; + } + char **paths_out = NULL; + cbm_compile_flags_t **flags_out = NULL; + int count = cbm_parse_compile_commands(json, repo_path, &paths_out, &flags_out); + free(json); + if (count <= 0) { + free(paths_out); + free(flags_out); + continue; + } + cbm_compile_commands_t *commands = calloc(1, sizeof(*commands)); + if (!commands) { + for (int i = 0; i < count; i++) { + free(paths_out[i]); + cbm_compile_flags_free(flags_out[i]); + } + free(paths_out); + free(flags_out); + return NULL; + } + commands->entries = calloc((size_t)count, sizeof(*commands->entries)); + if (!commands->entries) { + free(commands); + for (int i = 0; i < count; i++) { + free(paths_out[i]); + cbm_compile_flags_free(flags_out[i]); + } + free(paths_out); + free(flags_out); + return NULL; + } + commands->count = count; + for (int i = 0; i < count; i++) { + commands->entries[i].path = paths_out[i]; + commands->entries[i].flags = flags_out[i]; + } + free(paths_out); + free(flags_out); + qsort(commands->entries, (size_t)count, sizeof(*commands->entries), command_entry_compare); + cbm_log_info("index.compile_commands.loaded", "entries", "present", "path", paths[candidate]); + return commands; + } + return NULL; +} diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 7c7ed6884..9a82e64c0 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -25,6 +25,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 }; #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/limits.h" +#include "foundation/source_encoding.h" #include "foundation/str_util.h" #include "cbm.h" #include "arena.h" @@ -99,6 +100,19 @@ static char *read_file(const char *path, int *out_len, long *out_size, if (nread > (size_t)size) { nread = (size_t)size; } + size_t decoded_len = 0; + cbm_source_encoding_t encoding; + char *decoded = cbm_source_transcode_utf8(buf, nread, &decoded_len, &encoding); + if (encoding == CBM_SOURCE_INVALID) { + free(buf); + if (out_status) *out_status = CBM_READ_ENCODING; + return NULL; + } + if (decoded) { + free(buf); + buf = decoded; + nread = decoded_len; + } memset(buf + nread, 0, CBM_TS_LOOKAHEAD_PAD); *out_len = (int)nread; return buf; @@ -817,6 +831,9 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t cbm_log_warn("index.file_oversized", "path", rel, "size_mb", itoa_log((int)(file_size / (CBM_SZ_1K * CBM_SZ_1K))), "cap_mb", itoa_log((int)(cap / (CBM_SZ_1K * CBM_SZ_1K)))); + } else if (rst == CBM_READ_ENCODING) { + cbm_pipeline_add_file_error(ctx->pipeline, rel, "unsupported source encoding", + "encoding"); } else if (rst == CBM_READ_OPEN_FAIL || rst == CBM_READ_OOM) { cbm_pipeline_add_file_error(ctx->pipeline, rel, "read failed", "read"); } @@ -826,13 +843,17 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t /* Studio Export XML is transformed to one cacheable aggregate so later * passes see the same calls/usages/semantic carriers as native UDL. */ + const cbm_compile_flags_t *compile_flags = + cbm_compile_commands_find(ctx->compile_commands, rel); CBMFileResult *result = lang == CBM_LANG_OBJECTSCRIPT_EXPORT ? cbm_pipeline_extract_objectscript_export(source, source_len, ctx->project_name, rel, ctx->macro_table, NULL) : cbm_extract_file_ex( - source, source_len, lang, ctx->project_name, rel, CBM_EXTRACT_BUDGET, NULL, - NULL /* no extra defines or include paths */, ctx->macro_table, NULL); + source, source_len, lang, ctx->project_name, rel, CBM_EXTRACT_BUDGET, + compile_flags ? (const char **)compile_flags->defines : NULL, + compile_flags ? (const char **)compile_flags->include_paths : NULL, + ctx->macro_table, NULL); free(source); if (!result) { diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index dbf459814..821a8ee2f 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -11,6 +11,7 @@ * Depends on: worker_pool, graph_buffer (shared IDs + merge), extraction (cbm.h) */ #include "foundation/constants.h" +#include "foundation/source_encoding.h" enum { PP_RING = 4, @@ -281,6 +282,19 @@ static char *read_file(const char *path, int *out_len, long *out_size, } size_t nread = fread(buf, SKIP_ONE, (size_t)size, f); (void)fclose(f); + size_t decoded_len = 0; + cbm_source_encoding_t encoding; + char *decoded = cbm_source_transcode_utf8(buf, nread, &decoded_len, &encoding); + if (encoding == CBM_SOURCE_INVALID) { + free(buf); + if (out_status) *out_status = CBM_READ_ENCODING; + return NULL; + } + if (decoded) { + free(buf); + buf = decoded; + nread = decoded_len; + } buf[nread] = '\0'; *out_len = (int)nread; return buf; @@ -643,6 +657,7 @@ typedef struct { int file_count; const char *project_name; const char *repo_path; + const cbm_compile_commands_t *compile_commands; extract_worker_state_t *workers; int max_workers; @@ -1131,6 +1146,8 @@ static void extract_worker(int worker_id, void *ctx_ptr) { itoa_log((int)(file_size / (CBM_SZ_1K * CBM_SZ_1K))), "cap_mb", itoa_log((int)(cap / (CBM_SZ_1K * CBM_SZ_1K)))); } + } else if (rst == CBM_READ_ENCODING) { + pp_err_add(errs, fi->rel_path, "unsupported source encoding", "encoding"); } else if (rst == CBM_READ_OPEN_FAIL || rst == CBM_READ_OOM) { pp_err_add(errs, fi->rel_path, "read failed", "read"); } @@ -1158,14 +1175,18 @@ static void extract_worker(int worker_id, void *ctx_ptr) { /* Export XML uses the same cache slot as every physical file, so its * generated classes are composed before entering the common registry * and resolution lifecycle. */ + const cbm_compile_flags_t *flags = + cbm_compile_commands_find(ec->compile_commands, fi->rel_path); CBMFileResult *result = fi->language == CBM_LANG_OBJECTSCRIPT_EXPORT ? cbm_pipeline_extract_objectscript_export(source, source_len, ec->project_name, fi->rel_path, ec->macro_table, ec->return_type_table) : cbm_extract_file_ex(source, source_len, fi->language, ec->project_name, - fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL, ec->macro_table, - ec->return_type_table); + fi->rel_path, CBM_EXTRACT_BUDGET, + flags ? (const char **)flags->defines : NULL, + flags ? (const char **)flags->include_paths : NULL, + ec->macro_table, ec->return_type_table); uint64_t file_elapsed_ms = (extract_now_ns() - file_t0) / PP_USEC_PER_MS; @@ -1463,6 +1484,7 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file .retain_per_file_max_bytes = resolved_opts.retain_per_file_max_bytes, .macro_table = pp_macro_table, .return_type_table = ctx->return_type_table, + .compile_commands = ctx->compile_commands, }; atomic_init(&ec.next_worker_id, 0); atomic_init(&ec.next_file_idx, 0); diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 48dab0083..f9ba05b53 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -12,6 +12,7 @@ * Depends on: pass_definitions having populated the registry and graph buffer */ #include "foundation/constants.h" +#include "foundation/source_encoding.h" #include "foundation/mem_core.h" #include "foundation/str_util.h" // cbm_json_escape #include "pipeline/pipeline.h" @@ -60,6 +61,26 @@ static char *read_file(const char *path, int *out_len) { if (nread > (size_t)size) { nread = (size_t)size; } + size_t decoded_len = 0; + cbm_source_encoding_t encoding; + char *decoded = cbm_source_transcode_utf8(buf, nread, &decoded_len, &encoding); + if (encoding == CBM_SOURCE_INVALID) { + cbm_free(CBM_MEM_CLASS_SEMANTIC, buf); + return NULL; + } + if (decoded) { + char *owned = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, decoded_len + CBM_TS_LOOKAHEAD_PAD); + if (!owned) { + free(decoded); + cbm_free(CBM_MEM_CLASS_SEMANTIC, buf); + return NULL; + } + memcpy(owned, decoded, decoded_len + CBM_TS_LOOKAHEAD_PAD); + free(decoded); + cbm_free(CBM_MEM_CLASS_SEMANTIC, buf); + buf = owned; + nread = decoded_len; + } memset(buf + nread, 0, CBM_TS_LOOKAHEAD_PAD); *out_len = (int)nread; return buf; @@ -663,8 +684,13 @@ static CBMFileResult *sem_get_or_extract(cbm_pipeline_ctx_t *ctx, int file_idx, if (!source) { return NULL; } - CBMFileResult *r = cbm_extract_file(source, source_len, fi->language, ctx->project_name, - fi->rel_path, CBM_EXTRACT_BUDGET, NULL, NULL); + const cbm_compile_flags_t *flags = + cbm_compile_commands_find(ctx->compile_commands, fi->rel_path); + CBMFileResult *r = cbm_extract_file_ex( + source, source_len, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET, + flags ? (const char **)flags->defines : NULL, + flags ? (const char **)flags->include_paths : NULL, ctx->macro_table, + ctx->return_type_table); cbm_free(CBM_MEM_CLASS_SEMANTIC, source); if (r) { *owned = true; diff --git a/src/pipeline/pass_usages.c b/src/pipeline/pass_usages.c index 9738f7198..e1eb7b037 100644 --- a/src/pipeline/pass_usages.c +++ b/src/pipeline/pass_usages.c @@ -13,6 +13,7 @@ * Depends on: pass_definitions having populated the registry and graph buffer */ #include "foundation/constants.h" +#include "foundation/source_encoding.h" #include "foundation/str_util.h" // cbm_json_escape #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" @@ -79,6 +80,18 @@ static char *read_file(const char *path, int *out_len) { if (nread > (size_t)size) { nread = (size_t)size; } + size_t decoded_len = 0; + cbm_source_encoding_t encoding; + char *decoded = cbm_source_transcode_utf8(buf, nread, &decoded_len, &encoding); + if (encoding == CBM_SOURCE_INVALID) { + free(buf); + return NULL; + } + if (decoded) { + free(buf); + buf = decoded; + nread = decoded_len; + } memset(buf + nread, 0, CBM_TS_LOOKAHEAD_PAD); *out_len = (int)nread; return buf; @@ -352,8 +365,13 @@ int cbm_pipeline_pass_usages(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fil errors++; continue; } - result = cbm_extract_file(source, source_len, files[i].language, ctx->project_name, rel, - CBM_EXTRACT_BUDGET, NULL, NULL); + const cbm_compile_flags_t *flags = + cbm_compile_commands_find(ctx->compile_commands, rel); + result = cbm_extract_file_ex( + source, source_len, files[i].language, ctx->project_name, rel, + CBM_EXTRACT_BUDGET, flags ? (const char **)flags->defines : NULL, + flags ? (const char **)flags->include_paths : NULL, ctx->macro_table, + ctx->return_type_table); free(source); if (!result) { errors++; diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 6bc0310dd..924c564e4 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -2696,6 +2696,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { struct timespec t0; cbm_clock_gettime(CLOCK_MONOTONIC, &t0); cbm_path_alias_collection_t *path_aliases = NULL; + cbm_compile_commands_t *compile_commands = NULL; cbm_file_hash_t *baseline_manifest = NULL; int baseline_count = 0; char **requested_excluded_dirs = NULL; @@ -2834,6 +2835,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { * when no usable configs are found — non-TS projects pay nothing. */ path_aliases = cbm_load_path_aliases_excluded(p->repo_path, p->excluded_dirs, p->excluded_count); + compile_commands = cbm_compile_commands_load(p->repo_path); /* Build shared context for pass functions */ cbm_pipeline_ctx_t ctx = { @@ -2845,6 +2847,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { .pipeline = p, /* so passes can record per-file skips (Track B) */ .mode = (int)p->mode, .path_aliases = path_aliases, + .compile_commands = compile_commands, .excluded_dirs = p->excluded_dirs, .excluded_count = p->excluded_count, }; @@ -2873,6 +2876,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { cbm_registry_free(p->registry); p->registry = NULL; cbm_path_alias_collection_free(path_aliases); + cbm_compile_commands_free(compile_commands); if (restore_requested_discovery) { cbm_discover_free_excluded(p->excluded_dirs, p->excluded_count); cbm_discover_free_ignored(p->ignored_files, p->ignored_count); diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 894b73257..3bf6852cd 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -32,6 +32,7 @@ enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24 }; #include "foundation/sha256.h" #include +#include #include #include #include @@ -307,6 +308,48 @@ static void semantic_manifest_git_digest(const cbm_git_context_t *git_ctx, out[CBM_SHA256_HEX_LEN] = '\0'; } +static int semantic_manifest_add_compile_inputs(semantic_manifest_builder_t *builder, + const char *project, const char *repo_path) { + char setting[CBM_SZ_64] = ""; + (void)cbm_safe_getenv("CBM_SOURCE_ENCODING", setting, sizeof(setting), NULL); + char digest[CBM_SHA256_HEX_LEN + 1]; + cbm_sha256_hex(setting, strlen(setting), digest); + int rc = semantic_manifest_add_digest(builder, project, + CBM_SEMANTIC_INPUT_PREFIX "source-encoding-v1", + digest, 0, 0); + if (rc != 0) return rc; + + char configured[CBM_SZ_4K] = ""; + (void)cbm_safe_getenv("CBM_COMPILE_COMMANDS_PATH", configured, sizeof(configured), NULL); + char paths[4][CBM_SZ_4K]; + if (configured[0] && configured[0] != '/' && + !(isalpha((unsigned char)configured[0]) && configured[1] == ':')) { + snprintf(paths[0], sizeof(paths[0]), "%s/%s", repo_path, configured); + } else { + snprintf(paths[0], sizeof(paths[0]), "%s", configured); + } + snprintf(paths[1], sizeof(paths[1]), "%s/compile_commands.json", repo_path); + snprintf(paths[2], sizeof(paths[2]), "%s/build/compile_commands.json", repo_path); + snprintf(paths[3], sizeof(paths[3]), "%s/out/compile_commands.json", repo_path); + for (int i = 0; i < 4; i++) { + if (!paths[i][0] || !cbm_file_exists(paths[i])) continue; + char file_sha[CBM_SHA256_HEX_LEN + 1]; + int64_t mtime_ns = 0, size = 0; + if (semantic_manifest_hash_file(paths[i], file_sha, &mtime_ns, &size) != 0) { + return CBM_NOT_FOUND; + } + char key[CBM_SZ_128]; + char identity[CBM_SZ_4K + CBM_SHA256_HEX_LEN + 2]; + snprintf(key, sizeof(key), CBM_SEMANTIC_INPUT_PREFIX "compile-commands-%d", i); + int n = snprintf(identity, sizeof(identity), "%s|%s", paths[i], file_sha); + if (n < 0 || n >= (int)sizeof(identity)) return CBM_NOT_FOUND; + cbm_sha256_hex(identity, (size_t)n, digest); + rc = semantic_manifest_add_digest(builder, project, key, digest, mtime_ns, size); + if (rc != 0) return rc; + } + return 0; +} + static bool semantic_manifest_package_control(const char *name) { if (!name) { return false; @@ -454,6 +497,9 @@ int cbm_pipeline_build_semantic_manifest(const char *project, const char *repo_p rc = semantic_manifest_add_digest(&builder, project, CBM_SEMANTIC_INPUT_PROJECT_CONFIG, project_config_digest, 0, 0); } + if (rc == 0) { + rc = semantic_manifest_add_compile_inputs(&builder, project, repo_path); + } struct timespec t_hash; cbm_clock_gettime(CLOCK_MONOTONIC, &t_hash); int hash_workers = cbm_default_worker_count(true); @@ -1750,6 +1796,10 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p } for (int i = 0; i < stored_count; i++) { if (!cbm_ht_get(fresh_by_path, stored[i].rel_path)) { + if (semantic_manifest_is_virtual_path(stored[i].rel_path)) { + decline = "semantic_input_removed"; + goto done; + } if (!cbm_ht_get(files_by_path, stored[i].rel_path) && closure_is_alias_config(plan_aliases, stored[i].rel_path)) { /* Removed alias config: its governed files resolve without diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 78ce486a0..2ddf39f53 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -83,6 +83,7 @@ void cbm_pkg_entries_free(cbm_pkg_entries_t *e); /* Shared context passed to each pass function. * Derived from cbm_pipeline_t fields during run. */ +typedef struct cbm_compile_commands cbm_compile_commands_t; typedef struct { const char *project_name; /* borrowed from pipeline */ const char *repo_path; /* borrowed from pipeline */ @@ -105,6 +106,7 @@ typedef struct { * configs are an easy follow-on). NULL when no usable configs were found. * Owned by pipeline.c / pipeline_incremental.c. */ const cbm_path_alias_collection_t *path_aliases; + const cbm_compile_commands_t *compile_commands; /* Directory subtrees excluded during discovery. Borrowed from pipeline.c. */ char **excluded_dirs; @@ -399,6 +401,13 @@ typedef struct { char standard[CBM_SZ_32]; } cbm_compile_flags_t; +/* Load one compilation database for this run. The map and its flags remain + * read-only while extraction workers run. NULL means no usable database. */ +cbm_compile_commands_t *cbm_compile_commands_load(const char *repo_path); +const cbm_compile_flags_t *cbm_compile_commands_find(const cbm_compile_commands_t *commands, + const char *relative_path); +void cbm_compile_commands_free(cbm_compile_commands_t *commands); + /* Split a shell command string into arguments (handles quoting). * Writes args to out[]. Returns count. Caller must free each out[i]. */ int cbm_split_command(const char *cmd, char **out, int max_out); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 3a69a0877..93ec87943 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -9745,6 +9745,59 @@ TEST(compile_commands_parse_json) { PASS(); } +TEST(compile_commands_windows_msvc_flags) { + const char *json = + "[{\"directory\":\"C:\\\\synthetic\\\\build\"," + "\"file\":\"C:\\\\synthetic\\\\src\\\\sample.cpp\"," + "\"arguments\":[\"cl\",\"/IC:\\\\synthetic\\\\include\"," + "\"/DWIN_BUILD=1\",\"/std:c++20\"]}]"; + char **paths = NULL; + cbm_compile_flags_t **flags = NULL; + int count = cbm_parse_compile_commands(json, "C:/synthetic", &paths, &flags); + ASSERT_EQ(count, 1); + ASSERT_STR_EQ(paths[0], "src/sample.cpp"); + ASSERT_EQ(flags[0]->define_count, 1); + ASSERT_STR_EQ(flags[0]->defines[0], "WIN_BUILD=1"); + ASSERT_EQ(flags[0]->include_count, 1); + cbm_compile_flags_free(flags[0]); + free(paths[0]); + free(paths); + free(flags); + PASS(); +} + +TEST(compile_commands_load_and_lookup) { + char repo[256] = "/tmp/cbm_compile_db_XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(repo)); + for (char *p = repo; *p; p++) { + if (*p == '\\') *p = '/'; + } + char source_path[512]; + char database_path[512]; + snprintf(source_path, sizeof(source_path), "%s/sample.cpp", repo); + snprintf(database_path, sizeof(database_path), "%s/compile_commands.json", repo); + FILE *source = cbm_fopen(source_path, "wb"); + ASSERT_NOT_NULL(source); + fputs("int sample(void) { return 1; }\n", source); + fclose(source); + FILE *database = cbm_fopen(database_path, "wb"); + ASSERT_NOT_NULL(database); + fprintf(database, + "[{\"directory\":\"%s\",\"file\":\"%s\"," + "\"arguments\":[\"clang++\",\"-DACTIVE=1\",\"-c\",\"sample.cpp\"]}]", + repo, source_path); + fclose(database); + cbm_compile_commands_t *commands = cbm_compile_commands_load(repo); + ASSERT_NOT_NULL(commands); + const cbm_compile_flags_t *flags = cbm_compile_commands_find(commands, "sample.cpp"); + ASSERT_NOT_NULL(flags); + ASSERT_EQ(flags->define_count, 1); + ASSERT_STR_EQ(flags->defines[0], "ACTIVE=1"); + cbm_compile_commands_free(commands); + rm_rf(repo); + PASS(); +} + TEST(compile_commands_parse_empty) { char **paths = NULL; cbm_compile_flags_t **flags = NULL; @@ -14932,6 +14985,8 @@ SUITE(pipeline) { RUN_TEST(compile_commands_split_command); RUN_TEST(compile_commands_extract_flags); RUN_TEST(compile_commands_parse_json); + RUN_TEST(compile_commands_windows_msvc_flags); + RUN_TEST(compile_commands_load_and_lookup); RUN_TEST(compile_commands_parse_empty); RUN_TEST(compile_commands_parse_invalid); /* Infrascan helpers */ diff --git a/tests/test_platform.c b/tests/test_platform.c index 5995a4b01..b3da47666 100644 --- a/tests/test_platform.c +++ b/tests/test_platform.c @@ -8,6 +8,7 @@ #include "../src/foundation/compat_thread.h" #include "../src/foundation/platform.h" #include "../src/foundation/platform_internal.h" +#include "../src/foundation/source_encoding.h" #include "../src/foundation/system_info_internal.h" #include #include @@ -475,6 +476,28 @@ TEST(platform_mkstemp_and_mkdtemp_survive_non_ascii_directory) { PASS(); } +#ifdef _WIN32 +TEST(platform_mkstemp_retained_files_exceed_crt_namespace) { + char base[CBM_SZ_256] = "/tmp/cbm-many-temp-XXXXXX"; + ASSERT_NOT_NULL(cbm_mkdtemp(base)); + char paths[64][CBM_SZ_512] = {{0}}; + int created = 0; + for (; created < 64; created++) { + int written = snprintf(paths[created], sizeof(paths[created]), + "%s/.worker-log-XXXXXX", base); + ASSERT_TRUE(written > 0 && written < (int)sizeof(paths[created])); + int descriptor = cbm_mkstemp(paths[created]); + ASSERT_TRUE(descriptor >= 0); + ASSERT_EQ(_close(descriptor), 0); + } + for (int index = 0; index < created; index++) { + ASSERT_EQ(cbm_unlink(paths[index]), 0); + } + ASSERT_EQ(cbm_rmdir(base), 0); + PASS(); +} +#endif + typedef struct { atomic_int *ready; atomic_bool *go; @@ -1128,13 +1151,47 @@ TEST(cgroup_no_mem_files) { #endif /* __linux__ */ +TEST(platform_source_encoding_cp949_and_utf8) { + size_t length = 0; + cbm_source_encoding_t encoding = CBM_SOURCE_INVALID; + const char utf8[] = "int n; /* \xea\xb0\x80 */\n"; + char *converted = cbm_source_transcode_utf8(utf8, sizeof(utf8) - 1, &length, &encoding); + ASSERT_NULL(converted); + ASSERT_EQ(encoding, CBM_SOURCE_UTF8); + ASSERT_EQ(length, sizeof(utf8) - 1); + + const char cp949[] = "int n; /* \xb0\xa1 */\n"; + converted = cbm_source_transcode_utf8(cp949, sizeof(cp949) - 1, &length, &encoding); + ASSERT_NOT_NULL(converted); + ASSERT_EQ(encoding, CBM_SOURCE_CP949); + ASSERT_STR_EQ(converted, utf8); + free(converted); + + const char bom[] = "\xef\xbb\xbfint n;\n"; + converted = cbm_source_transcode_utf8(bom, sizeof(bom) - 1, &length, &encoding); + ASSERT_NOT_NULL(converted); + ASSERT_EQ(encoding, CBM_SOURCE_UTF8_BOM); + ASSERT_STR_EQ(converted, "int n;\n"); + free(converted); + + const char invalid[] = {'x', (char)0xff}; + converted = cbm_source_transcode_utf8(invalid, sizeof(invalid), &length, &encoding); + ASSERT_NULL(converted); + ASSERT_EQ(encoding, CBM_SOURCE_INVALID); + PASS(); +} + SUITE(platform) { + RUN_TEST(platform_source_encoding_cp949_and_utf8); RUN_TEST(platform_file_apis_survive_max_path_overflow); RUN_TEST(platform_mkdir_p_follows_own_symlink_only_when_opted_in); RUN_TEST(platform_mkdir_p_symlink_trust_is_bounded_by_owner_and_mode); RUN_TEST(platform_mkdir_p_follow_owned_is_per_call_site); RUN_TEST(platform_mkdir_p_resolves_link_text_from_the_link_directory); RUN_TEST(platform_mkstemp_and_mkdtemp_survive_non_ascii_directory); +#ifdef _WIN32 + RUN_TEST(platform_mkstemp_retained_files_exceed_crt_namespace); +#endif RUN_TEST(platform_mkdtemp_is_thread_safe); RUN_TEST(platform_counter_scaling_avoids_intermediate_overflow); RUN_TEST(platform_counter_scaling_preserves_monotonic_deadlines);