Fix Windows indexing recovery and source settings

Makefile.cbm

@@ -329,6 +329,7 @@ FOUNDATION_SRCS = \
     src/foundation/subprocess.c \
     src/foundation/sha256.c \
     src/foundation/secure_random.c \
+    src/foundation/source_encoding.c \
     src/foundation/macos_acl.c \
     src/foundation/private_file_lock.c \
     src/foundation/lock_registry.c

src/foundation/compat.c

@@ -269,36 +269,55 @@ int cbm_mkstemp(char *tmpl) {
         errno = ENAMETOOLONG;
         return CBM_NOT_FOUND;
     }
-    /* Wide-API expansion and open: worker staging files land inside
-     * CBM_CACHE_DIR, which users may place at non-ASCII paths; the ANSI CRT
-     * (_mktemp/_open) mangles those bytes in the local codepage. */
+    /* Keep the six-character mkstemp contract, but do not use _wmktemp:
+     * that CRT helper has a tiny name space on Windows and retained worker
+     * logs can exhaust it during recovery. The exclusive open closes races
+     * with other processes; collisions simply draw another random name. */
     wchar_t *wide_template = cbm_utf8_to_wide(buf);
-    if (!wide_template || !_wmktemp(wide_template)) {
-        free(wide_template);
+    if (!wide_template) {
+        errno = EINVAL;
         return CBM_NOT_FOUND;
     }
-    char *expanded_for_open = cbm_wide_to_utf8(wide_template);
-    wchar_t *wide_open = expanded_for_open ? cbm_path_to_wide(expanded_for_open) : NULL;
-    free(expanded_for_open);
-    if (!wide_open) {
+    size_t wide_len = wcslen(wide_template);
+    if (wide_len < 6 || wcscmp(wide_template + wide_len - 6, L"XXXXXX") != 0) {
         free(wide_template);
+        errno = EINVAL;
         return CBM_NOT_FOUND;
     }
-    int fd = _wopen(wide_open, _O_CREAT | _O_EXCL | _O_RDWR | _O_BINARY, _S_IREAD | _S_IWRITE);
-    free(wide_open);
-    if (fd >= 0) {
+    static const wchar_t hex[] = L"0123456789abcdef";
+    for (int attempt = 0; attempt < 128; attempt++) {
+        unsigned int random_bits = 0;
+        if (!cbm_secure_random(&random_bits, sizeof(random_bits))) {
+            errno = EIO;
+            break;
+        }
+        for (int digit = 0; digit < 6; digit++) {
+            wide_template[wide_len - 6 + digit] = hex[(random_bits >> (digit * 4)) & 0xf];
+        }
         char *expanded = cbm_wide_to_utf8(wide_template);
-        if (!expanded || strlen(expanded) >= sizeof(buf)) {
+        wchar_t *wide_open = expanded ? cbm_path_to_wide(expanded) : NULL;
+        if (!expanded || !wide_open || strlen(expanded) >= sizeof(buf)) {
+            free(expanded);
+            free(wide_open);
+            errno = ENAMETOOLONG;
+            break;
+        }
+        int fd = _wopen(wide_open, _O_CREAT | _O_EXCL | _O_RDWR | _O_BINARY,
+                        _S_IREAD | _S_IWRITE);
+        free(wide_open);
+        if (fd >= 0) {
+            strcpy(tmpl, expanded);
             free(expanded);
             free(wide_template);
-            (void)_close(fd);
-            return CBM_NOT_FOUND;
+            return fd;
         }
-        strcpy(tmpl, expanded);
         free(expanded);
+        if (errno != EEXIST) {
+            break;
+        }
     }
     free(wide_template);
-    return fd;
+    return CBM_NOT_FOUND;
 }
 #endif
 

src/foundation/compat.h

@@ -166,11 +166,9 @@ static inline int cbm_setenv(const char *name, const char *value, int overwrite)
         free(wide_value);
         return EINVAL;
     }
-    /* Keep the CRT's narrow environment useful for legacy getenv callers,
-     * then repair the process-wide Windows environment with the actual UTF-16
-     * value. _putenv_s alone routes UTF-8 path bytes through the active ANSI
-     * code page, which corrupts non-ASCII cache roots inherited by children. */
-    int status = _putenv_s(name, value);
+    /* Update the CRT through its wide environment. _putenv_s rejects valid
+     * UTF-8 values when the active ANSI code page cannot represent them. */
+    int status = _wputenv_s(wide_name, wide_value);
     if (status == 0 && !SetEnvironmentVariableW(wide_name, wide_value)) {
         status = EINVAL;
     }

src/foundation/limits.h

@@ -17,6 +17,7 @@ typedef enum {
     CBM_READ_EMPTY,     /* zero/negative size — benign, nothing to index */
     CBM_READ_OVERSIZED, /* size exceeds cbm_max_file_bytes() */
     CBM_READ_OOM,       /* buffer allocation failed */
+    CBM_READ_ENCODING,  /* source bytes are neither UTF-8 nor CP949 */
 } cbm_read_status_t;
 
 /* Maximum size (bytes) of a single source file the indexer will read into

src/foundation/mem_override_win.c

@@ -56,6 +56,7 @@ void *__real_malloc(size_t size);
 void *__real_calloc(size_t count, size_t size);
 void *__real_realloc(void *block, size_t size);
 void __real_free(void *block);
+void __real__aligned_free(void *block);
 char *__real_strdup(const char *text);
 size_t __real__msize(void *block);
 /* Wrappers must call __real_* for anything that is itself wrapped: a plain
@@ -155,9 +156,58 @@ void __wrap__aligned_free(void *block) {
         mi_free(block);
         return;
     }
-    __real_free(block);
+    /* A CRT aligned block has its own bookkeeping header. Plain free is not
+     * its matching deallocator. */
+    __real__aligned_free(block);
+}
+
+#if defined(CBM_MEM_GLOBAL_OVERRIDE) && CBM_MEM_GLOBAL_OVERRIDE
+/* LLVM-MinGW's static C++ runtime calls _aligned_malloc through the linker's
+ * wrapper but its aligned operator delete can call the CRT import pointer
+ * __imp__aligned_free directly. --wrap=_aligned_free does not rewrite that
+ * import-pointer reference. Provide the six C++ ABI delete entry points so
+ * every aligned new/delete pair reaches the same owner-aware deallocator. */
+void cbm_cxx_aligned_delete(void *block) __asm__("_ZdlPvSt11align_val_t");
+void cbm_cxx_aligned_delete(void *block) {
+    __wrap__aligned_free(block);
+}
+
+void cbm_cxx_aligned_delete_nothrow(void *block, const void *tag)
+    __asm__("_ZdlPvSt11align_val_tRKSt9nothrow_t");
+void cbm_cxx_aligned_delete_nothrow(void *block, const void *tag) {
+    (void)tag;
+    __wrap__aligned_free(block);
+}
+
+void cbm_cxx_aligned_delete_sized(void *block, size_t size, size_t alignment)
+    __asm__("_ZdlPvySt11align_val_t");
+void cbm_cxx_aligned_delete_sized(void *block, size_t size, size_t alignment) {
+    (void)size;
+    (void)alignment;
+    __wrap__aligned_free(block);
 }
 
+void cbm_cxx_aligned_array_delete(void *block) __asm__("_ZdaPvSt11align_val_t");
+void cbm_cxx_aligned_array_delete(void *block) {
+    __wrap__aligned_free(block);
+}
+
+void cbm_cxx_aligned_array_delete_nothrow(void *block, const void *tag)
+    __asm__("_ZdaPvSt11align_val_tRKSt9nothrow_t");
+void cbm_cxx_aligned_array_delete_nothrow(void *block, const void *tag) {
+    (void)tag;
+    __wrap__aligned_free(block);
+}
+
+void cbm_cxx_aligned_array_delete_sized(void *block, size_t size, size_t alignment)
+    __asm__("_ZdaPvySt11align_val_t");
+void cbm_cxx_aligned_array_delete_sized(void *block, size_t size, size_t alignment) {
+    (void)size;
+    (void)alignment;
+    __wrap__aligned_free(block);
+}
+#endif
+
 void *__wrap_realloc(void *block, size_t size) {
     if (!block) {
         void *fresh = mi_malloc(size);

src/foundation/source_encoding.c

@@ -0,0 +1,128 @@
+#include "foundation/source_encoding.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <errno.h>
+#include <iconv.h>
+#endif
+
+static int source_valid_utf8(const unsigned char *bytes, size_t length) {
+    for (size_t i = 0; i < length;) {
+        unsigned char first = bytes[i];
+        if (first < 0x80) {
+            if (first == 0) return 0;
+            i++;
+            continue;
+        }
+        unsigned int count;
+        uint32_t cp;
+        if (first >= 0xc2 && first <= 0xdf) {
+            count = 2;
+            cp = first & 0x1f;
+        } else if (first >= 0xe0 && first <= 0xef) {
+            count = 3;
+            cp = first & 0x0f;
+        } else if (first >= 0xf0 && first <= 0xf4) {
+            count = 4;
+            cp = first & 0x07;
+        } else {
+            return 0;
+        }
+        if (count > length - i) return 0;
+        for (unsigned int j = 1; j < count; j++) {
+            unsigned char next = bytes[i + j];
+            if ((next & 0xc0) != 0x80) return 0;
+            cp = (cp << 6) | (next & 0x3f);
+        }
+        if ((count == 2 && cp < 0x80) || (count == 3 && cp < 0x800) ||
+            (count == 4 && cp < 0x10000) || cp > 0x10ffff ||
+            (cp >= 0xd800 && cp <= 0xdfff)) return 0;
+        i += count;
+    }
+    return 1;
+}
+
+char *cbm_source_transcode_utf8(const char *input, size_t length, size_t *out_length,
+                                cbm_source_encoding_t *encoding) {
+    if (!input || !out_length || !encoding) return NULL;
+    *out_length = length;
+    *encoding = CBM_SOURCE_INVALID;
+    size_t bom = length >= 3 && (unsigned char)input[0] == 0xef &&
+                         (unsigned char)input[1] == 0xbb &&
+                         (unsigned char)input[2] == 0xbf ? 3 : 0;
+    const char *requested = getenv("CBM_SOURCE_ENCODING");
+    int force_cp949 = requested &&
+                      (strcmp(requested, "cp949") == 0 || strcmp(requested, "euc-kr") == 0);
+    int force_utf8 = requested && strcmp(requested, "utf8") == 0;
+    int valid_utf8 = source_valid_utf8((const unsigned char *)input + bom, length - bom);
+    if (bom && !valid_utf8) return NULL;
+    if (valid_utf8 && (!force_cp949 || bom)) {
+        *encoding = bom ? CBM_SOURCE_UTF8_BOM : CBM_SOURCE_UTF8;
+        if (!bom) return NULL;
+        char *copy = malloc(length - bom + 17);
+        if (!copy) {
+            *encoding = CBM_SOURCE_INVALID;
+            return NULL;
+        }
+        memcpy(copy, input + bom, length - bom);
+        memset(copy + length - bom, 0, 17);
+        *out_length = length - bom;
+        return copy;
+    }
+    if (force_utf8) return NULL;
+    if (length == 0 || length > (size_t)INT32_MAX) return NULL;
+    /* A NUL in a source buffer is binary data, not a legacy text encoding. */
+    if (memchr(input, 0, length)) return NULL;
+#ifdef _WIN32
+    int wide_length = MultiByteToWideChar(949, MB_ERR_INVALID_CHARS, input, (int)length, NULL, 0);
+    if (wide_length <= 0) return NULL;
+    wchar_t *wide = malloc((size_t)wide_length * sizeof(*wide));
+    if (!wide) return NULL;
+    if (MultiByteToWideChar(949, MB_ERR_INVALID_CHARS, input, (int)length, wide,
+                            wide_length) != wide_length) {
+        free(wide);
+        return NULL;
+    }
+    int needed = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wide, wide_length, NULL, 0,
+                                     NULL, NULL);
+    char *converted = needed > 0 ? malloc((size_t)needed + 17) : NULL;
+    if (!converted || WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wide, wide_length,
+                                          converted, needed, NULL, NULL) != needed) {
+        free(converted);
+        free(wide);
+        return NULL;
+    }
+    free(wide);
+    memset(converted + needed, 0, 17);
+    *out_length = (size_t)needed;
+#else
+    if (length > (SIZE_MAX - 17) / 4) return NULL;
+    iconv_t converter = iconv_open("UTF-8", "CP949");
+    if (converter == (iconv_t)-1) return NULL;
+    size_t capacity = length * 4 + 17;
+    char *converted = malloc(capacity);
+    if (!converted) {
+        iconv_close(converter);
+        return NULL;
+    }
+    char *read_ptr = (char *)input;
+    char *write_ptr = converted;
+    size_t remaining = length;
+    size_t available = capacity - 17;
+    size_t result = iconv(converter, &read_ptr, &remaining, &write_ptr, &available);
+    iconv_close(converter);
+    if (result == (size_t)-1 || remaining != 0) {
+        free(converted);
+        return NULL;
+    }
+    *out_length = (size_t)(write_ptr - converted);
+    memset(write_ptr, 0, 17);
+#endif
+    *encoding = CBM_SOURCE_CP949;
+    return converted;
+}

src/foundation/source_encoding.h

@@ -0,0 +1,21 @@
+#ifndef CBM_SOURCE_ENCODING_H
+#define CBM_SOURCE_ENCODING_H
+
+#include <stddef.h>
+
+typedef enum {
+    CBM_SOURCE_UTF8,
+    CBM_SOURCE_UTF8_BOM,
+    CBM_SOURCE_CP949,
+    CBM_SOURCE_INVALID
+} cbm_source_encoding_t;
+
+/* Auto-detect UTF-8 and CP949/EUC-KR, or honor CBM_SOURCE_ENCODING=utf8,
+ * cp949, or euc-kr for a repository with a known encoding.
+ * Return a malloc-owned UTF-8 replacement when conversion is needed.
+ * A NULL result with CBM_SOURCE_UTF8 means the input is already UTF-8;
+ * a NULL result with CBM_SOURCE_INVALID means conversion failed. */
+char *cbm_source_transcode_utf8(const char *input, size_t length, size_t *out_length,
+                                cbm_source_encoding_t *encoding);
+
+#endif

src/mcp/index_supervisor.c

@@ -13,6 +13,7 @@
 #include "ui/http_server.h"      /* cbm_http_server_resolve_binary_path */
 
 #include <limits.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
@@ -705,6 +706,11 @@ int cbm_index_worker_start_with_log(const char *args_json, size_t memory_budget_
     worker_result_init(&handle->result);
     if (!worker_unique_file(handle->response_path, sizeof(handle->response_path), "response") ||
         !worker_unique_file(handle->log_path, sizeof(handle->log_path), "log")) {
+        int saved_errno = errno;
+        char error_text[CBM_SZ_32];
+        (void)snprintf(error_text, sizeof(error_text), "%d", saved_errno);
+        cbm_log_error("index.supervisor.artifact_create_failed", "artifact",
+                      handle->response_path[0] ? "log" : "response", "errno", error_text);
         (void)cbm_unlink(handle->response_path);
         (void)cbm_unlink(handle->log_path);
         free(handle);

src/mcp/mcp.c

@@ -10776,7 +10776,11 @@ static char *index_run_supervised(cbm_mcp_server_t *srv, const char *args) {
         cbm_mcp_supervised_result_disposition_t recovery_disposition =
             cbm_mcp_supervised_result_disposition(rc2, &wr2);
         if (recovery_disposition == CBM_MCP_SUPERVISED_RESULT_FALLBACK) {
-            last_outcome = wr2.outcome;
+            /* A recovery setup failure must not replace the worker failure
+             * that made us enter recovery in the first place. */
+            cbm_log_error("index.supervisor.recovery_start_failed", "original_outcome",
+                          cbm_proc_outcome_str(last_outcome), "recovery_outcome",
+                          cbm_proc_outcome_str(wr2.outcome));
             cbm_index_worker_result_free(&wr2);
             break; /* spawn failed mid-recovery — give up */
         }

src/pipeline/pass_calls.c

@@ -10,6 +10,7 @@
  * Depends on: pass_definitions having populated the registry and graph buffer
  */
 #include "foundation/constants.h"
+#include "foundation/source_encoding.h"
 
 enum { PC_RING = 4, PC_RING_MASK = 3, PC_SIG_SCAN = 15, PC_REGEX_GRP = 2 };
 /* Confidence for a service-pattern HTTP/ASYNC edge emitted when registry
@@ -73,6 +74,18 @@ static char *read_file(const char *path, int *out_len) {
     if (nread > (size_t)size) {
         nread = (size_t)size;
     }
+    size_t decoded_len = 0;
+    cbm_source_encoding_t encoding;
+    char *decoded = cbm_source_transcode_utf8(buf, nread, &decoded_len, &encoding);
+    if (encoding == CBM_SOURCE_INVALID) {
+        free(buf);
+        return NULL;
+    }
+    if (decoded) {
+        free(buf);
+        buf = decoded;
+        nread = decoded_len;
+    }
     memset(buf + nread, 0, CBM_TS_LOOKAHEAD_PAD);
     *out_len = (int)nread;
     return buf;
@@ -761,9 +774,13 @@ static CBMFileResult *calls_get_or_extract(cbm_pipeline_ctx_t *ctx, int idx,
     if (!src) {
         return NULL;
     }
-    CBMFileResult *r = cbm_extract_file_ex(src, slen, fi->language, ctx->project_name, fi->rel_path,
-                                           CBM_EXTRACT_BUDGET, NULL, NULL, ctx->macro_table,
-                                           ctx->return_type_table);
+    const cbm_compile_flags_t *flags =
+        cbm_compile_commands_find(ctx->compile_commands, fi->rel_path);
+    CBMFileResult *r = cbm_extract_file_ex(
+        src, slen, fi->language, ctx->project_name, fi->rel_path, CBM_EXTRACT_BUDGET,
+        flags ? (const char **)flags->defines : NULL,
+        flags ? (const char **)flags->include_paths : NULL, ctx->macro_table,
+        ctx->return_type_table);
     free(src);
     if (r) {
         *owned = true;