From c06ceb163880bd3b22d13ebc64093d599028af2e Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Fri, 25 Sep 2026 18:37:16 +0200 Subject: [PATCH] feat(watcher): opt-in polling of non-git project roots (#1948) A project indexed from a directory that is not a git repository was never refreshed after its first index: init_baseline classified it is_git=false and poll_project/check_changes returned early for it forever, so the graph silently went stale. The comment in init_baseline even claimed such folders were "polled as a plain directory", which was never true. Add an opt-in config key, watch_non_git (default false, read once at daemon startup like watcher_enabled). When on, a non-git root is polled on the existing adaptive cadence (cbm_watcher_poll_interval_ms) by a tree signature: the indexer's own cbm_discover walk (same skip lists, .gitignore stack and .cbmignore), sorted by path and folded over (relative path, size, mtime). A changed signature takes the existing reindex path with the same commit-after-success discipline as the #937 dirty signature. The baseline leaves the signature "unknown", so the first poll reindexes once (at-least-once, covering edits made while the daemon was down). Runaway guard: only files discovery would index enter the signature, so cbm's own .codebase-memory output (built-in skip) and the cache directory (pruned by absolute path) can never make a finished reindex look like a new change; a scratch folder under an unrelated dirty repository is polled on its own files and never inherits the ancestor's dirty state. cbm_file_info_t gains mtime_ns, filled from the stat discovery already performs, so the signature needs no second stat per file and stays UTF-8-correct on Windows (discovery's wide stat). Docs: README and docs/CONFIGURATION.md now say non-git roots are not watched by default and document watch_non_git. The staleness signal in index_status is left to #1561's freshness object. Refs #1948. Signed-off-by: Martin Vogel --- README.md | 3 + docs/CONFIGURATION.md | 1 + src/cli/cli.c | 7 ++ src/cli/cli.h | 6 ++ src/daemon/host.c | 6 ++ src/discover/discover.c | 27 +++++- src/discover/discover.h | 1 + src/watcher/watcher.c | 148 ++++++++++++++++++++++++----- src/watcher/watcher.h | 12 +++ tests/test_cli.c | 35 +++++++ tests/test_watcher.c | 205 ++++++++++++++++++++++++++++++++++++++++ 11 files changed, 422 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 7564427eaf..7cc423dbc5 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,8 @@ Watcher registration is controlled separately by `auto_watch` (default `true`). To turn the watcher off entirely, set `config set watcher_enabled false` (default `true`): the background poll thread never starts and no project is registered, while `auto_index` and manual `index_repository` keep working. Unlike `auto_watch` — which is consulted per session — `watcher_enabled` is read once when the background daemon starts, so run `codebase-memory-mcp daemon stop` after changing it; reconnecting your MCP client alone will not restart the daemon. See [docs/CONFIGURATION.md](docs/CONFIGURATION.md#2-cli-managed-runtime-settings). +The watcher follows **git** projects only by default: a project indexed from a directory that is not a git repository is not watched and stays at its last index until you run `index_repository` again. To have such roots polled too, set `config set watch_non_git true` (default `false`, read when the daemon starts): they are then checked on the same cadence with a file-tree scan that uses the indexer's own ignore rules. + ### Keeping Up to Date **Updates run from the install script on every platform, not from inside the running binary.** `codebase-memory-mcp update` validates your flags and then prints the exact command to run: @@ -763,6 +765,7 @@ codebase-memory-mcp config set auto_index true # auto-index on session codebase-memory-mcp config set auto_index_limit 50000 # max files for auto-index codebase-memory-mcp config set auto_watch false # don't register background git watcher (default: true) codebase-memory-mcp config set watcher_enabled false # stop the watcher thread entirely (default: true) +codebase-memory-mcp config set watch_non_git true # also poll non-git project roots (default: false) codebase-memory-mcp config set index_max_files 250000 # optional per-index source-file limit codebase-memory-mcp config set index_max_source_mb 16384 # optional per-index source-size limit codebase-memory-mcp config reset auto_index # reset to default diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 024f37e930..fc9a0a9953 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -89,6 +89,7 @@ Current keys: | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | | `auto_watch` | `true` | Register the session's project with the background git watcher on connect. Set `false` to keep a session from registering its project (the watcher still runs for other projects). | | `watcher_enabled` | `true` | Master switch for the background watcher subsystem. Set `false` to stop the watcher from starting at all — no poll thread and no project registration. Reindex manually with `index_repository` when disabled. | +| `watch_non_git` | `false` | Also poll project roots that are **not git repositories**. By default the watcher only follows git projects, so a project indexed from a plain directory is never refreshed after its first index — reindex it manually with `index_repository`. Set `true` to poll such roots on the same adaptive cadence with a file-tree scan: the indexer's own discovery walk (same skip lists, `.gitignore` and `.cbmignore` rules) hashed over each file's path, size and mtime. Any change reindexes once; paths the indexer skips (including cbm's own `.codebase-memory/` output) never trigger. The first poll after the daemon starts reindexes each such project once, since nothing records which tree state the index holds. The scan walks the whole tree every poll, so it costs more than git polling on very large trees. Read once when the daemon starts, like `watcher_enabled`. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | diff --git a/src/cli/cli.c b/src/cli/cli.c index d7b0cc7406..7902f34d0f 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -7450,6 +7450,11 @@ bool cbm_config_watcher_enabled(cbm_config_t *cfg) { return cbm_config_get_bool(cfg, CBM_CONFIG_WATCHER_ENABLED, true); } +/* Opt-in tree polling of non-git roots (#1948); see cli.h. */ +bool cbm_config_watch_non_git(cbm_config_t *cfg) { + return cbm_config_get_bool(cfg, CBM_CONFIG_WATCH_NON_GIT, false); +} + bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (!cfg || !policy) { @@ -7490,6 +7495,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_AUTO_WATCH, "true", "Register background git watcher on session connect"}, {CBM_CONFIG_WATCHER_ENABLED, "true", "Run the background watcher thread (auto-reindex); false to disable"}, + {CBM_CONFIG_WATCH_NON_GIT, "false", + "Also poll non-git project roots for changes (file-tree scan); off by default"}, {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"}, {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"}, {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, diff --git a/src/cli/cli.h b/src/cli/cli.h index 0edc4d3eef..c45c057071 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -446,6 +446,7 @@ bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t #define CBM_CONFIG_AUTO_WATCH "auto_watch" #define CBM_CONFIG_UI_LANG "ui-lang" #define CBM_CONFIG_WATCHER_ENABLED "watcher_enabled" +#define CBM_CONFIG_WATCH_NON_GIT "watch_non_git" /* #1558: the graph UI's loopback listener. Stored in the UI config file rather * than the key-value store, but surfaced through `config` so it is findable. */ #define CBM_CONFIG_UI_ENABLED "ui_enabled" @@ -460,6 +461,11 @@ bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t * the watcher. */ bool cbm_config_watcher_enabled(cbm_config_t *cfg); +/* Whether the watcher also polls NON-GIT project roots (default false, #1948). + * Non-git roots are otherwise never refreshed after their first index. Read + * once at daemon startup, like watcher_enabled. NULL-safe (NULL → false). */ +bool cbm_config_watch_non_git(cbm_config_t *cfg); + /* ── Binary activation safety ─────────────────────────────────── */ typedef void *cbm_cli_activation_lock_t; diff --git a/src/daemon/host.c b/src/daemon/host.c index 3e509e2071..a44fcdaf1d 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -609,6 +609,12 @@ static bool host_state_prepare(host_state_t *host, const cbm_daemon_ipc_endpoint if (watcher_enabled) { host->watch_store = cbm_store_open_memory(); host->watcher = cbm_watcher_new(host->watch_store, host_watcher_index, host); + /* #1948: non-git roots stay unwatched unless watch_non_git opts in. + * Same read-once-at-startup contract as watcher_enabled. */ + if (host->watcher && cbm_config_watch_non_git(host->runtime_config)) { + cbm_watcher_set_poll_non_git(host->watcher, true); + cbm_log_info("watcher.non_git", "mode", "tree_poll"); + } } else { cbm_log_info("watcher.disabled", "reason", "config"); } diff --git a/src/discover/discover.c b/src/discover/discover.c index 4595a8b9e4..fa8e4bf2b5 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -549,8 +549,23 @@ static void file_list_add_ignored(file_list_t *fl, const char *rel_path, const c fl->ignored_count++; } +/* Nanosecond mtime from a stat result, per platform. Windows' wide_stat only + * carries whole seconds, so the value is seconds scaled to ns there. */ +#define DISCOVER_NS_PER_SEC 1000000000LL +static int64_t discover_stat_mtime_ns(const struct stat *st) { +#ifdef __APPLE__ + return ((int64_t)st->st_mtimespec.tv_sec * DISCOVER_NS_PER_SEC) + + (int64_t)st->st_mtimespec.tv_nsec; +#elif defined(_WIN32) + return (int64_t)st->st_mtime * DISCOVER_NS_PER_SEC; +#else + return ((int64_t)st->st_mtim.tv_sec * DISCOVER_NS_PER_SEC) + (int64_t)st->st_mtim.tv_nsec; +#endif +} + static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, CBMLanguage lang, - int64_t size) { + const struct stat *st) { + int64_t size = (int64_t)st->st_size; if (fl->max_files >= 0 && fl->count >= fl->max_files) { fl->limit_exceeded = true; return; @@ -605,6 +620,7 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, fi->rel_path = relative_copy; fi->language = lang; fi->size = size; + fi->mtime_ns = discover_stat_mtime_ns(st); } /* ── Recursive walk ─────────────────────────────── */ @@ -939,9 +955,10 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co const cbm_discover_opts_t *opts, const gitignore_link_t *ignore_chain, const cbm_gitignore_t *global_gi, - const cbm_gitignore_t *cbmignore, off_t size, file_list_t *out) { + const cbm_gitignore_t *cbmignore, const struct stat *st, + file_list_t *out) { const char *skip_reason = - file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, size); + file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size); if (skip_reason) { /* Deliberately not indexed (#963) — record so callers can surface it. * Unsupported-language files below are NOT recorded: "no grammar for @@ -954,7 +971,7 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co if (lang == CBM_LANG_COUNT) { return; } - fl_add(out, abs_path, rel_path, lang, size); + fl_add(out, abs_path, rel_path, lang, st); } typedef struct { @@ -1066,7 +1083,7 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram } } else if (S_ISREG(st.st_mode)) { walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi, - cbmignore, st.st_size, out); + cbmignore, &st, out); } } diff --git a/src/discover/discover.h b/src/discover/discover.h index 89ccca4f36..a9c2a5449b 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -147,6 +147,7 @@ typedef struct { char *rel_path; /* relative to repo root (heap-allocated) */ CBMLanguage language; /* detected language */ int64_t size; /* file size in bytes */ + int64_t mtime_ns; /* modification time, ns (whole seconds on Windows) */ } cbm_file_info_t; typedef struct { diff --git a/src/watcher/watcher.c b/src/watcher/watcher.c index f17d06511f..a83cab31a5 100644 --- a/src/watcher/watcher.c +++ b/src/watcher/watcher.c @@ -2,7 +2,10 @@ * watcher.c — Git-based file change watcher. * * Strategy: git status + HEAD tracking (the most reliable approach). - * For non-git projects, the watcher skips polling (no fsnotify/dirmtime yet). + * Non-git projects are NOT polled by default. With the opt-in + * cbm_watcher_set_poll_non_git(w, true) (config key `watch_non_git`, #1948) + * they are polled with a tree signature instead: the indexer's own discovery + * walk (same ignore rules), folded over (relative path, size, mtime). * * * Per-project state tracks: @@ -32,6 +35,7 @@ #include "foundation/str_util.h" #include "foundation/subprocess.h" #include "pipeline/artifact.h" /* CBM_ARTIFACT_DIR: the indexer's own output directory */ +#include "discover/discover.h" /* cbm_discover: the non-git tree signature (#1948) */ #ifdef _WIN32 #include "foundation/win_utf8.h" #define WIN32_LEAN_AND_MEAN @@ -61,7 +65,8 @@ typedef struct { cbm_subprocess_t *active_git; /* Room for Git's 64-hex SHA-256 object ID plus newline/NUL while reading. */ char last_head[CBM_SZ_128]; /* git HEAD hash (committed baseline) */ - bool is_git; /* false → skip polling */ + bool is_git; /* false → tree-polled if tree_poll, else skipped */ + bool tree_poll; /* non-git root polled by tree signature (#1948) */ bool baseline_done; /* true after first poll */ int missing_root_count; /* consecutive polls where root was missing (ENOENT/ENOTDIR) */ uint64_t first_missing_ms; /* cbm_now_ms() of the streak's first miss (0 = no streak) */ @@ -73,8 +78,14 @@ typedef struct { * are committed only after a SUCCESSFUL reindex (busy-skips and failed * runs retry); check_changes stages its observations in the pending_* * fields. 0 = clean tree. */ - uint64_t last_dirty_sig; /* committed dirty-state signature */ - uint64_t pending_dirty_sig; /* observed at check time */ + uint64_t last_dirty_sig; /* committed dirty-state signature */ + uint64_t pending_dirty_sig; /* observed at check time */ + /* Non-git tree signature (#1948), same commit discipline as the dirty + * signature. 0 = unknown: the baseline never adopts the tree as already + * indexed, so the first poll reindexes once (at-least-once, like a tree + * that is already dirty at a git baseline). Signatures are never 0. */ + uint64_t last_tree_sig; + uint64_t pending_tree_sig; char pending_head[CBM_SZ_128]; /* HEAD observed at check time */ /* Consecutive hard index failures (index_fn < 0). A hard error is * usually persistent — a poisoned coordination endpoint, an unreadable @@ -106,6 +117,9 @@ struct cbm_watcher { cbm_watcher_project_pruned_fn project_pruned; void *mutation_context; atomic_int stopped; + /* Opt-in (#1948): poll non-git roots by tree signature. Default false — + * non-git roots are not watched. Read at each project's baseline. */ + atomic_bool poll_non_git; /* Deferred-free list: freed after the next poll_once. */ project_state_t **pending_free; int pending_free_count; @@ -851,6 +865,46 @@ static watcher_git_status_t git_file_count(cbm_watcher_t *w, project_state_t *st return WATCHER_GIT_OK; } +/* ── Non-git tree signature (#1948) ─────────────────────────────── */ + +static int tree_entry_cmp(const void *a, const void *b) { + const cbm_file_info_t *fa = a; + const cbm_file_info_t *fb = b; + return strcmp(fa->rel_path, fb->rel_path); +} + +/* Signature of a non-git tree: FNV-1a over every file the INDEXER would + * discover (cbm_discover, full mode — the same skip lists, .gitignore stack + * and .cbmignore), folding each (relative path, size, mtime) in path order so + * readdir order cannot change the value. Adding, deleting, renaming or editing + * an indexable file yields a new signature; touching anything discovery skips + * does not. That includes everything cbm writes itself: the .codebase-memory + * artifact directory is a built-in skip and the cache directory is pruned by + * absolute path, so a successful reindex can never look like a new change + * (the runaway history in init_baseline and #1953). Never returns 0 in + * *sig_out, which stays reserved for "unknown". */ +static bool tree_signature(const char *root_path, uint64_t *sig_out, int *count_out) { + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int count = 0; + if (cbm_discover(root_path, &opts, &files, &count) != 0) { + return false; + } + if (count > 1) { + qsort(files, (size_t)count, sizeof(*files), tree_entry_cmp); + } + uint64_t h = SIG_FNV_OFFSET; + for (int i = 0; i < count; i++) { + h = sig_fold(h, files[i].rel_path, strlen(files[i].rel_path) + SKIP_ONE); + h = sig_fold(h, &files[i].size, sizeof(files[i].size)); + h = sig_fold(h, &files[i].mtime_ns, sizeof(files[i].mtime_ns)); + } + cbm_discover_free(files, count); + *sig_out = h ? h : 1; + *count_out = count; + return true; +} + /* ── Project state lifecycle ────────────────────────────────────── */ static void state_free(project_state_t *s); @@ -1042,9 +1096,16 @@ cbm_watcher_t *cbm_watcher_new(cbm_store_t *store, cbm_index_fn index_fn, void * cbm_mutex_init(&w->projects_lock); cbm_mutex_init(&w->coordination_lock); atomic_init(&w->stopped, 0); + atomic_init(&w->poll_non_git, false); return w; } +void cbm_watcher_set_poll_non_git(cbm_watcher_t *w, bool enabled) { + if (w) { + atomic_store_explicit(&w->poll_non_git, enabled, memory_order_release); + } +} + void cbm_watcher_free(cbm_watcher_t *w) { if (!w) { return; @@ -1238,8 +1299,10 @@ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) { * A directory is only really git-managed here if it carries its own .git, * or the ancestor repository actually tracks something inside it. A * genuine sub-package of a monorepo passes the second test; a scratch or - * gitignored folder sitting under a repo fails both and is polled as a - * plain directory instead. */ + * gitignored folder sitting under a repo fails both and is treated as a + * non-git root: NOT polled by default, or polled by its own tree signature + * when the watch_non_git opt-in is on (#1948). Either way the ancestor's + * dirty state never reaches it. */ if (s->is_git && !git_has_own_dot_git(s->root_path)) { bool tracked = false; watcher_git_status_t tracked_status = git_tracks_anything_here(w, s, &tracked); @@ -1282,6 +1345,14 @@ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) { s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count); cbm_log_info("watcher.baseline", "project", s->project_name, "strategy", "git", "files", s->file_count > 0 ? "yes" : "0"); + } else if (atomic_load_explicit(&w->poll_non_git, memory_order_acquire)) { + /* #1948 opt-in. last_tree_sig stays 0 ("unknown"): nothing records + * which tree state the DB holds (edits made while the daemon was down + * are exactly the silent-staleness case), so the first poll reindexes + * once and the signature gates every poll after it. The interval is + * sized from the file count of each signature walk. */ + s->tree_poll = true; + cbm_log_info("watcher.baseline", "project", s->project_name, "strategy", "tree"); } else { cbm_log_info("watcher.baseline", "project", s->project_name, "strategy", "none"); } @@ -1295,13 +1366,31 @@ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) { * poll_project commits them only after a SUCCESSFUL reindex so that * busy-skips and failed runs retry instead of silently losing the change * (#937). Observations are staged in the pending_* fields. */ +/* Non-git leg of check_changes (#1948). A failed walk reports "no verdict" + * (false) so the committed baseline is kept and a later poll retries. */ +static bool check_tree_changes(project_state_t *s, bool *changed_out) { + uint64_t sig = 0; + int file_count = 0; + if (!tree_signature(s->root_path, &sig, &file_count)) { + cbm_log_warn("watcher.tree_walk_failed", "project", s->project_name, "path", s->root_path); + /* Retry at the regular cadence, not on every run-loop tick. */ + s->next_poll_ns = now_ns() + ((int64_t)s->interval_ms * US_PER_MS); + return false; + } + s->pending_tree_sig = sig; + s->file_count = file_count; + s->interval_ms = cbm_watcher_poll_interval_ms(file_count); + *changed_out = sig != s->last_tree_sig; + return true; +} + static bool check_changes(cbm_watcher_t *w, project_state_t *s, bool *changed_out) { if (!changed_out) { return false; } *changed_out = false; if (!s->is_git) { - return true; + return s->tree_poll ? check_tree_changes(s, changed_out) : true; } bool changed = false; @@ -1416,6 +1505,28 @@ static void prune_missing_project(cbm_watcher_t *w, project_state_t *s) { free(root_path); } +/* After a SUCCESSFUL reindex, commit the baselines OBSERVED AT CHECK TIME — + * the state whose reindex just succeeded. A commit/edit landing during the + * reindex is deliberately not absorbed: the next poll sees it as a new delta + * (at-least-once, never lost). */ +static void commit_baselines(cbm_watcher_t *w, project_state_t *s) { + if (!s->is_git) { + /* Tree strategy (#1948): the walk already refreshed the file count. */ + s->last_tree_sig = s->pending_tree_sig; + return; + } + if (s->pending_head[0] != '\0') { + snprintf(s->last_head, sizeof(s->last_head), "%s", s->pending_head); + } + s->last_dirty_sig = s->pending_dirty_sig; + /* Refresh file count for interval */ + int file_count = 0; + if (git_file_count(w, s, &file_count) == WATCHER_GIT_OK) { + s->file_count = file_count; + s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count); + } +} + static void poll_project(const char *key, void *val, void *ud) { (void)key; poll_ctx_t *ctx = ud; @@ -1468,8 +1579,9 @@ static void poll_project(const char *key, void *val, void *ud) { return; } - /* Skip non-git projects */ - if (!s->is_git) { + /* Non-git projects are skipped unless the watch_non_git opt-in turned on + * tree polling for them at baseline (#1948). */ + if (!s->is_git && !s->tree_poll) { return; } @@ -1495,26 +1607,14 @@ static void poll_project(const char *key, void *val, void *ud) { * not admit an index callback after that ownership boundary. */ return; } - cbm_log_info("watcher.changed", "project", s->project_name, "strategy", "git"); + cbm_log_info("watcher.changed", "project", s->project_name, "strategy", + s->is_git ? "git" : "tree"); if (ctx->w->index_fn) { int rc = ctx->w->index_fn(s->project_name, s->root_path, ctx->w->user_data); if (rc == 0) { ctx->reindexed++; s->index_failures = 0; - /* Commit the baselines OBSERVED AT CHECK TIME — the state whose - * reindex just succeeded. A commit/edit landing during the - * reindex is deliberately not absorbed: the next poll sees it - * as a new delta (at-least-once, never lost). */ - if (s->pending_head[0] != '\0') { - snprintf(s->last_head, sizeof(s->last_head), "%s", s->pending_head); - } - s->last_dirty_sig = s->pending_dirty_sig; - /* Refresh file count for interval */ - int file_count = 0; - if (git_file_count(ctx->w, s, &file_count) == WATCHER_GIT_OK) { - s->file_count = file_count; - s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count); - } + commit_baselines(ctx->w, s); } else if (rc > 0) { /* Busy-skip: baseline stays uncommitted, next poll retries. */ cbm_log_info("watcher.index.retry", "project", s->project_name); diff --git a/src/watcher/watcher.h b/src/watcher/watcher.h index 2b33d78f49..6e8410138c 100644 --- a/src/watcher/watcher.h +++ b/src/watcher/watcher.h @@ -4,6 +4,7 @@ * Polls indexed projects for git changes (HEAD movement or dirty working tree) * and triggers re-indexing via a callback. Uses adaptive polling intervals * based on project size (5s base + 1s per 500 files, capped at 60s). + * Non-git roots are not polled unless cbm_watcher_set_poll_non_git() opts in. * * Depends on: foundation, store (for project metadata) */ @@ -59,6 +60,17 @@ void cbm_watcher_set_project_mutation_guard(cbm_watcher_t *w, cbm_watcher_project_mutation_end_fn end, cbm_watcher_project_pruned_fn pruned, void *context); +/* Opt in to polling NON-GIT project roots (#1948; default off — non-git roots + * are not watched). When on, a non-git root is polled on the same adaptive + * cadence by a tree signature: the indexer's own discovery walk (same skip + * lists, .gitignore and .cbmignore rules), folded over each file's (relative + * path, size, mtime). A changed signature triggers index_fn; the first poll + * after baseline reindexes once, since nothing records which tree state the + * index holds. Paths discovery skips — including cbm's own .codebase-memory + * artifact directory and cache directory — never change the signature. + * Read at each project's baseline, so set it before registering projects. */ +void cbm_watcher_set_poll_non_git(cbm_watcher_t *w, bool enabled); + /* ── Watch list management ──────────────────────────────────────── */ /* Add a project to the watch list. root_path is copied. Returns true only when diff --git a/tests/test_cli.c b/tests/test_cli.c index 267b403dde..7a2dd37c51 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -15557,6 +15557,40 @@ TEST(cli_config_watcher_enabled_default_and_persist) { PASS(); } +/* #1948: non-git roots are polled only when `watch_non_git` opts in. The key + * defaults OFF (behaviour unchanged, also for a NULL config) and is listed in + * the config-key table so `config list` / `config set` can find it. */ +TEST(cli_config_watch_non_git_default_off_and_listed_issue1948) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-cfg-ng-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + ASSERT_FALSE(cbm_config_watch_non_git(NULL)); + cbm_config_t *cfg = cbm_config_open(tmpdir); + ASSERT_NOT_NULL(cfg); + ASSERT_FALSE(cbm_config_watch_non_git(cfg)); + cbm_config_set(cfg, CBM_CONFIG_WATCH_NON_GIT, "true"); + ASSERT_TRUE(cbm_config_watch_non_git(cfg)); + cbm_config_close(cfg); + cfg = cbm_config_open(tmpdir); + ASSERT_NOT_NULL(cfg); + ASSERT_TRUE(cbm_config_watch_non_git(cfg)); + cbm_config_set(cfg, CBM_CONFIG_WATCH_NON_GIT, "false"); + ASSERT_FALSE(cbm_config_watch_non_git(cfg)); + cbm_config_close(cfg); + + bool listed = false; + for (size_t i = 0; i < cbm_cli_config_key_count_for_testing(); i++) { + const char *key = cbm_cli_config_key_at_for_testing(i); + listed = listed || (key && strcmp(key, CBM_CONFIG_WATCH_NON_GIT) == 0); + } + ASSERT_TRUE(listed); + + test_rmdir_r(tmpdir); + PASS(); +} + /* ═══════════════════════════════════════════════════════════════════ * Group H: cbm_replace_binary (update command helper) * ═══════════════════════════════════════════════════════════════════ */ @@ -17006,6 +17040,7 @@ SUITE(cli) { RUN_TEST(cli_config_delete); RUN_TEST(cli_config_persists); RUN_TEST(cli_config_watcher_enabled_default_and_persist); + RUN_TEST(cli_config_watch_non_git_default_off_and_listed_issue1948); /* Replace binary (update command helper — group H) */ #ifndef _WIN32 diff --git a/tests/test_watcher.c b/tests/test_watcher.c index 49f4d29900..91b4568c95 100644 --- a/tests/test_watcher.c +++ b/tests/test_watcher.c @@ -2303,6 +2303,207 @@ TEST(watcher_non_git_skips) { PASS(); } +/* ══════════════════════════════════════════════════════════════════ + * NON-GIT TREE POLLING (#1948, opt-in) + * ══════════════════════════════════════════════════════════════════ */ + +/* Bypass the adaptive interval and run one poll cycle; returns the number of + * reindexes that cycle performed. */ +static int nongit_poll(cbm_watcher_t *w, const char *project) { + cbm_watcher_touch(w, project); + return cbm_watcher_poll_once(w); +} + +/* Control: with the opt-in OFF (the default) a non-git root is never + * refreshed, however its indexable files change. watcher_non_git_skips covers + * this with .txt files discovery ignores anyway; this uses a real source file + * so the ON tests below differ from it ONLY in the opt-in. */ +TEST(watcher_non_git_opt_in_off_never_polls_issue1948) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_ng_off_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + th_write_file(TH_PATH(tmpdir, "app.py"), "def alpha():\n return 1\n"); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL); + cbm_watcher_watch(w, "ng-off", tmpdir); + index_call_count = 0; + + cbm_watcher_poll_once(w); /* baseline */ + ASSERT_EQ(nongit_poll(w, "ng-off"), 0); + th_append_file(TH_PATH(tmpdir, "app.py"), "\ndef gamma():\n return 3\n"); + ASSERT_EQ(nongit_poll(w, "ng-off"), 0); + th_write_file(TH_PATH(tmpdir, "new.py"), "def beta():\n return 2\n"); + ASSERT_EQ(nongit_poll(w, "ng-off"), 0); + ASSERT_EQ(remove(TH_PATH(tmpdir, "new.py")), 0); + ASSERT_EQ(nongit_poll(w, "ng-off"), 0); + ASSERT_EQ(index_call_count, 0); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + +/* #1948: a project indexed from a plain directory stayed frozen at its first + * index forever. With watch_non_git on, the watcher polls it by tree + * signature: the first poll after baseline catches up once (nothing records + * which tree state the index holds), then every edit, addition and deletion + * of an indexable file triggers exactly one reindex, and an untouched tree + * triggers none. */ +TEST(watcher_non_git_opt_in_detects_edit_add_delete_issue1948) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_ng_on_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + th_write_file(TH_PATH(tmpdir, "app.py"), "def alpha():\n return 1\n"); + th_write_file(TH_PATH(tmpdir, "pkg/util.py"), "def helper():\n return 0\n"); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL); + cbm_watcher_set_poll_non_git(w, true); + cbm_watcher_watch(w, "ng-on", tmpdir); + index_call_count = 0; + + cbm_watcher_poll_once(w); /* baseline: classification only */ + ASSERT_EQ(index_call_count, 0); + ASSERT_EQ(nongit_poll(w, "ng-on"), 1); /* at-least-once catch-up */ + ASSERT_EQ(nongit_poll(w, "ng-on"), 0); /* untouched tree: stable */ + ASSERT_EQ(nongit_poll(w, "ng-on"), 0); + + /* Edit (the reporter's repro: append a function). */ + th_append_file(TH_PATH(tmpdir, "app.py"), "\ndef gamma_uniquename():\n return 2\n"); + /* The adaptive interval still gates polling: no touch, no check. */ + ASSERT_EQ(cbm_watcher_poll_once(w), 0); + ASSERT_EQ(nongit_poll(w, "ng-on"), 1); + ASSERT_EQ(nongit_poll(w, "ng-on"), 0); + + /* Add, in a subdirectory. */ + th_write_file(TH_PATH(tmpdir, "pkg/extra.py"), "def extra():\n return 4\n"); + ASSERT_EQ(nongit_poll(w, "ng-on"), 1); + ASSERT_EQ(nongit_poll(w, "ng-on"), 0); + + /* Delete. */ + ASSERT_EQ(remove(TH_PATH(tmpdir, "pkg/extra.py")), 0); + ASSERT_EQ(nongit_poll(w, "ng-on"), 1); + ASSERT_EQ(nongit_poll(w, "ng-on"), 0); + + ASSERT_EQ(index_call_count, 4); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + +/* Simulates the daemon's index worker writing cbm's own output under the root + * on every successful reindex (the artifact re-export of #1953). */ +static int own_output_index_callback(const char *name, const char *path, void *ud) { + (void)name; + (void)ud; + index_call_count++; + th_append_file(TH_PATH(path, CBM_ARTIFACT_DIR "/graph.py"), "# rewritten by index\n"); + return 0; +} + +/* The signature must only see what the indexer would index. Paths discovery + * skips — cbm's own .codebase-memory output, built-in skip dirs, .cbmignore'd + * paths, non-source files — never trigger, and in particular a reindex that + * rewrites cbm's own output must not look like a new change on the next poll + * (the runaway-reindex history of #841/#937/#1953). */ +TEST(watcher_non_git_opt_in_ignores_skipped_paths_issue1948) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_ng_ign_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + th_write_file(TH_PATH(tmpdir, "app.py"), "def alpha():\n return 1\n"); + th_write_file(TH_PATH(tmpdir, ".cbmignore"), "ignored/\n"); + th_write_file(TH_PATH(tmpdir, CBM_ARTIFACT_DIR "/graph.py"), "# artifact\n"); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, own_output_index_callback, NULL); + cbm_watcher_set_poll_non_git(w, true); + cbm_watcher_watch(w, "ng-ign", tmpdir); + index_call_count = 0; + + cbm_watcher_poll_once(w); /* baseline */ + ASSERT_EQ(nongit_poll(w, "ng-ign"), 1); /* catch-up; rewrites own output */ + /* The reindex's own write must not retrigger. */ + ASSERT_EQ(nongit_poll(w, "ng-ign"), 0); + ASSERT_EQ(nongit_poll(w, "ng-ign"), 0); + + th_write_file(TH_PATH(tmpdir, "ignored/skip.py"), "def skipped():\n return 0\n"); + th_write_file(TH_PATH(tmpdir, "node_modules/dep/index.js"), "module.exports = 1;\n"); + th_write_file(TH_PATH(tmpdir, "logo.png"), "not really a png\n"); + th_append_file(TH_PATH(tmpdir, CBM_ARTIFACT_DIR "/graph.py"), "# touched\n"); + ASSERT_EQ(nongit_poll(w, "ng-ign"), 0); + ASSERT_EQ(index_call_count, 1); + + /* Positive control in the same tree: an indexable edit still triggers. */ + th_append_file(TH_PATH(tmpdir, "app.py"), "\ndef beta():\n return 2\n"); + ASSERT_EQ(nongit_poll(w, "ng-ign"), 1); + ASSERT_EQ(nongit_poll(w, "ng-ign"), 0); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + +/* The opt-in changes nothing for git roots: a clean repository does not get + * the tree strategy's catch-up reindex and still reacts to commits only. A + * plain folder under an unrelated dirty repository is tree-polled on its OWN + * files and never inherits the ancestor's dirty state. */ +TEST(watcher_non_git_opt_in_leaves_git_roots_unchanged_issue1948) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_ng_git_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + if (wt_git(tmpdir, "init -q") != 0) { + th_rmtree(tmpdir); + FAIL("git init failed"); + } + th_write_file(TH_PATH(tmpdir, "app.py"), "def alpha():\n return 1\n"); + wt_git(tmpdir, "add app.py"); + wt_git(tmpdir, "commit -q -m init"); + + cbm_store_t *store = cbm_store_open_memory(); + cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL); + cbm_watcher_set_poll_non_git(w, true); + cbm_watcher_watch(w, "ng-git", tmpdir); + index_call_count = 0; + + cbm_watcher_poll_once(w); /* baseline */ + ASSERT_EQ(nongit_poll(w, "ng-git"), 0); + ASSERT_EQ(nongit_poll(w, "ng-git"), 0); + th_append_file(TH_PATH(tmpdir, "app.py"), "\ndef beta():\n return 2\n"); + wt_git(tmpdir, "commit -q -am second"); + ASSERT_EQ(nongit_poll(w, "ng-git"), 1); + ASSERT_EQ(nongit_poll(w, "ng-git"), 0); + + /* Now a scratch folder under the (permanently dirty) repository. */ + th_write_file(TH_PATH(tmpdir, "dirty.txt"), "uncommitted\n"); + char nested[400]; + snprintf(nested, sizeof(nested), "%s/scratch", tmpdir); + th_write_file(TH_PATH(nested, "notes.py"), "def note():\n return 0\n"); + cbm_watcher_unwatch(w, "ng-git"); + cbm_watcher_watch(w, "ng-nested", nested); + index_call_count = 0; + cbm_watcher_poll_once(w); /* baseline: nested non-git */ + ASSERT_EQ(nongit_poll(w, "ng-nested"), 1); /* tree catch-up */ + for (int i = 0; i < 3; i++) { + ASSERT_EQ(nongit_poll(w, "ng-nested"), 0); /* ancestor dirt ignored */ + } + th_append_file(TH_PATH(nested, "notes.py"), "\ndef more():\n return 1\n"); + ASSERT_EQ(nongit_poll(w, "ng-nested"), 1); + + cbm_watcher_free(w); + cbm_store_close(store); + th_rmtree(tmpdir); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * ADAPTIVE INTERVAL BEHAVIOR * ══════════════════════════════════════════════════════════════════ */ @@ -3711,6 +3912,10 @@ SUITE(watcher) { /* Non-git project */ RUN_TEST(watcher_non_git_skips); + RUN_TEST(watcher_non_git_opt_in_off_never_polls_issue1948); + RUN_TEST(watcher_non_git_opt_in_detects_edit_add_delete_issue1948); + RUN_TEST(watcher_non_git_opt_in_ignores_skipped_paths_issue1948); + RUN_TEST(watcher_non_git_opt_in_leaves_git_roots_unchanged_issue1948); /* Adaptive interval behavior */ RUN_TEST(watcher_interval_blocks_repoll);