diff --git a/assets/guster/icon.png b/assets/guster/icon.png new file mode 100644 index 00000000..cd932818 Binary files /dev/null and b/assets/guster/icon.png differ diff --git a/external_plugins/guster/.codex-plugin/plugin.json b/external_plugins/guster/.codex-plugin/plugin.json new file mode 100644 index 00000000..357e82b3 --- /dev/null +++ b/external_plugins/guster/.codex-plugin/plugin.json @@ -0,0 +1,34 @@ +{ + "name": "guster", + "version": "0.9.5", + "description": "Discover, audit, plan and evolve agent context across projects, skills, tools, knowledge and credentials; optionally scaffold a lightweight Guster Hub for durable knowledge and project state.", + "author": { + "name": "Lumos-789", + "url": "https://github.com/Lumos-789" + }, + "homepage": "https://github.com/Lumos-789/guster", + "repository": "https://github.com/Lumos-789/guster", + "license": "MIT", + "keywords": [ + "agent", + "context", + "governance", + "skills", + "mcp", + "codex", + "zcode", + "dsh" + ], + "skills": "./skills/", + "interface": { + "displayName": "Guster", + "shortDescription": "Audit and evolve agent context", + "longDescription": "Guster discovers agent projects, instruction files, skills, tools, knowledge, state, evidence and credential references. It diagnoses drift, computes a context health score, produces reviewable migration plans, and can scaffold a lightweight hub for durable knowledge and project state.", + "developerName": "Lumos-789", + "category": "Developer Tools", + "capabilities": [ + "Read", + "Write with confirmation" + ] + } +} diff --git a/external_plugins/guster/.zcode-plugin/plugin.json b/external_plugins/guster/.zcode-plugin/plugin.json new file mode 100644 index 00000000..95ada09e --- /dev/null +++ b/external_plugins/guster/.zcode-plugin/plugin.json @@ -0,0 +1,27 @@ +{ + "name": "guster", + "version": "0.9.5", + "description": "Discover, audit, plan and evolve agent context across projects, skills, tools, knowledge and credentials; optionally scaffold a lightweight Guster Hub for durable knowledge and project state.", + "description_i18n": { + "en": "Discover, audit, plan and evolve agent context across projects, skills, tools, knowledge and credentials; optionally scaffold a lightweight Guster Hub for durable knowledge and project state.", + "zh-CN": "发现、诊断、规划和演进全机 agent 上下文:项目、指令、Skill、工具、知识、状态与凭据引用;可初始化一个轻量 Guster Hub,承载长期知识与项目状态。" + }, + "author": { + "name": "Lumos-789", + "url": "https://github.com/Lumos-789" + }, + "homepage": "https://github.com/Lumos-789/guster", + "repository": "https://github.com/Lumos-789/guster", + "license": "MIT", + "keywords": [ + "agent", + "context", + "governance", + "skills", + "mcp", + "codex", + "zcode", + "dsh" + ], + "skills": "skills" +} diff --git a/external_plugins/guster/CHANGELOG.md b/external_plugins/guster/CHANGELOG.md new file mode 100644 index 00000000..a245065a --- /dev/null +++ b/external_plugins/guster/CHANGELOG.md @@ -0,0 +1,521 @@ +# Changelog + +## 0.9.5 — 2026-11-19 + +- **apply/plan safety hardening**: plans are validated for schema + equivalence before execution, path boundaries keep every operation inside + its intended root (paths are resolved first, so a symlink cannot smuggle + a write past the boundary into the allowed roots), the legacy `pointer` + operation is removed from the executor, backups can no longer be + overwritten by a later plan, and a failed `mkdir` rolls back with the + rest of the plan. +- **Drift baseline fixed**: the machine↔disk drift check compares against + the registry captured *before* this scan overwrites it (the old code read + the file back after its own write, so write-path runs never saw drift); + a corrupt baseline is a visible finding instead of a silent skip. +- **Crash fix**: `sk-`-shaped tokens no longer crash the scan. +- **Index shape recognition**: llms.txt-style index forms are recognized for + index coverage. +- **Removed command surface** (breaking): the `upgrade` subcommand, the + `plan init` subcommand, the explicit `--dry-run` flag and the `--home` + flag are gone. DRY-run-by-default and `--yes` semantics are unchanged; + `guster init --project ` replaces `plan init` for scaffolding. +- **`hub init` preflight exits 4**: a plan refused at preflight (non-empty + target, bad type) now returns exit code 4, aligned with `apply`/`init`. +- **CLI polish**: `score --fail-under` failures emit a JSON error envelope + under `--json` (exit 2 unchanged); `init` refuses an existing broken + `config.json` instead of returning a success-shaped no-op; + credential-registry warnings no longer echo the broken row's text. +- **Documentation alignment and skill simplification**: the docs describe + the current architecture only (hub schema 2 declarative paths, five FRS + state prefixes, scaffold-shaped placement tree, CTX-09 date-key list, + credential-registry precondition); legacy directory names are marked as + compat-scan-only; the `guster` skill is compressed to 150 lines and the + reformat playbook to 75 without dropping commands, workflow, gates, exit + codes or rule IDs. + +## 0.9.4 — 2026-11-19 + +The hub contract catches up with the reference hub's 2026-11-19 restructure, +and the format spec gains its first structured-knowledge contract. The plugin +still ships no organization policy — it discovers what exists and validates +what files declare. + +- **Root README.md is optional** (relaxation): required root files are now + `AGENTS.md` + `hub.yaml`; an existing root README.md stays untouched (no + check depends on it), and `hub init` still scaffolds one. +1 test. +- **Hub scan alias `output/`** joins `Human Output/` (kept as the legacy + alias) for evidence/deliverables directories. +- **Knowledge subzones are discovered, not enumerated**: the scanner no + longer hardcodes `general/` + `projects/` + `workgroups/` under the + knowledge root. Every first-level directory (`machines/`, or anything) is + scanned as knowledge and index-checked against `knowledge/README.md`. + Strict validation now index-checks `.yaml`/`.yml` notes, not only `.md`, + and only requires the knowledge root plus its README — no specific subzone + names. +- **CTX-10 becomes register/v1** (breaking, spec_version 2): the hub's human + project table moves from `projects/_registry.md` (markdown table) to + `register/projects.yaml` — a `schema: register/v1` + `kind: projects` + + `updated:` header with one row per project (`id`, `status`, `as_of`, + `name`, `spec.path`). `hub init` scaffolds it; validation requires it; the + three-way drift diff (machine table ↔ register ↔ disk) and the entry + counter follow. New `registry-format` finding: wrong `schema`/`kind`, duplicate ids, + missing `spec.path` or non-date `as_of`/`status` (a missing `updated` + date is caught by strict `hub validate`, not by the finding). +- **register/v1 shape errors fail loud**: a register whose body carries + column-0 `id:`/`name:` mapping keys but parses zero `- id:` rows (the flat + single-project map) now raises `registry-format` instead of silently + validating nothing. `hub validate`'s `updated:` check is strict ISO at the + same time — `2026/1/5` no longer passes, only `2026-01-05`. +- **register/v1 vocabulary clarified**: `status` adds `on-hold`; `as_of` is + optional (checked when present); `name` is optional and never validated; + `spec.path: ./` is the legal placeholder for a row that owns no directory + of its own; unknown row keys and `spec` subkeys are ignored and never + validated — the vocabulary only grows, semantics change only on a + spec_version bump. Another register/ YAML declaring register/v1 with a + foreign `kind:` gets an info finding: accepted as register data, never + validated. +- **CTX-11: the knowledge/v1 contract** (new, opt-in): a YAML knowledge file + that declares `schema: knowledge/v1` gets strict checks — required metadata + (`id` = file stem, `type` required with vocabulary, optional in-vocabulary + `status`, non-empty `topic`, + YYYY-MM-DD `as_of`/`updated` with `updated` ≥ `as_of`), `see_also` bare-id + cross-references resolved against the knowledge root, one-id-one-file, and + a 90-day freshness SLA for active `state`/`index` files. Plain notes are + never touched. Findings: `knowledge-format`, `see-also-broken`, + `duplicate-knowledge-id`, `knowledge-stale`. +- **knowledge/v1 typos stay visible**: a knowledge file that declares a + `knowledge/*` or `register/*` schema other than knowledge/v1 reports an + info finding instead of silently opting out of every check, and a + `see-also-broken` reference whose file exists without the v1 header says + so in the finding. CTX-02 accepts an equivalent two-column markdown index + (backticked path + topic). The hub scaffold gains an all-commented + knowledge/v1 skeleton (`knowledge/general/knowledge-v1-skeleton.txt`) and + the register template a commented example row. +- **Dependency-free metadata parsing**: `parse_yaml_head` / + `parse_register_rows` read only the column-0 metadata face of YAML files + (scalars, one inline list, nested `spec.path`); the prose body is never + parsed, so the plugin still runs on a stock Python install. +- All CTX-11/registry-format findings stay outside CHI (the registry-drift + discipline), so health scores remain comparable across the upgrade. +- **`plan move` stops writing pointers** (breaking for plan outputs): the + plan is mkdir + move with a backup; the automatic `.moved.md` + transition pointer at the old location is gone. It outlived its + referencers, re-triggered `misplaced-artifact`, and read as reversible + after the source was gone — update the referencers in the same pass + instead. +- **`guster link` removed**: the standalone pointer listing duplicated what + `doctor`'s `broken-pointer` finding already reports. The command, its + exit-code row and the report hint are gone. +- **Zone and hub `profile` retired**: `zones add --profile` is gone and + `config.json` zone entries are `{path}` only (a legacy `profile` key is + tolerated and ignored); `hub.yaml` loses `profile` (a leftover key is + ignored, and unknown top-level manifest keys no longer fail the JSON + Schema — local extensions validate too). +- **GCR leaves every output**: after 0.9.3 made it reported-only, the + governance metric is deleted — no `metrics.gcr`, no report footnote. The + `hub.yaml` `governance_docs` declaration and the `governance-overlap` + finding (more than 4 framework docs) stay. +- **Score output says what it measured**: a CHI that cannot be computed + prints how many dimensions were (`n/a / 100 (5/7 dims scored)`), trend + lines render `n/a` instead of `None`, and the output points at + `skills/guster/reference/metrics.md` for definitions and targets. The + machine-scope summary labels its skills line "(user-level roots only)", + `guster init`'s dry run lists the files it would create, and finding + paths resolve to one spelling (`/private/tmp` vs `/tmp` no longer mix). +- **Templates and both SKILL playbooks rewritten for the register (the + new-project onboarding step now drafts a register/v1 row); `project cards` + vocabulary retired across docs. +- **Migration from 0.9.3**: create `register/projects.yaml` (header: + `schema: register/v1`, `kind: projects`, `updated:` date; then one row per + project), delete the old `projects/_registry.md`, and re-run + `guster hub validate`. Until then `hub attach`/`hub status` refuse the hub + and doctor reports `registry-drift: Hub has no register/projects.yaml`. + 192 py + 2 node tests green (+26 over 0.9.3, covering dynamic subzones, yaml + index checks, register drift/format/shape and vocabulary, knowledge/v1 + typo visibility, the pointer/link/profile/GCR retirements and the score + output). + +## 0.9.3 — 2026-11-18 + +The honest-third-party release: no measurement ships with a built-in list of +the author's own documents, no version drift can hide between the CLI and the +manifests, and no flag pretends to do something it does not. + +- **GCR is configured, not hardcoded** (breaking for the metric): the + governance complexity ratio counted five fixed framework doc names that only + exist in the original author's hub, so every third-party install measured + `0` and still looked scored. The list is now the hub's own `governance_docs` + key in `hub.yaml` (added to `schemas/hub.schema.json`); a hub that declares + no list reports GCR as `n/a` through the same empty-evidence rule as every + other dimension. `scores.gcr` no longer exists in any output — GCR was + documented as "reported, never scored", and now the JSON agrees. The + `governance-overlap` finding (more than 4 framework docs) still fires when + the list is declared. +- **`upgrade` compares the CLI too**: the command only diffed the three + manifests against each other, so a CLI that drifted from all of them stayed + green. The CLI `__version__` is now part of the comparison set and any + disagreement exits 2; `--json` reports a `versions` object (`cli`, `codex`, + `zcode`, `package`) instead of the old `manifests` map. +- **`export --format json` writes only the JSON report** (behavior change): + it used to write the Markdown report unconditionally and add a `.json` + sibling; with `--format json` the Markdown file is no longer produced. + `--format md` (the default) is unchanged. +- **Removed the dead `score --write` flag**: `cmd_score` never read it — + writing metrics is the default and `--no-write` is the real opt-out. +- **`.release-gate.json` left public tracking**: the local release-gate + allowlist now lives in the gitignored `.agents/` directory; the gate tool + reads it from either location. +- All five version sources (CLI, `pyproject.toml`, both plugin manifests, + `package.json`) realigned at 0.9.3; `test_manifest_versions_match` is green + again. + +Notable changes. Earlier history is in `git log`. + +## 0.9.2 — 2026-11-18 + +- New icon: a 1-bit pixel directory-tree base patrolled by four glowing + sparks — the audited-order home with agents shuttling between levels. + Part of the shared pixel family across the six plugins; no code changes. + +Notable changes. Earlier history is in `git log`. + +## 0.9.1 — 2026-11-14 + +The format-spec release: the context format contract is now externally +deliverable — schema-checked in the editor, sourced to open conventions, and +stated in attributable vocabulary. + +- **JSON Schema contracts**: `schemas/hub.schema.json` and + `schemas/plan.schema.json` (draft 2020-12, derived from the code that + reads and writes each format) pin the `hub.yaml` manifest and the plan + JSON. The scaffolded `hub.yaml` ships a yaml-language-server modeline on + its first line, so the VS Code YAML extension validates it with zero + configuration, and an agent hand-writing a batch plan has the exact field + contract to check against before the `apply` dry-run. `tests/test_schemas.py` + keeps both contracts honest with a stdlib-only mini validator. +- **CTX-09 optional keys**: a note header may carry `name` and `description` + (the one-fact-per-file shape inspired by the Claude Code memory tool) and + `superseded_by`, the path or stem of the note that replaced this one — pure + metadata, never part of the pointer scan, no new finding. The naked-pointer + scan now skips front-matter, so `superseded_by` values are not misread as + naked pointers. `spec_version` stays 1. +- **Open conventions, sourced**: the spec's conventions section now states + AGENTS.md as an open format (nearest-wins nested discovery, the same + narrow-never-contradict discipline as CTX-01, plus the + `ln -s AGENTS.md CLAUDE.md` recipe) and YAML front-matter as the de-facto + standard Jekyll, Hugo and Gray Matter share. +- **README positioning**: both READMEs gain a schema-contracts chapter, the + conventions sourcing, a compact prior-art section (Cline Memory Bank, + Claude Code memory tool, Letta, Mem0, Zep — inspired-by wording + throughout), and the Anthropic vocabulary (structured note-taking, + 'just in time' context, context rot, compaction); the project tree lists + `schemas/`. English and Chinese stay semantically aligned. + +## 0.9.0 — 2026-11-14 + +The "installed and fully usable" release: the new-user journey is closed end +to end — first run, configuration, onboarding a project, and the patrol loop. + +- **First run without a config**: guster probes the conventional roots + (`~/Projects`, `~/code`, `~/dev`, `~/src`), scans hits under an + "Unconfigured" banner telling you to run `guster init`, and fails loud + (exit 1, pointing at `init --root`) when none exists. The silent fallback + to scanning the whole home directory is gone. `init` now writes the + detected roots instead of hardcoding `~/`; a bare `init` with no candidates + requires `--root`. +- **`guster check`**: `scan` → `doctor` → `score --trend` in one pass, one + report; the exit code carries the doctor semantics (2 high, 3 critical). + Human-driven patrol only — no scheduled runs. +- **`guster zones list|add|remove`**: manage the machine zones in + `config.json` with the same fail-loud validation as the rest of the config. +- **Config validation is uniform**: `roots` must be a list of existing + absolute paths (a bare string is a type error, never a list of characters), + `roots: []` is rejected, as are relative paths, missing zone directories + and a `hub` that is not a directory. +- **Default hub de-privatized**: nothing defaults to a private layout anymore; + without a config there is no hub and the banner says so. `hub status` + prints "no hub attached" and exits 0; `--hub` pointing at a non-hub + directory is a `hub-manifest-missing` finding instead of a silent clean + report. Hub wording: "missing" → "absent". +- **Registry drift (CTX-10)**: the machine table (`registry.json`), the hub's + human table (`projects/_registry.md`, read through its path column) and the + git roots on disk are three-way diffed; dead rows, unscanned rows and + unregistered projects are findings, never a CHI dimension. `host:/path` + remote rows are counted apart. +- **Data safety**: state writes are atomic (tmp + `os.replace`) and serialized + by an flock on `/.lock`; `--no-write` genuinely suppresses the + state file of `scan`, `doctor`, `score` and `check`; a corrupt registry + warns and rescans instead of silently self-healing. +- **Freshness and dates**: `verified_at` is the canonical date key; English + prose keys (`Last updated:`/`updated:`) join the Chinese family; without a + declared date the ladder falls back to the last git commit and then to file + mtime (labeled in the raw column). Index matching requires the backtick + form, so `a.md` no longer hits `data.md`. +- **Credentials**: rows with too few columns raise parse warnings instead of + vanishing; `guster credentials` prints bilingual scope labels; the fixed + 9-column order is documented with a copyable sample (README and spec). +- **Plans and recovery**: a bad plan reports the expected schema shape; + `.moved.md` pointers carry a `verified_at` date; only the newest 10 plan + backup directories are kept; a failed apply rolls back automatically, and + if the rollback itself fails the error names the backup directory for + manual recovery (there is no separate rollback command — the README now + says exactly this). +- **Skill**: new "New project onboarding" playbook (inventory roots/zones → + propose placement → user confirms → create + `init --project` → draft the + CTX-10 registry row → `scan`/`doctor` close the loop; git operations follow + the user's own git conventions). Waivers now state explicitly that a waiver + greens the gate, not the score. "Company repository" wording is replaced by + the mechanism: tracked files you do not own or have no authority over. + Private skill references are gone. +- **Docs**: README/README_CN gain a roots-and-zones configuration chapter + (including `scan_max_depth`), the `check` command, a full exit-code table + (0–4, shared with the skill), and the four fixed inaccuracies (backups + manual recovery, five state files, `export`'s default output path, the + credential sample row). Hub templates emit `verified_at` dates. The main + `guster --help` carries a one-line epilog about state writes and exit codes. + +## 0.8.0 — 2026-11-14 + +- **Declarative hub contract (schema 2)**: `hub.yaml` sections are now + declared, not prescribed. A schema-2 hub declares what it has — only + `knowledge` and `projects` are mandatory — so a hub without a `tools` + section validates instead of failing. Declared sections may point outside + the hub with an absolute or `~/` path (validated for existence, not + containment). Schema-1 manifests behave exactly as before. +- **Machine zones with governance profiles** (`config.json` `zones`): + `{path, profile}` entries describe tool areas that are not repositories. + The `light` profile scans AGENTS.md/README.md entry docs (pointer health, + freshness) and `.env` secrets without demanding `.agents/` slots — a + non-git tool layer is governed without pretending each tool is a project. + Registry and `status` carry the zone summary; bad zone config fails loud. +- **Tool entry docs join pointer checks**: artifacts of kind `tool` (hub + tools sections and machine zones) are now pointer-scanned like other entry + documents; previously only their AGENTS.md was seen. +- Pointer scan cap raised from 200 to 500 files per run. +- `status` warns when the cached registry was written by a different CLI + version, and prints declared zones. +- Default workspace roots no longer include the private `~/ZCodeProject`; + candidates are `~/IdeaProjects` and `~/Projects` (config overrides). +- `hub init` template now emits a schema-2 manifest. + +## 0.7.4 — 2026-11-12 + +- **CTX-09 opt-in now wins over malformedness** (`parse_note_header`): a + header that never declares `type:` can no longer report + `note-header-malformed`. Previously the parser bailed out as malformed + before reaching the `type:` gate, so legacy prose notes that merely open + with `---` (unclosed fence, prose inside the fence, a `----` opener, plain + non key:value lines) were flagged against the documented contract. The + opening fence is now strictly `---` (the same rule `parse_frontmatter` + applies), and only headers that opted in can be malformed. +- Note headers parse literal/folded block scalars (`updated: |` with an + indented block, `>` folded), aligning `parse_note_header` with + `parse_frontmatter` — a legal block-scalar date was previously reported + as a malformed line. +- **Apply rollback leaves no lying artifacts**: a `move` with a `.moved.md` + pointer now registers the pointer write in the undo list — a pre-existing + pointer is backed up and restored, a freshly written one is deleted — and + fresh `write` targets register an undo entry too. A failed batch now + restores sources, removes destinations, pointers and fresh writes, matching + the documented "backs up every touched path and rolls back on failure". + +## 0.7.3 — 2026-11-11 + +- **Legacy context reformat playbook** (`skills/guster/reference/reformat.md`): + step-by-step guidance for bringing a scattered, pre-spec context site under + the format contract — the piece the 0.7.0 spec lacked. Four passes: + inventory and classify (doctor findings as the worklist, one destination + class per artifact, duplicates pick a canonical copy), placement first + (`plan init` for the slots, `plan move` or one hand-written batch plan — + apply backs up every path and rolls back), content second (the CTX symptom + → transform table: index lines, naked paths to links, date lines, duplicate + to pointer card), then verify (`doctor` zero new findings, `score --trend` + before/after). Principles are explicit: meaning-preserving (no invented + facts), classify-before-touch (garbage is proposed for deletion, not + formatted), placement before content (paths final before pointers are + written). Deletions and distillations ride the classification table for + user approval; company-tracked files are never migration targets. + Field-tested on two real projects (a migrated repo with a stale cross-repo + link and a stray root state doc): the lessons — search referencers by stem + before moving, override shape-based slot suggestions by classification, + delete `.moved.md` transition pointers after referencers update — are + folded back into the playbook. +- SKILL.md wiring: trigger phrases for scattered/legacy context, a + "Reformatting legacy context" section, and the reference entry. + +## 0.7.2 — 2026-11-09 + +- **CTX-09 structured note headers** (opt-in): knowledge notes may carry YAML + front-matter following the Obsidian properties convention — the interchange + format Foam/Quartz/Logseq also accept, so guster defines no schema of its + own. A header that declares `type:` is machine-checked (`note-header-malformed`): + it must parse and carry a machine-readable date (`updated`/`date`/`lastmod`/ + `created`, YYYY-MM-DD), and the note's body then joins the CTX-03 pointer + scan so a pointer card's truth links are verified. Truth sources live in the + body as ordinary links, never as invented header keys. Legacy prose notes + without `type:` are unaffected. + +## 0.7.1 — 2026-11-09 + +- Fixed the placement tree in `reference/framework.md`: the three arrows for + status/proof/durable-understanding questions pointed at the wrong plane + numbers (4/5/6 instead of 5/6/4), contradicting the plane definitions in the + same file. +- Documentation now teaches bare commands instead of the no-op flags + `score --write` and `init --home`: the flags are still accepted for existing + scripts, but writing metrics and initializing the state home are the defaults + with or without them (README, README_CN, AGENTS.md, SKILL.md, doctor's + next-action hint. + +## 0.7.0 — 2026-11-09 + +- **Context format spec v1** (`skills/guster/reference/spec.md`): the format + contract behind the metrics, written down and versioned — eight rules + (`CTX-01..08`, markdownlint-style stable IDs) covering instruction files, + indexes, pointers, dates, slots, single-source, skill metadata and + credentials, each naming the finding that enforces it. Follows open + conventions: `AGENTS.md` (agents.md) for instruction files, llms.txt-shaped + link indexes, stable rule IDs. The skill cites it when writing or moving + context assets; project and hub scaffolds now state the pointer rule. +- **New finding `naked-pointer` (medium)**: file paths left outside a markdown + link or code span in an instruction or index document are reported — until + now such paths were invisible to the pointer check, so a broken link written + in prose never fired. Directory references in prose are deliberately not + flagged. Advisory only: no CHI dimension changes, scores stay comparable. +- `metrics.raw.naked_pointers` counts the new finding; the guster skill + workflow adds a post-apply `doctor` pass and triggers on 上下文格式/格式规范. + +## 0.6.1 — 2026-11-05 + +- **Workspace and group roots are no longer scored as projects.** A directory + that holds other scanned projects (a per-user workspace root, a per-group + checkout root) carries group-level `AGENTS.md` files but is not a project: counting it + inflated `Projects:` and turned one container's partial `.agents/` into a + permanent placement failure. `registry.json` marks `container`, PCR/ICR skip + them, and the report reads `Projects: 62 (workspace containers: 2)`. +- **PCR explains itself.** The raw column breaks a shortfall into extra + top-level entries / missing slots / missing index. "9/41 · 25 extra entries" + is actionable; "21.4%" alone reads like a crisis. +- Two boundaries are now written down instead of being implicit, so they stop + coming back as "bugs": PCR's denominator is adopters only (a repo whose sole + context asset is one well-placed `AGENTS.md` has nothing to place — demanding + six empty slot directories there would be structure theater, and + `/skills/` is product source in a skill repository, not unplaced + context), and SIR does not scan arbitrary file content for token shapes (the + release gate owns that; SIR only claims host configuration it actually read). + +## 0.6.0 — 2026-11-05 + +Measurement honesty pass. Every change below removes a case where the score +claimed knowledge it did not have. + +- **An empty denominator is now `n/a`, not 1.0.** A dimension with no evidence + is dropped and the remaining weights are renormalized; a scope with no + evidence reports `CHI: n/a`. This reverses the 0.5.5 behavior ("empty + denominators still print `0/0` and score 1.0"), which let a scope with fewer + artifacts outrank a measured one. `project`/`group` CHI therefore drops when + their denominators are empty, and scores are no longer comparable across + scopes with different coverage. RBR follows the same rule per component: a + missing entry file is not a perfect budget. +- **Git probe failures are loud instead of clean.** `git ls-files` and + `git log` returning an error used to look like "no tracked files" and "no + commit dates": credentials read as isolated and state documents read as + undated. Both now return `None`, surface as a new `git-unavailable` finding, + and SIR counts the affected credentials as unverified. `registry.json` + records `git_state` per project. +- **SIR stops punishing registered credentials.** The numerator is now + unregistered-outside + tracked + unverified + inline, so a file declared in + `~/.agents/credentials/registry.md` no longer costs points; previously 30 of + 31 in-place company `.env` files were counted as failures while the matching + finding was suppressed. The `secret-outside-root` finding and SIR finally + share one definition. +- **Dates are read where a freshness claim actually lives.** A filename date + must open the name (`2026-11-05-x`, `20261105-x`); `ticket-20261105-notes.md` + no longer counts as today's work. `最后更新` is matched in the first 40 lines + only, so prose mentioning another file cannot date this one. +- **PHR sees directory pointers** (`knowledge/general/`) and resolves paths + against the enclosing repository root, so `.claude/CLAUDE.md` and + `.agents/README.md` stop reporting root-relative links as broken. The report + states when the 200-file pointer cap narrowed the scan. +- **RBR probes the host that is actually installed.** It used to read + `~/.zcode/AGENTS.md` only; a DSH-only machine measured 0 bytes and scored + full marks. It now sums the entry files that exist, deduplicated by real path. +- **Wider credential detection, same read-only boundary.** `.env` discovery + goes two levels deep instead of one, and inline detection covers `api_key`, + `secret_key`, `auth_token` and `sk-…` shapes besides `Bearer`. Values stay + redacted and are never printed. +- **ICR requires a non-empty index**: an empty `README.md` no longer counts as + indexing `.agents/`. +- **`status` reports a snapshot instead of mixing two scans.** It used to run + fresh policy against a cached registry; it now prints the cached registry plus + the last `metrics.jsonl` entry, and says so. +- **`score --fail-under N`** turns CHI into an opt-in gate (exit 2). `score` + still exits 0 by default. +- **`metrics.jsonl` records the scope and both value sets** (raw `metrics` and + capped `scores`), so a trend line no longer mixes a budget ratio with a score. +- Knowledge files up to 8 MB are hashed (was 2 MB); oversized ones are counted + in `knowledge_unhashed` instead of silently skipping single-source checks. +- The report now prints how many projects have no `.agents/` directory, which + dimension had no evidence, and when the pointer scan was truncated. + +## 0.5.5 — 2026-11-05 + +- FRS now counts state documents under the hub project root (`projects/`), + `handoff/`, `provision/` and `.agents/plans/`, not only `kind: state`. + Project `.agents/plans/*` therefore enters the FRS denominator, so a project + or group with a stale plan loses freshness points instead of reporting + `100.0 (0/0)`. Plans use the existing 90-day rule: a declared + `verified_at`/`最后更新` or filename date, otherwise the last git commit of the + repository that owns them. Stale-state findings now point at the offending + in-scope document, so a project/group report explains the lost points instead + of filtering the finding out as hub-level. Empty denominators still print + `0/0` and score 1.0. +- `doctor --scope hub` reports the project cards known to the hub + (`Projects: 3 (scanned roots: 0)`) instead of the number of scanned roots, and + omits the empty `Workspaces:` header line when the scope has no workspaces. + +## 0.5.4 — 2026-11-04 + +- Removed three skill sub-pages that nothing pointed at: `init/SETUP.md`, + `use/OPERATE.md`, `test/SELFTEST.md`. `SKILL.md` carries the workflow itself; + the sub-pages had drifted and were never loaded. + +## 0.5.3 — 2026-11-04 + +- `doctor` reports exactly the dimensions that enter CHI for the run's scope. + `--scope hub` no longer prints RBR/PCR/SIR rows it never scores, and the + always-informational GCR value is labeled `Not counted in CHI`. +- `project` and `group` runs now apply one boundary to artifacts and findings: + hub knowledge, hub state documents and the machine-level local wiki no longer + sit in ICR/FRS/PHR denominators while their findings are filtered out. A + denominator now only contains documents that can raise an in-scope finding, + so those scopes no longer inherit a score from out-of-scope health. +- Removed the unused `templates/` file skeletons and `schema/*.json`: nothing + read them, and the project `AGENTS.md` / `.agents/README.md` text already + lives in `planner.py`. `templates/hub/` is the only shipped template set and + `package.json` no longer ships `schema/`. + +## 0.5.2 — 2026-11-04 + +- Bad plan files no longer crash: a non-object plan, a malformed operation, or an + unreadable plan path fails through the `[guster] ...` / plan-error output and a + non-zero exit. +- `guster apply --dry-run` is accepted as the documented explicit dry run and is + rejected when combined with `--yes`. +- A missing `--root` workspace and an unreadable or non-object Guster config now + fail loud instead of silently scanning defaults. +- Pointer health counts each target once per file, so a link whose text is also + inline code is no longer double-counted. +- Project `.agents/README.md` and hub knowledge indexes are now actually + pointer-checked; the filter listed them but they were never scanned. +- A refused `apply`, `init` or `hub init` prints `FAILED` instead of `APPLIED`. +- The home directory is no longer classified as a project just because it holds + the user-level `~/.agents/`; a fresh `init --home` setup scores clean. +- The setup skill points at the shipped `skills/` directory. + +## 0.5.1 — 2026-11-03 + +- Added reasoned `doctor.waivers` and precise pointer checks. +- Fixed double-counting of linked wiki files and OpenSpec state. +- Added `GUSTER_EXTRA_SKILL_ROOTS` for additional skill hosts. diff --git a/external_plugins/guster/LICENSE b/external_plugins/guster/LICENSE new file mode 100644 index 00000000..b6c14d9c --- /dev/null +++ b/external_plugins/guster/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Lumos-789 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/external_plugins/guster/README.md b/external_plugins/guster/README.md new file mode 100644 index 00000000..b98e625a --- /dev/null +++ b/external_plugins/guster/README.md @@ -0,0 +1,360 @@ +# Guster + +Guster is a distributable agent plugin that audits and evolves the context of a whole machine: projects, instruction files, skills, tools, knowledge, project state, evidence and credential references. It can also scaffold a lightweight Hub for durable knowledge and project state. + +It is the **control plane** for agent context. A Guster Hub is the optional **data plane**: the local repository that holds the project register, knowledge, tools and reports. The plugin does not replace the hub; it discovers it, validates it, and helps keep it coherent. + +## Prior art + +The ideas are borrowed with credit; the implementation is guster's own. + +- **Cline Memory Bank** — inspired the layered responsibilities of the files (project register, knowledge, state) and the banking rhythm of refreshing them per working session; both banks are plain markdown. +- **Claude Code memory tool** — inspired the one-fact-per-file note shape and front-matter metadata; guster's note headers spell the keys `name`, `description`, `type`. +- **Letta (MemGPT)** — inspired the label/limit abstraction of memory blocks behind guster's resident budgets. +- **Mem0** — inspired the decide-first stance (ADD/UPDATE/DELETE/NOOP, soft invalidation by marking instead of deleting) behind plan/apply/waiver; a waiver greens the gate, not the score. +- **Zep/Graphiti** — inspired the dual-timeline fact model (`valid_at`/`invalid_at`); `superseded_by` is its one-file simplification. +- The framing vocabulary — structured note-taking (agentic memory), 'just in time' context, context rot, compaction — is inspired by Anthropic's "Effective context engineering for AI agents" (2025-09-29). + +## What it does + +- **Discovers** Git roots, `AGENTS.md` / `CLAUDE.md`, `.agents/` directories, skills, tools, knowledge, state, evidence and credential locations. +- **Diagnoses** resident budgets, skill metadata budgets, copy drift, duplicate canonical sources, orphan knowledge, broken pointers, misplaced task folders, stale state, inline tokens and credentials outside the canonical root. +- **Scores** context health with seven dimensions and a CHI dashboard. +- **Plans** structural changes as reviewable JSON plans. +- **Applies** only with `--yes`, backing up every touched path and rolling back + automatically when an operation fails. +- **Initializes** a standard `.agents/` layout for a new project or a new machine. +- **Scaffolds** a lite Guster Hub when the user does not have a data plane yet. + +## Quick start + +```bash +cd +./bin/guster init --yes +./bin/guster hub init --path --yes +./bin/guster scan +./bin/guster doctor +./bin/guster score +``` + +`guster check` runs the scan → doctor → score (with trend) pass as one command +when you want the whole picture at once. + +The CLI is Python 3.9+ and has no third-party Python dependencies. + +Install a stable command on your `PATH`: + +```bash +ln -s "$(pwd)/bin/guster" ~/.local/bin/guster +``` + +## Commands + +| Command | Purpose | +|---|---| +| `guster status` | Show plugin version, registry and health summary | +| `guster scan` | Discover context assets and write `registry.json` | +| `guster doctor` | Run policy checks and write `findings.json` | +| `guster score` | Compute metrics and CHI; append `metrics.jsonl`. `--fail-under N` turns it into a gate (exit 2) | +| `guster check` | `scan` + `doctor` + `score --trend` in one pass; exit code carries the doctor semantics | +| `guster zones list/add/remove` | Manage the machine zones in `config.json` | +| `guster plan findings` | Turn findings into a review checklist | +| `guster plan move` | Plan a move with backup (no pointer is written) | +| `guster apply ` | Dry-run by default; `--yes` writes with backup | +| `guster init` | Initialize the state home or a project | +| `guster hub init` | Create a lite Hub and attach it to the local config | +| `guster hub attach` | Attach an existing Hub without copying its content | +| `guster hub status` | Show the Hub manifest and contract checks | +| `guster hub validate` | Validate the lite Hub contract | +| `guster export` | Export a redacted Markdown/JSON report | +| `guster credentials` | List registered credentials by scope (company / personal) | + +## Configuration: roots and zones + +`~/.agents/guster/config.json` declares where your context lives: + +```json +{ + "version": 1, + "roots": ["/home/me/Projects"], + "zones": [{ "path": "/home/me/tools" }], + "scan_max_depth": 5 +} +``` + +- **`roots`** — the workspace roots scanned for git repositories. Must be a + list of existing absolute paths; a bare string, an empty list, a relative + path or a missing directory fails loud instead of being reinterpreted. +- **`hub`** — the attached Guster Hub path (written by `hub init`/`hub attach`). +- **`scan_max_depth`** — how deep under each root the scanner walks looking + for git repositories (default 5). Raise it for deeply nested layouts, lower + it to speed up huge trees. +- **`zones`** — tool areas that are not repositories: a tools directory you + want governed without pretending each tool is a project. A zone is scanned + light-style: its `AGENTS.md`/`README.md` entry docs join the pointer and + freshness checks, its `.env` files join the secret scan, and `.agents/` + slot compliance never applies. An entry is `{path}` only; extra keys a + legacy config may still carry (e.g. a retired `profile`) are tolerated + and ignored. + +Manage zones with the `zones` subcommand (`list`, `add `, +`remove `); it writes `config.json` and applies the same validation. + +**First run without a config**: guster probes the conventional directories +`~/Projects`, `~/code`, `~/dev` and `~/src`. Hits are scanned with an +"Unconfigured" banner telling you to run `guster init` to pin them; if none +exists the command fails loud (exit 1) pointing at +`guster init --root `. Guster never silently scans your home directory. + +## Lite Hub + +The plugin can create a deliberately small data plane: + +```text +Guster/ +├── AGENTS.md +├── README.md +├── hub.yaml +├── knowledge/ +│ ├── README.md +│ └── general/ +│ ├── start-here.md +│ └── knowledge-v1-skeleton.txt +├── projects/ +├── register/ +│ └── projects.yaml +└── tools/ + └── README.md +``` + +`tools/` is present as an index and contract, with no tool preinstalled. `handoff/` and `reports/` are optional and are not created by the lite scaffold. The scanner skips missing optional directories. + +`guster hub init` refuses a non-empty target, writes a reviewable plan, and attaches the new Hub to `~/.agents/guster/config.json`. Use `guster hub attach --path ` for an existing Hub. + +The core roots are declared at creation time: + +```bash +guster hub init --path --knowledge notes --projects work --tools scripts --yes +``` + +Under `hub.yaml` schema 2 the layout is declarative: `knowledge` and `projects` must be declared under `paths` — a missing declaration fails loud — while every other section (such as `tools`) is optional. Relative declarations stay inside the Hub; any section may instead be declared external (absolute or `~/`). Declarations are recorded in `hub.yaml` under `paths` and can be changed later. + +Scanning follows the directories that exist, not a mode: there is no `profile` concept in `hub.yaml` (a leftover key from an older version is ignored, as are unknown top-level manifest keys). Knowledge subzones are discovered, not enumerated — every first-level directory under the knowledge root (`general/`, `projects/`, `machines/`, ...) is scanned and index-checked. A hub is also scanned across the configured project state root, `register/`, `handoff/`, `reports/`, `output/`, `.agents/plans/` and `strategy.md` whenever those exist. Legacy layouts — `provision/`, `manuals/`, `cron-tasks/`, `openspec/`, `archive/` and the `Human Output/` alias of `output/` — are compat-scanned when present only; they are never structures to create. Generated or runtime areas such as `out/`, `.manual-site/`, `.playwright-mcp/` and `.codegraph/` are not treated as knowledge sources. `--scope hub` is Hub-only; local wiki and machine-level secrets belong to the machine scope. + +## Health model + +```text +CHI = 0.20 SSR + 0.15 RBR + 0.15 PCR + 0.15 ICR + + 0.15 FRS + 0.10 PHR + 0.10 SIR +``` + +- **SSR** single-source rate +- **RBR** resident budget ratio +- **PCR** placement compliance rate: share of `.agents/` dirs with all six + standard slots, a non-empty index and no unknown top-level entry. The + denominator is `.agents/` dirs only — a repository that never adopted the + layout is not scored, and a directory that holds other scanned projects (a + workspace or group root) is not a project. The report prints the failure + breakdown, so a low number says why instead of just being low. +- **ICR** index coverage rate +- **FRS** state freshness rate: share of state documents under the hub's + project root (`projects/`), `register/`, `handoff/`, `provision/` and + `.agents/plans/` that were verified within the 90-day SLA. The denominator + is those prefixes only — a hub `reports/` file or a machine-level note is + not state. + The date comes from a declared key — `verified_at` (canonical), the Chinese + `最后更新` family or the English `Last updated:`/`updated:` — in the first 40 + lines, or a date that opens the filename (`2026-11-05-…`, `20261105-…`); + without one, the ladder falls back to the owning repository's last git + commit, then to file mtime (labeled as such in the raw column). A date + buried in prose or a ticket number inside the name is not a freshness claim. +- **PHR** pointer health rate +- **SIR** secret isolation rate: credential files plus literal tokens in host + configuration. Guster does not scan arbitrary file content for token shapes — + that is the release gate's job; SIR only claims what it actually looked at. + +Critical findings do not rescale CHI: they make `doctor` exit 3 (any `high` +finding exits 2), and the score stays the weighted sum above. Treat the exit code +as the gate and CHI as the trend. + +A dimension with no evidence is reported as `n/a` and dropped from the weighted +sum instead of scoring 1.0: an empty denominator means "not measured", not +"perfect". When every dimension of a scope is empty, CHI itself is `n/a`, and +the `score` headline says what it measured — `n/a / 100 (5/7 dims scored)`. +Metric definitions and targets live in +[`skills/guster/reference/metrics.md`](skills/guster/reference/metrics.md). + +## Context format spec + +The metrics are static — filename stems, content hashes, path resolution, git +dates — so they only see documents that keep the format contract. The contract +is written down and versioned: [`skills/guster/reference/spec.md`](skills/guster/reference/spec.md) +(spec_version 2, rules `CTX-01..11` — `CTX-09` structured note header, + `CTX-10` register/v1 project register, `CTX-11` opt-in knowledge/v1 +structured knowledge contract — markdownlint-style stable IDs). It follows +open conventions instead of inventing new ones: the open `AGENTS.md` format +([agents.md](https://agents.md)) for instruction files — discovery is nested +and nearest-wins, the same discipline as CTX-01 (a nested `AGENTS.md` narrows +the parent, never contradicts it), and a host that only reads `CLAUDE.md` is +served with `ln -s AGENTS.md CLAUDE.md` — llms.txt-shaped link indexes +([llmstxt.org](https://llmstxt.org)), YAML front-matter for note headers (the +de-facto standard Jekyll, Hugo and Gray Matter share), and stable rule IDs +like markdownlint. + +A document that ignores the contract is not scored lower — it is invisible: a +naked path in prose never reaches the pointer check, a date written as "last +week" falls back to git, an index without filenames counts as missing. Since +0.7.0 the `naked-pointer` finding closes that gap from the other side: file +paths left outside a link or code span in an instruction or index document are +reported, so the unwritable form is also the flagged form. + +## Schema contracts + +The two machine-readable formats carry JSON Schema (draft 2020-12) contracts, +derived from the code that reads and writes them: + +- [`schemas/hub.schema.json`](schemas/hub.schema.json) — `hub.yaml`. The + scaffolded manifest ships with a yaml-language-server modeline on its first + line, so the VS Code YAML extension (or any editor speaking + yaml-language-server) validates `hub.yaml` with zero configuration: + + ```yaml + # yaml-language-server: $schema=https://raw.githubusercontent.com/Lumos-789/guster/main/schemas/hub.schema.json + ``` + +- [`schemas/plan.schema.json`](schemas/plan.schema.json) — the plan JSON that + `plan` writes and `apply` executes. An agent hand-writing a batch plan has + the exact field contract to check against before the `apply` dry-run. + +## Scopes + +| Level | Command | Scored dimensions | +|---|---|---| +| machine | `guster doctor` | SSR RBR PCR ICR FRS PHR SIR | +| hub | `guster doctor --scope hub` | SSR ICR FRS PHR | +| group | `guster doctor --root A --root B` | SSR PCR ICR FRS PHR | +| project | `guster doctor --root ` | SSR PCR ICR FRS PHR | + +Scoped runs write to `~/.agents/guster/scoped//`. + +Every scoped run applies one boundary to both sides of the report: an artifact +counts toward a denominator only when a finding about it could also appear in +that scope. `project` and `group` runs therefore drop hub knowledge, hub state +documents and the machine-level local wiki from ICR/FRS/PHR unless they live +under one of the given roots; the report table shows exactly the dimensions that +enter CHI, and any extra value is labeled `Not counted in CHI`. A dimension with +no in-scope artifacts stays vacuous (`0/0` in the raw column, `n/a` score) instead +of borrowing health from out-of-scope files. This changes CHI for `project` and +`group` scopes; hub health keeps being reported by `--scope hub` and the +machine-wide run. Because empty dimensions are dropped and the weights are +renormalized over what remains, CHI is not comparable across scopes with +different coverage. + +## Credential registry + +Guster reads `~/.agents/credentials/registry.md` and checks it against every credential file it finds: + +- `secret-undeclared` - a credential file is not registered. +- `secret-missing` - a registered location no longer exists. +- `git-unavailable` - a repository could not be queried, so tracked state and + commit dates are unknown. SIR counts those credentials as unverified rather + than clean. + +`secret-undeclared` and `secret-missing` apply only when the registry exists +and parses at least one row: with `~/.agents/credentials/registry.md` missing, +both checks stay off — `guster credentials` printing `registry: (missing)` +means exactly that. Create the table (header row plus one row, like the +copyable example below) to enable them. + +The registry holds scope (`company` / `personal`), site, account, kind, location and variable names. +It never holds values. `guster credentials` prints the split. A copyable row +(column order is fixed — see the format spec; the first six columns are +required, `keys`/`used_by`/`status` are optional): + +```markdown +| ID | scope | site | account | kind | locations | keys | used_by | status | +|---|---|---|---|---|---|---|---|---| +| example-host | personal | example.com | me@example.com | token | `~/.agents/credentials/example-host.env` | `EXAMPLE_HOST_TOKEN` | my-cli | active | +``` + +## The eight planes + +1. **Truth** — code, tests, build and runtime config. +2. **Instruction** — `AGENTS.md` and policies. +3. **Capability** — skills, tools, MCP, plugins, scripts. +4. **Knowledge** — durable principles, methods, business semantics, pitfalls. +5. **State** — the project register, handoffs, environment and port registries. +6. **Evidence** — reports, screenshots, videos, test output. +7. **Runtime** — sessions, caches, logs, temp files, browser profiles. +8. **Secret** — passwords, tokens, keys, cookies; references only. + +## Safety + +- Read-only on your workspace by default. `apply --yes`, `init --yes`, `hub init --yes` and `hub attach` are the only commands that change workspace files or Guster config on their own; `export` always writes one report file — `/context-health.md` by default, or the path you name with `--out`. `--format json` writes only the `.json` report (no Markdown). +- Guster keeps five files under its state directory (`~/.agents/guster/`, or `scoped//`): `config.json` (roots/zones/hub, written by `init`, `hub init`/`attach` and `zones`), `registry.json`, which every scanning command rewrites (`status` included, when no registry exists yet), `findings.json` from `doctor`, `metrics.jsonl` from `score`, and the transient `.lock` that serializes concurrent runs. `--no-write` suppresses the state writes of `scan`, `doctor`, `score` and `check` only. (`plans/` and `backups/` are directories, not files.) +- Credential values are never persisted or printed; scans keep only paths and redacted shapes. +- No automatic `git add`, `commit`, `push` or deploy. +- Tracked files you do not own or have no authority over — a shared or employer repository usually qualifies — are not ordinary migration targets. +- Plans are written to `~/.agents/guster/plans/`. `apply` copies every touched path under `~/.agents/guster/backups//` before changing it, and rolls back automatically when an operation fails; if the rollback itself fails, the error names that backup directory and you restore by copying the pre-change copies back. Only the newest 10 plan backup directories are kept. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | success — clean run, or findings no worse than `medium` | +| `1` | tool/config error: malformed or unreadable plan, an apply that failed mid-operation and was rolled back, missing workspace root, invalid `config.json` (non-list `roots`, a bad root/zone/hub path, empty `roots`, no scan candidates — run `guster init`); `hub status` also returns 1 when the attached hub fails its checks | +| `2` | `doctor`/`check` found `high` (but no `critical`) findings; `score --fail-under` below target or CHI not computable; `hub validate` failed the lite contract | +| `3` | `doctor`/`check` found `critical` findings | +| `4` | `apply`, `init` or `hub init` refused the plan at preflight — nothing was touched (missing move source, existing destination, invalid `hub init` target; an `init --project` target that only fails mid-apply exits 1 after rollback) | + +The same table ships in the `guster` skill so agents and humans read one +contract. Treat the exit code as the gate and CHI as the trend. + +## Plugin hosts + +| Host | Manifest | +|---|---| +| Codex | `.codex-plugin/plugin.json` | +| ZCode | `.zcode-plugin/plugin.json` | +| DSH | `package.json` + `cordis.patch.yml` | + +All three point at `skills/` (the `guster` and `guster-hub` skills). Keep their versions synchronized. + +```bash +# DSH bundle +dsh plugin add /path/to/guster +``` + +### Extra skill roots + +By default guster scans `~/.agents/skills`, `~/.codex/skills`, `~/.dsh/skills` and +`~/.claude/skills` (user-level roots only — project-local `.agents/skills/` +directories are not part of the skill scan). Additional hosts stay local and are +injected without changing the published source: + +```bash +export GUSTER_EXTRA_SKILL_ROOTS="name=/path/to/skills:other=/another/path" +``` + +## Project layout + +```text +skills/guster/ agent-facing skill +skills/guster-hub/ hub bootstrap and validation skill +src/guster/ Python CLI and policy engine +bin/guster executable wrapper +templates/hub/ lite Hub scaffold (the only shipped template set) +schemas/ JSON Schema contracts for hub.yaml and plan JSON +tests/ fixture and plugin tests +``` + +## Test + +```bash +python3 -m unittest discover -s tests -v +node --test tests/test_plugin.mjs +``` + +## License + +MIT © Lumos-789 diff --git a/external_plugins/guster/README_CN.md b/external_plugins/guster/README_CN.md new file mode 100644 index 00000000..ea2a00c4 --- /dev/null +++ b/external_plugins/guster/README_CN.md @@ -0,0 +1,290 @@ +# Guster + +Guster 是一个可分发的 Agent 插件,用来审计和演进整台机器的上下文:项目、指令文件、Skill、工具、知识、项目状态、证据和凭据引用;也可以初始化一个轻量 Hub,承载长期知识与项目状态。 + +它是上下文的**控制面**。Guster Hub 是可选的**数据面**:承载项目登记册、知识、工具和报告的本地仓库。插件不替代 Hub,而是发现它、校验它,并帮助它保持自洽。 + +## 理念来源 + +理念皆有出处,实现是 guster 自己的: + +- **Cline Memory Bank** —— 启发了分层文件的职责划分(项目登记册、知识、状态)与每个工作会话刷新一次的 banking 节奏;两边的记忆库都是纯 markdown。 +- **Claude Code memory tool** —— 启发了「一事实一文件」的笔记形状与 front-matter 元数据;guster 的笔记头把这些键拼写为 `name`、`description`、`type`。 +- **Letta (MemGPT)** —— 启发了 memory block 的 label/limit 抽象,guster 的常驻预算背后正是它。 +- **Mem0** —— 启发了先裁决再动手的语义(ADD/UPDATE/DELETE/NOOP、以标记代删除的软失效),落到 plan/apply/waiver;waiver 只让门禁变绿,不改分数。 +- **Zep/Graphiti** —— 启发了双时间轴事实建模(`valid_at`/`invalid_at`),`superseded_by` 是它的文件化简化。 +- 定位词汇——structured note-taking(agentic memory)、'just in time' context、context rot、compaction——借鉴自 Anthropic《Effective context engineering for AI agents》(2025-09-29)。 + +## 它做什么 + +- **发现** Git 根、`AGENTS.md` / `CLAUDE.md`、`.agents/`、Skill、工具、知识、状态、证据和凭据位置。 +- **诊断** 常驻预算、Skill 元数据预算、副本漂移、多规范源、孤儿知识、断链、错位任务目录、陈旧状态、内联 token 和凭据越界。 +- **评分** 用七个维度和一个 CHI 仪表盘衡量上下文健康。 +- **计划** 把结构变更写成可审阅的 JSON plan。 +- **执行** 只有带 `--yes` 才写入;每个被改路径先备份,操作失败自动回滚。 +- **初始化** 新项目或新机器的标准 `.agents/` 布局。 +- **脚手架** 用户还没有数据面时,生成一个 Lite Guster Hub。 + +## 快速开始 + +```bash +cd +./bin/guster init --yes +./bin/guster hub init --path --yes +./bin/guster scan +./bin/guster doctor +./bin/guster score +``` + +想一次拿到全貌时,`guster check` 一条命令跑 scan → doctor → score(含趋势)。 + +CLI 依赖 Python 3.9+,无第三方 Python 依赖。 + +安装一个稳定的 `guster` 命令: + +```bash +ln -s "$(pwd)/bin/guster" ~/.local/bin/guster +``` + +## 命令 + +| 命令 | 用途 | +|---|---| +| `guster status` | 查看插件版本、registry 和健康摘要 | +| `guster scan` | 发现上下文资产并写 `registry.json` | +| `guster doctor` | 跑规则并写 `findings.json` | +| `guster score` | 计算指标和 CHI,追加 `metrics.jsonl`;`--fail-under N` 变成门禁(低于则退出码 2) | +| `guster check` | `scan` + `doctor` + `score --trend` 一条命令跑完;退出码承接 doctor 语义 | +| `guster zones list/add/remove` | 管理 `config.json` 里的机器 zone | +| `guster plan findings` | 把发现转成审阅清单 | +| `guster plan move` | 计划一次带备份的移动(不再自动写指针) | +| `guster apply ` | 默认 dry-run;`--yes` 才写入 | +| `guster init` | 初始化状态目录或项目布局 | +| `guster hub init` | 创建 Lite Hub 并接入本地配置 | +| `guster hub attach` | 接入已有 Hub,不复制其内容 | +| `guster hub status` | 查看 Hub manifest 和契约检查 | +| `guster hub validate` | 校验 Lite Hub 契约 | +| `guster export` | 导出脱敏 Markdown/JSON 报告 | +| `guster credentials` | 按范围(公司 / 个人)列出已登记凭据 | + +## 配置:roots 与 zones + +`~/.agents/guster/config.json` 声明你的上下文在哪里: + +```json +{ + "version": 1, + "roots": ["/home/me/Projects"], + "zones": [{ "path": "/home/me/tools" }], + "scan_max_depth": 5 +} +``` + +- **`roots`** —— 扫 git 仓库的工作区根。必须是「已存在的绝对路径」列表; + 裸字符串、空列表、相对路径、不存在的目录都会 fail-loud,绝不悄悄换解释。 +- **`hub`** —— 已接入的 Guster Hub 路径(由 `hub init`/`hub attach` 写入)。 +- **`scan_max_depth`** —— 在每个 root 下找 git 仓库时的目录递归深度(默认 5)。 + 嵌套深的布局可以调大,巨大的目录树可以调小加速。 +- **`zones`** —— 不是仓库的工具区:想让一个工具目录被治理、又不必把每个工具 + 当成项目。zone 按 light 方式扫描:其 `AGENTS.md`/`README.md` 入口文档进指针 + 与新鲜度检查,`.env` 进密钥扫描,`.agents/` 六槽合规永不适用。条目只认 + `{path}`;旧配置可能残留的多余键(如已退役的 `profile`)容忍并忽略。 + +zone 用 `zones` 子命令管理(`list`、`add `、`remove `), +直接写 `config.json`,校验同上。 + +**没有 config 的首跑**:guster 探测约定目录 `~/Projects`、`~/code`、`~/dev`、 +`~/src`。命中即照常扫描并打「未配置」横幅提示跑 `guster init` 固定;一个都 +没有则 fail-loud(退出码 1)并指路 `guster init --root `。绝不静默扫 +整个 home 目录。 + +## Lite Hub + +插件可以生成一个刻意保持很小的数据面: + +```text +Guster/ +├── AGENTS.md +├── README.md +├── hub.yaml +├── knowledge/ +│ ├── README.md +│ └── general/ +│ ├── start-here.md +│ └── knowledge-v1-skeleton.txt +├── projects/ +├── register/ +│ └── projects.yaml +└── tools/ + └── README.md +``` + +`tools/` 只提供索引和收录契约,不预装任何工具;`handoff/`、`reports/` 仍是可选模块,Lite 脚手架不创建,扫描器会跳过缺失的可选目录。 + +`guster hub init` 会拒绝非空目录,先写可审阅的 plan,再把新 Hub 接入 `~/.agents/guster/config.json`。已有 Hub 用 `guster hub attach --path `。 + +核心目录在创建时声明: + +```bash +guster hub init --path --knowledge notes --projects work --tools scripts --yes +``` + +`hub.yaml` schema 2 下布局是声明式的:`knowledge` 与 `projects` 必须在 `paths` 下显式声明——缺一个直接 fail-loud——其余节(如 `tools`)可选。相对声明必须留在 Hub 内;任何节都可以改声明为外部路径(绝对路径或 `~/`)。声明记录在 `hub.yaml` 的 `paths` 下,之后也可以修改。 + +扫描按目录是否存在走,不是开关:`hub.yaml` 没有 `profile` 概念(旧版残留的键会被忽略,未知的 manifest 顶层键同样忽略)。knowledge 子区是动态发现的,不写死枚举——knowledge 根下任意一层目录(`general/`、`projects/`、`machines/`……)都会被扫描并做索引检查。Hub 只要有配置的项目状态根、`register/`、`handoff/`、`reports/`、`output/`、`.agents/plans/` 和 `strategy.md`,就会被扫到。旧结构——`provision/`、`manuals/`、`cron-tasks/`、`openspec/`、`archive/` 以及 `output/` 的旧别名 `Human Output/`——仅作兼容扫描(存在才扫),绝不是要新建的结构。`out/`、`.manual-site/`、`.playwright-mcp/`、`.codegraph/` 等生成物或运行态不作为知识源。`--scope hub` 只算 Hub;本地 wiki 和机器级凭据归机器范围。 + +## 健康模型 + +```text +CHI = 0.20 SSR + 0.15 RBR + 0.15 PCR + 0.15 ICR + + 0.15 FRS + 0.10 PHR + 0.10 SIR +``` + +- **SSR** 单源率 +- **RBR** 常驻预算比 +- **PCR** 落位合规率:`.agents/` 中同时具备 6 个标准槽、非空索引且无未知顶层项的比例。 + 分母只算 `.agents/` 目录本身 —— 从未采用该布局的仓库不计分,而「包含其他已扫描项目」 + 的目录(工作区或组根)不算项目。报告会打出不合格原因分布,低分能自己解释为什么低。 +- **ICR** 索引覆盖率 +- **FRS** 状态新鲜度:Hub 项目目录(`projects/`)、`register/`、`handoff/`、 + `provision/` 与 `.agents/plans/` 各前缀下状态文档中在 90 天 SLA 内核实过的 + 比例;分母只认这些前缀,Hub `reports/` 或机器级笔记不算状态文档。日期取自前 40 行内的声明键—— + `verified_at`(canonical)、中文 `最后更新` 族或英文 `Last updated:`/`updated:` + ——或文件名开头处的日期(`2026-11-05-…`、`20261105-…`);都没有时依次兜底到 + 所属仓库最后一次 git 提交、再兜底文件 mtime(Raw 列会标注来源)。正文深处的 + 日期和文件名里的工单号都不算新鲜度声明。 +- **PHR** 指针健康率 +- **SIR** 敏感隔离率:凭据文件 + host 配置里的字面令牌。Guster 不扫描任意文件内容找 + token 形状 —— 那是发布闸的职责;SIR 只声称它确实看过的东西。 + +严重 finding 不改写 CHI:它让 `doctor` 以 3 退出(任意 `high` 退出 2),分数仍是上面的加权和。门禁看退出码,CHI 看趋势。 + +某个维度没有证据时记为 `n/a` 并从加权和中剔除,而不是记 1.0:空分母表示「没测」,不是「满分」。某 scope 的所有维度都为空时,CHI 本身为 `n/a`,`score` 标题行也会说明实际测了几个维度——`n/a / 100 (5/7 dims scored)`。指标定义与目标值见 +[`skills/guster/reference/metrics.md`](skills/guster/reference/metrics.md)。 + +## 上下文格式规范 + +指标全是静态的——文件名主干、内容哈希、路径解析、git 日期——所以只有守住格式契约的文档才被测得到。契约本身已写成文档并版本化:[`skills/guster/reference/spec.md`](skills/guster/reference/spec.md)(spec_version 2,规则 `CTX-01..11`——`CTX-09` 结构化条目头、`CTX-10` register/v1 项目登记册、`CTX-11` 可选加入的 knowledge/v1 契约,markdownlint 式稳定编号)。它遵循开源惯例而不是另造一套:指令文件用 `AGENTS.md` 开放格式([agents.md](https://agents.md))——发现规则是嵌套 nearest-wins,与 CTX-01(嵌套只收窄、绝不矛盾)是同一条纪律,只认 `CLAUDE.md` 的宿主用 `ln -s AGENTS.md CLAUDE.md` 即可伺候——索引文件用 llms.txt 形态的链接列表([llmstxt.org](https://llmstxt.org))、笔记头用 YAML front-matter(Jekyll/Hugo/Gray Matter 共同坐实的业界事实标准)、规则编号学 markdownlint。 + +不守契约的文档不是被扣分——而是彻底隐身:正文里裸写的路径永远到不了指针检查、「上周更新」落到 git 时间兜底、索引里没有文件名等于没索引。0.7.0 起 `naked-pointer` finding 从另一侧堵上这个缺口:指令文档/索引页里留在链接和代码记号之外的文件路径会被直接报告,写不规范的形态同时也是会被标记的形态。 + +## Schema 契约 + +两份机器可读格式都带 JSON Schema(draft 2020-12)契约,字段全部从读写它们的代码实况推导: + +- [`schemas/hub.schema.json`](schemas/hub.schema.json) —— `hub.yaml` 清单。脚手架生成的 manifest 首行自带 yaml-language-server modeline,VS Code YAML 扩展(或任何接 yaml-language-server 的编辑器)零配置即可校验: + + ```yaml + # yaml-language-server: $schema=https://raw.githubusercontent.com/Lumos-789/guster/main/schemas/hub.schema.json + ``` + +- [`schemas/plan.schema.json`](schemas/plan.schema.json) —— `plan` 产出、`apply` 执行的 plan JSON。agent 手搓批量 plan 时,它就是完整的字段契约,`apply` dry-run 前先对照自查。 + +## 层级 + +| 层级 | 命令 | 计分维度 | +|---|---|---| +| 机器 | `guster doctor` | SSR RBR PCR ICR FRS PHR SIR | +| Hub | `guster doctor --scope hub` | SSR ICR FRS PHR | +| 组 | `guster doctor --root A --root B` | SSR PCR ICR FRS PHR | +| 项目 | `guster doctor --root ` | SSR PCR ICR FRS PHR | + +带范围的运行写入 `~/.agents/guster/scoped//`。 + +每个带范围的运行对报告两侧使用同一条边界:一个 artifact 只有在「针对它的 finding 也能出现在该 scope」时,才计入分母。因此 `project` / `group` 运行会把 Hub 知识、Hub 状态文档和机器级本地 wiki 从 ICR/FRS/PHR 分母中剔除,除非它们就在给定 root 之下;报告表格只列真正进入 CHI 的维度,额外展示的数值必须标注 `Not counted in CHI`。某个维度在该 scope 内没有 artifact 时保持空集口径(Raw 列显示 `0/0`,得分 `n/a`),而不是借用 scope 外文件的健康度。这会改变 `project` / `group` 的 CHI;Hub 健康度仍由 `--scope hub` 和全机运行报告。空维度被剔除后权重在剩余维度上归一,因此覆盖度不同的 scope 之间 CHI 不可横向比较。 + +## 凭据登记 + +Guster 读取 `~/.agents/credentials/registry.md`,并对照它检查发现的每个凭据文件: + +- `secret-undeclared` — 凭据文件没有登记。 +- `secret-missing` — 已登记的位置已不存在。 +- `git-unavailable` — 某个仓库查询失败,跟踪状态与提交日期未知;SIR 会把这类凭据记为「未核实」而不是「干净」。 + +`secret-undeclared` 与 `secret-missing` 只在登记表存在且解析出至少一行时生效:`~/.agents/credentials/registry.md` 缺失时这两项检查整体关闭——`guster credentials` 打印 `registry: (missing)` 就是这个意思。照下方的表头加一行样例把表建起来即可启用。 + +登记表保存范围(`company` / `personal`)、站点、账号、类型、位置和变量名,从不保存凭据值。`guster credentials` 按范围输出。一行可复制的样例(列序固定——详见格式规范;前 6 列必填,`keys`/`used_by`/`status` 可选): + +```markdown +| ID | scope | site | account | kind | locations | keys | used_by | status | +|---|---|---|---|---|---|---|---|---| +| example-host | personal | example.com | me@example.com | token | `~/.agents/credentials/example-host.env` | `EXAMPLE_HOST_TOKEN` | my-cli | active | +``` + +## 八个平面 + +1. **Truth** — 代码、测试、构建和运行配置。 +2. **Instruction** — `AGENTS.md` 和策略。 +3. **Capability** — Skill、工具、MCP、插件、脚本。 +4. **Knowledge** — 稳定原则、方法、业务语义、坑。 +5. **State** — 项目登记册、交接、环境和端口台账。 +6. **Evidence** — 报告、截图、视频、测试输出。 +7. **Runtime** — 会话、缓存、日志、临时文件、浏览器 profile。 +8. **Secret** — 密码、token、key、cookie;只存引用。 + +## 安全边界 + +- 默认不写工作区:只有 `apply --yes`、`init --yes`、`hub init --yes` 和 `hub attach` 会自行改动工作区文件或 Guster 配置;`export` 总会写一个报告文件——默认 `/context-health.md`,或 `--out` 指定的路径;`--format json` 只写 `.json` 报告(不再同时写 Markdown)。 +- Guster 会在状态目录(`~/.agents/guster/` 或 `scoped//`)维护五个文件:`config.json`(roots/zones/hub,由 `init`、`hub init`/`attach` 和 `zones` 写入)、所有扫描类命令都会重写的 `registry.json`(`status` 在没有 registry 时也会写)、`doctor` 的 `findings.json`、`score` 的 `metrics.jsonl`,以及串行化并发运行的临时 `.lock`。`--no-write` 只对 `scan`、`doctor`、`score`、`check` 关闭本命令的状态写入。(`plans/` 与 `backups/` 是目录,不算文件。) +- 永不持久化或输出凭据值;扫描只保留路径和脱敏形态。 +- 不自动 `git add` / `commit` / `push` / 部署。 +- 不拥有、也无权变更其 tracked 文件的仓库——共享仓或雇主仓通常是——不是普通迁移目标。 +- plan 写 `~/.agents/guster/plans/`。`apply` 改动前把每个被触碰路径复制到 `~/.agents/guster/backups//`,操作失败自动回滚;若回滚本身也失败,报错会点名该备份目录,手工把改动前副本拷回即完成恢复。plan 备份目录只保留最近 10 个。 + +## 退出码 + +| 码 | 含义 | +|---|---| +| `0` | 成功——干净运行,或 finding 严重度不高于 `medium` | +| `1` | 工具/配置错误:plan 损坏或不可读、执行中失败且已回滚的 apply、工作区根不存在、`config.json` 无效(`roots` 非列表、root/zone/hub 路径坏、`roots` 为空、无可扫候选——跑 `guster init`);已接入 hub 校验不过时 `hub status` 也返回 1 | +| `2` | `doctor`/`check` 发现 `high`(但无 `critical`)finding;`score --fail-under` 低于目标或 CHI 不可算;`hub validate` 未过 Lite 契约 | +| `3` | `doctor`/`check` 发现 `critical` finding | +| `4` | `apply`、`init` 或 `hub init` 在 preflight 拒绝了 plan——什么都没动(move 源不存在、目标已存在、`hub init` 目标无效;`init --project` 目标若在执行中才失败,回滚后按第 1 行退出 1) | + +同一张表也随 `guster` skill 发布,agent 与人读同一份契约。门禁看退出码,CHI 看趋势。 + +## 插件宿主 + +| 宿主 | Manifest | +|---|---| +| Codex | `.codex-plugin/plugin.json` | +| ZCode | `.zcode-plugin/plugin.json` | +| DSH | `package.json` + `cordis.patch.yml` | + +三端都指向 `skills/`(`guster` 与 `guster-hub` 两个 Skill),版本必须同步。 + +```bash +# DSH bundle +dsh plugin add /path/to/guster +``` + +### 额外 Skill 根 + +默认扫描 `~/.agents/skills`、`~/.codex/skills`、`~/.dsh/skills` 和 `~/.claude/skills`(仅用户级根——项目内 `.agents/skills/` 目录不在 skill 扫描面内)。其他宿主保持在本机,通过环境变量注入,不改发布源: + +```bash +export GUSTER_EXTRA_SKILL_ROOTS="name=/path/to/skills:other=/another/path" +``` + +## 项目结构 + +```text +skills/guster/ agent 侧治理 skill +skills/guster-hub/ Hub 初始化与校验 skill +src/guster/ Python CLI 与策略引擎 +bin/guster 可执行入口 +templates/hub/ Lite Hub 脚手架(唯一随包发布的模板集) +schemas/ hub.yaml 与 plan JSON 的 JSON Schema 契约 +tests/ fixture 与插件测试 +``` + +## 测试 + +```bash +python3 -m unittest discover -s tests -v +node --test tests/test_plugin.mjs +``` + +## License + +MIT © Lumos-789 diff --git a/external_plugins/guster/assets/guster/icon.png b/external_plugins/guster/assets/guster/icon.png new file mode 100644 index 00000000..cd932818 Binary files /dev/null and b/external_plugins/guster/assets/guster/icon.png differ diff --git a/external_plugins/guster/bin/guster b/external_plugins/guster/bin/guster new file mode 100644 index 00000000..e97baaeb --- /dev/null +++ b/external_plugins/guster/bin/guster @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +SOURCE="${BASH_SOURCE[0]}" +while [ -L "$SOURCE" ]; do + DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)" + SOURCE="$(readlink "$SOURCE")" + case "$SOURCE" in + /*) ;; + *) SOURCE="$DIR/$SOURCE" ;; + esac +done +HERE="$(cd -P "$(dirname "$SOURCE")/.." && pwd)" +PYTHONPATH="$HERE/src${PYTHONPATH:+:$PYTHONPATH}" exec python3 -m guster "$@" diff --git a/external_plugins/guster/cordis.patch.yml b/external_plugins/guster/cordis.patch.yml new file mode 100644 index 00000000..e172e1ba --- /dev/null +++ b/external_plugins/guster/cordis.patch.yml @@ -0,0 +1,3 @@ +- insert: + - id: guster + name: 'guster' diff --git a/external_plugins/guster/index.js b/external_plugins/guster/index.js new file mode 100644 index 00000000..bb454824 --- /dev/null +++ b/external_plugins/guster/index.js @@ -0,0 +1,84 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const skillRoots = { + guster: fileURLToPath(new URL("./skills/guster/", import.meta.url)), + "guster-hub": fileURLToPath(new URL("./skills/guster-hub/", import.meta.url)), +}; + +function parseFrontmatter(source, skillPath) { + const match = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(source); + if (!match) throw new Error(`guster: missing YAML frontmatter in ${skillPath}`); + const header = {}; + for (const rawLine of match[1].split(/\r?\n/)) { + const line = rawLine.trimEnd(); + if (!line || line.startsWith("#") || /^\s/.test(line)) continue; + const colon = line.indexOf(":"); + if (colon < 1) continue; + const key = line.slice(0, colon).trim(); + let value = line.slice(colon + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + header[key] = value; + } + return { header, body: source.slice(match[0].length).trimStart() }; +} + +function loadSkill(skillName) { + const skillRoot = skillRoots[skillName]; + const skillPath = join(skillRoot, "SKILL.md"); + const source = readFileSync(skillPath, "utf8"); + const { header, body } = parseFrontmatter(source, skillPath); + if (typeof header.name !== "string" || typeof header.description !== "string") { + throw new Error(`guster: invalid skill metadata in ${skillPath}`); + } + return { + name: header.name, + description: header.description, + content: body, + source: "bundled", + path: skillPath, + resourceBase: { kind: "directory", path: skillRoot }, + invocation: { modelInvocable: true, userInvocable: true }, + }; +} + +const skills = Object.keys(skillRoots).map((skillName) => loadSkill(skillName)); + +export const name = "guster"; +export const inject = ["skills", "agents"]; + +export function apply(ctx) { + for (const skill of skills) ctx.skills.register(skill); + + const agentRegistrations = new Map(); + const registerForAgent = (agent) => { + if (agentRegistrations.has(agent)) return; + const injection = agent.ctx.inject(["skills"], (agentCtx) => { + for (const skill of skills) agentCtx.skills.register(skill); + }); + agentRegistrations.set(agent, () => injection.dispose()); + }; + + for (const agent of ctx.agents.list()) registerForAgent(agent); + + ctx.on("agent/created", ({ agent }) => registerForAgent(agent)); + ctx.on("agent/disposed", ({ agent }) => { + agentRegistrations.get(agent)?.(); + agentRegistrations.delete(agent); + }); + + ctx.effect( + () => () => { + const disposals = [...agentRegistrations.values()].map((dispose) => dispose()); + agentRegistrations.clear(); + return Promise.all(disposals); + }, + "guster agent registrations", + ); +} diff --git a/external_plugins/guster/schemas/hub.schema.json b/external_plugins/guster/schemas/hub.schema.json new file mode 100644 index 00000000..9869b70c --- /dev/null +++ b/external_plugins/guster/schemas/hub.schema.json @@ -0,0 +1,113 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/Lumos-789/guster/main/schemas/hub.schema.json", + "title": "Guster hub manifest", + "description": "Format contract for hub.yaml, the Guster hub manifest. Field semantics follow src/guster/layout.py and src/guster/hub.py. Guster reads values in their string form, so quoted and unquoted spellings of schema_version are equivalent. Unknown top-level keys (e.g. a leftover `profile`) are ignored by the CLI, so local extensions validate too.", + "type": "object", + "required": [ + "schema_version" + ], + "properties": { + "schema_version": { + "description": "Manifest format major version; absent means 1.", + "enum": ["1", "2", 1, 2] + }, + "hub_version": { + "description": "Version of the hub's own contract; informational, shown by `hub status`.", + "type": "string" + }, + "min_guster": { + "description": "Minimum guster CLI version this hub needs; compared numerically by dotted segments.", + "type": "string" + }, + "name": { + "description": "Human-readable hub name.", + "type": "string" + }, + "created_at": { + "description": "ISO date the hub was created.", + "type": "string" + }, + "governance_docs": { + "description": "File names this hub counts as primary context-governance framework docs, looked up under the hub knowledge/general and ~/.agents/wiki. Absent or empty means GCR is reported as n/a instead of measured.", + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "paths": { + "description": "Declared hub sections. Relative values are hub-relative and must not contain '..'; a hub may also declare external sections with absolute or ~/ paths. Schema 2 must declare knowledge and projects.", + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1 + }, + "properties": { + "knowledge": { + "type": "string", + "minLength": 1 + }, + "projects": { + "type": "string", + "minLength": 1 + }, + "tools": { + "type": "string", + "minLength": 1 + } + } + }, + "doctor": { + "description": "Settings for `guster doctor`.", + "type": "object", + "additionalProperties": false, + "properties": { + "waivers": { + "description": "Silence a finding; every waiver must carry a written reason.", + "type": "array", + "items": { + "type": "object", + "required": [ + "rule", + "reason" + ], + "additionalProperties": false, + "properties": { + "rule": { + "description": "Rule name of the waived finding, e.g. broken-pointer; the name a doctor finding reports, not the CTX spec id.", + "type": "string", + "minLength": 1 + }, + "path": { + "description": "Optional scope; only findings for this path are waived.", + "type": "string" + }, + "reason": { + "description": "Written justification; a waiver without one fails loud.", + "type": "string", + "minLength": 1 + } + } + } + } + } + } + }, + "if": { + "required": ["schema_version"], + "properties": { + "schema_version": { + "enum": ["2", 2] + } + } + }, + "then": { + "required": ["paths"], + "properties": { + "paths": { + "required": ["knowledge", "projects"] + } + } + } +} diff --git a/external_plugins/guster/schemas/plan.schema.json b/external_plugins/guster/schemas/plan.schema.json new file mode 100644 index 00000000..15eaf136 --- /dev/null +++ b/external_plugins/guster/schemas/plan.schema.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/Lumos-789/guster/main/schemas/plan.schema.json", + "title": "Guster plan", + "description": "Format contract for the plan JSON produced by src/guster/planner.py (build_plan and the op_* constructors) and consumed by `guster apply` (src/guster/executor.py). mkdir/move/write run on apply; manual ops stay a skipped checklist.", + "type": "object", + "required": [ + "version", + "id", + "created_at", + "title", + "notes", + "operations" + ], + "additionalProperties": false, + "properties": { + "version": { + "description": "Plan format version; the integer 1.", + "type": "integer", + "enum": [1] + }, + "id": { + "description": "Unique plan id; also the plan file name stem.", + "type": "string", + "minLength": 1 + }, + "created_at": { + "description": "ISO timestamp of plan creation.", + "type": "string" + }, + "title": { + "description": "One-line plan title.", + "type": "string", + "minLength": 1 + }, + "notes": { + "description": "What the plan does and how to apply it; may be empty.", + "type": "string" + }, + "operations": { + "description": "Ordered operations applied one by one; a failure rolls back.", + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "required": [ + "kind", + "path" + ], + "additionalProperties": false, + "properties": { + "kind": { + "enum": ["mkdir"] + }, + "path": { + "description": "Directory to create; absolute path.", + "type": "string", + "minLength": 1, + "pattern": "^/" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "src", + "dst" + ], + "additionalProperties": false, + "properties": { + "kind": { + "enum": ["move"] + }, + "src": { + "description": "Existing file or directory to move; absolute path.", + "type": "string", + "minLength": 1, + "pattern": "^/" + }, + "dst": { + "description": "Destination path; must not exist yet; absolute path.", + "type": "string", + "minLength": 1, + "pattern": "^/" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "path", + "content" + ], + "additionalProperties": false, + "properties": { + "kind": { + "enum": ["write"] + }, + "path": { + "description": "File to write; absolute path.", + "type": "string", + "minLength": 1, + "pattern": "^/" + }, + "content": { + "description": "Full file content.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "kind", + "path", + "action", + "reason" + ], + "additionalProperties": false, + "properties": { + "kind": { + "enum": ["manual"] + }, + "path": { + "description": "Path the action is about; may be empty.", + "type": "string" + }, + "action": { + "description": "What a human should do.", + "type": "string" + }, + "reason": { + "description": "Why; usually the finding rule and message.", + "type": "string" + } + } + } + ] + } + } + } +} diff --git a/external_plugins/guster/skills/guster-hub/SKILL.md b/external_plugins/guster/skills/guster-hub/SKILL.md new file mode 100644 index 00000000..1d75d71e --- /dev/null +++ b/external_plugins/guster/skills/guster-hub/SKILL.md @@ -0,0 +1,95 @@ +--- +name: guster-hub +description: Create, attach or validate a lightweight Guster Hub data plane for the Guster plugin. Use when the user says 建文件阵地/Guster Hub/精简 Hub/初始化知识枢纽/接入现有 Hub. +--- + +# Guster Hub + +The Hub is the data plane: a plain repository for durable knowledge and project state. The Guster plugin is the control plane and reads this repository; it does not replace it. + +## Lite hub + +The lite Hub intentionally ships only: + +- `AGENTS.md`, `README.md` and `hub.yaml` +- `knowledge/` with a single index, the `general/` starter note, an + all-commented knowledge/v1 skeleton next to it, and an empty `projects/` + subzone to grow into +- `register/projects.yaml`, the project table (register/v1: one row per + project, with a commented example row in the scaffold) +- `tools/` with an index and inclusion rules, but no tool preinstalled + +It does **not** ship `handoff/` or `reports/`. Add those later only when the workflow needs them. Missing optional directories are ignored by the scanner. + +Under `hub.yaml` schema 2 the layout is declarative: `knowledge` and `projects` must be declared under `paths` — a missing declaration fails loud — while every other section (such as `tools`) is optional and may live outside the Hub. + +There is no mode switch: the scanner follows the directories that exist — `hub.yaml` has no `profile` concept, and a leftover key from an older version is ignored like any unknown top-level manifest key. Knowledge subzones are discovered, not enumerated: every first-level directory under the knowledge root (`general/`, `projects/`, `machines/`, ...) is scanned and index-checked, whatever it is named. A fuller hub is also scanned across `register/`, `handoff/`, `reports/`, `output/`, `.agents/plans/` and `strategy.md` when present. Legacy layouts — `provision/`, `manuals/`, `cron-tasks/`, `openspec/`, `archive/` and the `Human Output/` alias of `output/` — are compat-scanned when present only; they are never structures to create. Generated directories such as `out/` are not knowledge sources. `--scope hub` excludes local wiki and machine-level secrets. + +## Workflow + +1. Check whether a hub is already attached: + +```bash +guster hub status +``` + +2. Create a new lite hub: + +```bash +guster hub init --path --yes +``` + +This refuses a non-empty target and attaches the new hub to `~/.agents/guster/config.json`. + +Choose custom core paths at creation time: + +```bash +guster hub init --path --knowledge notes --projects work --tools scripts --yes +``` + +Relative paths stay inside the Hub (no `..`, no absolute); any section may instead be declared external (absolute or `~/`). To change them later, edit the `paths` map in `hub.yaml`, then run `guster hub validate`. + +The same manifest carries `doctor.waivers`: findings the owner knowingly accepts, each with a `reason` and an optional `path`. Keep them in the hub so the decision is versioned next to the corpus it protects. + +3. Validate and score it: + +```bash +guster hub validate +guster scan +guster doctor --scope hub +``` + +4. Add content: + +- Create one topic per file under any first-level subzone of `knowledge/` (`general/`, `projects/`, ...). +- Add the file name to `knowledge/README.md`. +- Register a project as a row in `register/projects.yaml` (id, status, as_of, name, spec.path); spec.path must resolve to the project directory on this machine. +- Add a tool only when it is a run-once command: one directory with its own `AGENTS.md`, then a row in `tools/README.md`. +- Put `verified_at: YYYY-MM-DD` in every state document (`最后更新:` is the + accepted Chinese form; `verified_at` is canonical — see the format spec). + `verified_at` is the state-document freshness key only; it is not one of + the CTX-09 header keys. +- Optional structured header (CTX-09): front-matter that declares `type:` plus + a date key opts the note into machine checks. The accepted date keys are + `updated`/`date`/`lastmod`/`created` as YYYY-MM-DD (Obsidian properties + convention). Keep truth sources as body markdown links, never as invented + header keys. +- YAML knowledge files may opt into the knowledge/v1 contract (CTX-11) by declaring `schema: knowledge/v1` with `id/type/status/topic/as_of/updated` metadata; opted-in files get strict format, see_also cross-reference and freshness checks. The scaffold ships an all-commented skeleton (`knowledge/general/knowledge-v1-skeleton.txt`) showing every field. + +## Existing hub + +If the directory already exists, use: + +```bash +guster hub attach --path +``` + +`attach` validates the core layout and backs up the Guster config before changing the hub path. Legacy hubs without `hub.yaml` are accepted with warnings; `guster hub validate` is the strict lite-contract check. Upgrading a 0.9.3 hub: create `register/projects.yaml` (header `schema: register/v1`, `kind: projects`, `updated:` date, one row per project — the old `projects/_registry.md` table), delete `_registry.md`, then re-validate; see CHANGELOG 0.9.4. + +## Boundaries + +- Never copy private content from another hub into a new one. +- Never add credentials, company secrets, generated caches or scratch files. +- Do not add optional modules just to fill the tree. +- Do not write Hub content from the governance skill; propose durable notes to + the user and let them (or their capture workflow of choice) file them. diff --git a/external_plugins/guster/skills/guster/SKILL.md b/external_plugins/guster/skills/guster/SKILL.md new file mode 100644 index 00000000..0e6a5d20 --- /dev/null +++ b/external_plugins/guster/skills/guster/SKILL.md @@ -0,0 +1,150 @@ +--- +name: guster +description: Audit and evolve agent context across projects, skills, tools, knowledge and credential references. Use when the user says 上下文治理/项目上下文/上下文结构/上下文格式/格式规范/整理 .agents/归位/搬迁/断链/健康评分/凭据登记, or asks where a fact, file, skill or credential belongs, or wants to reformat scattered legacy context into the standard format, or wants a before/after context health score. +--- + +# Guster + +Guster is the control plane for agent context: it discovers projects, +instruction files, skills, tools, knowledge, state, evidence and credential +references; diagnoses drift; scores health; plans reviewable migrations. The +optional Guster Hub is the data plane (`guster-hub` skill, `guster hub init`). + +## When to use + +- Inspect or reorganize machine agent context; place a file, skill, tool, register row, note or credential reference. +- Audit drift, duplicates, broken pointers, budgets, stale state; health scores and before/after comparisons. +- Initialize the standard `.agents/` layout; onboard a new project; reformat + scattered legacy (pre-spec) context into the standard format. +- Install the same context rules on another machine or workspace. +- Not for ordinary code edits, debugging or writing unless the request is about context structure itself. + +## Scope + +Four levels. `--scope` is inferred from `--root` when omitted. + +| Level | Command | Covers | Scored dimensions | +|---|---|---|---| +| machine | `guster doctor` | every repo under the workspace roots, declared `zones` and the hub | SSR RBR PCR ICR FRS PHR SIR | +| hub | `guster doctor --scope hub` | knowledge, the project register, tools, reports, handoffs | SSR ICR FRS PHR | +| group | `guster doctor --root A --root B [--root C]` | a requirement family or any set of related repos | SSR PCR ICR FRS PHR | +| project | `guster doctor --root ` | one repository | SSR PCR ICR FRS PHR | + +- Machine level owns resident budgets, skill budgets and credential layout; + hub level owns knowledge indexing, register freshness and hub pointers. +- Group exposes cross-repo facts (same stem different content, duplicates); + project shows only that repo plus how many machine findings were hidden. +- Scoped runs write to `~/.agents/guster/scoped//`, never the + machine-wide registry or trend. Pointer checks skip doc templates + (`draft-NNN.md`, `daily-*.md`, `0X-xxx.md`) and `~/` on `ssh`/`scp`/ + `rsync` command lines. + +## Workflow + +1. `guster status` +2. `guster scan --json` (add `--root ` for a single project) +3. `guster doctor` +4. `guster score` +5. `guster plan findings` or a targeted `guster plan move` +6. Read the plan with the user, then `guster apply --yes` +7. `guster doctor` again (zero new findings = contract kept), then `guster score --trend` + +Workspace writes are gated behind a plan and `--yes`; scans refresh +`registry.json` (`doctor` adds `findings.json`, `score` appends +`metrics.jsonl`) — `--no-write` skips that for `scan`, `doctor`, `score` +and `check`. `guster check` runs the patrol in one pass. + +## Commands + +```bash +guster status +guster scan --root --json +guster doctor +guster score --trend | --fail-under 80 # --fail-under: exit 2 below target +guster check # scan + doctor + score --trend +guster zones list|add |remove +guster plan findings +guster plan move --from --to +guster apply --yes +guster init +guster init --project --yes +guster hub init --path --yes +guster hub attach|status|validate +guster export --format md +guster credentials +``` + +## Gates + +- **Never persist or print credential values.** Paths, variable names and redacted forms only. +- **Never auto-commit, auto-push, auto-deploy; never apply a plan without explicit confirmation.** +- **Never treat tracked files you do not own as ordinary migration targets** — + the test is ownership/authority over them, not the word "company". +- **Fail loud** on an unknown workspace, malformed plan or missing backup. +- Prefer no-focus, CLI-only operation; if the scan finds a credential in + git, stop structural work and report it as critical. + +## Waivers + +A finding you knowingly accept is waived in `/hub.yaml` under `doctor.waivers`, never by deleting the check: + +```yaml +doctor: + waivers: + - rule: broken-pointer + path: ~/work/company-repo + reason: company-tracked doc, owned upstream +``` + +`path` is optional; without it the whole rule is waived. Every waiver needs +a `reason` — a malformed block fails loud. A waiver greens the gate, not the +score: waived findings stay visible in `doctor --json` and skip the exit +code, but CHI is computed before waivers apply — waive only what you cannot fix. + +## Format spec + +Context assets follow the versioned contract in `reference/spec.md` +(CTX-01..11): instruction files, llms.txt-shaped indexes, machine-verifiable +pointers and dates, the six `.agents/` slots, one home per fact, skill +budgets, credentials by reference, register/v1, opt-in knowledge/v1 — a doctor +finding names the broken rule; schema contracts live in `schemas/`. + +## New project onboarding + +Orchestration of existing commands — the user confirms every structural step (conventions: `reference/placement.md`): + +1. `guster status --json`: read configured `roots`, `zones`, attached hub. +2. Propose a placement under a root the user governs, per their own naming + conventions; they confirm the path before anything is created. +3. `mkdir` + `git init` per the user's git conventions (Guster never runs + git); scaffold with `guster init --project `, then `--yes`. +4. Draft a register/v1 row (`id`/`status`/`as_of`/`name`/`spec.path`) for the + hub's `register/projects.yaml`; the user pastes it — Guster never edits hub content. +5. `guster scan` + `guster doctor`: zero new findings is done. + +## Reformatting legacy context + +For an inherited pre-spec mess, run the four-pass playbook in +`reference/reformat.md`: inventory and classify (doctor findings as the +worklist) → placement (`guster init --project ` for the slots, `plan +move` or a batch plan) → content (indexes, links, dates, pointer notes) → +verify (`doctor` zero new findings, `score --trend`). Deletions and +distillations are proposed for approval, never silent. + +## References + +- `reference/spec.md` — format spec v2 (CTX rules and their findings). +- `reference/reformat.md` — four-pass legacy reformat playbook. +- `reference/framework.md` — 1-3-5-8 framework and placement tree. +- `reference/metrics.md` — metric definitions, targets and CHI. +- `reference/placement.md` — directory trees and the git boundary. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | success — clean run, or findings no worse than `medium` | +| `1` | tool/config error: malformed or unreadable plan, an apply that failed mid-operation and was rolled back, missing workspace root, invalid `config.json` (non-list `roots`, a bad root/zone/hub path, empty `roots`, no scan candidates — run `guster init`); `hub status` also returns 1 when the attached hub fails its checks | +| `2` | `doctor`/`check` found `high` (but no `critical`) findings; `score --fail-under` below target or CHI not computable; `hub validate` failed the lite contract | +| `3` | `doctor`/`check` found `critical` findings | +| `4` | `apply`, `init` or `hub init` refused the plan at preflight — nothing was touched (missing move source, existing destination, invalid `hub init` target; an `init --project` target that only fails mid-apply exits 1 after rollback) | diff --git a/external_plugins/guster/skills/guster/reference/framework.md b/external_plugins/guster/skills/guster/reference/framework.md new file mode 100644 index 00000000..ed77c737 --- /dev/null +++ b/external_plugins/guster/skills/guster/reference/framework.md @@ -0,0 +1,55 @@ +# Guster Framework + +## One principle + +One fact has one home; one home tells one story; show it only when needed. + +## Three laws + +1. **Single home** — one canonical source per topic; other locations are + pointers, generated copies or immutable evidence. +2. **Load on demand** — resident instructions only for rules that apply to + every task in scope, change default behavior, and have costly failure. +3. **Layer isolation** — a container holds content with the same scope, + authority, lifetime, sensitivity and load mode. + +## Five sameness checks + +- scope: global / workgroup / project / subdirectory / task / session +- authority: platform / instruction / code / runtime / knowledge / evidence +- lifetime: invariant / monthly / weekly / daily / hourly / one-shot +- sensitivity: public / internal / confidential / secret +- load: resident / run-start / triggered / retrieved / never + +If one container mixes two values on any axis, split it. + +## Eight planes + +1. Truth: code, tests, build and runtime config. +2. Instruction: AGENTS.md and policies. +3. Capability: skills, tools, MCP, plugins, scripts. +4. Knowledge: durable principles, methods, business semantics, pitfalls. +5. State: the project register, handoffs, environment and port registries. +6. Evidence: reports, screenshots, videos, test output. +7. Runtime: sessions, caches, logs, temp files, browser profiles. +8. Secret: passwords, tokens, keys, cookies; references only. + +## Placement tree + +1. Secret? → 8. +2. Code/config? → 1. +3. Changes default behavior for all tasks? → 2. +4. Repeatable action/procedure? → 3. +5. Current status/next step? → 5. +6. Proof/artifact? → 6. +7. Durable understanding? → 4. +8. Temporary working data? → 7. + +If content hits multiple planes, split it into conclusion + evidence + pointer. + +## Conflict rules + +- Safety: strictest wins. +- Fact: closest to runtime/code wins. +- Preference: most specific scope wins. +- Same level: do not guess; fix the missing canonical source. diff --git a/external_plugins/guster/skills/guster/reference/metrics.md b/external_plugins/guster/skills/guster/reference/metrics.md new file mode 100644 index 00000000..39b97f68 --- /dev/null +++ b/external_plugins/guster/skills/guster/reference/metrics.md @@ -0,0 +1,62 @@ +# Guster Metrics + +Every number is a static count: files, bytes, characters, dates, path +resolution and git metadata. Nothing measures runtime token usage. + +## Core metrics + +| ID | Metric | Target | Numerator / denominator | +|---|---|---:|---| +| SSR | single-source rate | ≥95% | topics with one source / all topics (filename stem + content hash) | +| RBR | resident budget ratio | ≤100% | bytes of existing host entry files + hub `AGENTS.md` over 20 KB; skill descriptions over 6,000 chars | +| PCR | placement compliance rate | ≥90% | `.agents/` dirs with all six slots, a non-empty index and no unknown top-level entry / dirs that have `.agents/` (workspace containers excluded) | +| ICR | index coverage rate | knowledge 100%, `.agents` ≥90% | mean of indexed knowledge files and `.agents/` dirs with an index | +| FRS | state freshness rate | ≥90% | state docs verified within 90 days / state docs | +| PHR | pointer health rate | 100% | resolvable pointers / resolvable-looking pointers | +| SIR | secret isolation rate | ≥95% | credential files that are unregistered, tracked, unverifiable or inline / credential files + inline tokens. Inline means host config, not arbitrary file content | + +## Score transforms + +- RBR: each sub-ratio (entry bytes / 20 KB, skill chars / 6,000) becomes + `min(1, 1/ratio)` and the two transformed scores are averaged — that score is + what enters CHI. The report's Value column shows the mean of the raw ratios + instead: budget used, lower is better. +- ICR: an unweighted mean of two heterogeneous coverage ratios (knowledge + files indexed, `.agents/` dirs with an index). When one denominator dwarfs + the other, the equal-weight average lets the small side dominate the + dimension. + +## Targets + +- Resident: one ≤20 KB budget for the entry files that exist (deduplicated by + real path, host symlinks do not double count) plus the hub instruction file. + Soft line budgets are 150 per user entry and 50 for the hub. +- Skills: ≤6,000 characters of description in total, ≤500 per skill. +- State: one 90-day SLA for every state document — hub `projects/`, `register/`, + `handoff/`, `provision/` and `.agents/plans/`. There is no separate 7- or 30-day tier. + The date is `verified_at`, a `最后更新` line in the first 40 lines, a date that + opens the filename, or the owning repository's last commit — that last one is + "touched", not "verified". +- Knowledge: index every durable file; review every 90–180 days. +- Secrets: 0 in git; 1 canonical root; 0 inline tokens. A file kept in place is + accepted when `registry.md` declares it; an unreadable repository counts as + unverified, never as clean. + +## No evidence means n/a + +An empty denominator produces `n/a`, not 1.0: the dimension is dropped and the +remaining weights are renormalized. A scope with no evidence at all reports +`CHI: n/a`. Scores from scopes with different coverage are not comparable. + +## CHI + +The Context Health Index is a dashboard, not a KPI: + +```text +CHI = 0.20 SSR + 0.15 RBR + 0.15 PCR + 0.15 ICR + + 0.15 FRS + 0.10 PHR + 0.10 SIR +``` + +Critical findings do not rescale the score. They set the exit code: `doctor` +returns 3 on any critical and 2 on any high finding. `score --fail-under N` +turns the index into an explicit gate. diff --git a/external_plugins/guster/skills/guster/reference/placement.md b/external_plugins/guster/skills/guster/reference/placement.md new file mode 100644 index 00000000..bbdbccfb --- /dev/null +++ b/external_plugins/guster/skills/guster/reference/placement.md @@ -0,0 +1,60 @@ +# Guster Placement + +## User level + +```text +~/.agents/ +├── skills/ # shared capabilities +├── wiki/ # local/sensitive knowledge +├── credentials/ # secret root (target) +├── tmp/ # scratch +└── archive/ +``` + +User instructions remain a single canonical file with host symlinks. + +## Guster Hub + +The standard skeleton — what `guster hub init` scaffolds: + +```text +/ +├── AGENTS.md # instruction +├── README.md # index +├── hub.yaml # manifest (schema 2: knowledge and projects declared, tools optional) +├── register/ # state — project register (projects.yaml, CTX-10) +├── projects/ # state +├── knowledge/ +│ ├── README.md # knowledge index +│ ├── general/ # starter note + knowledge/v1 skeleton; subzones are discovered, not fixed +│ └── projects/ # empty starter subzone +├── tools/ # capability — index only, no tool preinstalled +└── .agents/plans/ # local plans +``` + +Optional state and evidence directories — `handoff/`, `reports/`, `output/` +(legacy alias `Human Output/`) — are added when the workflow needs them, never +by the scaffold; the scanner picks them up when present. + +## Project level + +```text +/.agents/ +├── README.md # required index +├── plans/ # active plans +├── wiki/ # durable project knowledge +├── evidence/ # screenshots, videos, test output +├── skills/ # project-local skills +├── tmp/ # 14-day TTL +└── archive/ # superseded material +``` + +Topic directories are allowed only under a slot, e.g. +`evidence/brand-report-20261028/`, not at `.agents/` top level. + +## Git boundary + +- Portable, non-sensitive knowledge → hub `knowledge/` and git. +- Local paths, accounts, production IDs, sensitive pitfalls → `~/.agents/wiki/`. +- Plans, evidence, scratch → project `.agents/`, ignored. +- Credentials → secret root only; never git, docs, reports or chat. diff --git a/external_plugins/guster/skills/guster/reference/reformat.md b/external_plugins/guster/skills/guster/reference/reformat.md new file mode 100644 index 00000000..a49072e6 --- /dev/null +++ b/external_plugins/guster/skills/guster/reference/reformat.md @@ -0,0 +1,75 @@ +# Legacy Context Reformat + +The playbook for bringing a scattered, pre-spec context site under the format +contract (`spec.md`) — essay-style instructions, unindexed knowledge, naked +paths, undated state, duplicates, top-level task folders. The agent judges; +this page fixes the order of operations. + +Principles: **meaning-preserving** (never invent facts or rewrite for style); +**classify before touching** (formatting garbage launders it); **placement +before content** (a pointer written before its target moves is rewritten +after); **one scope, one pass** (leave it internally consistent). + +## Pass 1 — Inventory and classify + +1. `guster doctor --root ` (add `--json`): the findings list is the + worklist, each naming the CTX rule it broke; `guster scan` adds the assets. +2. Classify every artifact — flagged or not — into exactly one class: + + | Class | Destination | + |---|---| + | instruction | root `AGENTS.md`, narrowed per CTX-01 | + | durable knowledge | hub `knowledge/` or `.agents/wiki/` (project-local) | + | plan / task bundle | `.agents/plans/` | + | evidence | `.agents/evidence/` | + | skill | `.agents/skills/` or `~/.agents/skills/` | + | scratch | `.agents/tmp/` — or delete | + | retired | `.agents/archive/` — or distill, then delete | + | code or config | the repository itself, not context | + | no future reader | propose deletion, never format | + +3. Duplicate stems (CTX-06): pick the canonical copy by freshness and + completeness; the rest become pointer cards in pass 3. A credential value + found anywhere stops structural work: report critical (CTX-08). + +## Pass 2 — Placement + +- No `.agents/` yet: `guster init --project ` gives the six slots, README and a minimal `AGENTS.md`. +- Before any move, search the repo for the artifact's path stem: every + referencer found now is a pointer you update in pass 3 (doctor after the + move catches the rest). The slot a `misplaced-artifact` finding suggests + is shape-based — classification can override it. +- One artifact: `guster plan move --from --to ` (backup on apply; referencers updated in pass 3, no pointer written). +- A batch: hand-write one plan JSON — `version: 1`, an `operations` list of + `mkdir`/`move`/`write` objects, same shape `plan move` emits — then + dry-run, read it with the user, `--yes`. Apply backs up every touched path + and rolls back on failure. + +## Pass 3 — Content (per scope, table order) + +| Symptom | Transform | Rule | +|---|---|---| +| Scope has files but no index | Write the llms.txt-shaped README: H1, one-line blockquote, one link per file | CTX-02 | +| Knowledge file missing from its index | Add `- [title](path): one-line description` — the description says when to read it | CTX-02 | +| Naked path in prose or index | Markdown link in indexes, `` `code span` `` in prose, when-to-read in the same sentence | CTX-03 | +| State document without a machine-readable date | Add `verified_at: YYYY-MM-DD` / `最后更新: YYYY-MM-DD` — today only if you re-verified, otherwise the git date | CTX-04 | +| Task bundle at `.agents/` top level | Move into its slot (pass 2) | CTX-05 | +| Non-canonical duplicate | Replace with a pointer card: front-matter `type: pointer` plus a date key, truth links in the body; volatile values are deleted, not copied | CTX-06, CTX-09 | +| Instruction file over budget or contradicting a narrower one | Cut or split — nesting narrows, never contradicts | CTX-01 | +| Referencer still pointing at a pre-move path | Update it to the new path (doctor reports each as `broken-pointer`); `~`-prefixed links resolve against home, so cross-repo pointers to a hub stay verifiable | CTX-03 | +| Leftover `.moved.md` transition pointer (older guster versions wrote them on apply) | Delete once every referencer is updated — a transition pointer nobody reads re-triggers `misplaced-artifact`; keep it only for referencers outside your control | CTX-05 | + +Content edits are ordinary file edits by the agent; plans are for moves. + +## Pass 4 — Verify + +1. `guster doctor` — the pass kept the contract iff zero new findings appear. +2. `guster score --trend` for the before/after number; committing stays the + user's workflow — guster never auto-commits. + +## Scale and review + +For a large legacy site, deliver the pass-1 classification table (artifact → +class → keep/merge/delete) to the user before pass 2 — one approval for the +map, not one per file; deletions never execute silently. A company-tracked +file is never a migration target: report it, waive it, move on. diff --git a/external_plugins/guster/skills/guster/reference/spec.md b/external_plugins/guster/skills/guster/reference/spec.md new file mode 100644 index 00000000..149a396b --- /dev/null +++ b/external_plugins/guster/skills/guster/reference/spec.md @@ -0,0 +1,165 @@ +# Context Format Spec + +spec_version: 2. The format contract for context assets — the shapes that let +both an agent and `guster doctor` verify a document without reading minds. +Each rule carries a stable ID (markdownlint style) and names the finding that +mechanically enforces it. What no finding enforces is guidance for the agent +that writes the document. + +## Open conventions this spec follows + +- **AGENTS.md** (agents.md) — the open format for the tool-agnostic + instruction file. Discovery is nested: hosts collect `AGENTS.md` files + walking outward from the working directory and the nearest one wins. On + that precedence CTX-01 stacks a stricter discipline: a nested file narrows + the parent, never contradicts it. Guster reads `AGENTS.md` (and legacy + `CLAUDE.md`) wherever a host reads them and does not invent a competing + instruction format; a host that only knows `CLAUDE.md` is served with + `ln -s AGENTS.md CLAUDE.md`. +- **llms.txt** (llmstxt.org) — index files as curated link lists. A knowledge + index is an H1, a one-line blockquote summary, then one markdown link per + file with a one-line description. Same shape, local files instead of URLs. +- **YAML front-matter** — the `---`-delimited metadata block; the de-facto + standard Jekyll, Hugo and Gray Matter share. CTX-09 rides it rather than + inventing a note format. +- **Stable rule IDs** (markdownlint) — rules are numbered, cited by findings + and waivers, and only change meaning on a spec_version bump. + +## Rules + +| ID | Rule | Enforced by | +|---|---|---| +| CTX-01 | `AGENTS.md` is the instruction file; nested ones narrow, never contradict. Budgets: hub ≤50 lines, user entry ≤150 lines, resident total ≤20 KB. | `resident-budget` | +| CTX-02 | Every knowledge scope has a README index in llms.txt shape; every knowledge file appears there verbatim, as `- [title](path): one-line description`; an equivalent two-column markdown table (backticked path + topic) is accepted. An index that names nothing is empty. | `orphan-knowledge` | +| CTX-03 | Any file path in an instruction or index document is machine-verifiable: a markdown link or inline code, never naked in prose. A pointer also states when to read the target. | `broken-pointer`, `naked-pointer` | +| CTX-04 | State documents carry a machine-readable date. The canonical key is `verified_at: YYYY-MM-DD`; the Chinese prose keys (`最后更新`/`更新时间`/`更新于`/`最近核实`/`最后核实`/`已核实`) and the English prose keys (`Last updated:`/`updated:`) are accepted equivalents within the first 40 lines, as is a date that opens the filename. Without a declared date the ladder falls back to the repository's last git commit, then to file mtime; no signal at all means stale. | `stale-state` | +| CTX-05 | `.agents/` has exactly the six slots — `plans/`, `wiki/`, `evidence/`, `skills/`, `tmp/`, `archive/` — plus `README.md`/`AGENTS.md`. Task bundles live in a slot, never at the top level. | `misplaced-artifact` | +| CTX-06 | One home per fact. Same topic, same filename stem; the second home becomes a pointer to the first. Copies are either byte-identical or a mistake. | `duplicate-canonical`, `skill-copy-drift` | +| CTX-07 | Skill frontmatter carries `name` and `description` only; one trigger sentence, ≤500 chars each, ≤6,000 chars across the library. | `skill-budget` | +| CTX-08 | Credentials appear by reference — a path in the registry under the canonical root — never as a value in a document, a config or git. | `secret-in-git`, `secret-outside-root`, `secret-inline`, `secret-undeclared`, `secret-missing` | +| CTX-09 | A knowledge note may carry YAML front-matter following the Obsidian properties convention (the interchange format Foam/Quartz/Logseq also accept). The header is opt-in: it is machine-checked only when it declares `type:`. A checked header must parse and carry a machine-readable date (`updated`/`date`/`lastmod`/`created` as YYYY-MM-DD), and the note's body then joins the CTX-03 pointer scan — truth sources go in the body as links, never as invented header keys. Optional keys: `name` and `description` state what the note is (the one-fact-per-file shape inspired by the Claude Code memory tool), and `superseded_by` names the path or stem of the note that replaced this one; that key is metadata about the note, not a truth source, so it never joins the pointer scan and never raises a finding. | `note-header-malformed`, `broken-pointer`, `naked-pointer` | +| CTX-10 | The hub's project register (`register/projects.yaml`) follows the register/v1 format: a `schema: register/v1` + `kind: projects` + `updated:` header, then one row per project — top-level `- id:` list items under the `projects:` key — with `id` (unique), `status` (active, pending, on-hold, stale, retired; optional but in-vocabulary), `as_of` (YYYY-MM-DD, optional; checked when present), `name` (optional, never validated) and `spec.path`. v1 validates this one table; a register/ YAML that declares register/v1 with another `kind:` gets an info finding, not errors. `spec.path` is the project location — `~/...`, absolute, or relative to the hub root — and must resolve to a directory on this machine; `./` is the legal placeholder for a row with no directory of its own. Unknown row keys and spec subkeys are ignored and never validated; the vocabulary only grows — semantics change only on a spec_version bump. Remote forms (`host:/path`, URLs) are counted apart, never resolved. | `registry-drift`, `registry-format` | +| CTX-11 | A YAML knowledge file may declare `schema: knowledge/v1` to opt into the structured knowledge contract. An opted-in file must carry, at the top level: `id` (equal to the file stem), `type` (state, howto, principle, pitfall, decision, index, reference), `status` (active, dormant, superseded, archived; optional but must be in-vocabulary), `topic` (non-empty one-liner), `as_of` and `updated` (YYYY-MM-DD, `updated` ≥ `as_of`), and optionally `see_also` — an inline list of **ids of other files in the same knowledge root** (bare ids only; path-like references belong in the body under CTX-03). Local extensions (zone, host, history, ...) are allowed and ignored. Freshness: active `state`/`index` files older than 90 days (`updated`) raise `knowledge-stale`. | `knowledge-format`, `see-also-broken`, `duplicate-knowledge-id`, `knowledge-stale` | + +### CTX-10 in detail + +The register is read through `spec.path`, so a row is only as good as its +path. Conventions that keep the three-way diff (machine table `registry.json` +↔ this register ↔ git roots on disk) meaningful: + +- One project per row; `spec.path` names the project **directory** (or a + knowledge-backed entry such as a topic file — `spec.path: ./` is the legal + placeholder when the row owns no directory), not an arbitrary file. +- `~/`-relative paths resolve against the user's home; bare relative paths + resolve against the **hub root**; absolute paths resolve as-is. +- A malformed header (wrong `schema`/`kind`), a duplicated `id`, a missing + `spec.path` or a non-date `as_of` raises `registry-format` (medium); so does + a register that parses zero rows while column-0 `id:`/`name:` keys betray a + flat single-project map. The header `updated:` date is not part of that + check — strict `guster hub validate` reports it instead, doctor does not. +- Unknown row keys and `spec` subkeys (owner, repo, priority, tags, ...) are + ignored and never validated; the vocabulary only grows — semantics change + only on a spec_version bump. +- A row whose path does not exist on disk is a dead row (`registry-drift`, + medium); a row that exists but is never scanned points outside the + configured roots (medium); a scanned project with no row is unregistered + (low). Remote references are counted apart instead of drifting forever. +- register/v1 validates `kind: projects` only. Another YAML under register/ + that declares register/v1 with a different `kind:` reports an info finding: + accepted as register data, never validated. + +### CTX-11 in detail + +The knowledge/v1 contract is opt-in and metadata-only: the plugin reads the +column-0 keys of the YAML file (scalars, one inline list for `see_also`) and +never parses the `body`. That keeps the check dependency-free and the prose +free-form. A minimal complete file, saved as a `.yaml` under the knowledge +root (e.g. `knowledge/general/deploy-checklist.yaml`): + +```yaml +schema: knowledge/v1 +id: deploy-checklist +type: howto +status: active +topic: One-line topic +as_of: 2026-01-01 +updated: 2026-01-01 +see_also: [ci-pipeline] +body: | + Free-form prose; the checker never parses past the metadata face. +``` + +- `see_also` accepts bare ids; each id must belong to a file that also + declares knowledge/v1 in the same knowledge root (reference a plain note + and the checker cannot see its id — a see_also id whose file exists without + the header says so in the finding). Path-like or remote references are + left to CTX-03 and never flagged here. +- One id, one file: the same id in two files (even across subzones) raises + `duplicate-knowledge-id`. +- A file that declares a `knowledge/*` or `register/*` schema other than + knowledge/v1 reports an info finding (declares X — current supported is + knowledge/v1): a typo'd declaration stays visible instead of silently + opting out of every check. +- Freshness applies only to active `state` and `index` types — principles and + references do not rot. The 90-day window is fixed, not configurable from + hub.yaml; setting `status: dormant` (or `superseded`/`archived`) is the + only way to take a file out of the SLA deliberately. +- Subzones under the knowledge root are discovered, not enumerated: + `general/`, `projects/`, `machines/` or any first-level directory is scanned + and index-checked the same way. + +### Credential registry table + +CTX-08's registry (`~/.agents/credentials/registry.md`) is a markdown table +with a fixed column order; only paths and variable **names** live here, +never values. When the file is missing (or parses zero rows) the +`secret-undeclared`/`secret-missing` checks do not apply — create the table +with its header plus at least one row to enable them: + +| # | Column | Required | +|---|---|---| +| 1 | ID | yes | +| 2 | scope (`company`/`personal`) | yes | +| 3 | site | yes | +| 4 | account | yes | +| 5 | kind | yes | +| 6 | locations | yes | +| 7 | keys (variable names) | no | +| 8 | used_by | no | +| 9 | status | no | + +A pipe row with fewer than the six required columns is a parse warning, not +silence; `guster credentials` prints it. + +## Why a format contract + +The metrics are static: filename stems, content hashes, path resolution, git +dates. A document that ignores the contract is not scored lower — it is +invisible. A naked path in prose never reaches the pointer check; a date +written as "last week" falls back to git; an index without filenames counts as +missing. The contract is what makes measurement possible, and the score is +what keeps the contract from drifting. Both directions are load-bearing. + +## Writing guide (no finding enforces this) + +When creating or moving a context asset: + +1. Pick the home by CTX-06, the slot by CTX-05, the index entry by CTX-02; a + new project gets its register row by CTX-10. +2. Write pointers as `[label](relative/path.md)` in indexes and `` `path` `` + in prose; add when-to-read in the same sentence. +3. Add the date line before saving a state document; run `guster doctor` + after edits — a new finding names the rule you just broke. +4. A note that summarizes knowledge whose truth lives elsewhere (a pointer + card) opts into CTX-09: standard front-matter (`type: pointer`, a date + key, `tags` if useful) plus a truth section of ordinary markdown links. + `name` and `description` are optional metadata; so is `superseded_by`, + which marks the note a better one replaced — the chain is for readers; + the scanner never resolves it. Volatile values (latest numbers, current + picks) stay out of the card and are read from the truth sources instead. + +## Versioning + +Additive changes keep spec_version; a rule that changes meaning bumps it and +gets a CHANGELOG entry. Findings cite rule IDs, waivers accept them, so a +spec change never silently re-targets an existing waiver. diff --git a/external_plugins/guster/src/guster/__init__.py b/external_plugins/guster/src/guster/__init__.py new file mode 100644 index 00000000..5a93f106 --- /dev/null +++ b/external_plugins/guster/src/guster/__init__.py @@ -0,0 +1,3 @@ +"""Guster: agent context steward.""" + +__version__ = "0.9.5" diff --git a/external_plugins/guster/src/guster/__main__.py b/external_plugins/guster/src/guster/__main__.py new file mode 100644 index 00000000..a049ad7a --- /dev/null +++ b/external_plugins/guster/src/guster/__main__.py @@ -0,0 +1,5 @@ +from .cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/external_plugins/guster/src/guster/cli.py b/external_plugins/guster/src/guster/cli.py new file mode 100644 index 00000000..fef3f788 --- /dev/null +++ b/external_plugins/guster/src/guster/cli.py @@ -0,0 +1,1014 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from . import __version__ +from .discover import ScanConfig, count_zone_tools, default_roots, scan +from .executor import ApplyError, ConfigError, apply_plan, init_home +from .hub import attach_hub, default_hub_path, init_hub, validate_hub +from .metrics import compute_metrics, metric_line +from .planner import ( + load_plan, + plan_from_findings, + plan_init_project, + plan_move, + save_plan, +) +from .policy import run_policy +from .report import redacted_registry, render_report, render_status +from .util import ( + json_dumps, + load_json_strict, + now_iso, + save_json, + state_lock, + write_text, +) + + +def default_state_dir(home: Path) -> Path: + return home / ".agents" / "guster" + + +def state_dir_from(args: argparse.Namespace) -> Path: + """Explicit --state-dir or the machine default; used by commands that do + not need a full scan config (init/apply/plan/hub/zones).""" + if args.state_dir: + return Path(args.state_dir).expanduser() + return default_state_dir(Path.home()) + + +def load_config(config_path: Path) -> Dict[str, Any]: + if not config_path.is_file(): + return {} + try: + local = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise ValueError(f"cannot read Guster config {config_path}: {exc}") from exc + if not isinstance(local, dict): + raise ValueError(f"Guster config must be a JSON object: {config_path}") + return local + + +def resolve_root(root: Path) -> Path: + if not root.exists(): + raise ValueError(f"workspace root does not exist: {root.resolve()}") + if not root.is_dir(): + raise ValueError(f"workspace root is not a directory: {root.resolve()}") + return root + + +def parse_roots(raw: Any, config_path: Path) -> List[Path]: + """Config roots must be existing absolute directories; one bad entry stops + the run instead of being silently reinterpreted (a bare string is a type + error, never a list of characters).""" + if not isinstance(raw, list): + raise ValueError( + f"Guster config 'roots' must be a list of absolute paths, " + f"got {type(raw).__name__}: {config_path}" + ) + if not raw: + raise ValueError(f"Guster config 'roots' must list at least one path: {config_path}") + roots: List[Path] = [] + for item in raw: + if not isinstance(item, str) or not item.strip(): + raise ValueError( + f"Guster config 'roots' entries must be non-empty strings: {item!r} in {config_path}" + ) + path = Path(item).expanduser() + if not path.is_absolute(): + raise ValueError( + f"Guster config 'roots' entries must be absolute paths: {item!r} in {config_path}" + ) + roots.append(resolve_root(path)) + return roots + + +def parse_zones(raw: Any, config_path: Path) -> List[Dict[str, str]]: + """Only 'path' is read; extra keys a legacy config may still carry (e.g. a + retired 'profile') are tolerated silently.""" + if not isinstance(raw, list): + raise ValueError(f"Guster config 'zones' must be a list: {config_path}") + zones: List[Dict[str, str]] = [] + for item in raw: + if not isinstance(item, dict) or not str(item.get("path", "")).strip(): + raise ValueError(f"zone entry needs a 'path' in {config_path}: {item!r}") + zone_path = Path(str(item["path"])).expanduser() + if not zone_path.is_absolute(): + raise ValueError( + f"Guster config 'zones' entries must be absolute paths: {item!r} in {config_path}" + ) + if not zone_path.is_dir(): + raise ValueError(f"configured zone does not exist: {zone_path.resolve()}") + zones.append({"path": str(zone_path)}) + return zones + + +def add_common(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--root", action="append", default=None, help="Workspace root; repeatable.") + parser.add_argument( + "--scope", + choices=["auto", "machine", "hub", "project", "group"], + default="auto", + help="Context scope: machine (default), hub, one project, or a group of projects.", + ) + parser.add_argument("--hub", default=None, help="Guster Hub path.") + parser.add_argument("--state-dir", default=None, help="Guster state directory.") + parser.add_argument("--json", action="store_true", help="Emit JSON.") + parser.add_argument("--quiet", action="store_true", help="Only print errors.") + + +def cli_roots(raw: Optional[Sequence[str]]) -> List[Path]: + """--root values follow the config roots discipline: existing absolute dirs, + so the persisted registry never carries a path that drifts with CWD.""" + roots: List[Path] = [] + for item in raw or []: + path = Path(item).expanduser() + if not path.is_absolute(): + raise ValueError(f"--root paths must be absolute: {item!r}") + roots.append(resolve_root(path)) + return roots + + +def parse_max_depth(raw: Any, config_path: Path) -> int: + if isinstance(raw, bool) or not isinstance(raw, int) or raw <= 0: + raise ValueError( + f"Guster config 'scan_max_depth' must be a positive integer, " + f"got {raw!r} ({type(raw).__name__}) in {config_path}" + ) + return raw + + +def build_config(args: argparse.Namespace) -> ScanConfig: + home = Path.home() + base_state = Path(args.state_dir).expanduser() if args.state_dir else default_state_dir(home) + config_path = base_state / "config.json" + unconfigured = not config_path.is_file() + local = load_config(config_path) + + hub: Optional[Path] = None + if args.hub: + hub = Path(args.hub).expanduser() + elif "hub" in local: + raw_hub = local["hub"] + if not isinstance(raw_hub, str) or not raw_hub.strip(): + raise ValueError(f"Guster config 'hub' must be a path string: {config_path}") + hub = Path(raw_hub).expanduser() + if hub is not None: + if not hub.exists(): + raise ValueError(f"hub directory does not exist: {hub.resolve()}") + if not hub.is_dir(): + raise ValueError(f"hub is not a directory: {hub.resolve()}") + + explicit_roots = cli_roots(args.root) + scope = getattr(args, "scope", "auto") or "auto" + if scope == "auto": + scope = "machine" if not explicit_roots else ("project" if len(explicit_roots) == 1 else "group") + if scope in {"project", "group"} and not explicit_roots: + raise ValueError("--scope project|group needs at least one --root") + if scope == "hub" and hub is None: + raise ValueError("--scope hub needs --hub or an attached hub") + auto_roots = False + if scope == "hub": + roots: List[Path] = [] + elif explicit_roots: + roots = explicit_roots + elif "roots" in local: + roots = parse_roots(local.get("roots"), config_path) + else: + roots = default_roots(home) + if not roots: + raise ValueError( + "no workspace roots configured and none of " + "~/Projects, ~/code, ~/dev, ~/src exists — " + "run `guster init --root ` to configure Guster" + ) + auto_roots = True + + state_dir = base_state + if not args.state_dir and (scope != "machine" or explicit_roots): + # Scoped runs keep their own registry and trend so they never overwrite + # the machine-wide view. + key = "|".join([scope] + sorted(str(root.resolve()) for root in roots) + [str(hub or "")]) + state_dir = base_state / "scoped" / hashlib.sha1(key.encode()).hexdigest()[:10] + + plugin_root = Path(__file__).resolve().parents[2] + zones = parse_zones(local.get("zones", []), config_path) + + return ScanConfig( + home=home, + hub=hub, + roots=roots, + state_dir=state_dir, + plugin_root=plugin_root, + max_depth=parse_max_depth(local.get("scan_max_depth", 5), config_path), + scope=scope, + zones=zones, + unconfigured=unconfigured, + auto_roots=auto_roots, + ) + + +def ensure_state(config: ScanConfig) -> None: + config.state_dir.mkdir(parents=True, exist_ok=True) + (config.state_dir / "plans").mkdir(parents=True, exist_ok=True) + (config.state_dir / "backups").mkdir(parents=True, exist_ok=True) + + +def load_previous_registry(config: ScanConfig) -> Optional[Dict[str, Any]]: + """The persisted machine table this scan replaces, captured before the new + registry overwrites it; unreadable content reads as None so run_policy's + own strict disk read (and its corrupt-baseline finding) stays in charge.""" + data, error = load_json_strict(config.state_dir / "registry.json") + if error is not None or not isinstance(data, dict): + return None + return data + + +def do_scan( + config: ScanConfig, write: bool = True +) -> Tuple[Dict[str, Any], Optional[Dict[str, Any]]]: + """Scan fresh; the second value is the pre-overwrite registry handed to + run_policy as the drift baseline.""" + ensure_state(config) + with state_lock(config.state_dir): + previous = load_previous_registry(config) + registry = scan(config) + if write: + save_json(config.state_dir / "registry.json", registry) + return registry, previous + + +def load_registry(config: ScanConfig) -> Optional[Dict[str, Any]]: + """Cached registry; corrupt content warns and reads as absent so `status` + rescans instead of silently self-healing.""" + path = config.state_dir / "registry.json" + data, error = load_json_strict(path) + if error is not None: + print(f"[guster] warning: {path} is corrupt ({error}); rescanning", file=sys.stderr) + return None + if data is None: + return None + if not isinstance(data, dict): + print(f"[guster] warning: {path} is not a JSON object; rescanning", file=sys.stderr) + return None + return data + + +def evaluate( + registry: Dict[str, Any], + config: ScanConfig, + previous_registry: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + policy_result = run_policy(registry, config, previous_registry=previous_registry) + metric_result = compute_metrics(registry, policy_result) + return {"policy": policy_result, "metrics": metric_result} + + +def append_metric(config: ScanConfig, metric_result: Dict[str, Any]) -> None: + ensure_state(config) + with state_lock(config.state_dir): + path = config.state_dir / "metrics.jsonl" + line = { + "at": metric_result.get("generated_at", now_iso()), + "scope": metric_result.get("scope"), + "chi": metric_result.get("chi"), + "dims": metric_result.get("dims"), + "unscored_dims": metric_result.get("unscored_dims"), + "metrics": metric_result.get("metrics", {}), + "scores": metric_result.get("scores", {}), + } + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(line, ensure_ascii=False) + "\n") + + +def last_metric(config: ScanConfig) -> Optional[Dict[str, Any]]: + """Newest persisted snapshot; `status` reports this instead of mixing a stale scan with fresh policy.""" + path = config.state_dir / "metrics.jsonl" + if not path.is_file(): + return None + lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + if not lines: + return None + try: + item = json.loads(lines[-1]) + except ValueError as exc: + print(f"[guster] warning: {path} last line is corrupt ({exc}); no cached metric shown", file=sys.stderr) + return None + if not isinstance(item, dict): + print(f"[guster] warning: {path} last line is not a JSON object; no cached metric shown", file=sys.stderr) + return None + item.setdefault("generated_at", item.get("at")) + return item + + +def write_findings(config: ScanConfig, policy_result: Dict[str, Any]) -> None: + ensure_state(config) + with state_lock(config.state_dir): + save_json(config.state_dir / "findings.json", policy_result) + + +def print_scan_summary(registry: Dict[str, Any], config: ScanConfig) -> None: + summary = registry.get("summary", {}) + print("Guster scan") + print(f" workspaces: {', '.join(registry.get('workspaces', []))}") + print(f" hub: {registry.get('hub') or '(none)'}") + if config.scope == "hub": + print( + f" registered: {summary.get('registry_entries', 0)} " + f"(scanned roots: {summary.get('projects', 0)})" + ) + else: + print(f" projects: {summary.get('projects', 0)}") + print(f" .agents: {summary.get('agents_dirs', 0)} (indexed {summary.get('agents_with_index', 0)})") + skills_note = "" if config.scope == "hub" else " (user-level roots only)" + print(f" skills: {summary.get('skills', 0)} unique / {summary.get('skill_copies', 0)} copies{skills_note}") + print(f" knowledge: {summary.get('knowledge', 0)}") + print(f" state: {summary.get('state', 0)}") + print(f" evidence: {summary.get('evidence', 0)}") + print(f" credentials:{summary.get('secret_files', 0)} files; {summary.get('inline_bearers', 0)} inline") + print(f" registry: {config.state_dir / 'registry.json'}") + + +def cmd_status(args: argparse.Namespace) -> int: + config = build_config(args) + registry = load_registry(config) + cached = registry is not None + if registry is None: + registry, previous = do_scan(config, write=True) + policy_result: Optional[Dict[str, Any]] = None + if cached: + # The cached registry describes an earlier scan. Running policy against + # it would mix old artifacts with fresh file contents, so report the + # stored snapshot and let `doctor` own the fresh evaluation. + metric_result = last_metric(config) + else: + result = evaluate(registry, config, previous_registry=previous) + policy_result = result["policy"] + metric_result = result["metrics"] + if args.json: + print(json_dumps({ + "registry": { + "plugin": registry.get("plugin"), + "generated_at": registry.get("generated_at"), + "hub": registry.get("hub"), + "workspaces": registry.get("workspaces"), + "summary": registry.get("summary"), + }, + "metrics": metric_result, + "cached": cached, + "findings": None if policy_result is None else len(policy_result["findings"]), + "scope": registry.get("scope"), + "state_dir": str(config.state_dir), + })) + else: + print(render_status(registry, metric_result)) + if policy_result is None: + print("cached registry; run `guster doctor` for a fresh evaluation") + elif metric_result is not None: + print(metric_line(metric_result)) + print(f"findings: {len(policy_result['findings'])}") + return 0 + + +def cmd_scan(args: argparse.Namespace) -> int: + config = build_config(args) + registry, _ = do_scan(config, write=not args.no_write) + if args.json: + print(json_dumps(registry)) + elif not args.quiet: + print_scan_summary(registry, config) + return 0 + + +def cmd_doctor(args: argparse.Namespace) -> int: + config = build_config(args) + registry, previous = do_scan(config, write=not args.no_write) + result = evaluate(registry, config, previous_registry=previous) + policy_result = result["policy"] + if not args.no_write: + write_findings(config, policy_result) + findings = policy_result["findings"] + if args.json: + print(json_dumps(policy_result)) + elif not args.quiet: + print(render_report(registry, policy_result, result["metrics"])) + severities = {item["severity"] for item in findings} + if "critical" in severities: + return 3 + if "high" in severities: + return 2 + return 0 + + +def metrics_reference_line(config: ScanConfig) -> str: + """The metrics reference as an absolute pointer into the installed plugin; + a generic fallback when that layout does not ship the file.""" + if config.plugin_root is None: + return "Definitions & targets: see the guster skill reference (metrics.md)" + reference = config.plugin_root / "skills" / "guster" / "reference" / "metrics.md" + if reference.is_file(): + return f"Definitions & targets: {reference}" + return "Definitions & targets: see the guster skill reference (metrics.md)" + + +def cmd_score(args: argparse.Namespace) -> int: + config = build_config(args) + registry, previous = do_scan(config, write=not args.no_write) + result = evaluate(registry, config, previous_registry=previous) + metric_result = result["metrics"] + if not args.no_write: + append_metric(config, metric_result) + fail_reason: Optional[str] = None + if args.fail_under is not None: + chi = metric_result.get("chi") + if chi is None: + fail_reason = "CHI is not computable; no dimension has evidence" + elif chi < args.fail_under: + fail_reason = f"CHI {chi} is below --fail-under {args.fail_under}" + if fail_reason is not None and args.json: + print(json_dumps({"ok": False, "error": fail_reason, "metrics": metric_result})) + return 2 + if args.json: + print(json_dumps(metric_result)) + elif not args.quiet: + chi = metric_result.get("chi") + headline = f"Guster Context Health: {'n/a' if chi is None else chi} / 100" + unscored = metric_result.get("unscored_dims") or [] + if unscored: + scored = len(metric_result.get("scored_dims") or []) + total = len(metric_result.get("dims") or []) + headline += f" ({scored}/{total} dims scored)" + print(headline) + print("") + dims = set(metric_result.get("dims") or []) + for key, value in metric_result["scores"].items(): + if dims and key not in dims: + continue + print(f" {key.upper():4s} {'n/a' if value is None else f'{value:6.1f}'}") + print("") + print(f"registry: {config.state_dir / 'registry.json'}") + print(f"metrics: {config.state_dir / 'metrics.jsonl'}") + print(metrics_reference_line(config)) + if args.trend: + path = config.state_dir / "metrics.jsonl" + if path.is_file(): + lines = path.read_text(encoding="utf-8").splitlines()[-10:] + print("") + print("Recent trend:") + for line in lines: + try: + item = json.loads(line) + chi = item.get("chi") + print(f" {item.get('at')} CHI {'n/a' if chi is None else chi}") + except ValueError: + continue + if fail_reason is not None: + print(f"error: {fail_reason}", file=sys.stderr) + return 2 + return 0 + + +def cmd_check(args: argparse.Namespace) -> int: + """scan -> doctor -> score --trend in one pass; the exit code carries the + doctor semantics (3 critical, 2 high, 0 clean).""" + config = build_config(args) + registry, previous = do_scan(config, write=not args.no_write) + result = evaluate(registry, config, previous_registry=previous) + policy_result = result["policy"] + if not args.no_write: + write_findings(config, policy_result) + append_metric(config, result["metrics"]) + if args.json: + print(json_dumps({"policy": policy_result, "metrics": result["metrics"]})) + elif not args.quiet: + print(render_report(registry, policy_result, result["metrics"])) + path = config.state_dir / "metrics.jsonl" + if path.is_file(): + lines = [line for line in path.read_text(encoding="utf-8").splitlines() if line.strip()][-10:] + if lines: + print("") + print("Recent trend:") + for line in lines: + try: + item = json.loads(line) + chi = item.get("chi") + print(f" {item.get('at')} CHI {'n/a' if chi is None else chi}") + except ValueError: + continue + severities = {item["severity"] for item in policy_result["findings"]} + if "critical" in severities: + return 3 + if "high" in severities: + return 2 + return 0 + + +def cmd_plan(args: argparse.Namespace) -> int: + if args.plan_action == "findings": + config = build_config(args) + ensure_state(config) + registry, previous = do_scan(config, write=True) + result = evaluate(registry, config, previous_registry=previous) + plan = plan_from_findings(result["policy"]["findings"]) + state_dir = config.state_dir + elif args.plan_action == "move": + src = Path(args.source).expanduser() + dst = Path(args.destination).expanduser() + plan = plan_move(src, dst) + state_dir = state_dir_from(args) + else: + raise SystemExit(f"unknown plan action: {args.plan_action}") + + path = save_plan(state_dir, plan) + if args.json: + print(json_dumps(plan)) + elif not args.quiet: + print(f"plan: {path}") + print(f"operations: {len(plan.get('operations', []))}") + for index, operation in enumerate(plan.get("operations", []), start=1): + kind = operation.get("kind") + if kind == "move": + print(f" {index}. move {operation['src']} -> {operation['dst']}") + elif kind == "mkdir": + print(f" {index}. mkdir {operation['path']}") + elif kind == "write": + print(f" {index}. write {operation['path']}") + elif kind == "manual": + print(f" {index}. manual {operation['path']}: {operation['action']}") + print("review the plan; apply with `guster apply --yes`.") + return 0 + + +def cmd_apply(args: argparse.Namespace) -> int: + plan = load_plan(Path(args.plan).expanduser()) + result = apply_plan( + plan, + state_dir_from(args), + dry_run=not args.yes, + yes=args.yes, + ) + if args.json: + print(json_dumps(result)) + elif not args.quiet: + if result.get("dry_run"): + mode = "DRY RUN" + elif result.get("ok"): + mode = "APPLIED" + else: + mode = "FAILED" + print(f"{mode}: {result.get('plan')}") + print(f" applied: {result.get('applied', 0)}") + print(f" skipped: {result.get('skipped', 0)}") + if result.get("backup"): + print(f" backup: {result['backup']}") + if result.get("errors"): + for error in result["errors"]: + print(f" error: {error}") + return 0 if result.get("ok") else 4 + + +def cmd_init(args: argparse.Namespace) -> int: + state_dir = state_dir_from(args) + if args.project: + project = Path(args.project).expanduser() + plan = plan_init_project(project) + path = save_plan(state_dir, plan) + result = apply_plan( + plan, + state_dir, + dry_run=not args.yes, + yes=args.yes, + extra_bases=[project.resolve()], + ) + if not args.json and not args.quiet: + print(f"plan: {path}") + if result.get("dry_run"): + missing: List[Path] = [] + probe = project.resolve() + while not probe.exists() and probe != probe.parent: + missing.append(probe) + probe = probe.parent + if missing: + print(" these directories do not exist yet and will be created:") + for entry in sorted(missing, key=lambda item: len(item.parts)): + print(f" {entry}") + for index, operation in enumerate(plan.get("operations", []), start=1): + kind = operation.get("kind") + if kind == "mkdir": + print(f" {index}. mkdir {operation['path']}") + elif kind == "write": + print(f" {index}. write {operation['path']}") + else: + explicit_roots = cli_roots(args.root) + result = init_home( + state_dir, + yes=args.yes, + dry_run=not args.yes, + home=Path.home(), + roots=explicit_roots or None, + ) + if args.json: + print(json_dumps(result)) + elif not args.quiet: + if result.get("dry_run"): + mode = "DRY RUN" + elif result.get("ok"): + mode = "APPLIED" + else: + mode = "FAILED" + print(f"{mode}: init") + if result.get("dry_run"): + for index, operation in enumerate(result.get("operations") or [], start=1): + kind = operation.get("kind") + if kind == "mkdir": + print(f" {index}. mkdir {operation['path']}") + elif kind == "write": + print(f" {index}. write {operation['path']}") + else: + print(f" applied: {result.get('applied', 0)}") + print(f" skipped: {result.get('skipped', 0)}") + if result.get("errors"): + for error in result["errors"]: + print(f" error: {error}") + return 0 if result.get("ok") else 4 + + +def cmd_hub(args: argparse.Namespace) -> int: + state_dir = state_dir_from(args) + action = args.hub_action + local = load_config(state_dir / "config.json") + attached: Optional[Path] = None + if "hub" in local: + raw_hub = local["hub"] + if not isinstance(raw_hub, str) or not raw_hub.strip(): + raise ValueError( + f"Guster config 'hub' must be a path string: {state_dir / 'config.json'}" + ) + attached = Path(raw_hub).expanduser() + if attached is None and not getattr(args, "path", None) and not getattr(args, "hub", None): + if action == "status": + if args.json: + print(json_dumps({"attached": False, "hub": None})) + elif not args.quiet: + print("no hub attached") + return 0 + if action == "validate": + raise ValueError("no hub attached; pass --path to validate one") + target = Path( + getattr(args, "path", None) + or getattr(args, "hub", None) + or attached + or default_hub_path(Path.home()) + ).expanduser() + + if action == "init": + try: + result = init_hub( + target, + state_dir, + yes=args.yes, + dry_run=not args.yes, + name=args.name, + knowledge=args.knowledge, + projects=args.projects, + tools=args.tools, + ) + except ConfigError as exc: + # Config discipline errors (invalid config.json roots/hub) follow the + # exit-code table's tool/config-error row, not the preflight refusal. + if args.json: + print(json_dumps({"ok": False, "error": str(exc)})) + else: + print(f"[guster] {exc}", file=sys.stderr) + return 1 + except ValueError as exc: + # init_hub raises before building the plan (bad target type, a + # non-empty target, an invalid layout): nothing was touched, which + # is the preflight-refusal contract behind exit code 4. + if args.json: + print(json_dumps({"ok": False, "error": str(exc)})) + else: + print(f"[guster] {exc}", file=sys.stderr) + return 4 + if args.json: + print(json_dumps(result)) + elif not args.quiet: + if result.get("dry_run"): + mode = "DRY RUN" + elif result.get("ok"): + mode = "APPLIED" + else: + mode = "FAILED" + print(f"{mode}: hub {result['hub']}") + print(f" plan: {result.get('plan')}") + print(f" applied: {result.get('applied', 0)}") + print(f" skipped: {result.get('skipped', 0)}") + if result.get("attach"): + print(f" config: {result['attach']['config']}") + paths = result.get("paths") or {} + if paths: + print( + " paths: " + f"knowledge={paths['knowledge']} " + f"projects={paths['projects']} " + f"tools={paths['tools']}" + ) + if result.get("errors"): + for error in result["errors"]: + print(f" error: {error}") + return 0 if result.get("ok") else 4 + + if action == "attach": + result = attach_hub(state_dir, target) + if args.json: + print(json_dumps(result)) + elif not args.quiet: + print(f"attached: {result['hub']}") + print(f" config: {result['config']}") + if result.get("backup"): + print(f" backup: {result['backup']}") + print(" run `guster scan` to refresh the registry with the hub") + return 0 + + info = validate_hub(target, strict=action == "validate") + if args.json: + print(json_dumps(info)) + elif not args.quiet: + manifest = info.get("manifest") or {} + print(f"Hub: {info['hub']}") + if manifest: + print( + "Manifest: " + f"schema={manifest.get('schema_version') or '(absent)'} " + f"hub={manifest.get('hub_version') or '(absent)'} " + f"min_guster={manifest.get('min_guster') or '(absent)'}" + ) + else: + print("Manifest: (absent)") + print(f"Plugin: {info['plugin']}") + paths = info.get("paths") or {} + if paths: + print( + "Paths: " + f"knowledge={paths.get('knowledge', '(absent)')} " + f"projects={paths.get('projects', '(absent)')} " + f"tools={paths.get('tools', '(absent)')}" + ) + if info["errors"]: + for error in info["errors"]: + print(f" error: {error}") + for warning in info["warnings"]: + print(f" warning: {warning}") + if info["ok"]: + print("Checks: OK") + if action == "validate": + return 0 if info["ok"] else 2 + return 0 if info["ok"] else 1 + + +def cmd_zones(args: argparse.Namespace) -> int: + state_dir = state_dir_from(args) + config_path = state_dir / "config.json" + config = load_config(config_path) + action = args.zones_action + + if action == "list": + zones = parse_zones(config.get("zones", []), config_path) + if args.json: + print(json_dumps({"zones": zones})) + elif not args.quiet: + if not zones: + print("no zones configured; add one with `guster zones add `") + for item in zones: + zone_path = Path(item["path"]).expanduser() + print(f"{item['path']} tools={count_zone_tools(zone_path)}") + return 0 + + zone_path = Path(args.path).expanduser() + raw_zones = config.get("zones", []) + if not isinstance(raw_zones, list): + raise ValueError(f"Guster config 'zones' must be a list: {config_path}") + + if action == "add": + if not zone_path.is_dir(): + raise ValueError(f"zone directory does not exist: {zone_path.resolve()}") + for item in raw_zones: + if isinstance(item, dict) and str(item.get("path", "")).strip(): + if Path(str(item["path"])).expanduser().resolve() == zone_path.resolve(): + raise ValueError(f"zone already configured: {zone_path.resolve()}") + entry = {"path": str(zone_path)} + config.setdefault("version", 1) + config["zones"] = raw_zones + [entry] + zones = parse_zones(config["zones"], config_path) + state_dir.mkdir(parents=True, exist_ok=True) + save_json(config_path, config) + if args.json: + print(json_dumps({"ok": True, "added": entry, "zones": zones, "config": str(config_path)})) + elif not args.quiet: + print(f"added zone: {entry['path']}") + print(f"config: {config_path}") + return 0 + + if action == "remove": + remaining = [ + item + for item in raw_zones + if not ( + isinstance(item, dict) + and str(item.get("path", "")).strip() + and Path(str(item["path"])).expanduser().resolve() == zone_path.resolve() + ) + ] + if len(remaining) == len(raw_zones): + raise ValueError(f"zone is not configured: {zone_path.resolve()}") + if remaining: + config["zones"] = remaining + else: + config.pop("zones", None) + state_dir.mkdir(parents=True, exist_ok=True) + save_json(config_path, config) + if args.json: + print(json_dumps({"ok": True, "removed": str(zone_path), "config": str(config_path)})) + elif not args.quiet: + print(f"removed zone: {zone_path}") + print(f"config: {config_path}") + return 0 + + raise SystemExit(f"unknown zones action: {action}") + + +def cmd_export(args: argparse.Namespace) -> int: + config = build_config(args) + registry, previous = do_scan(config, write=True) + result = evaluate(registry, config, previous_registry=previous) + out = Path(args.out).expanduser() if args.out else config.state_dir / "context-health.md" + if args.format == "json": + out = out.with_suffix(".json") + save_json(out, { + "registry": redacted_registry(registry), + "policy": result["policy"], + "metrics": result["metrics"], + }) + else: + write_text(out, render_report(registry, result["policy"], result["metrics"])) + if not args.quiet: + print(f"report: {out}") + return 0 + + +def cmd_credentials(args: argparse.Namespace) -> int: + config = build_config(args) + registry, _ = do_scan(config, write=True) + info = (registry.get("secrets") or {}).get("registry") or {} + entries = info.get("entries") or [] + scopes: Dict[str, List[Dict[str, str]]] = {} + for entry in entries: + scopes.setdefault(entry.get("scope", "unknown"), []).append(entry) + if args.json: + print(json_dumps({"path": info.get("path"), "entries": entries})) + return 0 + if not args.quiet: + print(f"registry: {info.get('path') or '(missing)'}") + for warning in info.get("parse_warnings") or []: + print(f" warning: {warning}") + for scope in ("company", "personal", "unknown"): + items = scopes.pop(scope, []) + if not items: + continue + label = {"company": "company 公司", "personal": "personal 个人", "unknown": "unknown 未分类"}[scope] + print(f"\n{label}({len(items)})") + for entry in items: + keys = f" [{entry['keys']}]" if entry.get("keys") not in {"", "-"} else "" + print(f" {entry['id']:24} {entry['site'][:16]:18} {entry['kind']:8} {entry['status']}{keys}") + for scope, items in scopes.items(): + print(f"\n{scope}({len(items)})") + for entry in items: + print(f" {entry['id']:24} {entry['site'][:16]:18} {entry['kind']:8} {entry['status']}") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="guster", + description="Discover, audit, plan and evolve agent context.", + epilog="Scans refresh Guster's state files; workspace writes need a plan with --yes.", + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + subparsers = parser.add_subparsers(dest="command", required=True) + + common = argparse.ArgumentParser(add_help=False) + add_common(common) + + status = subparsers.add_parser("status", parents=[common], help="Show plugin, registry and health status.") + status.set_defaults(func=cmd_status) + + scan_parser = subparsers.add_parser("scan", parents=[common], help="Discover context assets.") + scan_parser.add_argument("--no-write", action="store_true", help="Do not write registry.json.") + scan_parser.set_defaults(func=cmd_scan) + + doctor = subparsers.add_parser("doctor", parents=[common], help="Run policy checks.") + doctor.add_argument("--no-write", action="store_true", help="Do not write registry.json or findings.json.") + doctor.set_defaults(func=cmd_doctor) + + score = subparsers.add_parser("score", parents=[common], help="Compute metrics and CHI.") + score.add_argument("--no-write", action="store_true", help="Do not write registry.json or append metrics.jsonl.") + score.add_argument("--trend", action="store_true", help="Show the last 10 metric snapshots.") + score.add_argument( + "--fail-under", + type=float, + default=None, + metavar="CHI", + help="Exit 2 when CHI is below this value (opt-in gate for CI).", + ) + score.set_defaults(func=cmd_score) + + check = subparsers.add_parser( + "check", + parents=[common], + help="Run scan, doctor and score --trend in one pass.", + ) + check.add_argument("--no-write", action="store_true", help="Do not write registry, findings or metrics.") + check.set_defaults(func=cmd_check) + + zones_parser = subparsers.add_parser("zones", help="List or manage configured machine zones.") + zones_sub = zones_parser.add_subparsers(dest="zones_action", required=True) + zones_list = zones_sub.add_parser("list", parents=[common], help="List configured zones.") + zones_list.set_defaults(func=cmd_zones) + zones_add = zones_sub.add_parser("add", parents=[common], help="Add a zone to config.json.") + zones_add.add_argument("path", help="Zone directory (must exist).") + zones_add.set_defaults(func=cmd_zones) + zones_remove = zones_sub.add_parser("remove", parents=[common], help="Remove a configured zone.") + zones_remove.add_argument("path", help="Zone directory as configured.") + zones_remove.set_defaults(func=cmd_zones) + + plan = subparsers.add_parser("plan", help="Create a reviewable plan.") + plan_sub = plan.add_subparsers(dest="plan_action", required=True) + plan_findings = plan_sub.add_parser("findings", parents=[common], help="Turn findings into a review checklist.") + plan_findings.set_defaults(func=cmd_plan) + plan_move_parser = plan_sub.add_parser("move", parents=[common], help="Plan a move with backup.") + plan_move_parser.add_argument("--from", dest="source", required=True, help="Source path.") + plan_move_parser.add_argument("--to", dest="destination", required=True, help="Destination path.") + plan_move_parser.set_defaults(func=cmd_plan) + + apply_parser = subparsers.add_parser("apply", parents=[common], help="Apply a plan.") + apply_parser.add_argument("plan", help="Plan JSON path.") + apply_parser.add_argument("--yes", action="store_true", help="Actually write; without it this is a dry run.") + apply_parser.set_defaults(func=cmd_apply) + + init = subparsers.add_parser("init", parents=[common], help="Initialize Guster home or a project layout.") + init.add_argument("--project", default=None, help="Project root; omit to initialize the Guster state home.") + init.add_argument("--yes", action="store_true", help="Actually write; without it this is a dry run.") + init.set_defaults(func=cmd_init) + + hub = subparsers.add_parser("hub", help="Create, attach or inspect a Guster Hub.") + hub_sub = hub.add_subparsers(dest="hub_action", required=True) + hub_init = hub_sub.add_parser("init", parents=[common], help="Create a lite Guster Hub.") + hub_init.add_argument("--path", default=None, help="Hub directory; defaults to the attached hub or the conventional Guster Hub path.") + hub_init.add_argument("--name", default=None, help="Hub display name.") + hub_init.add_argument("--knowledge", default="knowledge", help="Knowledge root relative to the hub.") + hub_init.add_argument("--projects", default="projects", help="Project state root relative to the hub.") + hub_init.add_argument("--tools", default="tools", help="Tool root relative to the hub.") + hub_init.add_argument("--yes", action="store_true", help="Actually write; without it this is a dry run.") + hub_init.set_defaults(func=cmd_hub) + hub_attach = hub_sub.add_parser("attach", parents=[common], help="Attach an existing hub to Guster config.") + hub_attach.add_argument("--path", default=None, help="Hub directory; defaults to the attached hub or the conventional Guster Hub path.") + hub_attach.set_defaults(func=cmd_hub) + hub_status = hub_sub.add_parser("status", parents=[common], help="Show hub manifest and validation status.") + hub_status.add_argument("--path", default=None, help="Hub directory; defaults to the attached hub or the conventional Guster Hub path.") + hub_status.set_defaults(func=cmd_hub) + hub_validate = hub_sub.add_parser("validate", parents=[common], help="Validate a hub against the lite contract.") + hub_validate.add_argument("--path", default=None, help="Hub directory; defaults to the attached hub or the conventional Guster Hub path.") + hub_validate.set_defaults(func=cmd_hub) + + export = subparsers.add_parser("export", parents=[common], help="Export a redacted health report.") + export.add_argument("--format", choices=["md", "json"], default="md") + export.add_argument("--out", default=None, help="Output path.") + export.set_defaults(func=cmd_export) + + creds = subparsers.add_parser("credentials", parents=[common], help="List registered credentials by scope.") + creds.set_defaults(func=cmd_credentials) + + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + return int(args.func(args)) + except (ApplyError, OSError, ValueError) as exc: + if getattr(args, "json", False): + print(json_dumps({"ok": False, "error": str(exc)})) + else: + print(f"[guster] {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/external_plugins/guster/src/guster/discover.py b/external_plugins/guster/src/guster/discover.py new file mode 100644 index 00000000..5ebb2313 --- /dev/null +++ b/external_plugins/guster/src/guster/discover.py @@ -0,0 +1,1102 @@ +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple + +from . import __version__ +from .layout import load_governance_docs, resolve_hub_layout +from .util import ( + EXCLUDE_DIRS, + find_git_roots, + first_heading, + git_tracked, + is_indexed, + is_under, + iter_files, + load_json, + now_iso, + parse_frontmatter, + parse_register_rows, + read_text, + relative, + sha256_file, + sha256_text, +) + + +STANDARD_AGENTS_SLOTS = ["plans", "wiki", "evidence", "skills", "tmp", "archive"] +ARTIFACT_KIND_PRIORITY = { + "instruction": 0, + "knowledge": 1, + "plan": 2, + "tool": 3, + "evidence": 4, + "state": 5, + "archive": 6, +} +GENERIC_TOPIC_STEMS = { + "readme", + "index", + "agents", + "notes", + "notesmd", + "proposal", + "tasks", + "design", + "spec", + "plan", + "todo", + "changelog", + "summary", + "skill", + "openspec", + "config", + "example", + "template", +} +SECRET_SUFFIX_NAMES = {".env", "config.json", "credentials.json", "credentials.local.json"} +INLINE_CONFIG_FILES = [ + "~/.codex/config.toml", + "~/.zcode/cli/config.json", + "~/.dsh/cordis.patch.yml", +] +INLINE_PATTERNS = [ + # Host configs carry tokens in more shapes than one: keep the detector + # narrow enough to stay quiet, broad enough not to miss a literal key. + r"\bBearer\s+([A-Za-z0-9._\-]{8,})", + r"\b(sk-[A-Za-z0-9]{16,})", + r"(?i)\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|secret[_-]?key)" + r"\s*[:=]\s*[\"']?([A-Za-z0-9._\-]{16,})", +] +SECRET_ENV_DEPTH = 2 +HASH_LIMIT = 8_000_000 + + +@dataclass +class ScanConfig: + home: Path + hub: Optional[Path] + roots: List[Path] + state_dir: Path + plugin_root: Optional[Path] = None + max_depth: int = 5 + scope: str = "machine" + now: str = field(default_factory=now_iso) + # Declared machine zones (`config.json` `zones`): {"path": ...}. Zones are + # tool areas — AGENTS/README artifacts and secrets, no project slots — so a + # non-git tool layer is governed without pretending each tool is a repository. + zones: List[Dict[str, str]] = field(default_factory=list) + # First-run UX: no config.json existed, and roots came from auto-detection. + # Drives the "run `guster init`" banner; empty candidates fail loud earlier. + unconfigured: bool = False + auto_roots: bool = False + + +def scope_roots(config: ScanConfig) -> Optional[List[Path]]: + """Directories that define the run's scope; None means the machine-wide view. + + The same boundary filters artifacts and findings, so a scoped report never + counts a document it can no longer raise a finding about. + """ + if config.scope == "hub": + return [config.hub.expanduser().resolve()] if config.hub else [] + if config.scope in {"project", "group"}: + return [root.expanduser().resolve() for root in config.roots] + return None + + +def in_scope_path(value: str, roots: Sequence[Path]) -> bool: + if not value: + return False + try: + candidate = Path(value).expanduser().resolve() + except OSError: + return False + return any(candidate == root or root in candidate.parents for root in roots) + + +def default_roots(home: Path) -> List[Path]: + """Generic first-run candidates; empty when none exist so callers fail loud.""" + candidates = [home / name for name in ("Projects", "code", "dev", "src")] + seen: Set[str] = set() + roots: List[Path] = [] + for candidate in candidates: + if candidate.is_dir(): + resolved = str(candidate.resolve()) + if resolved not in seen: + seen.add(resolved) + roots.append(candidate) + return roots + + +def dedupe_artifacts(items: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Keep one record per path so ratios count files, not scan passes.""" + best: Dict[str, Tuple[int, Dict[str, Any]]] = {} + order: List[str] = [] + for item in items: + path = item["path"] + rank = ARTIFACT_KIND_PRIORITY.get(str(item.get("kind")), 9) + current = best.get(path) + if current is None: + order.append(path) + best[path] = (rank, item) + elif rank < current[0]: + best[path] = (rank, item) + return [best[path][1] for path in order] + + +def topic_key(path: Path) -> str: + stem = path.stem.strip().lower().lstrip(".") + stem = re.sub(r"[-_ ]?20\d{6}$", "", stem) + stem = re.sub(r"[-_ ]?\d{4}-\d{2}-\d{2}$", "", stem) + return stem + + +def derive_topics( + artifacts: Sequence[Dict[str, Any]], + skills: Sequence[Dict[str, Any]], +) -> Dict[str, Any]: + """Group knowledge into facts and report the ones with more than one home.""" + knowledge_all = [ + item + for item in artifacts + if item.get("kind") == "knowledge" + and Path(item["path"]).name.lower() not in {"readme.md", "agents.md"} + ] + knowledge = [item for item in knowledge_all if item.get("hash")] + unhashed = sum(1 for item in knowledge_all if not item.get("hash")) + consumed: Set[str] = set() + topics: List[Dict[str, Any]] = [] + + by_stem: Dict[str, List[Dict[str, Any]]] = {} + for item in knowledge: + stem = topic_key(Path(item["path"])) + if not stem or stem in GENERIC_TOPIC_STEMS or len(stem) < 4: + continue + by_stem.setdefault(stem, []).append(item) + for stem, items in sorted(by_stem.items()): + hashes = {item["hash"] for item in items} + if len(hashes) > 1: + topics.append( + { + "key": stem, + "reason": "same-name-different-content", + "sources": len(hashes), + "paths": [item["path"] for item in items], + } + ) + consumed.update(item["path"] for item in items) + + by_hash: Dict[str, List[Dict[str, Any]]] = {} + for item in knowledge: + if item["path"] in consumed: + continue + by_hash.setdefault(item["hash"], []).append(item) + for digest, items in by_hash.items(): + topics.append( + { + "key": topic_key(Path(items[0]["path"])) or digest[:8], + "reason": "duplicate-content" if len(items) > 1 else "single", + "sources": len(items), + "paths": [item["path"] for item in items], + } + ) + + for skill in skills: + copies = skill.get("copies") or [] + hashes = {copy.get("hash", "") for copy in copies} or {skill.get("hash", "")} + topics.append( + { + "key": f"skill:{skill['name']}", + "reason": "skill-copy-drift" if len(hashes) > 1 else "single", + "sources": len(hashes) if len(hashes) > 1 else 1, + "paths": [copy.get("root", "") for copy in copies] or [skill.get("root", "")], + } + ) + + multi = [topic for topic in topics if int(topic.get("sources", 1)) > 1] + return { + "total": len(topics), + "multi": len(multi), + "multi_topics": multi, + "unhashed": unhashed, + } + + +def scan(config: ScanConfig) -> Dict[str, Any]: + home = config.home + hub = config.hub + scope = config.scope + roots = [root for root in config.roots if root.exists()] + git_roots = find_git_roots(roots, max_depth=config.max_depth) + project_roots: List[Path] = list(git_roots) + known = {item.resolve() for item in project_roots} + for root in roots: + resolved = root.resolve() + if resolved in known: + continue + if resolved == home.resolve() and not has_project_marker(root): + # `~/.agents/` is the user-level context root; the home directory is + # a workspace, not a project, unless it carries its own markers. + continue + if looks_like_project(root): + project_roots.append(root) + known.add(resolved) + + projects: List[Dict[str, Any]] = [] + project_tracked: Dict[str, Optional[Set[str]]] = {} + used_ids: Set[str] = set() + for root in sorted(project_roots, key=lambda item: str(item)): + tracked = git_tracked(root) + project_tracked[str(root)] = tracked + project = scan_project( + root=root, + group=root.parent.name, + tracked=tracked, + now=config.now, + taken=used_ids, + ) + used_ids.add(project["id"]) + projects.append(project) + + mark_containers(projects) + index_texts = load_index_texts(home, hub, projects) + + walk_errors: List[str] = [] + artifacts: List[Dict[str, Any]] = [] + zones = [ + item + for item in config.zones + if Path(item["path"]).expanduser().is_dir() + ] + if scope != "hub": + artifacts.extend(scan_project_agents(projects, index_texts, project_tracked, walk_errors)) + artifacts.extend(scan_zones(zones, walk_errors)) + artifacts.extend(scan_hub_knowledge(hub, index_texts, walk_errors)) + if scope != "hub": + artifacts.extend(scan_local_wiki(home, index_texts, walk_errors)) + artifacts = dedupe_artifacts(artifacts) + scoped = scope_roots(config) + if scoped is not None: + # Scoped runs must not carry artifacts whose findings they filter out: + # that inflates ICR/FRS/PHR denominators and depresses the score. + artifacts = [item for item in artifacts if in_scope_path(item["path"], scoped)] + + skills = scan_skills(home) if scope != "hub" else [] + topics = derive_topics(artifacts, skills) + secret_files = scan_secret_files( + home, + hub, + projects, + project_tracked, + zones=zones, + include_home=scope != "hub", + errors=walk_errors, + ) + credential_registry = scan_credential_registry(home) + inline_bearers = scan_inline_bearers(home) if scope != "hub" else [] + governance_docs = find_governance_docs(home, hub) + if governance_docs is not None and scope == "hub" and hub: + governance_docs = [ + item for item in governance_docs if is_under(Path(item), hub) + ] + manifests = scan_plugin_manifests(config.plugin_root) if scope != "hub" else {} + + registry: Dict[str, Any] = { + "version": 1, + "plugin": {"name": "guster", "version": __version__}, + "generated_at": config.now, + "scope": config.scope, + "home": str(home), + "hub": str(hub) if hub else None, + "state_dir": str(config.state_dir), + "unconfigured": config.unconfigured, + "auto_roots": config.auto_roots, + "workspaces": [str(root) for root in roots], + "scan_errors": {"count": len(walk_errors), "examples": walk_errors[:10]}, + "zones": [ + { + "path": str(Path(item["path"]).expanduser()), + "tools": count_zone_tools(Path(item["path"]).expanduser()), + } + for item in zones + ], + "projects": projects, + "skills": skills, + "topics": topics, + "artifacts": artifacts, + "secrets": { + "canonical_roots": [ + str(home / ".agents" / "credentials"), + ], + "files": secret_files, + "inline_bearers": inline_bearers, + "registry": credential_registry, + }, + "governance_docs": governance_docs, + "plugin_manifests": manifests, + "summary": summarize( + projects, + skills, + artifacts, + secret_files, + inline_bearers, + count_registry_entries(hub, walk_errors), + ), + } + return registry + + +def mark_containers(projects: List[Dict[str, Any]]) -> None: + """A directory holding other scanned projects is a workspace/group, not a project. + + Group-level AGENTS.md files are instruction assets of the group; counting the + container as a project inflates the project count and turns one container's + partial `.agents/` into a permanent placement failure. + """ + roots = [Path(project["path"]) for project in projects] + for project in projects: + root = Path(project["path"]) + project["container"] = any(root in other.parents for other in roots if other != root) + + +def looks_like_project(root: Path) -> bool: + return ( + has_project_marker(root) + or (root / ".agents").is_dir() + or (root / "skills").is_dir() + ) + + +def has_project_marker(root: Path) -> bool: + return ( + (root / "AGENTS.md").is_file() + or (root / "CLAUDE.md").is_file() + or (root / "package.json").is_file() + ) + + +def scan_project( + root: Path, + group: str, + tracked: Optional[Set[str]], + now: str, + taken: Optional[Set[str]] = None, +) -> Dict[str, Any]: + agent_files = find_agent_files(root, max_depth=3) + agents_dir = root / ".agents" + agents_entries: List[str] = [] + standard_slots: List[str] = [] + has_index = False + if agents_dir.is_dir(): + try: + agents_entries = sorted( + item.name for item in agents_dir.iterdir() if item.name != ".DS_Store" + ) + except OSError: + agents_entries = [] + standard_slots = [slot for slot in STANDARD_AGENTS_SLOTS if (agents_dir / slot).exists()] + has_index = agents_index_present(agents_dir) + + status = "unknown" + for agent_file in agent_files: + text = read_text(agent_file, limit=200_000) + match = re.search(r"\*\*状态\*\*\s*[::]\s*(.+)", text) + if match: + status = match.group(1).strip() + break + + project_id = unique_project_id(root.name, root, taken) + return { + "id": project_id, + "path": str(root), + "group": group, + "git_root": str(root), + "agent_files": [relative(path, root) for path in agent_files], + "agents_dir": str(agents_dir) if agents_dir.is_dir() else None, + "agents_entries": agents_entries, + "standard_slots": standard_slots, + "has_index": has_index, + "skills": find_project_skills(root), + "tools": find_project_tools(root), + "status": status, + "last_scan": now, + "container": False, + "git_state": "ok" if tracked is not None else "error", + "tracked_count": len(tracked) if tracked is not None else None, + } + + +def agents_index_present(agents_dir: Path) -> bool: + """An index must have content; an empty placeholder indexes nothing.""" + for name in ("README.md", "AGENTS.md"): + path = agents_dir / name + try: + if path.is_file() and path.stat().st_size > 0: + return True + except OSError: + continue + return False + + +def unique_project_id(name: str, root: Path, taken: Optional[Set[str]] = None) -> str: + """Slug id; a same-name sibling gets a short path-hash suffix so + `project:{id}` index keys never overwrite each other.""" + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", name.strip()).strip("-") + base = cleaned or sha256_text(str(root))[:10] + if taken is not None and base in taken: + return f"{base}-{sha256_text(str(root))[:8]}" + return base + + +def find_agent_files(root: Path, max_depth: int = 3) -> List[Path]: + results: List[Path] = [] + for name in ("AGENTS.md", "CLAUDE.md"): + candidate = root / name + if candidate.is_file(): + results.append(candidate) + root_depth = len(root.resolve().parts) + for current, dirs, files in os.walk(root): + current_path = Path(current) + depth = len(current_path.resolve().parts) - root_depth + if depth >= max_depth: + dirs[:] = [] + dirs[:] = sorted(d for d in dirs if d not in EXCLUDE_DIRS and not d.startswith(".git")) + for name in ("AGENTS.md", "CLAUDE.md"): + candidate = current_path / name + if candidate.is_file() and candidate not in results: + results.append(candidate) + return sorted(results, key=lambda item: str(item)) + + +def find_project_skills(root: Path) -> List[str]: + roots = [root / "skills", root / ".agents" / "skills", root / ".claude" / "skills"] + found: List[str] = [] + for skill_root in roots: + if not skill_root.is_dir(): + continue + for child in sorted(skill_root.iterdir()): + if (child / "SKILL.md").is_file(): + found.append(str(child)) + return found + + +def find_project_tools(root: Path) -> List[str]: + tools_root = root / "tools" + if not tools_root.is_dir(): + return [] + found: List[str] = [] + for child in sorted(tools_root.iterdir()): + if child.is_dir() and ((child / "AGENTS.md").exists() or (child / "README.md").exists()): + found.append(str(child)) + return found + + +def load_index_texts(home: Path, hub: Optional[Path], projects: Sequence[Dict[str, Any]]) -> Dict[str, str]: + texts: Dict[str, str] = {} + if hub: + layout = resolve_hub_layout(hub) + texts["hub"] = read_text(layout["knowledge"] / "README.md", limit=2_000_000) + tools_root = layout.get("tools") + if tools_root: + texts["tools"] = read_text(tools_root / "README.md", limit=2_000_000) + texts["local_wiki"] = read_text(home / ".agents" / "wiki" / "README.md", limit=2_000_000) + for project in projects: + agents_dir = project.get("agents_dir") + if agents_dir: + texts[f"project:{project['id']}"] = read_text( + Path(agents_dir) / "README.md", limit=1_000_000 + ) + return texts + + +def artifact_record( + path: Path, + kind: str, + plane: str, + scope: str, + project_id: Optional[str], + indexed: Optional[bool], + tracked: Optional[Set[str]] = None, + base: Optional[Path] = None, +) -> Dict[str, Any]: + text = "" + if path.suffix.lower() in {".md", ".markdown", ".txt", ".json", ".yaml", ".yml", ".toml"}: + text = read_text(path, limit=1_000_000) + frontmatter, body = parse_frontmatter(text) + size = 0 + mtime = 0.0 + try: + # One stat, no exists-then-stat window: a file deleted mid-scan reads + # as absent (0), never as a crash. + info = path.stat() + size = info.st_size + mtime = info.st_mtime + except OSError: + pass + tracked_flag: Optional[bool] = False + if tracked is None: + # Unknown is not "untracked": a failed git probe must stay visible. + tracked_flag = None + elif base is not None: + try: + tracked_flag = str(path.resolve().relative_to(base.resolve())) in tracked + except (OSError, ValueError): + tracked_flag = False + oversized = size > HASH_LIMIT + file_hash = "" if oversized else sha256_file(path) + return { + "path": str(path), + "rel": relative(path, base) if base else str(path), + "kind": kind, + "plane": plane, + "scope": scope, + "project": project_id, + "size": size, + "mtime": mtime, + "hash": file_hash, + "hash_error": not oversized and not file_hash, + "frontmatter": frontmatter, + "title": first_heading(body if text else "", path.stem), + "indexed": indexed, + "git_tracked": tracked_flag, + "verified_at": frontmatter.get("verified_at") or frontmatter.get("updated", ""), + "stale_after": frontmatter.get("stale_after", ""), + "status": frontmatter.get("status", ""), + } + + +def scan_project_agents( + projects: Sequence[Dict[str, Any]], + index_texts: Dict[str, str], + project_tracked: Dict[str, Optional[Set[str]]], + errors: Optional[List[str]] = None, +) -> List[Dict[str, Any]]: + records: List[Dict[str, Any]] = [] + for project in projects: + root = Path(project["path"]) + tracked = project_tracked.get(project["path"], set()) + for agent_rel in project.get("agent_files", []): + path = root / agent_rel + records.append( + artifact_record(path, "instruction", "instruction", "project", project["id"], None, tracked, root) + ) + agents_dir = project.get("agents_dir") + if not agents_dir: + continue + agents_path = Path(agents_dir) + index_text = index_texts.get(f"project:{project['id']}", "") + index_path = agents_path / "README.md" + if index_path.is_file() and not index_path.is_symlink(): + records.append( + artifact_record( + index_path, "instruction", "instruction", "project", project["id"], None, tracked, root + ) + ) + for slot, kind, plane in ( + ("plans", "plan", "state"), + ("wiki", "knowledge", "knowledge"), + ("evidence", "evidence", "evidence"), + ("archive", "archive", "evidence"), + ): + slot_root = agents_path / slot + if not slot_root.is_dir(): + continue + if slot_root.is_symlink(): + # A shared slot (e.g. a wiki linked from a sibling checkout) is + # scanned under the project that owns the real directory. + continue + for path in iter_files(slot_root, suffixes={".md", ".json", ".yaml", ".yml"}, errors=errors): + if "openspec" in path.parts[:-1]: + # OpenSpec store config and change metadata are state, not knowledge. + continue + if path.is_symlink(): + continue + indexed = is_indexed(path.name, index_text) if kind == "knowledge" else None + records.append( + artifact_record(path, kind, plane, "project", project["id"], indexed, tracked, root) + ) + return records + + +def scan_hub_knowledge( + hub: Optional[Path], + index_texts: Dict[str, str], + errors: Optional[List[str]] = None, +) -> List[Dict[str, Any]]: + if not hub: + return [] + records: List[Dict[str, Any]] = [] + tracked = git_tracked(hub) + knowledge_index = index_texts.get("hub", "") + layout = resolve_hub_layout(hub) + knowledge_index_path = layout["knowledge"] / "README.md" + if knowledge_index_path.is_file(): + records.append( + artifact_record( + knowledge_index_path, "instruction", "instruction", "project", None, None, tracked, hub + ) + ) + default_suffixes = {".md", ".json", ".yaml", ".yml"} + groups: List[Tuple[Path, str, str, str, bool, Set[str]]] = [] + # Knowledge subzones are discovered, not enumerated: every first-level + # directory under the knowledge root (general/, projects/, machines/, ...) + # is scanned as hub knowledge and index-checked against the README. + knowledge_root = layout["knowledge"] + if knowledge_root.is_dir(): + for child in sorted(knowledge_root.iterdir()): + if not child.is_dir() or child.name.startswith(".") or child.name in EXCLUDE_DIRS: + continue + groups.append((child, "knowledge", "knowledge", child.name, True, default_suffixes)) + groups.extend([ + (layout["projects"], "state", "state", "project", False, default_suffixes), + (hub / "register", "state", "state", "project", False, default_suffixes), + (hub / "handoff", "state", "state", "project", False, default_suffixes), + (hub / "provision", "state", "state", "project", False, default_suffixes), + (hub / "reports", "evidence", "evidence", "project", False, default_suffixes), + (hub / "output", "evidence", "evidence", "project", False, default_suffixes), + (hub / "Human Output", "evidence", "evidence", "project", False, default_suffixes), + (hub / "manuals", "state", "state", "project", False, default_suffixes), + (hub / "cron-tasks", "state", "state", "project", False, default_suffixes | {".plist"}), + (hub / "openspec", "plan", "state", "project", False, default_suffixes), + (hub / "archive", "archive", "archive", "project", False, default_suffixes), + (hub / ".agents" / "plans", "plan", "state", "project", False, default_suffixes), + ]) + for root, kind, plane, scope, check_index, suffixes in groups: + if not root.is_dir(): + continue + for path in iter_files(root, suffixes=suffixes, errors=errors): + indexed = None + if check_index and kind == "knowledge": + indexed = is_indexed(path.name, knowledge_index) + records.append( + artifact_record(path, kind, plane, scope, None, indexed, tracked, hub) + ) + strategy = hub / "strategy.md" + if strategy.is_file(): + records.append( + artifact_record( + strategy, "state", "state", "project", None, None, tracked, hub + ) + ) + tools_root = layout.get("tools") + for path in iter_files(tools_root, suffixes={".md"}, max_depth=3, errors=errors) if tools_root else []: + if path.name in {"AGENTS.md", "README.md"}: + records.append( + artifact_record(path, "tool", "capability", "project", None, None, tracked, hub) + ) + return records + + +def scan_local_wiki( + home: Path, + index_texts: Dict[str, str], + errors: Optional[List[str]] = None, +) -> List[Dict[str, Any]]: + root = home / ".agents" / "wiki" + if not root.is_dir(): + return [] + index_text = index_texts.get("local_wiki", "") + records: List[Dict[str, Any]] = [] + for path in iter_files(root, suffixes={".md"}, skip_dirs=EXCLUDE_DIRS | {"assets"}, errors=errors): + if path.is_symlink(): + # Compatibility links point at a canonical file already counted. + continue + records.append( + artifact_record( + path, + "knowledge", + "knowledge", + "local", + None, + is_indexed(path.name, index_text), + ) + ) + return records + + +def count_zone_tools(root: Path) -> int: + """Tool count for a zone summary: flat zones hold tools at depth 1, + grouped zones (like common/company) hold them at depth 2.""" + if not root.is_dir(): + return 0 + count = 0 + for child in sorted(root.iterdir()): + if not child.is_dir() or child.name.startswith("."): + continue + if (child / "AGENTS.md").exists() or (child / "README.md").exists(): + count += 1 + continue + for grandchild in sorted(child.iterdir()): + if not grandchild.is_dir() or grandchild.name.startswith("."): + continue + if (grandchild / "AGENTS.md").exists() or (grandchild / "README.md").exists(): + count += 1 + return count + + +def scan_zones( + zones: Sequence[Dict[str, str]], + errors: Optional[List[str]] = None, +) -> List[Dict[str, Any]]: + """Zone scan: entry docs and nothing else. + + A declared tool zone is not a project tree. The scan collects + AGENTS.md/README.md so pointer health and freshness can see them, and the + caller feeds the zone root to the secret scan; slot compliance never + applies because a tool is not a repository with `.agents/` slots. + """ + records: List[Dict[str, Any]] = [] + for zone in zones: + root = Path(zone["path"]).expanduser() + for path in iter_files(root, suffixes={".md"}, max_depth=3, errors=errors): + if path.is_symlink() or path.name not in {"AGENTS.md", "README.md"}: + continue + records.append( + artifact_record( + path, + "tool", + "capability", + "zone", + None, + None, + None, + root, + ) + ) + return records + + +def extra_skill_roots() -> List[Tuple[str, Path]]: + """Extra skill hosts from GUSTER_EXTRA_SKILL_ROOTS (name=path, os.pathsep separated). + + Declared hosts are configuration, not guesses: a malformed entry or a missing + directory stops the run instead of silently dropping the host.""" + roots: List[Tuple[str, Path]] = [] + for item in os.environ.get("GUSTER_EXTRA_SKILL_ROOTS", "").split(os.pathsep): + item = item.strip() + if not item: + continue + name, sep, raw = item.partition("=") + if not sep or not name.strip() or not raw.strip(): + raise ValueError(f"GUSTER_EXTRA_SKILL_ROOTS entries must be name=path: {item!r}") + root = Path(raw.strip()).expanduser() + if not root.is_dir(): + raise ValueError( + f"GUSTER_EXTRA_SKILL_ROOTS host {name.strip()!r} directory does not exist: {root}" + ) + roots.append((name.strip(), root)) + return roots + + +def scan_skills(home: Path) -> List[Dict[str, Any]]: + roots = [ + ("agents", home / ".agents" / "skills"), + ("codex", home / ".codex" / "skills"), + ("dsh", home / ".dsh" / "skills"), + ("claude", home / ".claude" / "skills"), + ] + extra_skill_roots() + raw: List[Dict[str, Any]] = [] + for host, root in roots: + if not root.is_dir(): + continue + for skill_md in sorted(root.glob("*/SKILL.md")): + text = read_text(skill_md, limit=500_000) + frontmatter, _ = parse_frontmatter(text) + name = frontmatter.get("name") or skill_md.parent.name + description = frontmatter.get("description", "") + raw.append( + { + "name": name, + "root": str(skill_md.parent), + "host": host, + "description": description, + "description_chars": len(description), + "hash": sha256_file(skill_md), + } + ) + + grouped: Dict[str, List[Dict[str, Any]]] = {} + for item in raw: + grouped.setdefault(item["name"], []).append(item) + + skills: List[Dict[str, Any]] = [] + for name in sorted(grouped): + copies = sorted(grouped[name], key=lambda item: item["root"]) + primary = copies[0] + skills.append( + { + "name": name, + "root": primary["root"], + "host": primary["host"], + "description": primary["description"], + "description_chars": len(primary["description"]), + "hash": primary["hash"], + "copies": [ + {"host": item["host"], "root": item["root"], "hash": item["hash"]} + for item in copies + ], + } + ) + return skills + + +def find_env_files(root: Path, max_depth: int = SECRET_ENV_DEPTH) -> List[Path]: + """`.env` files under a project, bounded by depth so the scan stays cheap.""" + found: List[Path] = [] + stack: List[Tuple[Path, int]] = [(root, 0)] + while stack: + current, depth = stack.pop() + try: + entries = list(current.iterdir()) + except OSError: + continue + for entry in entries: + if entry.is_file() and entry.name == ".env": + found.append(entry) + elif ( + entry.is_dir() + and depth < max_depth + and not entry.name.startswith(".") + and entry.name not in EXCLUDE_DIRS + and not entry.is_symlink() + ): + stack.append((entry, depth + 1)) + return sorted(found) + + +def scan_secret_files( + home: Path, + hub: Optional[Path], + projects: Sequence[Dict[str, Any]], + project_tracked: Dict[str, Optional[Set[str]]], + zones: Sequence[Dict[str, str]] = (), + include_home: bool = True, + errors: Optional[List[str]] = None, +) -> List[Dict[str, Any]]: + candidates: List[Path] = [] + if include_home: + for root in (home / ".agents" / "credentials",): + if root.is_dir(): + candidates.extend( + path + for path in iter_files(root, errors=errors) + if path.is_file() and path.suffix.lower() != ".md" + ) + if hub: + tools_root = resolve_hub_layout(hub).get("tools") + if tools_root and tools_root.is_dir(): + for path in iter_files(tools_root, max_depth=3, errors=errors): + if path.name in SECRET_SUFFIX_NAMES or path.name.startswith("credentials.") or path.name.endswith(".local.json"): + candidates.append(path) + for zone in zones: + candidates.extend(find_env_files(Path(zone["path"]).expanduser())) + for project in projects: + candidates.extend(find_env_files(Path(project["path"]))) + + seen: Set[str] = set() + records: List[Dict[str, Any]] = [] + for path in candidates: + try: + resolved = str(path.resolve()) + except OSError: + continue + if resolved in seen: + continue + seen.add(resolved) + canonical = is_under(path, home / ".agents" / "credentials") + tracked: Optional[bool] = None + for project in projects: + project_root = Path(project["path"]) + if not is_under(path, project_root): + continue + tracked_set = project_tracked.get(str(project_root)) + if tracked_set is None: + tracked = None + else: + try: + rel = str(path.resolve().relative_to(project_root.resolve())) + except ValueError: + rel = "" + tracked = rel in tracked_set + break + if tracked is None and not any(is_under(path, Path(p["path"])) for p in projects): + # Outside every scanned repository there is nothing that could track it. + tracked = False + try: + size = path.stat().st_size + except OSError: + size = 0 + records.append( + { + "path": str(path), + "canonical": canonical, + "git_tracked": tracked, + "size": size, + "kind": infer_secret_kind(path), + } + ) + return records + + +def scan_credential_registry(home: Path) -> Dict[str, Any]: + """Read the credential registry table; it never contains secret values.""" + path = home / ".agents" / "credentials" / "registry.md" + entries: List[Dict[str, str]] = [] + warnings: List[str] = [] + if path.is_file(): + for line_no, line in enumerate(read_text(path, limit=1_000_000).splitlines(), start=1): + stripped = line.strip() + if not stripped.startswith("|"): + continue + cells = [cell.strip() for cell in stripped.strip("|").split("|")] + if cells[0] in {"ID", ""} or set(cells[0]) <= {"-", ":"}: + continue + if len(cells) < 6: + # An entry-shaped row that cannot be an entry is a broken row, + # not silence: ID|scope|site|account|kind|locations are required. + # Line/column info only — the row text stays out because a + # broken row may carry a pasted secret value. + warnings.append( + f"line {line_no}: expected at least 6 columns " + f"(ID|scope|site|account|kind|locations), got {len(cells)}" + ) + continue + entries.append( + { + "id": cells[0], + "scope": cells[1], + "site": cells[2], + "account": cells[3], + "kind": cells[4], + "locations": cells[5], + "keys": cells[6] if len(cells) > 6 else "", + "used_by": cells[7] if len(cells) > 7 else "", + "status": cells[8] if len(cells) > 8 else "", + } + ) + return { + "path": str(path) if path.is_file() else None, + "entries": entries, + "parse_warnings": warnings, + } + + +def infer_secret_kind(path: Path) -> str: + name = path.name.lower() + if name.endswith(".env") or name.endswith(".env.example"): + return "env" + if "token" in name or "secret" in name or "credential" in name: + return "token" + if name.endswith((".json", ".yaml", ".yml", ".toml")): + return "config" + return "file" + + +def scan_inline_bearers(home: Path) -> List[Dict[str, Any]]: + records: List[Dict[str, Any]] = [] + for raw in INLINE_CONFIG_FILES: + path = home / raw[2:] if raw.startswith("~/") else Path(raw) + if not path.is_file(): + continue + text = read_text(path, limit=2_000_000) + values: List[str] = [] + for pattern in INLINE_PATTERNS: + for match in re.finditer(pattern, text): + value = match.group(1) if match.groups() else match.group(0) + redacted = f"{value[:4]}...{value[-4:]}" + if redacted not in values: + values.append(redacted) + if values: + records.append({"path": str(path), "count": len(values), "redacted": values}) + return records + + +def find_governance_docs(home: Path, hub: Optional[Path]) -> Optional[List[str]]: + # Framework docs, not topic authorities. A doc that labels itself a facet + # ("分册") belongs to the primary framework and does not compete with it. + named = load_governance_docs(hub) + if named is None: + return None + wanted = set(named) + candidates: List[Path] = [] + if hub: + general = resolve_hub_layout(hub)["knowledge"] / "general" + if general.is_dir(): + candidates.extend(path for path in general.iterdir() if path.name in wanted) + candidates.extend( + path + for path in (home / ".agents" / "wiki").glob("*.md") + if path.name in wanted + ) + primary = [] + for path in candidates: + header = "\n".join(read_text(path, limit=4000).splitlines()[:5]) + if "分册" not in header: + primary.append(str(path)) + return sorted(primary) + + +def scan_plugin_manifests(plugin_root: Optional[Path]) -> Dict[str, Any]: + if plugin_root is None: + return {} + manifests: Dict[str, Any] = {} + for label, rel in ( + ("codex", ".codex-plugin/plugin.json"), + ("zcode", ".zcode-plugin/plugin.json"), + ("package", "package.json"), + ): + path = plugin_root / rel + data = load_json(path, {}) + manifests[label] = { + "path": str(path), + "exists": path.is_file(), + "version": data.get("version") if isinstance(data, dict) else None, + "name": data.get("name") if isinstance(data, dict) else None, + } + return manifests + + +def count_registry_entries(hub: Optional[Path], errors: Optional[List[str]] = None) -> int: + """Projects registered in the hub's register (register/projects.yaml). + + Same parser as the drift reconciliation, so the summary and the CTX-10 + diff can never disagree about how many rows exist.""" + if not hub: + return 0 + path = hub.expanduser().resolve() / "register" / "projects.yaml" + if not path.is_file(): + return 0 + return len(parse_register_rows(read_text(path, limit=1_000_000))) + + +def summarize( + projects: Sequence[Dict[str, Any]], + skills: Sequence[Dict[str, Any]], + artifacts: Sequence[Dict[str, Any]], + secret_files: Sequence[Dict[str, Any]], + inline_bearers: Sequence[Dict[str, Any]], + registry_entries: int = 0, +) -> Dict[str, Any]: + return { + "projects": len(projects), + "containers": sum(1 for project in projects if project.get("container")), + "registry_entries": registry_entries, + "agents_dirs": sum(1 for project in projects if project.get("agents_dir")), + "agents_with_index": sum(1 for project in projects if project.get("has_index")), + "git_errors": sum(1 for project in projects if project.get("git_state") == "error"), + "skills": len(skills), + "skill_copies": sum(len(skill.get("copies", [])) for skill in skills), + "artifacts": len(artifacts), + "knowledge": sum(1 for item in artifacts if item.get("kind") == "knowledge"), + "state": sum(1 for item in artifacts if item.get("kind") == "state"), + "evidence": sum(1 for item in artifacts if item.get("kind") == "evidence"), + "secret_files": len(secret_files), + "secret_outside_root": sum(1 for item in secret_files if not item.get("canonical")), + "secret_in_git": sum(1 for item in secret_files if item.get("git_tracked")), + "inline_bearer_files": len(inline_bearers), + "inline_bearers": sum(item.get("count", 0) for item in inline_bearers), + } diff --git a/external_plugins/guster/src/guster/executor.py b/external_plugins/guster/src/guster/executor.py new file mode 100644 index 00000000..28dc92e0 --- /dev/null +++ b/external_plugins/guster/src/guster/executor.py @@ -0,0 +1,444 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from .discover import default_roots +from .util import is_under, load_json_strict, now_iso, unique_suffix, write_text + + +class ApplyError(RuntimeError): + pass + + +class ConfigError(ValueError): + """Invalid or unreadable config.json: exit-code contract treats these as + tool/config errors (exit 1), not plan refusals (exit 4).""" + + +BACKUPS_KEEP = 10 + + +def prune_backups(state_dir: Path, keep: int = BACKUPS_KEEP) -> List[str]: + """Keep the newest `keep` plan backup directories; hub-config files and + anything unexpected in backups/ stay untouched.""" + root = state_dir / "backups" + if not root.is_dir(): + return [] + + def stamp(item: Path) -> float: + try: + return item.stat().st_mtime + except OSError: + return 0.0 + + dirs = sorted( + (item for item in root.iterdir() if item.is_dir()), + key=stamp, + reverse=True, + ) + removed: List[str] = [] + for item in dirs[keep:]: + try: + shutil.rmtree(item) + except OSError: + continue + removed.append(str(item)) + return removed + + +def _config_roots(raw: Any, config_path: Path) -> List[Path]: + """Same discipline as cli.parse_roots: a list of existing absolute directories; + anything else is a config error, never silently widened into a write boundary.""" + if not isinstance(raw, list): + raise ConfigError( + f"Guster config 'roots' must be a list of absolute paths, " + f"got {type(raw).__name__}: {config_path}" + ) + if not raw: + raise ConfigError(f"Guster config 'roots' must list at least one path: {config_path}") + roots: List[Path] = [] + for item in raw: + if not isinstance(item, str) or not item.strip(): + raise ConfigError( + f"Guster config 'roots' entries must be non-empty strings: {item!r} in {config_path}" + ) + path = Path(item).expanduser() + if not path.is_absolute(): + raise ConfigError( + f"Guster config 'roots' entries must be absolute paths: {item!r} in {config_path}" + ) + if not path.exists(): + raise ConfigError(f"workspace root does not exist: {path.resolve()}") + if not path.is_dir(): + raise ConfigError(f"workspace root is not a directory: {path.resolve()}") + roots.append(path) + return roots + + +def allowed_bases(state_dir: Path, extra_bases: Optional[Sequence[Path]] = None) -> List[Path]: + """Directories a plan may write into: the state dir, the configured workspace + roots (auto-detected defaults when the config names none), the attached hub + (config "hub" key, written by `guster hub attach`) and any caller-declared + extra bases (e.g. the project being initialized).""" + bases: List[Path] = [state_dir] + config_path = state_dir / "config.json" + data, error = load_json_strict(config_path) + if error is not None: + raise ConfigError(f"cannot read Guster config {config_path}: {error}") + if data is not None and not isinstance(data, dict): + raise ConfigError(f"Guster config must be a JSON object: {config_path}") + if data is not None and "roots" in data: + bases.extend(_config_roots(data["roots"], config_path)) + else: + bases.extend(default_roots(Path.home())) + if data is not None and "hub" in data: + raw_hub = data["hub"] + if not isinstance(raw_hub, str) or not raw_hub.strip(): + raise ConfigError(f"Guster config 'hub' must be a path string: {config_path}") + hub = Path(raw_hub).expanduser() + if not hub.is_absolute(): + raise ConfigError( + f"Guster config 'hub' must be an absolute path: {raw_hub!r} in {config_path}" + ) + if not hub.is_dir(): + raise ConfigError(f"hub path from config is not a directory: {hub.resolve()}") + bases.append(hub) + bases.extend(extra_bases or []) + return bases + + +def _outside_error( + index: int, + kind: str, + field: str, + path: Path, + bases: Sequence[Path], +) -> Optional[str]: + try: + resolved = path.resolve() + except OSError: + return f"op {index}: cannot resolve {kind} {field}: {path}" + if any(is_under(resolved, base) for base in bases): + return None + return ( + f"op {index}: {kind} {field} is outside the allowed bases: {path} " + f"(resolved {resolved}); allowed bases: " + ", ".join(str(base) for base in bases) + ) + + +def preflight( + plan: Dict[str, Any], + bases: Optional[Sequence[Path]] = None, +) -> List[str]: + errors: List[str] = [] + operations = plan.get("operations", []) + if not isinstance(operations, list): + return ["plan operations must be a list"] + for index, operation in enumerate(operations, start=1): + if not isinstance(operation, dict): + errors.append(f"op {index}: operation is not a JSON object") + continue + kind = operation.get("kind") + if not isinstance(kind, str) or not kind: + errors.append(f"op {index}: missing operation kind") + continue + if kind == "move": + if "pointer" in operation: + errors.append( + f"op {index}: pointer updates are no longer applied by `apply`; " + "update the referenced file with a write op" + ) + continue + src_value = operation.get("src") + dst_value = operation.get("dst") + missing = [ + key + for key, value in (("src", src_value), ("dst", dst_value)) + if not isinstance(value, str) or not value + ] + if missing: + errors.append(f"op {index}: move needs string field(s): {', '.join(missing)}") + continue + src = Path(src_value) + dst = Path(dst_value) + if not src.exists(): + errors.append(f"op {index}: source does not exist: {src}") + if dst.exists(): + errors.append(f"op {index}: destination already exists: {dst}") + if src == dst: + errors.append(f"op {index}: source and destination are identical") + if bases is not None: + for field, value in (("src", src), ("dst", dst)): + error = _outside_error(index, kind, field, value, bases) + if error: + errors.append(error) + elif kind in {"write", "mkdir"}: + value = operation.get("path") + if not isinstance(value, str) or not value: + errors.append(f"op {index}: {kind} needs a string 'path'") + continue + path = Path(value) + if kind == "mkdir": + if path.exists() and not path.is_dir(): + errors.append(f"op {index}: target exists and is not a directory: {path}") + else: + if not isinstance(operation.get("content"), str): + errors.append(f"op {index}: write needs a string 'content'") + elif path.exists() and not path.is_file(): + errors.append(f"op {index}: target is not a file: {path}") + if bases is not None: + error = _outside_error(index, kind, "path", path, bases) + if error: + errors.append(error) + elif kind != "manual": + errors.append(f"op {index}: unknown operation kind: {kind}") + return errors + + +def apply_plan( + plan: Dict[str, Any], + state_dir: Path, + dry_run: bool = True, + yes: bool = False, + extra_bases: Optional[Sequence[Path]] = None, +) -> Dict[str, Any]: + if not dry_run and not yes: + raise ApplyError("refusing to write without --yes") + bases = allowed_bases(state_dir, extra_bases) + errors = preflight(plan, bases) + if errors: + return { + "ok": False, + "dry_run": dry_run, + "plan": plan.get("id"), + "errors": errors, + "applied": 0, + "skipped": 0, + } + + plan_id = plan.get("id") or "plan" + backup_dir = state_dir / "backups" / plan_id + if not is_under(backup_dir, state_dir / "backups"): + # A slug-hostile plan id must never escape the backup area, not even + # when apply_plan is called as a library without load_plan's validation. + raise ApplyError(f"plan id {plan_id!r} escapes the backup directory; refusing to apply") + undo: List[Tuple[str, Dict[str, Any]]] = [] + applied = 0 + skipped = 0 + + if dry_run: + for operation in plan.get("operations", []): + if operation.get("kind") == "manual": + skipped += 1 + else: + applied += 1 + return { + "ok": True, + "dry_run": True, + "plan": plan_id, + "errors": [], + "applied": applied, + "skipped": skipped, + "operations": plan.get("operations", []), + } + + if backup_dir.exists(): + # Never overwrite a previous run's rollback material for the same id. + backup_dir = state_dir / "backups" / f"{plan_id}-{unique_suffix()}" + backup_dir.mkdir(parents=True, exist_ok=True) + try: + for index, operation in enumerate(plan.get("operations", []), start=1): + kind = operation.get("kind") + if kind == "manual": + skipped += 1 + continue + if kind == "mkdir": + path = Path(operation["path"]) + late = _outside_error(index, kind, "path", path, bases) + if late: + raise ApplyError(late) + created: List[Path] = [] + probe = path + while not probe.exists() and probe != probe.parent: + created.append(probe) + probe = probe.parent + path.mkdir(parents=True, exist_ok=True) + for fresh in reversed(created): + # Reversed undo must rmdir the deepest level first, so the + # shallowest created level is appended last. + undo.append(("mkdir", {"path": str(fresh)})) + applied += 1 + elif kind == "move": + src = Path(operation["src"]) + dst = Path(operation["dst"]) + for field, value in (("src", src), ("dst", dst)): + late = _outside_error(index, kind, field, value, bases) + if late: + raise ApplyError(late) + backup = backup_dir / f"{index:03d}-{src.name}" + if src.is_dir(): + shutil.copytree(src, backup) + else: + shutil.copy2(src, backup) + undo.append(("move", {"src": str(src), "dst": str(dst), "backup": str(backup)})) + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), str(dst)) + applied += 1 + elif kind == "write": + path = Path(operation["path"]) + late = _outside_error(index, kind, "path", path, bases) + if late: + raise ApplyError(late) + backup = backup_dir / f"{index:03d}-{path.name}" + if path.exists(): + shutil.copy2(path, backup) + undo.append(("write", {"path": str(path), "backup": str(backup)})) + write_text(path, operation["content"]) + applied += 1 + else: + raise ApplyError(f"unsupported operation: {kind}") + except Exception as exc: + rollback_error: Optional[BaseException] = None + residual: List[str] = [] + try: + residual = rollback(undo) + except BaseException as undo_exc: # noqa: BLE001 - reported, never hidden + rollback_error = undo_exc + if rollback_error is not None: + # Both the apply and the rollback failed: keep the root cause and + # name the recovery material instead of swallowing either. + raise ApplyError( + f"apply failed ({exc}); rollback also failed ({rollback_error}) — " + f"backups for manual recovery: {backup_dir}" + ) from exc + if residual: + raise ApplyError( + f"apply failed and was rolled back: {exc}; " + f"directories left behind (not empty or in use): {', '.join(residual)}" + ) from exc + raise ApplyError(f"apply failed and was rolled back: {exc}") from exc + + pruned = prune_backups(state_dir) + return { + "ok": True, + "dry_run": False, + "plan": plan_id, + "errors": [], + "applied": applied, + "skipped": skipped, + "backup": str(backup_dir), + "pruned_backups": pruned, + "finished_at": now_iso(), + } + + +def rollback(undo: List[Tuple[str, Dict[str, Any]]]) -> List[str]: + """Reverse applied operations; returns created-directory paths that could + not be removed (non-empty or busy) so the caller can name them.""" + residual: List[str] = [] + for kind, payload in reversed(undo): + if kind == "mkdir": + path = Path(payload["path"]) + if not path.is_dir(): + continue + try: + path.rmdir() + except OSError: + residual.append(str(path)) + elif kind == "move": + src = Path(payload["src"]) + dst = Path(payload["dst"]) + backup = Path(payload["backup"]) + if dst.exists(): + if dst.is_dir(): + shutil.rmtree(dst) + else: + dst.unlink() + if backup.is_dir(): + shutil.copytree(backup, src, dirs_exist_ok=True) + elif backup.exists(): + src.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(backup, src) + elif kind == "write": + path = Path(payload["path"]) + backup = Path(payload["backup"]) + if backup.exists(): + shutil.copy2(backup, path) + elif path.exists(): + path.unlink() + return residual + + +def init_home( + state_dir: Path, + yes: bool = False, + dry_run: bool = True, + home: Optional[Path] = None, + roots: Optional[List[Path]] = None, +) -> Dict[str, Any]: + operations = [ + {"kind": "mkdir", "path": str(state_dir)}, + {"kind": "mkdir", "path": str(state_dir / "plans")}, + {"kind": "mkdir", "path": str(state_dir / "backups")}, + ] + config_path = state_dir / "config.json" + if not config_path.exists(): + resolved_home = (home or Path.home()).expanduser().resolve() + if roots is None: + roots = default_roots(resolved_home) + if not roots: + raise ValueError( + f"no workspace roots detected under {resolved_home} " + "(looked for ~/Projects, ~/code, ~/dev, ~/src); " + "pass --root to declare where your projects live" + ) + config: Dict[str, Any] = { + "version": 1, + "roots": [str(root) for root in roots], + "scan_max_depth": 5, + } + operations.append( + { + "kind": "write", + "path": str(config_path), + "content": json.dumps( + config, + ensure_ascii=False, + indent=2, + ) + + "\n", + } + ) + else: + # A broken config must not turn `guster init --root ` into a + # success-shaped no-op: validate it or refuse loudly. + data, error = load_json_strict(config_path) + reason = error + if reason is None and not isinstance(data, dict): + reason = f"config must be a JSON object, got {type(data).__name__}" + if reason is None and "roots" not in data: + reason = "config declares no 'roots'" + if reason is None: + try: + _config_roots(data["roots"], config_path) + except ValueError as exc: + reason = str(exc) + if reason is not None: + raise ValueError( + f"config exists but invalid ({reason}); fix or remove {config_path} " + "then re-run `guster init --root `" + ) + plan = { + "version": 1, + # Same slug discipline as planner.new_plan_id so a saved init plan can + # still pass load_plan's id check when re-applied by hand. + "id": f"init-home-{now_iso().replace(':', '').replace('-', '').replace('+', 'p')}-{unique_suffix()}", + "created_at": now_iso(), + "title": "Initialize Guster state home", + "notes": "Creates the local state directory and default config.", + "operations": operations, + } + return apply_plan(plan, state_dir, dry_run=dry_run, yes=yes) diff --git a/external_plugins/guster/src/guster/hub.py b/external_plugins/guster/src/guster/hub.py new file mode 100644 index 00000000..e73fe71a --- /dev/null +++ b/external_plugins/guster/src/guster/hub.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Dict, List, Optional + +from . import __version__ +from .executor import ConfigError, apply_plan +from .layout import DEFAULT_PATHS, load_manifest, resolve_hub_layout, validate_layout_value +from .planner import build_plan, op_mkdir, op_write, save_plan +from .util import DATE_ONLY_RE, EXCLUDE_DIRS, is_indexed, iter_files, load_json_strict, now_iso, read_text, save_json, today_iso, unique_suffix + + +def default_hub_path(home: Path) -> Path: + return home / "guster-hub" + + +def template_root() -> Path: + root = Path(__file__).resolve().parents[2] / "templates" / "hub" + if not root.is_dir(): + raise FileNotFoundError(f"lite hub template not found: {root}") + return root + + +def _version_tuple(value: str) -> tuple: + parts = [int(item) for item in re.findall(r"\d+", str(value))[:3]] + return tuple(parts + [0] * (3 - len(parts))) + + +def _render(text: str, replacements: Dict[str, str]) -> str: + for key, value in replacements.items(): + text = text.replace("{{" + key + "}}", value) + if "{{" in text: + raise ValueError("hub template still contains an unresolved placeholder") + return text + + +def _relative(path: Path, root: Path) -> str: + try: + return str(path.relative_to(root)) + except ValueError: + return str(path) + + +def plan_init_hub( + target: Path, + name: Optional[str] = None, + knowledge: str = DEFAULT_PATHS["knowledge"], + projects: str = DEFAULT_PATHS["projects"], + tools: str = DEFAULT_PATHS["tools"], +) -> Dict[str, Any]: + root = template_root() + hub = target.expanduser().resolve() + path_values = { + "knowledge": validate_layout_value(knowledge, "knowledge"), + "projects": validate_layout_value(projects, "projects"), + "tools": validate_layout_value(tools, "tools"), + } + layout = {key: (hub / value).resolve() for key, value in path_values.items()} + if len(set(layout.values())) != len(layout): + raise ValueError("hub paths must point to distinct directories") + replacements = { + "DATE": today_iso(), + "HUB_NAME": name or hub.name, + "PLUGIN_VERSION": __version__, + "KNOWLEDGE_PATH": path_values["knowledge"], + "PROJECTS_PATH": path_values["projects"], + "TOOLS_PATH": path_values["tools"], + } + operations: List[Dict[str, Any]] = [op_mkdir(hub)] + for path in layout.values(): + operations.append(op_mkdir(path)) + for path in sorted(root.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(root) + if rel.parts[0] in layout: + output = layout[rel.parts[0]].joinpath(*rel.parts[1:]) + else: + output = hub / rel + if path.name == "gitignore.tmpl": + output = hub / rel.parent / ".gitignore" + elif path.suffix == ".tmpl": + output = output.with_suffix("") + content = ( + "" + if path.name == ".gitkeep" + else _render(path.read_text(encoding="utf-8"), replacements) + ) + operations.append(op_write(output, content)) + return build_plan( + f"Initialize Guster Hub {hub.name}", + operations, + notes=( + "Creates the lite data plane: knowledge, project state, tool index " + "and the hub contract." + ), + ) + + +def validate_hub(target: Path, strict: bool = True) -> Dict[str, Any]: + hub = target.expanduser().resolve() + errors: List[str] = [] + warnings: List[str] = [] + if not hub.is_dir(): + return { + "ok": False, + "hub": str(hub), + "manifest": {}, + "errors": [f"hub directory does not exist: {hub}"], + "warnings": [], + "plugin": __version__, + } + + manifest = load_manifest(hub / "hub.yaml") + try: + layout = resolve_hub_layout(hub) + except ValueError as exc: + layout = {key: (hub / value).resolve() for key, value in DEFAULT_PATHS.items()} + errors.append(f"invalid hub layout: {exc}") + + legacy = ( + not manifest + and (hub / "AGENTS.md").is_file() + and (hub / "knowledge").is_dir() + and (hub / "projects").is_dir() + ) + required_files = [hub / "AGENTS.md", hub / "hub.yaml"] + required_dirs = [] + if "knowledge" in layout: + required_files.append(layout["knowledge"] / "README.md") + required_dirs.append(layout["knowledge"]) + if "projects" in layout: + register = hub / "register" / "projects.yaml" + required_files.append(register) + required_dirs.append(layout["projects"]) + migration_note = ( + " (migrate from projects/_registry.md - see CHANGELOG 0.9.4)" + if not register.is_file() and (layout["projects"] / "_registry.md").is_file() + else "" + ) + if "tools" in layout: + required_files.append(layout["tools"] / "README.md") + required_dirs.append(layout["tools"]) + for path in required_files: + if path.is_file(): + continue + rel = _relative(path, hub) + message = f"required file is absent: {rel}{migration_note if path == register and not legacy else ''}" + if legacy and not strict: + warnings.append(f"legacy hub lite contract file is absent: {rel}") + else: + errors.append(message) + for path in required_dirs: + if path.is_dir(): + continue + rel = _relative(path, hub) + if legacy and not strict: + warnings.append(f"legacy hub lite contract directory is absent: {rel}") + else: + errors.append(f"required directory is absent: {rel}") + + if not manifest: + message = "hub.yaml is absent or empty" + if legacy and not strict: + warnings.append(message + " (legacy hub)") + elif (hub / "hub.yaml").is_file(): + errors.append(message) + else: + if manifest.get("schema_version") not in {"1", "2"}: + errors.append(f"unsupported hub schema: {manifest.get('schema_version') or '(absent)'}") + minimum = manifest.get("min_guster") + if minimum and _version_tuple(minimum) > _version_tuple(__version__): + errors.append(f"hub requires Guster >= {minimum}; installed CLI is {__version__}") + + agents = hub / "AGENTS.md" + if agents.is_file() and len(read_text(agents).splitlines()) > 50: + warnings.append("AGENTS.md exceeds the 50-line soft budget") + + if strict: + index_path = layout["knowledge"] / "README.md" + if index_path.is_file(): + index = read_text(index_path, limit=2_000_000) + knowledge_root = layout["knowledge"] + try: + subzones = sorted(knowledge_root.iterdir()) if knowledge_root.is_dir() else [] + except OSError as exc: + subzones = [] + errors.append(f"cannot list knowledge root {knowledge_root}: {exc}") + for subzone in subzones: + if not subzone.is_dir() or subzone.name.startswith(".") or subzone.name in EXCLUDE_DIRS: + continue + for note in sorted( + iter_files(subzone, suffixes={".md", ".json", ".yaml", ".yml"}) + ): + if note.name.lower() == "readme.md": + continue + if not is_indexed(note.name, index): + errors.append(f"knowledge note is not indexed: {_relative(note, hub)}") + + register_path = hub / "register" / "projects.yaml" + if register_path.is_file(): + register = read_text(register_path) + declared = re.search(r"^updated:\s*['\"]?([^'\"\s#]+)", register, re.MULTILINE) + if declared is None or not DATE_ONLY_RE.fullmatch(declared.group(1)): + errors.append(f"{_relative(register_path, hub)} has no updated date") + + return { + "ok": not errors, + "hub": str(hub), + "manifest": manifest, + "paths": {key: _relative(value, hub) for key, value in layout.items()}, + "errors": errors, + "warnings": warnings, + "plugin": __version__, + } + + +def attach_hub(state_dir: Path, target: Path) -> Dict[str, Any]: + info = validate_hub(target, strict=False) + if not info["ok"]: + raise ValueError("hub validation failed: " + "; ".join(info["errors"])) + hub = target.expanduser().resolve() + state_dir.mkdir(parents=True, exist_ok=True) + config_path = state_dir / "config.json" + data, error = load_json_strict(config_path) + if error is not None: + raise ConfigError(f"cannot read Guster config {config_path}: {error}") + if data is None: + config: Dict[str, Any] = {} + elif not isinstance(data, dict): + raise ConfigError(f"Guster config must be a JSON object: {config_path}") + else: + config = data + backup: Optional[Path] = None + if config_path.is_file(): + backup_dir = state_dir / "backups" + backup_dir.mkdir(parents=True, exist_ok=True) + stamp = re.sub(r"[^0-9A-Za-z]", "", now_iso()) + backup = backup_dir / f"hub-config-{stamp}-{unique_suffix()}.json" + backup.write_bytes(config_path.read_bytes()) + config.setdefault("version", 1) + config["hub"] = str(hub) + save_json(config_path, config) + return { + "ok": True, + "hub": str(hub), + "config": str(config_path), + "backup": str(backup) if backup else None, + } + + +def init_hub( + target: Path, + state_dir: Path, + yes: bool = False, + dry_run: bool = True, + name: Optional[str] = None, + knowledge: str = DEFAULT_PATHS["knowledge"], + projects: str = DEFAULT_PATHS["projects"], + tools: str = DEFAULT_PATHS["tools"], +) -> Dict[str, Any]: + hub = target.expanduser().resolve() + if hub.exists(): + if not hub.is_dir(): + raise ValueError(f"hub target is not a directory: {hub}") + if any(item.name != ".DS_Store" for item in hub.iterdir()): + raise ValueError(f"hub target is not empty: {hub}") + path_values = { + "knowledge": validate_layout_value(knowledge, "knowledge"), + "projects": validate_layout_value(projects, "projects"), + "tools": validate_layout_value(tools, "tools"), + } + plan = plan_init_hub( + hub, + name=name, + knowledge=path_values["knowledge"], + projects=path_values["projects"], + tools=path_values["tools"], + ) + plan_path = save_plan(state_dir, plan) + result = apply_plan( + plan, + state_dir, + dry_run=dry_run, + yes=yes, + extra_bases=[target.resolve()], + ) + result["hub"] = str(hub) + result["plan"] = str(plan_path) + result["paths"] = path_values + if result.get("ok") and not dry_run: + result["attach"] = attach_hub(state_dir, hub) + return result diff --git a/external_plugins/guster/src/guster/layout.py b/external_plugins/guster/src/guster/layout.py new file mode 100644 index 00000000..867c707c --- /dev/null +++ b/external_plugins/guster/src/guster/layout.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Dict, List, Optional + +from .util import read_text + + +DEFAULT_PATHS = { + "knowledge": "knowledge", + "projects": "projects", + "tools": "tools", +} +# Schema 2 hubs declare what they have: only these two sections are mandatory, +# every other one (tools, ...) is optional and may live outside the hub. +HUB_CORE_SECTIONS = ("knowledge", "projects") + + +def load_manifest(path: Path) -> Dict[str, Any]: + """Parse the small flat YAML subset used by hub.yaml, including one paths map.""" + data: Dict[str, Any] = {} + current: Optional[str] = None + for raw_line in read_text(path).splitlines(): + if not raw_line.strip() or raw_line.lstrip().startswith("#"): + continue + indent = len(raw_line) - len(raw_line.lstrip(" ")) + line = raw_line.strip() + if ":" not in line: + continue + key, value = line.split(":", 1) + key = key.strip() + value = value.strip().strip("'\"") + if indent == 0: + current = key if key == "paths" and not value else None + if current: + data.setdefault("paths", {}) + else: + data[key] = value + elif current == "paths": + data.setdefault("paths", {})[key] = value + return data + + +def validate_layout_value(value: str, key: str) -> str: + raw = str(value).strip() + if not raw: + raise ValueError(f"hub path '{key}' is empty") + if raw.startswith("~"): + raise ValueError(f"hub path '{key}' must be relative to the hub") + path = Path(raw) + if path.is_absolute(): + raise ValueError(f"hub path '{key}' must be relative to the hub") + if ".." in path.parts: + raise ValueError(f"hub path '{key}' must not contain '..'") + if raw in {".", "./"}: + raise ValueError(f"hub path '{key}' must not be the hub root") + return str(path) + + +def is_external_layout_value(raw: str) -> bool: + text = str(raw).strip() + return text.startswith("~") or Path(text).is_absolute() + + +def resolve_layout_entry(raw: str, key: str, hub_root: Path) -> Path: + """One declared section: relative stays inside the hub, absolute/~/ is external. + + External sections describe a world where the directory does not belong to + the hub; the declaration still has to name a real directory, which the + caller checks, but it is not confined by the hub boundary. + """ + text = str(raw).strip() + if not text: + raise ValueError(f"hub path '{key}' is empty") + if is_external_layout_value(text): + return Path(text).expanduser().resolve() + value = validate_layout_value(text, key) + candidate = (hub_root / value).resolve() + if candidate == hub_root or hub_root not in candidate.parents: + raise ValueError(f"hub path '{key}' escapes the hub: {value}") + return candidate + + +def resolve_hub_layout(hub: Path) -> Dict[str, Path]: + root = hub.expanduser().resolve() + manifest = load_manifest(root / "hub.yaml") + raw_paths = manifest.get("paths") + paths = raw_paths if isinstance(raw_paths, dict) else {} + resolved: Dict[str, Path] = {} + if str(manifest.get("schema_version", "1")) == "2": + for key in HUB_CORE_SECTIONS: + if key not in paths: + raise ValueError(f"hub schema 2 must declare the '{key}' section") + for key, raw in paths.items(): + resolved[key] = resolve_layout_entry(str(raw), key, root) + else: + for key, default in DEFAULT_PATHS.items(): + resolved[key] = resolve_layout_entry(str(paths.get(key, default)), key, root) + if len(set(resolved.values())) != len(resolved): + raise ValueError("hub paths must point to distinct directories") + return resolved + + +def load_governance_docs(hub: Optional[Path]) -> Optional[List[str]]: + """Read the `governance_docs` file-name list from `/hub.yaml`. + + None means the hub declares no list, so governance complexity is not + applicable (n/a) rather than measured at zero; the framework doc names are + per-hub facts, never shipped defaults. + """ + if hub is None: + return None + manifest = Path(hub) / "hub.yaml" + if not manifest.is_file(): + return None + docs: List[str] = [] + in_block = False + for raw_line in read_text(manifest).splitlines(): + if not raw_line.strip() or raw_line.lstrip().startswith("#"): + continue + indent = len(raw_line) - len(raw_line.lstrip(" ")) + line = raw_line.strip() + if indent == 0: + in_block = line == "governance_docs:" + continue + if not in_block or not line.startswith("- "): + continue + name = line[2:].strip().strip("'\"") + if not name: + raise ValueError(f"governance_docs entry must name a file in {manifest}") + docs.append(name) + return docs or None + + +def load_waivers(hub: Optional[Path]) -> List[Dict[str, str]]: + """Read `doctor.waivers` from `/hub.yaml`. + + Waivers are the only way to silence a finding, so each entry must carry a + rule and a written reason; a broken block fails loud instead of guessing. + """ + if hub is None: + return [] + manifest = Path(hub) / "hub.yaml" + if not manifest.is_file(): + return [] + waivers: List[Dict[str, str]] = [] + current: Optional[Dict[str, str]] = None + in_block = False + for raw_line in read_text(manifest).splitlines(): + if not raw_line.strip() or raw_line.lstrip().startswith("#"): + continue + indent = len(raw_line) - len(raw_line.lstrip(" ")) + line = raw_line.strip() + if line.endswith("waivers:"): + in_block = True + current = None + continue + if not in_block: + continue + if indent == 0: + break + if line.startswith("- "): + current = {} + waivers.append(current) + line = line[2:].strip() + if current is None or ":" not in line: + continue + key, value = line.split(":", 1) + current[key.strip()] = value.strip().strip("'\"") + for entry in waivers: + rule = entry.get("rule", "").strip() + reason = entry.get("reason", "").strip() + if not rule or not reason: + raise ValueError(f"doctor waiver needs rule and reason in {manifest}") + entry["path"] = entry.get("path", "").strip() + return [ + {"rule": entry["rule"].strip(), "path": entry["path"], "reason": entry["reason"].strip()} + for entry in waivers + ] diff --git a/external_plugins/guster/src/guster/metrics.py b/external_plugins/guster/src/guster/metrics.py new file mode 100644 index 00000000..8786f1e8 --- /dev/null +++ b/external_plugins/guster/src/guster/metrics.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Sequence + +from .util import now_iso + + +CHI_WEIGHTS = { + "ssr": 0.20, + "rbr": 0.15, + "pcr": 0.15, + "icr": 0.15, + "frs": 0.15, + "phr": 0.10, + "sir": 0.10, +} +SCOPE_DIMS = { + "machine": ["ssr", "rbr", "pcr", "icr", "frs", "phr", "sir"], + "project": ["ssr", "pcr", "icr", "frs", "phr"], + "group": ["ssr", "pcr", "icr", "frs", "phr"], + "hub": ["ssr", "icr", "frs", "phr"], +} +RBR_BUDGET_BYTES = 20_000 +RBR_BUDGET_SKILL_CHARS = 6_000 + + +def dimensions_for_scope(scope: str) -> List[str]: + """Dimensions that enter CHI for a scope; the report must show exactly these.""" + return list(SCOPE_DIMS.get(scope) or SCOPE_DIMS["machine"]) + + +def safe_ratio(numerator: float, denominator: float) -> Optional[float]: + """A ratio, or None when the denominator is empty. + + An empty denominator means the run has no evidence, not a perfect score; a + default of 1.0 would let an unmeasured scope outrank a measured one. + """ + if denominator <= 0: + return None + return max(0.0, min(1.0, numerator / denominator)) + + +def mean_available(values: Sequence[Optional[float]]) -> Optional[float]: + present = [value for value in values if value is not None] + if not present: + return None + return sum(present) / len(present) + + +def percent(value: Optional[float]) -> Optional[float]: + return None if value is None else round(value * 100, 1) + + +def compute_metrics(registry: Dict[str, Any], policy_result: Dict[str, Any]) -> Dict[str, Any]: + stats = policy_result.get("stats", {}) + governance_declared = stats.get("governance_docs") is not None + governance_count = int(stats.get("governance_docs") or 0) + + topic_data = registry.get("topics") or {} + topic_total = int(topic_data.get("total", 0)) + topic_multi = int(topic_data.get("multi", 0)) + ssr = safe_ratio(topic_total - topic_multi, topic_total) + + resident_bytes = max(0, int(stats.get("resident_bytes", 0))) + skill_chars = max(0, int(stats.get("skill_chars", 0))) + resident_ratio = resident_bytes / RBR_BUDGET_BYTES if resident_bytes > 0 else None + skill_ratio = skill_chars / RBR_BUDGET_SKILL_CHARS if skill_chars > 0 else None + resident_score = None if resident_ratio is None else min(1.0, 1.0 / resident_ratio) + skill_score = None if skill_ratio is None else min(1.0, 1.0 / skill_ratio) + rbr = mean_available([resident_ratio, skill_ratio]) + rbr_score = mean_available([resident_score, skill_score]) + + pcr = safe_ratio( + int(stats.get("agents_dirs_compliant", 0)), + int(stats.get("agents_dirs_total", 0)), + ) + knowledge_icr = safe_ratio( + int(stats.get("knowledge_indexed", 0)), + int(stats.get("knowledge_total", 0)), + ) + agents_icr = safe_ratio( + int(stats.get("agents_indexed", 0)), + int(stats.get("agents_dirs_total", 0)), + ) + icr = mean_available([knowledge_icr, agents_icr]) + frs = safe_ratio( + int(stats.get("state_docs_fresh", 0)), + int(stats.get("state_docs_total", 0)), + ) + phr = safe_ratio( + int(stats.get("pointers_valid", 0)), + int(stats.get("pointers_checked", 0)), + ) + inline_bearers = int(stats.get("inline_bearers", 0)) + undeclared = int( + stats.get("secret_undeclared_outside", stats.get("secret_outside_root", 0)) + ) + secret_bad = ( + undeclared + + inline_bearers + + int(stats.get("secret_in_git", 0)) + + int(stats.get("secret_tracking_unknown", 0)) + ) + secret_total = int(stats.get("secret_total", 0)) + inline_bearers + sir = None if secret_total <= 0 else 1.0 - min(1.0, secret_bad / secret_total) + + dim_values: Dict[str, Optional[float]] = { + "ssr": ssr, + "rbr": rbr_score, + "pcr": pcr, + "icr": icr, + "frs": frs, + "phr": phr, + "sir": sir, + } + raw_values: Dict[str, Optional[float]] = { + "ssr": ssr, + "rbr": rbr, + "pcr": pcr, + "icr": icr, + "frs": frs, + "phr": phr, + "sir": sir, + } + scope = registry.get("scope", "machine") + scope_dims = dimensions_for_scope(scope) + scored_dims = [dim for dim in scope_dims if dim_values[dim] is not None] + if scored_dims: + total_weight = sum(CHI_WEIGHTS[dim] for dim in scored_dims) + chi = sum(CHI_WEIGHTS[dim] * dim_values[dim] for dim in scored_dims) / total_weight + else: + chi = None + + metrics: Dict[str, Optional[float]] = { + dim: (None if value is None else round(value, 4)) for dim, value in raw_values.items() + } + scores: Dict[str, Optional[float]] = {dim: percent(value) for dim, value in dim_values.items()} + + return { + "generated_at": now_iso(), + "metrics": metrics, + "scores": scores, + "chi": None if chi is None else round(chi * 100, 1), + "scope": scope, + "dims": scope_dims, + "scored_dims": scored_dims, + "unscored_dims": [dim for dim in scope_dims if dim not in scored_dims], + "raw": { + "resident_bytes": resident_bytes, + "resident_lines": int(stats.get("resident_lines", 0)), + "resident_entries": int(stats.get("resident_entries", 0)), + "skill_chars": skill_chars, + "topics_total": topic_total, + "topics_multi": topic_multi, + "knowledge_unhashed": int(topic_data.get("unhashed", 0)), + "agents_dirs_total": int(stats.get("agents_dirs_total", 0)), + "agents_dirs_compliant": int(stats.get("agents_dirs_compliant", 0)), + "agents_dirs_missing_index": int(stats.get("agents_dirs_missing_index", 0)), + "agents_dirs_missing_slots": int(stats.get("agents_dirs_missing_slots", 0)), + "agents_dirs_extra_entries": int(stats.get("agents_dirs_extra_entries", 0)), + "agents_indexed": int(stats.get("agents_indexed", 0)), + "knowledge_total": int(stats.get("knowledge_total", 0)), + "knowledge_indexed": int(stats.get("knowledge_indexed", 0)), + "state_docs_total": int(stats.get("state_docs_total", 0)), + "state_docs_fresh": int(stats.get("state_docs_fresh", 0)), + "state_docs_fresh_declared": int(stats.get("state_docs_fresh_declared", 0)), + "state_docs_fresh_from_git": int(stats.get("state_docs_fresh_from_git", 0)), + "state_docs_fresh_from_mtime": int(stats.get("state_docs_fresh_from_mtime", 0)), + "pointers_checked": int(stats.get("pointers_checked", 0)), + "pointers_valid": int(stats.get("pointers_valid", 0)), + "broken_pointers": int(stats.get("broken_pointers", 0)), + "naked_pointers": int(stats.get("naked_pointers", 0)), + "pointer_files_total": int(stats.get("pointer_files_total", 0)), + "pointer_files_scanned": int(stats.get("pointer_files_scanned", 0)), + "secret_total": int(stats.get("secret_total", 0)), + "secret_outside_root": int(stats.get("secret_outside_root", 0)), + "secret_undeclared_outside": undeclared, + "secret_tracking_unknown": int(stats.get("secret_tracking_unknown", 0)), + "secret_in_git": int(stats.get("secret_in_git", 0)), + "inline_bearers": inline_bearers, + "git_probe_failures": int(stats.get("git_probe_failures", 0)), + "scan_walk_errors": int(stats.get("scan_walk_errors", 0)), + "governance_docs": governance_count if governance_declared else None, + }, + } + + +def metric_line(result: Dict[str, Any]) -> str: + chi = result.get("chi") + parts = [f"CHI {'n/a' if chi is None else chi} ({result.get('scope')})"] + scores = result.get("scores") or {} + for dim in result.get("dims") or []: + value = scores.get(dim) + parts.append(f"{dim.upper()} {'n/a' if value is None else value}") + return " | ".join(parts) diff --git a/external_plugins/guster/src/guster/planner.py b/external_plugins/guster/src/guster/planner.py new file mode 100644 index 00000000..e76f3feb --- /dev/null +++ b/external_plugins/guster/src/guster/planner.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from .util import now_iso, save_json, slugify, unique_suffix + + +PLAN_SCHEMA_HINT = ( + '{"version": 1, "operations": [{"kind": "mkdir", "path": "..."}, ' + '{"kind": "move", "src": "...", "dst": "..."}, ' + '{"kind": "write", "path": "...", "content": "..."}, ' + '{"kind": "manual", "path": "...", "action": "...", "reason": "..."}]}' +) + +# Mirrors schemas/plan.schema.json (top-level required keys, per-kind fields, +# slug-shaped id); tests/test_schemas.py fails when the two drift apart. +PLAN_TOP_REQUIRED = ("version", "id", "created_at", "title", "notes", "operations") +PLAN_OPERATION_KINDS = ("mkdir", "move", "write", "manual") +PLAN_OPERATION_FIELDS: Dict[str, Tuple[Tuple[str, bool], ...]] = { + "mkdir": (("path", True),), + "move": (("src", True), ("dst", True)), + "write": (("path", True), ("content", False)), + "manual": (("path", False), ("action", False), ("reason", False)), +} +PLAN_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") + + +def new_plan_id(title: str) -> str: + stamp = now_iso().replace(":", "").replace("-", "").replace("+", "p") + return f"{stamp}-{slugify(title)[:40]}-{unique_suffix()}" + + +def build_plan(title: str, operations: Sequence[Dict[str, Any]], notes: Optional[str] = None) -> Dict[str, Any]: + plan_id = new_plan_id(title) + return { + "version": 1, + "id": plan_id, + "created_at": now_iso(), + "title": title, + "notes": notes or "", + "operations": list(operations), + } + + +def op_mkdir(path: Path) -> Dict[str, Any]: + return {"kind": "mkdir", "path": str(path)} + + +def op_move(src: Path, dst: Path) -> Dict[str, Any]: + return {"kind": "move", "src": str(src), "dst": str(dst)} + + +def op_write(path: Path, content: str) -> Dict[str, Any]: + return {"kind": "write", "path": str(path), "content": content} + + +def op_manual(path: str, action: str, reason: str) -> Dict[str, Any]: + return {"kind": "manual", "path": path, "action": action, "reason": reason} + + +def plan_move(src: Path, dst: Path) -> Dict[str, Any]: + # Absolute at plan time: the reviewed object must be the applied object, + # never re-resolved against whatever CWD `apply` later runs from. + src = src.expanduser().resolve() + dst = dst.expanduser().resolve() + if not src.exists(): + raise FileNotFoundError(f"source does not exist: {src}") + if dst.exists(): + raise FileExistsError(f"destination already exists: {dst}") + operations = [ + op_mkdir(dst.parent), + op_move(src, dst), + ] + return build_plan( + f"Move {src.name} to {dst.name}", + operations, + notes="Dry-run by default. Apply creates a backup.", + ) + + +def plan_init_project(project_root: Path) -> Dict[str, Any]: + root = project_root.resolve() + agents = root / ".agents" + operations: List[Dict[str, Any]] = [] + for slot in ("plans", "wiki", "evidence", "skills", "tmp", "archive"): + operations.append(op_mkdir(agents / slot)) + readme = agents / "README.md" + if not readme.exists(): + operations.append( + op_write( + readme, + "# Project Agent Context\n\n" + "| Area | Path | Purpose |\n" + "|---|---|---|\n" + "| Plans | `plans/` | Active plans, PRDs and checklists |\n" + "| Wiki | `wiki/` | Stable project knowledge |\n" + "| Evidence | `evidence/` | Screenshots, videos, test output, acceptance proof |\n" + "| Skills | `skills/` | Project-local skills |\n" + "| Tmp | `tmp/` | Scratch, 14-day TTL |\n" + "| Archive | `archive/` | Superseded plans and evidence |\n", + ) + ) + agents_md = root / "AGENTS.md" + if not agents_md.exists(): + operations.append( + op_write( + agents_md, + "# Project Instructions\n\n" + "## Scope\n\n" + "This project uses Guster's standard `.agents/` layout. " + "Put active plans in `.agents/plans/`, durable project knowledge in `.agents/wiki/`, " + "evidence in `.agents/evidence/`, and scratch files in `.agents/tmp/`.\n\n" + "## Boundaries\n\n" + "- Code and tests are the source of truth for behavior.\n" + "- Do not put credentials in this repository.\n" + "- Do not commit `.agents/` unless the repository explicitly chooses to share it.\n\n" + "## Context format\n\n" + "File paths in this file and `.agents/README.md` are markdown links or inline code, " + "never naked in prose, so `guster doctor` can verify them (CTX-03).\n", + ) + ) + return build_plan( + f"Initialize {root.name}", + operations, + notes="Creates the standard .agents layout, README index and a minimal AGENTS.md if absent.", + ) + + +def plan_from_findings(findings: Sequence[Dict[str, Any]]) -> Dict[str, Any]: + operations: List[Dict[str, Any]] = [] + for item in findings: + if item.get("severity") in {"critical", "high", "medium"}: + if item.get("rule") == "misplaced-artifact": + suggested = (item.get("evidence") or {}).get("suggested") or {} + for entry, slot in suggested.items(): + operations.append( + op_manual( + f"{item.get('path', '')}/.agents/{entry}", + f"Move under .agents/{slot}/ after updating any references", + f"misplaced-artifact: {entry} does not match a standard slot", + ) + ) + continue + raw_examples = (item.get("evidence") or {}).get("examples") + examples = ( + [str(example) for example in raw_examples if isinstance(example, str) and example] + if isinstance(raw_examples, list) + else [] + ) + for target in examples or [item.get("path", "")]: + operations.append( + op_manual( + target, + item.get("suggestion", ""), + f"{item.get('rule')}: {item.get('message')}", + ) + ) + return build_plan( + "Review context findings", + operations, + notes="This plan is a review checklist. It contains no automatic writes.", + ) + + +def save_plan(state_dir: Path, plan: Dict[str, Any]) -> Path: + path = state_dir / "plans" / f"{plan['id']}.json" + save_json(path, plan) + return path + + +def _validate_plan(data: Dict[str, Any], path: Path) -> None: + """Enforce the schemas/plan.schema.json contract before any write can happen; + a mistyped key (e.g. 'contnet') must stop the plan, not truncate a file.""" + problems: List[str] = [] + for key in PLAN_TOP_REQUIRED: + if key not in data: + problems.append(f"missing required key {key!r}") + for key in data: + if key not in PLAN_TOP_REQUIRED: + problems.append(f"unknown key {key!r}") + for key in ("title", "created_at", "notes"): + if key in data and not isinstance(data[key], str): + problems.append(f"{key} must be a string, got {type(data[key]).__name__}") + if "title" in data and data["title"] == "": + problems.append("title must be a non-empty string") + if "id" in data: + plan_id = data["id"] + if not isinstance(plan_id, str): + problems.append(f"id must be a string, got {type(plan_id).__name__}") + elif not PLAN_ID_RE.match(plan_id): + problems.append( + f"id must be a safe slug (letters, digits, '.', '_', '-'; " + f"no path separators or '..'): {plan_id!r}" + ) + for index, operation in enumerate(data["operations"], start=1): + if not isinstance(operation, dict): + problems.append(f"op {index}: operation is not a JSON object") + continue + kind = operation.get("kind") + if not isinstance(kind, str) or kind not in PLAN_OPERATION_FIELDS: + problems.append( + f"op {index}: unknown operation kind: {kind!r} " + f"(expected one of {', '.join(PLAN_OPERATION_KINDS)})" + ) + continue + fields = PLAN_OPERATION_FIELDS[kind] + allowed = {"kind"} | {field for field, _ in fields} + for key in operation: + if key not in allowed: + hint = ( + "; pointer updates are no longer applied by `apply`; " + "update the referenced file with a write op" + if key == "pointer" and kind == "move" + else "" + ) + problems.append(f"op {index}: unknown key {key!r} in {kind} operation{hint}") + for field, non_empty in fields: + if field not in operation: + problems.append(f"op {index}: {kind} operation needs {field!r}") + continue + value = operation[field] + if not isinstance(value, str): + problems.append( + f"op {index}: {kind} {field} must be a string, got {type(value).__name__}" + ) + continue + if non_empty and not value: + problems.append(f"op {index}: {kind} {field} must be a non-empty string") + elif field in ("path", "src", "dst") and not Path(value).is_absolute(): + problems.append(f"op {index}: {kind} {field} must be an absolute path: {value!r}") + if problems: + raise ValueError( + f"plan {path} does not match the plan contract: {'; '.join(problems)}; " + f"the contract is schemas/plan.schema.json, e.g. {PLAN_SCHEMA_HINT}" + ) + + +def load_plan(path: Path) -> Dict[str, Any]: + try: + raw = path.read_text(encoding="utf-8") + except OSError as exc: + raise ValueError(f"cannot read plan {path}: {exc}") from exc + try: + data = json.loads(raw) + except ValueError as exc: + raise ValueError(f"invalid plan JSON {path}: {exc}") from exc + if not isinstance(data, dict): + raise ValueError(f"plan must be a JSON object: {path}") + version = data.get("version") + if isinstance(version, bool) or not isinstance(version, int) or version != 1: + raise ValueError( + f"unsupported plan version: {version!r} (expected the integer 1); " + f"a plan is shaped {PLAN_SCHEMA_HINT}" + ) + if not isinstance(data.get("operations"), list): + raise ValueError( + "plan has no operations list; expected an 'operations' array of operation " + f"objects with a 'kind' of mkdir/move/write/manual, e.g. {PLAN_SCHEMA_HINT}" + ) + _validate_plan(data, path) + return data diff --git a/external_plugins/guster/src/guster/policy.py b/external_plugins/guster/src/guster/policy.py new file mode 100644 index 00000000..92b534b4 --- /dev/null +++ b/external_plugins/guster/src/guster/policy.py @@ -0,0 +1,1557 @@ +from __future__ import annotations + +import glob +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple + +from .discover import ScanConfig, STANDARD_AGENTS_SLOTS, in_scope_path, scope_roots +from .layout import load_waivers, resolve_hub_layout +from .util import ( + DATE_ONLY_RE, + NOTE_DATE_KEYS, + age_days, + extract_naked_pointers, + extract_pointers, + git_last_commit_dates, + has_top_level_row_keys, + is_under, + load_json_strict, + note_header_date, + parse_date, + parse_note_header, + parse_register_rows, + parse_yaml_head, + read_text, + relative, + resolve_pointer, +) + + +SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4} +COMMON_KNOWLEDGE_NAMES = {"readme.md", "agents.md", "index.md", "skill.md"} +MEDIA_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".mp4", ".mov", ".wav", ".mp3"} +RESIDENT_ENTRIES = ( + (".zcode", "AGENTS.md"), + (".dsh", "AGENTS.md"), + (".codex", "AGENTS.md"), + (".claude", "CLAUDE.md"), +) +DATE_HEADER_LINES = 40 +VERIFIED_DATE_KEYS = r"(?:最后更新|更新时间|更新于|最近核实|最后核实|已核实|verified_at|(?i:last updated|updated))" +FRESH_SOURCE_STATS = { + "declared": "state_docs_fresh_declared", + "git": "state_docs_fresh_from_git", + "mtime": "state_docs_fresh_from_mtime", +} + + +def resident_entries(home: Path) -> List[Path]: + """Host entry files that exist, deduped by real path (hosts often symlink one file).""" + seen: Set[str] = set() + entries: List[Path] = [] + for folder, name in RESIDENT_ENTRIES: + path = home / folder / name + if not path.is_file(): + continue + try: + real = str(path.resolve()) + except OSError: + continue + if real in seen: + continue + seen.add(real) + entries.append(path) + return entries + + +def finding( + rule: str, + severity: str, + title: str, + path: str, + message: str, + suggestion: str, + evidence: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + return { + "rule": rule, + "severity": severity, + "title": title, + "path": path, + "message": message, + "suggestion": suggestion, + "evidence": evidence or {}, + } + + +def suggest_slot(name: str, is_dir: bool) -> str: + """Guess the standard slot an unplaced top-level entry belongs to.""" + lowered = name.lower() + if is_dir: + if lowered in {"skills", "tools", "scripts"}: + return "skills" + if lowered in {"wiki", "knowledge"}: + return "wiki" + if any( + key in lowered + for key in ( + "shot", "screenshot", "截图", "review", "evidence", "audit", + "verify", "verification", "acceptance", "visual", "frame", + "panorama", "photo", + ) + ): + return "evidence" + if any(key in lowered for key in ("scratch", "tmp", "backup", "bak", "cache")): + return "archive" + return "plans" + suffix = Path(name).suffix.lower() + if suffix in MEDIA_SUFFIXES or suffix in {".pdf", ".xlsx"}: + return "evidence" + if suffix in {".py", ".sh", ".mjs"}: + return "skills" + if suffix in {".json", ".log", ".txt", ".bak", ".tsv", ".html"}: + return "tmp" + return "plans" + + +# `host:/path` (and URLs) reference another machine; a local three-way diff +# can never see them, so they are counted apart instead of drifting forever. +REGISTRY_REMOTE_FORM = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") + +# knowledge/v1 format contract (CTX-11). Opt-in: only files whose metadata +# face declares `schema: knowledge/v1` are validated, so plain notes stay +# untouched. Vocabularies ship with the format; zone/host/history-style local +# extensions are ignored, never validated. +KNOWLEDGE_SCHEMA = "knowledge/v1" +KNOWLEDGE_TYPES = {"state", "howto", "principle", "pitfall", "decision", "index", "reference"} +KNOWLEDGE_STATUSES = {"active", "dormant", "superseded", "archived"} +KNOWLEDGE_STALE_TYPES = {"state", "index"} +KNOWLEDGE_STALE_DAYS = 90 +REGISTER_SCHEMA = "register/v1" +REGISTER_KIND = "projects" +REGISTER_STATUSES = {"active", "pending", "on-hold", "stale", "retired"} + + +def resolve_registry_row_path(raw: str, hub_root: Path, home: Path) -> Optional[Path]: + """A register row path resolves against the hub root, `~` or an absolute.""" + value = raw.strip().rstrip("/") + if not value: + return None + if value.startswith("~"): + return home / value[2:] + candidate = Path(value) + if candidate.is_absolute(): + return candidate + return (hub_root / candidate).resolve() + + +def resolved_str(raw: str) -> str: + """Canonical spelling for finding paths: one doctor report must not mix + /tmp and /private/tmp forms of the same directory.""" + try: + return str(Path(raw).resolve()) + except OSError: + return raw + + +def collect_knowledge_v1( + artifacts: List[Dict[str, Any]], knowledge_root: Optional[Path] +) -> Tuple[List[Dict[str, Any]], List[Tuple[Path, str]]]: + """Read the metadata face of every knowledge/v1 file under the hub's + knowledge root. Files without the `schema: knowledge/v1` header are + skipped: the format contract is opt-in by declaration. A file that + declares a knowledge/* or register/* schema other than the current one is + returned as a mismatch, so a typo'd declaration stays visible instead of + silently opting out of every check.""" + docs: List[Dict[str, Any]] = [] + mismatches: List[Tuple[Path, str]] = [] + if knowledge_root is None: + return docs, mismatches + for item in artifacts: + if item.get("kind") != "knowledge": + continue + path = Path(item["path"]) + if path.suffix.lower() not in {".yaml", ".yml"}: + continue + if not is_under(path, knowledge_root): + continue + head = parse_yaml_head(read_text(path, limit=500_000)) + declared = head.get("schema") + if declared == KNOWLEDGE_SCHEMA: + docs.append({"path": path, "stem": path.stem, "head": head}) + elif isinstance(declared, str) and declared.startswith(("knowledge/", "register/")): + mismatches.append((path, declared)) + return docs, mismatches + + +def run_policy( + registry: Dict[str, Any], + config: ScanConfig, + previous_registry: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Orchestration only: seed stats, run every rule family in its own + _check_* function, then scope, waive and order the findings. + `previous_registry` is the machine table captured before this run + overwrote it; when omitted the baseline is read from disk, strictly.""" + findings: List[Dict[str, Any]] = [] + governance = registry.get("governance_docs") + stats: Dict[str, Any] = { + "pointers_checked": 0, + "pointers_valid": 0, + "broken_pointers": 0, + "naked_pointers": 0, + "agents_dirs_total": 0, + "agents_dirs_compliant": 0, + "agents_dirs_missing_index": 0, + "agents_dirs_missing_slots": 0, + "agents_dirs_extra_entries": 0, + "knowledge_total": 0, + "knowledge_indexed": 0, + "agents_index_total": 0, + "agents_indexed": 0, + "state_docs_total": 0, + "state_docs_fresh": 0, + "state_docs_fresh_declared": 0, + "state_docs_fresh_from_git": 0, + "state_docs_fresh_from_mtime": 0, + "duplicate_knowledge": 0, + "note_headers_total": 0, + "governance_docs": None if governance is None else len(governance), + "misplaced_entries": 0, + "resident_entries": 0, + "secret_total": 0, + "secret_outside_root": 0, + "secret_in_git": 0, + "secret_tracking_unknown": 0, + "secret_undeclared_outside": 0, + "pointer_files_total": 0, + "pointer_files_scanned": 0, + "git_probe_failures": 0, + "inline_bearers": 0, + "registry_rows": 0, + "registry_rows_remote": 0, + "registry_drift_dead_rows": 0, + "registry_drift_unscanned_rows": 0, + "registry_drift_unregistered": 0, + "registry_drift_stale_machine": 0, + "registry_format_bad": 0, + "knowledge_v1_total": 0, + "knowledge_format_bad": 0, + "see_also_broken": 0, + "knowledge_duplicate_ids": 0, + "knowledge_stale": 0, + "scan_walk_errors": int((registry.get("scan_errors") or {}).get("count") or 0), + } + home = config.home + hub = config.hub + scope = getattr(config, "scope", "machine") + hub_layout = resolve_hub_layout(hub) if hub else {} + artifacts = registry.get("artifacts", []) + projects = registry.get("projects", []) + git_failures: List[str] = [ + str(project.get("path") or project.get("id")) + for project in projects + if project.get("git_state") == "error" + ] + skills = registry.get("skills", []) + + findings.extend(_check_resident_budget(home, hub, scope, stats)) + if scope != "hub": + findings.extend(_check_skill_metadata(skills, home, stats)) + findings.extend(_check_skill_copy_drift(skills)) + knowledge, knowledge_index_findings = _check_knowledge_index(artifacts, stats) + findings.extend(knowledge_index_findings) + findings.extend(_check_knowledge_v1(artifacts, hub, hub_layout, stats)) + findings.extend(_check_duplicate_canonical(registry, stats)) + structured_notes, note_findings = _check_structured_notes(knowledge, stats) + findings.extend(note_findings) + findings.extend(_check_project_layout(projects, stats)) + findings.extend( + _check_state_freshness(artifacts, projects, hub, hub_layout, git_failures, stats) + ) + findings.extend(_check_pointers(artifacts, structured_notes, hub, home, config, stats)) + secrets_findings, secrets_ctx = _check_secrets(registry, home, stats) + findings.extend(secrets_findings) + findings.extend(_check_credential_registry(registry, secrets_ctx, stats)) + findings.extend(_check_governance_overlap(registry)) + findings.extend(_check_tmp_hygiene(projects)) + findings.extend(_check_plugin_manifests(registry)) + findings.extend( + _check_registry_drift(registry, config, hub_layout, stats, previous_registry) + ) + + scoped = scope_roots(config) + if scoped is not None: + kept = [ + item for item in findings if in_scope_path(item.get("path", ""), scoped) + ] + hidden = [ + item for item in findings if not in_scope_path(item.get("path", ""), scoped) + ] + stats["scoped_hidden_findings"] = len(hidden) + if hidden: + by_rule: Dict[str, int] = {} + for item in hidden: + by_rule[item["rule"]] = by_rule.get(item["rule"], 0) + 1 + kept.append( + finding( + "out-of-scope", + "info", + f"Machine-level findings hidden in {scope} scope", + "", + f"{len(hidden)} finding(s) sit outside the {scope} scope.", + "Run `guster doctor` without --root for the machine-wide view.", + {"by_rule": by_rule}, + ) + ) + findings = kept + + waived: List[Dict[str, Any]] = [] + waivers = load_waivers(hub) + if waivers: + kept, waived = [], [] + for item in findings: + reason = _waiver_reason(item, waivers) + if reason is None: + kept.append(item) + else: + item["waiver_reason"] = reason + waived.append(item) + stats["waived_findings"] = len(waived) + findings = kept + + findings.sort(key=lambda item: (SEVERITY_ORDER.get(item["severity"], 9), item["rule"], item["path"])) + for index, item in enumerate(findings, start=1): + item["id"] = f"F{index:04d}" + return {"findings": findings, "waived": waived, "stats": stats} + + +def _check_resident_budget( + home: Path, hub: Optional[Path], scope: str, stats: Dict[str, Any] +) -> List[Dict[str, Any]]: + """Resident budget: every host entry file that exists, counted once. + Owns stats keys: resident_entries, resident_bytes, resident_lines.""" + findings: List[Dict[str, Any]] = [] + user_agents = resident_entries(home) if scope != "hub" else [] + hub_agents = (hub / "AGENTS.md") if hub and (hub / "AGENTS.md").is_file() else None + resident_bytes = sum(file_size(path) for path in user_agents) + ( + file_size(hub_agents) if hub_agents else 0 + ) + resident_lines = sum(count_lines(path) for path in user_agents) + ( + count_lines(hub_agents) if hub_agents else 0 + ) + stats["resident_bytes"] = resident_bytes + stats["resident_lines"] = resident_lines + stats["resident_entries"] = len(user_agents) + if resident_bytes > 20_000: + findings.append( + finding( + "resident-budget", + "medium", + "Resident context exceeds 20 KB", + str(user_agents[0] if user_agents else (hub_agents or "")), + f"User + hub resident instruction is {resident_bytes} bytes / {resident_lines} lines.", + "Move procedures, examples and state tables into knowledge or skills; keep only rules and pointers.", + {"bytes": resident_bytes, "lines": resident_lines, "target_bytes": 20_000}, + ) + ) + for path in user_agents: + lines = count_lines(path) + if lines > 150: + findings.append( + finding( + "resident-budget", + "medium", + "User-level instruction exceeds soft line budget", + str(path), + f"{lines} lines; soft target is 150.", + "Demote long procedures and examples.", + ) + ) + if hub_agents and count_lines(hub_agents) > 50: + findings.append( + finding( + "resident-budget", + "medium", + "Hub instruction exceeds soft line budget", + str(hub_agents), + f"{count_lines(hub_agents)} lines; soft target is 50.", + "Move detail into a knowledge reference doc under knowledge/general/.", + ) + ) + return findings + + +def _check_skill_metadata( + skills: List[Dict[str, Any]], home: Path, stats: Dict[str, Any] +) -> List[Dict[str, Any]]: + """Skill metadata budget. Owns stats keys: skill_chars.""" + findings: List[Dict[str, Any]] = [] + total_skill_chars = sum(int(skill.get("description_chars", 0)) for skill in skills) + stats["skill_chars"] = total_skill_chars + for skill in skills: + chars = int(skill.get("description_chars", 0)) + if chars <= 500: + continue + findings.append( + finding( + "skill-budget", + "high", + "Skill description exceeds per-item budget", + skill.get("root", ""), + f"{skill.get('name')} description is {chars} chars; limit is 500.", + "Shorten the description to trigger conditions only; move detail into SKILL.md body.", + {"skill": skill.get("name"), "chars": chars}, + ) + ) + if total_skill_chars > 6_000: + findings.append( + finding( + "skill-budget", + "high", + "Skill metadata exceeds total budget", + str(home / ".agents" / "skills"), + f"Unique skill descriptions total {total_skill_chars} chars; budget is 6,000.", + "Trim the longest descriptions first; keep each description to one clear trigger sentence.", + {"chars": total_skill_chars, "target": 6_000}, + ) + ) + return findings + + +def _check_skill_copy_drift(skills: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Skill copy drift.""" + findings: List[Dict[str, Any]] = [] + for skill in skills: + copies = skill.get("copies", []) + hashes = {copy.get("hash") for copy in copies if copy.get("hash")} + if len(copies) > 1 and len(hashes) > 1: + findings.append( + finding( + "skill-copy-drift", + "high", + "Skill copies drifted", + skill.get("root", ""), + f"{skill.get('name')} has {len(hashes)} different copies.", + "Make copies byte-identical or replace the copy strategy with symlinks where the host supports it.", + {"copies": copies}, + ) + ) + return findings + + +def _check_knowledge_index( + artifacts: List[Dict[str, Any]], stats: Dict[str, Any] +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Knowledge index coverage and duplicates. + Owns stats keys: knowledge_total, knowledge_indexed. Returns the knowledge + artifact list for the downstream structured-notes check.""" + findings: List[Dict[str, Any]] = [] + knowledge = [item for item in artifacts if item.get("kind") == "knowledge"] + stats["knowledge_total"] = len(knowledge) + orphan_groups: Dict[str, List[Dict[str, Any]]] = {} + for item in knowledge: + if item.get("indexed") is True: + stats["knowledge_indexed"] += 1 + elif item.get("indexed") is False: + if Path(item["path"]).name.lower() in COMMON_KNOWLEDGE_NAMES: + # An index does not index itself. + continue + orphan_groups.setdefault(item.get("scope", "unknown"), []).append(item) + + for orphan_scope, items in orphan_groups.items(): + severity = "medium" if orphan_scope != "local" else "low" + examples = [str(Path(item["path"]).resolve()) for item in items[:10]] + findings.append( + finding( + "orphan-knowledge", + severity, + "Knowledge files are not indexed", + str(items[0]["path"]), + f"{len(items)} knowledge file(s) in scope '{orphan_scope}' are absent from the relevant README index.", + "Add each file to its README index, or merge/delete it if obsolete.", + {"count": len(items), "examples": examples}, + ) + ) + return knowledge, findings + + +def _check_knowledge_v1( + artifacts: List[Dict[str, Any]], + hub: Optional[Path], + hub_layout: Dict[str, Path], + stats: Dict[str, Any], +) -> List[Dict[str, Any]]: + """knowledge/v1 format contract (CTX-11): files that declare the schema + header get strict format, cross-reference and freshness checks; plain + notes are never touched. These findings stay outside CHI (same + discipline as registry-drift) so scores stay comparable. + Owns stats keys: knowledge_v1_total, knowledge_format_bad, see_also_broken, + knowledge_stale, knowledge_duplicate_ids.""" + findings: List[Dict[str, Any]] = [] + v1_docs: List[Dict[str, Any]] = [] + schema_mismatches: List[Tuple[Path, str]] = [] + knowledge_root = hub_layout.get("knowledge") if hub is not None else None + if knowledge_root is not None: + v1_docs, schema_mismatches = collect_knowledge_v1(artifacts, knowledge_root) + stats["knowledge_v1_total"] = len(v1_docs) + knowledge_yaml_stems: Set[str] = set() + if knowledge_root is not None: + for item in artifacts: + if item.get("kind") != "knowledge": + continue + candidate = Path(item["path"]) + if candidate.suffix.lower() in {".yaml", ".yml"} and is_under(candidate, knowledge_root): + knowledge_yaml_stems.add(candidate.stem) + for mismatch_path, declared in schema_mismatches: + findings.append( + finding( + "knowledge-format", + "info", + "knowledge file declares an unsupported schema", + str(mismatch_path), + f"declares schema {declared} — current supported is {KNOWLEDGE_SCHEMA}; " + "the file is skipped by every knowledge/v1 check.", + f"Fix the header to '{KNOWLEDGE_SCHEMA}', or drop the schema key to keep it a plain note.", + {"declared": declared, "current": KNOWLEDGE_SCHEMA}, + ) + ) + known_ids: Set[str] = set() + for doc in v1_docs: + doc_id = doc["head"].get("id") + if isinstance(doc_id, str) and doc_id: + known_ids.add(doc_id) + format_bad: List[Tuple[str, List[str]]] = [] + broken_refs: List[Tuple[str, List[str]]] = [] + stale_files: List[str] = [] + id_owners: Dict[str, List[str]] = {} + today = datetime.now(timezone.utc).date() + for doc in v1_docs: + head = doc["head"] + path = doc["path"] + issues: List[str] = [] + doc_id = head.get("id") + if not isinstance(doc_id, str) or not doc_id: + issues.append("id is missing") + elif doc_id != doc["stem"]: + issues.append(f"id {doc_id!r} does not match file stem {doc['stem']!r}") + else: + id_owners.setdefault(doc_id, []).append(str(path)) + doc_type = head.get("type") + if doc_type is None: + issues.append("type is missing (vocabulary: " + ", ".join(sorted(KNOWLEDGE_TYPES)) + ")") + elif doc_type not in KNOWLEDGE_TYPES: + issues.append(f"type {doc_type!r} not in vocabulary {sorted(KNOWLEDGE_TYPES)}") + if "status" in head and not isinstance(head.get("status"), str): + issues.append("status must be a scalar like active, not a block") + doc_status = head.get("status") + if isinstance(doc_status, str) and doc_status not in KNOWLEDGE_STATUSES: + issues.append(f"status {doc_status!r} not in vocabulary {sorted(KNOWLEDGE_STATUSES)}") + topic = head.get("topic") + if not isinstance(topic, str) or not topic.strip(): + issues.append("topic is missing or empty") + for date_key in ("as_of", "updated"): + value = head.get(date_key) + if not isinstance(value, str) or not DATE_ONLY_RE.match(value): + issues.append(f"{date_key} is missing or not YYYY-MM-DD: {value!r}") + as_of_value, updated_value = head.get("as_of"), head.get("updated") + if ( + isinstance(as_of_value, str) + and isinstance(updated_value, str) + and DATE_ONLY_RE.match(as_of_value) + and DATE_ONLY_RE.match(updated_value) + and updated_value < as_of_value + ): + issues.append(f"updated {updated_value} predates as_of {as_of_value}") + if "see_also" in head and not isinstance(head.get("see_also"), list): + issues.append("see_also must be an inline list like [id-a, id-b]") + see = head.get("see_also") + if isinstance(see, list): + missing = sorted( + ref + for ref in see + if isinstance(ref, str) + and ref + and "/" not in ref + and not ref.startswith("~") + and not REGISTRY_REMOTE_FORM.match(ref) + and ref not in known_ids + ) + if missing: + broken_refs.append((str(path), missing)) + if issues: + format_bad.append((str(path), issues)) + if ( + doc_type in KNOWLEDGE_STALE_TYPES + and doc_status == "active" + and isinstance(updated_value, str) + and DATE_ONLY_RE.match(updated_value) + ): + age = (today - datetime.strptime(updated_value, "%Y-%m-%d").date()).days + if age > KNOWLEDGE_STALE_DAYS: + stale_files.append(f"{path} (updated {updated_value})") + stats["knowledge_format_bad"] = len(format_bad) + stats["see_also_broken"] = len(broken_refs) + stats["knowledge_stale"] = len(stale_files) + if format_bad: + findings.append( + finding( + "knowledge-format", + "medium", + "knowledge/v1 files violate the format contract", + format_bad[0][0], + f"{len(format_bad)} file(s) declare knowledge/v1 but break the contract: " + + "; ".join( + f"{Path(p).name}: {', '.join(items[:3])}" for p, items in format_bad[:5] + ), + "Fix the metadata per the CTX-11 contract (reference/spec.md), or drop the schema header.", + { + "count": len(format_bad), + "issues": [ + f"{Path(p).name}: {', '.join(items[:3])}" + for p, items in format_bad[:20] + ], + }, + ) + ) + if broken_refs: + first_path, first_missing = broken_refs[0] + described = [ + f"{ref} (file exists but does not declare knowledge/v1)" + if ref in knowledge_yaml_stems + else ref + for ref in first_missing[:5] + ] + findings.append( + finding( + "see-also-broken", + "medium", + "see_also references point at ids that do not exist", + first_path, + f"{len(broken_refs)} file(s) reference unknown ids; " + f"first: {', '.join(described)}.", + "Point see_also at existing ids only; rename both sides together, or drop the stale reference.", + {"count": len(broken_refs), "refs": first_missing[:10]}, + ) + ) + duplicate_ids = {doc_id: owners for doc_id, owners in id_owners.items() if len(owners) > 1} + stats["knowledge_duplicate_ids"] = len(duplicate_ids) + if duplicate_ids: + first_id, owners = sorted(duplicate_ids.items())[0] + findings.append( + finding( + "duplicate-knowledge-id", + "medium", + "The same knowledge id lives in more than one file", + owners[0], + f"{len(duplicate_ids)} id(s) are duplicated; {first_id}: {owners[:3]}.", + "One fact, one file: merge the files or split the ids so each id has exactly one home.", + {"count": len(duplicate_ids), "ids": sorted(duplicate_ids)[:10]}, + ) + ) + if stale_files: + findings.append( + finding( + "knowledge-stale", + "medium", + "State-flavored knowledge files are past the freshness SLA", + stale_files[0], + f"{len(stale_files)} active state/index file(s) not updated within " + f"{KNOWLEDGE_STALE_DAYS} days.", + "Re-verify the content and bump `updated`, or set status dormant/archived when it no longer needs review.", + {"count": len(stale_files), "files": stale_files[:10]}, + ) + ) + return findings + + +def _check_duplicate_canonical( + registry: Dict[str, Any], stats: Dict[str, Any] +) -> List[Dict[str, Any]]: + """Same-fact-many-homes from the scan's topic grouping. + Owns stats keys: duplicate_knowledge.""" + findings: List[Dict[str, Any]] = [] + multi_topics = (registry.get("topics") or {}).get("multi_topics") or [] + stats["duplicate_knowledge"] = len(multi_topics) + if multi_topics: + examples = [path for topic in multi_topics[:4] for path in topic.get("paths", [])[:2]] + reasons = sorted({str(topic.get("reason")) for topic in multi_topics}) + findings.append( + finding( + "duplicate-canonical", + "medium", + "The same fact has more than one home", + examples[0] if examples else "", + f"{len(multi_topics)} topic(s) have more than one source ({', '.join(reasons)}).", + "Pick one canonical home and replace the others with pointers; keep a copy only when it is generated and verified.", + { + "count": len(multi_topics), + "topics": [topic.get("key") for topic in multi_topics[:10]], + "examples": examples[:10], + }, + ) + ) + return findings + + +def _check_structured_notes( + knowledge: List[Dict[str, Any]], stats: Dict[str, Any] +) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """Structured note headers (CTX-09). Opt-in: only notes whose front-matter + declares a ``type:`` are checked, so legacy prose notes never fire. The + format follows the Obsidian properties convention (see reference/spec.md); + truth pointers live in the body as CTX-03 links, not in the header. + Owns stats keys: note_headers_total. Returns the structured note list for + the pointer check.""" + findings: List[Dict[str, Any]] = [] + structured_notes: List[Dict[str, Any]] = [] + for item in knowledge: + if Path(item["path"]).suffix.lower() != ".md": + continue + text = read_text(Path(item["path"]), limit=200_000) + header = parse_note_header(text) + if header is None: + continue + stats["note_headers_total"] += 1 + structured_notes.append(item) + problems: List[str] = [] + if header.get("_malformed"): + problems.append( + f"front-matter cannot be parsed (line {header.get('line')}: " + f"{header.get('text')})" + ) + else: + if not note_header_date(header): + problems.append( + "no machine-readable date (one of " + f"{', '.join(NOTE_DATE_KEYS)} as YYYY-MM-DD)" + ) + if problems: + findings.append( + finding( + "note-header-malformed", + "medium", + "Structured note header is not well-formed", + item["path"], + "; ".join(problems) + ".", + "Fix the front-matter (Obsidian properties convention: type + a " + "date key), or drop the type key to make it a plain note.", + {"problems": problems}, + ) + ) + return structured_notes, findings + + +def _check_project_layout( + projects: List[Dict[str, Any]], stats: Dict[str, Any] +) -> List[Dict[str, Any]]: + """Project .agents layout. Owns stats keys: agents_dirs_total, + agents_indexed, agents_dirs_compliant, agents_dirs_missing_index, + agents_dirs_missing_slots, agents_dirs_extra_entries, misplaced_entries.""" + findings: List[Dict[str, Any]] = [] + for project in projects: + if not project.get("agents_dir"): + continue + if project.get("container"): + # A workspace/group root keeps container-level context; judging it as + # a project would score scaffolding the group never needed. + continue + stats["agents_dirs_total"] += 1 + if project.get("has_index"): + stats["agents_indexed"] += 1 + unknown = [ + entry + for entry in project.get("agents_entries", []) + if entry not in STANDARD_AGENTS_SLOTS and entry not in {"README.md", "AGENTS.md"} + ] + slots_ok = set(STANDARD_AGENTS_SLOTS).issubset(set(project.get("standard_slots", []))) + if project.get("has_index") and slots_ok and not unknown: + stats["agents_dirs_compliant"] += 1 + elif not project.get("has_index"): + stats["agents_dirs_missing_index"] += 1 + elif not slots_ok: + stats["agents_dirs_missing_slots"] += 1 + else: + stats["agents_dirs_extra_entries"] += 1 + if unknown: + stats["misplaced_entries"] += len(unknown) + agents_root = Path(project.get("agents_dir") or "") + suggested = { + entry: suggest_slot(entry, (agents_root / entry).is_dir()) + for entry in unknown + } + findings.append( + finding( + "misplaced-artifact", + "medium", + "Project .agents has task-shaped top-level entries", + resolved_str(str(project["path"])), + f"{len(unknown)} top-level entr{'y' if len(unknown) == 1 else 'ies'} do not match the standard slots.", + "Move task bundles under evidence/, plans/, wiki/ or archive/, then add a README index.", + {"entries": unknown[:100], "suggested": suggested}, + ) + ) + return findings + + +def _check_state_freshness( + artifacts: List[Dict[str, Any]], + projects: List[Dict[str, Any]], + hub: Optional[Path], + hub_layout: Dict[str, Path], + git_failures: List[str], + stats: Dict[str, Any], +) -> List[Dict[str, Any]]: + """State freshness. Plans are state-plane documents too, even though their + kind is "plan"; select on the plane so project/group scopes are not + vacuously fresh. The date ladder is declared (verified_at / updated / + 最后更新 / filename date) → owning repository's git date → file mtime, + each tier reported as its own freshness source. + Owns stats keys: state_docs_total, state_docs_fresh, + state_docs_fresh_declared, state_docs_fresh_from_git, + state_docs_fresh_from_mtime, git_probe_failures. Mutates git_failures.""" + findings: List[Dict[str, Any]] = [] + state_prefixes = ["handoff/", "provision/", "register/", ".agents/plans/"] + if hub_layout: + state_prefixes.append(relative(hub_layout["projects"], hub).rstrip("/") + "/") + state_docs = [ + item + for item in artifacts + if item.get("plane") == "state" + and any(item.get("rel", "").startswith(prefix) for prefix in state_prefixes) + ] + stats["state_docs_total"] = len(state_docs) + commit_dates = git_last_commit_dates(hub) if hub else {} + if commit_dates is None: + if str(hub) not in git_failures: + git_failures.append(str(hub)) + commit_dates = {} + project_roots = {str(project["id"]): Path(project["path"]) for project in projects} + project_commit_dates: Dict[str, Dict[str, str]] = {} + stale_missing: List[str] = [] + stale_old: List[Tuple[str, str]] = [] + for item in state_docs: + path = Path(item["path"]) + text = read_text(path, limit=500_000) + verified = ( + item.get("verified_at") + or extract_verified_date(text) + or extract_filename_date(path) + ) + source = "declared" + if not verified: + project_root = project_roots.get(str(item.get("project"))) + if project_root is None: + verified = commit_dates.get(item.get("rel", ""), "") + else: + key = str(project_root) + if key not in project_commit_dates: + dates = git_last_commit_dates(project_root) + if dates is None: + if key not in git_failures: + git_failures.append(key) + dates = {} + project_commit_dates[key] = dates + verified = project_commit_dates[key].get(item.get("rel", ""), "") + source = "git" + if not verified and item.get("mtime"): + # Files git never saw (e.g. gitignored `.agents/` docs) still carry + # an mtime; count it as its own source instead of a freshness cliff. + verified = ( + datetime.fromtimestamp(float(item["mtime"]), tz=timezone.utc) + .date() + .isoformat() + ) + source = "mtime" + if not verified: + stale_missing.append(str(path)) + continue + parsed = parse_date(verified) + if parsed is None: + stale_missing.append(str(path)) + continue + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + if (datetime.now(timezone.utc) - parsed.astimezone(timezone.utc)).days <= 90: + stats["state_docs_fresh"] += 1 + stats[FRESH_SOURCE_STATS[source]] += 1 + else: + stale_old.append((str(path), verified)) + if stale_missing: + findings.append( + finding( + "stale-state", + "medium", + "State documents lack a verifiable freshness date", + stale_missing[0], + f"{len(stale_missing)} state document(s) have no verified_at or update date.", + "Add verified_at to project state docs/handoffs/environment docs; refresh active ones before relying on them.", + {"count": len(stale_missing), "examples": stale_missing[:10]}, + ) + ) + if stale_old: + findings.append( + finding( + "stale-state", + "medium", + "State documents are older than 90 days", + stale_old[0][0], + f"{len(stale_old)} state document(s) are older than the 90-day SLA.", + "Re-verify against git/runtime and update verified_at, or archive the document.", + {"count": len(stale_old), "examples": stale_old[:10]}, + ) + ) + stats["git_probe_failures"] = len(git_failures) + if git_failures: + findings.append( + finding( + "git-unavailable", + "high", + "Repository state could not be read", + git_failures[0], + f"{len(git_failures)} repository(ies) could not be queried; tracked state and commit dates are unknown.", + "Fix git access for these repositories and re-run; SIR and FRS do not count them as clean.", + {"count": len(git_failures), "repos": git_failures[:10]}, + ) + ) + return findings + + +def _check_pointers( + artifacts: List[Dict[str, Any]], + structured_notes: List[Dict[str, Any]], + hub: Optional[Path], + home: Path, + config: ScanConfig, + stats: Dict[str, Any], +) -> List[Dict[str, Any]]: + """Pointers. Owns stats keys: pointer_files_total, pointer_files_scanned, + pointers_checked, pointers_valid, broken_pointers, naked_pointers.""" + findings: List[Dict[str, Any]] = [] + pointer_files = [ + item + for item in artifacts + if Path(item["path"]).suffix.lower() == ".md" + and ( + Path(item["path"]).name in {"AGENTS.md", "CLAUDE.md"} + or ( + Path(item["path"]).name == "README.md" + and ( + # Project .agents indexes and hub knowledge indexes are + # instruction artifacts, including custom hub layouts; tool + # entry docs (hub sections and machine zones) carry + # pointers like any other entry document. + item.get("kind") == "instruction" + or item.get("kind") == "tool" + or item.get("path", "").endswith( + ( + "/knowledge/README.md", + "/tools/README.md", + "/.agents/README.md", + "/.agents/wiki/README.md", + ) + ) + ) + ) + ) + ] + broken_by_file: List[Dict[str, Any]] = [] + naked_by_file: List[Dict[str, Any]] = [] + # Structured notes (CTX-09) opt their whole body into pointer checks, so a + # pointer card's truth links are verified like any CTX-03 pointer. + seen_pointer_paths = {item["path"] for item in pointer_files} + pointer_files += [ + item for item in structured_notes if item["path"] not in seen_pointer_paths + ] + stats["pointer_files_total"] = len(pointer_files) + scanned_pointer_files = pointer_files[:500] + stats["pointer_files_scanned"] = len(scanned_pointer_files) + for item in scanned_pointer_files: + path = Path(item["path"]) + text = read_text(path, limit=1_000_000) + pointers = extract_pointers(text) + broken: List[str] = [] + for pointer in pointers: + resolved = resolve_pointer(pointer, path, hub, home, config_roots=config.roots) + if resolved is None: + continue + stats["pointers_checked"] += 1 + if resolved.exists(): + stats["pointers_valid"] += 1 + else: + broken.append(pointer) + stats["broken_pointers"] += 1 + if broken: + broken_by_file.append( + finding( + "broken-pointer", + "high", + "Broken context pointer", + resolved_str(str(path)), + f"{len(broken)} pointer(s) do not resolve.", + "Fix the path or replace it with a pointer that states when to read the target.", + {"pointers": broken[:10]}, + ) + ) + naked = extract_naked_pointers(text) + if naked: + stats["naked_pointers"] += len(naked) + naked_by_file.append( + finding( + "naked-pointer", + "medium", + "Path in prose is not machine-verifiable", + resolved_str(str(path)), + f"{len(naked)} file path(s) sit outside a link or code span, so the pointer check never sees them.", + "Wrap each path in a markdown link or inline code (CTX-03, reference/spec.md).", + {"paths": naked[:10]}, + ) + ) + findings.extend(broken_by_file[:100]) + findings.extend(naked_by_file[:100]) + return findings + + +def _check_secrets( + registry: Dict[str, Any], home: Path, stats: Dict[str, Any] +) -> Tuple[ + List[Dict[str, Any]], + Tuple[Dict[str, Any], List[Dict[str, str]], Set[str], List[str]], +]: + """Secrets. Owns stats keys: secret_total, inline_bearers, + secret_outside_root, secret_tracking_unknown, secret_in_git, + secret_undeclared_outside. Returns the findings plus the + credential-registry context (registry_info, entries, declared_paths, + missing_locations) for the registry coverage check.""" + findings: List[Dict[str, Any]] = [] + secret_data = registry.get("secrets", {}) + secret_files = secret_data.get("files", []) + inline = secret_data.get("inline_bearers", []) + stats["secret_total"] = len(secret_files) + stats["inline_bearers"] = sum(int(item.get("count", 0)) for item in inline) + outside = [item for item in secret_files if not item.get("canonical")] + stats["secret_outside_root"] = len(outside) + stats["secret_tracking_unknown"] = sum( + 1 for item in secret_files if item.get("git_tracked") is None + ) + registry_info = secret_data.get("registry") or {} + entries = registry_info.get("entries") or [] + declared_paths: Set[str] = set() + missing_locations: List[str] = [] + declared_entries: Set[str] = set() + for entry in entries: + for location in str(entry.get("locations", "")).split(";"): + location = location.strip() + if not location or location == "-": + continue + pattern = str(home / location[2:]) if location.startswith("~/") else os.path.expanduser(location) + matches = glob.glob(pattern) + if not matches: + missing_locations.append(location) + continue + declared_entries.add(str(entry.get("id"))) + for match in matches: + try: + declared_paths.add(str(Path(match).resolve())) + except OSError: + continue + tracked = [item for item in secret_files if item.get("git_tracked")] + stats["secret_in_git"] = len(tracked) + for item in tracked[:20]: + findings.append( + finding( + "secret-in-git", + "critical", + "Credential file is tracked by git", + item["path"], + "A credential-bearing file is tracked; this must be removed from the index and history, not just deleted.", + "Stop, rotate the credential, remove it from the index/history, and add an ignore rule.", + ) + ) + outside_undeclared = [] + for item in outside: + try: + if str(Path(item["path"]).resolve()) not in declared_paths: + outside_undeclared.append(item["path"]) + except OSError: + continue + stats["secret_undeclared_outside"] = len(outside_undeclared) + if outside_undeclared: + findings.append( + finding( + "secret-outside-root", + "high", + "Credential files are unregistered outside the canonical root", + outside_undeclared[0], + f"{len(outside_undeclared)} credential-bearing file(s) live outside" + " ~/.agents/credentials and are not in the registry.", + "Register each location with its scope, or move the value into the canonical root.", + { + "count": len(outside_undeclared), + "declared_outside_root": len(outside) - len(outside_undeclared), + "examples": outside_undeclared[:10], + }, + ) + ) + for item in inline: + findings.append( + finding( + "secret-inline", + "high", + "Literal bearer token in host configuration", + item["path"], + f"{item.get('count', 0)} bearer token(s) are inline; values are redacted.", + "Replace with an environment/credential reference supported by the host; rotate if the file was shared.", + {"count": item.get("count", 0), "redacted": item.get("redacted", [])}, + ) + ) + return findings, (registry_info, entries, declared_paths, missing_locations) + + +def _check_credential_registry( + registry: Dict[str, Any], + secrets_ctx: Tuple[Dict[str, Any], List[Dict[str, str]], Set[str], List[str]], + stats: Dict[str, Any], +) -> List[Dict[str, Any]]: + """Credential registry coverage. Owns stats keys: credentials_registered, + credentials_by_scope, credentials_undeclared, credentials_missing.""" + registry_info, entries, declared_paths, missing_locations = secrets_ctx + findings: List[Dict[str, Any]] = [] + secret_files = (registry.get("secrets") or {}).get("files", []) + stats["credentials_registered"] = len(entries) + stats["credentials_by_scope"] = {} + for entry in entries: + scope = entry.get("scope", "unknown") + stats["credentials_by_scope"][scope] = stats["credentials_by_scope"].get(scope, 0) + 1 + if entries: + undeclared = [] + for item in secret_files: + try: + resolved = str(Path(item["path"]).resolve()) + except OSError: + continue + if resolved not in declared_paths: + undeclared.append(item["path"]) + stats["credentials_undeclared"] = len(undeclared) + stats["credentials_missing"] = len(missing_locations) + if undeclared: + findings.append( + finding( + "secret-undeclared", + "medium", + "Credential file is not registered", + undeclared[0], + f"{len(undeclared)} credential file(s) are absent from ~/.agents/credentials/registry.md.", + "Add an entry with scope (company/personal), site, account, location and keys.", + {"count": len(undeclared), "examples": undeclared[:10]}, + ) + ) + if missing_locations: + findings.append( + finding( + "secret-missing", + "medium", + "Registered credential location does not exist", + registry_info.get("path") or "", + f"{len(missing_locations)} registered location(s) resolve to nothing.", + "Rotate/restore the secret or drop the stale registry row.", + {"count": len(missing_locations), "examples": missing_locations[:10]}, + ) + ) + return findings + + +def _check_governance_overlap(registry: Dict[str, Any]) -> List[Dict[str, Any]]: + """Governance document overlap.""" + findings: List[Dict[str, Any]] = [] + governance_docs = registry.get("governance_docs") or [] + if len(governance_docs) > 4: + findings.append( + finding( + "governance-overlap", + "high", + "Too many documents claim to define context governance", + governance_docs[0], + f"{len(governance_docs)} primary framework documents are present.", + "Keep one primary framework and label the rest as facets or history.", + {"docs": governance_docs}, + ) + ) + return findings + + +def _check_tmp_hygiene(projects: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Tmp and evidence hygiene.""" + findings: List[Dict[str, Any]] = [] + tmp_old: List[str] = [] + for project in projects: + agents_dir = project.get("agents_dir") + if not agents_dir: + continue + tmp_root = Path(agents_dir) / "tmp" + if not tmp_root.is_dir(): + continue + for path in tmp_root.rglob("*"): + age = age_days(path) + if age is not None and age > 14 and path.is_file(): + tmp_old.append(str(path)) + if tmp_old: + findings.append( + finding( + "tmp-overdue", + "low", + "Temporary files exceed the 14-day TTL", + tmp_old[0], + f"{len(tmp_old)} file(s) in .agents/tmp are older than 14 days.", + "Archive what matters and delete the rest; tmp is not a knowledge store.", + {"count": len(tmp_old), "examples": tmp_old[:10]}, + ) + ) + return findings + + +def _check_plugin_manifests(registry: Dict[str, Any]) -> List[Dict[str, Any]]: + """Plugin manifest drift. A half-installed plugin (manifest file missing) + is a high finding in its own right and stays in the version comparison as + a null source, so it can never pass as green by being excluded.""" + findings: List[Dict[str, Any]] = [] + manifests = registry.get("plugin_manifests", {}) + for key, value in sorted(manifests.items()): + if isinstance(value, dict) and not value.get("exists"): + path = str(value.get("path") or "") + findings.append( + finding( + "host-manifest-drift", + "high", + "Plugin manifest is missing", + path, + f"plugin manifest missing: {path}", + "Reinstall or repair the plugin so every host manifest (codex, zcode, package) is present.", + {"key": key, "path": path}, + ) + ) + versions = { + key: (value.get("version") if value.get("exists") else None) + for key, value in manifests.items() + if isinstance(value, dict) and (not value.get("exists") or value.get("version")) + } + if len(set(versions.values())) > 1: + findings.append( + finding( + "host-manifest-drift", + "high", + "Plugin manifests disagree on version", + str(manifests.get("package", {}).get("path", "")), + f"Manifest versions: {versions}", + "Synchronize Codex, ZCode and package versions before release.", + {"versions": versions}, + ) + ) + return findings + + +def _check_registry_drift( + registry: Dict[str, Any], + config: ScanConfig, + hub_layout: Dict[str, Path], + stats: Dict[str, Any], + previous_registry: Optional[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Registry drift (CTX-10): the machine table (registry.json), the hub's + register (register/projects.yaml spec.path entries) and the git roots on + disk each claim to know which projects exist; disagreement is a finding, + never a CHI dimension. The human side needs a hub; the machine↔disk edge + stands on its own for hub-less users. + Owns stats keys: registry_rows, registry_rows_remote, + registry_drift_dead_rows, registry_drift_unscanned_rows, + registry_format_bad, registry_drift_unregistered, + registry_drift_stale_machine.""" + findings: List[Dict[str, Any]] = [] + home = config.home + hub = config.hub + projects = registry.get("projects", []) + if hub is not None and not (hub / "hub.yaml").is_file() and not (hub / "AGENTS.md").is_file(): + findings.append( + finding( + "hub-manifest-missing", + "high", + "Attached hub is not recognizable as a hub", + str(hub), + f"{hub} has neither hub.yaml nor AGENTS.md, so no hub contract applies to it.", + "Point config/--hub at a real hub (`guster hub init`), or drop the hub setting.", + {"hub": str(hub)}, + ) + ) + machine_paths: Set[str] = set() + machine_projects: Set[str] = set() + for project in projects: + try: + key = str(Path(project["path"]).resolve()) + except OSError: + continue + machine_paths.add(key) + if not project.get("container"): + machine_projects.add(key) + human_paths: Set[str] = set() + dead_rows: List[str] = [] + unscanned_rows: List[str] = [] + register_path: Optional[Path] = None + register_rows: List[Dict[str, str]] = [] + register_text = "" + register_display = "" + if hub is not None: + register_path = hub / "register" / "projects.yaml" + register_display = resolved_str(str(register_path)) + if not register_path.is_file() and hub_layout.get("projects") is not None: + findings.append( + finding( + "registry-drift", + "medium", + "Hub has no register/projects.yaml", + register_display, + "The hub declares a projects section but carries no register table, so the " + "CTX-10 three-way diff cannot run (0.9.4 replaced projects/_registry.md).", + "Create register/projects.yaml (register/v1 header + one row per project); " + "see CHANGELOG 0.9.4 for the migration.", + {"hub": resolved_str(str(hub))}, + ) + ) + if register_path.is_file(): + register_text = read_text(register_path, limit=1_000_000) + register_rows = parse_register_rows(register_text) + hub_root = hub.expanduser().resolve() + for row in register_rows: + raw = row.get("path") or "" + if not raw: + continue + if REGISTRY_REMOTE_FORM.match(raw): + stats["registry_rows_remote"] += 1 + continue + resolved = resolve_registry_row_path(raw, hub_root, home) + if resolved is None: + continue + try: + key = str(resolved.resolve()) + except OSError: + continue + human_paths.add(key) + if not resolved.exists(): + dead_rows.append(raw) + elif getattr(config, "scope", "machine") == "machine" and key not in machine_paths: + unscanned_rows.append(raw) + stats["registry_rows"] = len(register_rows) + stats["registry_drift_dead_rows"] = len(dead_rows) + stats["registry_drift_unscanned_rows"] = len(unscanned_rows) + if register_path is not None and register_path.is_file(): + format_issues: List[str] = [] + header = parse_yaml_head(register_text) + if header.get("schema") != REGISTER_SCHEMA: + format_issues.append(f"schema is {header.get('schema')!r}, expected '{REGISTER_SCHEMA}'") + if header.get("kind") != REGISTER_KIND: + format_issues.append(f"kind is {header.get('kind')!r}, expected '{REGISTER_KIND}'") + if not register_rows and has_top_level_row_keys(register_text): + format_issues.append( + "no project rows parsed — expected top-level `- id:` list items under `projects:`" + ) + seen_ids: Dict[str, int] = {} + for row in register_rows: + row_id = row.get("id") or "" + if not row_id: + format_issues.append("a row has no id") + else: + seen_ids[row_id] = seen_ids.get(row_id, 0) + 1 + if not row.get("path"): + format_issues.append(f"row {row_id or '(no id)'} has no spec.path") + as_of = row.get("as_of") or "" + if as_of and not DATE_ONLY_RE.match(as_of): + format_issues.append(f"row {row_id or '(no id)'} as_of is not YYYY-MM-DD: {as_of!r}") + row_status = row.get("status") or "" + if row_status and row_status not in REGISTER_STATUSES: + format_issues.append( + f"row {row_id or '(no id)'} status {row_status!r} not in vocabulary " + f"{sorted(REGISTER_STATUSES)}" + ) + for row_id, count in seen_ids.items(): + if count > 1: + format_issues.append(f"id {row_id!r} appears {count} times") + stats["registry_format_bad"] = len(format_issues) + if format_issues: + findings.append( + finding( + "registry-format", + "medium", + "Register table violates the register/v1 contract", + register_display, + "; ".join(format_issues[:10]), + "Fix the header (schema/kind) and each row (unique id, spec.path, YYYY-MM-DD as_of).", + {"count": len(format_issues), "issues": format_issues[:20]}, + ) + ) + if dead_rows: + findings.append( + finding( + "registry-drift", + "medium", + "Register rows point at paths that do not exist", + register_display, + f"{len(dead_rows)} row(s) in the register name a path that is not on disk.", + "Fix the row path, or delete the row; a register row must resolve to the project directory.", + {"count": len(dead_rows), "rows": dead_rows[:10]}, + ) + ) + if unscanned_rows: + findings.append( + finding( + "registry-drift", + "medium", + "Register rows are outside the machine scan", + register_display, + f"{len(unscanned_rows)} registered path(s) exist but the scan never records them; " + "the machine table cannot see what the register claims.", + "Add the containing directory to the configured roots (guster init/zones), or fix the row path.", + {"count": len(unscanned_rows), "rows": unscanned_rows[:10]}, + ) + ) + unregistered = sorted(machine_projects - human_paths) + stats["registry_drift_unregistered"] = len(unregistered) + if unregistered: + findings.append( + finding( + "registry-drift", + "low", + "Scanned projects are not registered in the hub", + unregistered[0], + f"{len(unregistered)} scanned project(s) have no row in the register.", + "Add a register/projects.yaml row per project, or waive deliberately unregistered ones in hub.yaml.", + {"count": len(unregistered), "examples": unregistered[:10]}, + ) + ) + if hub is not None and (hub / "register").is_dir(): + try: + register_siblings = sorted((hub / "register").iterdir()) + except OSError: + register_siblings = [] + for extra in register_siblings: + if not extra.is_file() or extra.suffix.lower() not in {".yaml", ".yml"}: + continue + if register_path is not None and extra == register_path: + continue + head = parse_yaml_head(read_text(extra, limit=10_000)) + if head.get("schema") == REGISTER_SCHEMA and head.get("kind") != REGISTER_KIND: + findings.append( + finding( + "registry-format", + "info", + "Register table declares a kind v1 does not validate", + resolved_str(str(extra)), + f"declares register/v1 kind={head.get('kind')!r} — " + "v1 validates kind: projects only.", + "This table is accepted as register data but never validated; keep the " + "projects table in register/projects.yaml.", + {"kind": head.get("kind")}, + ) + ) + # Machine↔disk baseline. `previous_registry` is the pre-overwrite capture + # (the default write path compares before storing); when it is not given, + # the baseline is read from disk, strictly — a corrupt baseline is a + # visible finding, never a silent skip of the drift check. + if previous_registry is None: + persisted_registry, baseline_error = load_json_strict(config.state_dir / "registry.json") + else: + persisted_registry, baseline_error = previous_registry, None + if baseline_error is not None: + findings.append( + finding( + "registry-drift", + "medium", + "Machine registry baseline is corrupt", + str(config.state_dir / "registry.json"), + "registry baseline corrupt; drift check skipped", + "Fix or delete registry.json so the machine↔disk drift check has a readable baseline.", + {"error": baseline_error}, + ) + ) + elif isinstance(persisted_registry, dict) and isinstance(persisted_registry.get("projects"), list): + persisted_paths: Set[str] = set() + for project in persisted_registry["projects"]: + if not isinstance(project, dict) or not project.get("path"): + continue + try: + persisted_paths.add(str(Path(project["path"]).resolve())) + except OSError: + continue + if persisted_paths != machine_paths: + vanished = sorted(persisted_paths - machine_paths) + appeared = sorted(machine_paths - persisted_paths) + stats["registry_drift_stale_machine"] = len(vanished) + len(appeared) + findings.append( + finding( + "registry-drift", + "low", + "Machine registry is out of date with disk", + str(config.state_dir / "registry.json"), + f"{len(vanished)} project(s) listed in registry.json no longer scan; " + f"{len(appeared)} on disk are missing from it.", + "Re-run `guster scan`/`guster doctor` without --no-write to refresh the machine table.", + {"vanished": vanished[:10], "appeared": appeared[:10]}, + ) + ) + return findings + + +def _waiver_reason(item: Dict[str, Any], waivers: Sequence[Dict[str, str]]) -> Optional[str]: + target = str(item.get("path") or "") + for waiver in waivers: + if waiver["rule"] != item.get("rule"): + continue + scope_path = waiver.get("path", "") + if not scope_path: + return waiver["reason"] + root = Path(os.path.expanduser(scope_path)) + try: + if Path(target).resolve() == root.resolve(): + return waiver["reason"] + if root.resolve() in Path(target).resolve().parents: + return waiver["reason"] + except OSError: + continue + return None + + +def extract_verified_date(text: str) -> str: + """Declared date from the document header; prose deeper down is not a claim about this file.""" + head = "\n".join(text.splitlines()[:DATE_HEADER_LINES]) + match = re.search( + VERIFIED_DATE_KEYS + r"\s*\*{0,2}\s*[::]\s*\*{0,2}\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})", + head, + ) + return match.group(1) if match else "" + + +def extract_filename_date(path: Path) -> str: + """Only a date that opens the filename counts; a ticket id is not a freshness claim.""" + stem = path.stem.lstrip(".") + match = re.match(r"(20\d{2})[-_]?(\d{2})[-_]?(\d{2})(?:$|[-_.])", stem) + if not match: + return "" + year, month, day = match.groups() + try: + datetime(int(year), int(month), int(day)) + except ValueError: + return "" + return f"{year}-{month}-{day}" + + +def count_lines(path: Path) -> int: + text = read_text(path, limit=5_000_000) + return len(text.splitlines()) if text else 0 + + +def file_size(path: Path) -> int: + try: + return path.stat().st_size + except OSError: + return 0 diff --git a/external_plugins/guster/src/guster/report.py b/external_plugins/guster/src/guster/report.py new file mode 100644 index 00000000..e06aafad --- /dev/null +++ b/external_plugins/guster/src/guster/report.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Set + +from . import __version__ +from .util import human_bytes, now_iso + + +SEVERITY_LABEL = { + "critical": "CRITICAL", + "high": "HIGH", + "medium": "MEDIUM", + "low": "LOW", + "info": "INFO", +} +RULE_ACTION_HINTS = { + "secret-in-git": "rotate the exposed credential, then remove it from the git index/history", + "secret-inline": "replace inline tokens with environment references and rotate them", + "secret-outside-root": "register or move unregistered credential files into the canonical root", + "secret-undeclared": "register credential files in ~/.agents/credentials/registry.md", + "secret-missing": "rotate or drop stale credential registry rows", + "broken-pointer": "fix the pointer path or replace it with a when-to-read pointer", + "naked-pointer": "wrap naked paths in a link or inline code (CTX-03)", + "misplaced-artifact": "move task bundles under a standard slot (`guster plan move`)", + "stale-state": "refresh verified_at on state documents or archive them", + "orphan-knowledge": "index knowledge files in their README, or delete them", + "duplicate-canonical": "keep one canonical home and turn the rest into pointers", + "skill-budget": "trim skill descriptions to one trigger sentence", + "skill-copy-drift": "make drifted skill copies byte-identical", + "resident-budget": "demote resident detail into knowledge or skills", + "registry-drift": "reconcile the register rows with scanned projects (CTX-10)", + "registry-format": "fix the register/v1 header and rows (unique id, spec.path, dates)", + "knowledge-format": "fix knowledge/v1 metadata per the CTX-11 contract, or drop the schema header", + "see-also-broken": "point see_also at existing ids, or drop the stale reference", + "duplicate-knowledge-id": "merge the files or split the ids so each id has one home", + "knowledge-stale": "re-verify and bump `updated`, or set status dormant/archived", + "hub-manifest-missing": "point the hub config at a real hub (`guster hub init`)", + "note-header-malformed": "fix note front-matter (type key + machine-readable date)", + "git-unavailable": "fix git access for the failing repositories", + "tmp-overdue": "archive or delete overdue files in .agents/tmp", + "governance-overlap": "reduce competing frameworks to one primary document", + "host-manifest-drift": "synchronize plugin manifest versions", +} + + +def next_actions(findings: Sequence[Dict[str, Any]]) -> List[str]: + """Actions derived from the findings that actually fired; an empty list + keeps the baseline ritual instead of naming problems that do not exist.""" + if not findings: + return [ + "No open findings — run `guster score` to record the baseline, and `guster check` after context edits." + ] + counts: Dict[str, int] = {} + for item in findings: + rule = str(item.get("rule") or "") + if rule == "out-of-scope": + continue + counts[rule] = counts.get(rule, 0) + 1 + actions: List[str] = [] + critical_rules = sorted( + { + str(item.get("rule") or "") + for item in findings + if item.get("severity") == "critical" and item.get("rule") != "out-of-scope" + } + ) + if critical_rules: + actions.append( + f"Fix the critical finding(s) first ({', '.join(critical_rules)}); " + "rotate any exposed credential before further edits." + ) + seen: Set[str] = set() + for item in findings: # findings arrive sorted by severity + rule = str(item.get("rule") or "") + if rule in seen or rule == "out-of-scope": + continue + seen.add(rule) + hint = RULE_ACTION_HINTS.get(rule) + if hint is None: + continue + actions.append(f"`{rule}` × {counts[rule]}: {hint}.") + if len(actions) >= 5: + break + actions.append( + "Turn the rest into a checklist with `guster plan findings`; structural changes go " + "through `guster plan move`/`guster init --project ` and `guster apply --yes`." + ) + actions.append("Re-run `guster score` after fixes and compare the trend.") + return actions + + +def render_report( + registry: Dict[str, Any], + policy_result: Dict[str, Any], + metric_result: Dict[str, Any], +) -> str: + summary = registry.get("summary", {}) + metrics = metric_result.get("metrics", {}) + scores = metric_result.get("scores", {}) + raw = metric_result.get("raw", {}) + findings = policy_result.get("findings", []) + scope = metric_result.get("scope", registry.get("scope", "machine")) + workspaces = registry.get("workspaces", []) + if scope == "hub": + registered = summary.get("registry_entries", 0) + projects_line = f"- Registered projects: **{registered}** (scanned roots: {summary.get('projects', 0)})" + else: + total_projects = int(summary.get("projects", 0)) + containers = int(summary.get("containers", 0)) + projects_line = f"- Projects: **{total_projects - containers}**" + if containers: + projects_line += f" (workspace containers: {containers})" + + lines: List[str] = [] + lines.append("# Guster Context Health") + lines.append("") + lines.append(f"> Generated: {metric_result.get('generated_at', now_iso())}") + if workspaces: + lines.append(f"> Workspaces: {', '.join(workspaces)}") + if registry.get("unconfigured"): + if registry.get("auto_roots") and workspaces: + lines.append( + f"> **Unconfigured:** auto-scanning {', '.join(workspaces)} — " + "run `guster init` to pin these roots." + ) + if not registry.get("hub"): + lines.append("> **No hub attached** — run `guster hub init` to create one.") + lines.append("") + lines.append("## Summary") + lines.append("") + lines.append(projects_line) + projects_total = int(summary.get("projects", 0)) + agents_dirs = int(summary.get("agents_dirs", 0)) + lines.append( + f"- `.agents/` dirs: **{agents_dirs}** " + f"(with index: {summary.get('agents_with_index', 0)}); " + f"projects without one: {max(0, projects_total - agents_dirs)}" + ) + lines.append( + f"- Skills: **{summary.get('skills', 0)}** unique / " + f"**{summary.get('skill_copies', 0)}** copies" + + ("" if scope == "hub" else " (user-level roots only)") + ) + lines.append( + f"- Knowledge: **{summary.get('knowledge', 0)}** files; " + f"state: {summary.get('state', 0)}; evidence: {summary.get('evidence', 0)}" + ) + lines.append( + f"- Credentials: **{summary.get('secret_files', 0)}** files " + f"({raw.get('secret_undeclared_outside', 0)} outside and unregistered, " + f"{raw.get('inline_bearers', 0)} inline)" + ) + lines.append("") + lines.append("## Context Health Index") + lines.append("") + dims = metric_result.get("dims") or [] + label = f"{scope} scope" + chi = metric_result.get("chi") + lines.append(f"**CHI: {'n/a' if chi is None else chi} / 100** ({label})") + lines.append("") + lines.append("| Dimension | Value | Score | Raw |") + lines.append("|---|---:|---:|---|") + rows = [ + ( + "Single source", + "ssr", + metrics.get("ssr"), + scores.get("ssr"), + f"{raw.get('topics_total', 0) - raw.get('topics_multi', 0)}/{raw.get('topics_total', 0)} topics single-source", + ), + ( + "Resident budget", + "rbr", + metrics.get("rbr"), + scores.get("rbr"), + f"{human_bytes(raw.get('resident_bytes', 0))} entry bytes " + f"({raw.get('resident_entries', 0)} host entry files + hub AGENTS.md); " + f"skills {raw.get('skill_chars', 0)} chars; budget used, lower is better", + ), + ( + "Placement", + "pcr", + metrics.get("pcr"), + scores.get("pcr"), + f"{raw.get('agents_dirs_compliant', 0)}/{raw.get('agents_dirs_total', 0)} compliant" + f" ({raw.get('agents_dirs_extra_entries', 0)} extra entries, " + f"{raw.get('agents_dirs_missing_slots', 0)} missing slots, " + f"{raw.get('agents_dirs_missing_index', 0)} no index)", + ), + ( + "Index coverage", + "icr", + metrics.get("icr"), + scores.get("icr"), + f"knowledge {raw.get('knowledge_indexed', 0)}/{raw.get('knowledge_total', 0)}; " + f".agents index {raw.get('agents_indexed', 0)}/{raw.get('agents_dirs_total', 0)}", + ), + ( + "State freshness", + "frs", + metrics.get("frs"), + scores.get("frs"), + f"{raw.get('state_docs_fresh', 0)}/{raw.get('state_docs_total', 0)} fresh " + f"({raw.get('state_docs_fresh_declared', 0)} declared, " + f"{raw.get('state_docs_fresh_from_git', 0)} from git, " + f"{raw.get('state_docs_fresh_from_mtime', 0)} from mtime)", + ), + ("Pointer health", "phr", metrics.get("phr"), scores.get("phr"), f"{raw.get('pointers_valid', 0)}/{raw.get('pointers_checked', 0)} valid"), + ( + "Secret isolation", + "sir", + metrics.get("sir"), + scores.get("sir"), + f"{raw.get('secret_undeclared_outside', 0)} undeclared outside; " + f"{raw.get('inline_bearers', 0)} inline; {raw.get('secret_in_git', 0)} in git; " + f"{raw.get('secret_tracking_unknown', 0)} unverifiable", + ), + ] + for label, key, value, score, detail in rows: + if dims and key not in dims: + continue + lines.append( + f"| {label} | {'n/a' if value is None else value} " + f"| {'n/a' if score is None else score} | {detail} |" + ) + lines.append("") + unscored = metric_result.get("unscored_dims") or [] + if unscored: + lines.append( + f"_No evidence: {', '.join(dim.upper() for dim in unscored)}. " + "CHI is normalized over the dimensions that had evidence, so it is not comparable " + "across scopes with different coverage._" + ) + lines.append("") + if raw.get("pointer_files_total", 0) > raw.get("pointer_files_scanned", 0): + lines.append( + f"_Pointer scan covered {raw.get('pointer_files_scanned')} of " + f"{raw.get('pointer_files_total')} pointer files._" + ) + lines.append("") + if raw.get("knowledge_unhashed", 0): + lines.append( + f"_Knowledge files excluded from single-source checks (above the hash limit or unreadable): " + f"{raw.get('knowledge_unhashed')}._" + ) + lines.append("") + lines.append("## Findings") + lines.append("") + if not findings: + lines.append("No findings.") + else: + counts: Dict[str, int] = {} + for item in findings: + counts[item["severity"]] = counts.get(item["severity"], 0) + 1 + lines.append( + "Counts: " + + ", ".join( + f"{SEVERITY_LABEL.get(level, level)} {counts.get(level, 0)}" + for level in ("critical", "high", "medium", "low", "info") + if counts.get(level) + ) + ) + lines.append("") + for item in findings[:80]: + label = SEVERITY_LABEL.get(item["severity"], item["severity"]) + lines.append(f"### [{label}] {item['title']}") + lines.append("") + lines.append(f"- Rule: `{item['rule']}`") + lines.append(f"- Path: `{item['path']}`") + lines.append(f"- {item['message']}") + lines.append(f"- Action: {item['suggestion']}") + evidence = item.get("evidence") or {} + if evidence: + compact = json.dumps(evidence, ensure_ascii=False) + if len(compact) > 500: + compact = compact[:500] + "..." + lines.append(f"- Evidence: `{compact}`") + lines.append("") + if len(findings) > 80: + lines.append(f"_...and {len(findings) - 80} more findings; use `guster doctor --json`._") + lines.append("") + waived = policy_result.get("waived", []) + if waived: + lines.append(f"## Waived ({len(waived)})") + lines.append("") + for item in waived: + lines.append(f"- `{item['rule']}` — `{item['path']}` — {item.get('waiver_reason', '')}") + lines.append("") + lines.append("## Next Actions") + lines.append("") + for index, action in enumerate(next_actions(findings), start=1): + lines.append(f"{index}. {action}") + lines.append("") + return "\n".join(lines) + + +def render_status(registry: Dict[str, Any], metric_result: Optional[Dict[str, Any]] = None) -> str: + plugin = registry.get("plugin", {}) + summary = registry.get("summary", {}) + projects = ( + summary.get("registry_entries", 0) + if registry.get("scope") == "hub" + else summary.get("projects", 0) + ) + lines = [ + f"Guster {plugin.get('version', '?')}", + f"generated: {registry.get('generated_at', '?')}", + f"hub: {registry.get('hub') or '(none)'}", + f"workspaces: {', '.join(registry.get('workspaces', []))}", + ] + zones = registry.get("zones") or [] + if zones: + lines.append( + "zones: " + + ", ".join( + f"{Path(item.get('path', '.')).name}({item.get('tools', 0)} tools)" + for item in zones + ) + ) + if registry.get("unconfigured"): + if registry.get("auto_roots") and registry.get("workspaces"): + lines.append( + "note: no config yet — auto-scanning " + + ", ".join(registry["workspaces"]) + + "; run `guster init` to pin roots" + ) + if not registry.get("hub"): + lines.append("note: no hub attached; run `guster hub init` to create one") + state_dir = registry.get("state_dir") + if state_dir and Path(state_dir).parent.name == "scoped": + lines.append(f"scope: {registry.get('scope', 'machine')} (state: {state_dir})") + lines.extend( + [ + f"projects: {projects}", + f"skills: {summary.get('skills', 0)} unique / {summary.get('skill_copies', 0)} copies" + + ("" if registry.get("scope") == "hub" else " (user-level roots only)"), + f"knowledge: {summary.get('knowledge', 0)}", + f"credentials: {summary.get('secret_files', 0)} files; {summary.get('inline_bearers', 0)} inline", + ] + ) + if plugin.get("version") and plugin.get("version") != __version__: + lines.append( + f"(registry was scanned by guster {plugin.get('version')}; " + f"current CLI is {__version__} — run `guster doctor` to refresh)" + ) + if metric_result: + chi = metric_result.get("chi") + stamp = metric_result.get("generated_at") + suffix = f" (scored {stamp})" if stamp else "" + lines.append(f"CHI: {'n/a' if chi is None else chi}{suffix}") + return "\n".join(lines) + + +def redacted_registry(registry: Dict[str, Any]) -> Dict[str, Any]: + def scrub(value: Any) -> Any: + if isinstance(value, dict): + # Key-level blacklist only; parse_warnings is allowed through + # because its text is redacted at the source (line/column info, + # never the broken row's content). + return { + key: scrub(item) + for key, item in value.items() + if key.lower() not in {"value", "password", "token", "secret", "authorization"} + } + if isinstance(value, list): + return [scrub(item) for item in value] + return value + + return scrub(registry) + diff --git a/external_plugins/guster/src/guster/util.py b/external_plugins/guster/src/guster/util.py new file mode 100644 index 00000000..de0a62fe --- /dev/null +++ b/external_plugins/guster/src/guster/util.py @@ -0,0 +1,866 @@ +from __future__ import annotations + +import contextlib +import hashlib +import json +import os +import re +import subprocess +import tempfile +import time +from contextlib import contextmanager +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple + +try: + import fcntl +except ImportError: # pragma: no cover - non-POSIX hosts + fcntl = None # type: ignore[assignment] + + +EXCLUDE_DIRS = { + ".git", + ".hg", + ".svn", + "node_modules", + "build", + "dist", + "out", + "__pycache__", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + ".next", + ".nuxt", + ".venv", + "venv", + "target", + ".idea", + ".vscode", +} + +TEXT_SUFFIXES = { + ".md", + ".markdown", + ".txt", + ".json", + ".yaml", + ".yml", + ".toml", + ".ini", + ".cfg", + ".conf", + ".sh", + ".bash", + ".zsh", + ".py", + ".js", + ".mjs", + ".ts", + ".tsx", + ".jsx", + ".go", + ".java", + ".kt", + ".rs", + ".rb", + ".php", + ".sql", +} + +POINTER_SUFFIXES = { + ".md", + ".json", + ".yaml", + ".yml", + ".toml", + ".sh", + ".bash", + ".zsh", + ".py", + ".js", + ".mjs", + ".ts", + ".tsx", + ".jsx", + ".go", + ".java", + ".kt", + ".rs", + ".sql", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".webp", + ".svg", + ".mp4", + ".mov", + ".jsonl", + ".csv", +} + + +def now_iso() -> str: + return datetime.now().astimezone().isoformat(timespec="seconds") + + +def today_iso() -> str: + return datetime.now().astimezone().date().isoformat() + + +def read_text(path: Path, limit: Optional[int] = None) -> str: + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + if limit is not None and len(text) > limit: + return text[:limit] + return text + + +def write_text(path: Path, text: str) -> None: + """Atomic replace: a concurrent reader sees the old or the new file, never a mix.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp( + dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp" + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(text) + os.replace(tmp_name, path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp_name) + raise + + +@contextmanager +def state_lock(state_dir: Path) -> Iterator[None]: + """Exclusive lock over one state dir so concurrent CLI runs cannot interleave + scan/policy writes. flock is POSIX-only; elsewhere the lock degrades to a no-op + while write_text itself stays atomic.""" + state_dir.mkdir(parents=True, exist_ok=True) + handle = (state_dir / ".lock").open("w") + try: + if fcntl is not None: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + yield + finally: + handle.close() + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_text(text: str) -> str: + return sha256_bytes(text.encode("utf-8")) + + +def sha256_file(path: Path) -> str: + try: + return sha256_bytes(path.read_bytes()) + except OSError: + return "" + + +def is_under(path: Path, root: Path) -> bool: + try: + path.resolve().relative_to(root.resolve()) + return True + except (OSError, ValueError): + return False + + +def relative(path: Path, base: Path) -> str: + try: + return str(path.resolve().relative_to(base.resolve())) + except (OSError, ValueError): + return str(path) + + +def slugify(value: str) -> str: + value = value.strip().lower() + value = re.sub(r"[^a-z0-9._-]+", "-", value) + value = re.sub(r"-{2,}", "-", value).strip("-") + return value or "item" + + +def unique_suffix(length: int = 3) -> str: + """Random hex tail; same-second plan ids and backups must not overwrite.""" + return os.urandom(length).hex() + + +def human_bytes(value: int) -> str: + if value < 1024: + return f"{value} B" + if value < 1024 * 1024: + return f"{value / 1024:.1f} KB" + return f"{value / (1024 * 1024):.1f} MB" + + +def json_dumps(data: Any, pretty: bool = True) -> str: + return json.dumps(data, ensure_ascii=False, indent=2 if pretty else None, sort_keys=False) + + +def load_json(path: Path, default: Any = None) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return default + + +def load_json_strict(path: Path) -> Tuple[Any, Optional[str]]: + """Value and error apart: absent is (None, None), corrupt is (None, reason). + + Missing and corrupt are different conditions; a caller that can rescan must + warn about corruption instead of silently treating it as absent.""" + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + return None, None + except OSError as exc: + return None, str(exc) + try: + return json.loads(raw), None + except ValueError as exc: + return None, str(exc) + + +def is_indexed(name: str, index_text: str) -> bool: + """An index row naming the file: backtick-wrapped (bare or as a `dir/name` + tail), a backtick-wrapped stem, a markdown table cell holding the stem, or + a markdown link whose target is the file (bare or a `dir/name` relative + path — the llms.txt-shaped `- [title](general/start-here.md)` index form). + + A bare substring does not count: `a.md` inside `data.md` is not an index row.""" + if f"`{name}`" in index_text or f"/{name}`" in index_text: + return True + stem = Path(name).stem + if f"`{stem}`" in index_text: + return True + if re.search(rf"\((?:[^)]*?/)?{re.escape(name)}\)", index_text): + return True + return re.search(rf"^\|\s*{re.escape(stem)}\s*\|", index_text, re.MULTILINE) is not None + + +def save_json(path: Path, data: Any) -> None: + write_text(path, json_dumps(data) + "\n") + + +def parse_frontmatter(text: str) -> Tuple[Dict[str, str], str]: + """Parse the small YAML frontmatter subset used by agent files. + + Supports scalar keys, quoted values and literal/folded blocks. It deliberately + avoids a PyYAML dependency so the plugin runs on a stock Python install. + """ + if not text.startswith("---"): + return {}, text + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + end: Optional[int] = None + for index in range(1, len(lines)): + if lines[index].strip() == "---": + end = index + break + if end is None: + return {}, text + + data: Dict[str, str] = {} + index = 1 + while index < end: + line = lines[index] + if not line.strip() or line.lstrip().startswith("#"): + index += 1 + continue + if line.startswith((" ", "\t")) or ":" not in line: + index += 1 + continue + key, raw = line.split(":", 1) + key = key.strip() + value = raw.strip() + if value in ("|", ">"): + block: List[str] = [] + index += 1 + while index < end: + candidate = lines[index] + if candidate.startswith(" "): + block.append(candidate[2:]) + elif not candidate.strip(): + block.append("") + else: + break + index += 1 + data[key] = ("\n" if value == "|" else " ").join(block).strip() + continue + data[key] = unquote(value) + index += 1 + return data, "\n".join(lines[end + 1 :]) + + +def unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"): + return value[1:-1] + return value + + +def _scalar(raw: str) -> str: + raw = raw.strip() + if raw[:1] in ('"', "'"): + raw = unquote(raw) + return unquote(raw.split(" #", 1)[0].strip()) + + +_TOP_KEY = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*):(?:[ \t]+(.*))?$") +_INLINE_LIST = re.compile(r"^\[(.*)\]$") + + +def parse_yaml_head(text: str) -> Dict[str, Any]: + """Parse the metadata face of a knowledge/v1 file without PyYAML. + + Only column-0 `key: value` lines are read: scalar keys get their unquoted + string, a `key: [a, b]` inline list becomes a list of unquoted strings, + and block keys (`history:`, `body:`, `key: |`...) are recorded as None so + callers can tell "present but opaque" from "absent". List items and + anything indented are skipped. This covers the frontmatter-like header of + knowledge/v1 documents; the prose body is never parsed.""" + data: Dict[str, Any] = {} + for line in text.lstrip("\ufeff").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if line[:1] in (" ", "\t") or stripped.startswith("- "): + continue + match = _TOP_KEY.match(line) + if match is None: + continue + key = match.group(1) + raw = (match.group(2) or "").strip() + inline = _INLINE_LIST.match(raw) + if inline: + items = [unquote(item) for item in inline.group(1).split(",") if item.strip()] + data[key] = items + elif raw in ("", "|", ">"): + # First block key ends the metadata face; prose below never + # overwrites header values with same-name column-0 lines. + data[key] = None + if key in ("body", "history"): + break + else: + data[key] = _scalar(raw) + return data + + +_ROW_KEY = re.compile(r"^([A-Za-z_][A-Za-z0-9_-]*):(?:[ \t]+(.*))?$") +_ROW_ITEM = re.compile(r"^-\s+(.*)$") + + +def parse_register_rows(text: str) -> List[Dict[str, str]]: + """Parse the project rows of a register/v1 table (register/projects.yaml). + + Returns one dict per top-level `- id: ...` entry with the flat keys the + drift reconciliation needs: id, status, as_of, name and the spec.path + nested one level down. Header keys (schema/kind/updated) and any other + nesting are ignored. Dependency-free by design, like parse_yaml_head.""" + rows: List[Dict[str, str]] = [] + current: Optional[Dict[str, str]] = None + in_spec = False + skipping_nested = False + for line in text.lstrip("\ufeff").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + indented = line[:1].isspace() + indent = len(line) - len(line.lstrip(" ")) + item = _ROW_ITEM.match(stripped) + if item is not None: + if not indented or indent <= 2: + payload = item.group(1) + match = _ROW_KEY.match(payload) + if match is not None and match.group(1) == "id": + current = {"id": _scalar(match.group(2) or "")} + rows.append(current) + in_spec = False + skipping_nested = False + continue + else: + # a deeper `- key:` list item (deps, aliases...) belongs to a + # nested block, never to the current top-level row + skipping_nested = True + continue + if current is None or skipping_nested: + continue + if line[:1].isspace(): + match = _ROW_KEY.match(stripped) + if match is None: + continue + key = match.group(1) + raw = (match.group(2) or "").strip() + if in_spec: + if key == "path" and raw: + current["path"] = _scalar(raw) + continue + if key == "spec" and raw == "": + in_spec = True + elif raw and key in ("status", "as_of", "name"): + current[key] = _scalar(raw) + else: + in_spec = False + return rows + + +_TOP_ROW_KEY_LINE = re.compile(r"^(?:id|name):(?:\s|$)", re.MULTILINE) + + +def has_top_level_row_keys(text: str) -> bool: + """Column-0 `id:`/`name:` mapping keys — the flat-map mistake behind a + register table that parsed zero `- id:` rows.""" + return _TOP_ROW_KEY_LINE.search(text) is not None + + +def first_heading(text: str, fallback: str = "") -> str: + for line in text.splitlines(): + if line.startswith("# "): + return line[2:].strip() + return fallback + + +def iter_files( + root: Path, + suffixes: Optional[Set[str]] = None, + max_depth: Optional[int] = None, + skip_dirs: Optional[Set[str]] = None, + errors: Optional[List[str]] = None, +) -> Iterator[Path]: + if not root.exists(): + return + skip = EXCLUDE_DIRS if skip_dirs is None else skip_dirs + root = root.resolve() + + def on_error(exc: OSError) -> None: + if errors is not None: + errors.append(f"{exc.filename or root}: {exc.strerror or exc}") + + for current, dirs, files in os.walk(root, onerror=on_error): + current_path = Path(current) + depth = len(current_path.relative_to(root).parts) + if max_depth is not None and depth >= max_depth: + dirs[:] = [] + dirs[:] = sorted(d for d in dirs if d not in skip) + for name in sorted(files): + path = current_path / name + if suffixes is not None and path.suffix.lower() not in suffixes: + continue + yield path + + +def find_git_roots(roots: Sequence[Path], max_depth: int = 5) -> List[Path]: + found: List[Path] = [] + seen: Set[str] = set() + + def visit(path: Path, depth: int) -> None: + try: + resolved = str(path.resolve()) + except OSError: + return + if resolved in seen: + return + seen.add(resolved) + if not path.is_dir(): + return + if (path / ".git").exists(): + found.append(path) + return + if depth >= max_depth: + return + try: + entries = sorted(path.iterdir(), key=lambda item: item.name) + except OSError: + return + for entry in entries: + if entry.name in EXCLUDE_DIRS or entry.name.startswith("."): + continue + if entry.is_dir(): + visit(entry, depth + 1) + + for root in roots: + visit(root, 0) + return sorted(found, key=lambda item: str(item)) + + +def git_tracked(root: Path) -> Optional[Set[str]]: + """Tracked paths; None means git could not answer, never a silent empty set.""" + if not (root / ".git").exists(): + return set() + try: + result = subprocess.run( + ["git", "-C", str(root), "ls-files", "-z"], + check=False, + capture_output=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + tracked: Set[str] = set() + for raw in result.stdout.split(b"\x00"): + if not raw: + continue + tracked.add(raw.decode("utf-8", errors="replace")) + return tracked + + +def git_last_commit_dates(root: Path) -> Optional[Dict[str, str]]: + """Map repo-relative paths to the newest commit date; None when git failed.""" + if not (root / ".git").exists(): + return {} + try: + result = subprocess.run( + ["git", "-C", str(root), "log", "--format=%x00%cI", "--name-only"], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + if result.returncode == 128 and "does not have any commits yet" in ( + result.stderr or "" + ): + # A freshly `git init`-ed repo has no history to read: no dates, + # not a git failure. + return {} + return None + dates: Dict[str, str] = {} + current = "" + for line in result.stdout.splitlines(): + if line.startswith("\x00"): + current = line[1:].strip() + continue + rel = line.strip() + if not rel or not current: + continue + dates.setdefault(rel, current) + return dates + + +NOTE_DATE_KEYS = ("updated", "date", "lastmod", "created") +_NOTE_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:[T ][\d:]+)?") +DATE_ONLY_RE = re.compile(r"^\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])$") + + +def parse_note_header(text: str) -> Optional[Dict[str, Any]]: + """Parse a knowledge note's YAML front-matter (CTX-09). + + Follows the Obsidian properties convention — the interchange format the + open-source markdown knowledge bases (Foam, Quartz, Logseq) all accept. + Hand-rolled for the simple subset (scalar keys, one list per key, ``#`` + comments, literal/folded block scalars) because the CLI runs on stock + Python without third-party deps. + + Returns None when the text has no front-matter or declares no ``type`` — + the structured header is opt-in, and that gate wins over malformedness, + so legacy prose notes that merely open with dashes never report. A dict + with ``_malformed`` reports a header that opted in but cannot be parsed. + """ + lines = text.splitlines() + if not text.startswith("---") or not lines or lines[0].strip() != "---": + return None + keys: Dict[str, Any] = {} + list_key: Optional[str] = None + malformed: Optional[Dict[str, Any]] = None + end: Optional[int] = None + limit = min(len(lines), 80) + index = 1 + while index < limit: + line = lines[index] + if line.strip() == "---": + end = index + break + stripped = line.strip() + if not stripped or stripped.startswith("#"): + index += 1 + continue + if stripped.startswith("- ") and list_key is not None: + value = stripped[2:].split("#", 1)[0].strip() + if value: + keys[list_key].append(value) + index += 1 + continue + match = re.match(r"^([A-Za-z0-9_-]+):\s*(.*)$", stripped) + if not match: + if malformed is None: + malformed = {"line": index + 1, "text": stripped[:60]} + index += 1 + continue + key = match.group(1) + value = match.group(2).split("#", 1)[0].strip() + if value in ("|", ">"): + block: List[str] = [] + index += 1 + while index < limit: + candidate = lines[index] + if candidate.startswith(" "): + block.append(candidate[2:]) + elif not candidate.strip(): + block.append("") + else: + break + index += 1 + keys[key] = ("\n" if value == "|" else " ").join(block).strip() + list_key = None + continue + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + if value: + keys[key] = value + list_key = None + else: + keys[key] = [] + list_key = key + index += 1 + note_type = keys.get("type") + if isinstance(note_type, list): + note_type = note_type[0] if note_type else "" + if not note_type or not str(note_type).strip().strip("[]"): + return None + if malformed is not None: + return {"_malformed": True, **malformed} + if end is None: + return {"_malformed": True, "line": len(lines), "text": "front-matter not closed"} + return keys + + +def note_header_date(header: Dict[str, Any]) -> Optional[str]: + """First machine-readable date value among the convention's date keys.""" + for key in NOTE_DATE_KEYS: + value = header.get(key) + if isinstance(value, str) and _NOTE_DATE_RE.match(value.strip()): + return value.strip() + return None + + +def extract_pointers(text: str) -> List[str]: + # A leading front-matter block is metadata, not prose (CTX-09: the body, + # never the header, joins the pointer scan). + text = _strip_frontmatter(text) + text = re.sub(r"```.*?```", "", text, flags=re.DOTALL) + pointers: List[str] = [] + + def add(candidate: str) -> None: + # `[`path`](path)` matches both the link and the inline-code pass; count + # each target once per file so pointer health is not double-weighted. + if _looks_like_pointer(candidate) and candidate not in pointers: + pointers.append(candidate) + + for match in re.finditer(r"\[[^\]]*\]\(([^)]+)\)", text): + add(match.group(1).strip()) + for match in re.finditer(r"`([^`\n]+)`", text): + candidate = match.group(1).strip() + if candidate.startswith("~/") and _is_remote_command(_line_at(text, match.start())): + continue + add(candidate) + return pointers + + +def _line_at(text: str, offset: int) -> str: + start = text.rfind("\n", 0, offset) + 1 + end = text.find("\n", offset) + return text[start : end if end != -1 else len(text)] + + +def _is_remote_command(line: str) -> bool: + # `ssh mini` + `~/...` in the same line names the remote home, not a local path. + if re.search(r"\bssh\s+(?:-[\w-]+\s+)*[A-Za-z0-9._-]+", line): + return True + return re.search(r"\b(?:scp|rsync)\b[^\n]*?[A-Za-z0-9._-]+:~/", line) is not None + + +def _looks_like_pointer(candidate: str) -> bool: + if not candidate or _is_placeholder(candidate): + return False + if any(char.isspace() for char in candidate): + return False + if candidate.startswith(("http://", "https://", "mailto:", "tel:", "#", "...")): + return False + suffix = Path(candidate).suffix.lower() + if candidate.startswith("~/"): + return True + if candidate.startswith(("./", "../")): + return True + if candidate.startswith("/"): + known_roots = ( + "/Users/", + "/home/", + "/tmp/", + "/Applications/", + "/System/", + "/opt/", + "/usr/", + "/var/", + "/etc/", + "/private/", + "/Volumes/", + ) + return candidate.startswith(known_roots) + if candidate.endswith("/"): + # A relative directory pointer (`knowledge/general/`) has no suffix and + # is still a link worth checking. + return True + return "/" in candidate and suffix in POINTER_SUFFIXES + + +def _is_placeholder(pointer: str) -> bool: + if any(token in pointer for token in ("<", ">", "*", "{", "}", "$", "|")): + return True + # File templates in docs: draft-NNN.md, daily-YYYY-MM-DD.md, 0X-xxx.md. + return re.search(r"(?:N{2,}|Y{4}|X{3,}|x{3,}|0[xX][-_/])", pointer) is not None + + +def _strip_pointer(pointer: str) -> str: + pointer = pointer.strip() + if pointer.startswith("<") and ">" in pointer: + pointer = pointer[1 : pointer.index(">")] + pointer = pointer.split(" ", 1)[0] + if pointer.startswith("file://"): + pointer = pointer[7:] + pointer = re.sub(r":\d+(?::\d+)?$", "", pointer) + return pointer + + +# Path-shaped token in prose: optional ~/ ./ ../ / prefix, then at least one +# path segment. The lookbehind keeps the match from starting inside a word, a +# URL or an email; CJK characters do not block, so `见docs/x.md` still matches. +NAKED_POINTER_TOKEN_RE = re.compile( + r"(? str: + """Body after a leading YAML front-matter block; header metadata is not prose.""" + if not text.startswith("---"): + return text + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return text + for index in range(1, len(lines)): + if lines[index].strip() == "---": + return "\n".join(lines[index + 1 :]) + return text + + +def extract_naked_pointers(text: str) -> List[str]: + """File-shaped paths left in prose after links, code spans and code blocks are removed. + + Only suffix-bearing paths count: directory references in prose (a bare + `~/some/repo`) name a place far more often than they point at a read, and + flagging them would drown the signal. A leading front-matter block is + metadata, not prose — a path-valued key (CTX-09 `superseded_by`) is never + a naked pointer. + """ + text = _strip_frontmatter(text) + residual = re.sub(r"```.*?```", "", text, flags=re.DOTALL) + residual = re.sub(r"\[[^\]]*\]\([^)]+\)", "", residual) + residual = re.sub(r"`[^`\n]+`", "", residual) + found: List[str] = [] + for match in NAKED_POINTER_TOKEN_RE.finditer(residual): + token = match.group(0).rstrip(".,;:!?\"')") + if not token or token in found: + continue + if _is_placeholder(token): + continue + if token.startswith(("http://", "https://", "mailto:", "tel:", "#")): + continue + if token.startswith(("/tmp/", "/private/tmp/", "/var/folders/")): + continue + if Path(token).suffix.lower() not in POINTER_SUFFIXES: + continue + found.append(token) + return found + + +def find_repo_root(start: Path) -> Optional[Path]: + """Nearest enclosing git work tree; `.claude/` and `.agents/` docs address the repo root.""" + cursor = start + for _ in range(8): + if (cursor / ".git").exists(): + return cursor + if cursor.parent == cursor: + return None + cursor = cursor.parent + return None + + +def resolve_pointer( + pointer: str, + base_file: Path, + hub: Optional[Path] = None, + home: Optional[Path] = None, + config_roots: Optional[Sequence[Path]] = None, +) -> Optional[Path]: + if not pointer or _is_placeholder(pointer): + return None + cleaned = _strip_pointer(pointer) + lowered = cleaned.lower() + if lowered.startswith(("http://", "https://", "mailto:", "tel:", "#")): + return None + if not cleaned: + return None + expanded = Path(os.path.expanduser(cleaned)) + if expanded.is_absolute(): + if str(expanded).startswith(("/tmp/", "/private/tmp/", "/var/folders/")): + # Runtime scratch paths are never expected to exist at rest. + return None + return expanded + if cleaned.startswith(("./", "../")): + return (base_file.parent / expanded).resolve() + anchor = cleaned.split("/", 1)[0] + anchors: List[Path] = [base_file.parent] + repo_root = find_repo_root(base_file.parent) + if repo_root is not None and repo_root != base_file.parent: + anchors.append(repo_root) + if hub is not None: + anchors.append(hub) + if home is not None: + anchors.append(home) + if config_roots: + # Anchors follow the user's declared roots, never hardcoded layouts. + anchors.extend(config_roots) + if not any((root / anchor).exists() for root in anchors): + # A bare relative path only counts as a pointer when its first segment + # exists in one of the known roots. Otherwise it is prose about another repo. + return None + candidates: List[Path] = [(root / expanded).resolve() for root in anchors] + for candidate in candidates: + if candidate.exists(): + return candidate + return candidates[0] + + +def age_days(path: Path) -> Optional[float]: + """Age in days; None when stat fails — unknown age, never a fake zero.""" + try: + return max(0.0, (time.time() - path.stat().st_mtime) / 86400.0) + except OSError: + return None + + +def parse_date(value: str) -> Optional[datetime]: + value = (value or "").strip() + if not value: + return None + for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%d %H:%M:%S"): + try: + return datetime.strptime(value, fmt) + except ValueError: + continue + try: + return datetime.fromisoformat(value) + except ValueError: + return None diff --git a/external_plugins/guster/templates/hub/AGENTS.md b/external_plugins/guster/templates/hub/AGENTS.md new file mode 100644 index 00000000..c089b03d --- /dev/null +++ b/external_plugins/guster/templates/hub/AGENTS.md @@ -0,0 +1,23 @@ +# Guster Hub + +This repository is the data plane for agent context. The Guster plugin reads it; it does not replace it. + +## Map + +- Knowledge index: [{{KNOWLEDGE_PATH}}/README.md]({{KNOWLEDGE_PATH}}/README.md) +- Project register: [register/projects.yaml](register/projects.yaml) +- Tool index: [{{TOOLS_PATH}}/README.md]({{TOOLS_PATH}}/README.md) +- Hub contract: [hub.yaml](hub.yaml) + +## Rules + +- One topic per knowledge file. First-level directories under `{{KNOWLEDGE_PATH}}/` are subzones (`general/`, `projects/`, `reference/`, ...), all discovered and index-checked. +- Register every project as a row in `register/projects.yaml` (id, status, as_of, spec.path); keep spec.path resolvable. +- State documents must carry `verified_at: YYYY-MM-DD`. +- File paths are markdown links or inline code, never naked in prose, so `guster doctor` can verify them. +- Code and tests remain the source of truth; this hub stores decisions and pointers. +- Never store credentials, company secrets, generated caches or scratch files here. + +## Optional modules + +Add `handoff/` or `reports/` only when the workflow needs them. diff --git a/external_plugins/guster/templates/hub/README.md b/external_plugins/guster/templates/hub/README.md new file mode 100644 index 00000000..2db1e050 --- /dev/null +++ b/external_plugins/guster/templates/hub/README.md @@ -0,0 +1,18 @@ +# Guster Hub (Lite) + +A minimal data plane for the Guster plugin. It holds durable knowledge and project state; governance lives in the plugin. + +## Start + +```bash +guster hub status +guster scan +guster doctor --scope hub +``` + +## Add content + +- Knowledge: create one topic per file under any first-level subzone of `{{KNOWLEDGE_PATH}}/` (`general/`, `projects/`, ...), then add it to `{{KNOWLEDGE_PATH}}/README.md`. +- Project: add a row to `register/projects.yaml` (id, status, as_of, spec.path, ...) pointing at the project directory. +- Tool: add one directory under `{{TOOLS_PATH}}/` with its own `AGENTS.md`, then register it in `{{TOOLS_PATH}}/README.md`. +- Handoff and reports are optional modules. Add them only when the workflow needs them. diff --git a/external_plugins/guster/templates/hub/gitignore.tmpl b/external_plugins/guster/templates/hub/gitignore.tmpl new file mode 100644 index 00000000..9fa88ac4 --- /dev/null +++ b/external_plugins/guster/templates/hub/gitignore.tmpl @@ -0,0 +1,2 @@ +.DS_Store +*.log diff --git a/external_plugins/guster/templates/hub/hub.yaml.tmpl b/external_plugins/guster/templates/hub/hub.yaml.tmpl new file mode 100644 index 00000000..daa9cd3b --- /dev/null +++ b/external_plugins/guster/templates/hub/hub.yaml.tmpl @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Lumos-789/guster/main/schemas/hub.schema.json +schema_version: 2 +hub_version: 0.1.0 +min_guster: {{PLUGIN_VERSION}} +name: {{HUB_NAME}} +created_at: {{DATE}} +paths: + knowledge: {{KNOWLEDGE_PATH}} + projects: {{PROJECTS_PATH}} + tools: {{TOOLS_PATH}} diff --git a/external_plugins/guster/templates/hub/knowledge/README.md b/external_plugins/guster/templates/hub/knowledge/README.md new file mode 100644 index 00000000..ff1efcbd --- /dev/null +++ b/external_plugins/guster/templates/hub/knowledge/README.md @@ -0,0 +1,5 @@ +# Knowledge Index + +| File | Topic | +|---|---| +| `general/start-here.md` | How this hub is organized | diff --git a/external_plugins/guster/templates/hub/knowledge/general/knowledge-v1-skeleton.txt b/external_plugins/guster/templates/hub/knowledge/general/knowledge-v1-skeleton.txt new file mode 100644 index 00000000..0bd7639f --- /dev/null +++ b/external_plugins/guster/templates/hub/knowledge/general/knowledge-v1-skeleton.txt @@ -0,0 +1,20 @@ +# knowledge/v1 skeleton — copy to .yaml next to this file, then uncomment +# and fill each line. The contract is opt-in: only files whose metadata face +# carries the uncommented `schema: knowledge/v1` header are validated; plain +# notes are never touched. Index the new file in knowledge/README.md. +# +# schema: knowledge/v1 +# id: my-topic # must equal the file stem (my-topic.yaml) +# type: howto # state | howto | principle | pitfall | decision | index | reference +# status: active # active | dormant | superseded | archived +# topic: one-line theme +# as_of: '2026-11-19' # date the facts were last checked (YYYY-MM-DD) +# updated: '2026-11-19' # last re-verification; must not predate as_of +# see_also: [] # inline list of other knowledge ids, e.g. [other-topic] +# body: # everything below the body key is free-form content +# - h: Heading +# text: | +# The durable knowledge itself. One topic per file. +# +# `type: state` and `type: index` files carry a 90-day freshness SLA while +# `status: active` — re-verify and bump `updated`, or set status dormant/archived. diff --git a/external_plugins/guster/templates/hub/knowledge/general/start-here.md b/external_plugins/guster/templates/hub/knowledge/general/start-here.md new file mode 100644 index 00000000..516278e6 --- /dev/null +++ b/external_plugins/guster/templates/hub/knowledge/general/start-here.md @@ -0,0 +1,14 @@ +# Start Here + +This hub is the data plane. The Guster plugin discovers it, checks its structure and scores its health. + +## What goes where + +- `general/` — reusable principles, methods and tool notes that apply across projects. +- `projects/` — durable business and architecture knowledge for one project. +- more subzones (e.g. `reference/`) may exist; every first-level directory here is discovered automatically. +- the hub's project register lives outside this knowledge root in `register/projects.yaml`. + +Keep one topic per file. Add every knowledge file to [the knowledge index](../README.md). + +Structured knowledge files opt in via a `schema: knowledge/v1` header; every field is shown in the all-commented skeleton `knowledge-v1-skeleton.txt` in this directory. diff --git a/external_plugins/guster/templates/hub/knowledge/projects/.gitkeep b/external_plugins/guster/templates/hub/knowledge/projects/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/external_plugins/guster/templates/hub/knowledge/projects/.gitkeep @@ -0,0 +1 @@ + diff --git a/external_plugins/guster/templates/hub/projects/.gitkeep b/external_plugins/guster/templates/hub/projects/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/external_plugins/guster/templates/hub/register/projects.yaml.tmpl b/external_plugins/guster/templates/hub/register/projects.yaml.tmpl new file mode 100644 index 00000000..9fec3c20 --- /dev/null +++ b/external_plugins/guster/templates/hub/register/projects.yaml.tmpl @@ -0,0 +1,19 @@ +schema: register/v1 +kind: projects +updated: '{{DATE}}' +# One row per project: a top-level `- id:` list item under the `projects:` key. +# Keys: id / status (active|pending|on-hold|stale|retired) / as_of "YYYY-MM-DD" / +# name / desc / spec.path (relative to the hub root, or ~/ or absolute) / +# spec.aliases / notes. spec.path must resolve to a directory that exists on +# this machine; `./` is the legal placeholder for a row with no directory of +# its own. Unknown keys are ignored. +# +# Example shape (uncomment and edit; the table may also start empty): +# projects: +# - id: example +# status: active +# as_of: '{{DATE}}' +# name: Example +# spec: +# path: ~/code/example +# notes: one line of free-form notes diff --git a/external_plugins/guster/templates/hub/tools/README.md b/external_plugins/guster/templates/hub/tools/README.md new file mode 100644 index 00000000..471d00b7 --- /dev/null +++ b/external_plugins/guster/templates/hub/tools/README.md @@ -0,0 +1,30 @@ +# Tools + +This is the index for manual tools in this hub. The Lite profile ships no tool by default; add one only when it earns a place. + +## What belongs here + +- Run-once commands that a person or agent invokes deliberately. +- One directory per tool, with an `AGENTS.md` covering purpose, entry point, inputs, outputs, failure behavior and credential references. +- The smallest executable entry point needed to do the job. + +## What does not + +- Long-running services, cron jobs or background daemons. +- Skills or agent instructions; those belong in the plugin or project `.agents/`. +- Knowledge notes; those belong in `knowledge/`. +- Generated caches, logs or scratch files. + +## Index + +| Tool | Purpose | Entry point | Credentials | +|---|---|---|---| +| _(none yet)_ | | | | + +## Add a tool + +1. Create `/` next to this file. +2. Add an `AGENTS.md` with the contract above. +3. Add the entry point. +4. Add a row to the index. +5. Reference credentials by path or variable name only; never commit values. diff --git a/marketplace.json b/marketplace.json index c1a9a6ee..4ce6a91a 100644 --- a/marketplace.json +++ b/marketplace.json @@ -10,7 +10,10 @@ "url": "https://z.ai" }, "unlistedPlugins": [ - { "name": "openpencil", "source": "./external_plugins/openpencil" } + { + "name": "openpencil", + "source": "./external_plugins/openpencil" + } ], "plugins": [ { @@ -827,7 +830,9 @@ }, { "name": "excalidraw", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "Excalidraw", "displayName_i18n": { "en": "Excalidraw", @@ -850,7 +855,9 @@ "name": "photocraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/photocraft/icon.png", "source": "./external_plugins/photocraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "PhotoCraft", "displayName_i18n": { "en": "PhotoCraft", @@ -877,7 +884,9 @@ "name": "vectorcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/vectorcraft/icon.png", "source": "./external_plugins/vectorcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "VectorCraft", "displayName_i18n": { "en": "VectorCraft", @@ -904,7 +913,9 @@ "name": "pdfcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/pdfcraft/icon.png", "source": "./external_plugins/pdfcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "PdfCraft", "displayName_i18n": { "en": "PdfCraft", @@ -931,7 +942,9 @@ "name": "designcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/designcraft/icon.png", "source": "./external_plugins/designcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "DesignCraft", "displayName_i18n": { "en": "DesignCraft", @@ -958,7 +971,9 @@ "name": "wordcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/wordcraft/icon.png", "source": "./external_plugins/wordcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "WordCraft", "displayName_i18n": { "en": "WordCraft", @@ -985,7 +1000,9 @@ "name": "gridcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/gridcraft/icon.png", "source": "./external_plugins/gridcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "GridCraft", "displayName_i18n": { "en": "GridCraft", @@ -1012,7 +1029,9 @@ "name": "deckcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/deckcraft/icon.png", "source": "./external_plugins/deckcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "DeckCraft", "displayName_i18n": { "en": "DeckCraft", @@ -1039,7 +1058,9 @@ "name": "cadcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/cadcraft/icon.png", "source": "./external_plugins/cadcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "CADCraft", "displayName_i18n": { "en": "CADCraft", @@ -1066,7 +1087,9 @@ "name": "soundcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/soundcraft/icon.png", "source": "./external_plugins/soundcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "SoundCraft", "displayName_i18n": { "en": "SoundCraft", @@ -1149,7 +1172,9 @@ }, "category": "developer-tools", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/kicad/icon.png", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "KiCad", "displayName_i18n": { "en": "KiCad", @@ -1531,6 +1556,46 @@ "repository": "https://github.com/geer1895/rfauto", "license": "GPL-3.0-only", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/rfauto/icon.png" + }, + { + "name": "guster", + "source": "./external_plugins/guster", + "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/guster/icon.png", + "description": "Discover, audit, plan and evolve agent context across projects, skills, tools, knowledge and credentials; optionally scaffold a lightweight Guster Hub for durable knowledge and project state.", + "description_i18n": { + "en": "Discover, audit, plan and evolve agent context across projects, skills, tools, knowledge and credentials; optionally scaffold a lightweight Guster Hub for durable knowledge and project state.", + "zh-CN": "发现、诊断、规划和演进全机 agent 上下文:项目、指令、Skill、工具、知识、状态与凭据引用;可初始化一个轻量 Guster Hub,承载长期知识与项目状态。" + }, + "version": "0.9.5", + "author": { + "name": "Lumos-789", + "url": "https://github.com/Lumos-789" + }, + "category": "developer-tools", + "keywords": [ + "agent", + "context", + "governance", + "skills", + "mcp", + "codex", + "zcode", + "dsh" + ], + "examplePrompts": [ + "Run a full context audit and show me the health score", + "Initialize this project with the standard .agents layout" + ], + "examplePrompts_i18n": { + "zh-CN": [ + "跑一遍全机上下文审计,给我健康分", + "用标准 .agents 布局初始化这个项目" + ] + }, + "displayName": "Guster", + "displayName_i18n": { + "zh-CN": "Guster" + } } ] -} \ No newline at end of file +}