feat(check): add control-plane validator, stale namespace detection, and registry credential checks

.github/workflows/bazel.yml

@@ -194,8 +194,10 @@ jobs:
           # subtrees plus the shared Bazel scaffold). Java and stargate no longer
           # ride the root row: discovered Java components and stargate have
           # their own scoped rows, so src/libraries/ is narrowed to
-          # src/libraries/go/.
-          ROOT_GLOBS=(MODULE.bazel .bazelignore .bazelrc .bazelversion BUILD.bazel WORKSPACE WORKSPACE.bazel go.work go.work.bazel rules/ platforms/ tools/ ci/ src/clis/ src/libraries/go/ .github/bazel-root-test-quarantine.txt)
+          # src/libraries/go/. The two stacks are the data of //:nvcf-cli-stacks,
+          # which the nvcf-cli tests that pin the CLI to the stacks read, so a
+          # change to a stack runs those tests.
+          ROOT_GLOBS=(MODULE.bazel .bazelignore .bazelrc .bazelversion BUILD.bazel WORKSPACE WORKSPACE.bazel go.work go.work.bazel rules/ platforms/ tools/ ci/ src/clis/ src/libraries/go/ deploy/stacks/self-managed/ deploy/stacks/nvcf-compute-plane/ .github/bazel-root-test-quarantine.txt)
 
           run_all=false
           changed=""
@@ -255,7 +257,7 @@ jobs:
           # subtrees a change actually touches (path prefix), plus the
           # reverse-dependency edges below (shared Java config -> every Java row;
           # a Java framework -> every Java service) and ROOT_GLOBS -> root row. A
-          # docs-only or helm-only change touches no subtree and runs nothing.
+          # docs-only or chart-only change touches no subtree and runs nothing.
           # The full-matrix fallbacks stay in the case above (workflow_dispatch,
           # an unreadable or rewritten diff, a new branch) so an untrustworthy
           # changeset is never under-tested.
@@ -416,7 +418,7 @@ jobs:
               hit=true
             elif [ "$path" = "." ]; then
               # Root is glob-gated on PRs and merges alike: it runs only when a
-              # ROOT_GLOB file changed (a docs- or helm-only change never triggers
+              # ROOT_GLOB file changed (a docs- or chart-only change never triggers
               # it). When it does run on a main push it uploads (CACHE_UPLOAD),
               # warming the cache; the rings do not evict, so a later root PR reads
               # that warm closure.

BUILD.bazel

@@ -51,6 +51,26 @@ filegroup(
     visibility = ["//visibility:public"],
 )
 
+# The self-managed and compute-plane stacks, also outside gazelle, for the
+# nvcf-cli tests that pin the CLI's view of the stacks to their source.
+# ROOT_GLOBS in .github/workflows/bazel.yml lists the same directories, so a
+# change to them runs those tests.
+filegroup(
+    name = "nvcf-cli-stacks",
+    srcs = glob(
+        [
+            "deploy/stacks/nvcf-compute-plane/**",
+            "deploy/stacks/self-managed/**",
+        ],
+        exclude = [
+            "deploy/stacks/*/out/**",
+            "deploy/stacks/*/testdata/**",
+            "deploy/stacks/*/tests/**",
+        ],
+    ),
+    visibility = ["//src/clis/nvcf-cli:__subpackages__"],
+)
+
 # Gazelle manages only the native Go subtrees listed in go.work.bazel.
 # nv-boot-parent and Cloud Tasks are also owned by the root Bazel module, but
 # their checked-in Java BUILD files are maintained manually and remain outside

ai-tooling/user/skills/nvcf-self-managed-cli/SKILL.md

@@ -59,13 +59,21 @@ KUBECONFIG=cp.yaml:gpu1.yaml nvcf-cli self-hosted up \
   --compute-plane-context=admin@gpu1 \
   --icms-url=https://icms.nvcf.example.com
 
-# Add a new compute plane to an existing control plane (no kubectl access to CP needed;
+# Add a new compute plane to an existing control plane, from the profile
+# `self-hosted control-plane profile export` wrote to the path it printed,
+# copied here as control-plane-profile.yaml (no kubectl access to CP needed;
 # reaches the control plane via the public ICMS HTTPRoute):
-nvcf-cli self-hosted add-compute-plane \
+nvcf-cli self-hosted compute-plane register \
+  --control-plane-profile=control-plane-profile.yaml \
   --cluster-name=ncp-local-2 \
-  --compute-plane-context=admin@gpu2 \
+  --kube-context=admin@gpu2 \
   --icms-url=https://icms.nvcf.example.com \
-  --token=$ADMIN_JWT
+  --token=$ADMIN_JWT \
+  --output=ncp-local-2-register-values.yaml
+nvcf-cli self-hosted compute-plane install \
+  --values=ncp-local-2-register-values.yaml \
+  --kube-context=admin@gpu2 \
+  --cluster-name=ncp-local-2
 
 # Tear down (always plan-only first):
 nvcf-cli self-hosted down --plan-only --cluster-name=ncp-local --json | jq
@@ -75,7 +83,7 @@ nvcf-cli self-hosted down --cluster-name=ncp-local
 nvcf-cli self-hosted uninstall --no-apply --compute-plane --cluster-name=ncp-local | kubectl delete -f -
 ```
 
-> **`up` vs `add-compute-plane`.** `up` always installs both planes — use it for the *first* install. `add-compute-plane` is the right subcommand any time the control plane is already running and you want to attach an Nth compute cluster.
+> `up` vs `compute-plane register` and `install`. `up` always installs both planes; use it for the first install. To attach an Nth compute cluster to a running control plane, register it with `compute-plane register` and install it with `compute-plane install`.
 >
 > **`down` always with `--plan-only` first.** Show the user the `willUninstall.commands[]` array before running for real.
 
@@ -135,12 +143,12 @@ After `init`, the credentials live in `~/.nvcf-cli.state`, so later commands wor
 
 | Subcommand | What it does | When to use |
 |---|---|---|
-| `nvcf-cli self-hosted check --pre [--local-only \| --control-plane-context=X \| --compute-plane-context=Y]` | Pre-flight: local-host tools + cluster-side prerequisites | Always run first on a new environment |
+| `nvcf-cli self-hosted check --pre [--local-only \| --control-plane-context=X --compute-plane-context=Y]` | Pre-flight: local-host tools, registry credentials, and cluster-side prerequisites for both planes. Pass both context flags or neither | Always run first on a new environment |
 | `nvcf-cli self-hosted install --control-plane \| kubectl apply -f -` | Render + apply the control plane | When you want manual control over apply (GitOps-friendly) |
 | `nvcf-cli self-hosted install --compute-plane --cluster-name=X \| kubectl apply -f -` | Register cluster + render compute plane | Same — manual apply path |
 | `nvcf-cli self-hosted up --cluster-name=X` | One-shot first install: pre-flight → control plane → register → compute plane | Standard install path (both planes from scratch) |
 | `nvcf-cli self-hosted up --plan-only --cluster-name=X` | Dry-run: emit phase-by-phase plan + ETA without changing state | Agent / CI preview before commit |
-| `nvcf-cli self-hosted add-compute-plane --cluster-name=X --compute-plane-context=Y --icms-url=… --token=$JWT` | Add a new compute plane to an *existing* control plane (no CP install) | Adding the 2nd, 3rd, … GPU cluster after the initial `up` |
+| `nvcf-cli self-hosted compute-plane register --control-plane-profile=P --cluster-name=X --kube-context=Y --output=V`, then `nvcf-cli self-hosted compute-plane install --values=V --kube-context=Y` | Add a new compute plane to an existing control plane (no CP install) | Adding the 2nd, 3rd, ... GPU cluster after the initial `up` |
 | `nvcf-cli self-hosted uninstall --control-plane` | Per-plane primitive: `helmfile destroy` on the control plane (refuses if compute planes still registered) | Final teardown after all compute planes are gone, or scripted pipelines |
 | `nvcf-cli self-hosted uninstall --compute-plane --cluster-name=X` | Per-plane primitive: `helmfile destroy` on a compute plane (no ICMS unregister, no drain) | Just remove the helm releases without the ICMS-side cleanup |
 | `nvcf-cli self-hosted uninstall --no-apply <plane>` | Render delete YAML via `helm get manifest` | GitOps; `\| kubectl delete -f -` or commit + Argo applies |
@@ -195,7 +203,7 @@ Session stickiness uses `x-multi-turn-session-id` for chat completions and Respo
 For step-by-step playbooks, load the prompt that matches the user's intent:
 
 - **Install from scratch.** [prompts/install-from-scratch.md](prompts/install-from-scratch.md) — k3d cluster → preflight → up → deploy a smoke function.
-- **Add a new compute plane.** [prompts/add-compute-plane.md](prompts/add-compute-plane.md) — split-cluster `up` against an existing control plane.
+- **Add a new compute plane.** [prompts/add-compute-plane.md](prompts/add-compute-plane.md): register and install a compute plane against an existing control plane.
 - **Deploy and invoke a function.** [prompts/deploy-and-invoke.md](prompts/deploy-and-invoke.md) — create → deploy → API key → invoke, including the LLM create/invoke variant.
 - **Diagnose a failed install.** [prompts/diagnose-failed-install.md](prompts/diagnose-failed-install.md) — `status --json` → identify failed component → kubectl describe → remediation.
 - **Rotate JWKS.** [prompts/rotate-cluster-jwks.md](prompts/rotate-cluster-jwks.md) — when PSAT auth starts failing.
@@ -250,16 +258,21 @@ For step-by-step playbooks, load the prompt that matches the user's intent:
 
 `nvcf-cli` subcommands that long-run (`up`, `status`, `check`) accept four output modes:
 
-- **`--json`** — JSONL events on stderr; one event per line; stable schema (`schemaVersion: 2`). **Use this when running under an agent.** Parse line-by-line with `jq -c .` or `json.loads()`.
-- **`--plain`** — Plain timestamped lines, RFC3339 UTC, `[NN/8]` phase prefix; grep-friendly. Default in non-TTY.
-- **`--accessible`** — Plain output without spinners, with verbose state markers (`[completed]`, `[running]`, `[pending]`, `[failed]`). For screen readers and constrained terminals.
-- **(no flag)** — Bubbletea TTY dashboard. Default in TTY ≥100×30. Don't use under an agent (cursor-up sequences are noisy).
+- `--json`: JSONL events on stderr, one event per line, stable schema (`schemaVersion: 1`). Use this when running under an agent. Redirect stderr to the parser (`2>&1 >/dev/null`), keep only lines that start with `{`, and parse line-by-line with `jq -c .` or `json.loads()`.
+- `--plain`: plain timestamped lines, RFC3339 UTC, `[NN/8]` phase prefix; grep-friendly. Default in non-TTY.
+- `--accessible`: plain output without spinners, with verbose state markers (`[completed]`, `[running]`, `[pending]`, `[failed]`). For screen readers and constrained terminals.
+- (no flag): Bubbletea TTY dashboard. Default in a TTY of at least 100x30. Don't use under an agent (cursor-up sequences are noisy).
 
 Auto-detect picks the right mode for whatever stdout/stderr is. The CLI also honors `NO_COLOR` (any value → forces plain), `TERM=dumb` (forces plain), and `CI=truthy` (forces plain even on a fake TTY). When the terminal is smaller than 100×30, the bubbletea renderer falls back to a compact layout. Explicit `--json` is the right call for agent piping.
 
 ## On failure
 
-`nvcf-cli` failures emit structured `phase_failed` events in JSON:
+`check` never emits `phase_failed`. It ends with one `final` event: gate on
+`success`, and treat a missing `final` event as a failure. See
+[examples/ci-pipelines.md](examples/ci-pipelines.md) and
+[reference/exit-codes.md](reference/exit-codes.md).
+
+Other `nvcf-cli` failures emit structured `phase_failed` events in JSON:
 
 ```json
 {"event":"phase_failed","phaseNum":4,"phase":"apply-cp",

ai-tooling/user/skills/nvcf-self-managed-cli/examples/ci-pipelines.md

@@ -12,20 +12,25 @@ deploy_nvcf:
   variables:
     KUBECONFIG: $KUBECONFIG_FILE
   script:
-    - nvcf-cli self-hosted check --pre --json | jq -e '.event != "phase_failed"' || exit 2
+    - nvcf-cli self-hosted check --pre --json 2>&1 >/dev/null | grep '^{' | jq -se 'any(.[]; .event == "final" and .success)' || exit 2
     - nvcf-cli self-hosted up --cluster-name=$CLUSTER_NAME --token=$NVCF_ADMIN_JWT --non-interactive --json
-    - nvcf-cli self-hosted status --json | jq -e '.verdict == "healthy"'
+    - nvcf-cli self-hosted status --json 2>&1 >/dev/null | grep '^{' | jq -se 'any(.[]; .verdict == "healthy")'
 ```
 
 Notes:
 
 - `--non-interactive --token=$JWT` is required in CI; never use interactive `init`.
 - Always `--json` for machine-parsing.
+- `--json` writes JSONL to stderr, not stdout, so redirect with `2>&1 >/dev/null` before a parser.
+- `check` reports its verdict in one `final` event, with `success: false` when any check failed at error severity or the run timed out (exit `5`, `verdict: "timeout"`). Gate on that event: `check` never emits `phase_failed`, so a condition on it always passes. Requiring the `final` event also fails the step when the command dies before emitting it.
+- Slurp with `jq -s` before testing a condition. Without it `jq -e` takes its exit status from the last event alone.
+- stderr also carries plain-text notices, so filter to JSON lines. Do not add `--show-logs` here: it appends a non-JSON transcript to the same stream.
+- The plain-text notice that says how to remove objects a validator run left in the cluster is filtered out with the rest. Read the same commands from the `final` event instead, for example `jq -r '.[] | select(.event == "final") | .cleanup[]?'` on the slurped stream.
 - Final status check gates downstream stages on `verdict == "healthy"`.
 
 ## GitOps (Argo / Flux) pattern
 
-CI doesn't `kubectl apply` — instead, render manifests, commit them, let the controller apply.
+CI doesn't `kubectl apply`. Instead, render manifests, commit them, let the controller apply.
 
 ```yaml
 render:
@@ -62,16 +67,20 @@ deploy_compute_planes:
       - CLUSTER: ncp-eu-1
         CONTEXT: admin@gpu-eu-1
   script:
-    - nvcf-cli self-hosted up
+    - nvcf-cli self-hosted compute-plane register
+        --control-plane-profile=control-plane-profile.yaml
         --cluster-name=$CLUSTER
-        --compute-plane-context=$CONTEXT
+        --kube-context=$CONTEXT
         --icms-url=$ICMS_PUBLIC_URL
         --token=$NVCF_ADMIN_JWT
-        --non-interactive
-        --json
+        --output=$CLUSTER-register-values.yaml
+    - nvcf-cli self-hosted compute-plane install
+        --values=$CLUSTER-register-values.yaml
+        --kube-context=$CONTEXT
+        --cluster-name=$CLUSTER
 ```
 
-Each parallel job registers + installs one compute plane against the shared control plane. Failures are isolated.
+Each parallel job registers + installs one compute plane against the shared control plane. Failures are isolated. The control-plane profile comes from `nvcf-cli self-hosted control-plane profile export`, run once against the control plane. It writes the profile to the stack's `out/` directory and prints that path; hand the file to each job as `control-plane-profile.yaml`, for example as an artifact.
 
 ## Plan-only preview in PRs
 

ai-tooling/user/skills/nvcf-self-managed-cli/examples/multi-cluster.md

@@ -11,19 +11,30 @@ The canonical multi-cluster topology. Each compute plane is registered separatel
 KUBECONFIG=cp.yaml nvcf-cli self-hosted install --control-plane | kubectl apply -f -
 nvcf-cli self-hosted check --control-plane --wait 5m
 
-# 2. Register + install each compute plane:
+# 2. Export the control-plane profile every compute plane registers from.
+#    It is written to the stack's out/ directory, and the command prints
+#    that path. Copy the file here:
+KUBECONFIG=cp.yaml nvcf-cli self-hosted control-plane profile export --cluster-name=ncp-cp
+cp <path it printed> control-plane-profile.yaml
+
+# 3. Register + install each compute plane. Both commands name the GPU
+#    cluster's context with --kube-context:
 for CTX in admin@gpu-east-1 admin@gpu-west-1 admin@gpu-eu-1; do
   NAME=$(echo "$CTX" | cut -d@ -f2)
-  nvcf-cli self-hosted up \
+  nvcf-cli self-hosted compute-plane register \
+    --control-plane-profile=control-plane-profile.yaml \
     --cluster-name=$NAME \
-    --compute-plane-context=$CTX \
+    --kube-context=$CTX \
     --icms-url=https://icms.nvcf.example.com \
     --token=$NVCF_ADMIN_JWT \
-    --non-interactive \
-    --json
+    --output=$NAME-register-values.yaml
+  nvcf-cli self-hosted compute-plane install \
+    --values=$NAME-register-values.yaml \
+    --kube-context=$CTX \
+    --cluster-name=$NAME
 done
 
-# 3. Verify all compute planes registered:
+# 4. Verify all compute planes registered:
 nvcf-cli cluster list-registered --nca-id=$NCA_ID --icms-url=$ICMS --json \
   | jq '.clusters[] | {name: .clusterName, status, nvcaVersion}'
 ```
@@ -37,24 +48,45 @@ The operator has kubectl access to ONE compute plane only — the control plane
 # service is reachable via api.nvcf.example.com — no kubectl access needed):
 nvcf-cli init --api-url=https://api.nvcf.example.com
 
-# Run pre-flight scoped to the compute plane only:
-nvcf-cli self-hosted check --pre \
-  --compute-plane-context=admin@gpu1 \
+# Run pre-flight scoped to the compute plane only. check takes both context
+# flags or neither and has no --kubeconfig flag, so give it, through
+# KUBECONFIG for this command only, a kubeconfig holding just the compute
+# cluster's context. The current context of your kubeconfig stays as it was.
+# --compute-plane checks the compute plane as installed, so a docker login
+# with no access to the org of the compute-plane charts passes where
+# NGC_API_KEY reaches it: treat a registry-credentials row that says the
+# docker login has no access as blocking, since compute-plane install pulls
+# its charts with that login. A docker login nvcr.io rejects fails either way.
+# With a control-plane context, check --pre --control-plane checks the new
+# compute plane as not yet installed instead; see prompts/add-compute-plane.md.
+KC=$(mktemp)
+kubectl config view --minify --flatten --context=admin@gpu1 > "$KC"
+KUBECONFIG="$KC" nvcf-cli self-hosted check --compute-plane \
   --icms-url=https://icms.nvcf.example.com \
   --json
 
-# Bring up just the compute plane:
-nvcf-cli self-hosted up \
+# Register and install just the compute plane, from the profile the
+# control-plane operator exported with `self-hosted control-plane profile export`,
+# saved here as control-plane-profile.yaml:
+nvcf-cli self-hosted compute-plane register \
+  --control-plane-profile=control-plane-profile.yaml \
   --cluster-name=my-compute-1 \
-  --compute-plane-context=admin@gpu1 \
+  --kube-context=admin@gpu1 \
   --icms-url=https://icms.nvcf.example.com \
   --token=$ADMIN_JWT \
-  --non-interactive
-
-# Status (compute-only — operator can't see control-plane component health):
-nvcf-cli self-hosted status \
-  --compute-plane-context=admin@gpu1 \
+  --output=my-compute-1-register-values.yaml
+nvcf-cli self-hosted compute-plane install \
+  --values=my-compute-1-register-values.yaml \
+  --kube-context=admin@gpu1 \
+  --cluster-name=my-compute-1
+
+# Status of admin@gpu1, from the same kubeconfig (compute-only: the operator
+# can't see control-plane component health). Then remove the file, which
+# holds the context's credentials:
+KUBECONFIG="$KC" nvcf-cli self-hosted status \
+  --cluster-name=my-compute-1 \
   --icms-url=https://icms.nvcf.example.com
+rm -f "$KC"
 ```
 
 ## Functions targeting specific compute planes
@@ -84,13 +116,17 @@ The `clusters` array (using cluster names, not IDs) limits scheduling to those c
 nvcf-cli cluster list-registered --nca-id=$NCA_ID --icms-url=$ICMS --json \
   | jq -r '.clusters[].clusterName' > clusters.txt
 
-# Status snapshot per compute plane (assuming each context name matches):
+# Status snapshot per compute plane (assuming each context name matches).
+# Each run reads a temporary kubeconfig holding only that cluster's context,
+# so the loop leaves the current context of your kubeconfig alone:
 while read NAME; do
   echo "=== $NAME ==="
-  nvcf-cli self-hosted status \
-    --cluster-name=$NAME \
-    --compute-plane-context=admin@$NAME \
-    --json | jq -c '{cluster:.cluster, verdict:.verdict, reconcile:.reconcileAgeSec}'
+  KC=$(mktemp)
+  kubectl config view --minify --flatten --context="admin@$NAME" > "$KC" &&
+    KUBECONFIG="$KC" nvcf-cli self-hosted status \
+      --cluster-name=$NAME \
+      --json | jq -c '{cluster:.cluster, verdict:.verdict, reconcile:.reconcileAgeSec}'
+  rm -f "$KC"
 done < clusters.txt
 ```