feat: add bench diagnostics

Author: wolone

README.md

@@ -195,6 +195,7 @@ Each bench lives on a single dataset (`<pool>/<bench>`) holding both its files a
 | `bench start` | Start all processes (web, workers, Redis, admin UI) |
 | `bench stop` | Stop a running bench from another terminal |
 | `bench restart` | Restart all processes — supervisor, systemd, or OpenRC (production only) |
+| `bench diagnostics` | Run health checks for resources, processes, Redis, database reachability, and workers |
 | `bench get-app <repo>` | Clone and install an app |
 | `bench new-site <name>` | Create a site |
 | `bench rename-site <old> <new>` | Rename a site (checks the hostname is free across all benches) |

admin/backend/app.py

@@ -14,6 +14,7 @@
 from .views.benches import benches_bp
 from .views.dashboard import dashboard_bp
 from .views.database import database_bp
+from .views.diagnostics import diagnostics_bp
 from .views.git import git_bp
 from .views.logs import logs_bp
 from .views.processes import processes_bp
@@ -217,6 +218,7 @@ def api_logout():
     app.register_blueprint(processes_bp, url_prefix="/api/processes")
     app.register_blueprint(logs_bp, url_prefix="/api/logs")
     app.register_blueprint(database_bp, url_prefix="/api/database")
+    app.register_blueprint(diagnostics_bp, url_prefix="/api/diagnostics")
     app.register_blueprint(tasks_bp, url_prefix="/api/tasks")
     app.register_blueprint(settings_bp, url_prefix="/api/settings")
     app.register_blueprint(updates_bp, url_prefix="/api/updates")

admin/backend/views/diagnostics.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import json
+
+from flask import Blueprint, current_app, jsonify
+
+from pilot.config.toml_store import BenchTomlStore
+from pilot.core.bench import Bench
+from pilot.core.diagnostics import DiagnosticReport, DiagnosticRunner
+
+diagnostics_bp = Blueprint("diagnostics", __name__)
+
+
+@diagnostics_bp.route("/")
+def index():
+    bench_root = current_app.config["BENCH_ROOT"]
+    try:
+        config = BenchTomlStore.for_bench(bench_root).read()
+        checks = DiagnosticRunner(Bench(config, bench_root)).run()
+    except Exception as error:
+        return jsonify({"error": str(error)}), 500
+    return jsonify(json.loads(DiagnosticReport(config.name, checks).to_json()))

docs/commands.md

@@ -337,6 +337,25 @@ Lists every bench found under `benches/`, with its runtime status (running/stopp
 
 ---
 
+## `bench diagnostics`
+
+Runs read-only health checks for the active bench.
+
+```bash
+bench diagnostics
+bench diagnostics --json
+```
+
+Checks cover bench structure, generated config files, site config JSON, core
+runtime dependencies, CPU load, disk space, Linux memory, process state, Redis
+ports, database reachability, workers, and Chromium availability for PDF
+generation. The command does not install packages, start services, modify
+configuration, or run migrations.
+
+The Admin backend exposes the same report at `GET /api/diagnostics/`.
+
+---
+
 ## `bench stop`
 
 Stops a running bench. `ProcessManagerFactory.detect_running()` picks the right manager (dev runner, systemd, or supervisor), then stops the workload and the admin service. Does **not** rely on `pids/bench.pid` — a dev bench with no PID file is stopped by killing the port-bound processes. Works across terminal sessions.

pilot/commands/diagnostics.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import argparse
+
+from pilot.commands.base import Command
+
+
+class DiagnosticsCommand(Command):
+    name = "diagnostics"
+    help = "Run health checks for the bench."
+
+    @classmethod
+    def add_arguments(cls, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument("--json", action="store_true", help="Print diagnostics as JSON.")
+
+    @classmethod
+    def from_args(cls, args: argparse.Namespace, bench):
+        return cls(bench, json_output=args.json)
+
+    def __init__(self, bench, json_output: bool = False) -> None:
+        self.bench = bench
+        self.json_output = json_output
+
+    def run(self) -> None:
+        from pilot.core.diagnostics import DiagnosticReport, DiagnosticRunner
+
+        report = DiagnosticReport(self.bench.config.name, DiagnosticRunner(self.bench).run())
+        if self.json_output:
+            print(report.to_json())
+            return
+        self._print(report)
+
+    def _print(self, report: DiagnosticReport) -> None:
+        print(f"Diagnostics for {report.bench_name}\n")
+        current_group = ""
+        for check in report.checks:
+            if check.group != current_group:
+                current_group = check.group
+                print(_group_title(current_group))
+            print(f"  {_badge(check.status)} {check.name}: {check.detail}")
+            if check.hint:
+                print(f"      {check.hint}")
+        print("\nResult: " + ("issues found" if report.failed else "all checks passed"))
+
+
+def _group_title(group: str) -> str:
+    return f"{group.capitalize()}:"
+
+
+def _badge(status: str) -> str:
+    icons = {"ok": _green("OK"), "warn": _yellow("WARN"), "fail": _red("FAIL"), "skip": _dim("SKIP")}
+    return icons.get(status, status.upper())
+
+
+def _green(text: str) -> str:
+    return f"\033[32m{text}\033[0m"
+
+
+def _yellow(text: str) -> str:
+    return f"\033[33m{text}\033[0m"
+
+
+def _red(text: str) -> str:
+    return f"\033[31m{text}\033[0m"
+
+
+def _dim(text: str) -> str:
+    return f"\033[90m{text}\033[0m"

pilot/core/diagnostics.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import socket
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pilot.core.bench import Bench
+
+
+@dataclass
+class DiagnosticCheck:
+    group: str
+    name: str
+    status: str
+    detail: str
+    hint: str = ""
+
+    def to_dict(self) -> dict[str, str]:
+        data = {"group": self.group, "name": self.name, "status": self.status, "detail": self.detail}
+        if self.hint:
+            data["hint"] = self.hint
+        return data
+
+
+class DiagnosticRunner:
+    def __init__(self, bench: "Bench") -> None:
+        self.bench = bench
+
+    def run(self) -> list[DiagnosticCheck]:
+        checks: list[DiagnosticCheck] = []
+        checks.extend(self._bench_checks())
+        checks.extend(self._configuration_checks())
+        checks.extend(self._site_checks())
+        checks.extend(self._dependency_checks())
+        checks.extend(self._resource_checks())
+        checks.extend(self._process_checks())
+        checks.extend(self._redis_checks())
+        checks.extend(self._database_checks())
+        checks.extend(self._worker_checks())
+        return checks
+
+    def _bench_checks(self) -> list[DiagnosticCheck]:
+        return [
+            self._path_check("bench", "bench.toml", self.bench.path / "bench.toml", "Run bench new <name>."),
+            self._path_check("bench", "python env", self.bench.python, "Run bench init."),
+            self._path_check("bench", "apps directory", self.bench.apps_path, "Run bench init."),
+            self._path_check("bench", "sites directory", self.bench.sites_path, "Run bench init."),
+            self._path_check("bench", "config directory", self.bench.config_path, "Run bench setup config."),
+        ]
+
+    def _configuration_checks(self) -> list[DiagnosticCheck]:
+        checks = [
+            self._path_check("config", "common site config", self.bench.sites_path / "common_site_config.json", "Run bench setup config."),
+            self._path_check("config", "procfile", self.bench.config_path / "Procfile", "Run bench setup config."),
+        ]
+        if self.bench.config.admin.enabled and not self.bench.config.admin.password:
+            checks.append(DiagnosticCheck("config", "admin password", "fail", "admin is enabled without a password", "Run bench set-admin-password."))
+        elif self.bench.config.admin.enabled:
+            checks.append(DiagnosticCheck("config", "admin password", "ok", "configured"))
+        else:
+            checks.append(DiagnosticCheck("config", "admin", "warn", "admin UI is disabled"))
+        return checks
+
+    def _site_checks(self) -> list[DiagnosticCheck]:
+        if not self.bench.sites_path.exists():
+            return [DiagnosticCheck("sites", "sites", "fail", f"missing: {self.bench.sites_path}", "Run bench init.")]
+        site_dirs = [p for p in sorted(self.bench.sites_path.iterdir()) if (p / "site_config.json").exists()]
+        if not site_dirs:
+            return [DiagnosticCheck("sites", "sites", "warn", "no sites found", "Create one with bench new-site <name>.")]
+        checks = [DiagnosticCheck("sites", "site count", "ok", f"{len(site_dirs)} site(s) found")]
+        checks.extend(self._site_config_check(site_dir) for site_dir in site_dirs)
+        return checks
+
+    def _dependency_checks(self) -> list[DiagnosticCheck]:
+        checks = [
+            self._executable_check("dependencies", "redis-server", "redis-server", "Install Redis or run bench init."),
+            self._executable_check("dependencies", "node", "node", "Install Node.js or run bench init."),
+        ]
+        checks.append(self._chromium_check())
+        return checks
+
+    def _resource_checks(self) -> list[DiagnosticCheck]:
+        return [
+            self._cpu_check(),
+            self._disk_check(),
+            self._memory_check(),
+        ]
+
+    def _process_checks(self) -> list[DiagnosticCheck]:
+        from pilot.managers.process_manager import ProcessManager
+
+        try:
+            manager = ProcessManager.detect_running(self.bench)
+            workload = manager.is_running()
+            admin = manager.is_admin_running()
+        except Exception as exc:
+            return [DiagnosticCheck("process", "process manager", "fail", str(exc))]
+        if workload:
+            return [DiagnosticCheck("process", "bench workload", "ok", "workload is running")]
+        if admin:
+            return [DiagnosticCheck("process", "admin", "warn", "admin is running, workload is stopped", "Run bench start or setup production.")]
+        return [DiagnosticCheck("process", "bench workload", "fail", "bench is not running", "Run bench start.")]
+
+    def _redis_checks(self) -> list[DiagnosticCheck]:
+        redis = self.bench.config.redis
+        return [
+            self._port_check("redis", "cache", "127.0.0.1", redis.cache_port),
+            self._port_check("redis", "queue", "127.0.0.1", redis.queue_port),
+        ]
+
+    def _database_checks(self) -> list[DiagnosticCheck]:
+        db_type = self.bench.config.db_type
+        if db_type == "sqlite":
+            return [DiagnosticCheck("database", "sqlite", "skip", "sqlite has no shared database server")]
+        if db_type == "postgres":
+            pg = self.bench.config.postgres
+            return [self._port_check("database", "postgres", pg.host, pg.port)]
+        mdb = self.bench.config.mariadb
+        if mdb.socket_path:
+            return [self._socket_check("database", "mariadb socket", Path(mdb.socket_path))]
+        return [self._port_check("database", "mariadb", mdb.host, mdb.port)]
+
+    def _worker_checks(self) -> list[DiagnosticCheck]:
+        workers = [p for p in self.bench.pids_path.glob("worker*.pid")] if self.bench.pids_path.exists() else []
+        if workers:
+            live = sum(1 for path in workers if self._pid_alive(path))
+            status = "ok" if live else "fail"
+            return [DiagnosticCheck("workers", "worker pids", status, f"{live}/{len(workers)} worker pid files are live")]
+        if self.bench.config.production.use_companion_manager:
+            return [DiagnosticCheck("workers", "workers", "skip", "workers run inside the companion web process")]
+        return [DiagnosticCheck("workers", "worker pids", "warn", "no worker pid files found", "Start the bench and re-run diagnostics.")]
+
+    def _disk_check(self) -> DiagnosticCheck:
+        usage = shutil.disk_usage(self.bench.path)
+        free_gb = usage.free / 1024**3
+        used_pct = (usage.used / usage.total) * 100 if usage.total else 0
+        detail = f"{free_gb:.1f} GB free, {used_pct:.0f}% used at {self.bench.path}"
+        if free_gb < 1:
+            return DiagnosticCheck("resources", "disk", "fail", detail, "Free disk space before running migrations or backups.")
+        if free_gb < 5:
+            return DiagnosticCheck("resources", "disk", "warn", detail, "Keep at least 5 GB free for updates and backups.")
+        return DiagnosticCheck("resources", "disk", "ok", detail)
+
+    def _cpu_check(self) -> DiagnosticCheck:
+        try:
+            load_1m = os.getloadavg()[0]
+        except (AttributeError, OSError):
+            return DiagnosticCheck("resources", "cpu load", "skip", "load average is not available")
+        cpu_count = os.cpu_count() or 1
+        detail = f"1m load {load_1m:.2f} across {cpu_count} CPU(s)"
+        if load_1m > cpu_count * 2:
+            return DiagnosticCheck("resources", "cpu load", "fail", detail, "Server is heavily loaded.")
+        if load_1m > cpu_count:
+            return DiagnosticCheck("resources", "cpu load", "warn", detail, "CPU pressure may slow requests and jobs.")
+        return DiagnosticCheck("resources", "cpu load", "ok", detail)
+
+    def _memory_check(self) -> DiagnosticCheck:
+        memory = self._linux_memory()
+        if not memory:
+            return DiagnosticCheck("resources", "memory", "skip", "memory check is only available on Linux")
+        available_gb = memory["available"] / 1024**2
+        detail = f"{available_gb:.1f} GB available"
+        if available_gb < 0.5:
+            return DiagnosticCheck("resources", "memory", "fail", detail, "Free memory or add swap before starting workers.")
+        if available_gb < 1:
+            return DiagnosticCheck("resources", "memory", "warn", detail, "Low memory can make builds and migrations unreliable.")
+        return DiagnosticCheck("resources", "memory", "ok", detail)
+
+    def _path_check(self, group: str, name: str, path: Path, hint: str) -> DiagnosticCheck:
+        if path.exists():
+            return DiagnosticCheck(group, name, "ok", str(path))
+        return DiagnosticCheck(group, name, "fail", f"missing: {path}", hint)
+
+    def _port_check(self, group: str, name: str, host: str, port: int) -> DiagnosticCheck:
+        if self._tcp_open(host, port):
+            return DiagnosticCheck(group, name, "ok", f"{host}:{port} is reachable")
+        return DiagnosticCheck(group, name, "fail", f"{host}:{port} is not reachable")
+
+    def _socket_check(self, group: str, name: str, path: Path) -> DiagnosticCheck:
+        if path.exists():
+            return DiagnosticCheck(group, name, "ok", str(path))
+        return DiagnosticCheck(group, name, "fail", f"missing: {path}", "Start the database service.")
+
+    def _executable_check(self, group: str, name: str, executable: str, hint: str) -> DiagnosticCheck:
+        path = shutil.which(executable)
+        if path:
+            return DiagnosticCheck(group, name, "ok", path)
+        return DiagnosticCheck(group, name, "fail", f"{executable} not found in PATH", hint)
+
+    def _chromium_check(self) -> DiagnosticCheck:
+        candidates = ["chromium", "chromium-browser", "google-chrome", "google-chrome-stable"]
+        found = next((path for name in candidates if (path := shutil.which(name))), "")
+        if found:
+            return DiagnosticCheck("dependencies", "chromium", "ok", found)
+        return DiagnosticCheck("dependencies", "chromium", "warn", "not found in PATH", "PDF generation may fail until Chromium is installed.")
+
+    def _site_config_check(self, site_dir: Path) -> DiagnosticCheck:
+        path = site_dir / "site_config.json"
+        try:
+            json.loads(path.read_text())
+        except Exception as exc:
+            return DiagnosticCheck("sites", site_dir.name, "fail", f"invalid site_config.json: {exc}")
+        return DiagnosticCheck("sites", site_dir.name, "ok", "site_config.json is valid")
+
+    def _pid_alive(self, path: Path) -> bool:
+        try:
+            os.kill(int(path.read_text().strip()), 0)
+            return True
+        except (ValueError, ProcessLookupError, PermissionError, OSError):
+            return False
+
+    def _linux_memory(self) -> dict[str, int]:
+        path = Path("/proc/meminfo")
+        if not path.exists():
+            return {}
+        fields = {}
+        for line in path.read_text().splitlines():
+            key, value = line.split(":", 1)
+            fields[key] = int(value.strip().split()[0])
+        return {"available": fields.get("MemAvailable", 0)}
+
+    def _tcp_open(self, host: str, port: int) -> bool:
+        try:
+            with socket.create_connection((host, port), timeout=0.5):
+                return True
+        except OSError:
+            return False
+
+
+class DiagnosticReport:
+    def __init__(self, bench_name: str, checks: list[DiagnosticCheck]) -> None:
+        self.bench_name = bench_name
+        self.checks = checks
+
+    @property
+    def failed(self) -> bool:
+        return any(check.status == "fail" for check in self.checks)
+
+    def to_json(self) -> str:
+        data = {"bench": self.bench_name, "checks": [check.to_dict() for check in self.checks]}
+        return json.dumps(data, indent=2)