diff --git a/boomtick-pkg/cli/aggregate-prs.sh b/boomtick-pkg/cli/aggregate-prs.sh deleted file mode 100755 index 2a5a2af7f0..0000000000 --- a/boomtick-pkg/cli/aggregate-prs.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# WARNING: Destructive operation - Modifies local git tracking state, pushes upstream branches, and generates remote Pull Requests. -if [ "$#" -lt 2 ]; then echo "Usage: $0 ..."; exit 1; fi - -# Load base branch from project_config.json if possible, fallback to origin/main -CONFIG_FILE="project_config.json" -BASE_BRANCH="origin/main" -if [ -f "$CONFIG_FILE" ]; then - # Silently attempt to load base_branch using jq if available - if command -v jq &> /dev/null; then - if LOADED_BRANCH=$(jq -r '.base_branch' "$CONFIG_FILE" 2>/dev/null); then - if [ "$LOADED_BRANCH" != "null" ] && [ -n "$LOADED_BRANCH" ]; then - BASE_BRANCH="$LOADED_BRANCH" - fi - else - echo "⚠️ Warning: Failed to parse '$CONFIG_FILE'. Using default base branch '$BASE_BRANCH'." >&2 - fi - else - echo "⚠️ Warning: 'jq' not found. Cannot parse '$CONFIG_FILE'. Using default base branch '$BASE_BRANCH'." >&2 - fi -fi -# Extract name without remote prefix (handles origin/main, upstream/develop, etc) -BASE_BRANCH_NAME=$(echo "$BASE_BRANCH" | sed 's/.*\///') - -T_BR="$1"; shift; PRs=("$@"); git checkout "$BASE_BRANCH_NAME" && git pull origin "$BASE_BRANCH_NAME" && git checkout -b "$T_BR" -P_BODY="" -for pr in "${PRs[@]}"; do - DATA=$(gh pr view "$pr" --json headRefName,body,title --jq '.') - REF=$(echo "$DATA" | jq -r '.headRefName') && TITLE=$(echo "$DATA" | jq -r '.title') && BODY=$(echo "$DATA" | jq -r '.body') - gh pr checkout "$pr" 2>/dev/null && git checkout "$T_BR" - if ! git merge "$REF" -m "Merging PR $pr: $TITLE" 2>/dev/null; then - echo "Conflict detected in PR #$pr. Attempting automatic resolution..." - CONFLICTED_FILES=$(git diff --name-only --diff-filter=U) - if [ -z "$CONFLICTED_FILES" ]; then - echo "CRITICAL: Merge failed but no conflicted files found. Aborting." - git merge --abort - exit 1 - fi - - if ! td-cli gh resolve; then - echo "CRITICAL: Conflict resolution failed in PR #$pr" - git merge --abort - exit 1 - fi - - git add $CONFLICTED_FILES - if ! git commit --no-edit; then - echo "CRITICAL: Failed to commit resolved merge for PR #$pr" - git merge --abort - exit 1 - fi - fi - P_BODY="${P_BODY}Closes #$pr"$'\n\n'"### Description from PR #$pr ($TITLE):"$'\n'"$BODY"$'\n\n'"---"$'\n' -done -git push -u origin "$T_BR" && gh pr create --title "Aggregated Feature: $T_BR" --body "$P_BODY" --head "$T_BR" --base "$BASE_BRANCH_NAME" diff --git a/boomtick-pkg/cli/analyze_overlaps.sh b/boomtick-pkg/cli/analyze_overlaps.sh deleted file mode 100755 index c0d35e62cd..0000000000 --- a/boomtick-pkg/cli/analyze_overlaps.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -OVERLAP_FILE="pr_overlaps.txt" - -if [[ ! -f "$OVERLAP_FILE" ]]; then - gh pr list --json number --jq '.[].number' | xargs -I{} sh -c "gh pr diff {} --name-only | sed 's|^|{} |'" | \ - awk '{file=$2; pr=$1; count[file]++; prs[file] = prs[file] " PR #" pr} END {for (f in count) if (count[f] > 1) print f " overlaps in:" prs[f]}' > "$OVERLAP_FILE" -fi - -awk -F' overlaps in: ' '{ - f=$1; p=$2; - split(f, path, "/"); - dir = (path[2] == "") ? "root" : path[1] "/"; - results[dir] = results[dir] "File: " f "\n PRs: " p "\n" -} -END { - for (d in results) { - print "=== Directory: " d " ==="; - print results[d]; - } -}' "$OVERLAP_FILE" diff --git a/boomtick-pkg/cli/dev_tools/cli-schema.json b/boomtick-pkg/cli/dev_tools/cli-schema.json index f86bbf0e5a..9b80da3306 100644 --- a/boomtick-pkg/cli/dev_tools/cli-schema.json +++ b/boomtick-pkg/cli/dev_tools/cli-schema.json @@ -746,7 +746,7 @@ ] }, "agent dispatch": { - "description": "", + "description": "\nInitialize a Jules session for a specific branch and task.\nNote: Use 'main' branch for PR consolidation tasks to avoid rebasing issues.\n", "exact_usage": "python3 boomtick-pkg/cli/dev_tools/td_cli.py agent dispatch ", "required_arguments": [ { @@ -861,7 +861,7 @@ ] }, "jules dispatch": { - "description": "", + "description": "\nInitialize a Jules session for a specific branch and task.\nNote: Use 'main' branch for PR consolidation tasks to avoid rebasing issues.\n", "exact_usage": "python3 boomtick-pkg/cli/dev_tools/td_cli.py jules dispatch ", "required_arguments": [ { diff --git a/boomtick-pkg/cli/dev_tools/dev_tools_sdk/config.py b/boomtick-pkg/cli/dev_tools/dev_tools_sdk/config.py index 8d083874ca..3c4fa873f3 100644 --- a/boomtick-pkg/cli/dev_tools/dev_tools_sdk/config.py +++ b/boomtick-pkg/cli/dev_tools/dev_tools_sdk/config.py @@ -13,45 +13,21 @@ class ProjectConfig: github_token_env: str = "GITHUB_TOKEN" gh_token_env: str = "GH_TOKEN" jules_api_url: str | None = None - core_dirs: List[str] = field(default_factory=lambda: ["src/layouts/", "src/components/"]) + core_dirs: List[str] = field(default_factory=list) monolithic_pr_threshold: int = 3 base_branch: str = "origin/main" max_diff_chars: int = 40000 - content_scopes: Dict[str, str] = field(default_factory=lambda: { - "resources": "content/resources/", - "posts": "content/posts/", - "blog": "content/blog/", - "studies": "content/studies/" - }) + content_scopes: Dict[str, str] = field(default_factory=dict) ai_synthesis_model: str = "gpt-4o-mini" ai_review_model: str = "gpt-4o" ai_vision_model: str = "gpt-4o" - ui_indicators: List[str] = field(default_factory=lambda: [ - "src/components", "src/pages", "src/layouts", "src/index.css", "tailwind" - ]) - tailwind_indicators: List[str] = field(default_factory=lambda: [ - "px-", "py-", "mt-", "flex", "grid", "text-[" - ]) - audit_check_dirs: List[str] = field(default_factory=lambda: [ - "src/features", "src/pages", "src/components", "src/layouts", "src/App.tsx" - ]) - allowed_bots: List[str] = field(default_factory=lambda: [ - "github-actions[bot]" - ]) + ui_indicators: List[str] = field(default_factory=list) + tailwind_indicators: List[str] = field(default_factory=list) + audit_check_dirs: List[str] = field(default_factory=list) + allowed_bots: List[str] = field(default_factory=lambda: ["github-actions[bot]"]) worktree_prefix: str = "bt-repair-" - spec_sections: List[str] = field(default_factory=lambda: [ - "Problem Statement", - "Goal", - "Non-Goals", - "Proposed Approach", - "Alternatives Considered", - "Architectural Impact", - "Scope", - "UNDERSTAND THE ISSUE", - "DETERMINE APPROACH", - "SPECIFY SCOPE", - "DEFINITION OF DONE" - ]) + spec_sections: List[str] = field(default_factory=list) + @property def base_branch_name(self) -> str: @@ -70,8 +46,7 @@ def load_project_config(path: str | Path = "project_config.json") -> ProjectConf raw = json.loads(p.read_text(encoding="utf-8")) except (json.JSONDecodeError, IOError): pass - else: - return ProjectConfig() + def get_list(key: str) -> Optional[List[str]]: val = raw.get(key) @@ -90,8 +65,15 @@ def get_dict(key: str) -> Optional[Dict[str, str]]: return None kwargs: Dict[str, Any] = {} - if "github_repo" in raw or "repo_name" in raw: - kwargs["github_repo"] = raw.get("github_repo") or raw.get("repo_name") + + # Critical Field Resolution (Fail-Fast Pattern) + github_repo = raw.get("github_repo") or raw.get("repo_name") or os.environ.get("GITHUB_REPOSITORY") or os.environ.get("GH_REPO") + if not github_repo and os.environ.get("CI") == "true": + raise RuntimeError("Configuration Error: 'github_repo' must be defined in project_config.json or environment (GITHUB_REPOSITORY) in CI.") + + if github_repo: + kwargs["github_repo"] = github_repo + if "github_token_env" in raw: kwargs["github_token_env"] = raw["github_token_env"] if "gh_token_env" in raw: diff --git a/boomtick-pkg/cli/dev_tools/generate_aggregate_prs_workflow.py b/boomtick-pkg/cli/dev_tools/generate_aggregate_prs_workflow.py index 4343679c54..c732fe4aa2 100644 --- a/boomtick-pkg/cli/dev_tools/generate_aggregate_prs_workflow.py +++ b/boomtick-pkg/cli/dev_tools/generate_aggregate_prs_workflow.py @@ -17,7 +17,7 @@ def main(): print("Generating workflow plan for Aggregate PRs...") # 1. Environment Validation - env_output = run_command(["bash", "boomtick-pkg/cli/verify.sh"]) + env_output = run_command(["td-cli", "doctor"]) # 2. Get Open PRs (Limit 100 per conventions) prs_output = run_command(["td-cli", "gh", "overlaps", "--limit", "100"]) diff --git a/boomtick-pkg/cli/dev_tools/generate_review_workflow.py b/boomtick-pkg/cli/dev_tools/generate_review_workflow.py index e2f5d7228a..8e098bb2d3 100644 --- a/boomtick-pkg/cli/dev_tools/generate_review_workflow.py +++ b/boomtick-pkg/cli/dev_tools/generate_review_workflow.py @@ -24,7 +24,7 @@ def main(): print(f"Generating workflow plan for PR #{pr_number}...") # 1. Environment Validation - env_output = run_command(["bash", "dev-tools/verify.sh"]) + env_output = run_command(["td-cli", "doctor"]) # 2. Issue Validation issue_output = "No issue number provided." diff --git a/boomtick-pkg/cli/dev_tools/pr_overlap.py b/boomtick-pkg/cli/dev_tools/pr_overlap.py deleted file mode 100755 index a77e20af61..0000000000 --- a/boomtick-pkg/cli/dev_tools/pr_overlap.py +++ /dev/null @@ -1,119 +0,0 @@ -import subprocess -import json -import argparse -import sys -import os -# nosemgrep: python.lang.security.deserialization.pickle.avoid-pickle -import pickle -from collections import defaultdict - -sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from utils import get_github_client, get_repo_name - - -def main(): - parser = argparse.ArgumentParser(description="Identify and propose consolidation of Pull Requests (PRs) that demonstrate high levels of functional or structural overlap.") - parser.add_argument("--limit", type=int, default=50, help="Limit the number of open PRs to process (default: 50)") - parser.add_argument("--no-cache", action="store_true", help="Bust the cache and force fetching data from GitHub") - args = parser.parse_args() - - CACHE_FILE = ".pr_cache.pkl" - limit = args.limit - - def get_open_prs(limit): - try: - client = get_github_client() - repo = client.get_repo(get_repo_name()) - pulls = repo.get_pulls(state='open') - prs = [] - for pr in list(pulls)[:limit]: - prs.append({"number": pr.number, "title": pr.title}) - return prs - except Exception as e: - print(f"Error fetching open PRs: {e}", file=sys.stderr) - sys.exit(1) - - def get_pr_files(pr_number): - try: - client = get_github_client() - repo = client.get_repo(get_repo_name()) - pr = repo.get_pull(int(pr_number)) - files = pr.get_files() - return {f.filename for f in files if not f.filename.startswith("tests/visual.spec.ts-snapshots/")} - except Exception as e: - print(f"Error fetching files for PR #{pr_number}: {e}", file=sys.stderr) - return set() - - if not args.no_cache and os.path.exists(CACHE_FILE): - with open(CACHE_FILE, 'rb') as f: - # nosemgrep: python.lang.security.deserialization.pickle.avoid-pickle - cache = pickle.load(f) - else: - cache = {"prs": {}, "files": {}} - - current_prs = get_open_prs(limit) - for pr in current_prs: - num = str(pr['number']) - cache["prs"][num] = pr['title'] - cache["files"][num] = get_pr_files(num) - - with open(CACHE_FILE, 'wb') as f: - # nosemgrep: python.lang.security.deserialization.pickle.avoid-pickle - pickle.dump(cache, f) - - # 1. Report specific exact-match overlap groups - overlap_groups = defaultdict(list) - for pr_num, files in cache["files"].items(): - for file in files: - touching_prs = [p for p, fs in cache["files"].items() if file in fs] - if len(touching_prs) > 1: - overlap_groups[frozenset(touching_prs)].append(file) - - print("--- EXACT OVERLAP GROUPS ---") - for pr_set, files in sorted(overlap_groups.items(), key=lambda x: len(x[1]), reverse=True): - pr_list = sorted(list(pr_set), key=int) - print(f"PRs {', '.join(pr_list)} overlap on {len(files)} files:") - for pr in pr_list: - print(f" [{pr}] {cache['prs'].get(pr, 'Unknown PR')}") - for f in sorted(files): - print(f" - {f}") - - # 2. Report connected clusters - print("\n--- CONNECTED CLUSTERS ---") - graph = defaultdict(set) - all_prs = list(cache["files"].keys()) - for i, pr1 in enumerate(all_prs): - for pr2 in all_prs[i+1:]: - if cache["files"][pr1] & cache["files"][pr2]: - graph[pr1].add(pr2) - graph[pr2].add(pr1) - - visited = set() - for pr in all_prs: - if pr not in visited and pr in graph: - component = {pr} - stack = [pr] - while stack: - curr = stack.pop() - for neighbor in graph[curr]: - if neighbor not in visited: - visited.add(neighbor) - component.add(neighbor) - stack.append(neighbor) - visited.add(pr) - - comp_list = sorted(list(component), key=int) - involved_files = set() - for p in component: - involved_files |= cache["files"][p] - - print(f"Cluster PRs {', '.join(comp_list)}:") - for p in comp_list: - print(f" [{p}] {cache['prs'].get(p, 'Unknown PR')}") - print(" All files touched by this cluster:") - for f in sorted(list(involved_files)): - print(f" - {f}") - print("-" * 40) - -if __name__ == "__main__": - main() diff --git a/boomtick-pkg/cli/dev_tools/repair.py b/boomtick-pkg/cli/dev_tools/repair.py deleted file mode 100755 index 2dcf13f5b0..0000000000 --- a/boomtick-pkg/cli/dev_tools/repair.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -""" -repair.py - Agentic CI Repair via Orchestrator -Part of the Tech-Dancer 'Self-Healing' CI pipeline. -""" - -import os -import sys -import json -import re - -from tdw_services.orchestrator import Orchestrator -from utils import extract_failing_info - -MAX_RETRIES = 3 - -def log(msg): - print(f"🤖 [Repair Agent] {msg}") - -def parse_eslint_json(json_path: str) -> list: - if not os.path.exists(json_path): - return [] - try: - with open(json_path, 'r') as f: - data = json.load(f) - - findings = [] - for file_entry in data: - file_path = file_entry['filePath'] - if file_path.startswith(os.getcwd()): - file_path = os.path.relpath(file_path, os.getcwd()) - - for msg in file_entry.get('messages', []): - if msg.get('severity') >= 2: # Error - findings.append({ - "file": file_path, - "line": msg.get('line'), - "message": f"{msg.get('message')} ({msg.get('ruleId')})", - "type": "eslint" - }) - return findings - except: - return [] - -def agent_loop(file_path, initial_errors): - current_errors = initial_errors - orch = Orchestrator() - - for attempt in range(MAX_RETRIES): - log(f"Attempt {attempt + 1} for {file_path}") - - success = orch.repair_ci(file_path, current_errors) - if not success: - log(f"Failed to apply fix for {file_path}") - break - - import subprocess - res = subprocess.run(["pnpm", "run", "type-check"], capture_output=True, text=True) - new_findings = extract_failing_info(res.stdout + res.stderr) - new_errors = [f["message"] for f in new_findings if f["file"] == file_path] - - if not new_errors: - log(f"✅ Fixed all identified errors in {file_path}") - return True - else: - log(f"⚠️ Still has {len(new_errors)} errors in {file_path}. Retrying...") - current_errors = new_errors - - return False - -def main(): - json_findings = [] - if "--eslint-json" in sys.argv: - idx = sys.argv.index("--eslint-json") - json_findings = parse_eslint_json(sys.argv[idx+1]) - - logs = sys.stdin.read() if "--stdin" in sys.argv else "" - if not logs and len(sys.argv) > 1 and os.path.exists(sys.argv[1]): - with open(sys.argv[1], "r") as f: - logs = f.read() - - findings = json_findings + extract_failing_info(logs) - - if not findings: - log("No actionable errors found.") - sys.exit(0) - - files_to_fix = {} - for f in findings: - if f["file"] not in files_to_fix: - files_to_fix[f["file"]] = [] - files_to_fix[f["file"]].append(f["message"]) - - for file_path, errors in files_to_fix.items(): - if not os.path.exists(file_path): - continue - agent_loop(file_path, errors) - -if __name__ == "__main__": - main() diff --git a/boomtick-pkg/cli/dev_tools/utils.py b/boomtick-pkg/cli/dev_tools/utils.py index ddf83d5a9a..aeab11306d 100644 --- a/boomtick-pkg/cli/dev_tools/utils.py +++ b/boomtick-pkg/cli/dev_tools/utils.py @@ -96,6 +96,21 @@ def to_standard_schema(schema, uppercase: bool = False): return [to_standard_schema(item, uppercase=uppercase) for item in schema] return schema +def _call_api_with_retry(req: urllib.request.Request, max_retries: int = 3) -> Optional[dict]: + """Helper to perform urllib requests with retries and exponential backoff.""" + for attempt in range(max_retries): + try: + with urllib.request.urlopen(req, timeout=30) as response: + if response.status == 200: + return json.loads(response.read().decode("utf-8")) + except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as e: + if attempt == max_retries - 1: + log_error(f"API call failed after {max_retries} attempts: {e}") + return None + wait_time = (2 ** attempt) + (random.random() * 0.1) + time.sleep(wait_time) + return None + def call_ai(prompt: str, model: str = None, url: Optional[str] = None, max_retries: int = 3, schema = None) -> Optional[str]: """Unified helper to call AI API using LangChain ChatOpenAI with retries.""" try: @@ -327,29 +342,34 @@ def run_command(cmd: Union[str, List[str]], shell: bool = False, check: bool = T return proc -def get_github_token() -> Optional[str]: - """Retrieves the GitHub token from environment (prioritizing GITHUB_TOKEN).""" - return os.getenv("GITHUB_TOKEN") +def get_github_token() -> str: + """Retrieves the GitHub token from environment. Fails fast in CI.""" + token = os.getenv("GITHUB_TOKEN") + if not token and os.environ.get("CI") == "true": + raise CLIError("GITHUB_TOKEN is missing. This is required for operation in CI.", code=401) + return token or "" -def get_repo_name() -> Optional[str]: - """Auto-detect repo from environment variables or git remote.""" +def get_repo_name() -> str: + """Auto-detect repo from environment variables. Fails fast in CI.""" repo = os.getenv("GITHUB_REPOSITORY") or os.getenv("GH_REPO") if repo: return repo + if os.environ.get("CI") == "true": + raise CLIError("GITHUB_REPOSITORY is missing. This is required for operation in CI.", code=400) + try: - # Using check=False here to avoid noisy logs for a common discovery step + # Local discovery as fallback only res = run_command(['git', 'config', '--get', 'remote.origin.url'], check=False, log_on_error=False) - if res.returncode != 0: - return os.getenv("GH_REPO") - url = res.stdout.strip() - if not url: - return os.getenv("GH_REPO") - import re - match = re.search(r'[:/]([^/]+/[^/.]+)(\.git)?$', url) - return match.group(1) if match else url + if res.returncode == 0 and res.stdout: + url = res.stdout.strip() + match = re.search(r'[:/]([^/]+/[^/.]+)(\.git)?$', url) + if match: + return match.group(1) except Exception: - return None + pass + + return "" class GHAConfigManager: """Manages GitHub Actions variables with local caching and robust error handling.""" @@ -406,37 +426,11 @@ def get_variable(self, name: str) -> Optional[str]: import requests token = get_github_token() repo = get_repo_name() - if token and repo: - try: - import requests - url = f"https://api.github.com/repos/{repo}/actions/variables/{name}" - headers = { - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.github+json" - } - response = requests.get(url, headers=headers, timeout=10) - if response.status_code == 200: - data = response.json() - val = str(data.get("value", "")) - self.cache[name] = val - self._save_cache() - return val - except Exception: - pass - - # 3. Check gh CLI availability - if self.gh_available is None: - try: - run_command(["gh", "--version"], log_on_error=False) - self.gh_available = True - except (CLIError, FileNotFoundError): - self.gh_available = False - - if not self.gh_available: + if not (token and repo): return None try: - url = f"https://api.github.com/repos/{repo_name}/actions/variables/{name}" + url = f"https://api.github.com/repos/{repo}/actions/variables/{name}" headers = { "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json", @@ -444,12 +438,10 @@ def get_variable(self, name: str) -> Optional[str]: } response = requests.get(url, headers=headers, timeout=10) if response.status_code == 200: - val = response.json().get("value") + val = str(response.json().get("value", "")) self.cache[name] = val self._save_cache() return val - elif response.status_code != 404: - log_error(f"fetching GHA variable '{name}' via API: {response.status_code} {response.text}") except Exception as e: log_error(f"Unexpected error fetching GHA variable '{name}': {e}") diff --git a/boomtick-pkg/cli/snapshot.sh b/boomtick-pkg/cli/snapshot.sh deleted file mode 100755 index a136e577c9..0000000000 --- a/boomtick-pkg/cli/snapshot.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Ensure we are in the project root -cd "$(dirname "$0")/../.." - -echo "=== DevTools Snapshot & Debug Utility ===" - -CONFIG_FILE="boomtick-pkg/project_config.json" - -if [ -f "$CONFIG_FILE" ]; then - echo "Using configuration from $CONFIG_FILE" - if command -v jq &> /dev/null; then - echo "Configuration state:" - jq '.' "$CONFIG_FILE" - else - echo "jq not installed. Raw configuration:" - cat "$CONFIG_FILE" - fi -else - echo "⚠️ Warning: $CONFIG_FILE not found. Using defaults." -fi - -# Basic environment check -echo "--- Environment ---" -echo "Python version: $(python3 --version 2>&1 || echo 'Not installed')" - -# Node version validation -NODE_VERSION=$(node --version 2>&1 | sed 's/^v//' || echo 'Not installed') -echo "Node version: v$NODE_VERSION" - -if [ -f ".nvmrc" ]; then - PINNED_NODE=$(cat .nvmrc | sed 's/^v//') - PINNED_MAJOR=$(echo "$PINNED_NODE" | cut -d. -f1) - CURRENT_MAJOR=$(echo "$NODE_VERSION" | cut -d. -f1) - # Bypass engine checks for Jules agents - IS_JULES=0 - if [[ "$USER" == *"jules"* ]] || [ -n "$JULES_API_KEY" ]; then - IS_JULES=1 - fi - - if [ "$CURRENT_MAJOR" != "$PINNED_MAJOR" ] && [ "$IS_JULES" -eq 0 ]; then - echo "❌ Error: Node version mismatch!" - echo " Expected: v$PINNED_NODE (from .nvmrc)" - echo " Actual: v$NODE_VERSION" - echo " Please install and use the pinned version." - else - if [ "$IS_JULES" -eq 1 ] && [ "$CURRENT_MAJOR" != "$PINNED_MAJOR" ]; then - echo "✅ Node version mismatch bypassed for Jules agent (v$NODE_VERSION)" - else - echo "✅ Node major version matches .nvmrc" - fi - fi -fi - -echo "pnpm version: $(pnpm --version 2>&1 || echo 'Not installed')" - -# Token check (do not print token!) -if [ -n "$GITHUB_TOKEN" ]; then - echo "GitHub token: Present" -else - echo "GitHub token: Missing" -fi - -if [ -n "$JULES_API_KEY" ]; then - echo "Jules API key: Present" -else - echo "Jules API key: Missing" -fi - -if [ -n "$GEMINI_API_KEY" ]; then - echo "Gemini API key: Present" -else - echo "Gemini API key: Missing" -fi - -echo "=== Snapshot Complete ===" diff --git a/boomtick-pkg/cli/tdw_services/cli.py b/boomtick-pkg/cli/tdw_services/cli.py index 9933b7cc47..a839cc4a52 100644 --- a/boomtick-pkg/cli/tdw_services/cli.py +++ b/boomtick-pkg/cli/tdw_services/cli.py @@ -319,9 +319,7 @@ def resolve_conflicts(ctx, pr, allow_unrelated, strategy, push): @click.pass_context def verify_versions(ctx, diff_input): """Verify version changes in a diff for downgrades or hard blocks.""" - import subprocess - import tempfile - script_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'verify_versions.py') + orch = ctx.obj['ORCHESTRATOR'] if not diff_input: # If no input provided, try to get diff against main @@ -330,31 +328,20 @@ def verify_versions(ctx, diff_input): except Exception as e: err(ctx, f"Failed to get git diff: {e}") - # Use a temporary file to avoid E2BIG/ARG_MAX issues with large diffs - with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp: - tmp.write(diff_input) - tmp_path = tmp.name + findings = orch.verify_versions(diff_input) - cmd = [sys.executable, script_path, tmp_path] - try: - proc = subprocess.run(cmd, capture_output=True, text=True) - os.unlink(tmp_path) - if proc.stdout: - try: - findings = json.loads(proc.stdout) - if findings: - status = "error" if any(f['severity'] == 'error' for f in findings) else "success" - out(ctx, f"Found {len(findings)} version issues.", data={"status": status, "findings": findings}) - if status == "error": - sys.exit(1) - else: - out(ctx, "✅ No version issues detected.", data={"status": "success", "findings": []}) - except json.JSONDecodeError: - err(ctx, f"Invalid validator output: {proc.stdout}") - else: - err(ctx, f"Validator failed: {proc.stderr}") - except Exception as e: - err(ctx, f"Error running validator: {e}") + if findings: + status = "error" if any(f['severity'] == 'error' for f in findings) else "success" + if not ctx.obj['JSON']: + for f in findings: + icon = "❌" if f['severity'] == "error" else "⚠️" + click.echo(f"{icon} {f['message']} ({f['file']})") + + out(ctx, f"Found {len(findings)} version issues.", data={"status": status, "findings": findings}) + if status == "error": + sys.exit(1) + else: + out(ctx, "✅ No version issues detected.", data={"status": "success", "findings": []}) @gh.command() @click.option('--pr', type=int) @@ -539,19 +526,31 @@ def pre_submit(ctx): @click.pass_context def overlaps(ctx, limit, no_cache): """Identify and propose consolidation of PRs with high functional or structural overlap.""" - import subprocess - import sys - import os - - script_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'dev_tools', 'pr_overlap.py') - cmd = [sys.executable, script_path, '--limit', str(limit)] - if no_cache: - cmd.append('--no-cache') + orch = ctx.obj['ORCHESTRATOR'] + res = orch.analyze_pr_overlaps(limit=limit, use_cache=not no_cache) - try: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as e: - err(ctx, f"pr_overlap.py failed with exit code {e.returncode}") + if not ctx.obj['JSON']: + click.echo("--- EXACT OVERLAP GROUPS ---") + for group in res['exact_overlaps']: + pr_info = ", ".join([f"#{p['number']}" for p in group['prs']]) + click.echo(f"PRs {pr_info} overlap on {len(group['files'])} files:") + for pr in group['prs']: + click.echo(f" [{pr['number']}] {pr['title']}") + for f in group['files']: + click.echo(f" - {f}") + + click.echo("\n--- CONNECTED CLUSTERS ---") + for cluster in res['clusters']: + pr_info = ", ".join([f"#{p['number']}" for p in cluster['prs']]) + click.echo(f"Cluster PRs {pr_info}:") + for pr in cluster['prs']: + click.echo(f" [{pr['number']}] {pr['title']}") + click.echo(" All files touched by this cluster:") + for f in cluster['all_files']: + click.echo(f" - {f}") + click.echo("-" * 40) + + out(ctx, "PR overlap analysis complete.", data=res) # ========================================== # UX COMMAND GROUP diff --git a/boomtick-pkg/cli/tdw_services/orchestrator.py b/boomtick-pkg/cli/tdw_services/orchestrator.py index af82fe610f..bd4133e5fa 100644 --- a/boomtick-pkg/cli/tdw_services/orchestrator.py +++ b/boomtick-pkg/cli/tdw_services/orchestrator.py @@ -13,7 +13,9 @@ from tdw_services.services.github import GitHubClient from tdw_services.services.ai_service import AIClient from tdw_services.services.jules import JulesClient +from tdw_services.services.version_service import VersionService from tdw_services.utils import log_error, get_or_create_log_dir, CLIError + from tdw_services.handlers.command_handler import CommandHandler from utils import ( get_github_token, @@ -51,6 +53,7 @@ def __init__(self) -> None: self._github: Optional[GitHubClient] = None self._ai: Optional[AIClient] = None self._jules: Optional[JulesClient] = None + self._version: Optional[VersionService] = None @property def github(self) -> GitHubClient: @@ -70,6 +73,12 @@ def jules(self) -> JulesClient: self._jules = JulesClient() return self._jules + @property + def version_service(self) -> VersionService: + if self._version is None: + self._version = VersionService() + return self._version + def _hash_content(self, content: str) -> str: return hashlib.md5(content.encode('utf-8')).hexdigest() @@ -252,6 +261,11 @@ def resolve_baseline(self, file_path: Optional[str], env_var: str, fallback_valu if val is not None and str(val).strip() != "": return int(val) return fallback_value + def verify_versions(self, diff_text: str) -> List[Dict[str, Any]]: + """Verifies version changes in a diff for downgrades or hard blocks.""" + changes = self.version_service.parse_diff(diff_text) + return self.version_service.verify_changes(changes) + def get_audit_results(self, content: Optional[str] = None, targets: Optional[List[str]] = None) -> Dict[str, Any]: cmd = ["node", "scripts/detect-antipatterns.mjs", "--json"] if targets: @@ -651,7 +665,14 @@ def runtime_check(self) -> Dict[str, str]: log_error(f"pnpm version mismatch\nExpected: {expected_pnpm}\nActual: {actual_pnpm}") raise CLIError(f"Run: corepack enable && corepack prepare pnpm@{expected_pnpm} --activate") - return {"node": actual_node, "pnpm": actual_pnpm} + # Consolidated token check from snapshot.sh + tokens = { + "GITHUB_TOKEN": bool(os.environ.get("GITHUB_TOKEN")), + "JULES_API_KEY": bool(os.environ.get("JULES_API_KEY")), + "GEMINI_API_KEY": bool(os.environ.get("GEMINI_API_KEY")) + } + + return {"node": actual_node, "pnpm": actual_pnpm, "tokens": tokens} def verify_ci_metrics(self, **kwargs): @@ -719,42 +740,131 @@ def run_step(name: str, cmd: List[str]) -> None: except Exception: pass return results - def repair_local(self, logs_path: Optional[str] = None, stdin: bool = False, worktree: bool = False) -> Dict[str, Any]: + def repair_ci(self, file_path: str, errors: List[str]) -> bool: + """Uses AI to attempt to repair a file based on CI error messages.""" + if not os.path.exists(file_path): + return False + + with open(file_path, 'r', encoding='utf-8') as f: + content = f.read() + + error_block = "\n".join([f"- {e}" for e in errors]) + prompt = f"""Repair the following file to resolve these CI errors: + +ERRORS: +{error_block} + +FILE CONTENT: +{content} + +Provide the complete repaired file content. Output ONLY the code without markdown markers.""" + + try: + repaired = self.ai.generate(prompt) + repaired = self.ai.clean_llm_output(repaired) + with open(file_path, 'w', encoding='utf-8') as f: + f.write(repaired) + return True + except Exception as e: + log_error(f"AI repair failed for {file_path}: {e}") + return False + + def repair_local(self, logs_path: Optional[str] = None, stdin: bool = False, worktree: bool = False, eslint_json_path: Optional[str] = None) -> Dict[str, Any]: logs_content = "" - if stdin: logs_content = sys.stdin.read() + if stdin: + logs_content = sys.stdin.read() elif logs_path: if os.path.exists(logs_path): - with open(logs_path, 'r') as f: logs_content = f.read() - else: raise CLIError(f"Log file not found: {logs_path}") + with open(logs_path, 'r', encoding='utf-8') as f: + logs_content = f.read() + else: + raise CLIError(f"Log file not found: {logs_path}") else: res_lint = run_command(["pnpm", "run", "lint:ox"], check=False) res_tsc = run_command(["pnpm", "run", "type-check"], check=False) logs_content = res_lint.stdout + res_lint.stderr + "\n" + res_tsc.stdout + res_tsc.stderr - if not logs_content.strip(): return {"status": "success", "message": "No errors found."} - import tempfile, shutil - original_cwd = os.getcwd(); repair_script = os.path.abspath(os.path.join(original_cwd, "dev-tools", "repair.py")) - worktree_path = None; branch_name = None + + findings = extract_failing_info(logs_content) + + if eslint_json_path and os.path.exists(eslint_json_path): + try: + with open(eslint_json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + for file_entry in data: + file_path = file_entry['filePath'] + if file_path.startswith(os.getcwd()): + file_path = os.path.relpath(file_path, os.getcwd()) + for msg in file_entry.get('messages', []): + if msg.get('severity') >= 2: # Error + findings.append({ + "file": file_path, + "line": msg.get('line'), + "message": f"{msg.get('message')} ({msg.get('ruleId')})", + "type": "eslint" + }) + except Exception as e: + log_error(f"Failed to parse ESLint JSON: {e}") + + if not findings: + return {"status": "success", "message": "No actionable errors found."} + + original_cwd = os.getcwd() + worktree_path = None + branch_name = None + try: if worktree: branch_name = f"repair/local-{datetime.now().strftime('%H%M%S')}" prefix = PROJECT_CONFIG.worktree_prefix - # Create temporary worktree within repo root to avoid Security Error worktree_path = os.path.join(original_cwd, f"{prefix}{datetime.now().strftime('%H%M%S')}") os.makedirs(worktree_path, exist_ok=True) run_command(["git", "worktree", "add", "-b", branch_name, worktree_path, "HEAD"]) os.chdir(worktree_path) if os.path.exists(os.path.join(original_cwd, "node_modules")): os.symlink(os.path.join(original_cwd, "node_modules"), os.path.join(worktree_path, "node_modules")) - # Create temporary log file within repo root logs/ - log_dir = get_or_create_log_dir("repair") - with tempfile.NamedTemporaryFile(mode='w', suffix=".log", delete=False, dir=log_dir) as tmp_log: - tmp_log.write(logs_content); tmp_log_path = tmp_log.name - cmd = [sys.executable, repair_script, tmp_log_path] - proc = run_command(cmd, check=False) - os.unlink(tmp_log_path) - if proc.returncode == 0: return {"status": "success", "message": "Repair completed.", "worktree": worktree_path, "branch": branch_name} - else: return {"status": "error", "message": f"Repair failed with code {proc.returncode}"} - finally: os.chdir(original_cwd) + + files_to_fix = {} + for f in findings: + if f["file"] not in files_to_fix: + files_to_fix[f["file"]] = [] + files_to_fix[f["file"]].append(f["message"]) + + results = {} + for file_path, errors in files_to_fix.items(): + if not os.path.exists(file_path): + continue + + # Agentic Loop + success = False + current_errors = errors + for attempt in range(3): # MAX_RETRIES = 3 + from tdw_services.utils import log_info + log_info(f"Attempt {attempt + 1} for {file_path}") + if self.repair_ci(file_path, current_errors): + # Verify + res = run_command(["pnpm", "run", "type-check"], check=False) + new_findings = extract_failing_info(res.stdout + res.stderr) + new_errors = [f["message"] for f in new_findings if f["file"] == file_path] + if not new_errors: + log_info(f"✅ Fixed all identified errors in {file_path}") + success = True + break + else: + log_info(f"⚠️ Still has {len(new_errors)} errors in {file_path}. Retrying...") + current_errors = new_errors + else: + break + results[file_path] = success + + return { + "status": "success", + "message": "Repair triage completed.", + "results": results, + "worktree": worktree_path, + "branch": branch_name + } + finally: + os.chdir(original_cwd) def handle_audit_gate(self) -> Dict[str, Any]: current_count = int(run_command(["node", "scripts/detect-antipatterns.mjs", "--count-only"]) or 0) @@ -1187,6 +1297,95 @@ def list_prs(self, state: str = "open", limit: int = 100, include_drafts: bool = return {"prs": prs} + def analyze_pr_overlaps(self, limit: int = 50, use_cache: bool = True) -> Dict[str, Any]: + """Identify and propose consolidation of PRs with high functional or structural overlap.""" + cache_file = os.path.join(get_or_create_log_dir("cache"), "pr_cache.json") + + cache = {"prs": {}, "files": {}} + if use_cache and os.path.exists(cache_file): + try: + with open(cache_file, 'r', encoding='utf-8') as f: + cache = json.load(f) + # Convert file lists back to sets for efficient overlap calculation + for k in cache["files"]: + cache["files"][k] = set(cache["files"][k]) + except Exception: + pass + + repo_name = get_repo_name() + prs = self.github.list_pull_requests(state='open', limit=limit) + + for pr in prs: + num = str(pr['number']) + if num not in cache["prs"] or not use_cache: + cache["prs"][num] = pr['title'] + files = self.github.fetch_pr_files(pr['number']) + cache["files"][num] = {f.get('filename') for f in files if not f.get('filename', '').startswith("tests/visual.spec.ts-snapshots/")} + + try: + # Convert sets to lists for JSON serialization + serializable_cache = { + "prs": cache["prs"], + "files": {k: list(v) for k, v in cache["files"].items()} + } + with open(cache_file, 'w', encoding='utf-8') as f: + json.dump(serializable_cache, f, indent=2) + except Exception: + pass + + overlap_groups = defaultdict(list) + for pr_num, files in cache["files"].items(): + for file in files: + touching_prs = [p for p, fs in cache["files"].items() if file in fs] + if len(touching_prs) > 1: + overlap_groups[frozenset(touching_prs)].append(file) + + exact_overlaps = [] + for pr_set, files in sorted(overlap_groups.items(), key=lambda x: len(x[1]), reverse=True): + pr_list = sorted(list(pr_set), key=int) + exact_overlaps.append({ + "prs": [{"number": p, "title": cache["prs"].get(p, "Unknown PR")} for p in pr_list], + "files": sorted(files) + }) + + graph = defaultdict(set) + all_prs = list(cache["files"].keys()) + for i, pr1 in enumerate(all_prs): + for pr2 in all_prs[i+1:]: + if cache["files"][pr1] & cache["files"][pr2]: + graph[pr1].add(pr2) + graph[pr2].add(pr1) + + clusters = [] + visited = set() + for pr in all_prs: + if pr not in visited and pr in graph: + component = {pr} + stack = [pr] + while stack: + curr = stack.pop() + for neighbor in graph[curr]: + if neighbor not in visited: + visited.add(neighbor) + component.add(neighbor) + stack.append(neighbor) + visited.add(pr) + + comp_list = sorted(list(component), key=int) + involved_files = set() + for p in component: + involved_files |= cache["files"][p] + + clusters.append({ + "prs": [{"number": p, "title": cache["prs"].get(p, "Unknown PR")} for p in comp_list], + "all_files": sorted(list(involved_files)) + }) + + return { + "exact_overlaps": exact_overlaps, + "clusters": clusters + } + def aggregate_prs(self, target_branch: str, pr_numbers: List[int]) -> Dict[str, Any]: """ Aggregates multiple PRs into a single target branch and creates a consolidated PR. diff --git a/boomtick-pkg/cli/tdw_services/services/version_service.py b/boomtick-pkg/cli/tdw_services/services/version_service.py new file mode 100644 index 0000000000..fa726c3fff --- /dev/null +++ b/boomtick-pkg/cli/tdw_services/services/version_service.py @@ -0,0 +1,276 @@ +import os +import re +import json +import requests +from packaging import version +from typing import Dict, Optional, List + +class VersionService: + # Registry Cache + _NPM_CACHE = {} + _GITHUB_CACHE = {} + + def fetch_latest_npm(self, package_name: str) -> Optional[str]: + if package_name in self._NPM_CACHE: + return self._NPM_CACHE[package_name] + try: + url = f"https://registry.npmjs.org/{package_name}/latest" + res = requests.get(url, timeout=5) + res.raise_for_status() + ver = res.json().get("version") + if ver: + self._NPM_CACHE[package_name] = ver + return ver + except Exception as e: + from tdw_services.utils import log_warn + log_warn(f"Failed to fetch latest NPM version for {package_name}: {e}") + return None + + def fetch_latest_node(self) -> Optional[str]: + if "node" in self._NPM_CACHE: + return self._NPM_CACHE["node"] + try: + url = "https://nodejs.org/dist/index.json" + res = requests.get(url, timeout=5) + res.raise_for_status() + data = res.json() + if data and isinstance(data, list) and len(data) > 0: + # Latest is typically the first one in the list + ver = data[0].get("version", "").lstrip('v') + if ver: + self._NPM_CACHE["node"] = ver + return ver + except Exception as e: + from tdw_services.utils import log_warn + log_warn(f"Failed to fetch latest Node.js version: {e}") + return None + + def fetch_latest_gh_action(self, action_path: str) -> Optional[str]: + if action_path in self._GITHUB_CACHE: + return self._GITHUB_CACHE[action_path] + try: + url = f"https://api.github.com/repos/{action_path}/releases/latest" + headers = {"Accept": "application/vnd.github+json"} + # Try to use token if available + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + + res = requests.get(url, headers=headers, timeout=5) + res.raise_for_status() + tag = res.json().get("tag_name") + if tag: + self._GITHUB_CACHE[action_path] = tag + return tag + except Exception as e: + from tdw_services.utils import log_warn + log_warn(f"Failed to fetch latest GitHub Action version for {action_path}: {e}") + return None + + def compare_versions(self, v1: str, v2: str) -> int: + """Returns 1 if v1 > v2, -1 if v1 < v2, 0 if v1 == v2.""" + if v1 == v2: return 0 + try: + v1_clean = v1.lstrip('v') + v2_clean = v2.lstrip('v') + + if '.x' in v1_clean: v1_clean = v1_clean.replace('.x', '.0') + if '.x' in v2_clean: v2_clean = v2_clean.replace('.x', '.0') + + pv1 = version.parse(v1_clean) + pv2 = version.parse(v2_clean) + if pv1 > pv2: return 1 + if pv1 < pv2: return -1 + return 0 + except Exception: + if v1 > v2: return 1 + if v1 < v2: return -1 + return 0 + + def get_stack_versions(self, fetch_latest: bool = False) -> Dict[str, str]: + """Extracts core versions (Node, pnpm, GHA) from the repository.""" + versions = { + "node": "24.16.0", + "pnpm": "10.28.2", + "actions/checkout": "v4", + "actions/setup-node": "v4", + "actions/upload-artifact": "v4", + } + + try: + if os.path.exists(".node-version"): + with open(".node-version", "r") as f: + v = f.read().strip().lstrip('v') + if v: versions["node"] = v + elif os.path.exists(".nvmrc"): + with open(".nvmrc", "r") as f: + v = f.read().strip().lstrip('v') + if v: versions["node"] = v + + if os.path.exists("package.json"): + with open("package.json", "r") as f: + pkg = json.load(f) + if "packageManager" in pkg: + versions["pnpm"] = pkg["packageManager"].replace("pnpm@", "") + elif "engines" in pkg and "pnpm" in pkg["engines"]: + versions["pnpm"] = pkg["engines"]["pnpm"] + + if "engines" in pkg and "node" in pkg["engines"] and not os.path.exists(".node-version"): + versions["node"] = pkg["engines"]["node"] + + workflow_dir = ".github/workflows" + if os.path.exists(workflow_dir): + for filename in os.listdir(workflow_dir): + if not (filename.endswith(".yml") or filename.endswith(".yaml")): + continue + try: + with open(os.path.join(workflow_dir, filename), "r") as f: + content = f.read() + matches = re.findall(r"uses:\s+([\w\-/]+)@([\w\.]+)", content) + for action, v_str in matches: + if not action.startswith("actions/"): continue + current_v = versions.get(action) + if not current_v or self.compare_versions(v_str, current_v) > 0: + versions[action] = v_str + except Exception: pass + + if fetch_latest: + latest_node = self.fetch_latest_node() + if latest_node: versions["latest_node"] = latest_node + + latest_pnpm = self.fetch_latest_npm("pnpm") + if latest_pnpm: versions["latest_pnpm"] = latest_pnpm + + for action in ["actions/checkout", "actions/setup-node"]: + latest_a = self.fetch_latest_gh_action(action) + if latest_a: versions[f"latest_{action}"] = latest_a + + except Exception: + pass + + return versions + + def parse_diff(self, diff_text: str) -> List[Dict]: + """Parses a git diff to find version changes.""" + changes = [] + + ACTION_PATTERN = re.compile(r"uses:\s+([\w\-/]+)@([\w\.]+)") + PKG_JSON_VERSION_PATTERN = re.compile(r'"(node|pnpm|[\w\-\./@]+)":\s*"([\d\.\^x~<>=\| v]+)"') + PM_PATTERN = re.compile(r'"packageManager":\s*"pnpm@([\d\.]+)"') + + SENSITIVE_FILES = [".nvmrc", ".node-version", "package.json"] + SENSITIVE_DIRS = [".github/workflows/"] + + hunks = re.split(r"^(?=--- )", diff_text, flags=re.MULTILINE) + for hunk in hunks: + if not hunk.strip(): continue + + lines = hunk.splitlines() + current_file = None + for line in lines: + if line.startswith("--- a/"): + current_file = line[6:] + break + elif line.startswith("+++ b/"): + current_file = line[6:] + break + + if not current_file: + continue + + is_sensitive = (current_file in SENSITIVE_FILES or + any(current_file.startswith(sd) for sd in SENSITIVE_DIRS)) + if not is_sensitive: + continue + + removals = {} # name -> version + additions = {} # name -> version + + for line in lines: + if line.startswith("--- ") or line.startswith("+++ ") or line.startswith("@@ "): + continue + + if line.startswith("-"): + content = line[1:].strip() + m = ACTION_PATTERN.search(content) + if m: removals[m.group(1)] = m.group(2) + m = PKG_JSON_VERSION_PATTERN.search(content) + if m: removals[m.group(1)] = m.group(2) + m = PM_PATTERN.search(content) + if m: removals["pnpm"] = m.group(1) + if current_file in [".nvmrc", ".node-version"]: + removals["node"] = content.replace("v", "") + + elif line.startswith("+"): + content = line[1:].strip() + m = ACTION_PATTERN.search(content) + if m: additions[m.group(1)] = m.group(2) + m = PKG_JSON_VERSION_PATTERN.search(content) + if m: additions[m.group(1)] = m.group(2) + m = PM_PATTERN.search(content) + if m: additions["pnpm"] = m.group(1) + if current_file in [".nvmrc", ".node-version"]: + additions["node"] = content.replace("v", "") + + for name, new_v in additions.items(): + old_v = removals.get(name, "unknown") + type_val = "action" if "/" in name and "pnpm" not in name else "dependency" + if name == "node" or current_file in [".nvmrc", ".node-version"]: type_val = "runtime" + + changes.append({ + "file": current_file, + "type": type_val, + "name": name, + "old": old_v, + "new": new_v + }) + + return changes + + def verify_changes(self, changes: List[Dict]) -> List[Dict]: + """Verifies changes against HEAD and registries.""" + findings = [] + stack = self.get_stack_versions() + + for c in changes: + head_v = stack.get(c["name"]) + if not head_v and c["name"] == "node": head_v = stack.get("node") + if not head_v and c["name"] == "pnpm": head_v = stack.get("pnpm") + + if head_v: + if self.compare_versions(c["new"], head_v) < 0: + findings.append({ + "severity": "error", + "file": c["file"], + "message": f"Version downgrade detected for {c['name']}: {head_v} -> {c['new']}", + "type": "downgrade" + }) + + latest = None + if c["name"] == "node": + latest = self.fetch_latest_node() + elif c["type"] == "action": + latest = self.fetch_latest_gh_action(c["name"]) + elif c["name"] in ["pnpm"] or c["type"] == "dependency": + latest = self.fetch_latest_npm(c["name"]) + + if latest: + if self.compare_versions(c["new"], latest) < 0: + findings.append({ + "severity": "warn", + "file": c["file"], + "message": f"Proposed version for {c['name']} ({c['new']}) is outdated. Latest is {latest}.", + "type": "outdated" + }) + + if c["name"] == "node": + if head_v and self.compare_versions(c["new"], head_v) != 0: + if os.environ.get("ALLOW_NODE_VERSION_CHANGE") != "true": + findings.append({ + "severity": "error", + "file": c["file"], + "message": f"Hard block: Node.js version modification detected ({head_v} -> {c['new']}). Modification is forbidden unless ALLOW_NODE_VERSION_CHANGE=true.", + "type": "hard_block" + }) + + return findings diff --git a/boomtick-pkg/cli/tests/services/test_version_service.py b/boomtick-pkg/cli/tests/services/test_version_service.py new file mode 100644 index 0000000000..50732d1db4 --- /dev/null +++ b/boomtick-pkg/cli/tests/services/test_version_service.py @@ -0,0 +1,68 @@ +import pytest +from unittest.mock import MagicMock, patch +from tdw_services.services.version_service import VersionService + +@pytest.fixture +def service(): + # Clear caches before each test + VersionService._NPM_CACHE = {} + VersionService._GITHUB_CACHE = {} + return VersionService() + +def test_compare_versions(service): + assert service.compare_versions("1.0.0", "1.0.0") == 0 + assert service.compare_versions("1.0.1", "1.0.0") == 1 + assert service.compare_versions("0.9.9", "1.0.0") == -1 + assert service.compare_versions("v1.2.3", "1.2.2") == 1 + assert service.compare_versions("24.x", "24.16.0") == -1 # 24.0.0 < 24.16.0 + +@patch("requests.get") +def test_fetch_latest_npm(mock_get, service): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"version": "10.0.0"} + mock_get.return_value = mock_response + + assert service.fetch_latest_npm("pnpm") == "10.0.0" + assert "pnpm" in service._NPM_CACHE + + # Test caching + service.fetch_latest_npm("pnpm") + assert mock_get.call_count == 1 + +@patch("requests.get") +def test_fetch_latest_node(mock_get, service): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = [{"version": "v22.0.0"}] + mock_get.return_value = mock_response + + assert service.fetch_latest_node() == "22.0.0" + assert "node" in service._NPM_CACHE + +def test_parse_diff(service): + diff = """--- a/package.json ++++ b/package.json +@@ -10,1 +10,1 @@ +- "pnpm": "10.0.0" ++ "pnpm": "10.1.0" +""" + changes = service.parse_diff(diff) + assert len(changes) == 1 + assert changes[0]["name"] == "pnpm" + assert changes[0]["new"] == "10.1.0" + +def test_verify_changes_downgrade(service): + changes = [{ + "file": "package.json", + "name": "pnpm", + "new": "9.0.0", + "old": "10.0.0", + "type": "dependency" + }] + + with patch.object(service, "get_stack_versions", return_value={"pnpm": "10.0.0"}): + with patch.object(service, "fetch_latest_npm", return_value="10.0.0"): + findings = service.verify_changes(changes) + # Should have downgrade error, and outdated warning if new < latest + assert any(f["severity"] == "error" and "downgrade" in f["message"].lower() for f in findings) diff --git a/boomtick-pkg/cli/tests/test_config.py b/boomtick-pkg/cli/tests/test_config.py index eadef54c1e..f421e46d8b 100644 --- a/boomtick-pkg/cli/tests/test_config.py +++ b/boomtick-pkg/cli/tests/test_config.py @@ -10,7 +10,8 @@ def test_load_default_config(tmp_path): assert isinstance(config, ProjectConfig) assert config.base_branch == "origin/main" assert config.monolithic_pr_threshold == 3 - assert "src/layouts/" in config.core_dirs + # core_dirs now defaults to empty list in code, loaded from json at runtime + assert config.core_dirs == [] def test_load_custom_config(tmp_path): config_file = tmp_path / "project_config.json" diff --git a/boomtick-pkg/cli/verify.sh b/boomtick-pkg/cli/verify.sh deleted file mode 100755 index a58becaf1c..0000000000 --- a/boomtick-pkg/cli/verify.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# Ensure we are in the project root -cd "$(dirname "$0")/.." - -echo "=== DevTools Verification & Setup Utility ===" - -check_tool() { - if ! command -v "$1" &> /dev/null; then - return 1 - fi - return 0 -} - -echo "--- 1. Checking Environment Variables ---" -if [ -z "$GITHUB_TOKEN" ]; then - echo "⚠️ Warning: GITHUB_TOKEN is not set. GitHub Operations will fail." -else - echo "✅ GitHub token is present." -fi - -echo "--- 2. Checking Python Environment ---" -if check_tool python3; then - echo "✅ Python is available." -else - echo "❌ Error: python3 is required." - return 1 -fi - -echo "--- 3. Setting up Python Virtual Environment ---" -if [ ! -d ".venv" ]; then - if check_tool uv; then - echo "Using uv for high-speed setup..." - uv venv .venv - source .venv/bin/activate - uv pip install -e ./cli/ - elif check_tool python3; then - echo "Falling back to python3 -m venv..." - python3 -m venv .venv - source .venv/bin/activate - pip install -e ./cli/ - else - echo "❌ Error: Cannot set up Python environment." - return 1 - fi -else - source .venv/bin/activate - echo "✅ Python environment already set up." -fi - -echo "--- 4. Checking CLI Tooling ---" -# Just test if the CLI runs -if PYTHONPATH=./cli:./cli/dev_tools python3 ./cli/tdw_services/cli.py --help > /dev/null; then - echo "✅ tdw_services CLI is functional." -else - echo "❌ Error: tdw_services CLI failed to execute." - return 1 -fi - -echo "--- 5. Checking Node Environment ---" -if check_tool pnpm; then - echo "✅ pnpm is available." -else - echo "⚠️ Warning: pnpm is not installed. Node builds may fail." -fi - -echo "=== Verification Complete! ===" diff --git a/boomtick-pkg/cli/verify_versions.py b/boomtick-pkg/cli/verify_versions.py deleted file mode 100644 index 8fdf26b265..0000000000 --- a/boomtick-pkg/cli/verify_versions.py +++ /dev/null @@ -1,183 +0,0 @@ -import os -import sys -import re -import json -from typing import Dict, List, Optional, Tuple - -# Add dev-tools to path -sys.path.append(os.path.dirname(os.path.abspath(__file__))) -from utils import ( - get_stack_versions, - log_info, - log_error, - log_warn, - fetch_latest_npm, - fetch_latest_gh_action, - compare_versions -) - -def parse_diff(diff_text: str) -> List[Dict]: - """Parses a git diff to find version changes.""" - changes = [] - current_file = None - - # Regex patterns - ACTION_PATTERN = re.compile(r"uses:\s+([\w\-/]+)@([\w\.]+)") - PKG_JSON_VERSION_PATTERN = re.compile(r'"(node|pnpm|[\w\-\./@]+)":\s*"([\d\.\^x~<>=\| v]+)"') - PM_PATTERN = re.compile(r'"packageManager":\s*"pnpm@([\d\.]+)"') - - # Files we care about - SENSITIVE_FILES = [".nvmrc", ".node-version", "package.json"] - SENSITIVE_DIRS = [".github/workflows/"] - - hunks = re.split(r"^(?=--- )", diff_text, flags=re.MULTILINE) - for hunk in hunks: - if not hunk.strip(): continue - - lines = hunk.splitlines() - current_file = None - for line in lines: - if line.startswith("--- a/"): - current_file = line[6:] - break - elif line.startswith("+++ b/"): - current_file = line[6:] - break - - if not current_file: - continue - - is_sensitive = (current_file in SENSITIVE_FILES or - any(current_file.startswith(sd) for sd in SENSITIVE_DIRS)) - if not is_sensitive: - continue - - removals = {} # name -> version - additions = {} # name -> version - - for line in lines: - if line.startswith("--- ") or line.startswith("+++ ") or line.startswith("@@ "): - continue - - if line.startswith("-"): - content = line[1:].strip() - # Check Actions - m = ACTION_PATTERN.search(content) - if m: removals[m.group(1)] = m.group(2) - # Check Dependencies - m = PKG_JSON_VERSION_PATTERN.search(content) - if m: removals[m.group(1)] = m.group(2) - # Check pnpm - m = PM_PATTERN.search(content) - if m: removals["pnpm"] = m.group(1) - # Check node files - if current_file in [".nvmrc", ".node-version"]: - removals["node"] = content.replace("v", "") - - elif line.startswith("+"): - content = line[1:].strip() - m = ACTION_PATTERN.search(content) - if m: additions[m.group(1)] = m.group(2) - m = PKG_JSON_VERSION_PATTERN.search(content) - if m: additions[m.group(1)] = m.group(2) - m = PM_PATTERN.search(content) - if m: additions["pnpm"] = m.group(1) - if current_file in [".nvmrc", ".node-version"]: - additions["node"] = content.replace("v", "") - - # Correlate changes - for name, new_v in additions.items(): - old_v = removals.get(name, "unknown") - type_val = "action" if "/" in name and "pnpm" not in name else "dependency" - if name == "node" or current_file in [".nvmrc", ".node-version"]: type_val = "runtime" - - changes.append({ - "file": current_file, - "type": type_val, - "name": name, - "old": old_v, - "new": new_v - }) - - return changes - -def verify_changes(changes: List[Dict]) -> List[Dict]: - """Verifies changes against HEAD and registries.""" - findings = [] - stack = get_stack_versions() - - for c in changes: - # 1. Compare against HEAD (Downgrade detection) - head_v = stack.get(c["name"]) - if not head_v and c["name"] == "node": head_v = stack.get("node") - if not head_v and c["name"] == "pnpm": head_v = stack.get("pnpm") - - if head_v: - if compare_versions(c["new"], head_v) < 0: - findings.append({ - "severity": "error", - "file": c["file"], - "message": f"Version downgrade detected for {c['name']}: {head_v} -> {c['new']}", - "type": "downgrade" - }) - - # 2. Compare against Latest (Outdated detection - optional warning) - from utils import fetch_latest_node # Import node fetcher - latest = None - if c["name"] == "node": - latest = fetch_latest_node() - elif c["type"] == "action": - latest = fetch_latest_gh_action(c["name"]) - elif c["name"] in ["pnpm"] or c["type"] == "dependency": - latest = fetch_latest_npm(c["name"]) - - if latest: - if compare_versions(c["new"], latest) < 0: - findings.append({ - "severity": "warn", - "file": c["file"], - "message": f"Proposed version for {c['name']} ({c['new']}) is outdated. Latest is {latest}.", - "type": "outdated" - }) - - # 3. Node.js Hard Block - if c["name"] == "node": - # Only trigger hard block if the version is ACTUALLY changing from HEAD - if head_v and compare_versions(c["new"], head_v) != 0: - if os.environ.get("ALLOW_NODE_VERSION_CHANGE") != "true": - findings.append({ - "severity": "error", - "file": c["file"], - "message": f"Hard block: Node.js version modification detected ({head_v} -> {c['new']}). Modification is forbidden unless ALLOW_NODE_VERSION_CHANGE=true.", - "type": "hard_block" - }) - - return findings - -def main(): - if len(sys.argv) < 2: - print("Usage: python3 verify_versions.py ") - sys.exit(1) - - input_val = sys.argv[1] - if os.path.exists(input_val): - with open(input_val, "r") as f: - diff_text = f.read() - else: - diff_text = input_val - - changes = parse_diff(diff_text) - findings = verify_changes(changes) - - if findings: - print(json.dumps(findings, indent=2)) - # Exit with error code if any 'error' severity exists - if any(f["severity"] == "error" for f in findings): - sys.exit(1) - else: - print(json.dumps([], indent=2)) - - sys.exit(0) - -if __name__ == "__main__": - main() diff --git a/boomtick-pkg/cli/version_utils.py b/boomtick-pkg/cli/version_utils.py deleted file mode 100644 index 3356ff2177..0000000000 --- a/boomtick-pkg/cli/version_utils.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -import re -import json -import requests -from packaging import version -from typing import Dict, Optional - -# Registry Cache -_NPM_CACHE = {} -_GITHUB_CACHE = {} - -def fetch_latest_npm(package_name: str) -> Optional[str]: - if package_name in _NPM_CACHE: - return _NPM_CACHE[package_name] - try: - url = f"https://registry.npmjs.org/{package_name}/latest" - res = requests.get(url, timeout=5) - if res.status_code == 200: - ver = res.json().get("version") - _NPM_CACHE[package_name] = ver - return ver - except Exception: - pass - return None - -def fetch_latest_node() -> Optional[str]: - if "node" in _NPM_CACHE: - return _NPM_CACHE["node"] - try: - url = "https://nodejs.org/dist/index.json" - res = requests.get(url, timeout=5) - if res.status_code == 200: - # Latest is the first one - ver = res.json()[0].get("version").lstrip('v') - _NPM_CACHE["node"] = ver - return ver - except Exception: - pass - return None - -def fetch_latest_gh_action(action_path: str) -> Optional[str]: - if action_path in _GITHUB_CACHE: - return _GITHUB_CACHE[action_path] - try: - url = f"https://api.github.com/repos/{action_path}/releases/latest" - headers = {} - # Try to use token if available - token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") - if token: - headers["Authorization"] = f"token {token}" - - res = requests.get(url, headers=headers, timeout=5) - if res.status_code == 200: - tag = res.json().get("tag_name") - _GITHUB_CACHE[action_path] = tag - return tag - except Exception: - pass - return None - -def compare_versions(v1: str, v2: str) -> int: - """Returns 1 if v1 > v2, -1 if v1 < v2, 0 if v1 == v2.""" - if v1 == v2: return 0 - try: - v1_clean = v1.lstrip('v') - v2_clean = v2.lstrip('v') - - if '.x' in v1_clean: v1_clean = v1_clean.replace('.x', '.0') - if '.x' in v2_clean: v2_clean = v2_clean.replace('.x', '.0') - - pv1 = version.parse(v1_clean) - pv2 = version.parse(v2_clean) - if pv1 > pv2: return 1 - if pv1 < pv2: return -1 - return 0 - except Exception: - if v1 > v2: return 1 - if v1 < v2: return -1 - return 0 - -def get_stack_versions(fetch_latest: bool = False) -> Dict[str, str]: - """Extracts core versions (Node, pnpm, GHA) from the repository.""" - versions = { - "node": "24.16.0", - "pnpm": "10.28.2", - "actions/checkout": "v4", - "actions/setup-node": "v4", - "actions/upload-artifact": "v4", - } - - try: - if os.path.exists(".node-version"): - with open(".node-version", "r") as f: - v = f.read().strip().lstrip('v') - if v: versions["node"] = v - elif os.path.exists(".nvmrc"): - with open(".nvmrc", "r") as f: - v = f.read().strip().lstrip('v') - if v: versions["node"] = v - - if os.path.exists("package.json"): - with open("package.json", "r") as f: - pkg = json.load(f) - if "packageManager" in pkg: - versions["pnpm"] = pkg["packageManager"].replace("pnpm@", "") - elif "engines" in pkg and "pnpm" in pkg["engines"]: - versions["pnpm"] = pkg["engines"]["pnpm"] - - if "engines" in pkg and "node" in pkg["engines"] and not os.path.exists(".node-version"): - versions["node"] = pkg["engines"]["node"] - - workflow_dir = ".github/workflows" - if os.path.exists(workflow_dir): - for filename in os.listdir(workflow_dir): - if not (filename.endswith(".yml") or filename.endswith(".yaml")): - continue - try: - with open(os.path.join(workflow_dir, filename), "r") as f: - content = f.read() - matches = re.findall(r"uses:\s+([\w\-/]+)@([\w\.]+)", content) - for action, v_str in matches: - if not action.startswith("actions/"): continue - current_v = versions.get(action) - if not current_v or compare_versions(v_str, current_v) > 0: - versions[action] = v_str - except Exception: pass - - if fetch_latest: - latest_node = fetch_latest_node() - if latest_node: versions["latest_node"] = latest_node - - latest_pnpm = fetch_latest_npm("pnpm") - if latest_pnpm: versions["latest_pnpm"] = latest_pnpm - - for action in ["actions/checkout", "actions/setup-node"]: - latest_a = fetch_latest_gh_action(action) - if latest_a: versions[f"latest_{action}"] = latest_a - - except Exception: - pass - - return versions diff --git a/boomtick-pkg/mcp/src/config.ts b/boomtick-pkg/mcp/src/config.ts index 5882231f31..41d0436aad 100644 --- a/boomtick-pkg/mcp/src/config.ts +++ b/boomtick-pkg/mcp/src/config.ts @@ -11,10 +11,16 @@ dotenv.config({ export const config = { githubToken: process.env.GITHUB_TOKEN, - githubOwner: process.env.GITHUB_OWNER || "arii", - githubRepo: process.env.GITHUB_REPO || "tech-dancer", + githubOwner: process.env.GITHUB_OWNER, + githubRepo: process.env.GITHUB_REPO, repoPath: process.env.BOOMTICK_REPO_PATH || path.resolve(__dirname, "../../../../"), defaultBaseBranch: process.env.DEFAULT_BASE_BRANCH || "main", viteBasePath: process.env.VITE_BASE_PATH || "/tech-dancer/", ghPath: process.env.GH_PATH || "gh" }; + +if (process.env.CI === "true") { + if (!config.githubToken) throw new Error("GITHUB_TOKEN is required in CI"); + if (!config.githubOwner) throw new Error("GITHUB_OWNER is required in CI"); + if (!config.githubRepo) throw new Error("GITHUB_REPO is required in CI"); +}