From 0e4a9a23b648b7e3f5d421733e30b29c443ec6d3 Mon Sep 17 00:00:00 2001 From: Jayapal Reddy Myaka Date: Thu, 14 May 2026 00:03:50 -0500 Subject: [PATCH 1/9] ci: pin setup-uv to v8.1.0 (#18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: stop pre-commit hook from touching .release-please-manifest.json release-please owns the manifest; overwriting it on every commit breaks release detection — it sees the bumped version and finds nothing new to release. * fix: allow release-please branches to merge to main release-please creates PRs from 'release-please--branches--main' directly to main — these must pass the enforce check. * fix: use RELEASE_TOKEN in release-please so release:published event fires GITHUB_TOKEN-created releases suppress same-repo webhook events, which blocked pypi-publish from triggering. RELEASE_TOKEN (PAT) fires the event normally. * chore: bump action versions to v9, update CLAUDE.md, refresh org profile * chore: upgrade trivy to 0.36.0, setup-uv to v6.1.0 (#17) * chore: remove redundant CI/release workflows from org .github repo ci.yml, package-validation.yml, release-dataenginex.yml, pypi-publish.yml are all superseded by the dex repo's own ci.yml and release.yml workflows. label-sync.yml duplicate of sync-labels.yml. WORKFLOWS_GUIDE.md updated. * fix: use v-prefixed tag for trivy-action (0.36.0 → v0.36.0) * fix(auto-pr): remove auto-merge call, just create the PR gh pr merge --auto requires "Allow auto-merge" to be enabled in each repo's settings. Drop it — the PR is created and merged manually when checks pass. * feat(workflows): add auto-pr-to-main reusable workflow For app repos (dex-studio, careerdex) that use ephemeral preview environments — feature branches open PRs directly to main instead of to a dev branch. * ci: pin setup-uv to v8.1.0 setup-uv dropped floating major/minor tags — @v7 no longer resolves. --- .github/workflows/auto-pr-dev-to-main.yml | 21 +- .github/workflows/auto-pr-to-main.yml | 62 +++++ .github/workflows/enforce-dev-to-main.yml | 9 +- .github/workflows/release-please.yml | 12 +- .github/workflows/security.yml | 10 +- .vscode/mcp.json | 14 - CLAUDE.md | 112 +++----- profile/README.md | 58 ++--- scripts/pre-commit-version-bump.sh | 19 +- wiki-content/Home.md | 2 +- workflows/WORKFLOWS_GUIDE.md | 18 +- workflows/ci.yml | 245 ------------------ workflows/label-sync.yml | 23 -- workflows/package-validation.yml | 143 ---------- workflows/pypi-publish.yml | 302 ---------------------- workflows/release-dataenginex.yml | 228 ---------------- workflows/security.yml | 10 +- 17 files changed, 154 insertions(+), 1134 deletions(-) create mode 100644 .github/workflows/auto-pr-to-main.yml delete mode 100644 workflows/ci.yml delete mode 100644 workflows/label-sync.yml delete mode 100644 workflows/package-validation.yml delete mode 100644 workflows/pypi-publish.yml delete mode 100644 workflows/release-dataenginex.yml diff --git a/.github/workflows/auto-pr-dev-to-main.yml b/.github/workflows/auto-pr-dev-to-main.yml index 543a6a2..ae08d8c 100644 --- a/.github/workflows/auto-pr-dev-to-main.yml +++ b/.github/workflows/auto-pr-dev-to-main.yml @@ -1,14 +1,3 @@ -# Reusable workflow — auto-creates a PR from dev → main and enables auto-merge. -# Triggered on every push to dev (i.e., after a feature PR is merged to dev). -# Uses merge commit (not squash) to preserve individual conventional commits for -# release-please to read when determining the next version bump. -# -# Prerequisites (one-time per repo, GitHub UI): -# 1. Settings → General → Allow auto-merge ✓ -# 2. Settings → Branches → branch protection on 'main': -# - Require status checks: CI + enforce-dev-to-main -# - Require branches to be up to date before merging - name: Auto PR — dev → main (reusable) on: @@ -28,7 +17,6 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} run: | - # Skip if dev has no commits ahead of main AHEAD=$(gh api "repos/$GH_REPO/compare/main...dev" --jq '.ahead_by') if [ "$AHEAD" = "0" ]; then echo "dev is already in sync with main — nothing to do" @@ -36,7 +24,6 @@ jobs: fi echo "dev is $AHEAD commit(s) ahead of main" - # Check for existing open PR PR_NUMBER=$(gh pr list \ --base main \ --head dev \ @@ -45,14 +32,13 @@ jobs: --jq '.[0].number') if [ -n "$PR_NUMBER" ]; then - echo "PR #$PR_NUMBER already exists — ensuring auto-merge is enabled" + echo "PR #$PR_NUMBER already exists" else - BODY="Automated PR: dev -> main. Auto-merge enabled - merge-commits when all required checks pass (CI + enforce-dev-to-main)." gh pr create \ --base main \ --head dev \ --title "chore: sync dev -> main" \ - --body "$BODY" + --body "Automated PR: dev → main. Merge when all required checks pass." PR_NUMBER=$(gh pr list \ --base main \ @@ -62,6 +48,3 @@ jobs: --jq '.[0].number') echo "Created PR #$PR_NUMBER" fi - - gh pr merge "$PR_NUMBER" --auto --merge - echo "Auto-merge (merge commit) enabled on PR #$PR_NUMBER" diff --git a/.github/workflows/auto-pr-to-main.yml b/.github/workflows/auto-pr-to-main.yml new file mode 100644 index 0000000..f5689f0 --- /dev/null +++ b/.github/workflows/auto-pr-to-main.yml @@ -0,0 +1,62 @@ +name: Auto PR — feature → main (reusable) + +on: + workflow_call: + +jobs: + create-pr: + name: Create PR to main + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Create or update PR (feature/fix → main) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + HEAD: ${{ github.ref_name }} + run: | + # Derive conventional commit title from branch name + # feature/feat-add-pipeline → "feat: add pipeline" + # feature/add-pipeline → "feat: add pipeline" + # fix/fix-null-ptr → "fix: null ptr" + BRANCH_DESC=$(echo "$HEAD" | sed 's|^feature/||;s|^fix/||') + if echo "$BRANCH_DESC" | grep -qE '^feat-'; then + TITLE="feat: $(echo "$BRANCH_DESC" | sed 's|^feat-||;s|-| |g')" + elif echo "$BRANCH_DESC" | grep -qE '^fix-'; then + TITLE="fix: $(echo "$BRANCH_DESC" | sed 's|^fix-||;s|-| |g')" + elif echo "$HEAD" | grep -qE '^fix/'; then + TITLE="fix: $(echo "$BRANCH_DESC" | sed 's|-| |g')" + else + TITLE="feat: $(echo "$BRANCH_DESC" | sed 's|-| |g')" + fi + + echo "PR title: $TITLE" + + PR_NUMBER=$(gh pr list \ + --base main \ + --head "$HEAD" \ + --state open \ + --json number \ + --jq '.[0].number') + + if [ -n "$PR_NUMBER" ]; then + echo "PR #$PR_NUMBER already exists" + else + BODY="Automated PR: ${HEAD} → main. Review the preview environment before merging." + gh pr create \ + --base main \ + --head "$HEAD" \ + --title "$TITLE" \ + --body "$BODY" + + PR_NUMBER=$(gh pr list \ + --base main \ + --head "$HEAD" \ + --state open \ + --json number \ + --jq '.[0].number') + echo "Created PR #$PR_NUMBER" + fi diff --git a/.github/workflows/enforce-dev-to-main.yml b/.github/workflows/enforce-dev-to-main.yml index 1c4afb7..d5f4741 100644 --- a/.github/workflows/enforce-dev-to-main.yml +++ b/.github/workflows/enforce-dev-to-main.yml @@ -15,9 +15,10 @@ jobs: steps: - name: Check source branch run: | - if [ "${{ github.head_ref }}" != "dev" ]; then - echo "PRs to main must come from the 'dev' branch." - echo "Source branch: ${{ github.head_ref }}" + branch="${{ github.head_ref }}" + if [ "$branch" != "dev" ] && [[ "$branch" != release-please--* ]]; then + echo "PRs to main must come from 'dev' or a release-please branch." + echo "Source branch: $branch" exit 1 fi - echo "Source branch is 'dev' — OK" + echo "Source branch is '$branch' — OK" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 53718c4..5d99bb2 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -5,6 +5,12 @@ # 2. Creates/updates a Release PR that bumps pyproject.toml version + CHANGELOG.md # 3. When the Release PR is merged → creates git tag + GitHub Release # +# Uses RELEASE_TOKEN (PAT) instead of GITHUB_TOKEN so the release:published +# event fires and triggers downstream workflows (e.g. pypi-publish). +# GITHUB_TOKEN-created releases suppress same-repo webhook events. +# +# Required secret: RELEASE_TOKEN — PAT with contents:write + pull-requests:write +# # After a Release PR is opened/updated, uv.lock is regenerated and committed # to the PR branch so the lock file stays in sync with the version bump. # @@ -30,7 +36,7 @@ jobs: - uses: googleapis/release-please-action@v4 id: release-please with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.RELEASE_TOKEN }} config-file: release-please-config.json manifest-file: .release-please-manifest.json @@ -43,11 +49,11 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ fromJson(steps.release-please.outputs.pr).headBranchName }} - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.RELEASE_TOKEN }} - name: Set up uv if: ${{ steps.release-please.outputs.pr }} - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@v8.1.0 with: version: "latest" diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 5d7045f..1a4f8bf 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -22,7 +22,7 @@ jobs: security-events: write steps: - name: Notify Security Scan Start - uses: actions/github-script@v8 + uses: actions/github-script@v9 if: always() with: script: | @@ -49,7 +49,7 @@ jobs: - uses: actions/checkout@v6 - name: Run Trivy repo security scan - uses: aquasecurity/trivy-action@0.35.0 + uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: fs scan-ref: . @@ -66,7 +66,7 @@ jobs: sarif_file: trivy-repo-results.sarif - name: Enforce Trivy misconfig gate (HIGH/CRITICAL) - uses: aquasecurity/trivy-action@0.35.0 + uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: fs scan-ref: . @@ -86,7 +86,7 @@ jobs: steps: - name: Notify Security Success if: ${{ needs.trivy-security.result == 'success' }} - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | if (!process.env.SLACK_WEBHOOK) return; @@ -123,7 +123,7 @@ jobs: - name: Notify Security Issues if: ${{ needs.trivy-security.result == 'failure' }} - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | if (!process.env.SLACK_WEBHOOK) return; diff --git a/.vscode/mcp.json b/.vscode/mcp.json index a58dbd4..fb73a97 100644 --- a/.vscode/mcp.json +++ b/.vscode/mcp.json @@ -32,20 +32,6 @@ "args": [ "mcp-server-fetch" ] - }, - "cloudflare": { - "type": "stdio", - "command": "npx", - "args": [ - "-y", - "@cloudflare/mcp-server-cloudflare", - "run", - "${env:CLOUDFLARE_ACCOUNT_ID}" - ], - "env": { - "CLOUDFLARE_API_TOKEN": "${env:CLOUDFLARE_API_TOKEN}", - "CLOUDFLARE_ACCOUNT_ID": "${env:CLOUDFLARE_ACCOUNT_ID}" - } } } } \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 54b5001..279b201 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,108 +1,54 @@ # CLAUDE.md — DEX Workspace -Always Be pragmatic, straight forward and challenge my ideas and system design focus on creating a consistent, scalable, and accessible user experience while improving development efficiency. Always refer to up to date resources as of today. Question my assumptions, point out the blank/blind spots and highlight opportunity costs. No sugarcoating. No pandering. No bias. No both siding. No retro active reasoning. If there is something wrong or will not work let me know even if I don't ask it specifically. If it is an issue/bug/problem find the root problem and suggest a solution refering to latest day resources — don't skip, bypass, supress or don't fallback to a defense mode. +Be pragmatic, direct, and challenge assumptions. No sugarcoating. Point out blank spots and opportunity costs. If something is wrong or won't work, say so. Find root causes, never workarounds. -Workspace-wide rules for all repos: **DEX · dex-studio · infradex** - -Repo-specific context is in each repo's own `CLAUDE.md`. - -______________________________________________________________________ +Workspace-wide rules: **dex · dex-studio · infradex**. Repo-specific context in each repo's `CLAUDE.md`. ## Repo Map | Repo | Package | Port | Purpose | |------|---------|------|---------| -| `dataenginex` (dex) | `dataenginex` | 17000 | Core framework — config, registry, CLI, API, ML, AI | -| `dex-studio` | `dex-studio` | 7860 | Web UI — single pane of glass (NiceGUI) | +| `dex` | `dataenginex` | 17000 | Core framework — config, registry, CLI, API, ML, AI (routing, runtime, memory, observability, workflows) | +| `dex-studio` | `dex-studio` | 7860 | Web UI — NiceGUI single pane of glass | | `infradex` | — | — | K3s / Helm / Terraform infrastructure | -> **Note:** datadex, agentdex, and careerdex have been consolidated into the `dataenginex` monorepo (data, agents, and templates modules). - -______________________________________________________________________ - -## Git & Branch Conventions +## Git -- **Branch naming:** `feature/` or `fix/` — never commit directly to `dev` or `main` -- **Flow:** feature branch → PR to `dev` → PR `dev` to `main` -- **dataenginex first:** When changes span repos, commit dataenginex before dex-studio/infradex — it is upstream -- **Author:** `git commit --author="jaymyaka "` on every commit -- **No Co-Authored-By** trailer in commit messages +- **Branches:** `feature/` or `fix/` — never commit to `dev` or `main` directly +- **Flow:** feature → PR to `dev` → PR to `main` +- **Order:** commit `dex` before `dex-studio`/`infradex` (dex is upstream) +- **Author:** `git commit --author="jaymyaka "` — no Co-Authored-By ## Conventional Commits -Commit messages drive release-please version bumps: - -| Type | Bump | Use for | -|------|------|---------| -| `feat:` | minor | New backward-compatible feature | -| `feat!:` / `BREAKING CHANGE:` | major | Breaking API change | -| `fix:` | patch | Bug fix | -| `perf:` | patch | Performance improvement | -| `chore:`, `refactor:`, `test:`, `ci:`, `docs:` | none | No release | - -______________________________________________________________________ - -## Release Flow (automated via release-please) - -``` -conventional commits → push to main - → release-please creates Release PR (bumps pyproject.toml + CHANGELOG + uv.lock) - → merge PR - → release-please creates tag v{version} + GitHub Release - → pypi-publish.yml publishes to PyPI (dex only) - → release-dex.yml attaches SBOM (dex only) -``` +| Prefix | Bump | Use for | +|--------|------|---------| +| `feat:` | minor | New feature | +| `feat!:` / `BREAKING CHANGE:` | major | Breaking change | +| `fix:` / `perf:` | patch | Bug fix / perf | +| `chore:` `refactor:` `test:` `ci:` `docs:` | none | No release | -**Version is managed automatically.** Never manually run version bump commands. -Pre-commit hook auto-bumps patch if you commit without changing the version. - -______________________________________________________________________ +Release-please automates versioning. Never bump manually. Pre-commit hook handles patch bump on commit. ## Coding Standards -See `.github/instructions/` for per-domain checklists applied during `/review`. - -**Universal rules (all repos, all files):** - `from __future__ import annotations` — first import in every source file -- Type hints on all public functions — `mypy --strict` must pass -- No `print()` — use `structlog` (standardized across all repos, no loguru) -- No hardcoded secrets — use env vars or Vault +- Type hints everywhere — `mypy --strict` must pass +- No `print()` — use `structlog` +- No hardcoded secrets — env vars or Vault - Tests required for all new code — 80%+ coverage on new paths -- DRY enforced — no copy-paste across repos; extract to shared base - -______________________________________________________________________ - -## Shared Tooling - -| Tool | Config source | -|------|--------------| -| poethepoet tasks | Each repo owns `poe_tasks.toml` — standard task names: `lint`, `test`, `check-all`, `dev`, `version`, `clean` | -| Git hooks | `.github/scripts/pre-commit-version-bump.sh` (install: `uv run poe install-hooks`) | -| Workflow sync | `.github/scripts/sync-workflows.sh` (dry-run) / `--apply` | -| Reusable workflows | `.github/.github/workflows/` (security, claude, enforce-dev-to-main, release-please) | - -______________________________________________________________________ - -## Commands (Claude slash commands) +- DRY — no copy-paste across repos -| Command | Purpose | -|---------|---------| -| `/validate` | Full lint → typecheck → test → smoke test pipeline | -| `/review` | Code review against `.github/CHECKLISTS.md` + instruction files | -| `/pr` | Generate PR description for current branch | -| `/release` | Release checklist (release-please flow) | -| `/new-feature` | Scaffold new feature with tests | -| `/sync` | Sync task files, check CLAUDE.md freshness | -| `/techdebt` | Identify and prioritize tech debt | -| `/debug` | Structured debugging session | -| `/fix-lint` | Run lint fixes and typecheck | +## Tooling -______________________________________________________________________ +- **Tasks:** `poe lint` / `poe test` / `poe check-all` / `poe dev` — defined in `poe_tasks.toml` +- **Hooks:** `.github/scripts/pre-commit-version-bump.sh` (`uv run poe install-hooks`) +- **Workflows:** `.github/.github/workflows/` (security, release-please, enforce-dev-to-main) +- **Slash commands:** `/validate` `/review` `/pr` `/new-feature` `/fix-lint` `/debug` ## Production Rules -- **Root-cause fixes only** — no workarounds, no `# type: ignore` without explanation -- **No shortcuts** — DRY enforced, no backwards-compatibility shims for dead code -- **Ask before committing** — always show staged diff and wait for approval -- **Ask before pushing** — confirm before any `git push` -- **Verify external versions** — always WebSearch/WebFetch for latest versions; never assume from training knowledge +- Root-cause fixes only — no `# type: ignore` without explanation +- Ask before committing — show diff, wait for approval +- Ask before pushing +- Always WebSearch/WebFetch for latest package versions — never assume from training diff --git a/profile/README.md b/profile/README.md index eb5aa37..8af3f99 100644 --- a/profile/README.md +++ b/profile/README.md @@ -11,11 +11,11 @@ Unified Data + ML + AI framework — config-driven, self-hosted, production-read --- One `dex.yaml` defines your entire pipeline — from data ingestion through ML training to AI agents. -Opinionated defaults that work out of the box, swap any layer for industry tools via extras. +Opinionated defaults that work out of the box, swap any layer via extras. > **Self-hosted.** Your data never leaves your infrastructure. > **Config-driven.** One YAML file defines everything. -> **Production-grade.** DuckDB, FastAPI, structlog, Prometheus metrics out of the box. +> **Production-grade.** DuckDB, FastAPI, structlog, Prometheus, OpenTelemetry out of the box. --- @@ -30,29 +30,30 @@ graph LR AC[ai:\nLLM, retrieval, agents] end - subgraph Core ["dataenginex"] + subgraph Core ["dataenginex (pip install)"] direction TB - REG[Backend Registry\nBase* ABCs + swappable backends] - API[FastAPI\nauth · rate limit · metrics] - OB[Observability\nstructlog · Prometheus · OTEL] + REG[Plugin Registry\nConnectors · Transforms · Backends] + API[FastAPI\nJWT · RBAC · SCIM · rate limit · metrics] + OB[Observability\nstructlog · Prometheus · OTEL · Langfuse] end - subgraph Extras ["pip install dataenginex\[extra\]"] - DAG[dagster] - MLF[mlflow] - LG[langgraph] - QD[qdrant] + subgraph Extras ["pip install dataenginex[extra]"] + CL[cloud: S3 · GCS · BigQuery] + AU[auth: RS256/JWKS · SCIM v2 · LDAP] + OV[observability: Langfuse LLM tracing] end Config --> Core Extras -->|same interface| Core - subgraph Deploy - UI[DEX Studio\nweb UI] + subgraph Products + B2B[DEX Studio\nB2B platform UI] + B2C[CareerDEX\nB2C career AI] INF[InfraDEX\nK3s · Helm · Terraform] end - UI -->|HTTP| Core + B2B -->|HTTP :17000| Core + B2C -->|Python lib| Core INF -->|deploys| Core ``` @@ -62,9 +63,10 @@ graph LR | Repo | What it does | Status | | --- | --- | --- | -| [**DEX**](https://github.com/TheDataEngineX/DEX) | Core framework: config system, backend registry, CLI, API, ML, AI agents | [![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/) | -| [**dex-studio**](https://github.com/TheDataEngineX/dex-studio) | Web UI: single pane of glass for projects, pipelines, ML, agents (NiceGUI) | Alpha | -| [**infradex**](https://github.com/TheDataEngineX/infradex) | IaC + monitoring: Terraform, Helm, K3s, Prometheus, Grafana | Alpha | +| [**DEX**](https://github.com/TheDataEngineX/DEX) | Core framework: config, CLI, FastAPI, ML registry, LLM routing (LiteLLM/vLLM), AI agents, RBAC, SCIM v2 | [![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/) | +| [**dex-studio**](https://github.com/TheDataEngineX/dex-studio) | B2B web UI: pipelines, ML experiments, AI playground, SQL console (Reflex/Python→React) | Alpha | +| [**careerdex**](https://github.com/TheDataEngineX/careerdex) | B2C career AI: job matching, resume analysis, interview prep, application tracking (Reflex) | Alpha | +| [**infradex**](https://github.com/TheDataEngineX/infradex) | IaC: K3s, Helm charts, Terraform — Authentik, Langfuse, Qdrant, Prometheus, Grafana, ArgoCD | Alpha | --- @@ -72,26 +74,23 @@ graph LR ```bash pip install dataenginex -dex init my-project --template full-stack dex validate dex.yaml -dex serve # → http://localhost:17000 +dex serve --config dex.yaml # → http://localhost:17000 ``` ```bash # From source git clone https://github.com/TheDataEngineX/DEX && cd DEX -uv sync && uv run poe dev # → http://localhost:17000 +uv sync && uv run poe dev # → http://localhost:17000 ``` **Optional extras:** ```bash -pip install dataenginex[dagster] # Dagster orchestration -pip install dataenginex[mlflow] # MLflow tracking -pip install dataenginex[agents] # LangGraph agent runtime -pip install dataenginex[vectors] # Qdrant + LanceDB -pip install dataenginex[embeddings] # sentence-transformers + ONNX -pip install dataenginex[all] # Everything +pip install "dataenginex[cloud]" # S3 · GCS · BigQuery connectors +pip install "dataenginex[auth]" # RS256/JWKS · SCIM v2 · LDAP sync +pip install "dataenginex[observability]" # Langfuse LLM tracing +pip install 'litellm>=1.83.3' --no-deps # 100+ LLM providers (separate install) ``` --- @@ -104,7 +103,8 @@ pip install dataenginex[all] # Everything | **Install** | `pip install dataenginex` | 10+ packages to wire together | | **Backends** | Swap via extras, same interface | Rewrite integration code | | **Self-hosted** | Works on laptop, VPS, or K8s | Cloud lock-in or complex setup | -| **Observability** | structlog + Prometheus + OTEL built-in | Manual instrumentation | +| **Observability** | structlog + Prometheus + OTEL + Langfuse | Manual instrumentation | +| **Enterprise** | RBAC, SCIM v2, LDAP, OIDC ready | Build it yourself | --- @@ -112,12 +112,12 @@ pip install dataenginex[all] # Everything | | | | --- | --- | -| **Documentation** | [docs.dataenginex.org](https://docs.dataenginex.org) | +| **Documentation** | [docs.thedataenginex.org](https://docs.thedataenginex.org) | | **Discussions** | [github.com/orgs/TheDataEngineX/discussions](https://github.com/orgs/TheDataEngineX/discussions) | | **Bug reports** | Open an issue in the relevant repo | | **Contributing** | [CONTRIBUTING.md](https://github.com/TheDataEngineX/.github/blob/main/CONTRIBUTING.md) | | **Security** | [SECURITY.md](https://github.com/TheDataEngineX/.github/blob/main/SECURITY.md) | -| **Website** | [dataenginex.org](https://dataenginex.org) | +| **Website** | [thedataenginex.org](https://thedataenginex.org) | --- diff --git a/scripts/pre-commit-version-bump.sh b/scripts/pre-commit-version-bump.sh index 8e38ad8..fa78651 100644 --- a/scripts/pre-commit-version-bump.sh +++ b/scripts/pre-commit-version-bump.sh @@ -10,14 +10,14 @@ # Behavior: # - Reads version from HEAD:pyproject.toml (last committed) # - Reads version from index (:pyproject.toml) (about to be committed) -# - If same → bumps patch in pyproject.toml + .release-please-manifest.json -# and re-stages both files +# - If same → bumps patch in pyproject.toml and re-stages it # - If different → user manually changed it; do nothing set -euo pipefail PYPROJECT="pyproject.toml" -MANIFEST=".release-please-manifest.json" +# NOTE: .release-please-manifest.json is intentionally NOT touched here — +# release-please owns that file; modifying it breaks its release detection. # Nothing to do on initial commit (no HEAD yet) if ! git rev-parse --verify HEAD > /dev/null 2>&1; then @@ -49,18 +49,5 @@ fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" NEW_VERSION=$(bash "$SCRIPT_DIR/bump-version.sh" patch "$PYPROJECT" | tail -n1) -# Sync .release-please-manifest.json if it exists -if [[ -f "$MANIFEST" ]]; then - python3 - "$MANIFEST" "$HEAD_VERSION" "$NEW_VERSION" <<'EOF' -import sys, json, re - -manifest_path, old_ver, new_ver = sys.argv[1], sys.argv[2], sys.argv[3] -content = open(manifest_path).read() -updated = content.replace(f'"{old_ver}"', f'"{new_ver}"') -open(manifest_path, 'w').write(updated) -EOF - git add "$MANIFEST" -fi - git add "$PYPROJECT" echo "auto-bumped version: $HEAD_VERSION → $NEW_VERSION" diff --git a/wiki-content/Home.md b/wiki-content/Home.md index c832221..767590e 100644 --- a/wiki-content/Home.md +++ b/wiki-content/Home.md @@ -104,7 +104,7 @@ Part of the [DataEngineX](https://github.com/TheDataEngineX) ecosystem: | [dex-studio](https://github.com/TheDataEngineX/dex-studio) | Desktop UI | 7860 | | [infradex](https://github.com/TheDataEngineX/infradex) | Infrastructure | — | -> **Note:** datadex, agentdex, and careerdex have been consolidated into [DEX](https://github.com/TheDataEngineX/DEX). Those repos are archived. +> **Note:** datadex and agentdex have been consolidated into [DEX](https://github.com/TheDataEngineX/DEX). careerdex is a standalone B2C product at [github.com/TheDataEngineX/careerdex](https://github.com/TheDataEngineX/careerdex). Full observability stack (Prometheus + Grafana + Jaeger): see [infradex](https://github.com/TheDataEngineX/infradex) `docker-compose.monitoring.yml`. diff --git a/workflows/WORKFLOWS_GUIDE.md b/workflows/WORKFLOWS_GUIDE.md index aaf6fac..fadbc9f 100644 --- a/workflows/WORKFLOWS_GUIDE.md +++ b/workflows/WORKFLOWS_GUIDE.md @@ -88,7 +88,7 @@ graph TB **What it does**: 1. Detects version bump in `pyproject.toml` -2. Creates git tag: `dataenginex-vX.Y.Z` +2. Creates git tag: `vX.Y.Z` 3. Creates GitHub release → **automatically triggers `pypi-publish.yml`** --- @@ -117,16 +117,6 @@ graph TB --- -### `label-sync.yml` - Label Taxonomy Sync -**Triggers**: Push to `main` when `.github/labels.yml` changes, manual dispatch - -**Jobs**: -- **sync-labels**: Synchronizes repository labels from `.github/labels.yml` - -**Purpose**: Keeps issue/PR labels consistent with maintainer taxonomy. - ---- - ### `project-automation.yml` - Project Intake Automation **Triggers**: Issue/PR opened/reopened events, manual dispatch @@ -152,7 +142,7 @@ gh run list --workflow ci.yml gh run view --log # Trigger manual PyPI publish -gh workflow run pypi-publish.yml -f tag=dataenginex-v0.6.0 +gh workflow run pypi-publish.yml -f tag=v0.6.0 ``` --- @@ -180,7 +170,7 @@ graph TD FixCode --> Push[Push changes] Type -->|PyPI Publish Failure| CheckRelease["Check release-dataenginex logs"] - CheckRelease --> CheckTag[Verify tag format dataenginex-vX.Y.Z] + CheckRelease --> CheckTag[Verify tag format vX.Y.Z] CheckTag --> CheckPyPI[Check pypi-publish.yml logs] Push --> End[✓ Resolved] @@ -198,7 +188,7 @@ uv run poe test ``` ### PyPI Publish Not Triggering -- Verify release tag format: `dataenginex-vMAJOR.MINOR.PATCH` +- Verify release tag format: `vMAJOR.MINOR.PATCH` - Check that `release-dataenginex.yml` created the GitHub release - Confirm `src/dataenginex/` files changed since last tag diff --git a/workflows/ci.yml b/workflows/ci.yml deleted file mode 100644 index 88d4386..0000000 --- a/workflows/ci.yml +++ /dev/null @@ -1,245 +0,0 @@ -# DEX Continuous Integration -# Tests dataenginex (core package) as a pure library. -# - dataenginex: src/dataenginex/ → core framework -# See docs/CI_CD.md for workflow architecture. - -name: Continuous Integration - -on: - push: - branches: - - main - - dev - pull_request: - branches: - - main - - dev - -permissions: - contents: read - checks: write - -env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - -jobs: - # Primary lint + test job (required, runs on dev deps only) - # Covers: dataenginex core API, FastAPI endpoints, shared tests - lint-and-test: - name: Lint and Test - runs-on: ubuntu-latest - - steps: - - name: Notify CI Start - uses: actions/github-script@v8 - if: always() - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '🧪 Testing Pipeline Started', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*🧪 Testing Pipeline Started*\n*Linting & Running Tests*\n*Event:* ${process.env.EVENT}\n*Branch:* ${process.env.REF}\n*Commit:* ${process.env.SHA?.substring(0, 8)}\n*By:* ${process.env.ACTOR}` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - EVENT: ${{ github.event_name }} - REF: ${{ github.ref_name }} - SHA: ${{ github.sha }} - ACTOR: ${{ github.actor }} - - - name: Checkout code - uses: actions/checkout@v6 - - - name: Set up Python 3.13 - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - version: "latest" - - - name: Install dependencies (uv) - env: - UV_PROJECT_ENVIRONMENT: .venv - UV_PYTHON: "3.13" - run: | - uv lock - uv sync --group ml - - - name: Run lint suite - run: uv run poe lint - - - name: Run import-order check - run: uv run poe imports-check - - - name: Run type checking - run: uv run poe typecheck - - - name: Run pytest - run: uv run poe test-cov - - - name: Smoke-test examples - run: | - uv run python examples/01_hello_pipeline.py - uv run python examples/03_quality_gate.py - uv run python examples/04_ml_training.py - uv run python examples/05_rag_demo.py - uv run python examples/06_llm_quickstart.py - - - name: Upload coverage to SonarCloud - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - run: echo "SonarCloud integration is not enabled in this repository; coverage.xml is generated by test-cov." - - - name: Notify CI Success - if: success() - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '✅ Tests Passed', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*✅ All Tests Passed*\n*Lint, Type Checks & Unit Tests*\n*Branch:* ${process.env.REF}\n*By:* ${process.env.ACTOR}` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - WORKFLOW: ${{ github.workflow }} - REF: ${{ github.ref_name }} - ACTOR: ${{ github.actor }} - - - name: Notify CI Failure - if: failure() - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '❌ Tests Failed', - blocks: [ - { - type: 'section', - text: { - type: 'mrkdwn', - text: `*❌ Tests Failed*\n⚠️ Lint, type checks or unit tests failed` - } - }, - { - type: 'section', - fields: [ - { type: 'mrkdwn', text: `*Branch:*\n${process.env.REF}` }, - { type: 'mrkdwn', text: `*Commit:*\n${process.env.SHA?.substring(0, 8)}` }, - { type: 'mrkdwn', text: `*Triggered by:*\n${process.env.ACTOR}` }, - { type: 'mrkdwn', text: `*Event:*\n${process.env.EVENT}` } - ] - }, - { - type: 'section', - text: { - type: 'mrkdwn', - text: `*<${process.env.URL}|View Test Results>*` - } - } - ] - }) - }); - } catch (e) { console.log('Slack notification failed:', e.message); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - WORKFLOW: ${{ github.workflow }} - REF: ${{ github.ref_name }} - SHA: ${{ github.sha }} - ACTOR: ${{ github.actor }} - EVENT: ${{ github.event_name }} - URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} - - # Optional integration test job (manual or for full coverage) - # Runs with --group data (PySpark, Airflow) + --group notebook (pandas, numpy) - integration-test: - name: Integration Test (Data + Notebook Groups) - runs-on: ubuntu-latest - if: | - github.event_name == 'workflow_dispatch' || - contains(github.event.pull_request.labels.*.name, 'full-test') - - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Set up Python 3.13 - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - version: "latest" - - - name: Install dependencies (with data + notebook groups) - env: - UV_PROJECT_ENVIRONMENT: .venv - UV_PYTHON: "3.13" - run: | - uv lock - uv sync --group dev --group data --group notebook --group ml - - - name: Run tests with PySpark + Airflow + Pandas - run: uv run poe test-cov - - - name: Notify Integration Test Complete - if: always() - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - const result = process.env.RESULT; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: `${result === 'success' ? '⚙️' : '⚠️'} Integration Tests Complete`, - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*${result === 'success' ? '⚙️' : '⚠️'} Integration Tests (PySpark + Airflow + Pandas)*\n*Status:* ${result === 'success' ? '✅ Passed' : '❌ Failed'}\n*Branch:* ${process.env.REF}` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - REF: ${{ github.ref_name }} - RESULT: ${{ job.status }} diff --git a/workflows/label-sync.yml b/workflows/label-sync.yml deleted file mode 100644 index 0b49d6d..0000000 --- a/workflows/label-sync.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Label Sync - -on: - push: - branches: [main] - paths: [labels.yml] - workflow_dispatch: - -jobs: - sync: - runs-on: ubuntu-latest - permissions: - issues: write - steps: - - uses: actions/checkout@v6 - - - name: Sync labels across org repos - uses: EndBug/label-sync@v2 - with: - config-file: labels.yml - delete-other-labels: false - env: - GITHUB_TOKEN: ${{ secrets.ORG_ADMIN_TOKEN }} diff --git a/workflows/package-validation.yml b/workflows/package-validation.yml deleted file mode 100644 index c2255af..0000000 --- a/workflows/package-validation.yml +++ /dev/null @@ -1,143 +0,0 @@ -# DataEngineX Package Validation -# Validates that pyproject.toml builds a valid wheel + passes twine checks. - -name: Package Validation - -on: - pull_request: - branches: - - main - - dev - paths: - - 'src/dataenginex/**' - - 'pyproject.toml' - - '.github/workflows/package-validation.yml' - push: - branches: - - main - - dev - -permissions: - contents: read - -env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - -jobs: - validate-dataenginex-package: - name: Validate dataenginex Package - runs-on: ubuntu-latest - - steps: - - name: Notify Package Validation Start - uses: actions/github-script@v8 - if: always() - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '📦 Package Validation Started', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*📦 Package Validation Started*\n*Building & Validating Package Distributions*\n*Event:* ${process.env.EVENT}\n*Branch:* ${process.env.REF}\n*By:* ${process.env.ACTOR}` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - EVENT: ${{ github.event_name }} - REF: ${{ github.ref_name }} - ACTOR: ${{ github.actor }} - - - name: Checkout code - uses: actions/checkout@v6 - - - name: Set up Python 3.13 - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - version: "latest" - - - name: Build and validate dataenginex - run: | - uvx --from build pyproject-build - uvx twine check dist/* - - - name: Notify Package Validation Success - if: success() - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '✅ Package Valid', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*✅ Package Validation Passed*\n*Distribution builds successfully*\n*Branch:* ${process.env.REF}\n*By:* ${process.env.ACTOR}` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - REF: ${{ github.ref_name }} - ACTOR: ${{ github.actor }} - - - name: Notify Package Validation Failure - if: failure() - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '❌ Package Validation Failed', - blocks: [ - { - type: 'section', - text: { - type: 'mrkdwn', - text: `*❌ Package Validation Failed*\n⚠️ Distribution build or twine checks failed` - } - }, - { - type: 'section', - fields: [ - { type: 'mrkdwn', text: `*Branch:*\n${process.env.REF}` }, - { type: 'mrkdwn', text: `*Triggered by:*\n${process.env.ACTOR}` }, - { type: 'mrkdwn', text: `*Event:*\n${process.env.EVENT}` }, - { type: 'mrkdwn', text: `*Run:*\n<${process.env.URL}|View Details>` } - ] - } - ] - }) - }); - } catch (e) { console.log('Slack notification failed:', e.message); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - REF: ${{ github.ref_name }} - ACTOR: ${{ github.actor }} - EVENT: ${{ github.event_name }} - URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/workflows/pypi-publish.yml b/workflows/pypi-publish.yml deleted file mode 100644 index 1edeefd..0000000 --- a/workflows/pypi-publish.yml +++ /dev/null @@ -1,302 +0,0 @@ -# DataEngineX PyPI Publishing -# Triggered by GitHub release from release-dataenginex.yml (tag: dataenginex-vX.Y.Z). -# Detects if src/dataenginex/ actually changed since previous dataenginex tag, then publishes. - -name: PyPI Publish - -on: - release: - types: - - published - workflow_run: - workflows: - - Release DataEngineX - types: - - completed - branches: - - main - workflow_dispatch: - inputs: - tag: - description: "Tag to publish (e.g., dataenginex-v0.6.0)" - required: false - type: string - -permissions: - contents: read - -env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - -jobs: - detect-dataenginex-changes: - name: Detect DataEngineX Changes - runs-on: ubuntu-latest - if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} - outputs: - dataenginex_changed: ${{ steps.detect.outputs.dataenginex_changed }} - previous_tag: ${{ steps.detect.outputs.previous_tag }} - approved_tag: ${{ steps.detect.outputs.approved_tag }} - current_tag: ${{ steps.detect.outputs.current_tag }} - package_version: ${{ steps.detect.outputs.package_version }} - - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Detect changed paths for release tag - id: detect - env: - RELEASE_TAG: ${{ github.event.release.tag_name }} - DISPATCH_TAG: ${{ github.event.inputs.tag }} - EVENT_NAME: ${{ github.event_name }} - shell: bash - run: | - set -euo pipefail - - git fetch --tags --force - - PACKAGE_VERSION=$(grep -m 1 '^version = ' pyproject.toml | sed 's/version = "//;s/".*//') - echo "package_version=${PACKAGE_VERSION}" >> "$GITHUB_OUTPUT" - - CURRENT_TAG="${RELEASE_TAG:-${DISPATCH_TAG:-}}" - if [[ -z "${CURRENT_TAG}" ]]; then - CURRENT_TAG="$(git tag --list 'dataenginex-v*' --sort=-version:refname | head -n 1 || true)" - fi - - if [[ -z "${CURRENT_TAG}" ]]; then - echo "Unable to determine current DataEngineX release tag." - exit 1 - fi - - echo "current_tag=${CURRENT_TAG}" >> "$GITHUB_OUTPUT" - - STABLE_REGEX='^dataenginex-v[0-9]+\.[0-9]+\.[0-9]+$' - ANY_SEMVER_REGEX='^dataenginex-v[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$' - - if [[ "${CURRENT_TAG}" =~ $STABLE_REGEX ]]; then - echo "approved_tag=true" >> "$GITHUB_OUTPUT" - else - echo "approved_tag=false" >> "$GITHUB_OUTPUT" - fi - - if [[ ! "${CURRENT_TAG}" =~ $ANY_SEMVER_REGEX ]]; then - echo "Tag ${CURRENT_TAG} is not a valid DataEngineX semver tag." - exit 1 - fi - - TAG_VERSION="${CURRENT_TAG#dataenginex-v}" - if [[ "$EVENT_NAME" == "release" && "$TAG_VERSION" != "$PACKAGE_VERSION" ]]; then - echo "Release tag (${CURRENT_TAG}) does not match pyproject.toml version (${PACKAGE_VERSION})." - exit 1 - fi - - PREVIOUS_TAG="$(git tag --list 'dataenginex-v*' --sort=-version:refname | grep -Fxv "${CURRENT_TAG}" | head -n 1 || true)" - echo "previous_tag=${PREVIOUS_TAG}" >> "$GITHUB_OUTPUT" - - if [[ -z "${PREVIOUS_TAG}" ]]; then - echo "No previous DataEngineX tag found. Allowing first publish for ${CURRENT_TAG}." - echo "dataenginex_changed=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - CHANGED_FILES="$(git diff --name-only "${PREVIOUS_TAG}" "${CURRENT_TAG}")" - - if echo "${CHANGED_FILES}" | grep -Eq '^src/dataenginex/|^pyproject\.toml$'; then - echo "DataEngineX changes found between ${PREVIOUS_TAG} and ${CURRENT_TAG}." - echo "dataenginex_changed=true" >> "$GITHUB_OUTPUT" - else - echo "No DataEngineX changes found between ${PREVIOUS_TAG} and ${CURRENT_TAG}." - echo "dataenginex_changed=false" >> "$GITHUB_OUTPUT" - fi - - build-distributions: - name: Build Distributions - runs-on: ubuntu-latest - needs: detect-dataenginex-changes - if: needs.detect-dataenginex-changes.outputs.dataenginex_changed == 'true' - - steps: - - name: Notify Package Build Start - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '📤 Package Publishing Pipeline Started', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*📤 DataEngineX Publishing Started*\n*Tag:* ${process.env.TAG}\n*Building distributions for TestPyPI/PyPI*` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - TAG: ${{ needs.detect-dataenginex-changes.outputs.current_tag }} - - - name: Checkout code - uses: actions/checkout@v6 - - - name: Set up Python 3.13 - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Install uv - uses: astral-sh/setup-uv@v7 - with: - version: "latest" - - - name: Build package - run: | - uv tool install build - uvx --from build pyproject-build - - - name: Check distributions - run: | - uv tool install twine - uvx twine check dist/* - - - name: Upload distributions - uses: actions/upload-artifact@v7 - with: - name: python-distributions - path: dist/ - - publish-testpypi: - name: Publish to TestPyPI - runs-on: ubuntu-latest - needs: [detect-dataenginex-changes, build-distributions] - if: contains(fromJson('["release","workflow_run","workflow_dispatch"]'), github.event_name) && needs.detect-dataenginex-changes.outputs.dataenginex_changed == 'true' && vars.TESTPYPI_ENVIRONMENT != '' - environment: ${{ vars.TESTPYPI_ENVIRONMENT }} - permissions: - id-token: write - - steps: - - name: Download distributions - uses: actions/download-artifact@v8 - with: - name: python-distributions - path: dist/ - - - name: Publish package distributions to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository-url: https://test.pypi.org/legacy/ - - publish-pypi: - name: Promote to PyPI - runs-on: ubuntu-latest - needs: [detect-dataenginex-changes, publish-testpypi] - if: contains(fromJson('["release","workflow_run","workflow_dispatch"]'), github.event_name) && needs.detect-dataenginex-changes.outputs.dataenginex_changed == 'true' && needs.detect-dataenginex-changes.outputs.approved_tag == 'true' && vars.PYPI_ENVIRONMENT != '' - environment: ${{ vars.PYPI_ENVIRONMENT }} - permissions: - id-token: write - - steps: - - name: Download distributions - uses: actions/download-artifact@v8 - with: - name: python-distributions - path: dist/ - - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - - - name: Notify PyPI Publish Success - if: success() - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '🎉 Package Published to PyPI', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*🎉 DataEngineX Published to PyPI*\n*Version:* ${process.env.VERSION}\n*Tag:* ${process.env.TAG}\n*Release:* ` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - VERSION: ${{ needs.detect-dataenginex-changes.outputs.package_version }} - TAG: ${{ needs.detect-dataenginex-changes.outputs.current_tag }} - - skip-publish: - name: Skip PyPI Publish - runs-on: ubuntu-latest - needs: [detect-dataenginex-changes, publish-testpypi, publish-pypi] - if: always() && (!contains(fromJson('["release","workflow_run","workflow_dispatch"]'), github.event_name) || needs.detect-dataenginex-changes.outputs.dataenginex_changed == 'false' || needs.detect-dataenginex-changes.outputs.approved_tag == 'false') - - steps: - - name: Notify Publish Skipped - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - const eventName = process.env.EVENT_NAME; - const dataenginexChanged = process.env.DEX_CHANGED; - const approvedTag = process.env.APPROVED_TAG; - const currentTag = process.env.CURRENT_TAG; - const previousTag = process.env.PREVIOUS_TAG; - - let reason = ''; - if (eventName !== 'release' && eventName !== 'workflow_run' && eventName !== 'workflow_dispatch') { - reason = 'Build validation only (no release/workflow_run/manual publish event)'; - } else if (dataenginexChanged === 'false') { - reason = `No changes in src/dataenginex/ since ${previousTag}`; - } else if (approvedTag === 'false') { - reason = `Tag ${currentTag} is not a stable release (requires dataenginex-vMAJOR.MINOR.PATCH)`; - } - - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '⏭️ Package Publishing Skipped', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*⏭️ PyPI Publishing Skipped*\n${reason}\n*Tag:* ${currentTag}` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - EVENT_NAME: ${{ github.event_name }} - DEX_CHANGED: ${{ needs.detect-dataenginex-changes.outputs.dataenginex_changed }} - APPROVED_TAG: ${{ needs.detect-dataenginex-changes.outputs.approved_tag }} - CURRENT_TAG: ${{ needs.detect-dataenginex-changes.outputs.current_tag }} - PREVIOUS_TAG: ${{ needs.detect-dataenginex-changes.outputs.previous_tag }} - - - name: Skip publish - run: | - if [[ "${{ github.event_name }}" != "release" && "${{ github.event_name }}" != "workflow_run" && "${{ github.event_name }}" != "workflow_dispatch" ]]; then - echo "Build validation only: publish jobs run only for release:published, workflow_run, or workflow_dispatch events." - elif [[ "${{ needs.detect-dataenginex-changes.outputs.dataenginex_changed }}" == "false" ]]; then - echo "Skipping TestPyPI/PyPI publish: no changes in src/dataenginex/ since previous tag ${{ needs.detect-dataenginex-changes.outputs.previous_tag }}." - elif [[ "${{ needs.detect-dataenginex-changes.outputs.approved_tag }}" == "false" ]]; then - echo "Published to TestPyPI only: tag ${{ needs.detect-dataenginex-changes.outputs.current_tag }} is not an approved stable tag (requires dataenginex-vMAJOR.MINOR.PATCH)." - fi diff --git a/workflows/release-dataenginex.yml b/workflows/release-dataenginex.yml deleted file mode 100644 index 6abde4d..0000000 --- a/workflows/release-dataenginex.yml +++ /dev/null @@ -1,228 +0,0 @@ -# DataEngineX Release Automation -# Triggered when pyproject.toml version changes on main. -# Creates git tag + GitHub release, which triggers pypi-publish.yml. - -name: Release DataEngineX - -on: - push: - branches: - - main - paths: - - 'pyproject.toml' - -permissions: - contents: write - packages: read - -env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - -jobs: - release-dataenginex: - name: Release DataEngineX Package - runs-on: ubuntu-latest - if: github.event.repository.default_branch == github.ref_name - - steps: - - name: Notify Release Start - uses: actions/github-script@v8 - if: always() - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '💎 DataEngineX Core Package Release Started', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*💎 DataEngineX Release Process Started*\n*Core Package Library for PyPI*\n*Branch:* main` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract DataEngineX version - id: version - run: | - VERSION=$(grep -m 1 "^version = " pyproject.toml | sed 's/version = "//;s/".*//') - if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$ ]]; then - echo "Invalid DataEngineX version in pyproject.toml: $VERSION" - exit 1 - fi - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "tag=dataenginex-v$VERSION" >> "$GITHUB_OUTPUT" - echo "Extracted DataEngineX version: $VERSION" - - - name: Extract DataEngineX changelog notes - id: changelog - run: | - CHANGELOG="CHANGELOG.md" - if [[ -f "$CHANGELOG" ]]; then - NOTES=$(awk "/^## \\[${{ steps.version.outputs.version }}\\]/{found=1; next} /^## \\[/{if(found) exit} found" "$CHANGELOG") - if [[ -z "$NOTES" ]]; then - NOTES="DataEngineX v${{ steps.version.outputs.version }} — see CHANGELOG.md for details." - fi - else - NOTES="DataEngineX v${{ steps.version.outputs.version }} — see RELEASE_NOTES.md for details." - fi - { - echo "notes<> "$GITHUB_OUTPUT" - - - name: Check if tag already exists - id: check_tag - run: | - if git rev-parse "${{ steps.version.outputs.tag }}" >/dev/null 2>&1; then - echo "tag_exists=true" >> "$GITHUB_OUTPUT" - echo "Tag ${{ steps.version.outputs.tag }} already exists" - else - echo "tag_exists=false" >> "$GITHUB_OUTPUT" - echo "Tag ${{ steps.version.outputs.tag }} is new" - fi - - - name: Configure git - if: steps.check_tag.outputs.tag_exists == 'false' - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - - name: Create git tag - if: steps.check_tag.outputs.tag_exists == 'false' - run: | - git tag -a "${{ steps.version.outputs.tag }}" -m "Release DataEngineX ${{ steps.version.outputs.tag }}" - git push origin "${{ steps.version.outputs.tag }}" - - - name: Create GitHub Release - if: steps.check_tag.outputs.tag_exists == 'false' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release create "${{ steps.version.outputs.tag }}" \ - --target main \ - --title "DataEngineX ${{ steps.version.outputs.tag }}" \ - --notes "${{ steps.changelog.outputs.notes }}" \ - --generate-notes \ - --draft=false - - - name: Notify docs site of new release - if: steps.check_tag.outputs.tag_exists == 'false' - uses: peter-evans/repository-dispatch@v3 - with: - token: ${{ secrets.DOCS_DISPATCH_TOKEN }} - repository: TheDataEngineX/docs - event-type: release - client-payload: >- - {"repo": "${{ github.repository }}", "tag": "${{ steps.version.outputs.tag }}"} - - - name: Set up Python for SBOM generation - if: steps.check_tag.outputs.tag_exists == 'false' - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - - name: Generate CycloneDX SBOM - if: steps.check_tag.outputs.tag_exists == 'false' - run: | - pip install cyclonedx-bom - cyclonedx-py environment \ - --output sbom-dataenginex-${{ steps.version.outputs.version }}.json \ - --output-format json - echo "SBOM generated: sbom-dataenginex-${{ steps.version.outputs.version }}.json" - - - name: Upload SBOM to release - if: steps.check_tag.outputs.tag_exists == 'false' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh release upload "${{ steps.version.outputs.tag }}" \ - "sbom-dataenginex-${{ steps.version.outputs.version }}.json" \ - --clobber - - - name: Notify Release Success - if: success() && steps.check_tag.outputs.tag_exists == 'false' - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '✅ DataEngineX Core Package Released', - blocks: [{ - type: 'section', - text: { - type: 'mrkdwn', - text: `*✅ DataEngineX Release Created*\n*Core Package Library*\n*Version:* ${process.env.VERSION}\n*Tag:* ${process.env.TAG}\n*Release:* <${process.env.URL}|View Release>` - } - }] - }) - }); - } catch (e) { console.log('Slack notification failed'); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - VERSION: ${{ steps.version.outputs.version }} - TAG: ${{ steps.version.outputs.tag }} - URL: ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ steps.version.outputs.tag }} - - - name: Notify Release Failure - if: failure() - uses: actions/github-script@v8 - with: - script: | - if (!process.env.SLACK_WEBHOOK) return; - try { - await fetch(process.env.SLACK_WEBHOOK, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - text: '❌ DataEngineX Release Failed', - blocks: [ - { - type: 'section', - text: { - type: 'mrkdwn', - text: `*❌ DataEngineX Release Failed*` - } - }, - { - type: 'section', - fields: [ - { type: 'mrkdwn', text: `*Workflow:*\nDataEngineX Release` }, - { type: 'mrkdwn', text: `*Branch:*\nmain` }, - { type: 'mrkdwn', text: `*Commit:*\n\`${process.env.SHA?.substring(0, 8)}\`` }, - { type: 'mrkdwn', text: `*Triggered by:*\n${process.env.ACTOR}` }, - { type: 'mrkdwn', text: `*Run:*\n<${process.env.URL}|View Details>` } - ] - }, - { - type: 'context', - elements: [{ type: 'mrkdwn', text: `🏷️ DataEngineX release tag creation failed — check logs` }] - } - ] - }) - }); - } catch (e) { console.log('Slack notification failed:', e.message); } - env: - SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} - SHA: ${{ github.sha }} - ACTOR: ${{ github.actor }} - URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/workflows/security.yml b/workflows/security.yml index ee0db48..7c5ac3b 100644 --- a/workflows/security.yml +++ b/workflows/security.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Security Scan Start - uses: actions/github-script@v8 + uses: actions/github-script@v9 if: always() with: script: | @@ -49,7 +49,7 @@ jobs: - uses: actions/checkout@v6 - name: Run Trivy repo security scan - uses: aquasecurity/trivy-action@0.35.0 + uses: aquasecurity/trivy-action@0.36.0 with: scan-type: fs scan-ref: . @@ -66,7 +66,7 @@ jobs: sarif_file: trivy-repo-results.sarif - name: Enforce Trivy misconfig gate (HIGH/CRITICAL) - uses: aquasecurity/trivy-action@0.35.0 + uses: aquasecurity/trivy-action@0.36.0 with: scan-type: fs scan-ref: . @@ -101,7 +101,7 @@ jobs: steps: - name: Notify Security Success if: ${{ needs.trivy-security.result == 'success' && needs.codeql.result == 'success' }} - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | if (!process.env.SLACK_WEBHOOK) return; @@ -139,7 +139,7 @@ jobs: - name: Notify Security Issues if: ${{ needs.trivy-security.result == 'failure' || needs.codeql.result == 'failure' }} - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | if (!process.env.SLACK_WEBHOOK) return; From 775be2d0d0441b883ce7d628a198d77745ef7991 Mon Sep 17 00:00:00 2001 From: Jayapal Reddy Myaka Date: Wed, 20 May 2026 02:05:07 -0500 Subject: [PATCH 2/9] ci: update workflow templates to pinned action versions (#19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: stop pre-commit hook from touching .release-please-manifest.json release-please owns the manifest; overwriting it on every commit breaks release detection — it sees the bumped version and finds nothing new to release. * fix: allow release-please branches to merge to main release-please creates PRs from 'release-please--branches--main' directly to main — these must pass the enforce check. * fix: use RELEASE_TOKEN in release-please so release:published event fires GITHUB_TOKEN-created releases suppress same-repo webhook events, which blocked pypi-publish from triggering. RELEASE_TOKEN (PAT) fires the event normally. * chore: bump action versions to v9, update CLAUDE.md, refresh org profile * chore: upgrade trivy to 0.36.0, setup-uv to v6.1.0 (#17) * chore: remove redundant CI/release workflows from org .github repo ci.yml, package-validation.yml, release-dataenginex.yml, pypi-publish.yml are all superseded by the dex repo's own ci.yml and release.yml workflows. label-sync.yml duplicate of sync-labels.yml. WORKFLOWS_GUIDE.md updated. * fix: use v-prefixed tag for trivy-action (0.36.0 → v0.36.0) * fix(auto-pr): remove auto-merge call, just create the PR gh pr merge --auto requires "Allow auto-merge" to be enabled in each repo's settings. Drop it — the PR is created and merged manually when checks pass. * feat(workflows): add auto-pr-to-main reusable workflow For app repos (dex-studio, careerdex) that use ephemeral preview environments — feature branches open PRs directly to main instead of to a dev branch. * ci: pin setup-uv to v8.1.0 setup-uv dropped floating major/minor tags — @v7 no longer resolves. * ci: update workflow templates to pinned action versions - checkout@v4 → @v6, setup-uv@v5 → @v8.1.0, setup-python@v5 → @v6 - delete empty .vscode/tasks.json --- .vscode/tasks.json | 4 ---- workflow-templates/docker-build.yml | 2 +- workflow-templates/helm-release.yml | 4 ++-- workflow-templates/python-ci.yml | 12 ++++++------ workflow-templates/security-scan.yml | 10 +++++----- 5 files changed, 14 insertions(+), 18 deletions(-) delete mode 100644 .vscode/tasks.json diff --git a/.vscode/tasks.json b/.vscode/tasks.json deleted file mode 100644 index 513d537..0000000 --- a/.vscode/tasks.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [] -} diff --git a/workflow-templates/docker-build.yml b/workflow-templates/docker-build.yml index 23f17f0..0017b4d 100644 --- a/workflow-templates/docker-build.yml +++ b/workflow-templates/docker-build.yml @@ -27,7 +27,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: docker/login-action@v3 with: diff --git a/workflow-templates/helm-release.yml b/workflow-templates/helm-release.yml index 156a3a7..6b9b1a7 100644 --- a/workflow-templates/helm-release.yml +++ b/workflow-templates/helm-release.yml @@ -16,7 +16,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: azure/setup-helm@v4 - run: helm lint ${{ inputs.chart-path }} @@ -24,7 +24,7 @@ jobs: needs: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: azure/setup-helm@v4 - name: Package chart diff --git a/workflow-templates/python-ci.yml b/workflow-templates/python-ci.yml index 1751f74..f278f77 100644 --- a/workflow-templates/python-ci.yml +++ b/workflow-templates/python-ci.yml @@ -23,8 +23,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.1.0 with: version: "latest" - run: uv sync --group dev @@ -34,8 +34,8 @@ jobs: typecheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.1.0 with: version: "latest" - run: uv sync --group dev @@ -44,8 +44,8 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.1.0 with: version: "latest" - run: uv sync --group dev diff --git a/workflow-templates/security-scan.yml b/workflow-templates/security-scan.yml index a7c14a8..af4f24b 100644 --- a/workflow-templates/security-scan.yml +++ b/workflow-templates/security-scan.yml @@ -17,7 +17,7 @@ jobs: name: Trivy vulnerability scan runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Run Trivy (SARIF) uses: aquasecurity/trivy-action@0.28.0 @@ -41,7 +41,7 @@ jobs: matrix: language: [python] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -58,11 +58,11 @@ jobs: name: Dependency audit runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v5 + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.1.0 with: version: "latest" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6 with: python-version: "3.13" - run: uv sync From 8fa40cece0a94e0982b6f7fc3825279b8d2f0599 Mon Sep 17 00:00:00 2001 From: jaymyaka Date: Tue, 21 Jul 2026 12:37:47 -0500 Subject: [PATCH 3/9] Update docs, upgrade deps, fix actions --- .claude/commands/fix-lint.md | 2 +- .claude/commands/release.md | 4 +- .claude/commands/validate.md | 2 +- .claude/memory/long-term-memory.md | 2 +- .claude/memory/project-memory.md | 2 - CHECKLISTS.md | 2 +- CLAUDE.md | 6 +- CONTRIBUTING.md | 4 +- GUIDELINES.md | 133 -- ISSUE_TEMPLATE/bug_report.yml | 2 +- PULL_REQUEST_TEMPLATE.md | 2 +- SUPPORT.md | 2 +- agents/devops-engineer.agent.md | 13 +- agents/llm-architect.agent.md | 8 +- agents/mcp-developer.agent.md | 2 +- agents/mlops-engineer.agent.md | 6 +- agents/multi-agent-coordinator.agent.md | 10 +- agents/security-auditor.agent.md | 4 +- agents/sre-engineer.agent.md | 9 +- copilot-instructions.md | 11 +- dependabot.yml | 2 +- ...026-03-17-workspace-config-optimization.md | 1215 ----------------- ...17-workspace-config-optimization-design.md | 385 ------ instructions/data-pipelines.instructions.md | 2 +- instructions/fastapi.instructions.md | 2 +- instructions/ml.instructions.md | 2 +- instructions/workflows.instructions.md | 10 +- learning/GUIDE.md | 21 - learning/__init__.py | 1 - learning/async_dependency_injection.py | 22 - learning/async_http_client.py | 65 - learning/context_managers.py | 41 - learning/strict_decorators.py | 27 - learning/var_len_args.py | 7 - learning/yield_keyword.py | 20 - profile/README.md | 38 +- prompts/debug.prompt.md | 3 +- prompts/new-feature.prompt.md | 1 - prompts/pr-description.prompt.md | 55 +- prompts/write-tests.prompt.md | 2 +- wiki-content/Architecture.md | 135 -- wiki-content/Changelog.md | 50 - wiki-content/Development.md | 128 -- wiki-content/Home.md | 120 -- wiki-content/ML-Guide.md | 172 --- wiki-content/push-all-wikis.sh | 36 - workflow-templates/docker-build.yml | 2 +- workflow-templates/python-ci.yml | 2 +- workflow-templates/security-scan.yml | 10 +- workflows/WORKFLOWS_GUIDE.md | 261 +--- .../reusable-auto-pr-dev-to-main.yml | 0 .../reusable-auto-pr-to-dev.yml | 0 .../reusable-auto-pr-to-main.yml | 0 .../reusable-claude.yml | 0 .../reusable-enforce-dev-to-main.yml | 0 .../reusable-release-please.yml | 2 +- .../reusable-security.yml | 0 workflows/security.yml | 42 +- workflows/sync-labels.yml | 8 +- 59 files changed, 175 insertions(+), 2940 deletions(-) delete mode 100644 GUIDELINES.md delete mode 100644 docs/superpowers/plans/2026-03-17-workspace-config-optimization.md delete mode 100644 docs/superpowers/specs/2026-03-17-workspace-config-optimization-design.md delete mode 100644 learning/GUIDE.md delete mode 100644 learning/__init__.py delete mode 100644 learning/async_dependency_injection.py delete mode 100644 learning/async_http_client.py delete mode 100644 learning/context_managers.py delete mode 100644 learning/strict_decorators.py delete mode 100644 learning/var_len_args.py delete mode 100644 learning/yield_keyword.py delete mode 100644 wiki-content/Architecture.md delete mode 100644 wiki-content/Changelog.md delete mode 100644 wiki-content/Development.md delete mode 100644 wiki-content/Home.md delete mode 100644 wiki-content/ML-Guide.md delete mode 100755 wiki-content/push-all-wikis.sh rename .github/workflows/auto-pr-dev-to-main.yml => workflows/reusable-auto-pr-dev-to-main.yml (100%) rename .github/workflows/auto-pr-to-dev.yml => workflows/reusable-auto-pr-to-dev.yml (100%) rename .github/workflows/auto-pr-to-main.yml => workflows/reusable-auto-pr-to-main.yml (100%) rename .github/workflows/claude.yml => workflows/reusable-claude.yml (100%) rename .github/workflows/enforce-dev-to-main.yml => workflows/reusable-enforce-dev-to-main.yml (100%) rename .github/workflows/release-please.yml => workflows/reusable-release-please.yml (97%) rename .github/workflows/security.yml => workflows/reusable-security.yml (100%) diff --git a/.claude/commands/fix-lint.md b/.claude/commands/fix-lint.md index d31f760..3f708ac 100644 --- a/.claude/commands/fix-lint.md +++ b/.claude/commands/fix-lint.md @@ -5,6 +5,6 @@ Fix all lint errors in the project. 1. Run `uv run poe lint` again to check remaining issues 1. Manually fix any issues Ruff couldn't auto-fix 1. Run `uv run poe typecheck` to verify no type errors introduced -1. Run `uv run poe test` to verify fixes didn't break anything +1. Run `uv run poe check-all` to verify fixes didn't break anything Report what was fixed and what remains (if anything). diff --git a/.claude/commands/release.md b/.claude/commands/release.md index 933dad4..15eb97d 100644 --- a/.claude/commands/release.md +++ b/.claude/commands/release.md @@ -1,6 +1,6 @@ # Release -Release is fully automated via **release-please**. Claude's role is to verify readiness and monitor the pipeline — never to create tags, bump versions manually, or trigger workflows directly. +Release is intended to be automated via **release-please** (not yet configured). Claude's role is to verify readiness and monitor the pipeline — never to create tags, bump versions manually, or trigger workflows directly. ## How it works @@ -24,7 +24,7 @@ conventional commits merged to main 1. **Verify pipeline triggered** ```bash - gh run list --workflow=release-please.yml --limit 5 + gh run list --workflow=reusable-release-please.yml --limit 5 ``` 1. **Check for open Release PR** diff --git a/.claude/commands/validate.md b/.claude/commands/validate.md index 5f2987c..f8f32eb 100644 --- a/.claude/commands/validate.md +++ b/.claude/commands/validate.md @@ -25,7 +25,7 @@ Run the full validation pipeline for the current repo. Stop and report on first 1. **Tests** ```bash - uv run poe test + uv run poe check-all # fallback: uv run pytest tests/ -x --tb=short -q ``` diff --git a/.claude/memory/long-term-memory.md b/.claude/memory/long-term-memory.md index 450d2bf..3cc253c 100644 --- a/.claude/memory/long-term-memory.md +++ b/.claude/memory/long-term-memory.md @@ -96,5 +96,5 @@ _Nothing yet._ - **Source:** - **Finding:** uv 0.10.8 switches default binary/CPython mirror from python.org to Astral mirrors. Raw `curl | sh` uv installs in CI are unpinned. -- **Impact:** Replaced `curl -LsSf https://astral.sh/uv/install.sh | sh` with `astral-sh/setup-uv@v7` in `.github/workflows/ci.yml` (both jobs) and `.github/workflows/package-validation.yml`. +- **Impact:** Replaced `curl -LsSf https://astral.sh/uv/install.sh | sh` with `astral-sh/setup-uv@v7` in `.github/workflows/ci.yml` (both jobs) and `.github/workflows/package-validation.yml`. Note: these workflows were removed in the 2026-07-16 `.github` cleanup; the UV setup pattern now lives in individual repos (`dataenginex`, `dex-studio`). - **Status:** fixed — 2026-03-19 diff --git a/.claude/memory/project-memory.md b/.claude/memory/project-memory.md index ad8c1ed..2aa5204 100644 --- a/.claude/memory/project-memory.md +++ b/.claude/memory/project-memory.md @@ -20,8 +20,6 @@ _Nothing yet._ | dex-studio | — | — | | infradex | — | — | -> **Note:** datadex, agentdex, and careerdex consolidated into dataenginex monorepo. - ## Blockers _None._ diff --git a/CHECKLISTS.md b/CHECKLISTS.md index 7817b60..261294f 100644 --- a/CHECKLISTS.md +++ b/CHECKLISTS.md @@ -251,6 +251,6 @@ uv run python -c "import dataenginex; print('OK', dataenginex.__file__)" **Full validation pipeline order:** 1. `uv run poe lint` (or `uv run python -m ruff check src/ tests/`) 2. `uv run poe typecheck` (or `uv run python -m mypy src/dataenginex/ --strict`) -3. `uv run poe test` (or `uv run python -m pytest tests/ -x --tb=short -q`) +3. `uv run poe check-all` (or `uv run python -m pytest tests/ -x --tb=short -q`) 4. Start real server and curl all endpoints (above) 5. Run standalone module imports (above) diff --git a/CLAUDE.md b/CLAUDE.md index 279b201..883099c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ Workspace-wide rules: **dex · dex-studio · infradex**. Repo-specific context i | Repo | Package | Port | Purpose | |------|---------|------|---------| | `dex` | `dataenginex` | 17000 | Core framework — config, registry, CLI, API, ML, AI (routing, runtime, memory, observability, workflows) | -| `dex-studio` | `dex-studio` | 7860 | Web UI — NiceGUI single pane of glass | +| `dex-studio` | `dex-studio` | 7860 | Web UI — FastAPI + Jinja2 + HTMX single pane of glass | | `infradex` | — | — | K3s / Helm / Terraform infrastructure | ## Git @@ -41,9 +41,9 @@ Release-please automates versioning. Never bump manually. Pre-commit hook handle ## Tooling -- **Tasks:** `poe lint` / `poe test` / `poe check-all` / `poe dev` — defined in `poe_tasks.toml` +- **Tasks:** `poe lint` / `poe check-all` / `poe dev` — defined in `poe_tasks.toml` - **Hooks:** `.github/scripts/pre-commit-version-bump.sh` (`uv run poe install-hooks`) -- **Workflows:** `.github/.github/workflows/` (security, release-please, enforce-dev-to-main) +- **Workflows:** `.github/workflows/` (security, sync-labels, project-automation, stale-issues) - **Slash commands:** `/validate` `/review` `/pr` `/new-feature` `/fix-lint` `/debug` ## Production Rules diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7a6f78..e063e93 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,7 @@ All repos use the same toolchain: ```bash # Prerequisites -python >= 3.12 # 3.13+ recommended +python >= 3.13 uv # Package manager (https://docs.astral.sh/uv/) # Clone and setup @@ -41,7 +41,7 @@ uv sync # Quality checks uv run poe lint # Ruff lint uv run poe typecheck # mypy --strict -uv run poe test # pytest +uv run poe check-all # lint + typecheck + tests ``` ## Coding Standards diff --git a/GUIDELINES.md b/GUIDELINES.md deleted file mode 100644 index c158376..0000000 --- a/GUIDELINES.md +++ /dev/null @@ -1,133 +0,0 @@ -# .github Instructions - Code Review Standards - -Welcome! This directory contains code review standards for the DataEngineX project. - -## 📚 Core Guidance - -**[copilot-instructions.md](copilot-instructions.md)** — Universal Principles -- Applies to all code, all languages -- Security, testing, performance, clarity -- ~200 lines of core guidance -- **Start here** if unsure - -**[CHECKLISTS.md](CHECKLISTS.md)** — Quick Reference by Domain -- Python, APIs, Data Pipelines, Databases, Tests, Docs, Security -- Pre-commit checklist for each domain -- Links to examples in actual project code (better than detailed docs) -- ~10-minute read - -**[PULL_REQUEST_TEMPLATE.md](PULL_REQUEST_TEMPLATE.md)** — PR Format -- Use when creating pull requests - -**[MAINTAINER_PLAYBOOK.md](MAINTAINER_PLAYBOOK.md)** — Sustainability & Governance -- Keep/Change/Remove audit standard -- Cloud-neutral and cost-aware contribution expectations -- Issue triage, label strategy, and deprecation policy - ---- - -## 🚀 Quick Start - -### I'm writing code -1. Check [CHECKLISTS.md](CHECKLISTS.md) for your domain -2. Use checklist items before submitting PR -3. Refer to code examples in project for patterns - -### I'm reviewing code -1. Glance at [copilot-instructions.md](copilot-instructions.md) principles -2. Check domain-specific checklist in [CHECKLISTS.md](CHECKLISTS.md) -3. Look for red flags (security, performance, testing) -4. Provide specific, actionable feedback - -### I'm not sure about something -1. Search [CHECKLISTS.md](CHECKLISTS.md) for your domain -2. Look at similar code in [src/](../src/) or [tests/](../tests/) -3. Ask the team - ---- - -## 🎯 Principles (Everywhere) - -✅ **Security first** - Never hardcode secrets, validate inputs, use parameterized queries - -✅ **Clear code** - Single responsibility, obvious naming, "why" comments (not "what") - -✅ **Explicit errors** - Catch specific exceptions, log with context, don't swallow errors - -✅ **Always test** - New code needs tests, 80%+ coverage target, independent tests - -✅ **Know before you optimize** - Profile first, optimize second - -✅ **Type hints & validation** - On public functions, at boundaries - -✅ **Document publicly exposed code** - Docstrings, examples, clear parameters - ---- - -## 📋 By Domain - -| Domain | Read | Examples | -|--------|------|----------| -| **Python code** | [CHECKLISTS.md#python-code](CHECKLISTS.md#python-code) | [src/dataenginex/](../src/dataenginex/) | -| **REST APIs** | [CHECKLISTS.md#rest-api-endpoints-fastapi](CHECKLISTS.md#rest-api-endpoints-fastapi) | [src/dataenginex/api/](../src/dataenginex/api/) | -| **Data Pipelines** | [CHECKLISTS.md#data-pipelines](CHECKLISTS.md#data-pipelines) | [examples/](../examples/) | -| **Databases** | [CHECKLISTS.md#database--sql](CHECKLISTS.md#database--sql) | [src/dataenginex/core/](../src/dataenginex/core/) | -| **Tests** | [CHECKLISTS.md#tests](CHECKLISTS.md#tests) | [tests/](../tests/) | -| **Documentation** | [CHECKLISTS.md#documentation--comments](CHECKLISTS.md#documentation--comments) | Any file in [src/](../src/) | -| **Security** | [CHECKLISTS.md#security](CHECKLISTS.md#security-critical---check-every-pr) | [src/dataenginex/api/auth.py](../src/dataenginex/api/auth.py) | - ---- - -## 🚨 Red Flags (Always Check) - -- 🔴 Hardcoded secrets or credentials -- 🔴 No input validation -- 🔴 SQL with string concatenation -- 🔴 Bare `except:` catching all exceptions -- 🔴 N+1 database queries -- 🔴 New feature with no tests -- 🔴 No error logging - ---- - -## 🤝 Code Review Style - -- **Be specific**: "This could be SQL injection" not "This looks risky" -- **Explain the why**: Help people learn, not just follow rules -- **Acknowledge good work**: Point out solid patterns -- **Ask questions**: "What if...?" not "You should..." -- **Reference principles**: Ground feedback here, not personal preference - ---- - -## For AI Agents 🤖 - -1. Start with [copilot-instructions.md](copilot-instructions.md) for principles -2. Use [CHECKLISTS.md](CHECKLISTS.md) to find domain-specific expectations -3. Reference actual project code in [src/](../src/) and [tests/](../tests/) for patterns -4. Check [pyproject.toml](../pyproject.toml) for tool settings (Python version, dependencies, etc.) -5. When in doubt, look at the code - it's the best documentation - ---- - -## When You Disagree - -Principles are intentional. If you think something should change: - -1. **Try it** - Follow it for a sprint -2. **Measure it** - Does it actually help? -3. **Propose change** - Open an issue with evidence - -Standards improve when based on experience, not preference. - ---- - -## That's It - -Core files: -- **Principles** → [copilot-instructions.md](copilot-instructions.md) -- **Checklists** → [CHECKLISTS.md](CHECKLISTS.md) -- **Maintainer governance** → [MAINTAINER_PLAYBOOK.md](MAINTAINER_PLAYBOOK.md) -- **Examples** → [src/](../src/) and [tests/](../tests/) - -Questions? Ask the team or open an issue. diff --git a/ISSUE_TEMPLATE/bug_report.yml b/ISSUE_TEMPLATE/bug_report.yml index d9cc7f8..0943691 100644 --- a/ISSUE_TEMPLATE/bug_report.yml +++ b/ISSUE_TEMPLATE/bug_report.yml @@ -69,4 +69,4 @@ body: id: python attributes: label: Python Version - placeholder: "3.12.0" + placeholder: "3.13.0" diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index a59cfd3..ecc4b80 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -20,5 +20,5 @@ - [ ] Documentation updated (if applicable) - [ ] `uv run poe lint` passes - [ ] `uv run poe typecheck` passes -- [ ] `uv run poe test` passes +- [ ] `uv run poe check-all` passes - [ ] No breaking changes (or documented in description) diff --git a/SUPPORT.md b/SUPPORT.md index 57dc982..12ed834 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -21,7 +21,7 @@ For bugs and feature requests, use the issue tracker in the relevant repo: | Repo | Issues | |------|--------| -| DEX | [Issues](https://github.com/TheDataEngineX/DEX/issues) | +| DEX | [Issues](https://github.com/TheDataEngineX/dataenginex/issues) | | dex-studio | [Issues](https://github.com/TheDataEngineX/dex-studio/issues) | | infradex | [Issues](https://github.com/TheDataEngineX/infradex/issues) | diff --git a/agents/devops-engineer.agent.md b/agents/devops-engineer.agent.md index 13933bb..92024fb 100644 --- a/agents/devops-engineer.agent.md +++ b/agents/devops-engineer.agent.md @@ -9,8 +9,8 @@ You are a DevOps/SRE engineer for the DataEngineX project, managing Docker, Kube - Docker: multi-stage builds, `python:3.13-slim`, non-root `dex` user (UID 1000) - Kubernetes: Kustomize base + overlays (dev/stage/prod), ArgoCD GitOps -- CI/CD: GitHub Actions (`ci.yml`, `release-dataenginex.yml`, `pypi-publish.yml`, `security.yml`) -- Monitoring: Prometheus (`http_*` metrics), Grafana dashboards, AlertManager rules +- CI/CD: GitHub Actions (`security.yml`, `sync-labels.yml`, `project-automation.yml`, `stale-issues.yml`) +- Monitoring: Prometheus (`http_*` metrics), Grafana dashboards, AlertManager rules (in `infradex/monitoring/`) - Observability: Jaeger + OTLP tracing via Docker Compose - Security: Semgrep + CodeQL scanning, `dependabot.yml`, `poe security` @@ -25,15 +25,14 @@ You are a DevOps/SRE engineer for the DataEngineX project, managing Docker, Kube ## Key Project Files -- Dockerfile: `Dockerfile` (multi-stage, `PYTHONPATH="/app/src"`) -- Compose: `docker-compose.yml` (dataenginex, prometheus, alertmanager, grafana, jaeger) -- Workflows: `.github/workflows/` (ci, release-dataenginex, security, pypi-publish) -- Monitoring: `monitoring/` (prometheus.yml, alertmanager.yml, alerts/, grafana/) +- Dockerfile: None in dataenginex (only `docker-compose.test.yml` for testing) +- Compose: `docker-compose.test.yml` (dataenginex test services) +- Workflows: `.github/workflows/` (security, sync-labels, project-automation, stale-issues) +- Monitoring: `infradex/monitoring/` (prometheus.yml, alertmanager.yml, alerts/, grafana/) - Config: `pyproject.toml`, `poe_tasks.toml` ## Guidelines - YAML: 2-space indent, lowercase hyphenated names (e.g., `dex-dev`) - Workflow naming: files lowercase `.yml`, workflows title case, jobs kebab-case -- Releases published to PyPI via `release-dataenginex.yml` → `pypi-publish.yml` - Always maintain `.dockerignore` — no dev deps in production image diff --git a/agents/llm-architect.agent.md b/agents/llm-architect.agent.md index 300c916..1946fd0 100644 --- a/agents/llm-architect.agent.md +++ b/agents/llm-architect.agent.md @@ -27,10 +27,10 @@ You are an LLM systems architect for DataEngineX, responsible for the AI agent a ## Key Project Files -- LLM provider: `${WORKSPACE_ROOT}/dex/src/dataenginex/ml/llm.py` -- Vector store: `${WORKSPACE_ROOT}/dex/src/dataenginex/ml/vectorstore.py` -- Model serving: `${WORKSPACE_ROOT}/dex/src/dataenginex/ml/serving.py` -- Interfaces (BaseAgentRuntime, BaseLLMProvider, BaseVectorStore): `${WORKSPACE_ROOT}/dex/src/dataenginex/core/interfaces.py` +- LLM provider: `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/ai/llm.py` +- Vector store: `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/ai/vectorstore.py` +- Model serving: `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/ml/serving.py` +- Interfaces (BaseAgentRuntime, BaseLLMProvider, BaseVectorStore): `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/core/interfaces.py` - Hardware constraints: see `.github/workspace.env` (auto-generated — run `setup-workspace.sh` to refresh) ## Guidelines diff --git a/agents/mcp-developer.agent.md b/agents/mcp-developer.agent.md index d4d8e45..323f403 100644 --- a/agents/mcp-developer.agent.md +++ b/agents/mcp-developer.agent.md @@ -25,7 +25,7 @@ You are an MCP (Model Context Protocol) developer for DataEngineX, building and ## Key Project Files - Claude Code MCP config: `${WORKSPACE_ROOT}/.claude/settings.local.json` -- Agent integrations: `${WORKSPACE_ROOT}/dex/src/dataenginex/ml/` +- Agent integrations: `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/` - Context7 MCP: already configured — add `use context7` to prompts for library docs ## Registered MCP Servers diff --git a/agents/mlops-engineer.agent.md b/agents/mlops-engineer.agent.md index 63ff5be..89fb9cd 100644 --- a/agents/mlops-engineer.agent.md +++ b/agents/mlops-engineer.agent.md @@ -8,7 +8,7 @@ You are an MLOps engineer specializing in the DataEngineX ML lifecycle — model ## Your Expertise - Model lifecycle: development → staging → production → archived (`ModelRegistry`, JSON-persisted) -- Drift detection: PSI-based feature and prediction drift — `DriftDetector`, `StatisticalDriftDetector` +- Drift detection: PSI-based feature and prediction drift — `DriftDetector`, `PSIDriftDetector` - Training pipelines: PySpark ML `Pipeline` + `PipelineModel`, reproducible experiment runs - Model CI/CD: automated retraining triggers, validation gates before promotion - Experiment tracking: run metadata, metrics, artifact versioning @@ -26,8 +26,8 @@ You are an MLOps engineer specializing in the DataEngineX ML lifecycle — model - Model Registry: `src/dataenginex/ml/registry.py` - Drift Detection: `src/dataenginex/ml/drift.py` -- ML examples: `examples/07_ml_registry.py`, `examples/08_spark_ml.py` -- Metrics: `src/dataenginex/observability/metrics.py` +- ML examples: `examples/08_spark_ml.py`, `examples/09_feature_engineering.py` +- Metrics: `src/dataenginex/ml/metrics.py` - Tests: `tests/unit/test_ml.py`, `tests/unit/test_drift.py` ## Guidelines diff --git a/agents/multi-agent-coordinator.agent.md b/agents/multi-agent-coordinator.agent.md index 3126003..7889598 100644 --- a/agents/multi-agent-coordinator.agent.md +++ b/agents/multi-agent-coordinator.agent.md @@ -26,11 +26,11 @@ You are a multi-agent systems architect for DataEngineX, designing and implement ## Key Project Files -- Agent source: `${WORKSPACE_ROOT}/dex/src/dataenginex/ml/` -- Agent runtime: `${WORKSPACE_ROOT}/dex/src/dataenginex/ml/serving.py` -- LLM providers: `${WORKSPACE_ROOT}/dex/src/dataenginex/ml/llm.py` -- Interfaces: `${WORKSPACE_ROOT}/dex/src/dataenginex/core/interfaces.py` -- Tests: `${WORKSPACE_ROOT}/dex/tests/` +- Agent source: `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/ai/agents/` +- Agent runtime: `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/ml/serving.py` +- LLM providers: `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/ai/llm.py` +- Interfaces: `${WORKSPACE_ROOT}/dataenginex/src/dataenginex/core/interfaces.py` +- Tests: `${WORKSPACE_ROOT}/dataenginex/tests/` ## Guidelines diff --git a/agents/security-auditor.agent.md b/agents/security-auditor.agent.md index 3922aa5..b461d33 100644 --- a/agents/security-auditor.agent.md +++ b/agents/security-auditor.agent.md @@ -3,7 +3,7 @@ description: "Read-only security auditor for vulnerability analysis, dependency tools: ["search/codebase", "read/terminalLastCommand", "read/terminalSelection"] --- -You are a security auditor for DataEngineX — read-only analysis of vulnerabilities, misconfigurations, supply chain risks, and compliance gaps across all 6 repos. +You are a security auditor for DataEngineX — read-only analysis of vulnerabilities, misconfigurations, supply chain risks, and compliance gaps across all 3 repos. ## Your Expertise @@ -28,7 +28,7 @@ You are a security auditor for DataEngineX — read-only analysis of vulnerabili - Dependencies: `pyproject.toml` + `uv.lock` — known CVEs, abandoned packages - Config files: `.env.example`, K8s manifests, Dockerfiles — secrets, misconfigs - Workflows: `.github/workflows/` — permissions, secret exposure, action versions -- Trivy findings: `argocd/`, `helm/` — misconfigs, image vulnerabilities +- Trivy findings: `infradex/argocd/`, `infradex/helm/` — misconfigs, image vulnerabilities ## Red Flags to Always Check diff --git a/agents/sre-engineer.agent.md b/agents/sre-engineer.agent.md index e1b112c..1e84a20 100644 --- a/agents/sre-engineer.agent.md +++ b/agents/sre-engineer.agent.md @@ -24,11 +24,10 @@ You are an SRE for DataEngineX, owning reliability targets, Prometheus/Grafana m ## Key Project Files -- Prometheus config: `monitoring/prometheus.yml` -- Alert rules: `monitoring/alerts/` -- Alertmanager: `monitoring/alertmanager.yml` -- Grafana dashboards: `monitoring/grafana/` -- Docker Compose: `docker-compose.monitoring.yml` +- Prometheus config: `infradex/monitoring/prometheus.yml` +- Alert rules: `infradex/monitoring/alerts/` +- Alertmanager: `infradex/monitoring/alertmanager.yml` +- Grafana dashboards: `infradex/monitoring/grafana/` ## Monitoring Stack Ports diff --git a/copilot-instructions.md b/copilot-instructions.md index 57419ce..4fa5f3b 100644 --- a/copilot-instructions.md +++ b/copilot-instructions.md @@ -11,11 +11,8 @@ Be pragmatic, straight forward and challenge my ideas and system design focus on | Repo | Package | Purpose | Port | |------|---------|---------|------| | `dataenginex` (dex) | `dataenginex` | Core framework — config, registry, CLI, API, ML, AI | 17000 | -| `dex-studio` | `dex-studio` | Web UI — single pane of glass (NiceGUI) | 7860 | +| `dex-studio` | `dex-studio` | Web UI — single pane of glass (FastAPI + Jinja2 + HTMX) | 7860 | | `infradex` | — | IaC + monitoring (Terraform, Helm, K3s) | — | - -> datadex, agentdex, and careerdex consolidated into `dataenginex` as submodules. - --- ## Coding Standards @@ -58,7 +55,7 @@ Be pragmatic, straight forward and challenge my ideas and system design focus on 1. `uv run poe lint` — Ruff lint 2. `uv run poe typecheck` — mypy strict -3. `uv run poe test` — pytest +3. `uv run poe check-all` — lint + typecheck + tests 4. Start real server and verify endpoints (see repo `CLAUDE.md`) 5. Standalone import check @@ -66,7 +63,7 @@ Be pragmatic, straight forward and challenge my ideas and system design focus on ### Context7 MCP -Always use Context7 MCP for library/API docs (FastAPI, PySpark, Pydantic, Airflow, NiceGUI) — without the user having to ask. +Always use Context7 MCP for library/API docs (FastAPI, PySpark, Pydantic, Airflow, Jinja2) — without the user having to ask. --- @@ -76,7 +73,7 @@ Verify consistency across all 3 repos on every `dev → main` PR. ### Required workflows in all repos -`ci.yml` · `enforce-dev-to-main.yml` · `claude.yml` · `security.yml` · `release-please.yml` +`security.yml` · `sync-labels.yml` · `project-automation.yml` · `stale-issues.yml` ### Required files in all repos diff --git a/dependabot.yml b/dependabot.yml index ddc8218..43a5439 100755 --- a/dependabot.yml +++ b/dependabot.yml @@ -1,7 +1,7 @@ version: 2 updates: - package-ecosystem: pip - directory: /dex + directory: /dataenginex schedule: interval: weekly day: monday diff --git a/docs/superpowers/plans/2026-03-17-workspace-config-optimization.md b/docs/superpowers/plans/2026-03-17-workspace-config-optimization.md deleted file mode 100644 index db68e2a..0000000 --- a/docs/superpowers/plans/2026-03-17-workspace-config-optimization.md +++ /dev/null @@ -1,1215 +0,0 @@ -# Workspace Config Optimization Implementation Plan - -> **STATUS: HISTORICAL** — Implemented March 2026. Release flow, version bump tasks, and tag formats here are superseded by release-please. See `/release.md` and `.github/CLAUDE.md` for current state. **For agentic workers:** Use superpowers:subagent-driven-development or superpowers:executing-plans to implement remaining steps. - -**Goal:** Optimize all DataEngineX workspace config files — fix broken deps, upgrade to Python 3.13, eliminate CI race conditions, consolidate Claude Code permissions, trim duplicate documentation, and consolidate agent definitions. - -**Architecture:** Config-only changes across `.github/`, `dex/pyproject.toml`, `dex/.github/workflows/`, and `~/.claude/`. No production source code changes. Each task is independently committable. - -**Tech Stack:** Python 3.13, uv, GitHub Actions, Claude Code, Ruff, mypy - -**Spec:** `.github/docs/superpowers/specs/2026-03-17-workspace-config-optimization-design.md` - -**Git identity for all commits:** `--author="jaymyaka "` - -**Working directory:** `/home/jay/workspace/DataEngineX/` unless stated otherwise. - ---- - -## File Map - -| Task | Files Modified | -| ---- | -------------- | -| 1 | Delete `.claude.bak/`, `.vscode.bak/` | -| 2 | `dex/.github/workflows/pypi-publish.yml` | -| 3 | `.github/dependabot.yml` | -| 4 | `dex/.github/workflows/ci.yml` | -| 5 | `dex/pyproject.toml` | -| 6 | `dex/.github/workflows/security.yml`, `dex/.github/workflows/pypi-publish.yml` | -| 7 | `dex/pyproject.toml`, `dex/uv.lock` | -| 8 | `dex/pyproject.toml`, `dex/.github/workflows/ci.yml`, `dex/Dockerfile` | -| 9 | `.github/.claude/settings.json`, `.github/.claude/settings.local.json` | -| 10 | `~/.claude/settings.json` | -| 11 | `.github/.claude/commands/validate.md`, `pr.md`, `review.md`, new `release.md` | -| 12 | `.github/CLAUDE.md` | -| 13 | `.github/copilot-instructions.md` | -| 14 | `.github/agents/` (create `senior-engineer.agent.md`, delete 3 files, update all) | -| 15 | `.github/instructions/` (7 files), `.github/instructions/workflows.instructions.md` | - ---- - -## Task 1: Delete Backup Directories - -**Files:** - -- Delete: `/home/jay/workspace/DataEngineX/.claude.bak/` -- Delete: `/home/jay/workspace/DataEngineX/.vscode.bak/` - -- [ ] **Step 1: Verify symlinks are intact before deleting** - - ```bash - ls -la /home/jay/workspace/DataEngineX/.claude - ls -la /home/jay/workspace/DataEngineX/.vscode - ls -la /home/jay/workspace/DataEngineX/CLAUDE.md - ``` - - Expected: all three are symlinks pointing into `.github/`. - -- [ ] **Step 2: Delete backup directories** - - ```bash - rm -rf /home/jay/workspace/DataEngineX/.claude.bak - rm -rf /home/jay/workspace/DataEngineX/.vscode.bak - ``` - -- [ ] **Step 3: Verify deletion and symlinks still work** - - ```bash - ls /home/jay/workspace/DataEngineX/.claude.bak 2>&1 || echo "deleted OK" - ls /home/jay/workspace/DataEngineX/.vscode.bak 2>&1 || echo "deleted OK" - cat /home/jay/workspace/DataEngineX/.github/.claude/settings.json | head -3 - ``` - -- [ ] **Step 4: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX - git add -A - git commit --author="jaymyaka " \ - -m "chore: remove stale .claude.bak and .vscode.bak migration artifacts" - ``` - ---- - -## Task 2: Fix `pypi-publish.yml` Race Condition - -**Files:** - -- Modify: `dex/.github/workflows/pypi-publish.yml` - -The `workflow_run` trigger causes `pypi-publish.yml` to fire twice when `release-dataenginex.yml` creates a GitHub Release — once via `workflow_run` and once via `release:published`. Remove the `workflow_run` trigger AND all its downstream references. - -- [ ] **Step 1: Remove the `workflow_run:` trigger block (lines 11–17)** - - Delete this block from the `on:` section: - - ```yaml - # DELETE THIS ENTIRE BLOCK: - workflow_run: - workflows: - - Release DataEngineX - types: - - completed - branches: - - main - ``` - -- [ ] **Step 2: Simplify the `detect-dataenginex-changes` job `if:` condition (line 35)** - - Old: - ```yaml - if: ${{ github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' }} - ``` - - Delete the entire `if:` line — it becomes always-true and is no longer needed. - -- [ ] **Step 3: Remove `workflow_run` from the three `fromJson` arrays (lines 180, 201, 247)** - - In all three occurrences, change: - ```yaml - contains(fromJson('["release","workflow_run","workflow_dispatch"]'), github.event_name) - ``` - to: - ```yaml - contains(fromJson('["release","workflow_dispatch"]'), github.event_name) - ``` - -- [ ] **Step 4: Rewrite the inline JavaScript at the `skip-publish` Slack notification** - - Both lines 262 and 263 must change — the `if` condition AND the reason string: - - Old (lines 262–263): - ```javascript - if (eventName !== 'release' && eventName !== 'workflow_run' && eventName !== 'workflow_dispatch') { - reason = 'Build validation only (no release/workflow_run/manual publish event)'; - ``` - - New (both lines): - ```javascript - if (eventName !== 'release' && eventName !== 'workflow_dispatch') { - reason = 'Build validation only (no release/workflow_dispatch event)'; - ``` - -- [ ] **Step 5: Rewrite the inline bash at `skip-publish` (around lines 296–297)** - - Old: - ```bash - if [[ "${{ github.event_name }}" != "release" && "${{ github.event_name }}" != "workflow_run" && "${{ github.event_name }}" != "workflow_dispatch" ]]; then - echo "Build validation only: publish jobs run only for release:published, workflow_run, or workflow_dispatch events." - ``` - - New: - ```bash - if [[ "${{ github.event_name }}" != "release" && "${{ github.event_name }}" != "workflow_dispatch" ]]; then - echo "Build validation only: publish jobs run only for release:published or workflow_dispatch events." - ``` - -- [ ] **Step 6: Verify no `workflow_run` references remain** - - ```bash - grep -n "workflow_run" /home/jay/workspace/DataEngineX/dex/.github/workflows/pypi-publish.yml - ``` - - Expected: no output. - -- [ ] **Step 7: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - git add .github/workflows/pypi-publish.yml - git commit --author="jaymyaka " \ - -m "fix: remove workflow_run trigger from pypi-publish to eliminate race condition" - ``` - ---- - -## Task 3: Expand `dependabot.yml` - -**Files:** - -- Modify: `/home/jay/workspace/DataEngineX/.github/dependabot.yml` - -- [ ] **Step 1: Read current file** - - ```bash - cat /home/jay/workspace/DataEngineX/.github/dependabot.yml - ``` - -- [ ] **Step 2: Rewrite with expanded config** - - Current file has only two entries (pip + github-actions, both `directory: "/"`). No other - ecosystems to preserve. Replace the full file: - - ```yaml - version: 2 - updates: - - package-ecosystem: pip - directory: /dex - schedule: - interval: weekly - day: monday - open-pull-requests-limit: 5 - labels: - - dependencies - commit-message: - prefix: "chore(deps):" - - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - day: monday - open-pull-requests-limit: 5 - labels: - - dependencies - commit-message: - prefix: "chore(deps):" - ``` - -- [ ] **Step 3: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX - git add .github/dependabot.yml - git commit --author="jaymyaka " \ - -m "chore: expand dependabot.yml with labels, limits, and commit prefix" - ``` - ---- - -## Task 4: Add CI Version-Drift Check - -**Files:** - -- Modify: `dex/.github/workflows/ci.yml` - -Adds a step to the `lint` job that, when `workspace.env` is present, asserts `VERSION_DEX` matches `pyproject.toml`. Skips silently if `workspace.env` is absent (it is gitignored). - -- [ ] **Step 1: Read current ci.yml** - - ```bash - cat /home/jay/workspace/DataEngineX/dex/.github/workflows/ci.yml - ``` - -- [ ] **Step 2: Add version-drift check step to the `lint` job** - - Append this step to the end of the `lint` job's `steps:` list: - - ```yaml - - name: Check workspace.env version matches pyproject.toml - if: ${{ hashFiles('../.github/workspace.env') != '' }} - run: | - set -euo pipefail - PYPROJECT_VERSION=$(python3 -c "import tomllib; t=tomllib.load(open('pyproject.toml','rb')); print(t['project']['version'])") - WORKSPACE_VERSION=$(grep '^VERSION_DEX=' ../.github/workspace.env | cut -d= -f2) - if [[ "$PYPROJECT_VERSION" != "$WORKSPACE_VERSION" ]]; then - echo "Version drift: pyproject.toml=$PYPROJECT_VERSION but workspace.env=$WORKSPACE_VERSION" - echo "Re-run .github/scripts/setup-workspace.sh to fix." - exit 1 - fi - echo "Version check passed: $PYPROJECT_VERSION" - ``` - -- [ ] **Step 3: Re-run setup-workspace.sh to refresh workspace.env** - - ```bash - bash /home/jay/workspace/DataEngineX/.github/scripts/setup-workspace.sh - grep VERSION_DEX /home/jay/workspace/DataEngineX/.github/workspace.env - ``` - - Expected: `VERSION_DEX=0.7.1` - -- [ ] **Step 4: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - git add .github/workflows/ci.yml - git commit --author="jaymyaka " \ - -m "ci: add workspace.env version-drift check to lint job" - ``` - ---- - -## Task 5: Fix Broken Dependency Version Bounds - -**Files:** - -- Modify: `dex/pyproject.toml` - -Two packages have minimum bounds that exceed the latest available on PyPI and will fail to resolve. - -- [ ] **Step 1: Fix `mkdocstrings[python]` bound** - - In `pyproject.toml` `[dependency-groups] dev`, change: - ```toml - "mkdocstrings[python]>=1.0.0", - ``` - to: - ```toml - "mkdocstrings[python]>=0.29.0", - ``` - -- [ ] **Step 2: Fix `zensical` bound** - - `zensical` latest on PyPI is `0.0.19`. The `>=0.0.24` bound has never been released. - - In `[dependency-groups] dev`, change: - ```toml - "zensical>=0.0.24", - ``` - to: - ```toml - "zensical>=0.0.19", - ``` - -- [ ] **Step 3: Verify uv can now resolve** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - uv lock - ``` - - Expected: exits 0, `uv.lock` updated. - -- [ ] **Step 4: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - git add pyproject.toml uv.lock - git commit --author="jaymyaka " \ - -m "fix: correct mkdocstrings and zensical version bounds (were above PyPI latest)" - ``` - ---- - -## Task 6: Update Outdated GitHub Actions - -**Files:** - -- Modify: `dex/.github/workflows/security.yml` -- Modify: `dex/.github/workflows/pypi-publish.yml` - -### security.yml changes - -| Location | Old | New | -| -------- | --- | --- | -| Line 52 | `actions/checkout@v4` | `actions/checkout@v6` | -| Line 66 | `github/codeql-action/upload-sarif@v3` | `github/codeql-action/upload-sarif@v4` | -| Lines 27, 89 | `actions/github-script@v7` (×2) | `actions/github-script@v8` | - -### pypi-publish.yml changes - -| Location | Old | New | -| -------- | --- | --- | -| Multiple | `actions/github-script@v7` | `actions/github-script@v8` | - -- [ ] **Step 1: Update security.yml** - - ```bash - sed -i \ - -e 's|actions/checkout@v4|actions/checkout@v6|g' \ - -e 's|github/codeql-action/upload-sarif@v3|github/codeql-action/upload-sarif@v4|g' \ - -e 's|actions/github-script@v7|actions/github-script@v8|g' \ - /home/jay/workspace/DataEngineX/dex/.github/workflows/security.yml - ``` - -- [ ] **Step 2: Update pypi-publish.yml** - - ```bash - sed -i \ - -e 's|actions/github-script@v7|actions/github-script@v8|g' \ - /home/jay/workspace/DataEngineX/dex/.github/workflows/pypi-publish.yml - ``` - -- [ ] **Step 3: Verify changes** - - ```bash - grep -n "github-script\|codeql-action\|actions/checkout" \ - /home/jay/workspace/DataEngineX/dex/.github/workflows/security.yml \ - /home/jay/workspace/DataEngineX/dex/.github/workflows/pypi-publish.yml - ``` - - Expected: all occurrences show `@v8` for github-script, `@v4` for codeql, `@v6` for checkout. - -- [ ] **Step 4: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - git add .github/workflows/security.yml .github/workflows/pypi-publish.yml - git commit --author="jaymyaka " \ - -m "chore(deps): bump codeql-action@v4, github-script@v8, checkout@v6 in workflows" - ``` - ---- - -## Task 7: Minor Dependency Bound Bumps - -**Files:** - -- Modify: `dex/pyproject.toml`, `dex/uv.lock` - -- [ ] **Step 1: Update three bounds in `pyproject.toml`** - - In `[dependency-groups] dev`: - - - `"poethepoet>=0.42.0"` → `"poethepoet>=0.42.1"` - - `"types-pyyaml>=6.0.12.20250822"` → `"types-pyyaml>=6.0.12.20250915"` - - In `[dependency-groups] data`: - - - `"databricks-cli"` → `"databricks-cli>=0.18.0"` - -- [ ] **Step 2: Regenerate lockfile** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - uv lock - ``` - - Expected: exits 0. - -- [ ] **Step 3: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - git add pyproject.toml uv.lock - git commit --author="jaymyaka " \ - -m "chore(deps): bump poethepoet, types-pyyaml, pin databricks-cli>=0.18.0" - ``` - ---- - -## Task 8: Python 3.13 Upgrade - -**Files:** - -- Modify: `dex/pyproject.toml` -- Modify: `dex/.github/workflows/ci.yml` -- Modify: `dex/Dockerfile` - -- [ ] **Step 1: Update `pyproject.toml`** - - Make these three changes: - - ```toml - # [project] - requires-python = ">=3.13" # was >=3.12 - - # [tool.ruff] - target-version = "py313" # was py312 - - # [tool.mypy] - python_version = "3.13" # was 3.12 - ``` - -- [ ] **Step 2: Update `ci.yml` — all three jobs** - - Change every occurrence of `python-version: "3.12"` to `python-version: "3.13"`. - - ```bash - sed -i 's/python-version: "3.12"/python-version: "3.13"/g' \ - /home/jay/workspace/DataEngineX/dex/.github/workflows/ci.yml - ``` - - Verify: - ```bash - grep "python-version" /home/jay/workspace/DataEngineX/dex/.github/workflows/ci.yml - ``` - -- [ ] **Step 3: Update `Dockerfile`** - - Change both `FROM python:3.12-slim` lines to `python:3.13-slim`: - - ```bash - sed -i 's/python:3.12-slim/python:3.13-slim/g' \ - /home/jay/workspace/DataEngineX/dex/Dockerfile - ``` - - Verify: - ```bash - grep "FROM python" /home/jay/workspace/DataEngineX/dex/Dockerfile - ``` - -- [ ] **Step 4: Regenerate lockfile with Python 3.13** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - uv lock - ``` - -- [ ] **Step 5: Run full validation on Python 3.13** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - uv run poe check-all - ``` - - Expected: all pass. If any failures, diagnose and fix before committing. - -- [ ] **Step 6: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - git add pyproject.toml uv.lock .github/workflows/ci.yml Dockerfile - git commit --author="jaymyaka " \ - -m "feat: upgrade to Python 3.13 (pyproject, ci, Dockerfile)" - ``` - ---- - -## Task 9: Consolidate Workspace `settings.json` - -**Files:** - -- Modify: `.github/.claude/settings.json` -- Modify: `.github/.claude/settings.local.json` - -Move all non-credential CLI permissions from `settings.local.json` into `settings.json`. -Remove tool-bypass entries (`grep *`, `find *`, `cat *`, `head *`, `tail *`) since dedicated tools exist. - -- [ ] **Step 1: Rewrite `.github/.claude/settings.json`** - - Replace the entire file with (preserves `autoUpdatesChannel`, `statusLine`, `deny` from original; - removes `Bash(grep *)`, `Bash(find *)`, `Bash(cat *)`, `Bash(head *)`, `Bash(tail *)`; - adds `git fetch`, `echo`, `python3`, `gh issue/release`, `git status *`, `uv run *`): - - ```json - { - "autoUpdatesChannel": "latest", - "statusLine": { - "type": "command", - "command": "jq -r '\"[\\(.model.display_name)] \\(.context_window.used_percentage // 0)% context\"'", - "padding": 2 - }, - "permissions": { - "defaultMode": "default", - "allow": [ - "Read", - "Edit", - "Bash(uv run poe *)", - "Bash(uv run uvicorn *)", - "Bash(uv run python *)", - "Bash(uv run *)", - "Bash(uv lock *)", - "Bash(uv sync *)", - "Bash(curl *)", - "Bash(git status)", - "Bash(git status *)", - "Bash(git diff *)", - "Bash(git log *)", - "Bash(git branch *)", - "Bash(git add *)", - "Bash(git fetch *)", - "Bash(ls *)", - "Bash(pwd)", - "Bash(which *)", - "Bash(wc *)", - "Bash(echo *)", - "Bash(python -c *)", - "Bash(python3 *)", - "Bash(gh pr *)", - "Bash(gh issue *)", - "Bash(gh release *)", - "Bash(gh run *)", - "Bash(gh api *)", - "Bash(llmfit *)", - "Bash(npx -y @upstash/context7-mcp*)" - ], - "deny": [ - "Bash(git commit *)", - "Bash(git push *)", - "Bash(rm -rf *)", - "Bash(git push --force*)", - "Bash(git reset --hard*)", - "Bash(pip install*)", - "Bash(sudo *)", - "Bash(chmod 777*)", - "Bash(curl * | bash*)" - ] - } - } - ``` - -- [ ] **Step 2: Strip CLI permissions from `.github/.claude/settings.local.json`** - - The file should retain only credentials and MCP config: - - ```json - { - "env": { - "ANTHROPIC_BASE_URL": "", - "ANTHROPIC_AUTH_TOKEN": "", - "ANTHROPIC_API_KEY": "", - "ANTHROPIC_MODEL": "" - }, - "enableAllProjectMcpServers": true, - "enabledMcpjsonServers": [ - "context7" - ] - } - ``` - - Remove the entire `permissions` block — all entries have been moved to `settings.json`. - -- [ ] **Step 3: Verify** - - ```bash - cat /home/jay/workspace/DataEngineX/.github/.claude/settings.json | python3 -m json.tool > /dev/null && echo "valid JSON" - cat /home/jay/workspace/DataEngineX/.github/.claude/settings.local.json | python3 -m json.tool > /dev/null && echo "valid JSON" - ``` - -- [ ] **Step 4: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX - git add .github/.claude/settings.json .github/.claude/settings.local.json - git commit --author="jaymyaka " \ - -m "chore: consolidate Claude Code permissions into settings.json, strip tool-bypass entries" - ``` - ---- - -## Task 10: Update Global `~/.claude/settings.json` - -**Files:** - -- Modify: `~/.claude/settings.json` - -Add read-only git ops and ripgrep to the global allow list, using the colon syntax that file already uses. - -- [ ] **Step 1: Read current file** - - ```bash - cat /home/jay/.claude/settings.json - ``` - -- [ ] **Step 2: Add entries to `permissions.allow`** - - Add alongside the existing `"Bash(uv run:*)"`: - - ```json - "Bash(git status:*)", - "Bash(git diff:*)", - "Bash(git log:*)", - "Bash(rg:*)" - ``` - - Final `permissions.allow` should be: - - ```json - "allow": [ - "Bash(uv run:*)", - "Bash(git status:*)", - "Bash(git diff:*)", - "Bash(git log:*)", - "Bash(rg:*)" - ] - ``` - -- [ ] **Step 3: Verify JSON is valid** - - ```bash - python3 -m json.tool /home/jay/.claude/settings.json > /dev/null && echo "valid JSON" - ``` - - Note: `~/.claude/settings.json` is not in a git repo — no commit needed for this task. - ---- - -## Task 11: Update Claude Code Commands - -**Files:** - -- Modify: `.github/.claude/commands/validate.md` -- Modify: `.github/.claude/commands/pr.md` -- Modify: `.github/.claude/commands/review.md` -- Create: `.github/.claude/commands/release.md` - -### `validate.md` — promote dev-server smoke test to numbered step - -Replace the current file content with: - -```markdown -# Validate - -Run the full validation pipeline for the current repo. Stop and report on first failure — do not skip steps. - -1. **Lint** - - ```bash - uv run poe lint - # fallback: uv run ruff check src/ tests/ - ``` - -1. **Format check** - - ```bash - uv run ruff format --check src/ tests/ - ``` - -1. **Typecheck** - - ```bash - uv run poe typecheck - # fallback: uv run mypy src// --strict - ``` - -1. **Tests** - - ```bash - uv run poe test - # fallback: uv run pytest tests/ -x --tb=short -q - ``` - -1. **Dev server smoke test** *(dex only)* - - ```bash - uv run poe dev & - sleep 3 - curl -sf http://localhost:17000/health && echo "health OK" - curl -sf http://localhost:17000/metrics | head -5 && echo "metrics OK" - kill %1 - ``` - -1. **Import check** — verify the package loads without hidden import-time errors - - ```bash - uv run python -c "import ; print('OK', .__file__)" - ``` - -Report pass/fail for each step with the exact error output. If a step fails, diagnose the root cause before stopping. -``` - -### `pr.md` — add author, branch convention, dev-target enforcement - -Replace the current file content with: - -```markdown -Generate a PR description for the current branch's changes. - -## Branch Convention - -Branch must follow `feature/` or `fix/` pattern. -Target must be `dev` — NEVER `main` directly. - -## Steps - -1. Verify branch naming: `git branch --show-current` -1. Run `git log --oneline dev..HEAD` to see commits -1. Run `git diff dev --stat` to see changed files -1. Run `git diff dev` for the actual changes -1. Categorize changes: - - **What** — What changed and why - - **How** — Implementation approach - - **Testing** — How it was validated (include test results) - - **Breaking Changes** — Any API contract changes - - **Checklist** — Based on `.github/CHECKLISTS.md` - -## Format - -Format as a proper GitHub PR description with: - -- Title following conventional commit format -- Summary paragraph -- Bullet-point change list -- Test evidence -- Reviewer notes - -Reference related issues where applicable. - -## Git Identity - -All commits on this branch must use: -`git commit --author="jaymyaka "` -``` - -### `review.md` — align with `CHECKLISTS.md` (add API contract and migration checks) - -Add the following checks to the "Review priorities" section: - -```markdown -5. **API contracts** — versioned routes (`/api/v1/`), `response_model=` on every endpoint, - no silent breaking changes to existing endpoints -6. **Migrations** — DB migrations are reversible, include rollback, tested in isolation -``` - -### `release.md` — new file - -```markdown -# Release - -Canonical release flow for dataenginex. Claude must show the staged diff and wait for -explicit approval before executing any commit. - -## Prerequisites - -- All tests passing: `uv run poe check-all` -- On a `feature/*` or `fix/*` branch (not directly on `dev` or `main`) - -## Steps - -1. **Bump version** (choose one): - - ```bash - uv run poe version-patch # bug fixes - uv run poe version-minor # new features - uv run poe version-major # breaking changes - ``` - -1. **Stage the version bump files** - - ```bash - git add pyproject.toml uv.lock CHANGELOG.md - git diff --staged - ``` - -1. **STOP — show diff and request explicit approval before committing.** - Do not proceed until the user approves the staged changes. - -1. **Commit with author identity** (after approval only): - - ```bash - git commit --author="jaymyaka " \ - -m "chore: bump version to $(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")" - ``` - -1. **Push branch and open PR to `dev`** - - ```bash - git push -u origin HEAD - gh pr create --base dev --title "chore: release vX.Y.Z" --body "Version bump" - ``` - -1. **After dev PR merges — open PR `dev` → `main`** - - ```bash - gh pr create --base main --head dev --title "release: vX.Y.Z" --body "Release vX.Y.Z" - ``` - -1. **After main PR merges — create GitHub Release** - - ```bash - VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") - gh release create "dataenginex-v${VERSION}" --generate-notes --title "dataenginex v${VERSION}" - ``` - -1. **Monitor PyPI publish** - - ```bash - gh run list --workflow=pypi-publish.yml --limit 5 - ``` -``` - -- [ ] **Step 1: Update `validate.md`** with the content above -- [ ] **Step 2: Update `pr.md`** with the content above -- [ ] **Step 3: Update `review.md`** — add the two new review priorities (API contracts, Migrations) -- [ ] **Step 4: Create `release.md`** with the content above -- [ ] **Step 5: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX - git add .github/.claude/commands/ - git commit --author="jaymyaka " \ - -m "feat: update Claude commands — promote smoke test, add release workflow, align review checklist" - ``` - ---- - -## Task 12: Restructure and Trim `CLAUDE.md` - -**Files:** - -- Modify: `.github/CLAUDE.md` - -**Goal:** Preserve all rules. Remove tooling-config that lives in `pyproject.toml`. Reorder sections for onboarding flow. Consolidate Design Philosophy to heading + one-liner + enforcement note format. - -- [ ] **Step 1: Read the current file** - - ```bash - wc -l /home/jay/workspace/DataEngineX/.github/CLAUDE.md - cat /home/jay/workspace/DataEngineX/.github/CLAUDE.md - ``` - -- [ ] **Step 2: Remove tooling-redundant content from Coding Standards section** - - Remove the following content blocks (they live in `pyproject.toml`): - - The Ruff rule list: `E, F, I, B, UP, SIM, C90` - - `line-length = 100` - - `max complexity = 8` - - `functions ≤ 50 lines` - - `asyncio_mode = "auto"` note - - `@pytest.mark.asyncio` note - - Replace all of the above with one line: - > *"Ruff, mypy strict, and pytest are pre-configured in each repo's `pyproject.toml` — run `uv run poe check-all` to validate."* - -- [ ] **Step 3: Reorder sections** - - Move sections to this order (all content preserved): - 1. Design Philosophy - 2. Workspace Layout - 3. Git Conventions *(moved up from below Workflow Orchestration)* - 4. Semantic Versioning *(moved up)* - 5. Dependencies *(moved up)* - 6. Workflow Orchestration - 7. Task Management - 8. Coding Standards - 9. Core Principles - 10. Red Flags - 11. Cross-Repo Sync Policy - 12. Developer Tools - -- [ ] **Step 4: Consolidate Design Philosophy** - - Convert each of the 10 principle blocks from multi-paragraph to: - **`### Principle Name`** + one-sentence summary + `*Enforced by: ...*` note. - - Example: - ```markdown - ### Self-Hosted & Portable - Runs identically on local Docker Compose or any cloud provider via environment-specific config only. - *Enforced by: standard protocols (S3-compatible, PostgreSQL, Redis, Kubernetes API) — no cloud-provider SDKs in app code.* - ``` - -- [ ] **Step 5: Verify word count reduction** - - ```bash - wc -l /home/jay/workspace/DataEngineX/.github/CLAUDE.md - ``` - - Expected: ≥30% fewer lines than the original count from Step 1. If not, review what was missed. - -- [ ] **Step 6: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX - git add .github/CLAUDE.md - git commit --author="jaymyaka " \ - -m "docs: restructure CLAUDE.md — trim tooling-config duplication, reorder sections, consolidate philosophy" - ``` - ---- - -## Task 13: Trim `copilot-instructions.md` - -**Files:** - -- Modify: `.github/copilot-instructions.md` - -**Goal:** Remove all content that duplicates CLAUDE.md. Target ≤4KB. Keep only Copilot-specific directives. - -- [ ] **Step 1: Read the current file size** - - ```bash - wc -c /home/jay/workspace/DataEngineX/.github/copilot-instructions.md - ``` - -- [ ] **Step 2: Strip duplicated CLAUDE.md content** - - Remove these sections that exist verbatim in CLAUDE.md: - - Full Design Philosophy block (all 10 principles) - - Coding Standards (Ruff rules, mypy, logging, error handling, testing, security) - - Git Conventions - - Semantic Versioning rules - -- [ ] **Step 3: Add cross-reference and keep Copilot-specific content only** - - Add at the top of the file: - ```markdown - > For coding standards, architecture principles, git conventions, and versioning rules, - > see `CLAUDE.md` — Copilot reads the same workspace. - ``` - - Retain only: - - Workspace layout table - - Copilot Chat slash command hints - - `@workspace`, `#file` context directives - - Agent/instruction file references - - Tool references (llmfit, Context7) - -- [ ] **Step 4: Verify size** - - ```bash - wc -c /home/jay/workspace/DataEngineX/.github/copilot-instructions.md - ``` - - Expected: ≤4096 bytes. - -- [ ] **Step 5: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX - git add .github/copilot-instructions.md - git commit --author="jaymyaka " \ - -m "docs: trim copilot-instructions.md to Copilot-specific content only (~4KB)" - ``` - ---- - -## Task 14: Consolidate Agent Definitions - -**Files:** - -- Create: `.github/agents/senior-engineer.agent.md` -- Delete: `.github/agents/backend-engineer.agent.md` -- Delete: `.github/agents/code-reviewer.agent.md` -- Delete: `.github/agents/python-pro.agent.md` -- Modify: All remaining 8 agent files (add Python 3.13 where 3.12 is referenced) - -### Create `senior-engineer.agent.md` - -Merge the best of `backend-engineer`, `code-reviewer`, and `python-pro`: - -```markdown ---- -description: "Senior Python/FastAPI engineer for DataEngineX — code quality, review, and backend development" -tools: ["search/codebase", "execute/runInTerminal", "execute/getTerminalOutput", "read/terminalLastCommand", "read/terminalSelection", "web/githubRepo"] ---- - -You are a senior Python 3.13+ engineer and code reviewer for the DataEngineX project. - -## Your Expertise - -- FastAPI: lifespan, middleware, routers, Pydantic v2, `response_model=` on every endpoint -- Type system: `from __future__ import annotations`, `mypy --strict`, `TypeVar`, `Protocol` -- Async: `asyncio`, `asyncio.TaskGroup`, structured concurrency -- Logging: `structlog` only across entire codebase — never `loguru`, never `print()` -- Observability: Prometheus metrics (`http_*` prefix), OpenTelemetry tracing, structured logging -- Auth: pure-Python HS256 JWT (no pyjwt), `BaseHTTPMiddleware` pattern -- Performance: `__slots__`, lazy imports, generator pipelines, connection pooling -- Packaging: `hatchling` + `uv`, `[dependency-groups]`, `uv.lock` as reproducibility contract - -## Your Approach - -- Always read existing patterns in `src/dataenginex/` before writing new code -- Run `uv run poe check-all` (lint + typecheck + tests) before marking anything done -- Max 4 parameters per function — use dataclasses or Pydantic models for more -- No bare `except:` — catch specific exceptions with full context -- No mutable default arguments — use `field(default_factory=...)` or `None` + early return -- `raise NotImplementedError("descriptive message")` for stubs — never return fake data -- Conventional commits: `feat:`, `fix:`, `refactor:`, `test:` - -## Code Review Priorities - -1. **Security** — No hardcoded secrets, parameterized queries, no PII in logs -2. **Correctness** — Specific exceptions, error context logged, type safety -3. **Testing** — Tests exist for new code, 80%+ coverage, AAA pattern -4. **Standards** — `from __future__ import annotations`, type hints, correct logging stack - -For each issue: **File & location** | **Severity**: critical/warning/info | **Issue** | **Fix** - -End with: total issues by severity + merge-ready verdict. - -## Key Project Files - -- Entry point: `examples/02_api_quickstart.py` -- Schemas: `src/dataenginex/core/schemas.py` -- Errors: `src/dataenginex/api/errors.py` -- Middleware: `src/dataenginex/middleware/` -- Tests: `tests/unit/`, `tests/integration/` -- Ruff/mypy config: `pyproject.toml` -- Poe tasks: `poe_tasks.toml` -``` - -- [ ] **Step 1: Create `senior-engineer.agent.md`** with content above -- [ ] **Step 2: Delete the three merged files** - - ```bash - rm /home/jay/workspace/DataEngineX/.github/agents/backend-engineer.agent.md - rm /home/jay/workspace/DataEngineX/.github/agents/code-reviewer.agent.md - rm /home/jay/workspace/DataEngineX/.github/agents/python-pro.agent.md - ``` - -- [ ] **Step 3: Update Python version references in remaining agent files** - - ```bash - grep -l "3\.12" /home/jay/workspace/DataEngineX/.github/agents/*.agent.md - ``` - - For each file found, change `Python 3.12+` to `Python 3.13+`. - -- [ ] **Step 4: Verify agent count** - - ```bash - ls /home/jay/workspace/DataEngineX/.github/agents/*.agent.md | wc -l - ``` - - Expected: `9` (8 domain-specific + `senior-engineer`). - -- [ ] **Step 5: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX - git add .github/agents/ - git commit --author="jaymyaka " \ - -m "refactor: consolidate backend-engineer, code-reviewer, python-pro into senior-engineer agent" - ``` - ---- - -## Task 15: Instruction File Deduplication - -**Files:** - -- Modify: `.github/instructions/workflows.instructions.md` *(critical — contains wrong action versions)* -- Audit and modify: `.github/instructions/python.instructions.md` -- Audit and modify: `.github/instructions/fastapi.instructions.md` -- Audit and modify: `.github/instructions/data-pipelines.instructions.md` -- Audit and modify: `.github/instructions/ml.instructions.md` -- Audit and modify: `.github/instructions/testing.instructions.md` -- Audit and modify: `.github/instructions/infrastructure.instructions.md` - -### `workflows.instructions.md` — critical fix (wrong action versions) - -- [ ] **Step 1: Update the pinned version table** - - Replace the table and the prohibition line: - - ```markdown - ## Action versions — current pinned versions - - | Action | Version | - | --- | --- | - | `actions/checkout` | `@v6` | - | `actions/setup-python` | `@v6` | - | `actions/upload-artifact` | `@v7` | - | `actions/download-artifact` | `@v8` | - | `astral-sh/setup-uv` | `@v7` | - | `codecov/codecov-action` | `@v5` | - | `github/codeql-action/*` | `@v4` | - | `actions/github-script` | `@v8` | - - Always pin to the current major tag. Verify against actual workflow files before documenting. - ``` - - Remove the line: `Never use @latest, @main, or non-existent versions (@v6, @v7, @v8).` - - Update the Python setup example to use `@v7` and `@v6`: - - ```yaml - - uses: astral-sh/setup-uv@v7 - with: - version: "latest" - - uses: actions/setup-python@v6 - with: - python-version: "3.13" - - run: uv sync - ``` - -### Remaining 6 instruction files — deduplication - -- [ ] **Step 2: Read each file and remove paragraphs that are verbatim copies of CLAUDE.md** - - For each file: - 1. Read the file - 2. Identify paragraphs that exist word-for-word in CLAUDE.md - 3. Replace each removed block with a comment: `` - 4. Update `python-version` references from 3.12 to 3.13 where present - -- [ ] **Step 3: Commit** - - ```bash - cd /home/jay/workspace/DataEngineX - git add .github/instructions/ - git commit --author="jaymyaka " \ - -m "docs: fix workflows.instructions.md action versions, deduplicate instruction files vs CLAUDE.md" - ``` - ---- - -## Final Verification - -- [ ] **Run full validation** - - ```bash - cd /home/jay/workspace/DataEngineX/dex - uv run poe check-all - ``` - - Expected: all pass on Python 3.13. - -- [ ] **Check all success criteria from spec** - - ```bash - # No .bak dirs - ls /home/jay/workspace/DataEngineX/.claude.bak 2>&1 || echo "OK — no .bak" - - # workflow_run gone from pypi-publish - grep "workflow_run" dex/.github/workflows/pypi-publish.yml 2>&1 || echo "OK — no workflow_run" - - # release-dataenginex still exists - ls dex/.github/workflows/release-dataenginex.yml && echo "OK — release workflow present" - - # settings.local.json clean - cat .github/.claude/settings.local.json | python3 -m json.tool - - # agent count - ls .github/agents/*.agent.md | wc -l # expected: 9 - - # Python 3.13 in ci.yml - grep "python-version" dex/.github/workflows/ci.yml # expected: 3.13 - - # pyproject.toml Python version - grep "requires-python\|python_version\|target-version" dex/pyproject.toml # expected: 3.13 - - # copilot-instructions size - wc -c .github/copilot-instructions.md # expected: ≤4096 - ``` - -- [ ] **Open PR: feature branch → dev** - - ```bash - gh pr create --base dev \ - --title "chore: workspace config optimization — Python 3.13, dep fixes, CI cleanup, doc trim" \ - --body "Implements .github/docs/superpowers/specs/2026-03-17-workspace-config-optimization-design.md" - ``` diff --git a/docs/superpowers/specs/2026-03-17-workspace-config-optimization-design.md b/docs/superpowers/specs/2026-03-17-workspace-config-optimization-design.md deleted file mode 100644 index b48f34a..0000000 --- a/docs/superpowers/specs/2026-03-17-workspace-config-optimization-design.md +++ /dev/null @@ -1,385 +0,0 @@ -# Workspace Config Optimization — Design Spec - -> **STATUS: HISTORICAL** — Implemented March 2026. Release flow, version bump tasks (`version-patch/minor/major`), and `dataenginex-v` tag format documented here are superseded. Current state: release-please automation, `v{version}` tags, pre-commit hook for version bumps. - -**Date:** 2026-03-17 -**Author:** jaymyaka -**Scope:** `.github/`, `.claude/`, `.vscode/`, global `~/.claude/`, workspace root config files - ---- - -## Goal - -Optimize all workspace configuration files across three axes: - -1. **Housekeeping** — remove stale artifacts, fix version drift, fix release workflow race condition -2. **Claude Code config** — consolidate permissions, align commands with actual workflows, add missing flows -3. **Content quality** — reduce CLAUDE.md redundancy with enforced tooling config, shrink - copilot-instructions, consolidate overlapping agent definitions - -All existing rules are preserved. Sections that duplicate what Ruff/mypy/pytest config already -enforces are removed from prompt files (they add noise without adding value). - ---- - -## Section 1: Housekeeping - -### 1.1 Delete Backup Directories - -- **Delete** `/home/jay/workspace/DataEngineX/.claude.bak/` -- **Delete** `/home/jay/workspace/DataEngineX/.vscode.bak/` -- **Reason:** Migration to centralized `.github/` with symlinks is complete. Backups are stale and - misleading. `setup-workspace.sh` only creates `.bak` directories when it finds a real (non-symlink) - directory at the target path, which cannot happen on a clean clone — safe to delete permanently. - -### 1.2 Fix `workspace.env` Version Staleness - -- **Problem:** `workspace.env` shows `VERSION_DEX=0.6.1`; actual version is `0.7.1`. -- **Root cause:** `setup-workspace.sh` is correct — it already uses `tomllib` via Python to read - `[project].version` from each `pyproject.toml`. The stale value exists because the script was not - re-run after the 0.7.1 release (VSCode's `folderOpen` task should have caught this). -- **Fix (two-part):** - 1. Re-run `setup-workspace.sh` now to regenerate `workspace.env` with current versions. - 2. Add a step to the `lint` job in `dex/.github/workflows/ci.yml` that, when `workspace.env` - is present, asserts `VERSION_DEX` matches the `version` in `pyproject.toml`. Skip silently - if `workspace.env` is absent (it is gitignored and will not exist in CI or fresh clones — - this check is only meaningful in the developer's local environment). -- **Do NOT change** the `tomllib`-based extraction logic — it is correct. - -### 1.3 Fix Race Condition in Release Workflows - -- **Keep:** Both `dex/.github/workflows/release-dataenginex.yml` AND - `dex/.github/workflows/pypi-publish.yml` -- **Problem:** `pypi-publish.yml` has two simultaneous automatic triggers — `release:published` - AND `workflow_run` (on "Release DataEngineX"). Flow: push to main → - `release-dataenginex.yml` runs → creates GitHub Release → triggers `pypi-publish.yml` via - BOTH paths at once. Two parallel publish runs, potential duplicate upload. -- **Fix:** In `pypi-publish.yml`, in a single commit: - 1. Remove the entire `workflow_run:` trigger block (lines 11–17) - 2. Simplify the `detect-dataenginex-changes` job `if:` condition (line 35): remove the - `github.event_name != 'workflow_run'` guard — it becomes unreachable - 3. Remove `workflow_run` from `fromJson(...)` arrays in job `if:` conditions (lines 180, 201, - 247): change `'["release","workflow_run","workflow_dispatch"]'` to - `'["release","workflow_dispatch"]'` in each occurrence - 4. Rewrite the inline JavaScript at lines 262–263 (`skip-publish` Slack notification): remove the - `eventName !== 'workflow_run'` condition check and its corresponding reason string - 5. Rewrite the inline bash at lines 296–297: remove the `workflow_run` branch from the `if/elif` - chain and the echo string that references it - Canonical flow after fix: `release-dataenginex.yml` creates GitHub Release → `pypi-publish.yml` - fires via `release:published` exactly once. `workflow_dispatch` is kept for manual runs. -- **Why `release-dataenginex.yml` must stay:** It owns tag creation, release notes, and SBOM - generation — none of which exist in `pypi-publish.yml`. - -### 1.4 Expand `dependabot.yml` - -- **File:** `.github/dependabot.yml` (workspace root — the only `dependabot.yml` in the monorepo) -- **Add to each ecosystem entry:** - - `schedule.day: monday` (explicit day alongside existing `interval: weekly`) - - `open-pull-requests-limit: 5` (prevent PR flood) - - `labels: ["dependencies"]` for easy filtering - - `commit-message.prefix: "chore(deps):"` to match conventional commits -- **Add** a `github-actions` ecosystem entry if not already present - ---- - -## Section 2: Claude Code Config - -### 2.1 Workspace `settings.json` (`.github/.claude/settings.json`) - -**Current state (confirmed by reading the file):** - -Already present in allow list: `Read`, `Edit`, `Bash(uv run poe *)`, `Bash(uv run uvicorn *)`, -`Bash(uv lock *)`, `Bash(uv sync *)`, `Bash(curl *)`, `Bash(git status)`, `Bash(git diff *)`, -`Bash(git log *)`, `Bash(git branch *)`, `Bash(git add *)`, `Bash(grep *)`, `Bash(find *)`, -`Bash(cat *)`, `Bash(head *)`, `Bash(tail *)`, `Bash(wc *)`, `Bash(ls *)`, `Bash(pwd)`, -`Bash(which *)`, `Bash(python -c *)`, `Bash(uv run python *)`, `Bash(llmfit *)`, -`Bash(npx -y @upstash/context7-mcp*)` - -**Changes:** - -Move from `settings.local.json` to `settings.json` (non-credential CLI patterns): - -- `Bash(gh run *)`, `Bash(gh api *)`, `Bash(gh pr *)` — already in local -- `Bash(git fetch *)` — already in local -- `Bash(echo *)` — already in local -- `Bash(python3 *)` — already in local -- `Bash(uv run *)` — already in local (broader than workspace's `uv run poe *` etc.) - -Add new entries not in either file: - -- `Bash(gh issue *)`, `Bash(gh release *)` — needed for release workflow - -Remove from allow list: - -- `Bash(grep *)` — the Grep tool exists for this; shell grep bypasses tool tracking -- `Bash(find *)` — the Glob tool exists for this -- `Bash(cat *)`, `Bash(head *)`, `Bash(tail *)` — the Read tool exists for these - -**Keep the poe pattern as-is:** `Bash(uv run poe *)` is already correct and covers all tasks. -Do NOT change it to `Bash(uv run poe *:*)` — the colon syntax does not match poe invocations. - -After moving non-credentials from `settings.local.json` to `settings.json`, `settings.local.json` -retains only: `env` block (API keys/tokens), `enableAllProjectMcpServers`, `enabledMcpjsonServers`. - -**No changes to deny list** — current denials are correct and intentional. - -### 2.2 Global `~/.claude/settings.json` - -**Current state:** Single allow entry using colon syntax: `"Bash(uv run:*)"`. New entries must -match this format — NOT the space-glob format used in workspace `settings.json`. - -**Add to allow list:** - -- `"Bash(git status:*)"`, `"Bash(git diff:*)"`, `"Bash(git log:*)"` — safe read-only git ops -- `"Bash(rg:*)"` — ripgrep, used by many tools as a subprocess - -**Verify plugin list** matches installed: superpowers (5.0.4), context7, code-review, -code-simplifier, huggingface-skills, github. - -### 2.3 Claude Code Commands (`.github/.claude/commands/`) - -**`validate.md`** — Align with the mandatory 5-step pipeline from `dex/CLAUDE.md`. Promote the -dev-server smoke test from a note to a full numbered step: - -1. `uv run poe lint` -2. `uv run poe typecheck` -3. `uv run poe test` -4. Smoke test dev server: `uv run poe dev` + curl `/health`, `/metrics` -5. Standalone import check of key public classes - -**`pr.md`** — Add: - -- `--author="jaymyaka "` to all git commit invocations -- Branch naming convention: `feature/` or `fix/` -- Explicit check: confirm target branch is `dev`, never `main` - -**`review.md`** — Align checklist exactly with `.github/CHECKLISTS.md` (currently diverges on -API contract and migration checks). - -**Add `release.md`** — New command documenting canonical release flow. Claude must NOT auto-commit; -it must show the staged diff and wait for explicit approval at the commit step: - -1. Bump version: `uv run poe version-patch|minor|major` -2. Stage: `git add pyproject.toml uv.lock CHANGELOG.md` -3. Show diff (`git diff --staged`) and request explicit approval before committing -4. After approval: `git commit --author="jaymyaka " -m "chore: bump version to x.y.z"` -5. Push branch, open PR to `dev`, merge -6. Open PR `dev` → `main`, merge -7. Create release: `gh release create vx.y.z --generate-notes` -8. Monitor: `gh run list --workflow=pypi-publish.yml` - -**Git identity note:** The `release.md` command is the only place commit operations happen under -approval. Git identity (`jaymyaka `) belongs here, not scattered across -all agent files (where `git commit` is blocked by the deny list anyway). - ---- - -## Section 3: Content Quality - -### 3.1 `CLAUDE.md` (workspace-level) - -**Preserve:** All rules, all principles, all workflow steps. - -**Remove (enforced by tooling, not prompt):** - -- Full Ruff rule list (`E, F, I, B, UP, SIM, C90`), line-length 100, max complexity 8, function - length 50 lines — all configured in `pyproject.toml` -- `asyncio_mode = "auto"` and `@pytest.mark.asyncio` note — configured in `pyproject.toml` -- Replace removed items with one line: - *"Ruff, mypy strict, and pytest are pre-configured in each repo's `pyproject.toml` — - run `uv run poe check-all` to validate."* - -**Restructure sections (all content preserved, reordered for onboarding flow):** - -| New Order | Section | Change | -| --------- | ------- | ------ | -| 1 | Design Philosophy | Consolidated — see below | -| 2 | Workspace Layout | Unchanged | -| 3 | Git Conventions | Moved earlier — devs hit this immediately | -| 4 | Semantic Versioning | Moved after Git — natural progression | -| 5 | Dependencies | Moved after versioning | -| 6 | Workflow Orchestration | Tightened — remove steps covered by superpowers skills | -| 7 | Task Management | Unchanged | -| 8 | Coding Standards | Trimmed as above; tooling config cross-referenced | -| 9 | Core Principles | Unchanged | -| 10 | Red Flags | Unchanged | -| 11 | Cross-Repo Sync Policy | Unchanged | -| 12 | Developer Tools | Unchanged | - -**Design Philosophy consolidation:** - -- Keep all 10 principles; zero information loss -- Convert each from a multi-paragraph block to: **heading** + one-sentence summary + enforcement - note (e.g., *"Enforced by: OSI license check in CI"*) -- Estimated reduction: ~40% word count - -### 3.2 `copilot-instructions.md` - -**Current:** ~18KB, largely duplicates CLAUDE.md verbatim. - -**Target:** ~4KB — Copilot-specific directives only: - -- Inline completion hints (file context, symbol awareness) -- Copilot Chat slash commands and agent mode behavior -- Workspace-level context directives (`@workspace`, `#file`) -- One cross-reference line: *"For coding standards, architecture principles, and git conventions, - see CLAUDE.md — Copilot reads the same workspace."* - -**Remove:** All verbatim Design Philosophy, Coding Standards, Git Conventions. - -### 3.3 Agent Definitions (`agents/`) - -**Consolidate overlapping agents (3 files → 1):** - -| Remove | Merge into | Reason | -| ------ | ---------- | ------ | -| `code-reviewer.agent.md` | `senior-engineer.agent.md` (new) | Scope fully overlaps the other two | -| `python-pro.agent.md` | `senior-engineer.agent.md` (new) | Python expertise is a senior baseline | -| `backend-engineer.agent.md` | `senior-engineer.agent.md` (new) | Repeats python-pro content entirely | - -**Keep (domain-specific, non-overlapping):** -`data-engineer`, `mlops-engineer`, `llm-architect`, `sre-engineer`, `devops-engineer`, -`mcp-developer`, `security-auditor`, `multi-agent-coordinator` - -**Git identity:** Do NOT add git identity instructions to agent bodies. `git commit` is on the -deny list — agents cannot commit regardless of identity. Git identity lives in `release.md` -(the only approved commit flow). Adding it to agent bodies adds noise without effect. - -### 3.4 Instruction Files (`instructions/`) - -- Audit each of the 7 files against CLAUDE.md -- Remove paragraphs that are verbatim copies of CLAUDE.md sections -- Add `` cross-references where content was removed -- No content additions — strictly deduplication - -**Critical fix — `workflows.instructions.md`:** - -The file currently states (line 19): *"Never use `@latest`, `@main`, or non-existent versions -(`@v6`, `@v7`, `@v8`)."* This directly contradicts the actual `ci.yml`, which uses `@v6` and `@v7` -for checkout and setup-uv. The file is actively giving wrong guidance that would cause reviewers -to flag correct code. Update the pinned version table to match current workflow reality across all `.github/workflows/*.yml` files: - -| Action | Old (wrong) | New (correct) | -| ------ | ----------- | ------------- | -| `actions/checkout` | `@v4` | `@v6` | -| `actions/setup-python` | `@v5` | `@v6` | -| `actions/upload-artifact` | `@v4` | `@v7` | -| `actions/download-artifact` | `@v4` | `@v8` | -| `astral-sh/setup-uv` | `@v5` | `@v7` | -| `codecov/codecov-action` | `@v4` | `@v5` | - -Remove the "non-existent versions" prohibition line entirely — it is factually wrong and will -be wrong again as versions increment. - ---- - -## Section 4: Dependency & Runtime Audit Fixes - -*Findings from live PyPI and GitHub Actions audit on 2026-03-17.* - -### 4.1 Python 3.13 Upgrade - -- **Current:** Python 3.12 (`requires-python = ">=3.12"`, `target-version = "py312"`) -- **Target:** Python 3.13 (stable as of October 2024; 3.14 is pre-release — skip it) -- **Files to update:** - - `pyproject.toml`: `requires-python = ">=3.13"`, `[tool.ruff] target-version = "py313"`, - `[tool.mypy] python_version = "3.13"` - - `dex/.github/workflows/ci.yml`: `python-version: "3.13"` in all three jobs - - `Dockerfile` (if present): `FROM python:3.13-slim` - - `setup-workspace.sh`: no change needed — reads live `python3 --version` -- **Verify before committing:** run `uv run poe check-all` with Python 3.13 to confirm no - compatibility regressions. All current deps have 3.13 wheels on PyPI. - -### 4.2 Broken Version Bounds - -Two packages have minimum bounds that exceed the latest available version on PyPI — they will -fail to resolve in any environment that cannot satisfy the constraint. - -**`mkdocstrings[python]>=1.0.0`** — latest on PyPI is `0.29.1`: - -- Change to `>=0.29.0` -- This is a docs-only dev dependency; no runtime impact - -**`zensical>=0.0.24`** — latest on PyPI is `0.0.19`: - -- Investigate first: check if this is a private/internal package or if the version was yanked -- Run `uv lock` to see if uv can resolve it; if not, either drop the constraint to `>=0.0.19` - or remove the package if unused -- If `zensical` is an internal tool not on PyPI, document this explicitly in `pyproject.toml` - with a comment - -### 4.3 Outdated GitHub Actions - -| File | Action | Current | Latest | Change | -| ---- | ------ | ------- | ------ | ------ | -| `security.yml` | `github/codeql-action/*` | `@v3` | `@v4` | Update all occurrences | -| `pypi-publish.yml` | `actions/github-script` | `@v7` | `@v8` | Update all occurrences | -| `workflows.instructions.md` | `github/codeql-action/*` | `@v3` (documented) | `@v4` | Update table | -| `workflows.instructions.md` | `actions/github-script` | not in table | `@v8` | Add to table | - -### 4.4 Minor Dependency Bound Bumps - -Update lower bounds to match current latest (no breaking changes, strictly newer patches): - -- `poethepoet`: `>=0.42.0` → `>=0.42.1` -- `types-pyyaml`: `>=6.0.12.20250822` → `>=6.0.12.20250915` -- `databricks-cli`: unpinned → `>=0.18.0` (add minimum bound, consistent with project standards) - -After changes, run `uv lock` to regenerate `uv.lock`. - ---- - -## Out of Scope - -- `wiki-content/` — git submodule complexity, high blast radius, no functional benefit -- `dex/CLAUDE.md` — repo-specific, not part of this workspace-level optimization -- `pyproject.toml` / `ruff.toml` — correct and authoritative; being used as the source of truth - ---- - -## Implementation Order - -1. Housekeeping: delete `.bak` dirs, re-run `setup-workspace.sh`, add CI version-drift check, - remove `workflow_run` trigger AND all `workflow_run` condition references from `pypi-publish.yml`, - expand `dependabot.yml` -2. **Dependency fixes**: fix `mkdocstrings` and `zensical` bounds, bump `poethepoet`/`types-pyyaml`/ - `databricks-cli`, update `codeql-action@v4` and `github-script@v8` in workflows -3. **Python 3.13 upgrade**: update `pyproject.toml`, `ci.yml`; run `uv run poe check-all` to verify -4. `settings.json`: move non-credential permissions from `settings.local.json`, add `gh issue *` - and `gh release *`, remove `grep *` / `find *` / `cat *` / `head *` / `tail *` -5. Global `~/.claude/settings.json`: add read-only git ops and `rg *` -6. Commands: update `validate.md`, `pr.md`, `review.md`; add `release.md` - *(depends on step 4 for `gh release *` permission)* -7. `CLAUDE.md` restructure and trim -8. `copilot-instructions.md` trim -9. Agent consolidation: create `senior-engineer.agent.md`, delete 3 files -10. Instruction file deduplication (including `workflows.instructions.md` action version update) - ---- - -## Success Criteria - -- `workspace.env` version matches `pyproject.toml` after running `setup-workspace.sh` -- `pypi-publish.yml` has only `release:published` and `workflow_dispatch` triggers; no `workflow_run` - references anywhere in the file -- `release-dataenginex.yml` still exists and owns tag + release + SBOM -- No `.bak` directories at workspace root -- `settings.local.json` contains only: `env` block + `enableAllProjectMcpServers` + - `enabledMcpjsonServers` (no CLI permission entries) -- CLAUDE.md is ≥30% shorter with all rules intact (verified by human review of diff) -- `copilot-instructions.md` is ≤4KB -- Agent directory has 9 files: 8 domain-specific + `senior-engineer.agent.md` -- All commands reference correct author and branch conventions -- `release.md` command exists with explicit approval gate before commit -- `workflows.instructions.md` pinned version table matches current workflow reality across all - `.github/workflows/*.yml` files; "non-existent versions" prohibition line removed -- `pyproject.toml` uses `requires-python = ">=3.13"`, ruff `target-version = "py313"`, - mypy `python_version = "3.13"` -- `ci.yml` runs Python 3.13 -- `mkdocstrings[python]` bound is satisfiable (`>=0.29.0`) -- `zensical` bound resolved (either satisfiable or package removed with explanation) -- `databricks-cli` has version pin `>=0.18.0` -- `codeql-action` updated to `@v4` in `security.yml` -- `actions/github-script` updated to `@v8` in all workflows -- `uv lock` regenerated after all `pyproject.toml` changes -- `uv run poe check-all` passes on Python 3.13 diff --git a/instructions/data-pipelines.instructions.md b/instructions/data-pipelines.instructions.md index d470664..95e41b0 100644 --- a/instructions/data-pipelines.instructions.md +++ b/instructions/data-pipelines.instructions.md @@ -38,7 +38,7 @@ Every pipeline must be safe to re-run: ## Lineage -- Track column-level lineage via `LineageTracker` — every transform must be registered +- Track column-level lineage via `PersistentLineage` or `LineageBackend` — every transform must be registered - Never drop lineage events silently — log failures ## Performance diff --git a/instructions/fastapi.instructions.md b/instructions/fastapi.instructions.md index baa5aa6..b7098b6 100644 --- a/instructions/fastapi.instructions.md +++ b/instructions/fastapi.instructions.md @@ -22,7 +22,7 @@ Extends `python.instructions.md`. ## Auth - Protected endpoints check auth via lifespan middleware — not inline per endpoint -- Auth: pure-Python HS256 JWT (no `pyjwt` dependency) +- Auth: HS256 JWT (PyJWT dependency) - Return `401` for missing/invalid token, `403` for insufficient permissions - Never log the token or any credential diff --git a/instructions/ml.instructions.md b/instructions/ml.instructions.md index 491c7fa..7f27366 100644 --- a/instructions/ml.instructions.md +++ b/instructions/ml.instructions.md @@ -1,5 +1,5 @@ --- -applyTo: "src/**/ml/**/*.py,src/**/models/**/*.py,src/**/registry/**/*.py,src/**/drift/**/*.py" +applyTo: "src/**/ml/**/*.py,src/**/models/**/*.py,src/**/registry/**/*.py" --- # ML Standards diff --git a/instructions/workflows.instructions.md b/instructions/workflows.instructions.md index cdecd07..3c2a62c 100644 --- a/instructions/workflows.instructions.md +++ b/instructions/workflows.instructions.md @@ -10,12 +10,12 @@ applyTo: ".github/workflows/**/*.yml" | --- | --- | | `actions/checkout` | `@v6` | | `actions/setup-python` | `@v6` | -| `actions/upload-artifact` | `@v7` | -| `actions/download-artifact` | `@v8` | -| `astral-sh/setup-uv` | `@v7` | +| `actions/upload-artifact` | `@v4` | +| `actions/download-artifact` | `@v4` | +| `astral-sh/setup-uv` | `@v8.1.0` | | `codecov/codecov-action` | `@v5` | | `github/codeql-action/*` | `@v4` | -| `actions/github-script` | `@v8` | +| `actions/github-script` | `@v9` | Always pin to the current major tag. Verify against actual workflow files before documenting. @@ -44,7 +44,7 @@ test: ## Python setup (always use uv) ```yaml -- uses: astral-sh/setup-uv@v7 +- uses: astral-sh/setup-uv@v8.1.0 with: version: "latest" - uses: actions/setup-python@v6 diff --git a/learning/GUIDE.md b/learning/GUIDE.md deleted file mode 100644 index 6fecc25..0000000 --- a/learning/GUIDE.md +++ /dev/null @@ -1,21 +0,0 @@ -# Learning & Examples - -Python learning examples demonstrating various concepts and patterns. - -## Files - -- `async_http_client.py` - Async HTTP client patterns with FastAPI -- `context_managers.py` - Context manager patterns with database connections -- `strict_decorators.py` - Decorator patterns and strict validation -- `var_len_args.py` - Variable-length arguments (\*args, \*\*kwargs) -- `yield_keyword.py` - Generator and yield patterns - -## Purpose - -These files demonstrate Python concepts and can be used for: - -- Learning and reference -- Proof-of-concepts -- Quick prototyping - -They are not part of the main application code. diff --git a/learning/__init__.py b/learning/__init__.py deleted file mode 100644 index 975ef5e..0000000 --- a/learning/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Python learning examples and proof-of-concepts.""" diff --git a/learning/async_dependency_injection.py b/learning/async_dependency_injection.py deleted file mode 100644 index b45e265..0000000 --- a/learning/async_dependency_injection.py +++ /dev/null @@ -1,22 +0,0 @@ -# Use this pattern to manage the lifecycle of long-running asynchronous -# tasks in your application. - -import asyncio -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager - - -@asynccontextmanager -async def lifespan() -> AsyncIterator[None]: - print("startup") - yield - print("shutdown") - - -async def main(): - async with lifespan(): - print("Application is running...") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/learning/async_http_client.py b/learning/async_http_client.py deleted file mode 100644 index 422f35a..0000000 --- a/learning/async_http_client.py +++ /dev/null @@ -1,65 +0,0 @@ -import json -from io import StringIO - -import httpx -import pandas as pd -import uvicorn -from fastapi import FastAPI, HTTPException -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse - -""" -async def fetch(url: str) -> httpx.Response: - async with httpx.AsyncClient() as client: - return await client.get(url) - -async def fetch_stock_data(symbol: str) -> httpx.Response: - #url = f"https://stooq.com/q/l/?s={symbol.lower()}&f=sd2t2ohlcv&h&e=csv" - url = f"https://stooq.com/q/d/l/?s={symbol.lower()}.us&i=d" - return await fetch(url) - -async def main(): - url = "https://httpbin.org/get" - response = await fetch(url) - print("Response status code:", response.status_code) - print("Response JSON:", response.json()) - - # Get Top 5 oldest stocks data from stooq.com - stock_symbols = ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA"] - - tasks = [fetch_stock_data(symbol) for symbol in stock_symbols] - responses = await asyncio.gather(*tasks) - for response in responses: - print(response.text) - -if __name__ == "__main__": - asyncio.run(main()) -""" - -app = FastAPI() -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - - -@app.get("/stock/{symbol}") -async def get_stock_data(symbol: str): - url = f"https://stooq.com/q/d/l/?s={symbol.lower()}.us&i=d" - async with httpx.AsyncClient() as client: - try: - response = await client.get(url) - response.raise_for_status() - # format the response from csv to json - response = pd.read_csv(StringIO(response.text)) - json_response = response.to_json(orient="records") - return JSONResponse(content={"data": json.loads(json_response)}) - except httpx.HTTPError as e: - raise HTTPException(status_code=500, detail=str(e)) from e - - -if __name__ == "__main__": - uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/learning/context_managers.py b/learning/context_managers.py deleted file mode 100644 index 47907fb..0000000 --- a/learning/context_managers.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -from collections.abc import Iterator -from contextlib import contextmanager - -import pyodbc -from dotenv import load_dotenv - -load_dotenv() - - -def get_connection_string() -> str: - driver = os.getenv("DB_DRIVER") - server_name = os.getenv("DB_SERVER") - database_name = os.getenv("DB_NAME") - username = os.getenv("DB_USER") - password = os.getenv("DB_PASSWORD") - return ( - f"DRIVER={driver};" - f"SERVER={server_name};" - f"DATABASE={database_name};" - f"UID={username};" - f"PWD={password}" - ) - - -@contextmanager -def db_connection() -> Iterator[pyodbc.Connection]: - conn_str = get_connection_string() - conn = pyodbc.connect(conn_str) - try: - yield conn - finally: - conn.close() - - -if __name__ == "__main__": - with db_connection() as connection: - cursor = connection.cursor() - cursor.execute("SELECT 1") - row = cursor.fetchone() - print("Database connection test result:", row) diff --git a/learning/strict_decorators.py b/learning/strict_decorators.py deleted file mode 100644 index 28a65be..0000000 --- a/learning/strict_decorators.py +++ /dev/null @@ -1,27 +0,0 @@ -from collections.abc import Callable -from typing import ParamSpec, TypedDict, TypeVar - -P = ParamSpec("P") -R = TypeVar("R") - - -def log(fn: Callable[P, R]) -> Callable[P, R]: - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - print(f"Calling {fn.__name__}") - return fn(*args, **kwargs) - - return wrapper - - -class User(TypedDict): - id: int - - -@log -def get_user(user_id: int) -> User: - return {"id": user_id} - - -if __name__ == "__main__": - user = get_user(1) - print(user) diff --git a/learning/var_len_args.py b/learning/var_len_args.py deleted file mode 100644 index 1d5d6b2..0000000 --- a/learning/var_len_args.py +++ /dev/null @@ -1,7 +0,0 @@ -def my_function(*args, **kwargs): - print("Positional arguments:", args) - print("Keyword arguments:", kwargs) - - -if __name__ == "__main__": - my_function(1, 2, name="John", age=30) diff --git a/learning/yield_keyword.py b/learning/yield_keyword.py deleted file mode 100644 index b44833b..0000000 --- a/learning/yield_keyword.py +++ /dev/null @@ -1,20 +0,0 @@ -# yield is used when we want to return a generator object from a function -# on demand instead of all at once. -# A generator function allows us to iterate over a sequence of values -# without storing them all in memory. -# On demand, the next value is produced when requested using the next() -# function or a loop. - - -def count_numbers(n): - num = 1 - for _ in range(n): - yield num - num += 1 - - -if __name__ == "__main__": - # Using the generator function - numbers = count_numbers(5) - for number in numbers: - print(number) diff --git a/profile/README.md b/profile/README.md index 8af3f99..254d95b 100644 --- a/profile/README.md +++ b/profile/README.md @@ -2,10 +2,9 @@ Unified Data + ML + AI framework — config-driven, self-hosted, production-ready -[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/TheDataEngineX/DEX/blob/main/LICENSE) +[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/TheDataEngineX/dataenginex/blob/main/LICENSE) [![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) [![PyPI](https://img.shields.io/pypi/v/dataenginex?label=dataenginex)](https://pypi.org/project/dataenginex/) -[![Docs](https://img.shields.io/badge/docs-thedataenginex.org-informational)](https://docs.thedataenginex.org) [![Discussions](https://img.shields.io/github/discussions/TheDataEngineX/.github?label=discussions)](https://github.com/orgs/TheDataEngineX/discussions) --- @@ -15,7 +14,7 @@ Opinionated defaults that work out of the box, swap any layer via extras. > **Self-hosted.** Your data never leaves your infrastructure. > **Config-driven.** One YAML file defines everything. -> **Production-grade.** DuckDB, FastAPI, structlog, Prometheus, OpenTelemetry out of the box. +> **Local-first.** DuckDB embedded, Ollama for LLMs — no external services required. --- @@ -47,13 +46,11 @@ graph LR Extras -->|same interface| Core subgraph Products - B2B[DEX Studio\nB2B platform UI] - B2C[CareerDEX\nB2C career AI] + ST[DEX Studio\nWeb UI] INF[InfraDEX\nK3s · Helm · Terraform] end - B2B -->|HTTP :17000| Core - B2C -->|Python lib| Core + ST -->|HTTP :7860| Core INF -->|deploys| Core ``` @@ -63,10 +60,9 @@ graph LR | Repo | What it does | Status | | --- | --- | --- | -| [**DEX**](https://github.com/TheDataEngineX/DEX) | Core framework: config, CLI, FastAPI, ML registry, LLM routing (LiteLLM/vLLM), AI agents, RBAC, SCIM v2 | [![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/) | -| [**dex-studio**](https://github.com/TheDataEngineX/dex-studio) | B2B web UI: pipelines, ML experiments, AI playground, SQL console (Reflex/Python→React) | Alpha | -| [**careerdex**](https://github.com/TheDataEngineX/careerdex) | B2C career AI: job matching, resume analysis, interview prep, application tracking (Reflex) | Alpha | -| [**infradex**](https://github.com/TheDataEngineX/infradex) | IaC: K3s, Helm charts, Terraform — Authentik, Langfuse, Qdrant, Prometheus, Grafana, ArgoCD | Alpha | +| [**dataenginex**](https://github.com/TheDataEngineX/dataenginex) | Core framework: config, CLI, pipelines, ML registry, LLM routing, AI agents, PrivacyGuard | [![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/) | +| [**dex-studio**](https://github.com/TheDataEngineX/dex-studio) | Web UI: pipelines, ML experiments, AI playground, SQL console (FastAPI + Jinja2 + HTMX) | Alpha | +| [**infradex**](https://github.com/TheDataEngineX/infradex) | IaC: K3s, Helm charts, Terraform — Authentik, Prometheus, Grafana, ArgoCD | Alpha | --- @@ -80,17 +76,22 @@ dex serve --config dex.yaml # → http://localhost:17000 ```bash # From source -git clone https://github.com/TheDataEngineX/DEX && cd DEX +git clone https://github.com/TheDataEngineX/dataenginex && cd dataenginex uv sync && uv run poe dev # → http://localhost:17000 ``` **Optional extras:** ```bash -pip install "dataenginex[cloud]" # S3 · GCS · BigQuery connectors -pip install "dataenginex[auth]" # RS256/JWKS · SCIM v2 · LDAP sync -pip install "dataenginex[observability]" # Langfuse LLM tracing -pip install 'litellm>=1.83.3' --no-deps # 100+ LLM providers (separate install) +pip install "dataenginex[cloud]" # S3 · GCS · BigQuery connectors +pip install "dataenginex[ml]" # scikit-learn · sentence-transformers +pip install "dataenginex[delta]" # Delta Lake connector +pip install "dataenginex[postgres]" # asyncpg for Postgres lineage +pip install "dataenginex[qdrant]" # Qdrant vector store +pip install "dataenginex[queue]" # arq background jobs +pip install "dataenginex[pytorch]" # PyTorch ML +pip install "dataenginex[tracking]" # MLflow experiment tracking +pip install 'litellm>=1.83.3' --no-deps # 100+ LLM providers (separate install) ``` --- @@ -103,8 +104,8 @@ pip install 'litellm>=1.83.3' --no-deps # 100+ LLM providers (separate install) | **Install** | `pip install dataenginex` | 10+ packages to wire together | | **Backends** | Swap via extras, same interface | Rewrite integration code | | **Self-hosted** | Works on laptop, VPS, or K8s | Cloud lock-in or complex setup | -| **Observability** | structlog + Prometheus + OTEL + Langfuse | Manual instrumentation | -| **Enterprise** | RBAC, SCIM v2, LDAP, OIDC ready | Build it yourself | +| **Observability** | structlog + Prometheus + OTEL | Manual instrumentation | +| **Local-first** | DuckDB + Ollama defaults | External services required | --- @@ -117,7 +118,6 @@ pip install 'litellm>=1.83.3' --no-deps # 100+ LLM providers (separate install) | **Bug reports** | Open an issue in the relevant repo | | **Contributing** | [CONTRIBUTING.md](https://github.com/TheDataEngineX/.github/blob/main/CONTRIBUTING.md) | | **Security** | [SECURITY.md](https://github.com/TheDataEngineX/.github/blob/main/SECURITY.md) | -| **Website** | [thedataenginex.org](https://thedataenginex.org) | --- diff --git a/prompts/debug.prompt.md b/prompts/debug.prompt.md index 1c61529..58186c2 100644 --- a/prompts/debug.prompt.md +++ b/prompts/debug.prompt.md @@ -15,8 +15,7 @@ Debug the issue in or related to ${file}. - Type mismatches (check with `poe typecheck`) - Missing validation at boundaries 4. **Verify** — Run targeted tests: - - `poe test-unit` for unit tests - - `poe test-integration` for API/integration tests + - `poe test-cov-core` for core tests - Check lint/type errors: `poe lint` and `poe typecheck` 5. **Fix** — Apply the minimal change that resolves the issue - Follow project error patterns from `src/dataenginex/api/errors.py` diff --git a/prompts/new-feature.prompt.md b/prompts/new-feature.prompt.md index 73ac645..ed58632 100644 --- a/prompts/new-feature.prompt.md +++ b/prompts/new-feature.prompt.md @@ -33,7 +33,6 @@ Scaffold a new feature for the DataEngineX project. 5. **Validate**: - `poe check-all` — lint + typecheck + tests - - `poe test-cov` — verify 80%+ coverage maintained 6. **Commit** — Use conventional commit: `feat: (#issue)` diff --git a/prompts/pr-description.prompt.md b/prompts/pr-description.prompt.md index eb60d44..3ec9291 100644 --- a/prompts/pr-description.prompt.md +++ b/prompts/pr-description.prompt.md @@ -12,55 +12,30 @@ Generate a pull request description for the current changes using the project's 3. Fill in the PR template from `.github/PULL_REQUEST_TEMPLATE.md`: ```markdown -## Description +## Summary -**Related Issue**: Closes #XXX + -## Keep / Change / Remove +## Related Issues -- **Keep**: -- **Change**: -- **Remove**: + -## Type of Change +## Changes -- [ ] Bug fix (non-breaking change which fixes an issue) -- [ ] New feature (non-breaking change which adds functionality) -- [ ] Breaking change (fix or feature that would cause existing functionality to change) -- [ ] Documentation update -- [ ] Infrastructure/DevOps change - -## Changes Made - -- Change 1 -- Change 2 - -## Testing - -- [ ] Unit tests added/updated -- [ ] Integration tests added/updated -- [ ] Tests pass locally (`uv run poe test`) -- [ ] Coverage maintained/improved (target: 80%+) - -## Architecture Impact - -- [ ] Cloud-neutral by default (no hard dependency on a single provider) -- [ ] Cost-aware default path (works locally or free/open-source tier) -- [ ] Extension points documented for future adapters/plugins - -## Deprecation / Migration - -- [ ] No deprecation impact -- [ ] If behavior changed, deprecation/migration notes were added to docs/release notes +- [ ] New feature +- [ ] Bug fix +- [ ] Refactor +- [ ] Documentation +- [ ] CI/CD ## Checklist -- [ ] Linting passes (`uv run poe lint` ✓) -- [ ] Type checking passes (`uv run poe typecheck` ✓) +- [ ] Tests added/updated - [ ] Documentation updated (if applicable) -- [ ] Pre-commit hooks pass - -## Notes for Reviewers +- [ ] `uv run poe lint` passes +- [ ] `uv run poe typecheck` passes +- [ ] `uv run poe check-all` passes +- [ ] No breaking changes (or documented in description) ``` 4. Check the appropriate boxes based on actual changes diff --git a/prompts/write-tests.prompt.md b/prompts/write-tests.prompt.md index d73d706..058f44c 100644 --- a/prompts/write-tests.prompt.md +++ b/prompts/write-tests.prompt.md @@ -21,6 +21,6 @@ Write tests for the code in ${file}. - Happy path + error paths + edge cases (empty, None, boundary) - Mock external services only, not code under test - Use `tmp_path` for file operations -5. **Verify** — Run `poe test-unit` to confirm tests pass +5. **Verify** — Run `poe test-cov-core` to confirm tests pass Follow patterns in existing tests under `tests/unit/`. diff --git a/wiki-content/Architecture.md b/wiki-content/Architecture.md deleted file mode 100644 index 135b623..0000000 --- a/wiki-content/Architecture.md +++ /dev/null @@ -1,135 +0,0 @@ -# Architecture - -## Medallion Data Pipeline - -DEX implements the medallion (lakehouse) architecture as its core data processing pattern: - -``` -Raw Sources (APIs, files, streams) - ↓ - BRONZE LAYER — Raw ingestion (Parquet) - ↓ quality gate ≥ 75% - SILVER LAYER — Cleaned & validated - ↓ quality gate ≥ 90% - GOLD LAYER — Enriched & aggregated - ↓ - API / ML / Analytics -``` - -Each layer transition enforces a `QualityGate`. The gate is injectable — you can supply custom `scorer`, `required_fields`, and `uniqueness_key` arguments. - -```python -from dataenginex.core import MedallionArchitecture, QualityGate - -arch = MedallionArchitecture() -gate = QualityGate( - scorer=my_scorer, - required_fields=["id", "timestamp"], - uniqueness_key="id", -) -``` - -## Module Dependency Graph - -``` -dataenginex.core ← always available (pydantic, pyyaml, structlog) -dataenginex.data ← always available -dataenginex.lakehouse ← always available; cloud extras unlock backends -dataenginex.warehouse ← always available -dataenginex.ml ← always available -dataenginex.plugins ← always available - -dataenginex.api ← requires [api] extra (FastAPI, uvicorn, structlog, OpenTelemetry) -dataenginex.middleware ← requires [api] extra -dataenginex.dashboard ← requires [dashboard] extra (Streamlit) -``` - -Key design decision (v0.6.0): FastAPI is **optional**. The core install ships only lightweight deps. API/middleware consumers must opt in with `pip install dataenginex[api]`. - -## Optional Extras - -| Extra | Unlocks | Key Deps | -|-------|---------|----------| -| `[api]` | `dataenginex.api`, `dataenginex.middleware` | FastAPI, uvicorn, structlog, OpenTelemetry | -| `[s3]` | S3 storage backend | boto3 | -| `[gcs]` | GCS storage backend | google-cloud-storage | -| `[bq]` | BigQuery storage backend | google-cloud-bigquery | -| `[cloud]` | S3 + GCS backends | boto3, google-cloud-storage | -| `[ml]` | `SentenceTransformerEmbedder` | sentence-transformers | -| `[notebook]` | Jupyter utilities | ipykernel | -| `[dashboard]` | Streamlit dashboard | streamlit | -| `[all]` | Everything above | — | - -## Storage Backends - -Storage is accessed via a unified `StorageBackend` ABC and a `get_storage(uri)` factory: - -| URI Scheme | Backend | Extra Required | -|------------|---------|----------------| -| `file://` | `LocalParquetStorage` | — | -| `json://` | `JsonStorage` | — | -| `parquet://` | `ParquetStorage` | — | -| `s3://` | `S3Storage` | `[s3]` | -| `gs://` | `GCSStorage` | `[gcs]` | -| `bq://` | `BigQueryStorage` | `[bq]` | - -All backends implement `read()`, `write()`, `list_objects(prefix)`, and `exists(path)`. - -## API Architecture - -The API module provides reusable primitives and route definitions. Applications use `dataenginex.api` routers directly or define their own. - -Provided utilities: -- **Auth** — Pure-Python HS256 JWT (no pyjwt dependency) -- **Health checks** — `HealthChecker` for liveness/readiness probes -- **Pagination** — Cursor-based `paginate()` helper -- **Rate limiting** — Configurable middleware -- **Error handling** — Structured HTTP error responses - -## Tech Stack - -| Layer | Technology | -|-------|------------| -| Language | Python 3.13+ | -| Package Manager | uv + Hatchling | -| Web Framework | FastAPI + Uvicorn (optional `[api]`) | -| Orchestration | Apache Airflow | -| Big Data | PySpark | -| Code Quality | Ruff + mypy (strict) | -| Testing | pytest + coverage (94%) | -| Observability | Prometheus, Grafana, Jaeger (OpenTelemetry) | -| Containers | Docker (multi-stage, non-root) | -| Kubernetes | K3s + ArgoCD (GitOps) | -| CI/CD | GitHub Actions | - -## Project Structure - -``` -DEX/ -├── src/ -│ └── dataenginex/ -│ ├── api/ # FastAPI utilities (auth, health, pagination, rate limiting) -│ ├── core/ # Medallion architecture, validators, schemas -│ ├── data/ # Connectors, profiler, schema registry -│ ├── dashboard/ # Streamlit dashboard -│ ├── lakehouse/ # Catalog, partitioning, storage backends -│ ├── middleware/ # Structured logging, Prometheus metrics, tracing -│ ├── ml/ # Training, registry, serving, drift, LLM, RAG -│ ├── plugins/ # Plugin system (entry-point based discovery) -│ └── warehouse/ # SQL/Spark transforms, column-level lineage -├── examples/ # Runnable scripts (01–10) -├── tests/ -│ ├── unit/ # Unit tests -│ ├── integration/ # End-to-end tests (requires docker-compose.test.yml) -│ └── fixtures/ # Sample data -├── Dockerfile # Multi-stage, non-root, port 17000 -└── docker-compose.test.yml # S3 + GCS emulators for integration tests -``` - -## DEX Ecosystem Data Flow - -``` -Ingest → Process (Spark/Flink) → Lakehouse → Warehouse → Feature Store → Model Serving → AI Apps & Agents - ↑ - Terraform → K8s → GitOps (infradex) -``` diff --git a/wiki-content/Changelog.md b/wiki-content/Changelog.md deleted file mode 100644 index adb9865..0000000 --- a/wiki-content/Changelog.md +++ /dev/null @@ -1,50 +0,0 @@ -# Changelog - -See [CHANGELOG.md](https://github.com/TheDataEngineX/DEX/blob/main/CHANGELOG.md) for the full history. - -## Latest: v0.6.1 — 2026-03-15 - -### Added - -- **`SentenceTransformerEmbedder`** — thin wrapper over `sentence-transformers` (`all-MiniLM-L6-v2` default). Install via `uv add 'dataenginex[ml]'`. Implements the `embed_fn` protocol for `RAGPipeline`. -- **`RAGPipeline.answer(question, llm, ...)`** — full retrieve → augment → generate loop in one call. Combines `build_context` with any `LLMProvider.generate_with_context`. -- **GitHub Actions upgraded to Node.js 24** — `ci.yml`, `pypi-publish.yml`, `release-dataenginex.yml`, `security.yml` now use `actions/checkout@v6`, `actions/setup-python@v6`, `astral-sh/setup-uv@v7`. -- **`examples/05_rag_demo.py`** — end-to-end RAG demo with `--embed`, `--llm`, `--model` CLI flags; Ollama fallback to MockProvider; uses `RAGPipeline.answer()`. - ---- - -## v0.6.0 — 2026-03-03 - -### Breaking Changes - -- **Routers removed** — `api/routers/v1.py` and `api/routers/ml.py` consolidated into the dataenginex monorepo. `dataenginex` provides reusable API utilities (auth, health, errors, pagination, rate limiting). -- **FastAPI is now optional** — Core install (`pip install dataenginex`) includes only lightweight deps: `pydantic`, `pyyaml`, `structlog`, `httpx`, `python-dotenv`, `prometheus-client`. API/middleware consumers must install `pip install dataenginex[api]`. -- **Root `__init__.py` slimmed** — `from dataenginex import ...` no longer re-exports `HealthChecker`, `HealthStatus`, `configure_logging`, `configure_tracing`, `get_logger`. Use `from dataenginex.api import ...` or `from dataenginex.middleware import ...` directly. -- **Domain consolidation** — All domain schemas and validators consolidated into the dataenginex monorepo. Domain models live alongside the core library. - -### Added - -- **RAG Vector DB adapter** — `VectorStoreBackend` ABC with `InMemoryBackend` and `ChromaDBBackend` implementations; `RAGPipeline` orchestrator; `Document` and `SearchResult` dataclasses. -- **LLM integration** — `LLMProvider` ABC with `OllamaProvider` and `MockProvider`; `generate_with_context()` for RAG-style augmented generation; `ChatMessage`, `LLMConfig`, `LLMResponse` dataclasses. -- **Injectable `QualityGate`** — `QualityGate.__init__` now accepts `scorer`, `required_fields`, and `uniqueness_key` keyword arguments. -- **Real `LocalParquetStorage`** — Reads/writes actual Parquet files via `pyarrow`. -- **`BigQueryStorage` stubbed** — All methods raise `NotImplementedError` until implemented. - -### Fixed - -- **Pickle safety** — `ml/training.py` now uses `SafeUnpickler` restricting deserialization to sklearn/numpy namespaces only; HMAC signature verification on model load. -- **LLM error handling** — `OllamaProvider.generate()`/`chat()` raise `ConnectionError` on HTTP failures instead of returning empty `LLMResponse`. -- **Pagination cursor** — `decode_cursor()` raises `ValueError` on invalid input instead of silently returning 0. -- **Storage backends** — `S3Storage.exists()` catches `NoSuchKey` specifically; `GCSStorage.exists()` returns `blob.exists()` with specific exception handling. - ---- - -## v0.5.0 — 2026-03-01 - -### Added - -- **Storage abstraction** — `list_objects(prefix)` and `exists(path)` on `StorageBackend` ABC; concrete implementations across all backends; `get_storage(uri)` factory function. - ---- - -For earlier versions, see the [full CHANGELOG](https://github.com/TheDataEngineX/DEX/blob/main/CHANGELOG.md). diff --git a/wiki-content/Development.md b/wiki-content/Development.md deleted file mode 100644 index 986f87e..0000000 --- a/wiki-content/Development.md +++ /dev/null @@ -1,128 +0,0 @@ -# Development - -## Prerequisites - -- Python 3.13+ -- [uv](https://docs.astral.sh/uv/) package manager - -```bash -git clone https://github.com/TheDataEngineX/DEX && cd DEX -uv run poe setup # install deps + pre-commit hooks -``` - -## Task Runner - -All commands use [poethepoet](https://github.com/nat-n/poethepoet) via `uv run poe `. Task definitions live in `poe_tasks.toml`. - -## Quality Commands - -| Command | Description | -|---------|-------------| -| `uv run poe lint` | Ruff lint (rules: E, F, I, B, UP, SIM, C90) | -| `uv run poe lint-fix` | Ruff lint + auto-fix | -| `uv run poe typecheck` | mypy --strict on `src/dataenginex/` only | -| `uv run poe check-all` | lint + typecheck + test | - -## Test Commands - -| Command | Description | -|---------|-------------| -| `uv run poe test` | All tests | -| `uv run poe test-unit` | Unit tests only (`tests/unit/`) | -| `uv run poe test-integration` | Integration tests only (`tests/integration/`) | -| `uv run poe test-cov` | Tests with HTML coverage report | - -**Integration tests** require running storage emulators: - -```bash -docker compose -f docker-compose.test.yml up -d -uv run poe test-integration -docker compose -f docker-compose.test.yml down -``` - -## Run Commands - -| Command | Description | -|---------|-------------| -| `uv run poe dev` | Dev server (uvicorn reload, port 17000) | -| `uv run poe docker-up` | Docker Compose up | -| `uv run poe docker-down` | Docker Compose down | - -## Dependency Management - -| Command | Description | -|---------|-------------| -| `uv run poe uv-sync` | Sync deps from lockfile | -| `uv run poe uv-lock` | Regenerate lockfile | -| `uv run poe security` | Audit deps for vulnerabilities | - -The `uv.lock` file **must always be committed** — it is the reproducibility contract. - -## Mandatory Validation Pipeline - -Run in this exact order after any code change: - -```bash -# 1. Lint -uv run poe lint - -# 2. Type check -uv run poe typecheck - -# 3. Tests -uv run poe test - -# 4. Real server smoke test -uv run python examples/02_api_quickstart.py -# Then verify endpoints: -# curl http://localhost:17000/health -# curl http://localhost:17000/ -# curl http://localhost:17000/metrics - -# 5. Standalone import check -python -c "from dataenginex.core import MedallionArchitecture, QualityGate" -python -c "from dataenginex.ml import ModelRegistry, RAGPipeline" -``` - -Tests passing does not equal the app working. Step 4 is non-negotiable. - -## API Endpoints (Dev Server) - -| Endpoint | Method | Description | -|----------|--------|-------------| -| `/` | GET | Root | -| `/echo` | POST | Echo endpoint | -| `/health` | GET | Health check | -| `/metrics` | GET | Prometheus metrics | - -## Coding Standards - -- `from __future__ import annotations` in all source files -- Line length: 100 characters -- Max function complexity: 8 -- Functions: under 50 lines, max 4 parameters -- Type hints on all public functions -- `mypy --strict` must pass on `src/dataenginex/` - -### Logging - -- **All code**: `import structlog; logger = structlog.get_logger()` with `logger.info("event", key=value)` -- Never use `print()`, `loguru`, stdlib `logging`, or f-strings in log calls - -## Git Conventions - -- Branches: `main` (prod), `dev` (integration), `feature/`, `fix/` -- Commits: Conventional — `feat:`, `fix:`, `docs:`, `chore:`, `refactor:`, `test:` -- Reference issues: `feat: add drift detection (#42)` -- Deployment: `dev` → staging/dev cluster, `main` → production - -## Running Examples - -```bash -uv run python examples/01_core_demo.py -uv run python examples/02_api_quickstart.py -uv run python examples/05_rag_demo.py --embed --llm ollama --model llama3.2 -uv run python examples/08_spark_ml.py -``` - -Examples are numbered 01–10 and cover progressively more advanced features. diff --git a/wiki-content/Home.md b/wiki-content/Home.md deleted file mode 100644 index 767590e..0000000 --- a/wiki-content/Home.md +++ /dev/null @@ -1,120 +0,0 @@ -# DEX - -> Core framework for data engineering, ML, and observability. - -[![CI](https://github.com/TheDataEngineX/DEX/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/TheDataEngineX/DEX/actions/workflows/ci.yml) -[![PyPI](https://img.shields.io/pypi/v/dataenginex)](https://pypi.org/project/dataenginex/) -[![Python 3.13+](https://img.shields.io/badge/python-3.13+-blue.svg)](https://www.python.org/downloads/) -[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) -[![Coverage](https://img.shields.io/badge/coverage-94%25-brightgreen)](https://github.com/TheDataEngineX/DEX) - -## What is DEX? - -A production-focused Python framework for data engineering — medallion architecture, ML lifecycle management, and enterprise observability out of the box. - -`dataenginex` is the core library. It is the only published package — applications and ecosystem repos are built on top of it. - -## Quick Start - -```bash -# Install core (no web framework dependencies) -pip install dataenginex - -# With FastAPI, middleware, auth, health checks -pip install dataenginex[api] - -# With cloud storage backends -pip install dataenginex[s3] # AWS S3 via boto3 -pip install dataenginex[gcs] # Google Cloud Storage -pip install dataenginex[bq] # Google BigQuery -pip install dataenginex[cloud] # All cloud storage (S3 + GCS) - -# Everything -pip install dataenginex[all] -``` - -```bash -# Clone and develop -git clone https://github.com/TheDataEngineX/DEX && cd DEX -uv run poe setup # install deps + pre-commit hooks -uv run poe dev # dev server → http://localhost:17000 -uv run poe test # run tests -``` - -## Quick Usage - -```python -# Core — always available -from dataenginex.core import MedallionArchitecture, QualityGate -from dataenginex.data import SchemaRegistry -from dataenginex.ml import ModelRegistry - -# API — requires pip install dataenginex[api] -from dataenginex.api import HealthChecker, AuthMiddleware, paginate -from dataenginex.middleware import configure_logging, configure_tracing - -# Storage — requires the relevant extra -from dataenginex.lakehouse import JsonStorage, get_storage -storage = get_storage("file://./data") # always works -storage = get_storage("s3://my-bucket") # requires [s3] -storage = get_storage("gs://my-bucket") # requires [gcs] -storage = get_storage("bq://my-project/ds") # requires [bq] -``` - -## Module Overview - -| Module | Requires Extra | Description | -|--------|----------------|-------------| -| `dataenginex.core` | — | Medallion architecture, schemas, quality gates, validators | -| `dataenginex.data` | — | Schema registry, data contracts, catalog | -| `dataenginex.lakehouse` | optional `[s3]` `[gcs]` `[bq]` | Storage backends (JSON, Parquet, S3, GCS, BigQuery), catalog, partitioning | -| `dataenginex.warehouse` | — | Warehouse layers, lineage tracking | -| `dataenginex.ml` | — | Model registry, vectorstore, LLM adapters, drift detection | -| `dataenginex.api` | `[api]` | Auth (JWT), health checks, error handling, pagination, rate limiting | -| `dataenginex.middleware` | `[api]` | Structured logging, Prometheus metrics, OpenTelemetry tracing | - -## Plugin System - -Extend the framework by implementing `DataEngineXPlugin` and registering an entry point: - -```toml -# pyproject.toml -[project.entry-points."dataenginex.plugins"] -my_plugin = "my_package.plugin:MyPlugin" -``` - -```python -from dataenginex.plugins import discover, PluginRegistry - -plugins = discover() # auto-loads all installed plugins -registry = PluginRegistry() -for plugin in plugins: - registry.register(plugin) - -status = registry.health_check_all() -``` - -## Workspace - -Part of the [DataEngineX](https://github.com/TheDataEngineX) ecosystem: - -| Repo | Purpose | Port | -|------|---------|------| -| [DEX](https://github.com/TheDataEngineX/DEX) | Core framework (data + ML + AI) | 17000 | -| [dex-studio](https://github.com/TheDataEngineX/dex-studio) | Desktop UI | 7860 | -| [infradex](https://github.com/TheDataEngineX/infradex) | Infrastructure | — | - -> **Note:** datadex and agentdex have been consolidated into [DEX](https://github.com/TheDataEngineX/DEX). careerdex is a standalone B2C product at [github.com/TheDataEngineX/careerdex](https://github.com/TheDataEngineX/careerdex). - -Full observability stack (Prometheus + Grafana + Jaeger): see [infradex](https://github.com/TheDataEngineX/infradex) `docker-compose.monitoring.yml`. - -## Pages - -- [[Architecture]] — Medallion pipeline, module graph, extras -- [[ML-Guide]] — ModelRegistry, RAG, LLM providers, drift detection -- [[Development]] — Build, test, and run commands -- [[Changelog]] — Recent version history - ---- - -**License**: MIT | **Python**: 3.13+ diff --git a/wiki-content/ML-Guide.md b/wiki-content/ML-Guide.md deleted file mode 100644 index 294b0a8..0000000 --- a/wiki-content/ML-Guide.md +++ /dev/null @@ -1,172 +0,0 @@ -# ML Guide - -All ML functionality lives in `dataenginex.ml`. The core install includes the module — no extra required unless you use `SentenceTransformerEmbedder` (which needs `[ml]`). - -## ModelRegistry - -`ModelRegistry` provides a JSON-persisted model lifecycle manager with four stages: `development → staging → production → archived`. - -```python -from dataenginex.ml import ModelRegistry - -registry = ModelRegistry(storage_path="./model-registry.json") - -# Register a model -registry.register( - name="salary-predictor", - version="1.0.0", - artifact_path="./models/salary_predictor.pkl", - metadata={"framework": "xgboost", "accuracy": 0.91}, -) - -# Promote through lifecycle -registry.promote("salary-predictor", "1.0.0", target_stage="staging") -registry.promote("salary-predictor", "1.0.0", target_stage="production") - -# List models -models = registry.list_models() -``` - -Model serialization uses `SafeUnpickler` (restricts deserialization to sklearn/numpy namespaces) with HMAC signature verification on load. Set `DATAENGINEX_MODEL_HMAC_KEY` in your environment. - -## RAG Pipeline - -`RAGPipeline` orchestrates document ingestion and semantic retrieval. It wraps a `VectorStoreBackend` and an optional embedding function. - -### Vector Store Backends - -| Backend | Description | -|---------|-------------| -| `InMemoryBackend` | In-process vector store (dev/test) | -| `ChromaDBBackend` | Persistent ChromaDB store (production) | - -```python -from dataenginex.ml import RAGPipeline, InMemoryBackend - -backend = InMemoryBackend() -pipeline = RAGPipeline(vector_store=backend) - -# Ingest documents -pipeline.ingest([ - {"id": "doc1", "text": "Data engineering best practices..."}, - {"id": "doc2", "text": "Medallion architecture overview..."}, -]) - -# Retrieve relevant documents -results = pipeline.retrieve("How does medallion architecture work?", top_k=3) -``` - -### SentenceTransformerEmbedder (v0.6.1+) - -A thin wrapper over `sentence-transformers` using `all-MiniLM-L6-v2` by default. Implements the `embed_fn` protocol for `RAGPipeline`. - -```bash -pip install dataenginex[ml] -``` - -```python -from dataenginex.ml import RAGPipeline, InMemoryBackend, SentenceTransformerEmbedder - -embedder = SentenceTransformerEmbedder(model_name="all-MiniLM-L6-v2") -pipeline = RAGPipeline(vector_store=InMemoryBackend(), embed_fn=embedder) -``` - -### Full RAG Loop with RAGPipeline.answer() (v0.6.1+) - -`RAGPipeline.answer(question, llm, ...)` performs the complete retrieve → augment → generate loop in one call: - -```python -from dataenginex.ml import RAGPipeline, InMemoryBackend, OllamaProvider - -llm = OllamaProvider(model="llama3.2") -pipeline = RAGPipeline(vector_store=InMemoryBackend(), embed_fn=embedder) -pipeline.ingest(documents) - -answer = pipeline.answer( - question="What is the medallion architecture?", - llm=llm, - top_k=3, -) -print(answer) -``` - -See `examples/05_rag_demo.py` for a complete end-to-end demo with CLI flags (`--embed`, `--llm`, `--model`). - -## LLM Providers - -`LLMProvider` is an ABC with two concrete implementations: - -### OllamaProvider - -Connects to a local [Ollama](https://ollama.ai) instance via REST API. - -```python -from dataenginex.ml import OllamaProvider, LLMConfig - -llm = OllamaProvider( - config=LLMConfig( - model="llama3.2", - base_url="http://localhost:11434", - temperature=0.7, - max_tokens=512, - ) -) - -response = llm.generate("Summarize the medallion architecture in one paragraph.") -print(response.text) - -# RAG-style augmented generation -response = llm.generate_with_context( - question="What is data quality?", - context="Data quality measures completeness, freshness, and uniqueness...", -) -``` - -`OllamaProvider` raises `ConnectionError` on HTTP failures (no silent empty responses). - -### MockProvider - -For testing without a running LLM: - -```python -from dataenginex.ml import MockProvider - -llm = MockProvider(responses=["Mock answer 1", "Mock answer 2"]) -response = llm.generate("Any question") -``` - -### Key Dataclasses - -| Class | Purpose | -|-------|---------| -| `LLMConfig` | Model name, base URL, temperature, max tokens | -| `LLMResponse` | Generated text, token counts, model metadata | -| `ChatMessage` | Role + content for multi-turn conversations | - -## Drift Detector - -PSI (Population Stability Index) based drift detection: - -```python -from dataenginex.ml import DriftDetector - -detector = DriftDetector(psi_threshold=0.2) - -# Compare reference vs current distribution -report = detector.detect( - reference=reference_dataframe["feature"], - current=current_dataframe["feature"], -) - -if report.is_drifted: - print(f"Drift detected: PSI={report.psi_score:.4f}") -``` - -PSI interpretation: -- `< 0.1` — No drift -- `0.1–0.2` — Moderate drift, investigate -- `> 0.2` — Significant drift, retrain model - -## PySpark ML Pipelines - -For Spark-based ML, see `examples/08_spark_ml.py` in the repository. PySpark is an optional dependency — the core package does not require it. diff --git a/wiki-content/push-all-wikis.sh b/wiki-content/push-all-wikis.sh deleted file mode 100755 index 5cdca51..0000000 --- a/wiki-content/push-all-wikis.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -# Push pre-written wiki content to all DataEngineX GitHub wiki repos. -# PREREQUISITE: Visit each repo's /wiki tab in the browser and create -# any first page (saves the placeholder) before running this script. -# -# Usage: bash /home/jay/workspace/DataEngineX/.github/wiki-content/push-all-wikis.sh -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -GH_TOKEN=$(gh auth token) -BOT_EMAIL="github-actions[bot]@users.noreply.github.com" -BOT_NAME="github-actions[bot]" - -push_wiki() { - local repo=$1 - local src_dir="$2" - local wiki_dir="/tmp/push-wiki-${repo}" - - echo "--- Cloning ${repo} wiki ---" - rm -rf "$wiki_dir" - git clone "https://x-access-token:${GH_TOKEN}@github.com/TheDataEngineX/${repo}.wiki.git" "$wiki_dir" 2>&1 \ - || { echo "SKIP: ${repo} wiki not initialized (visit https://github.com/TheDataEngineX/${repo}/wiki)"; return; } - - cp -r "${src_dir}"/*.md "$wiki_dir/" - cd "$wiki_dir" - git config user.email "$BOT_EMAIL" - git config user.name "$BOT_NAME" - git add . - git diff --cached --quiet && { echo "OK: ${repo} (no changes)"; return; } - git commit -m "docs: update wiki content" - git push origin HEAD 2>&1 && echo "OK: ${repo}" || echo "FAILED: ${repo}" -} - -push_wiki dataenginex "${SCRIPT_DIR}" -push_wiki dex-studio "${SCRIPT_DIR}/wiki-dex-studio" -push_wiki infradex "${SCRIPT_DIR}/wiki-infradex" diff --git a/workflow-templates/docker-build.yml b/workflow-templates/docker-build.yml index 0017b4d..c5b3d9b 100644 --- a/workflow-templates/docker-build.yml +++ b/workflow-templates/docker-build.yml @@ -18,7 +18,7 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME: thedataenginex/${{ inputs.image-name }} + IMAGE_NAME: TheDataEngineX/${{ inputs.image-name }} jobs: build: diff --git a/workflow-templates/python-ci.yml b/workflow-templates/python-ci.yml index f278f77..767efdd 100644 --- a/workflow-templates/python-ci.yml +++ b/workflow-templates/python-ci.yml @@ -7,7 +7,7 @@ on: description: "Python version to use" required: false type: string - default: "3.12" + default: "3.13" package-path: description: "Path to source package" required: false diff --git a/workflow-templates/security-scan.yml b/workflow-templates/security-scan.yml index af4f24b..2a72665 100644 --- a/workflow-templates/security-scan.yml +++ b/workflow-templates/security-scan.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v6 - name: Run Trivy (SARIF) - uses: aquasecurity/trivy-action@0.28.0 + uses: aquasecurity/trivy-action@0.36.0 with: scan-type: fs scan-ref: . @@ -29,7 +29,7 @@ jobs: severity: CRITICAL,HIGH - name: Upload Trivy results to GitHub Security - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 if: always() with: sarif_file: trivy-results.sarif @@ -44,15 +44,15 @@ jobs: - uses: actions/checkout@v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 pip-audit: name: Dependency audit diff --git a/workflows/WORKFLOWS_GUIDE.md b/workflows/WORKFLOWS_GUIDE.md index fadbc9f..e611670 100644 --- a/workflows/WORKFLOWS_GUIDE.md +++ b/workflows/WORKFLOWS_GUIDE.md @@ -1,201 +1,60 @@ -# GitHub Actions Workflows - -This folder contains CI/CD automation for DEX. - -> **📖 Full Documentation**: See [docs/CI_CD.md](../../docs/CI_CD.md) for comprehensive CI/CD pipeline documentation. - -## Workflow Architecture - -```mermaid -graph TB - subgraph "Triggers" - PR[Pull Request] - PushDev[Push to dev] - PushMain[Push to main] - end - - subgraph "CI Workflows" - CI[ci.yml
Lint & Test] - Security[security.yml
CodeQL & Semgrep] - PkgVal[package-validation.yml
Build & Twine Check] - end - - subgraph "Release Workflows" - RelDEX[release-dataenginex.yml
DataEngineX Release] - PyPI[pypi-publish.yml
PyPI Publishing] - end - - subgraph "Outputs" - PyPIReg[PyPI Registry] - end - - PR --> CI - PR --> Security - PR --> PkgVal - PushDev --> CI - PushDev --> Security - PushMain --> CI - PushMain --> Security - PushMain --> PkgVal - - PushMain --> RelDEX - RelDEX --> PyPI - - PyPI --> PyPIReg - - style CI fill:#e1f5ff - style Security fill:#e1f5ff - style PkgVal fill:#e1f5ff - style RelDEX fill:#f8f5ff - style PyPI fill:#f8f5ff - style PyPIReg fill:#d4edda -``` - -## Workflows - -### `ci.yml` - Continuous Integration -**Triggers**: Push to `main`/`dev`, Pull Requests - -**Jobs**: -- **lint-and-test**: Runs Ruff, import checks, mypy, and pytest with coverage - -**Required for merge**: ✅ All checks must pass - ---- - -### `security.yml` - Security Scans -**Triggers**: Push to `main`/`dev`, Pull Requests to `main`/`dev` - -**Jobs**: -- **CodeQL**: Static analysis for vulnerabilities -- **Semgrep**: OWASP Top 10 checks - -**Results**: GitHub Security tab - ---- - -### `package-validation.yml` - Package Build Validation -**Triggers**: Changes to `src/dataenginex/**` or `pyproject.toml` on push/PR - -**Jobs**: -- Builds wheel distribution -- Runs `twine check` to validate PyPI metadata - ---- - -### `release-dataenginex.yml` - DataEngineX Release -**Triggers**: Version change in root `pyproject.toml` on `main` branch - -**What it does**: -1. Detects version bump in `pyproject.toml` -2. Creates git tag: `vX.Y.Z` -3. Creates GitHub release → **automatically triggers `pypi-publish.yml`** - ---- - -### `pypi-publish.yml` - PyPI Publishing -**Triggers**: GitHub release published (from `release-dataenginex.yml`) - -**What it does**: -1. Detects changes in `src/dataenginex/` since last tag -2. Builds wheel and publishes to TestPyPI (dry-run), then PyPI - -**Publish gates**: -- Only publishes if code actually changed -- Stable semver tags only for PyPI; pre-releases go to TestPyPI only - ---- - -### `docs-pages.yml` - Documentation Deployment -**Triggers**: Push to `main` on docs/MkDocs changes, manual dispatch - -**Jobs**: -- **build**: Builds MkDocs site and uploads Pages artifact -- **deploy**: Publishes site to GitHub Pages - -**Custom Domain**: `docs.thedataenginex.org` - ---- - -### `project-automation.yml` - Project Intake Automation -**Triggers**: Issue/PR opened/reopened events, manual dispatch - -**Jobs**: -- **add-to-org-project**: Adds new issues/PRs to org project board - -**Configuration Required**: -- Variable: `ORG_PROJECT_URL` -- Secret: `ORG_PROJECT_TOKEN` - ---- - -## Quick Reference - -```bash -# Check CI status -gh pr checks - -# View workflow runs -gh run list --workflow ci.yml - -# View logs -gh run view --log - -# Trigger manual PyPI publish -gh workflow run pypi-publish.yml -f tag=v0.6.0 -``` - ---- - -## Required Secrets - -**Repository Secrets**: -- `GITHUB_TOKEN` - Auto-provided by GitHub Actions -- `ORG_PROJECT_TOKEN` - Required for project automation auto-add - -**Repository Variables**: -- `ORG_PROJECT_URL` - Required for project automation auto-add - ---- - -## Troubleshooting - -```mermaid -graph TD - Issue[CI/CD Issue] --> Type{Issue Type?} - - Type -->|CI Failure| CheckCI[Check CI logs] - CheckCI --> LocalTest["Run lint locally"] - LocalTest --> FixCode[Fix code issues] - FixCode --> Push[Push changes] - - Type -->|PyPI Publish Failure| CheckRelease["Check release-dataenginex logs"] - CheckRelease --> CheckTag[Verify tag format vX.Y.Z] - CheckTag --> CheckPyPI[Check pypi-publish.yml logs] - - Push --> End[✓ Resolved] - CheckPyPI --> End - - style Issue fill:#f8d7da - style End fill:#d4edda -``` - -### CI Failures -```bash -# Run lint locally -uv run poe lint -uv run poe test -``` - -### PyPI Publish Not Triggering -- Verify release tag format: `vMAJOR.MINOR.PATCH` -- Check that `release-dataenginex.yml` created the GitHub release -- Confirm `src/dataenginex/` files changed since last tag - ---- - -## Related Documentation - -- **[CI/CD Pipeline Guide](../../docs/CI_CD.md)** - Complete pipeline documentation -- **[SDLC](../../docs/SDLC.md)** - Development lifecycle -- **[Deploy Runbook](../../docs/DEPLOY_RUNBOOK.md)** - Release procedures +# GitHub Actions Workflows + +CI/CD automation for the DataEngineX `.github` meta-repo. +Reusable workflows shared across all repos are also in this directory (prefixed `reusable-`). + +## Workflows (run in `.github`) + +### `security.yml` — Security Scans +**Triggers**: Push/PR to `main`/`dev` + +Runs Trivy (misconfig + secret scan). CodeQL is handled by GitHub's default setup. + +### `sync-labels.yml` — Label Synchronization +**Triggers**: Push to `main` on `labels.yml` changes, manual dispatch + +Syncs labels from `labels.yml` to all repos (dataenginex, dex-studio, infradex). + +### `project-automation.yml` — Project Intake Automation +**Triggers**: Issue/PR opened/reopened events, manual dispatch + +Adds new issues/PRs to org project board. Requires `ORG_PROJECT_URL` variable + `ORG_PROJECT_TOKEN` secret. + +### `stale-issues.yml` — Stale Issue Management +**Triggers**: Scheduled (daily at 6 AM UTC) + +Marks issues/PRs stale after 60 days, closes after 30 more. + +## Reusable Workflows (called by other repos) + +| Workflow | Callers | Purpose | +|----------|---------|---------| +| `reusable-security.yml` | dataenginex, dex-studio, infradex | Trivy scan + Slack notifications | +| `reusable-auto-pr-to-main.yml` | dataenginex, dex-studio | Auto-create PR from feature/fix → main | +| `reusable-auto-pr-to-dev.yml` | (available) | Auto-create PR from feature/fix → dev + auto-merge | +| `reusable-auto-pr-dev-to-main.yml` | (available) | Auto-create PR from dev → main | +| `reusable-enforce-dev-to-main.yml` | (available) | Verify PRs to main come from dev | +| `reusable-claude.yml` | (available) | Claude Code action for @claude mentions | +| `reusable-release-please.yml` | (available) | Release please automation | + +## Quick Reference + +```bash +# Check CI status +gh pr checks + +# View workflow runs +gh run list --workflow security.yml + +# Trigger manual label sync +gh workflow run sync-labels.yml +``` + +## Required Secrets + +| Secret | Used by | +|--------|---------| +| `GITHUB_TOKEN` | Auto-provided | +| `ORG_PROJECT_TOKEN` | project-automation.yml | +| `LABELS_SYNC_TOKEN` | sync-labels.yml | +| `SLACK_WEBHOOK` | security.yml (optional) | diff --git a/.github/workflows/auto-pr-dev-to-main.yml b/workflows/reusable-auto-pr-dev-to-main.yml similarity index 100% rename from .github/workflows/auto-pr-dev-to-main.yml rename to workflows/reusable-auto-pr-dev-to-main.yml diff --git a/.github/workflows/auto-pr-to-dev.yml b/workflows/reusable-auto-pr-to-dev.yml similarity index 100% rename from .github/workflows/auto-pr-to-dev.yml rename to workflows/reusable-auto-pr-to-dev.yml diff --git a/.github/workflows/auto-pr-to-main.yml b/workflows/reusable-auto-pr-to-main.yml similarity index 100% rename from .github/workflows/auto-pr-to-main.yml rename to workflows/reusable-auto-pr-to-main.yml diff --git a/.github/workflows/claude.yml b/workflows/reusable-claude.yml similarity index 100% rename from .github/workflows/claude.yml rename to workflows/reusable-claude.yml diff --git a/.github/workflows/enforce-dev-to-main.yml b/workflows/reusable-enforce-dev-to-main.yml similarity index 100% rename from .github/workflows/enforce-dev-to-main.yml rename to workflows/reusable-enforce-dev-to-main.yml diff --git a/.github/workflows/release-please.yml b/workflows/reusable-release-please.yml similarity index 97% rename from .github/workflows/release-please.yml rename to workflows/reusable-release-please.yml index 5d99bb2..d2a129a 100644 --- a/.github/workflows/release-please.yml +++ b/workflows/reusable-release-please.yml @@ -17,7 +17,7 @@ # Callers need only: # jobs: # release: -# uses: TheDataEngineX/.github/.github/workflows/release-please.yml@main +# uses: TheDataEngineX/.github/workflows/release-please.yml@main # secrets: inherit name: Release Please (reusable) diff --git a/.github/workflows/security.yml b/workflows/reusable-security.yml similarity index 100% rename from .github/workflows/security.yml rename to workflows/reusable-security.yml diff --git a/workflows/security.yml b/workflows/security.yml index 7c5ac3b..b6d8eaf 100644 --- a/workflows/security.yml +++ b/workflows/security.yml @@ -1,6 +1,6 @@ -# Security Scans -# Runs Trivy (misconfig + secret scan) and CodeQL (static analysis) on push/PR. -# Results appear in GitHub Security tab. +# Security Scans for the .github meta-repo. +# Runs Trivy (misconfig + secret scan) on push/PR. +# CodeQL is handled by GitHub's default setup — do NOT add a custom codeql job. name: Security Scans @@ -37,7 +37,7 @@ jobs: type: 'section', text: { type: 'mrkdwn', - text: `*🔒 Security Scans Started*\n*Branch:* ${process.env.REF}\n*Tools:* Trivy (repo), CodeQL` + text: `*🔒 Security Scans Started*\n*Branch:* ${process.env.REF}\n*Tools:* Trivy (repo)` } }] }) @@ -49,7 +49,7 @@ jobs: - uses: actions/checkout@v6 - name: Run Trivy repo security scan - uses: aquasecurity/trivy-action@0.36.0 + uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: fs scan-ref: . @@ -66,7 +66,7 @@ jobs: sarif_file: trivy-repo-results.sarif - name: Enforce Trivy misconfig gate (HIGH/CRITICAL) - uses: aquasecurity/trivy-action@0.36.0 + uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: fs scan-ref: . @@ -75,32 +75,17 @@ jobs: severity: HIGH,CRITICAL exit-code: '1' - codeql: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - language: [ python, actions ] - steps: - - uses: actions/checkout@v6 - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: ${{ matrix.language }} - - name: Perform CodeQL Scan - uses: github/codeql-action/analyze@v4 - with: - category: /language:${{ matrix.language }} - notify-security: name: Notify Security Status runs-on: ubuntu-latest - needs: [trivy-security, codeql] + needs: [trivy-security] if: always() + permissions: + contents: read steps: - name: Notify Security Success - if: ${{ needs.trivy-security.result == 'success' && needs.codeql.result == 'success' }} + if: ${{ needs.trivy-security.result == 'success' }} uses: actions/github-script@v9 with: script: | @@ -123,8 +108,7 @@ jobs: type: 'section', fields: [ { type: 'mrkdwn', text: `*Branch:*\n${process.env.REF}` }, - { type: 'mrkdwn', text: `*Trivy (repo):*\n✅ Passed` }, - { type: 'mrkdwn', text: `*CodeQL:*\n✅ Passed` }, + { type: 'mrkdwn', text: `*Trivy:*\n✅ Passed` }, { type: 'mrkdwn', text: `*Run:*\n<${process.env.URL}|View Details>` } ] } @@ -138,7 +122,7 @@ jobs: URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - name: Notify Security Issues - if: ${{ needs.trivy-security.result == 'failure' || needs.codeql.result == 'failure' }} + if: ${{ needs.trivy-security.result == 'failure' }} uses: actions/github-script@v9 with: script: | @@ -170,7 +154,7 @@ jobs: }, { type: 'context', - elements: [{ type: 'mrkdwn', text: `🔍 Trivy or CodeQL found issues — review security tab for findings` }] + elements: [{ type: 'mrkdwn', text: `🔍 Trivy found issues — review security tab for findings` }] } ] }) diff --git a/workflows/sync-labels.yml b/workflows/sync-labels.yml index 6044a34..e1cd7e1 100644 --- a/workflows/sync-labels.yml +++ b/workflows/sync-labels.yml @@ -18,17 +18,21 @@ jobs: fail-fast: false matrix: repo: - - DEX + - dataenginex - dex-studio - infradex steps: - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v8.1.0 + with: + version: "latest" + - name: Sync labels to ${{ matrix.repo }} env: GH_TOKEN: ${{ secrets.LABELS_SYNC_TOKEN }} run: | - pip install pyyaml -q + uv pip install pyyaml -q python3 - <<'PYEOF' import json, subprocess, yaml with open('labels.yml') as f: From 8fff3ba069bfa596d2b9b2e85b541b12b3619f5c Mon Sep 17 00:00:00 2001 From: jaymyaka Date: Tue, 21 Jul 2026 12:57:14 -0500 Subject: [PATCH 4/9] Merge main: keep .github/workflows/ as canonical location, restore our versions with Slack/conventional commits --- .github/workflows/WORKFLOWS_GUIDE.md | 60 +++++++ .github/workflows/project-automation.yml | 49 +++++ .../reusable-auto-pr-dev-to-main.yml | 50 ++++++ .github/workflows/reusable-auto-pr-to-dev.yml | 77 ++++++++ .../workflows/reusable-auto-pr-to-main.yml | 60 +++++-- .github/workflows/reusable-claude.yml | 35 ++++ .../reusable-enforce-dev-to-main.yml | 24 +++ .github/workflows/reusable-release-please.yml | 72 ++++++++ .github/workflows/reusable-security.yml | 159 +++++++++++++++- .github/workflows/security.yml | 169 ++++++++++++++++++ .github/workflows/stale-issues.yml | 33 ++++ .github/workflows/sync-labels.yml | 53 ++++++ 12 files changed, 818 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/WORKFLOWS_GUIDE.md create mode 100644 .github/workflows/project-automation.yml create mode 100644 .github/workflows/reusable-auto-pr-dev-to-main.yml create mode 100644 .github/workflows/reusable-auto-pr-to-dev.yml create mode 100644 .github/workflows/reusable-claude.yml create mode 100644 .github/workflows/reusable-enforce-dev-to-main.yml create mode 100644 .github/workflows/reusable-release-please.yml create mode 100644 .github/workflows/security.yml create mode 100644 .github/workflows/stale-issues.yml create mode 100644 .github/workflows/sync-labels.yml diff --git a/.github/workflows/WORKFLOWS_GUIDE.md b/.github/workflows/WORKFLOWS_GUIDE.md new file mode 100644 index 0000000..e611670 --- /dev/null +++ b/.github/workflows/WORKFLOWS_GUIDE.md @@ -0,0 +1,60 @@ +# GitHub Actions Workflows + +CI/CD automation for the DataEngineX `.github` meta-repo. +Reusable workflows shared across all repos are also in this directory (prefixed `reusable-`). + +## Workflows (run in `.github`) + +### `security.yml` — Security Scans +**Triggers**: Push/PR to `main`/`dev` + +Runs Trivy (misconfig + secret scan). CodeQL is handled by GitHub's default setup. + +### `sync-labels.yml` — Label Synchronization +**Triggers**: Push to `main` on `labels.yml` changes, manual dispatch + +Syncs labels from `labels.yml` to all repos (dataenginex, dex-studio, infradex). + +### `project-automation.yml` — Project Intake Automation +**Triggers**: Issue/PR opened/reopened events, manual dispatch + +Adds new issues/PRs to org project board. Requires `ORG_PROJECT_URL` variable + `ORG_PROJECT_TOKEN` secret. + +### `stale-issues.yml` — Stale Issue Management +**Triggers**: Scheduled (daily at 6 AM UTC) + +Marks issues/PRs stale after 60 days, closes after 30 more. + +## Reusable Workflows (called by other repos) + +| Workflow | Callers | Purpose | +|----------|---------|---------| +| `reusable-security.yml` | dataenginex, dex-studio, infradex | Trivy scan + Slack notifications | +| `reusable-auto-pr-to-main.yml` | dataenginex, dex-studio | Auto-create PR from feature/fix → main | +| `reusable-auto-pr-to-dev.yml` | (available) | Auto-create PR from feature/fix → dev + auto-merge | +| `reusable-auto-pr-dev-to-main.yml` | (available) | Auto-create PR from dev → main | +| `reusable-enforce-dev-to-main.yml` | (available) | Verify PRs to main come from dev | +| `reusable-claude.yml` | (available) | Claude Code action for @claude mentions | +| `reusable-release-please.yml` | (available) | Release please automation | + +## Quick Reference + +```bash +# Check CI status +gh pr checks + +# View workflow runs +gh run list --workflow security.yml + +# Trigger manual label sync +gh workflow run sync-labels.yml +``` + +## Required Secrets + +| Secret | Used by | +|--------|---------| +| `GITHUB_TOKEN` | Auto-provided | +| `ORG_PROJECT_TOKEN` | project-automation.yml | +| `LABELS_SYNC_TOKEN` | sync-labels.yml | +| `SLACK_WEBHOOK` | security.yml (optional) | diff --git a/.github/workflows/project-automation.yml b/.github/workflows/project-automation.yml new file mode 100644 index 0000000..15c6433 --- /dev/null +++ b/.github/workflows/project-automation.yml @@ -0,0 +1,49 @@ +name: Project Automation + +on: + issues: + types: + - opened + - reopened + - transferred + pull_request: + types: + - opened + - reopened + - ready_for_review + workflow_dispatch: + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + add-to-org-project: + name: Add item to organization project + runs-on: ubuntu-latest + env: + ORG_PROJECT_URL: ${{ vars.ORG_PROJECT_URL }} + ORG_PROJECT_TOKEN: ${{ secrets.ORG_PROJECT_TOKEN }} + + steps: + - name: Add issue/PR to organization project + if: ${{ env.ORG_PROJECT_URL != '' && env.ORG_PROJECT_TOKEN != '' }} + id: add_to_project + continue-on-error: true + uses: actions/add-to-project@v1.0.2 + with: + project-url: ${{ env.ORG_PROJECT_URL }} + github-token: ${{ env.ORG_PROJECT_TOKEN }} + + - name: Explain action failure + if: ${{ steps.add_to_project.outcome == 'failure' }} + run: | + echo "Project automation failed but is non-blocking." + echo "Check ORG_PROJECT_URL and ORG_PROJECT_TOKEN scopes/permissions." + + - name: Explain skipped automation + if: ${{ env.ORG_PROJECT_URL == '' || env.ORG_PROJECT_TOKEN == '' }} + run: | + echo "Project automation skipped." + echo "Set repository/org variable ORG_PROJECT_URL and secret ORG_PROJECT_TOKEN to enable." diff --git a/.github/workflows/reusable-auto-pr-dev-to-main.yml b/.github/workflows/reusable-auto-pr-dev-to-main.yml new file mode 100644 index 0000000..ae08d8c --- /dev/null +++ b/.github/workflows/reusable-auto-pr-dev-to-main.yml @@ -0,0 +1,50 @@ +name: Auto PR — dev → main (reusable) + +on: + workflow_call: + +jobs: + create-pr: + name: Create PR to main + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Create or update PR (dev → main) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + run: | + AHEAD=$(gh api "repos/$GH_REPO/compare/main...dev" --jq '.ahead_by') + if [ "$AHEAD" = "0" ]; then + echo "dev is already in sync with main — nothing to do" + exit 0 + fi + echo "dev is $AHEAD commit(s) ahead of main" + + PR_NUMBER=$(gh pr list \ + --base main \ + --head dev \ + --state open \ + --json number \ + --jq '.[0].number') + + if [ -n "$PR_NUMBER" ]; then + echo "PR #$PR_NUMBER already exists" + else + gh pr create \ + --base main \ + --head dev \ + --title "chore: sync dev -> main" \ + --body "Automated PR: dev → main. Merge when all required checks pass." + + PR_NUMBER=$(gh pr list \ + --base main \ + --head dev \ + --state open \ + --json number \ + --jq '.[0].number') + echo "Created PR #$PR_NUMBER" + fi diff --git a/.github/workflows/reusable-auto-pr-to-dev.yml b/.github/workflows/reusable-auto-pr-to-dev.yml new file mode 100644 index 0000000..3a118d3 --- /dev/null +++ b/.github/workflows/reusable-auto-pr-to-dev.yml @@ -0,0 +1,77 @@ +# Reusable workflow — auto-creates a PR from a feature/fix branch → dev and enables +# auto-merge. The squash commit title is derived from the branch name in conventional +# commit format (feature/feat-add-foo → "feat: add foo"). +# +# Prerequisites (one-time per repo, GitHub UI): +# 1. Settings → General → Allow auto-merge ✓ +# 2. Settings → Branches → branch protection on 'dev': +# - Require status checks: CI (lint-and-test) +# - Require branches to be up to date before merging + +name: Auto PR — feature → dev (reusable) + +on: + workflow_call: + +jobs: + create-pr: + name: Create PR to dev + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Create or update PR (feature/fix → dev) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + HEAD: ${{ github.ref_name }} + run: | + # Derive conventional commit title from branch name + # feature/feat-add-pipeline → "feat: add pipeline" + # feature/add-pipeline → "feat: add pipeline" + # fix/fix-null-ptr → "fix: null ptr" + # fix/null-ptr → "fix: null ptr" + BRANCH_DESC=$(echo "$HEAD" | sed 's|^feature/||;s|^fix/||') + if echo "$BRANCH_DESC" | grep -qE '^feat-'; then + TITLE="feat: $(echo "$BRANCH_DESC" | sed 's|^feat-||;s|-| |g')" + elif echo "$BRANCH_DESC" | grep -qE '^fix-'; then + TITLE="fix: $(echo "$BRANCH_DESC" | sed 's|^fix-||;s|-| |g')" + elif echo "$HEAD" | grep -qE '^fix/'; then + TITLE="fix: $(echo "$BRANCH_DESC" | sed 's|-| |g')" + else + TITLE="feat: $(echo "$BRANCH_DESC" | sed 's|-| |g')" + fi + + echo "PR title: $TITLE" + + # Check for existing open PR + PR_NUMBER=$(gh pr list \ + --base dev \ + --head "$HEAD" \ + --state open \ + --json number \ + --jq '.[0].number') + + if [ -n "$PR_NUMBER" ]; then + echo "PR #$PR_NUMBER already exists" + else + BODY="Automated PR: ${HEAD} -> dev. Auto-merge enabled - squash-merges when all required checks pass. Edit the PR title to a conventional commit format if needed." + gh pr create \ + --base dev \ + --head "$HEAD" \ + --title "$TITLE" \ + --body "$BODY" + + PR_NUMBER=$(gh pr list \ + --base dev \ + --head "$HEAD" \ + --state open \ + --json number \ + --jq '.[0].number') + echo "Created PR #$PR_NUMBER" + fi + + gh pr merge "$PR_NUMBER" --auto --squash + echo "Auto-merge (squash) enabled on PR #$PR_NUMBER" diff --git a/.github/workflows/reusable-auto-pr-to-main.yml b/.github/workflows/reusable-auto-pr-to-main.yml index 290e392..f5689f0 100644 --- a/.github/workflows/reusable-auto-pr-to-main.yml +++ b/.github/workflows/reusable-auto-pr-to-main.yml @@ -1,36 +1,62 @@ -name: Reusable Auto PR to Main +name: Auto PR — feature → main (reusable) on: workflow_call: jobs: - auto-pr: + create-pr: name: Create PR to main runs-on: ubuntu-latest permissions: contents: write pull-requests: write - steps: - - uses: actions/checkout@v4 - - name: Create or update PR to main + steps: + - name: Create or update PR (feature/fix → main) env: - GH_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + HEAD: ${{ github.ref_name }} run: | - BRANCH="${{ github.ref_name }}" - REPO="${{ github.repository }}" + # Derive conventional commit title from branch name + # feature/feat-add-pipeline → "feat: add pipeline" + # feature/add-pipeline → "feat: add pipeline" + # fix/fix-null-ptr → "fix: null ptr" + BRANCH_DESC=$(echo "$HEAD" | sed 's|^feature/||;s|^fix/||') + if echo "$BRANCH_DESC" | grep -qE '^feat-'; then + TITLE="feat: $(echo "$BRANCH_DESC" | sed 's|^feat-||;s|-| |g')" + elif echo "$BRANCH_DESC" | grep -qE '^fix-'; then + TITLE="fix: $(echo "$BRANCH_DESC" | sed 's|^fix-||;s|-| |g')" + elif echo "$HEAD" | grep -qE '^fix/'; then + TITLE="fix: $(echo "$BRANCH_DESC" | sed 's|-| |g')" + else + TITLE="feat: $(echo "$BRANCH_DESC" | sed 's|-| |g')" + fi - # Check if PR already exists - EXISTING=$(gh pr list --repo "$REPO" --head "$BRANCH" --base main --json number --jq '.[0].number') + echo "PR title: $TITLE" - if [ -n "$EXISTING" ]; then - echo "PR #$EXISTING already exists for $BRANCH → main" + PR_NUMBER=$(gh pr list \ + --base main \ + --head "$HEAD" \ + --state open \ + --json number \ + --jq '.[0].number') + + if [ -n "$PR_NUMBER" ]; then + echo "PR #$PR_NUMBER already exists" else + BODY="Automated PR: ${HEAD} → main. Review the preview environment before merging." gh pr create \ - --repo "$REPO" \ - --head "$BRANCH" \ --base main \ - --title "chore: merge \`$BRANCH\` → main" \ - --body "Automated PR: merging \`$BRANCH\` into \`main\`." \ - --label "automated-pr" || true + --head "$HEAD" \ + --title "$TITLE" \ + --body "$BODY" + + PR_NUMBER=$(gh pr list \ + --base main \ + --head "$HEAD" \ + --state open \ + --json number \ + --jq '.[0].number') + echo "Created PR #$PR_NUMBER" fi diff --git a/.github/workflows/reusable-claude.yml b/.github/workflows/reusable-claude.yml new file mode 100644 index 0000000..41a1b71 --- /dev/null +++ b/.github/workflows/reusable-claude.yml @@ -0,0 +1,35 @@ +# Reusable workflow — called from each repo's .github/workflows/claude.yml +# Logic lives here once; repo files are thin callers. + +name: Claude Code (reusable) + +on: + workflow_call: + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + additional_permissions: | + actions: read diff --git a/.github/workflows/reusable-enforce-dev-to-main.yml b/.github/workflows/reusable-enforce-dev-to-main.yml new file mode 100644 index 0000000..d5f4741 --- /dev/null +++ b/.github/workflows/reusable-enforce-dev-to-main.yml @@ -0,0 +1,24 @@ +# Reusable workflow — called from each repo's .github/workflows/enforce-dev-to-main.yml +# Logic lives here once; repo files are thin callers. + +name: Enforce dev-to-main (reusable) + +on: + workflow_call: + +jobs: + check-source-branch: + name: Verify PR source is dev + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Check source branch + run: | + branch="${{ github.head_ref }}" + if [ "$branch" != "dev" ] && [[ "$branch" != release-please--* ]]; then + echo "PRs to main must come from 'dev' or a release-please branch." + echo "Source branch: $branch" + exit 1 + fi + echo "Source branch is '$branch' — OK" diff --git a/.github/workflows/reusable-release-please.yml b/.github/workflows/reusable-release-please.yml new file mode 100644 index 0000000..d2a129a --- /dev/null +++ b/.github/workflows/reusable-release-please.yml @@ -0,0 +1,72 @@ +# Reusable workflow — called from each repo's .github/workflows/release-please.yml +# +# On push to main, release-please: +# 1. Reads conventional commits since the last release tag +# 2. Creates/updates a Release PR that bumps pyproject.toml version + CHANGELOG.md +# 3. When the Release PR is merged → creates git tag + GitHub Release +# +# Uses RELEASE_TOKEN (PAT) instead of GITHUB_TOKEN so the release:published +# event fires and triggers downstream workflows (e.g. pypi-publish). +# GITHUB_TOKEN-created releases suppress same-repo webhook events. +# +# Required secret: RELEASE_TOKEN — PAT with contents:write + pull-requests:write +# +# After a Release PR is opened/updated, uv.lock is regenerated and committed +# to the PR branch so the lock file stays in sync with the version bump. +# +# Callers need only: +# jobs: +# release: +# uses: TheDataEngineX/.github/workflows/release-please.yml@main +# secrets: inherit + +name: Release Please (reusable) + +on: + workflow_call: + +jobs: + release-please: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - uses: googleapis/release-please-action@v4 + id: release-please + with: + token: ${{ secrets.RELEASE_TOKEN }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + + # ── Keep uv.lock in sync with version bump ────────────────────────────── + # When release-please opens or updates a Release PR, the PR bumps + # pyproject.toml but not uv.lock. This step checks out the PR branch, + # regenerates uv.lock, and pushes the update back. + - name: Checkout Release PR branch + if: ${{ steps.release-please.outputs.pr }} + uses: actions/checkout@v6 + with: + ref: ${{ fromJson(steps.release-please.outputs.pr).headBranchName }} + token: ${{ secrets.RELEASE_TOKEN }} + + - name: Set up uv + if: ${{ steps.release-please.outputs.pr }} + uses: astral-sh/setup-uv@v8.1.0 + with: + version: "latest" + + - name: Regenerate uv.lock for version bump + if: ${{ steps.release-please.outputs.pr }} + run: | + uv lock + if git diff --quiet uv.lock; then + echo "uv.lock already up to date — no commit needed" + else + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add uv.lock + git commit -m "chore: regenerate uv.lock for version bump" + git push + fi diff --git a/.github/workflows/reusable-security.yml b/.github/workflows/reusable-security.yml index 66a1baa..1a4f8bf 100644 --- a/.github/workflows/reusable-security.yml +++ b/.github/workflows/reusable-security.yml @@ -1,22 +1,169 @@ -name: Reusable Security Scan +# Reusable workflow — called from each repo's .github/workflows/security.yml +# Logic lives here once; repo files are thin callers. +# +# Runs Trivy (misconfig + secret scan) on push/PR. +# CodeQL is handled by GitHub's default setup — do NOT add a custom codeql job +# (GitHub rejects SARIF uploads when default setup is enabled). +# Results appear in GitHub Security tab. + +name: Security Scans (reusable) on: workflow_call: +env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + jobs: trivy-security: - name: Trivy Security Scan runs-on: ubuntu-latest permissions: contents: read + security-events: write steps: - - uses: actions/checkout@v4 + - name: Notify Security Scan Start + uses: actions/github-script@v9 + if: always() + with: + script: | + if (!process.env.SLACK_WEBHOOK) return; + try { + await fetch(process.env.SLACK_WEBHOOK, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: '🔒 Security Scans Started', + blocks: [{ + type: 'section', + text: { + type: 'mrkdwn', + text: `*🔒 Security Scans Started*\n*Branch:* ${process.env.REF}\n*Tools:* Trivy (repo)` + } + }] + }) + }); + } catch (e) { console.log('Slack notification failed'); } + env: + SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} + REF: ${{ github.ref_name }} + + - uses: actions/checkout@v6 + - name: Run Trivy repo security scan + uses: aquasecurity/trivy-action@v0.36.0 + with: + scan-type: fs + scan-ref: . + scanners: misconfig,secret + format: sarif + output: trivy-repo-results.sarif + severity: HIGH,CRITICAL + exit-code: '0' + + - name: Upload Trivy repo results to GitHub Security + if: always() + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: trivy-repo-results.sarif - - name: Run Trivy vulnerability scanner + - name: Enforce Trivy misconfig gate (HIGH/CRITICAL) uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: fs scan-ref: . + scanners: misconfig format: table - severity: CRITICAL,HIGH - exit-code: "0" + severity: HIGH,CRITICAL + exit-code: '1' + + notify-security: + name: Notify Security Status + runs-on: ubuntu-latest + needs: [trivy-security] + if: always() + permissions: + contents: read + + steps: + - name: Notify Security Success + if: ${{ needs.trivy-security.result == 'success' }} + uses: actions/github-script@v9 + with: + script: | + if (!process.env.SLACK_WEBHOOK) return; + try { + await fetch(process.env.SLACK_WEBHOOK, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: '✅ Security Scans Passed', + blocks: [ + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*✅ Security Scans Completed*` + } + }, + { + type: 'section', + fields: [ + { type: 'mrkdwn', text: `*Branch:*\n${process.env.REF}` }, + { type: 'mrkdwn', text: `*Trivy:*\n✅ Passed` }, + { type: 'mrkdwn', text: `*Run:*\n<${process.env.URL}|View Details>` } + ] + } + ] + }) + }); + } catch (e) { console.log('Slack notification failed:', e.message); } + env: + SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} + REF: ${{ github.ref_name }} + URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Notify Security Issues + if: ${{ needs.trivy-security.result == 'failure' }} + uses: actions/github-script@v9 + with: + script: | + if (!process.env.SLACK_WEBHOOK) return; + try { + await fetch(process.env.SLACK_WEBHOOK, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: '⚠️ Security Issues Detected', + blocks: [ + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*⚠️ Security Issues Detected*` + } + }, + { + type: 'section', + fields: [ + { type: 'mrkdwn', text: `*Workflow:*\nSecurity Scans` }, + { type: 'mrkdwn', text: `*Branch:*\n${process.env.REF}` }, + { type: 'mrkdwn', text: `*Commit:*\n\`${process.env.SHA?.substring(0, 8)}\`` }, + { type: 'mrkdwn', text: `*Triggered by:*\n${process.env.ACTOR}` }, + { type: 'mrkdwn', text: `*Event:*\n${process.env.EVENT}` }, + { type: 'mrkdwn', text: `*Run:*\n<${process.env.URL}|View Details>` } + ] + }, + { + type: 'context', + elements: [{ type: 'mrkdwn', text: `🔍 Trivy found issues — review security tab for findings` }] + } + ] + }) + }); + } catch (e) { console.log('Slack notification failed:', e.message); } + env: + SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} + REF: ${{ github.ref_name }} + SHA: ${{ github.sha }} + ACTOR: ${{ github.actor }} + EVENT: ${{ github.event_name }} + URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..b6d8eaf --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,169 @@ +# Security Scans for the .github meta-repo. +# Runs Trivy (misconfig + secret scan) on push/PR. +# CodeQL is handled by GitHub's default setup — do NOT add a custom codeql job. + +name: Security Scans + +on: + push: + branches: [ main, dev ] + pull_request: + branches: [ main, dev ] + +permissions: + contents: read + security-events: write + +env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + +jobs: + trivy-security: + runs-on: ubuntu-latest + steps: + - name: Notify Security Scan Start + uses: actions/github-script@v9 + if: always() + with: + script: | + if (!process.env.SLACK_WEBHOOK) return; + try { + await fetch(process.env.SLACK_WEBHOOK, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: '🔒 Security Scans Started', + blocks: [{ + type: 'section', + text: { + type: 'mrkdwn', + text: `*🔒 Security Scans Started*\n*Branch:* ${process.env.REF}\n*Tools:* Trivy (repo)` + } + }] + }) + }); + } catch (e) { console.log('Slack notification failed'); } + env: + SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} + REF: ${{ github.ref_name }} + + - uses: actions/checkout@v6 + - name: Run Trivy repo security scan + uses: aquasecurity/trivy-action@v0.36.0 + with: + scan-type: fs + scan-ref: . + scanners: misconfig,secret + format: sarif + output: trivy-repo-results.sarif + severity: HIGH,CRITICAL + exit-code: '0' + + - name: Upload Trivy repo results to GitHub Security + if: always() + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: trivy-repo-results.sarif + + - name: Enforce Trivy misconfig gate (HIGH/CRITICAL) + uses: aquasecurity/trivy-action@v0.36.0 + with: + scan-type: fs + scan-ref: . + scanners: misconfig + format: table + severity: HIGH,CRITICAL + exit-code: '1' + + notify-security: + name: Notify Security Status + runs-on: ubuntu-latest + needs: [trivy-security] + if: always() + permissions: + contents: read + + steps: + - name: Notify Security Success + if: ${{ needs.trivy-security.result == 'success' }} + uses: actions/github-script@v9 + with: + script: | + if (!process.env.SLACK_WEBHOOK) return; + try { + await fetch(process.env.SLACK_WEBHOOK, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: '✅ Security Scans Passed', + blocks: [ + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*✅ Security Scans Completed*` + } + }, + { + type: 'section', + fields: [ + { type: 'mrkdwn', text: `*Branch:*\n${process.env.REF}` }, + { type: 'mrkdwn', text: `*Trivy:*\n✅ Passed` }, + { type: 'mrkdwn', text: `*Run:*\n<${process.env.URL}|View Details>` } + ] + } + ] + }) + }); + } catch (e) { console.log('Slack notification failed:', e.message); } + env: + SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} + REF: ${{ github.ref_name }} + URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + - name: Notify Security Issues + if: ${{ needs.trivy-security.result == 'failure' }} + uses: actions/github-script@v9 + with: + script: | + if (!process.env.SLACK_WEBHOOK) return; + try { + await fetch(process.env.SLACK_WEBHOOK, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: '⚠️ Security Issues Detected', + blocks: [ + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*⚠️ Security Issues Detected*` + } + }, + { + type: 'section', + fields: [ + { type: 'mrkdwn', text: `*Workflow:*\nSecurity Scans` }, + { type: 'mrkdwn', text: `*Branch:*\n${process.env.REF}` }, + { type: 'mrkdwn', text: `*Commit:*\n\`${process.env.SHA?.substring(0, 8)}\`` }, + { type: 'mrkdwn', text: `*Triggered by:*\n${process.env.ACTOR}` }, + { type: 'mrkdwn', text: `*Event:*\n${process.env.EVENT}` }, + { type: 'mrkdwn', text: `*Run:*\n<${process.env.URL}|View Details>` } + ] + }, + { + type: 'context', + elements: [{ type: 'mrkdwn', text: `🔍 Trivy found issues — review security tab for findings` }] + } + ] + }) + }); + } catch (e) { console.log('Slack notification failed:', e.message); } + env: + SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }} + REF: ${{ github.ref_name }} + SHA: ${{ github.sha }} + ACTOR: ${{ github.actor }} + EVENT: ${{ github.event_name }} + URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/stale-issues.yml b/.github/workflows/stale-issues.yml new file mode 100644 index 0000000..fa55c7c --- /dev/null +++ b/.github/workflows/stale-issues.yml @@ -0,0 +1,33 @@ +name: Stale Issues + +on: + schedule: + - cron: "0 6 * * *" # Daily at 6 AM UTC + workflow_dispatch: + +jobs: + stale: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: > + This issue has been automatically marked as stale because it has not + had activity in the last 60 days. It will be closed in 30 days if no + further activity occurs. If this issue is still relevant, please + comment or update it. + stale-pr-message: > + This pull request has been automatically marked as stale because it + has not had activity in the last 60 days. It will be closed in 30 days + if no further activity occurs. + close-issue-message: "Closed due to inactivity. Reopen if still relevant." + close-pr-message: "Closed due to inactivity. Reopen if still relevant." + days-before-stale: 60 + days-before-close: 30 + stale-issue-label: "stale" + stale-pr-label: "stale" + exempt-issue-labels: "priority: P1,status: blocked" + exempt-pr-labels: "priority: P1,status: blocked" diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml new file mode 100644 index 0000000..e1cd7e1 --- /dev/null +++ b/.github/workflows/sync-labels.yml @@ -0,0 +1,53 @@ +name: Sync Labels + +on: + push: + branches: [main] + paths: + - labels.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + sync: + name: Sync labels to ${{ matrix.repo }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + repo: + - dataenginex + - dex-studio + - infradex + steps: + - uses: actions/checkout@v6 + + - uses: astral-sh/setup-uv@v8.1.0 + with: + version: "latest" + + - name: Sync labels to ${{ matrix.repo }} + env: + GH_TOKEN: ${{ secrets.LABELS_SYNC_TOKEN }} + run: | + uv pip install pyyaml -q + python3 - <<'PYEOF' + import json, subprocess, yaml + with open('labels.yml') as f: + labels = yaml.safe_load(f) or [] + repo = "TheDataEngineX/${{ matrix.repo }}" + r = subprocess.run( + ['gh', 'label', 'list', '--repo', repo, '--json', 'name', '--limit', '200'], + capture_output=True, text=True, check=True + ) + existing = {l['name'] for l in json.loads(r.stdout)} + for label in labels: + name = label.get('name', '') + color = str(label.get('color', '')).lstrip('#') + desc = label.get('description', '') or '' + cmd = ['gh', 'label', 'edit' if name in existing else 'create', + name, '--repo', repo, '--color', color, '--description', desc] + subprocess.run(cmd, check=True) + PYEOF From 4e37aa5a7bddbd624da7a4b29b193793b707bf14 Mon Sep 17 00:00:00 2001 From: jaymyaka Date: Tue, 21 Jul 2026 13:16:59 -0500 Subject: [PATCH 5/9] Fix ci.yml: correct markdownlint config path, update action versions --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8361d4a..74c5292 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,17 +23,17 @@ jobs: markdownlint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: DavidAnson/markdownlint-cli2-action@v19 with: globs: "**/*.md" - config: ".github/.markdownlint-cli2.jsonc" + config: ".markdownlint-cli2.jsonc" linkcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - uses: lycheeverse/lychee-action@v2 with: fail: true - args: --no-progress -- '**/*.md' '**/*.yml' + args: --no-progress --no-local -- '**/*.md' '**/*.yml' From c3380781244d7e250eca8dbe4c2317c34e62760b Mon Sep 17 00:00:00 2001 From: jaymyaka Date: Tue, 21 Jul 2026 13:19:11 -0500 Subject: [PATCH 6/9] Fix CI: disable MD060 table check, remove invalid lychee --no-local flag --- .github/workflows/ci.yml | 2 +- .markdownlint-cli2.jsonc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74c5292..60105cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,4 +36,4 @@ jobs: - uses: lycheeverse/lychee-action@v2 with: fail: true - args: --no-progress --no-local -- '**/*.md' '**/*.yml' + args: --no-progress -- '**/*.md' '**/*.yml' diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc index 8859cfe..948de44 100644 --- a/.markdownlint-cli2.jsonc +++ b/.markdownlint-cli2.jsonc @@ -9,6 +9,7 @@ "MD033": false, "MD034": false, "MD040": false, - "MD041": false + "MD041": false, + "MD060": false } } From 492d965bd2f3c06292f643b7f6f9e5fa15bf2c27 Mon Sep 17 00:00:00 2001 From: jaymyaka Date: Tue, 21 Jul 2026 13:20:14 -0500 Subject: [PATCH 7/9] Fix linkcheck: exclude local file paths that don't exist in meta-repo --- lychee.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lychee.toml b/lychee.toml index 5d1c566..b6cabff 100644 --- a/lychee.toml +++ b/lychee.toml @@ -1,5 +1,13 @@ -# Exclude URLs that are planned but not yet live +# Exclude URLs that are planned but not yet live, or local paths that don't exist in this meta-repo exclude = [ "https://github.com/TheDataEngineX/infradex.*", "https://docs.thedataenginex.org.*", + "file://*", + "**/.env.example", + "**/examples/**", + "**/src/**", + "**/tests/**", + "**/docs/**", + "**/pyproject.toml", + "**/README.md", ] From ea5b1355f6362d7ad6b681b6526db27651edc947 Mon Sep 17 00:00:00 2001 From: jaymyaka Date: Tue, 21 Jul 2026 13:23:10 -0500 Subject: [PATCH 8/9] Fix markdownlint: fix list numbering in pr-description.prompt.md, fix lychee exclude patterns --- copilot-instructions.md | 1 + lychee.toml | 14 +++++++------- prompts/pr-description.prompt.md | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/copilot-instructions.md b/copilot-instructions.md index 4fa5f3b..2f3f954 100644 --- a/copilot-instructions.md +++ b/copilot-instructions.md @@ -13,6 +13,7 @@ Be pragmatic, straight forward and challenge my ideas and system design focus on | `dataenginex` (dex) | `dataenginex` | Core framework — config, registry, CLI, API, ML, AI | 17000 | | `dex-studio` | `dex-studio` | Web UI — single pane of glass (FastAPI + Jinja2 + HTMX) | 7860 | | `infradex` | — | IaC + monitoring (Terraform, Helm, K3s) | — | + --- ## Coding Standards diff --git a/lychee.toml b/lychee.toml index b6cabff..c0f74ed 100644 --- a/lychee.toml +++ b/lychee.toml @@ -3,11 +3,11 @@ exclude = [ "https://github.com/TheDataEngineX/infradex.*", "https://docs.thedataenginex.org.*", "file://*", - "**/.env.example", - "**/examples/**", - "**/src/**", - "**/tests/**", - "**/docs/**", - "**/pyproject.toml", - "**/README.md", + "**.env.example", + "**examples/**", + "**src/**", + "**tests/**", + "**docs/**", + "**pyproject.toml", + "**README.md", ] diff --git a/prompts/pr-description.prompt.md b/prompts/pr-description.prompt.md index 3ec9291..5d118f7 100644 --- a/prompts/pr-description.prompt.md +++ b/prompts/pr-description.prompt.md @@ -38,6 +38,6 @@ Generate a pull request description for the current changes using the project's - [ ] No breaking changes (or documented in description) ``` -4. Check the appropriate boxes based on actual changes -5. Write a concise but complete description -6. Use conventional commit style for the summary line +1. Check the appropriate boxes based on actual changes +2. Write a concise but complete description +3. Use conventional commit style for the summary line From 1c24abcf3094ceb60fcf5b4a8ab80c6c9c0800d5 Mon Sep 17 00:00:00 2001 From: jaymyaka Date: Tue, 21 Jul 2026 13:24:51 -0500 Subject: [PATCH 9/9] Fix lychee exclude patterns: use proper regex syntax --- lychee.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lychee.toml b/lychee.toml index c0f74ed..27691ee 100644 --- a/lychee.toml +++ b/lychee.toml @@ -1,13 +1,13 @@ # Exclude URLs that are planned but not yet live, or local paths that don't exist in this meta-repo exclude = [ - "https://github.com/TheDataEngineX/infradex.*", - "https://docs.thedataenginex.org.*", - "file://*", - "**.env.example", - "**examples/**", - "**src/**", - "**tests/**", - "**docs/**", - "**pyproject.toml", - "**README.md", + "https://github.com/TheDataEngineX/infradex", + "https://docs.thedataenginex.org", + "file://", + "\\.env\\.example", + "examples/", + "src/", + "tests/", + "docs/", + "pyproject\\.toml", + "README\\.md", ]