feat: add versioned docs to website

.github/workflows/deploy.yml

@@ -78,6 +78,7 @@ jobs:
     environment:
       name: preview
     permissions:
+      deployments: write
       pull-requests: write
     steps:
       - name: Download build artifact

.github/workflows/sync.yml

@@ -0,0 +1,76 @@
+name: Sync Docs from Source
+
+on:
+  repository_dispatch:
+    types: [sync-docs]
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: 'Repository to sync'
+        required: true
+        default: 'dataenginex'
+        type: choice
+        options:
+          - dataenginex
+          - dex-studio
+      version:
+        description: 'Version tag to sync'
+        required: true
+        default: 'v0.1.0'
+
+permissions:
+  contents: write
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Parse payload
+        id: parse
+        run: |
+          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+            echo "repo=${{ github.event.inputs.repo }}" >> $GITHUB_OUTPUT
+            echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
+          else
+            echo "repo=${{ github.event.client_payload.repo }}" >> $GITHUB_OUTPUT
+            echo "version=${{ github.event.client_payload.version }}" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Clone source docs
+        run: |
+          REPO="${{ steps.parse.outputs.repo }}"
+          VERSION="${{ steps.parse.outputs.version }}"
+          git clone --depth 1 --branch "$VERSION" \
+            "https://github.com/TheDataEngineX/$REPO.git" /tmp/$REPO
+          rm -rf "docs-store/$REPO/$VERSION"
+          mkdir -p "docs-store/$REPO/$VERSION"
+          if [ -d "/tmp/$REPO/docs" ]; then
+            cp -r "/tmp/$REPO/docs/"* "docs-store/$REPO/$VERSION/"
+          fi
+
+      - name: Build site
+        run: |
+          uv python install 3.13
+          uv sync
+          PYTHONPATH=. uv run poe build
+
+      - name: Commit docs-store
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add docs-store/
+          git diff --cached --quiet || git commit -m "chore: sync docs [skip ci]"
+          git push
+
+      - name: Deploy to Cloudflare Pages
+        uses: cloudflare/pages-action@v1
+        with:
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+          projectName: thedataenginex-org
+          directory: output/
+          gitHubToken: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -5,3 +5,4 @@ output/
 .uv/
 *.egg-info/
 .DS_Store
+docs-store/

CLAUDE.md

@@ -1,31 +1,50 @@
 # CLAUDE.md — DEX Website
 
-> Workspace-wide rules in `../CLAUDE.md`.
-
 ## Project Overview
+Marketing site + versioned documentation for thedataenginex.org.
+Static site generated with Pelican (marketing) + custom build script (docs).
+Deployed to Cloudflare Pages.
 
-Marketing site for thedataenginex.org. Static site generated with Pelican, deployed to Cloudflare Pages.
-
-**Stack:** Pelican · Python 3.13+ · uv · Custom `dex` theme
-
+**Stack:** Pelican 4.12 · Python 3.13+ · uv · Custom `dex` theme · Jinja2 · Markdown
 **Live site:** https://thedataenginex.org
 
 ## Commands
-
-```bash
-uv run poe dev     # Dev server at http://localhost:8080 (live reload)
-uv run poe build   # Production build → output/
-uv run poe serve   # Serve output/ locally
-uv run poe clean   # Remove output/
-```
+uv run poe dev        # Dev server at http://localhost:8080 (Pelican only, live reload)
+uv run poe build      # Production build: Pelican + docs -> output/
+uv run poe build-pelican  # Pelican only
+uv run poe build-docs     # Docs only (after Pelican build)
+uv run poe serve      # Serve output/ locally
+uv run poe clean      # Remove output/
 
 ## Key Files
-
 | Path | Purpose |
 |------|---------|
-| `pelicanconf.py` | Dev config (localhost) |
-| `publishconf.py` | Production config (thedataenginex.org) |
-| `content/pages/` | Static pages (why.md, changelog.md) |
-| `themes/dex/` | Custom Pelican theme |
-| `content/extra/CNAME` | Cloudflare/GitHub Pages CNAME |
-| `.github/workflows/deploy.yml` | CI deploy via Cloudflare Pages Action |
+| pelicanconf.py | Pelican dev config |
+| publishconf.py | Pelican production config |
+| content/pages/ | Marketing pages (why.md, changelog.md) |
+| themes/dex/ | Custom Pelican theme + docs CSS/JS |
+| scripts/build_site.py | Docs build script (runs after Pelican) |
+| templates/docs/ | Docs Jinja2 templates |
+| config/site.yml | Docs site configuration |
+| docs-store/ | Versioned documentation (synced from source repos) |
+| .github/workflows/deploy.yml | CI deploy via Cloudflare Pages Action |
+| .github/workflows/sync.yml | Docs sync triggered by source repo releases |
+
+## Docs Build
+
+`scripts/build_site.py` walks `docs-store/` and renders .md -> .html.
+- Output goes to Pelican's `output/docs/` directory
+- Generates `output/sitemap.xml`, `output/robots.txt` (overwriting Pelican's)
+- Generates `output/search-index.json` for client-side search
+- Minifies HTML output
+
+### Adding docs from a new version
+```bash
+# Manually add to docs-store/
+mkdir -p docs-store/dataenginex/v0.2.0/
+cp -r /path/to/docs/* docs-store/dataenginex/v0.2.0/
+uv run poe build-docs
+```
+
+### Sync from source repo release
+Source repos fire `repository_dispatch` -> `.github/workflows/sync.yml`.

config/site.yml

@@ -0,0 +1,14 @@
+site:
+  name: TheDataEngineX
+  url: https://thedataenginex.org
+  description: DataEngineX platform — run AI agents, data pipelines, and ML workflows on your own infrastructure
+
+repos:
+  dataenginex:
+    display_name: DataEngineX
+    description: Python library for data engineering & AI agents
+  dex-studio:
+    display_name: DEX Studio
+    description: Self-hosted web UI for the DataEngineX platform
+
+og_default_image: /theme/img/og-card.png

content/pages/changelog.md

@@ -11,7 +11,7 @@ Full release notes are on [GitHub Releases](https://github.com/TheDataEngineX/DE
 
 The current version is tracked in [`pyproject.toml`](https://github.com/TheDataEngineX/DEX/blob/main/pyproject.toml).
 
-**Current:** `v1.1.1` — CLI compatibility fix (removed `rich` dep; Reflex pin conflict). `v1.1.0` added enterprise auth (JWKS/RS256), RBAC, SCIM v2, LiteLLM + vLLM providers, Langfuse LLM tracing, LightRAG graph retrieval, domain Prometheus metrics.
+**Current:** `v0.4.0` — Post-scope-reset milestone with focused feature set. See [GitHub Releases](https://github.com/TheDataEngineX/dataenginex/releases) for the full history.
 
 [View all releases on GitHub →](https://github.com/TheDataEngineX/DEX/releases)
 

content/pages/why.md

@@ -1,7 +1,7 @@
 ---
-title: Why DataEngineX — Complete Data + ML + AI Engineering
+title: Why DataEngineX — Data + ML + AI Engineering
 slug: why
-description: DataEngineX is an open-source, self-hosted Python framework for the complete platform for Data + ML + AI engineering — production-ready pipelines, ML lifecycle, and AI agents unified under one dex.yaml. Native integrations with Airflow, MLflow, LangChain, and more.
+description: DataEngineX is an open-source, self-hosted Python framework for data pipelines, ML lifecycle, and AI agents — unified under one dex.yaml. Native integrations with DuckDB, Spark, MLflow, LiteLLM, and more.
 template: page
 ---
 
@@ -28,7 +28,7 @@ DataEngineX does not rip out those tools. It ships production-ready implementati
 
 ## One File, Entire Stack
 
-DataEngineX is the complete platform for your Data + ML + AI engineering lifecycle. Define once, run anywhere:
+DataEngineX is a Python framework for your Data + ML + AI engineering lifecycle. Define once, run anywhere:
 
 ```yaml
 # dex.yaml
@@ -51,28 +51,24 @@ ai:
     - name: analyst
       tools: [sql, search]
 
-server:
-  auth: jwt
-  rate_limit: 100/min
-
 observability:
   metrics: prometheus
   tracing: otel
 ```
 
-One `dex serve` command starts everything. No glue code.
+Use it from Python, the `dex` CLI, or the self-hosted web UI. No glue code.
 
 ---
 
 ## Complete Platform — Built-In and Integrated
 
 DataEngineX ships production-ready implementations of every layer:
 
-- **Data pipelines** — DuckDB engine, S3/GCS connectors, quality gates, medallion lakehouse
+- **Data pipelines** — DuckDB engine, S3/GCS connectors, quality gates, medallion lakehouse, optional Spark backend
 - **ML lifecycle** — experiment tracking, model registry, training, serving, drift detection
 - **AI agents** — LLM routing via LiteLLM, hybrid BM25+dense retrieval, LangGraph runtime
-- **API server** — FastAPI with JWT auth, rate limiting, RBAC, SCIM v2
-- **Observability** — Prometheus metrics, OpenTelemetry tracing, Langfuse LLM tracing
+- **DEX Studio** — self-hosted web UI (FastAPI + Jinja2 + HTMX, port 7860)
+- **Observability** — structlog structured logging, Prometheus metrics, OpenTelemetry tracing — built in, no extra packages
 
 When you already run external tools, DEX integrates natively — not against them. Airflow only schedules. MLflow only tracks. LangChain only chains. DEX gives you the complete end-to-end stack:
 
@@ -82,7 +78,7 @@ When you already run external tools, DEX integrates natively — not against the
 | MLflow | Point `ml.backend: mlflow` — tracking_uri wired automatically |
 | LangChain / LiteLLM | LLM routing layer — 100+ providers, swap without code changes |
 | Qdrant | Vector store for RAG — configured in `ai.retrieval` |
-| Langfuse | LLM observability — enabled via `[observability]` extra |
+| Langfuse | LLM observability — enabled via config, not code |
 | PySpark | Big-data backend — swap `data.backend: spark` in config |
 
 ---
@@ -93,21 +89,14 @@ Opinionated defaults, zero lock-in. Every layer is swappable:
 
 ```bash
 pip install "dataenginex[cloud]"         # S3 + GCS + BigQuery connectors
-pip install "dataenginex[auth]"          # RS256/JWKS + SCIM v2 + LDAP sync
-pip install "dataenginex[observability]" # Langfuse LLM tracing
+pip install "dataenginex[qdrant]"        # Qdrant vector store
 pip install 'litellm>=1.83.3' --no-deps  # 100+ LLM providers (separate install)
 ```
 
 The config stays the same. The backend changes.
 
 ---
 
-## Enterprise-Ready Auth
-
-RBAC, SCIM v2 user provisioning, LDAP/AD sync, and OIDC federation (RS256/JWKS) ship as first-class extras — not bolt-ons. Enable them with env vars, not code changes.
-
----
-
 ## Self-Hosted
 
 Your data never leaves your infrastructure. No SaaS subscription. No vendor lock-in. Run on a VPS, K3s cluster, or bare metal.

docs-store/dataenginex/v0.1.0/getting-started.md

@@ -0,0 +1,13 @@
+# Getting Started
+
+Install DataEngineX and run your first pipeline.
+
+## Installation
+
+```bash
+pip install dataenginex
+```
+
+## First Pipeline
+
+Create a `dex.yaml` file and run `dex run`.

docs-store/dataenginex/v0.1.0/index.md

@@ -0,0 +1,12 @@
+# DataEngineX Documentation
+
+Welcome to the DataEngineX documentation. Build and run AI agents, data pipelines, and ML workflows.
+
+## Quick Start
+
+Install with `pip install dataenginex`.
+
+## Guides
+
+- [Getting Started](getting-started)
+- [Configuration](configuration)

docs-store/dex-studio/v0.4.0/index.md

@@ -0,0 +1,11 @@
+# DEX Studio Documentation
+
+Welcome to DEX Studio, the self-hosted web UI for the DataEngineX platform.
+
+## Quick Start
+
+```bash
+dex-studio
+```
+
+Then open http://localhost:7860.

pelicanconf.py

@@ -22,17 +22,16 @@
 # Theme
 THEME = "themes/dex"
 
-# Navigation
+# Navigation — Docs now points to local
 MENUITEMS = [
     ("Why DEX", "/why/"),
-    ("Docs", "https://docs.thedataenginex.org"),
+    ("Docs", "/docs/dataenginex/latest/"),
     ("Changelog", "/changelog/"),
     ("GitHub", "https://github.com/TheDataEngineX/DEX"),
 ]
 
-# Plugins
-PLUGINS = ["pelican.plugins.sitemap"]
-SITEMAP = {"format": "xml"}
+# Plugins — disabled sitemap; build_site.py generates complete sitemap
+PLUGINS = []
 
 # Static paths
 STATIC_PATHS = ["extra"]
@@ -50,4 +49,4 @@
 # Social
 GITHUB_URL = "https://github.com/TheDataEngineX/DEX"
 PYPI_URL = "https://pypi.org/project/dataenginex/"
-DOCS_URL = "https://docs.thedataenginex.org"
+DOCS_URL = "https://thedataenginex.org/docs/dataenginex/latest/"

pyproject.toml

@@ -1,17 +1,24 @@
 [project]
 name = "dataenginex-website"
 version = "0.1.0"
-description = "thedataenginex.org marketing site"
+description = "thedataenginex.org marketing site + docs"
 requires-python = ">=3.13"
 dependencies = [
     "pelican[markdown]>=4.10",
-    "pelican-sitemap>=1.0",
     "Pygments>=2.18",
+    "jinja2>=3.1",
+    "markdown>=3.6",
+    "pyyaml>=6.0",
 ]
 
+[project.optional-dependencies]
+images = ["pillow>=10.0"]
+
 [tool.poe.tasks]
 dev = { shell = "PYTHONPATH=. pelican --autoreload --listen --port 8080 content -s pelicanconf.py" }
-build = { shell = "PYTHONPATH=. pelican content -o output -s publishconf.py" }
+build = { shell = "PYTHONPATH=. pelican content -o output -s publishconf.py && python scripts/build_site.py" }
+build-pelican = { shell = "PYTHONPATH=. pelican content -o output -s publishconf.py" }
+build-docs = { shell = "python scripts/build_site.py" }
 clean = { shell = "rm -rf output" }
 serve = { shell = "cd output && python -m http.server 8080" }