From e0066e867a3a2b62b31dc0e3f39181e9f11d2c77 Mon Sep 17 00:00:00 2001 From: data-literate Date: Thu, 11 Jun 2026 20:42:29 -0500 Subject: [PATCH 1/4] feat: add versioned docs to website Consolidate documentation from dataenginex and dex-studio repos into the existing Pelican marketing site. - Pelican config: update Docs nav URL to local, remove sitemap plugin - New deps: jinja2, markdown, pyyaml, poe build-docs/build-pelican tasks - Templates: docs/base.html and docs/doc.html with sidebar, version selector, breadcrumbs, TOC, SEO meta, JSON-LD - CSS: docs layout, search UI, responsive sidebar - JS: client-side search overlay with highlight - Build script: walks docs-store/, renders .md->.html, generates sitemap.xml, robots.txt, search-index.json, minifies output - Sync workflow: repository_dispatch + workflow_dispatch to pull docs from source repo releases - CLAUDE.md updated with docs build architecture --- .github/workflows/sync.yml | 73 ++++ .gitignore | 1 + CLAUDE.md | 59 ++- config/site.yml | 14 + .../dataenginex/v0.1.0/getting-started.md | 13 + docs-store/dataenginex/v0.1.0/index.md | 12 + docs-store/dex-studio/v0.4.0/index.md | 11 + pelicanconf.py | 11 +- pyproject.toml | 13 +- scripts/build_site.py | 346 ++++++++++++++++++ templates/docs/base.html | 97 +++++ templates/docs/doc.html | 99 +++++ themes/dex/static/css/docs.css | 289 +++++++++++++++ themes/dex/static/css/main.css | 2 + themes/dex/static/js/search.js | 116 ++++++ uv.lock | 75 +++- 16 files changed, 1193 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/sync.yml create mode 100644 config/site.yml create mode 100644 docs-store/dataenginex/v0.1.0/getting-started.md create mode 100644 docs-store/dataenginex/v0.1.0/index.md create mode 100644 docs-store/dex-studio/v0.4.0/index.md create mode 100644 scripts/build_site.py create mode 100644 templates/docs/base.html create mode 100644 templates/docs/doc.html create mode 100644 themes/dex/static/css/docs.css create mode 100644 themes/dex/static/js/search.js diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml new file mode 100644 index 0000000..1a6663e --- /dev/null +++ b/.github/workflows/sync.yml @@ -0,0 +1,73 @@ +name: Sync Docs from Source + +on: + repository_dispatch: + types: [sync-docs] + workflow_dispatch: + inputs: + repo: + description: 'Repository to sync' + required: true + default: 'dataenginex' + type: choice + options: + - dataenginex + - dex-studio + version: + description: 'Version tag to sync' + required: true + default: 'v0.1.0' + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Parse payload + id: parse + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "repo=${{ github.event.inputs.repo }}" >> $GITHUB_OUTPUT + echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT + else + echo "repo=${{ github.event.client_payload.repo }}" >> $GITHUB_OUTPUT + echo "version=${{ github.event.client_payload.version }}" >> $GITHUB_OUTPUT + fi + + - name: Clone source docs + run: | + REPO="${{ steps.parse.outputs.repo }}" + VERSION="${{ steps.parse.outputs.version }}" + git clone --depth 1 --branch "$VERSION" \ + "https://github.com/TheDataEngineX/$REPO.git" /tmp/$REPO + rm -rf "docs-store/$REPO/$VERSION" + mkdir -p "docs-store/$REPO/$VERSION" + if [ -d "/tmp/$REPO/docs" ]; then + cp -r "/tmp/$REPO/docs/"* "docs-store/$REPO/$VERSION/" + fi + + - name: Build site + run: | + uv python install 3.13 + uv sync + PYTHONPATH=. uv run poe build + + - name: Commit docs-store + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add docs-store/ + git diff --cached --quiet || git commit -m "chore: sync docs [skip ci]" + git push + + - name: Deploy to Cloudflare Pages + uses: cloudflare/pages-action@v1 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + projectName: thedataenginex-org + directory: output/ + gitHubToken: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index d36290c..fdf8437 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ output/ .uv/ *.egg-info/ .DS_Store +docs-store/ diff --git a/CLAUDE.md b/CLAUDE.md index 86355e4..a9b98c1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,31 +1,50 @@ # CLAUDE.md — DEX Website -> Workspace-wide rules in `../CLAUDE.md`. - ## Project Overview +Marketing site + versioned documentation for thedataenginex.org. +Static site generated with Pelican (marketing) + custom build script (docs). +Deployed to Cloudflare Pages. -Marketing site for thedataenginex.org. Static site generated with Pelican, deployed to Cloudflare Pages. - -**Stack:** Pelican · Python 3.13+ · uv · Custom `dex` theme - +**Stack:** Pelican 4.12 · Python 3.13+ · uv · Custom `dex` theme · Jinja2 · Markdown **Live site:** https://thedataenginex.org ## Commands - -```bash -uv run poe dev # Dev server at http://localhost:8080 (live reload) -uv run poe build # Production build → output/ -uv run poe serve # Serve output/ locally -uv run poe clean # Remove output/ -``` +uv run poe dev # Dev server at http://localhost:8080 (Pelican only, live reload) +uv run poe build # Production build: Pelican + docs -> output/ +uv run poe build-pelican # Pelican only +uv run poe build-docs # Docs only (after Pelican build) +uv run poe serve # Serve output/ locally +uv run poe clean # Remove output/ ## Key Files - | Path | Purpose | |------|---------| -| `pelicanconf.py` | Dev config (localhost) | -| `publishconf.py` | Production config (thedataenginex.org) | -| `content/pages/` | Static pages (why.md, changelog.md) | -| `themes/dex/` | Custom Pelican theme | -| `content/extra/CNAME` | Cloudflare/GitHub Pages CNAME | -| `.github/workflows/deploy.yml` | CI deploy via Cloudflare Pages Action | +| pelicanconf.py | Pelican dev config | +| publishconf.py | Pelican production config | +| content/pages/ | Marketing pages (why.md, changelog.md) | +| themes/dex/ | Custom Pelican theme + docs CSS/JS | +| scripts/build_site.py | Docs build script (runs after Pelican) | +| templates/docs/ | Docs Jinja2 templates | +| config/site.yml | Docs site configuration | +| docs-store/ | Versioned documentation (synced from source repos) | +| .github/workflows/deploy.yml | CI deploy via Cloudflare Pages Action | +| .github/workflows/sync.yml | Docs sync triggered by source repo releases | + +## Docs Build + +`scripts/build_site.py` walks `docs-store/` and renders .md -> .html. +- Output goes to Pelican's `output/docs/` directory +- Generates `output/sitemap.xml`, `output/robots.txt` (overwriting Pelican's) +- Generates `output/search-index.json` for client-side search +- Minifies HTML output + +### Adding docs from a new version +```bash +# Manually add to docs-store/ +mkdir -p docs-store/dataenginex/v0.2.0/ +cp -r /path/to/docs/* docs-store/dataenginex/v0.2.0/ +uv run poe build-docs +``` + +### Sync from source repo release +Source repos fire `repository_dispatch` -> `.github/workflows/sync.yml`. diff --git a/config/site.yml b/config/site.yml new file mode 100644 index 0000000..f1fe140 --- /dev/null +++ b/config/site.yml @@ -0,0 +1,14 @@ +site: + name: TheDataEngineX + url: https://thedataenginex.org + description: DataEngineX platform — run AI agents, data pipelines, and ML workflows on your own infrastructure + +repos: + dataenginex: + display_name: DataEngineX + description: Python library for data engineering & AI agents + dex-studio: + display_name: DEX Studio + description: Self-hosted web UI for the DataEngineX platform + +og_default_image: /theme/img/og-card.png diff --git a/docs-store/dataenginex/v0.1.0/getting-started.md b/docs-store/dataenginex/v0.1.0/getting-started.md new file mode 100644 index 0000000..b87a4c0 --- /dev/null +++ b/docs-store/dataenginex/v0.1.0/getting-started.md @@ -0,0 +1,13 @@ +# Getting Started + +Install DataEngineX and run your first pipeline. + +## Installation + +```bash +pip install dataenginex +``` + +## First Pipeline + +Create a `dex.yaml` file and run `dex run`. diff --git a/docs-store/dataenginex/v0.1.0/index.md b/docs-store/dataenginex/v0.1.0/index.md new file mode 100644 index 0000000..dd18636 --- /dev/null +++ b/docs-store/dataenginex/v0.1.0/index.md @@ -0,0 +1,12 @@ +# DataEngineX Documentation + +Welcome to the DataEngineX documentation. Build and run AI agents, data pipelines, and ML workflows. + +## Quick Start + +Install with `pip install dataenginex`. + +## Guides + +- [Getting Started](getting-started) +- [Configuration](configuration) diff --git a/docs-store/dex-studio/v0.4.0/index.md b/docs-store/dex-studio/v0.4.0/index.md new file mode 100644 index 0000000..c511c66 --- /dev/null +++ b/docs-store/dex-studio/v0.4.0/index.md @@ -0,0 +1,11 @@ +# DEX Studio Documentation + +Welcome to DEX Studio, the self-hosted web UI for the DataEngineX platform. + +## Quick Start + +```bash +dex-studio +``` + +Then open http://localhost:7860. diff --git a/pelicanconf.py b/pelicanconf.py index be83556..5a8c1f9 100644 --- a/pelicanconf.py +++ b/pelicanconf.py @@ -22,17 +22,16 @@ # Theme THEME = "themes/dex" -# Navigation +# Navigation — Docs now points to local MENUITEMS = [ ("Why DEX", "/why/"), - ("Docs", "https://docs.thedataenginex.org"), + ("Docs", "/docs/dataenginex/latest/"), ("Changelog", "/changelog/"), ("GitHub", "https://github.com/TheDataEngineX/DEX"), ] -# Plugins -PLUGINS = ["pelican.plugins.sitemap"] -SITEMAP = {"format": "xml"} +# Plugins — disabled sitemap; build_site.py generates complete sitemap +PLUGINS = [] # Static paths STATIC_PATHS = ["extra"] @@ -50,4 +49,4 @@ # Social GITHUB_URL = "https://github.com/TheDataEngineX/DEX" PYPI_URL = "https://pypi.org/project/dataenginex/" -DOCS_URL = "https://docs.thedataenginex.org" +DOCS_URL = "https://thedataenginex.org/docs/dataenginex/latest/" diff --git a/pyproject.toml b/pyproject.toml index e0b6d70..b46b34f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,24 @@ [project] name = "dataenginex-website" version = "0.1.0" -description = "thedataenginex.org marketing site" +description = "thedataenginex.org marketing site + docs" requires-python = ">=3.13" dependencies = [ "pelican[markdown]>=4.10", - "pelican-sitemap>=1.0", "Pygments>=2.18", + "jinja2>=3.1", + "markdown>=3.6", + "pyyaml>=6.0", ] +[project.optional-dependencies] +images = ["pillow>=10.0"] + [tool.poe.tasks] dev = { shell = "PYTHONPATH=. pelican --autoreload --listen --port 8080 content -s pelicanconf.py" } -build = { shell = "PYTHONPATH=. pelican content -o output -s publishconf.py" } +build = { shell = "PYTHONPATH=. pelican content -o output -s publishconf.py && python scripts/build_site.py" } +build-pelican = { shell = "PYTHONPATH=. pelican content -o output -s publishconf.py" } +build-docs = { shell = "python scripts/build_site.py" } clean = { shell = "rm -rf output" } serve = { shell = "cd output && python -m http.server 8080" } diff --git a/scripts/build_site.py b/scripts/build_site.py new file mode 100644 index 0000000..3a18afb --- /dev/null +++ b/scripts/build_site.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +"""Build documentation pages for thedataenginex.org. + +Runs after Pelican. Walks docs-store/, renders .md -> .html via Jinja2, +generates sitemap.xml, robots.txt (overwriting Pelican's), and +search-index.json. Outputs to Pelican's output/ directory. +""" + +import json +import os +import re +import shutil +from datetime import datetime +from pathlib import Path + +import yaml +from jinja2 import Environment, FileSystemLoader +from markdown import markdown + +REPO_ROOT = Path(__file__).resolve().parent.parent +CONFIG_PATH = REPO_ROOT / "config" / "site.yml" +DOCS_STORE = REPO_ROOT / "docs-store" +TEMPLATES_DIR = REPO_ROOT / "templates" +OUTPUT_DIR = REPO_ROOT / "output" + + +def load_config(): + with open(CONFIG_PATH) as f: + return yaml.safe_load(f) + + +def discover_docs_versions(docs_store: Path) -> dict: + """Return {repo_name: [version_str, ...]} sorted newest first.""" + result = {} + if not docs_store.exists(): + return result + for repo_dir in sorted(docs_store.iterdir()): + if not repo_dir.is_dir() or repo_dir.name.startswith("."): + continue + versions = [] + for ver_dir in sorted(repo_dir.iterdir(), reverse=True): + if ver_dir.is_dir() and not ver_dir.name.startswith("."): + versions.append(ver_dir.name) + if versions: + result[repo_dir.name] = sorted(versions, key=parse_semver, reverse=True) + return result + + +def parse_semver(version_str: str) -> tuple: + match = re.match(r"^v?(\d+)\.(\d+)\.(\d+)$", version_str) + if match: + return tuple(int(g) for g in match.groups()) + return (0, 0, 0) + + +def get_doc_tree(repo_dir: Path) -> list[dict]: + """Return [{slug, title, path}] sorted alphabetically.""" + docs = [] + for f in sorted(repo_dir.iterdir()): + if f.suffix == ".md": + slug = f.stem + content = f.read_text(encoding="utf-8") + title = extract_title(content) or slug.replace("-", " ").title() + docs.append({"slug": slug, "title": title, "path": f}) + return docs + + +def extract_title(md_content: str) -> str | None: + match = re.search(r"^#\s+(.+)$", md_content, re.MULTILINE) + return match.group(1).strip() if match else None + + +def extract_description(md_content: str) -> str | None: + lines = md_content.split("\n") + in_para = False + for line in lines: + stripped = line.strip() + if stripped.startswith("# "): + in_para = True + continue + if in_para and stripped: + return (stripped[:152] + "...") if len(stripped) > 155 else stripped + if in_para and not stripped: + continue + if in_para and stripped.startswith("#"): + break + return None + + +def extract_toc(md_content: str) -> list[dict]: + items = [] + for line in md_content.split("\n"): + match = re.match(r"^(#{2,3})\s+(.+)$", line) + if match: + text = match.group(2).strip() + anchor_id = text.lower().replace(" ", "-").replace(".", "").replace(",", "") + items.append({"level": len(match.group(1)), "text": text, "id": anchor_id}) + return items + + +def render_markdown_to_html(md_content: str) -> str: + extensions = ["fenced_code", "codehilite", "tables", "sane_lists", "attr_list"] + return markdown(md_content, extensions=extensions) + + +def get_highest_semver(versions: list[str]) -> str: + if not versions: + return "" + sorted_vers = sorted(versions, key=parse_semver, reverse=True) + return sorted_vers[0] + + +def compute_is_stale(last_updated_str: str | None, months_threshold: int = 6) -> tuple[bool, int]: + if not last_updated_str: + return False, 0 + try: + updated = datetime.strptime(last_updated_str, "%Y-%m-%d").date() + except ValueError: + return False, 0 + delta = datetime.now().date() - updated + months = delta.days // 30 + return months > months_threshold, months + + +def generate_sitemap(pages: list[dict], site_url: str) -> str: + lines = [ + '', + '', + ] + for p in pages: + loc = f"{site_url}{p['url']}" + lastmod = p.get("last_updated") or datetime.now().strftime("%Y-%m-%d") + priority = p.get("priority", "0.7") + changefreq = p.get("changefreq", "monthly") + lines.append(" ") + lines.append(f" {loc}") + lines.append(f" {lastmod}") + lines.append(f" {priority}") + lines.append(f" {changefreq}") + lines.append(" ") + lines.append("") + return "\n".join(lines) + + +def generate_robots_txt(site_url: str) -> str: + return f"User-agent: *\nAllow: /\n\nSitemap: {site_url}/sitemap.xml\n" + + +def minify_html(html: str) -> str: + html = re.sub(r"\s+", " ", html) + html = re.sub(r">\s+<", "><", html) + return html.strip() + + +def minify_css(css: str) -> str: + css = re.sub(r"/\*.*?\*/", "", css, flags=re.DOTALL) + css = re.sub(r"\s+", " ", css) + css = re.sub(r"\s*([{}:;,])\s*", r"\1", css) + css = re.sub(r";}", "}", css) + return css.strip() + + +def minify_js(js: str) -> str: + js = re.sub(r"//.*", "", js) + js = re.sub(r"/\*.*?\*/", "", js, flags=re.DOTALL) + js = re.sub(r"\s+", " ", js) + return js.strip() + + +def check_thin_content(md_content: str, slug: str) -> None: + word_count = len(md_content.split()) + if word_count < 100: + print(f" WARNING: '{slug}' has only {word_count} words (below 100)") + + +def main(): + config = load_config() + site = config["site"] + repos_config = config.get("repos", {}) + all_versions = discover_docs_versions(DOCS_STORE) + + if not OUTPUT_DIR.exists(): + print(f"ERROR: Pelican output directory '{OUTPUT_DIR}' does not exist. Run Pelican build first.") + return + + # Setup Jinja2 + env = Environment(loader=FileSystemLoader(str(REPO_ROOT))) + env.globals["now"] = datetime.now() + + # Collect all pages for sitemap + sitemap_pages = [] + + # Scan Pelican output for existing pages (/, /why/, /changelog/) to include in sitemap + pelican_html_dirs = ["", "why", "changelog"] + for subdir in pelican_html_dirs: + index_path = OUTPUT_DIR / subdir / "index.html" + if index_path.exists(): + url = f"/{subdir}/" if subdir else "/" + sitemap_pages.append({ + "url": url, + "last_updated": None, + "priority": "1.0" if not subdir else "0.8", + "changefreq": "monthly", + }) + + # Render docs + search_pages = [] + + for repo_name, versions in all_versions.items(): + repo_display = repos_config.get(repo_name, {}).get("display_name", repo_name) + latest_version = get_highest_semver(versions) + + for version in versions: + repo_dir = DOCS_STORE / repo_name / version + if not repo_dir.is_dir(): + continue + + docs = get_doc_tree(repo_dir) + is_latest = version == latest_version + + for doc in docs: + md_content = doc["path"].read_text(encoding="utf-8") + check_thin_content(md_content, doc["slug"]) + + html_content = render_markdown_to_html(md_content) + title = doc["title"] + description = extract_description(md_content) + last_updated = None # placeholder; could come from git date + is_stale, months_since = compute_is_stale(last_updated) + + toc_items = extract_toc(md_content) + slug = doc["slug"] + page_url = f"/docs/{repo_name}/{version}/" if slug == "index" else f"/docs/{repo_name}/{version}/{slug}/" + canonical_url = page_url + + # Sidebar + sidebar_items = [ + { + "title": d["title"], + "slug": d["slug"], + "active": d["slug"] == slug, + "url": f"/docs/{repo_name}/{version}/" if d["slug"] == "index" else f"/docs/{repo_name}/{version}/{d['slug']}/", + } + for d in docs + ] + + page_ctx = { + "title": title, + "description": description or "", + "url": page_url, + "canonical_url": canonical_url, + "content": html_content, + "slug": slug, + "type": "doc", + "last_updated": last_updated, + "is_stale": is_stale, + "months_since_update": months_since if is_stale else 0, + "og_image": None, + "toc_items": toc_items, + } + + repo_ctx = { + "name": repo_name, + "display_name": repo_display, + "version": version, + "latest_version": latest_version, + "available_versions": versions, + } + + nav_ctx = {"sidebar": sidebar_items} + + template = env.get_template("templates/docs/doc.html") + output = template.render( + site=site, + page=page_ctx, + repo=repo_ctx, + navigation=nav_ctx, + ) + + # Write versioned page + out_path = OUTPUT_DIR / "docs" / repo_name / version / "index.html" if slug == "index" else OUTPUT_DIR / "docs" / repo_name / version / slug / "index.html" + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(minify_html(output), encoding="utf-8") + + # Write to latest/ + if is_latest: + latest_ctx = dict(page_ctx) + latest_ctx["canonical_url"] = canonical_url + latest_output = template.render( + site=site, + page=latest_ctx, + repo=dict(repo_ctx, version="latest"), + navigation=nav_ctx, + ) + latest_path = OUTPUT_DIR / "docs" / repo_name / "latest" / "index.html" if slug == "index" else OUTPUT_DIR / "docs" / repo_name / "latest" / slug / "index.html" + latest_path.parent.mkdir(parents=True, exist_ok=True) + latest_path.write_text(minify_html(latest_output), encoding="utf-8") + + # Sitemap entry (only for real version, not latest alias) + sitemap_pages.append({ + "url": page_url, + "last_updated": last_updated or datetime.now().strftime("%Y-%m-%d"), + "priority": "0.8", + "changefreq": "monthly", + }) + + # Search index entry + search_pages.append({ + "title": title, + "url": page_url, + "description": description or "", + "repo": repo_name, + "version": version, + }) + + # Write sitemap (overwrites Pelican's) + sitemap_xml = generate_sitemap(sitemap_pages, site["url"]) + (OUTPUT_DIR / "sitemap.xml").write_text(sitemap_xml, encoding="utf-8") + + # Write robots.txt (overwrites Pelican's static one) + robots_txt = generate_robots_txt(site["url"]) + (OUTPUT_DIR / "robots.txt").write_text(robots_txt, encoding="utf-8") + + # Write search index + search_json = json.dumps(search_pages, ensure_ascii=False, indent=2) + (OUTPUT_DIR / "search-index.json").write_text(search_json, encoding="utf-8") + + # Minify Pelican's CSS and JS + css_path = OUTPUT_DIR / "theme" / "css" / "main.css" + if css_path.exists(): + css_path.write_text(minify_css(css_path.read_text(encoding="utf-8")), encoding="utf-8") + + docs_css_path = OUTPUT_DIR / "theme" / "css" / "docs.css" + if docs_css_path.exists(): + docs_css_path.write_text(minify_css(docs_css_path.read_text(encoding="utf-8")), encoding="utf-8") + + for js_path in (OUTPUT_DIR / "theme" / "js").rglob("*.js") if (OUTPUT_DIR / "theme" / "js").exists() else []: + js_path.write_text(minify_js(js_path.read_text(encoding="utf-8")), encoding="utf-8") + + print(f"Docs built at {OUTPUT_DIR}") + print(f" Docs pages: {len(search_pages)}") + print(f" Sitemap URLs: {len(sitemap_pages)}") + + +if __name__ == "__main__": + main() diff --git a/templates/docs/base.html b/templates/docs/base.html new file mode 100644 index 0000000..718343d --- /dev/null +++ b/templates/docs/base.html @@ -0,0 +1,97 @@ + + + + + + + + {% block title %}{{ site.name }} Docs{% endblock %} + + + + + + + + + + + + + + + + + + {% if page.type == "doc" and page.last_updated %} + + + {% endif %} + + + + + + + {% block json_ld %}{% endblock %} + {% block head_extra %}{% endblock %} + + + +
+ +
+ +
+ {% block sidebar %}{% endblock %} +
+ {% block content %}{% endblock %} +
+
+ + + + + + + diff --git a/templates/docs/doc.html b/templates/docs/doc.html new file mode 100644 index 0000000..71ff2e7 --- /dev/null +++ b/templates/docs/doc.html @@ -0,0 +1,99 @@ +{% extends "templates/docs/base.html" %} + +{% block title %}{{ page.title }} | {{ repo.display_name }} Docs{% endblock %} +{% block description %}{{ page.description or site.description }}{% endblock %} + +{% block sidebar %} + +{% endblock %} + +{% block content %} +
+ {% if repo.version != repo.latest_version %} +
+ This is an older version of the {{ repo.display_name }} docs. + View the latest version. +
+ {% endif %} + + + +

{{ page.title }}

+ + {% if page.toc_items %} + + {% endif %} + + {{ page.content | safe }} + + {% if page.last_updated %} +

Last updated: {{ page.last_updated }}

+ {% endif %} + + {% if page.is_stale %} +
+ This doc was last updated {{ page.months_since_update }} months ago. + Some information may be outdated. +
+ {% endif %} +
+{% endblock %} + +{% block json_ld %} + +{% endblock %} diff --git a/themes/dex/static/css/docs.css b/themes/dex/static/css/docs.css new file mode 100644 index 0000000..f9b9169 --- /dev/null +++ b/themes/dex/static/css/docs.css @@ -0,0 +1,289 @@ +/* ── Docs layout: sidebar + content ─────────────────────── */ + +.docs-layout { + display: flex; + margin-top: 60px; /* matches nav height */ + min-height: calc(100vh - 60px); +} + +.docs-sidebar { + width: 280px; + flex-shrink: 0; + background: var(--color-bg-alt, #f8f9fa); + border-right: 1px solid var(--color-border, #dee2e6); + padding: 1.5rem; + overflow-y: auto; + position: sticky; + top: 60px; + height: calc(100vh - 60px); +} + +.docs-content { + flex: 1; + padding: 2rem 3rem; + max-width: 800px; + min-width: 0; +} + +/* ── Version selector ──────────────────────────────────── */ + +.docs-version-select { + margin-bottom: 1.5rem; +} + +.docs-version-select label { + font-size: 0.6875rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--color-text-muted, #6c757d); + display: block; + margin-bottom: 0.375rem; +} + +.docs-version-select select { + width: 100%; + padding: 0.5rem; + font-size: 0.8125rem; + border: 1px solid var(--color-border, #dee2e6); + border-radius: 6px; + background: #fff; + color: var(--color-text, #212529); + cursor: pointer; +} + +/* ── Sidebar nav ────────────────────────────────────────── */ + +.docs-sidebar-nav { + list-style: none; + margin: 0; + padding: 0; +} + +.docs-sidebar-nav li { + margin-bottom: 0.125rem; +} + +.docs-sidebar-nav a { + display: block; + padding: 0.375rem 0.5rem; + font-size: 0.8125rem; + border-radius: 4px; + color: var(--color-text-muted, #6c757d); + text-decoration: none; + transition: background 0.15s, color 0.15s; +} + +.docs-sidebar-nav a:hover { + background: var(--color-bg-hover, #e9ecef); + color: var(--color-text, #212529); +} + +.docs-sidebar-nav a.active { + background: var(--color-primary, #00897b); + color: #fff; + font-weight: 600; +} + +/* ── Breadcrumbs ────────────────────────────────────────── */ + +.docs-breadcrumbs { + font-size: 0.75rem; + margin-bottom: 1rem; + color: var(--color-text-muted, #6c757d); +} + +.docs-breadcrumbs ol { + list-style: none; + display: flex; + flex-wrap: wrap; + gap: 0.375rem; + padding: 0; + margin: 0; +} + +.docs-breadcrumbs li:not(:last-child)::after { + content: "/"; + margin-left: 0.375rem; + color: var(--color-border, #dee2e6); +} + +.docs-breadcrumbs li[aria-current="page"] { + font-weight: 600; + color: var(--color-text, #212529); +} + +/* ── On this page TOC ───────────────────────────────────── */ + +.docs-page-toc { + background: var(--color-bg-alt, #f8f9fa); + padding: 1rem 1.25rem; + border-radius: 8px; + margin-bottom: 1.5rem; + border: 1px solid var(--color-border, #dee2e6); +} + +.docs-page-toc strong { + font-size: 0.6875rem; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--color-text-muted, #6c757d); +} + +.docs-page-toc ul { + list-style: none; + padding: 0; + margin: 0.5rem 0 0; +} + +.docs-page-toc li { + margin-bottom: 0.125rem; +} + +.docs-page-toc a { + font-size: 0.8125rem; +} + +/* ── Version banner ─────────────────────────────────────── */ + +.docs-version-banner { + background: #fff3cd; + border: 1px solid #ffc107; + padding: 0.75rem 1rem; + border-radius: 6px; + margin-bottom: 1.5rem; + font-size: 0.8125rem; +} + +/* ── Last updated ────────────────────────────────────────── */ + +.docs-last-updated { + font-size: 0.75rem; + color: var(--color-text-muted, #6c757d); + margin-top: 2rem; + padding-top: 1rem; + border-top: 1px solid var(--color-border, #dee2e6); +} + +.docs-staleness-banner { + background: #fff3cd; + border: 1px solid #ffc107; + padding: 0.75rem 1rem; + border-radius: 6px; + margin-top: 1rem; + font-size: 0.8125rem; +} + +/* ── Search ──────────────────────────────────────────────── */ + +.docs-search-toggle { + background: none; + border: none; + cursor: pointer; + font-size: 1rem; + padding: 0.375rem; + color: var(--color-text-muted, #6c757d); + transition: color 0.15s; +} + +.docs-search-toggle:hover { + color: var(--color-text, #212529); +} + +.docs-search-overlay { + position: fixed; + inset: 0; + z-index: 999; + background: rgba(0, 0, 0, 0.35); + display: flex; + flex-direction: column; + align-items: center; + padding-top: 20vh; +} + +.docs-search-input { + width: 90%; + max-width: 520px; + padding: 0.75rem 1rem; + font-size: 1.125rem; + border: 2px solid var(--color-primary, #00897b); + border-radius: 8px; + outline: none; + background: #fff; +} + +.docs-search-results { + width: 90%; + max-width: 520px; + max-height: 50vh; + overflow-y: auto; + background: #fff; + border-radius: 8px; + margin-top: 0.5rem; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.1); +} + +.docs-search-result { + display: block; + padding: 0.75rem 1rem; + border-bottom: 1px solid var(--color-border, #dee2e6); + text-decoration: none; + color: inherit; + transition: background 0.15s; +} + +.docs-search-result:last-child { + border-bottom: none; +} + +.docs-search-result:hover { + background: var(--color-bg-alt, #f8f9fa); +} + +.docs-search-result-title { + display: block; + font-weight: 600; + font-size: 0.875rem; + color: var(--color-text, #212529); +} + +.docs-search-result-desc { + display: block; + font-size: 0.75rem; + color: var(--color-text-muted, #6c757d); + margin-top: 0.125rem; +} + +.docs-search-result-meta { + display: block; + font-size: 0.6875rem; + color: var(--color-text-muted, #6c757d); + margin-top: 0.125rem; +} + +.docs-search-empty { + padding: 1.5rem; + text-align: center; + color: var(--color-text-muted, #6c757d); +} + +mark { + background: #cfe8ff; + padding: 0 0.125em; + border-radius: 2px; +} + +/* ── Responsive ──────────────────────────────────────────── */ + +@media (max-width: 768px) { + .docs-sidebar { + display: none; + } + .docs-content { + padding: 1.5rem; + } + .docs-search-overlay { + padding-top: 15vh; + } +} diff --git a/themes/dex/static/css/main.css b/themes/dex/static/css/main.css index 7b99d68..04e850a 100644 --- a/themes/dex/static/css/main.css +++ b/themes/dex/static/css/main.css @@ -17,6 +17,8 @@ --color-bg: #ffffff; --color-bg-light: #eceff1; --color-bg-code: #1e272e; + --color-bg-alt: #f8f9fa; + --color-bg-hover: #e9ecef; --color-border: #cfd8dc; --font-sans: system-ui, -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; diff --git a/themes/dex/static/js/search.js b/themes/dex/static/js/search.js new file mode 100644 index 0000000..5531e0c --- /dev/null +++ b/themes/dex/static/js/search.js @@ -0,0 +1,116 @@ +(function () { + 'use strict'; + var searchIndex = []; + var overlay = null; + var input = null; + var results = null; + var toggle = null; + + function esc(str) { + var d = document.createElement('div'); + d.appendChild(document.createTextNode(str)); + return d.innerHTML; + } + + function init() { + fetch('/search-index.json') + .then(function (r) { return r.json(); }) + .then(function (data) { searchIndex = data; }) + .catch(function () {}); + + input = document.createElement('input'); + input.type = 'search'; + input.placeholder = 'Search docs...'; + input.className = 'docs-search-input'; + input.setAttribute('aria-label', 'Search documentation'); + + results = document.createElement('div'); + results.className = 'docs-search-results'; + results.style.display = 'none'; + + overlay = document.createElement('div'); + overlay.className = 'docs-search-overlay'; + overlay.style.display = 'none'; + overlay.appendChild(input); + overlay.appendChild(results); + document.body.appendChild(overlay); + + toggle = document.createElement('button'); + toggle.className = 'docs-search-toggle'; + toggle.textContent = '\uD83D\uDD0D'; + toggle.setAttribute('aria-label', 'Open search'); + toggle.addEventListener('click', function (e) { + e.preventDefault(); + overlay.style.display = 'flex'; + input.focus(); + }); + + var navLinks = document.querySelector('.nav-links'); + if (navLinks) { + var li = document.createElement('li'); + li.appendChild(toggle); + navLinks.appendChild(li); + } + + input.addEventListener('input', onInput); + input.addEventListener('keydown', onKeydown); + overlay.addEventListener('click', onOverlayClick); + } + + function onInput() { + var q = this.value.toLowerCase().trim(); + if (q.length < 2) { + results.style.display = 'none'; + return; + } + var hits = searchIndex.filter(function (p) { + return p.title.toLowerCase().indexOf(q) !== -1 || + p.description.toLowerCase().indexOf(q) !== -1; + }); + if (hits.length === 0) { + results.innerHTML = '
No results found
'; + } else { + results.innerHTML = hits.slice(0, 20).map(function (h) { + return '' + + '' + highlight(esc(h.title), q) + '' + + '' + highlight(esc(h.description), q) + '' + + '' + esc(h.repo) + ' ' + esc(h.version) + '' + + ''; + }).join(''); + } + results.style.display = 'block'; + } + + function onKeydown(e) { + if (e.key === 'Escape') { + close(); + } + } + + function onOverlayClick(e) { + if (e.target === overlay) { + close(); + } + } + + function close() { + overlay.style.display = 'none'; + results.style.display = 'none'; + input.value = ''; + if (toggle) toggle.focus(); + } + + function highlight(text, query) { + var idx = text.toLowerCase().indexOf(query); + if (idx === -1) return text; + return text.slice(0, idx) + + '' + text.slice(idx, idx + query.length) + '' + + text.slice(idx + query.length); + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', init); + } else { + init(); + } +})(); diff --git a/uv.lock b/uv.lock index c307317..b20222f 100644 --- a/uv.lock +++ b/uv.lock @@ -28,9 +28,16 @@ name = "dataenginex-website" version = "0.1.0" source = { virtual = "." } dependencies = [ + { name = "jinja2" }, + { name = "markdown" }, { name = "pelican", extra = ["markdown"] }, - { name = "pelican-sitemap" }, { name = "pygments" }, + { name = "pyyaml" }, +] + +[package.optional-dependencies] +images = [ + { name = "pillow" }, ] [package.dev-dependencies] @@ -40,10 +47,14 @@ dev = [ [package.metadata] requires-dist = [ + { name = "jinja2", specifier = ">=3.1" }, + { name = "markdown", specifier = ">=3.6" }, { name = "pelican", extras = ["markdown"], specifier = ">=4.10" }, - { name = "pelican-sitemap", specifier = ">=1.0" }, + { name = "pillow", marker = "extra == 'images'", specifier = ">=10.0" }, { name = "pygments", specifier = ">=2.18" }, + { name = "pyyaml", specifier = ">=6.0" }, ] +provides-extras = ["images"] [package.metadata.requires-dev] dev = [{ name = "poethepoet", specifier = ">=0.31" }] @@ -218,15 +229,61 @@ markdown = [ ] [[package]] -name = "pelican-sitemap" -version = "1.2.2" +name = "pillow" +version = "12.2.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pelican" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/c1/d9e1d539decfba947d711bcb37427ad15914557a728418bd928d56e58a32/pelican_sitemap-1.2.2.tar.gz", hash = "sha256:93f72d035dd0ec2f5cee38b2766bbd7f6a54bf186635ae1261a6deac422dcfb7", size = 8013, upload-time = "2025-10-12T15:24:36.727Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/c2/d2f98364b49a40bac9f315f29d8c22322b47bc4ebdec6a8e3a4f657ebc24/pelican_sitemap-1.2.2-py3-none-any.whl", hash = "sha256:09ba150dd1b54ef855f18e7ec920160d7118e932e0dac1c6e8bf3a41e938e682", size = 8046, upload-time = "2025-10-12T15:24:35.678Z" }, + { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, + { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, + { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, + { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, + { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, + { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, + { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, + { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, + { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, + { url = "https://files.pythonhosted.org/packages/23/c4/7349421080b12fb35414607b8871e9534546c128a11965fd4a7002ccfbee/pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e", size = 6375896, upload-time = "2026-04-01T14:44:11.197Z" }, + { url = "https://files.pythonhosted.org/packages/3f/82/8a3739a5e470b3c6cbb1d21d315800d8e16bff503d1f16b03a4ec3212786/pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b", size = 7081266, upload-time = "2026-04-01T14:44:13.947Z" }, + { url = "https://files.pythonhosted.org/packages/c3/25/f968f618a062574294592f668218f8af564830ccebdd1fa6200f598e65c5/pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06", size = 2463508, upload-time = "2026-04-01T14:44:16.312Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, + { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, + { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, + { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, + { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, + { url = "https://files.pythonhosted.org/packages/36/a3/f9a77144231fb8d40ee27107b4463e205fa4677e2ca2548e14da5cf18dce/pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4", size = 7115667, upload-time = "2026-04-01T14:44:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fc/ac4ee3041e7d5a565e1c4fd72a113f03b6394cc72ab7089d27608f8aaccb/pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4", size = 6538966, upload-time = "2026-04-01T14:44:35.252Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a8/27fb307055087f3668f6d0a8ccb636e7431d56ed0750e07a60547b1e083e/pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea", size = 7238241, upload-time = "2026-04-01T14:44:37.875Z" }, + { url = "https://files.pythonhosted.org/packages/ad/4b/926ab182c07fccae9fcb120043464e1ff1564775ec8864f21a0ebce6ac25/pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24", size = 6379592, upload-time = "2026-04-01T14:44:40.336Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c4/f9e476451a098181b30050cc4c9a3556b64c02cf6497ea421ac047e89e4b/pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98", size = 7085542, upload-time = "2026-04-01T14:44:43.251Z" }, + { url = "https://files.pythonhosted.org/packages/00/a4/285f12aeacbe2d6dc36c407dfbbe9e96d4a80b0fb710a337f6d2ad978c75/pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453", size = 2465765, upload-time = "2026-04-01T14:44:45.996Z" }, + { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, + { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, + { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, + { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, + { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, + { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, + { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, + { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, + { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, + { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, + { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, + { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, + { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, + { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, + { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, ] [[package]] From fc0a7df20d12468e1adcb4a5f1d24971dafca864 Mon Sep 17 00:00:00 2001 From: data-literate Date: Thu, 11 Jun 2026 22:14:04 -0500 Subject: [PATCH 2/4] org page updates --- content/pages/changelog.md | 2 +- content/pages/why.md | 29 +++++++++-------------------- themes/dex/templates/base.html | 2 +- themes/dex/templates/index.html | 10 +++++----- 4 files changed, 16 insertions(+), 27 deletions(-) diff --git a/content/pages/changelog.md b/content/pages/changelog.md index dacb82b..24a257d 100644 --- a/content/pages/changelog.md +++ b/content/pages/changelog.md @@ -11,7 +11,7 @@ Full release notes are on [GitHub Releases](https://github.com/TheDataEngineX/DE The current version is tracked in [`pyproject.toml`](https://github.com/TheDataEngineX/DEX/blob/main/pyproject.toml). -**Current:** `v1.1.1` — CLI compatibility fix (removed `rich` dep; Reflex pin conflict). `v1.1.0` added enterprise auth (JWKS/RS256), RBAC, SCIM v2, LiteLLM + vLLM providers, Langfuse LLM tracing, LightRAG graph retrieval, domain Prometheus metrics. +**Current:** `v0.4.0` — Post-scope-reset milestone with focused feature set. See [GitHub Releases](https://github.com/TheDataEngineX/dataenginex/releases) for the full history. [View all releases on GitHub →](https://github.com/TheDataEngineX/DEX/releases) diff --git a/content/pages/why.md b/content/pages/why.md index 54f645a..31d7492 100644 --- a/content/pages/why.md +++ b/content/pages/why.md @@ -1,7 +1,7 @@ --- -title: Why DataEngineX — Complete Data + ML + AI Engineering +title: Why DataEngineX — Data + ML + AI Engineering slug: why -description: DataEngineX is an open-source, self-hosted Python framework for the complete platform for Data + ML + AI engineering — production-ready pipelines, ML lifecycle, and AI agents unified under one dex.yaml. Native integrations with Airflow, MLflow, LangChain, and more. +description: DataEngineX is an open-source, self-hosted Python framework for data pipelines, ML lifecycle, and AI agents — unified under one dex.yaml. Native integrations with DuckDB, Spark, MLflow, LiteLLM, and more. template: page --- @@ -28,7 +28,7 @@ DataEngineX does not rip out those tools. It ships production-ready implementati ## One File, Entire Stack -DataEngineX is the complete platform for your Data + ML + AI engineering lifecycle. Define once, run anywhere: +DataEngineX is a Python framework for your Data + ML + AI engineering lifecycle. Define once, run anywhere: ```yaml # dex.yaml @@ -51,16 +51,12 @@ ai: - name: analyst tools: [sql, search] -server: - auth: jwt - rate_limit: 100/min - observability: metrics: prometheus tracing: otel ``` -One `dex serve` command starts everything. No glue code. +Use it from Python, the `dex` CLI, or the self-hosted web UI. No glue code. --- @@ -68,11 +64,11 @@ One `dex serve` command starts everything. No glue code. DataEngineX ships production-ready implementations of every layer: -- **Data pipelines** — DuckDB engine, S3/GCS connectors, quality gates, medallion lakehouse +- **Data pipelines** — DuckDB engine, S3/GCS connectors, quality gates, medallion lakehouse, optional Spark backend - **ML lifecycle** — experiment tracking, model registry, training, serving, drift detection - **AI agents** — LLM routing via LiteLLM, hybrid BM25+dense retrieval, LangGraph runtime -- **API server** — FastAPI with JWT auth, rate limiting, RBAC, SCIM v2 -- **Observability** — Prometheus metrics, OpenTelemetry tracing, Langfuse LLM tracing +- **DEX Studio** — self-hosted web UI (FastAPI + Jinja2 + HTMX, port 7860) +- **Observability** — structlog structured logging, Prometheus metrics, OpenTelemetry tracing — built in, no extra packages When you already run external tools, DEX integrates natively — not against them. Airflow only schedules. MLflow only tracks. LangChain only chains. DEX gives you the complete end-to-end stack: @@ -82,7 +78,7 @@ When you already run external tools, DEX integrates natively — not against the | MLflow | Point `ml.backend: mlflow` — tracking_uri wired automatically | | LangChain / LiteLLM | LLM routing layer — 100+ providers, swap without code changes | | Qdrant | Vector store for RAG — configured in `ai.retrieval` | -| Langfuse | LLM observability — enabled via `[observability]` extra | +| Langfuse | LLM observability — enabled via config, not code | | PySpark | Big-data backend — swap `data.backend: spark` in config | --- @@ -93,8 +89,7 @@ Opinionated defaults, zero lock-in. Every layer is swappable: ```bash pip install "dataenginex[cloud]" # S3 + GCS + BigQuery connectors -pip install "dataenginex[auth]" # RS256/JWKS + SCIM v2 + LDAP sync -pip install "dataenginex[observability]" # Langfuse LLM tracing +pip install "dataenginex[qdrant]" # Qdrant vector store pip install 'litellm>=1.83.3' --no-deps # 100+ LLM providers (separate install) ``` @@ -102,12 +97,6 @@ The config stays the same. The backend changes. --- -## Enterprise-Ready Auth - -RBAC, SCIM v2 user provisioning, LDAP/AD sync, and OIDC federation (RS256/JWKS) ship as first-class extras — not bolt-ons. Enable them with env vars, not code changes. - ---- - ## Self-Hosted Your data never leaves your infrastructure. No SaaS subscription. No vendor lock-in. Run on a VPS, K3s cluster, or bare metal. diff --git a/themes/dex/templates/base.html b/themes/dex/templates/base.html index f1796e0..d13d62a 100644 --- a/themes/dex/templates/base.html +++ b/themes/dex/templates/base.html @@ -70,7 +70,7 @@ "priceCurrency": "USD" }, "softwareRequirements": "Python 3.13+", - "softwareVersion": "1.1.1", + "softwareVersion": "0.4.0", "downloadUrl": "https://pypi.org/project/dataenginex/", "installUrl": "https://docs.thedataenginex.org/getting-started/", "sameAs": [ diff --git a/themes/dex/templates/index.html b/themes/dex/templates/index.html index d248f9d..d259d50 100644 --- a/themes/dex/templates/index.html +++ b/themes/dex/templates/index.html @@ -12,7 +12,7 @@
- v1.1.2 + v0.4.0 stable

@@ -321,7 +321,7 @@

One core. Four components.<

- v1.1.2 + v0.4.0

dataenginex

pip install dataenginex

@@ -415,9 +415,9 @@

Ready to unify your stack?

Extras
- 
pip install "dataenginex[cloud]"         # S3 · GCS · BigQuery
-pip install "dataenginex[observability]" # Langfuse LLM tracing
-pip install 'litellm>=1.83.3' --no-deps  # 100+ LLM providers
+ 
pip install "dataenginex[cloud]"  # S3 · GCS · BigQuery
+pip install "dataenginex[qdrant]" # Qdrant vector store
+pip install 'litellm>=1.83.3' --no-deps # 100+ LLM providers
From 374d016b3992b54054e40c89e91f98760b745e8c Mon Sep 17 00:00:00 2001 From: Jayapal Reddy Myaka Date: Thu, 11 Jun 2026 22:14:33 -0500 Subject: [PATCH 3/4] Potential fix for pull request finding 'CodeQL / Workflow does not contain permissions' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/sync.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 1a6663e..d2e6bf2 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -18,6 +18,9 @@ on: required: true default: 'v0.1.0' +permissions: + contents: write + jobs: sync: runs-on: ubuntu-latest From 0ccb8c4c3bf582cf90f40266134405b6ffa63431 Mon Sep 17 00:00:00 2001 From: data-literate Date: Thu, 11 Jun 2026 22:19:56 -0500 Subject: [PATCH 4/4] fix: add deployments:write permission to deploy-preview job --- .github/workflows/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 376c359..7c5f473 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -78,6 +78,7 @@ jobs: environment: name: preview permissions: + deployments: write pull-requests: write steps: - name: Download build artifact