jaymyaka · Aug 26, 2026 · Aug 15, 2026 · Aug 15, 2026 · Aug 25, 2026 · Aug 26, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: astral-sh/setup-uv@v9.0.0
+      - uses: astral-sh/setup-uv@v10.0.1
         with:
           version: "latest"
           python-version: "3.13"
@@ -29,7 +29,7 @@ jobs:
           UV_PROJECT_ENVIRONMENT: .venv
       - uses: reviewdog/action-actionlint@v1
 
-      - run: uv run poe quality
+      - run: uv run poe check-all
 
       - run: uv build
 
@@ -39,14 +39,14 @@ jobs:
     needs: quality
     steps:
       - uses: actions/checkout@v7
-      - uses: astral-sh/setup-uv@v9.0.0
+      - uses: astral-sh/setup-uv@v10.0.1
         with:
           version: "latest"
           python-version: "3.13"
       - run: uv sync --group ml
         env:
           UV_PROJECT_ENVIRONMENT: .venv
-      - run: uv run poe test-cov-core
+      - run: uv run poe test-cov
       - uses: codecov/codecov-action@v7
         with:
           flags: dataenginex
@@ -60,7 +60,7 @@ jobs:
     if: github.event_name == 'schedule'
     steps:
       - uses: actions/checkout@v7
-      - uses: astral-sh/setup-uv@v9.0.0
+      - uses: astral-sh/setup-uv@v10.0.1
         with:
           version: "latest"
           python-version: "3.13"

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -26,7 +26,7 @@ jobs:
       - uses: actions/checkout@v7
         with:
           ref: ${{ inputs.ref || github.ref }}
-      - uses: astral-sh/setup-uv@v9.0.0
+      - uses: astral-sh/setup-uv@v10.0.1
         with:
           version: "latest"
           python-version: "3.13"
@@ -61,7 +61,7 @@ jobs:
       - uses: actions/checkout@v7
         with:
           ref: ${{ inputs.ref || github.ref }}
-      - uses: astral-sh/setup-uv@v9.0.0
+      - uses: astral-sh/setup-uv@v10.0.1
         with:
           version: "latest"
       - name: Extract version

diff --git a/.gitignore b/.gitignore
@@ -66,6 +66,8 @@ desktop.ini
 *.log
 *.sqlite3
 *.duckdb
+*.duckdb.wal
+*.duckdb.tmp
 logs/
 .dex/
 
@@ -87,11 +89,13 @@ Notes.txt
 
 # Claude Code — local overrides (personal, never commit)
 .claude/settings.local.json
+.claude/.headroom_wrap_marker.json
 .superpowers/
 
 # AI Coding Assistant
 .continue/
 .playwright-mcp/
+.serena/
 
 # Local task notes (tracked separately)
 tasks/todo.md
diff --git a/.gitleaks.toml b/.gitleaks.toml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,12 +1,32 @@
 # Changelog
 
-See [docs/release-notes.md](docs/release-notes.md) for the complete release history.
+See [src/dataenginex/RELEASE_NOTES.md](src/dataenginex/RELEASE_NOTES.md) for the complete release history.
 
 All notable changes to `dataenginex` will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.0] - 2026-08-12
+
+### Changed
+
+- Architecture restructure: `core/`, `data/`, `ml/`, `ai/`, `warehouse/`, `lakehouse/` replaced by `domains/`, `providers/`, `runtime/`, `foundation/`, `interfaces/`
+- Domains organized by concern: `domains/ai`, `domains/analytics`, `domains/data`, `domains/execution`, `domains/governance`, `domains/ml`, `domains/plugins`, `domains/security`
+- New `application/` layer for domain services (governance, projects, runs, resources)
+- New `bootstrap/` module for engine wiring and startup
+- New `orm/` module for SQLAlchemy models
+- New `spark/` module for PySpark connector
+- `duckdb/` extracted as standalone module
+- Version bumped to 0.7.0
+
+## [0.6.0] - 2026-08-01
+
+### Changed
+
+- Monorepo restructuring (ADR-0001): single uv workspace with members in `packages/` and `apps/`
+- Version bumped to 0.6.0
+
 ## [0.5.2] - 2026-07-28
 
 ### Added
@@ -21,6 +41,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `sentence-transformers` 5.x mock fix in `test_ml_finetune.py` (submodule import path)
 - Documentation updated across all guides to reflect current architecture and test stats
 
+[0.5.2]: https://github.com/TheDataEngineX/dataenginex/compare/v0.5.1...v0.5.2
+
 ## [0.5.1] - 2026-07-27
 
 ### Changed
@@ -144,4 +166,6 @@ ______________________________________________________________________
 [0.4.2]: https://github.com/TheDataEngineX/dataenginex/compare/v0.4.1...v0.4.2
 [0.5.0]: https://github.com/TheDataEngineX/dataenginex/compare/v0.4.2...v0.5.0
 [0.5.1]: https://github.com/TheDataEngineX/dataenginex/compare/v0.5.0...v0.5.1
-[0.5.2]: https://github.com/TheDataEngineX/dataenginex/compare/v0.5.1...v0.5.2
+[0.5.2]: https://github.com/TheDataEngineX/dataenginex/compare/v0.5.1...v0.5.2
+[0.6.0]: https://github.com/TheDataEngineX/dataenginex/compare/v0.5.2...v0.6.0
+[0.7.0]: https://github.com/TheDataEngineX/dataenginex/compare/v0.6.0...v0.7.0
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -9,9 +9,12 @@ Goal is to save Claude code tokens for lower cost without losing quality.
 
 **DataEngineX** — unified Data + ML + AI library. Config-driven, self-hosted, local-first. Pure Python — no bundled HTTP server.
 
-| Package | Location | Purpose |
-|---------|----------|---------|
-| `dataenginex` | `src/dataenginex/` | Core library — config, registry, CLI, pipelines, ML, AI, PrivacyGuard |
+| Directory | Purpose |
+|-----------|---------|
+| `src/dataenginex/` | Core library — engine, domains, providers, foundation, CLI |
+| `schemas/` | Published JSON Schema for manifests |
+| `tests/` | Unit, integration, and architecture tests |
+| `docs/` | Design specs and integration plans |
 
 **Stack:** Python 3.13+ · DuckDB · SQLite (WAL, persistence store) · structlog · Pydantic · Click · pyarrow · croniter · httpx · prometheus-client · uv · Ruff · mypy strict · pytest
 
@@ -27,35 +30,28 @@ ______________________________________________________________________
 # Quality
 uv run poe lint           # Ruff lint
 uv run poe lint-fix       # Ruff lint + auto-fix
-uv run poe typecheck      # mypy --strict (src/dataenginex/ only)
+uv run poe typecheck      # mypy --strict
 uv run poe check-all      # lint + typecheck + test
 
 # Test
-uv run poe test           # All tests
-uv run poe test-unit      # Unit tests only
-uv run poe test-integration  # Integration tests only
-uv run poe test-cov       # Tests with coverage
+uv run poe test           # Core library tests
+uv run poe test-cov       # tests + coverage
 
 # CLI
 dex validate dex.yaml     # Validate config file
 dex version               # Show version + environment
 
-# Dev
-uv run poe dev            # Dev server (uvicorn reload, port 17000) — for examples/API testing only (dataenginex has no built-in HTTP server)
-
 # Deps
-uv run poe uv-sync        # Sync deps from lockfile
-uv run poe uv-lock        # Regenerate lockfile
+uv sync                   # Sync dependencies
+uv lock                   # Regenerate lockfile
 uv run poe security       # Audit deps for vulnerabilities
 ```
 
 ## Optional Extras
 
 ```bash
 pip install "dataenginex[cloud]"        # S3, GCS, BigQuery connectors
-pip install "dataenginex[postgres]"     # asyncpg for Postgres lineage
 pip install "dataenginex[qdrant]"       # Qdrant vector store
-pip install "dataenginex[queue]"        # arq background jobs
 pip install "dataenginex[delta]"        # Delta Lake connector
 pip install "dataenginex[pytorch]"      # PyTorch ML
 pip install 'litellm>=1.83.3' --no-deps # LLM routing (separate: pins python-dotenv)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -2,18 +2,27 @@
 
 Thank you for your interest in contributing to DataEngineX!
 
-**→ Full contributor guide: [docs/contributing.md](docs/contributing.md)**
-
-Quick essentials:
-
-- Development setup: [docs/development.md](docs/development.md)
-- Governance & legal: See [docs/contributing.md](docs/contributing.md)
-- Code of Conduct: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
-
 ## Quick Start
 
 1. Fork and create a feature branch from `main`
 1. Run local checks: `uv run poe check-all`
-1. Open a PR using `.github/PULL_REQUEST_TEMPLATE.md`
+1. Open a PR
+
+## Development Setup
+
+```bash
+git clone https://github.com/TheDataEngineX/dataenginex && cd dataenginex
+uv sync
+uv run poe check-all          # lint + typecheck + tests
+```
+
+## Code Style
+
+- Python 3.13+, `from __future__ import annotations` in all public modules
+- Ruff lint + mypy strict
+- structlog for logging
+- Pydantic for data validation
+
+## Code of Conduct
 
-See [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md) for complete guidelines on commits, code style, and PR process.
+See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md).
diff --git a/README.md b/README.md
@@ -73,21 +73,32 @@ ______________________________________________________________________
 
 ```text
 src/dataenginex/
+├── ai/                 # LLM providers, routing, agents, retrieval, tools, memory, workflows
+├── api/                # Pydantic response models, GraphQL schema (no HTTP server)
+├── application/        # Domain services (governance, projects, runs, resources)
+├── bootstrap/          # Engine wiring and startup
 ├── cli/                # `dex` CLI: validate, version, init
-├── config/             # dex.yaml schema, loader
-├── core/               # Base ABCs, registry, exceptions
+├── config/             # dex.yaml schema, loader, env-var resolution
+├── core/               # Core abstractions and shared utilities
+├── data/               # Connectors (CSV, Parquet, DuckDB, REST, HTTP, Kafka), transforms
+├── domains/            # Domain logic (ai, analytics, data, execution, governance, ml, plugins, security)
+├── duckdb/             # DuckDB connection and helpers
 ├── engine.py           # DexEngine — entry point
-├── store.py            # DexStore — SQLite (WAL) persistence
-├── api/                # Pydantic response models (no HTTP server)
-├── data/               # Connectors, pipeline runner, transforms, quality
-├── ml/                 # Classical ML: training, registry, serving, drift
-├── ai/                 # LLM providers, agents, RAG, vector store, memory
-├── orchestration/      # Scheduler, background workers
+├── engines/            # Engine implementations and registry (DuckDB, Spark, base)
+├── foundation/         # Base ABCs, plugin contracts, errors, events, resources
+├── interfaces/         # Gateway, embedded, external service interfaces
+├── lakehouse/          # Bronze/Silver/Gold lakehouse storage and catalog
 ├── middleware/         # structlog, Prometheus metrics
-├── lakehouse/          # Storage backends, catalog, partitioning
-├── warehouse/          # Transforms, lineage
+├── ml/                 # ML training, registry, serving, drift, features
+├── orchestration/      # Scheduler, background workers, queue backends
+├── orm/                # SQLAlchemy ORM models
+├── plugins/            # Entry-point plugin discovery
+├── providers/          # Backend implementations (connectors, vector, tracking)
+├── runtime/            # Compiler, registry, ML runtime, sandbox
 ├── secops/             # PII detection, masking, audit
-└── plugins/            # Entry-point discovery
+├── spark/              # PySpark connector (SQL, streaming, ML, catalog, lineage)
+├── store.py            # DexStore — DuckDB (WAL) persistence
+└── warehouse/          # Warehouse operations
 ```
 
 ______________________________________________________________________
@@ -104,8 +115,6 @@ dex validate dex.yaml         # validate a config file
 dex version                   # show version + environment
 ```
 
-See [docs/development.md](docs/development.md) for the full setup.
-
 ______________________________________________________________________
 
 ## Want the full workbench?
@@ -128,12 +137,9 @@ ______________________________________________________________________
 
 | Guide | Description |
 | --- | --- |
-| [Quickstart](docs/quickstart.md) | Get running in 5 minutes |
-| [Architecture](docs/architecture.md) | System design and patterns |
-| [Development](docs/development.md) | Local setup and workflow |
-| [API Reference](docs/api-reference/index.md) | Module-by-module reference |
 | [CHANGELOG](CHANGELOG.md) | Release history |
+| [Release Notes](src/dataenginex/RELEASE_NOTES.md) | Package release details |
 
 ______________________________________________________________________
 
-**License:** MIT • **Python:** 3.13+ • **Status:** Pre-1.0 • **Version:** 0.5.2
+**License:** MIT • **Python:** 3.13+ • **Status:** Pre-1.0 • **Version:** 0.7.0
diff --git a/SECURITY.md b/SECURITY.md
@@ -4,7 +4,7 @@
 
 | Version | Supported |
 |---------|-----------|
-| Latest (0.4.x) | ✅ |
+| Latest (0.7.x) | ✅ |
 | Previous minor release | ✅ (security fixes only) |
 | Older versions | ❌ |
 

diff --git a/docker-compose.test.yml b/docker-compose.test.yml