fix: apache-airflow<3.0.0 — resolve 8 Dependabot security alerts + sync dev

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v5
+      - uses: astral-sh/setup-uv@v7
         with:
           version: "latest"
       - uses: actions/setup-python@v6
@@ -28,7 +28,7 @@ jobs:
     needs: lint
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v5
+      - uses: astral-sh/setup-uv@v7
         with:
           version: "latest"
       - uses: actions/setup-python@v6
@@ -42,7 +42,7 @@ jobs:
     needs: [lint, typecheck]
     steps:
       - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v5
+      - uses: astral-sh/setup-uv@v7
         with:
           version: "latest"
       - uses: actions/setup-python@v6

.github/workflows/pypi-publish.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
@@ -145,10 +145,10 @@ jobs:
           TAG: ${{ needs.detect-dataenginex-changes.outputs.current_tag }}
 
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Set up Python 3.12
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.12"
 

.github/workflows/release-dataenginex.yml

@@ -51,7 +51,7 @@ jobs:
           SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }}
 
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -123,7 +123,7 @@ jobs:
 
       - name: Set up Python for SBOM generation
         if: steps.check_tag.outputs.tag_exists == 'false'
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.12"
 

.github/workflows/security.yml

@@ -47,7 +47,7 @@ jobs:
           SLACK_WEBHOOK: ${{ env.SLACK_WEBHOOK }}
           REF: ${{ github.ref_name }}
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Run Trivy repo security scan
         uses: aquasecurity/trivy-action@0.35.0
         with:
@@ -82,7 +82,7 @@ jobs:
       matrix:
         language: [ python, actions ]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v4
         with:

CHANGELOG.md

@@ -7,11 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.6.1] - 2026-03-15
+
 ### Added
 
-- **Plugin system** — `DataEngineXPlugin` ABC, `PluginRegistry` with register/get/all/health_check_all, `discover()` for auto-loading plugins via `importlib.metadata` entry_points (`dataenginex.plugins` group)
-- **Streamlit dashboard** — `BaseDashboard` and `DashboardConfig` (Pydantic) with 4 reusable panels: pipeline status, quality scores, model drift, alerts. Optional `dashboard` dependency group (`streamlit>=1.40.0`)
-- **Coverage gate** — `--cov-fail-under=80` enforced in `test-cov-core` poe task
+- **`SentenceTransformerEmbedder`** — thin wrapper over `sentence-transformers` (`all-MiniLM-L6-v2` default). Install via `uv add 'dataenginex[ml]'`. Implements the `embed_fn` protocol for `RAGPipeline`.
+- **`RAGPipeline.answer(question, llm, ...)`** — full retrieve → augment → generate loop in one call. Combines `build_context` with any `LLMProvider.generate_with_context`.
+- **GitHub Actions upgraded to Node.js 24** — `ci.yml`, `pypi-publish.yml`, `release-dataenginex.yml`, `security.yml` now use `actions/checkout@v6`, `actions/setup-python@v6`, `astral-sh/setup-uv@v7`.
+- **`examples/05_rag_demo.py`** — end-to-end RAG demo with `--embed`, `--llm`, `--model` CLI flags; Ollama fallback to MockProvider; uses `RAGPipeline.answer()`.
 
 ## [0.6.0] - 2026-03-03
 

examples/05_rag_demo.py

@@ -2,83 +2,110 @@
 
 Demonstrates:
     1. Creating an in-memory vector store backend
-    2. Ingesting sample documents into a RAG pipeline
-    3. Querying the pipeline to retrieve relevant context
-    4. Using the MockProvider to generate an LLM response with context
+    2. Embedding documents with SentenceTransformerEmbedder (or hash fallback)
+    3. Ingesting documents into the RAG pipeline
+    4. Using RAGPipeline.answer() for the full retrieve → augment → generate loop
+    5. Swapping providers: MockProvider for tests, OllamaProvider for production
 
-Run::
+Run with hash-based embeddings (no extra deps)::
 
     uv run python examples/05_rag_demo.py
+
+Run with real sentence-transformer embeddings::
+
+    uv sync --extra ml
+    uv run python examples/05_rag_demo.py --embed sentence-transformers
+
+Run against a live Ollama instance::
+
+    uv run python examples/05_rag_demo.py --llm ollama --model qwen3-coder:30b-a3b-q4_K_M
 """
 
 from __future__ import annotations
 
-from dataenginex.ml.llm import MockProvider
-from dataenginex.ml.vectorstore import InMemoryBackend, RAGPipeline
+import argparse
+
+from dataenginex.ml.llm import MockProvider, OllamaProvider
+from dataenginex.ml.vectorstore import InMemoryBackend, RAGPipeline, SentenceTransformerEmbedder
+
+DOCS = [
+    (
+        "DataEngineX is a data engineering and ML platform. "
+        "It provides quality gates, medallion architecture, "
+        "and model lifecycle management."
+    ),
+    (
+        "The medallion architecture organises data into three layers: "
+        "Bronze (raw ingestion), Silver (cleaned and validated), "
+        "and Gold (business-ready aggregations)."
+    ),
+    (
+        "DataEngineX examples demonstrate end-to-end pipelines. "
+        "Examples 07-10 cover API ingestion, PySpark ML, "
+        "feature engineering, and model analysis."
+    ),
+    (
+        "The DataEngineX ML module provides ModelRegistry, DriftDetector, "
+        "RAGPipeline, and LLM provider adapters for production ML workflows."
+    ),
+    (
+        "Quality gates in DataEngineX enforce data completeness, "
+        "consistency, and freshness checks before data moves "
+        "between medallion layers."
+    ),
+]
+
+QUERIES = [
+    "What is the medallion architecture?",
+    "How does the model registry work?",
+    "What are quality gates?",
+]
+
+
+def build_pipeline(use_sentence_transformers: bool) -> RAGPipeline:
+    if use_sentence_transformers:
+        embedder = SentenceTransformerEmbedder()
+        return RAGPipeline(store=InMemoryBackend(dimension=384), embed_fn=embedder, dimension=384)
+    print("Using hash-based embeddings (install dataenginex[ml] for real embeddings)\n")
+    return RAGPipeline(store=InMemoryBackend(dimension=64), dimension=64)
 
 
 def main() -> None:
-    """Run the RAG demo pipeline."""
-    # 1. Set up components
-    backend = InMemoryBackend(dimension=64)
-    pipeline = RAGPipeline(store=backend, dimension=64)
-    llm = MockProvider(default_response="Based on the provided context, here is my answer.")
-
-    # 2. Prepare sample documents
-    texts = [
-        (
-            "DataEngineX is a data engineering and ML platform. "
-            "It provides quality gates, medallion architecture, "
-            "and model lifecycle management."
-        ),
-        (
-            "The medallion architecture organises data into three layers: "
-            "Bronze (raw ingestion), Silver (cleaned and validated), "
-            "and Gold (business-ready aggregations)."
-        ),
-        (
-            "DataEngineX examples demonstrate end-to-end pipelines. "
-            "Examples 07-10 cover API ingestion, PySpark ML, "
-            "feature engineering, and model analysis."
-        ),
-        (
-            "The DataEngineX ML module provides ModelRegistry, DriftDetector, "
-            "RAGPipeline, and LLM provider adapters for production ML workflows."
-        ),
-        (
-            "Quality gates in DataEngineX enforce data completeness, "
-            "consistency, and freshness checks before data moves "
-            "between medallion layers."
-        ),
-    ]
-    doc_ids = ["doc-1", "doc-2", "doc-3", "doc-4", "doc-5"]
-
-    # 3. Ingest documents into the RAG pipeline
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--embed", choices=["hash", "sentence-transformers"], default="hash")
+    parser.add_argument("--llm", choices=["mock", "ollama"], default="mock")
+    parser.add_argument("--model", default="qwen3-coder:30b-a3b-q4_K_M")
+    args = parser.parse_args()
+
+    # 1. Build pipeline
+    pipeline = build_pipeline(use_sentence_transformers=args.embed == "sentence-transformers")
+
+    # 2. Ingest
     print("=== Ingesting documents ===")
-    count = pipeline.ingest(texts, ids=doc_ids)
-    print(f"Ingested {count} documents into the vector store.\n")
-
-    # 4. Query the pipeline
-    queries = [
-        "What is the medallion architecture?",
-        "How does CareerDEX work?",
-        "What are quality gates?",
-    ]
-
-    for query in queries:
-        print(f"--- Query: {query} ---")
-
-        # Retrieve relevant context
-        context = pipeline.build_context(query, top_k=2)
-        print(f"Retrieved context ({len(context)} chars):\n{context[:200]}...\n")
-
-        # Generate response using the LLM with retrieved context
-        response = llm.generate_with_context(question=query, context=context)
-        print(f"LLM Response: {response.text}\n")
-
-    # 5. Show call history
-    print(f"=== LLM was called {len(llm.call_history)} times ===")
-    print("Done!")
+    count = pipeline.ingest(DOCS)
+    print(f"Stored {count} documents\n")
+
+    # 3. Choose LLM provider
+    if args.llm == "ollama":
+        llm = OllamaProvider(model=args.model)
+        if not llm.is_available():
+            print(
+                f"Ollama not reachable or model '{args.model}' not loaded — falling back to mock\n"
+            )
+            llm = MockProvider()  # type: ignore[assignment]
+    else:
+        llm = MockProvider()  # type: ignore[assignment]
+
+    print(f"LLM provider: {llm.__class__.__name__}\n")
+
+    # 4. Full RAG loop via RAGPipeline.answer()
+    for query in QUERIES:
+        print(f"Q: {query}")
+        response = pipeline.answer(query, llm, top_k=2)
+        print(f"A: {response.text[:300]}")
+        print(f"   tokens={response.total_tokens} finish={response.finish_reason}\n")
+
+    print("Done.")
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dataenginex"
-version = "0.6.0"
+version = "0.6.1"
 description = "DataEngineX - Core framework for data engineering projects"
 authors = [
     {name = "Jay", email = "jayapal.myaka99@gmail.com"}
@@ -52,7 +52,7 @@ dev = [
 data = [
     "pyspark>=4.1.1",
     "databricks-cli",
-    "apache-airflow>=2.8.0",
+    "apache-airflow>=2.8.0,<3.0.0",
     "requests>=2.32.0",
 ]
 notebook = [
@@ -115,6 +115,14 @@ ignore_missing_imports = true
 module = ["boto3.*", "google.auth.*", "google.cloud.*", "pyarrow.*"]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+module = ["sentence_transformers.*"]
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = ["dataenginex.ml.vectorstore"]
+warn_unused_ignores = false  # sentence-transformers optional: ignore used in CI, unused when installed
+
 [[tool.mypy.overrides]]
 module = ["dataenginex.lakehouse.storage"]
 warn_unused_ignores = false  # pre-commit installs google-cloud-storage (ignore used), CI does not (ignore unused)

src/dataenginex/__init__.py

@@ -36,7 +36,7 @@
 try:
     __version__ = version("dataenginex")
 except PackageNotFoundError:
-    __version__ = "0.6.0"
+    __version__ = "0.6.1"
 
 # Re-export core symbols that don't require optional dependencies
 from dataenginex.core import DataLayer, MedallionArchitecture, QualityGate