diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
index 096c80ff..eec4a35e 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.yml
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -7,6 +7,7 @@ body:
attributes:
value: |
Thanks for taking the time to report a bug in memU! Please fill in the following details.
+ If this report involves a vulnerability, leaked credential, private user data, or authentication bypass, do not open a public issue. Follow the Security Policy instead: https://github.com/NevaMind-AI/MemU/security/policy
- type: textarea
id: description
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
index 6a94220c..3f23dfc9 100644
--- a/.github/ISSUE_TEMPLATE/config.yml
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -1,8 +1,14 @@
blank_issues_enabled: false
contact_links:
- name: GitHub Discussions
- url: https://github.com/NevaMind-AI/memU/discussions
+ url: https://github.com/NevaMind-AI/MemU/discussions
about: For questions and general discussions about memU
+ - name: Security Reports
+ url: https://github.com/NevaMind-AI/MemU/security/policy
+ about: Please report suspected vulnerabilities privately
- name: Documentation
- url: https://github.com/NevaMind-AI/memU/blob/main/README.md
+ url: https://github.com/NevaMind-AI/MemU/blob/main/README.md
about: Check out the memU documentation
+ - name: Community Chat
+ url: https://discord.com/invite/hQZntfGsbJ
+ about: Join the memU Discord community
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 0467b332..ee78e9e6 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,44 +1,41 @@
-## 📝 Pull Request Summary
+## Summary
-Please provide a short summary explaining the purpose of this PR.
+Describe what this pull request changes and why it matters.
----
-
-## ✅ What does this PR do?
-- Clearly describe the change introduced.
-- Mention the motivation or problem it solves.
-
----
-
-## 🤔 Why is this change needed?
-- Explain the context or user impact.
-- Link any relevant issue or discussion.
-
----
-
-## 🔍 Type of Change
-Please check what applies:
+## Type of Change
- [ ] Bug fix
- [ ] New feature
- [ ] Documentation update
-- [ ] Refactor / cleanup
-- [ ] Other (please explain)
+- [ ] Refactor or cleanup
+- [ ] CI, build, or release change
+- [ ] Test-only change
----
+## Validation
-## ✅ PR Quality Checklist
+- [ ] `make check`
+- [ ] `make test`
+- [ ] `make docs-build`
+- [ ] Focused tests or manual verification listed below
-- [ ] PR title follows the conventional format (feat:, fix:, docs:)
-- [ ] Changes are limited in scope and easy to review
-- [ ] Documentation updated where applicable
-- [ ] No breaking changes (or clearly documented)
-- [ ] Related issues or discussions linked
+Validation notes:
+
+```text
+Paste commands, relevant output, or explain why a check was not run.
+```
----
+## Review Notes
-## 📌 Optional
+- Public API impact:
+- Storage/backend impact:
+- Security or privacy impact:
+- Breaking changes or migration steps:
-- [ ] Screenshots or examples added (if applicable)
-- [ ] Edge cases considered
-- [ ] Follow-up tasks mentioned
+## Checklist
+
+- [ ] PR title uses a supported conventional prefix (`feat`, `fix`, `docs`, `test`, `refactor`, `perf`, `style`, `ci`, `build`, `chore`, `revert`)
+- [ ] Changes are limited in scope and easy to review
+- [ ] Tests cover new behavior or regression risk
+- [ ] Documentation, examples, or ADRs are updated when behavior changes
+- [ ] No secrets, credentials, or private user data are included
+- [ ] Related issues or discussions are linked
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000..3e94deb6
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,47 @@
+version: 2
+updates:
+ - package-ecosystem: "uv"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ day: "monday"
+ time: "09:00"
+ timezone: "Etc/UTC"
+ open-pull-requests-limit: 5
+ commit-message:
+ prefix: "chore(deps)"
+ prefix-development: "chore(deps-dev)"
+ include: "scope"
+ labels:
+ - "dependencies"
+ - "python"
+
+ - package-ecosystem: "cargo"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ day: "monday"
+ time: "09:15"
+ timezone: "Etc/UTC"
+ open-pull-requests-limit: 5
+ commit-message:
+ prefix: "chore(deps)"
+ include: "scope"
+ labels:
+ - "dependencies"
+ - "rust"
+
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "weekly"
+ day: "monday"
+ time: "09:30"
+ timezone: "Etc/UTC"
+ open-pull-requests-limit: 5
+ commit-message:
+ prefix: "ci(deps)"
+ include: "scope"
+ labels:
+ - "dependencies"
+ - "ci"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7218e5f2..e96190b8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -2,13 +2,15 @@ name: build
on: [push, pull_request]
+permissions:
+ contents: read
+
jobs:
build:
runs-on: ubuntu-latest
- if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
strategy:
matrix:
- python-version: ["3.13"]
+ python-version: ["3.12", "3.13"]
steps:
- uses: actions/checkout@v6
@@ -24,12 +26,13 @@ jobs:
run: uv python install ${{ matrix.python-version }}
- name: Sync dependencies
- run: |
- uv sync --frozen
- uv run pre-commit install
+ run: uv sync --frozen
- name: Run style checks
- run: uv run make check
+ run: make check
- name: Run tests
- run: uv run make test
+ run: make test
+
+ - name: Build docs
+ run: make docs-build
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 00000000..d4340403
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,44 @@
+name: codeql
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+ schedule:
+ - cron: "37 3 * * 1"
+
+permissions:
+ actions: read
+ contents: read
+ security-events: write
+
+jobs:
+ analyze:
+ name: Analyze (${{ matrix.language }})
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - language: python
+ build-mode: none
+ - language: rust
+ build-mode: none
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v6
+
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@v4
+ with:
+ languages: ${{ matrix.language }}
+ build-mode: ${{ matrix.build-mode }}
+
+ - name: Perform CodeQL analysis
+ uses: github/codeql-action/analyze@v4
+ with:
+ category: "/language:${{ matrix.language }}"
diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
index efaa9e1f..4d59976c 100644
--- a/.github/workflows/release-please.yml
+++ b/.github/workflows/release-please.yml
@@ -4,15 +4,21 @@ on:
- main
permissions:
- contents: write
- issues: write
- pull-requests: write
+ contents: read
+
+concurrency:
+ group: release-please-${{ github.ref }}
+ cancel-in-progress: false
name: release-please
jobs:
release-please:
runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ issues: write
+ pull-requests: write
outputs:
releases_created: ${{ steps.release.outputs.releases_created }}
tag_name: ${{ steps.release.outputs.tag_name }}
@@ -36,27 +42,27 @@ jobs:
label: linux-x86_64
target: x86_64-unknown-linux-gnu
extra-args: "--compatibility manylinux_2_39"
- python-version: "3.13"
+ python-version: "3.12"
- os: ubuntu-latest
label: linux-aarch64
target: aarch64-unknown-linux-gnu
extra-args: "--compatibility manylinux_2_39"
- python-version: "3.13"
+ python-version: "3.12"
- os: macos-15-intel
label: macos-x86_64
target: ""
extra-args: ""
- python-version: "3.13"
+ python-version: "3.12"
- os: macos-latest
label: macos-aarch64
target: ""
extra-args: ""
- python-version: "3.13"
+ python-version: "3.12"
- os: windows-latest
label: windows-x86_64
target: ""
extra-args: ""
- python-version: "3.13"
+ python-version: "3.12"
steps:
- uses: actions/checkout@v6
@@ -85,7 +91,7 @@ jobs:
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
- name: Upload wheel artifact
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
name: wheels-${{ matrix.label }}
path: dist/*.whl
@@ -102,7 +108,7 @@ jobs:
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
- python-version: "3.13"
+ python-version: "3.12"
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
@@ -114,7 +120,7 @@ jobs:
run: uvx maturin sdist --out dist
- name: Upload sdist artifact
- uses: actions/upload-artifact@v6
+ uses: actions/upload-artifact@v7
with:
name: sdist
path: dist/*.tar.gz
@@ -130,18 +136,19 @@ jobs:
if: ${{ needs.release-please.outputs.releases_created == 'true' && needs.release-please.outputs.tag_name != '' }}
environment: pypi
permissions:
+ actions: read
id-token: write
contents: write
steps:
- name: Download wheel artifacts
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
pattern: wheels-*
merge-multiple: true
path: dist
- name: Download sdist artifact
- uses: actions/download-artifact@v7
+ uses: actions/download-artifact@v8
with:
name: sdist
path: dist
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index b7dffddb..1f17a214 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -9,7 +9,12 @@ repos:
- id: check-json
- id: pretty-format-json
args: [--autofix, --no-sort-keys]
+ - id: check-added-large-files
+ args: [--maxkb=1024]
+ - id: detect-private-key
- id: end-of-file-fixer
+ - id: mixed-line-ending
+ args: [--fix=lf]
- id: trailing-whitespace
- repo: https://github.com/astral-sh/ruff-pre-commit
diff --git a/.python-version b/.python-version
index 24ee5b1b..e4fba218 100644
--- a/.python-version
+++ b/.python-version
@@ -1 +1 @@
-3.13
+3.12
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 00000000..e5e34be8
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,31 @@
+# Code of Conduct
+
+## Our Pledge
+
+We want MemU to be a welcoming, useful, and technically serious open source project. We expect everyone who participates in the project to treat other contributors, users, and maintainers with respect.
+
+## Expected Behavior
+
+- Be kind, patient, and constructive.
+- Assume good intent while still being clear about technical risks.
+- Keep discussions focused on the work, the project, and the people affected by decisions.
+- Welcome newcomers and help them find the right issue, document, or maintainer.
+- Respect privacy, security boundaries, and confidential reports.
+
+## Unacceptable Behavior
+
+- Harassment, intimidation, threats, or personal attacks.
+- Demeaning language related to identity, background, experience level, or technical choices.
+- Publishing private information without explicit permission.
+- Repeatedly derailing discussions after maintainers ask to refocus.
+- Any behavior that makes the project unsafe or hostile for others.
+
+## Reporting
+
+Please report conduct concerns privately by emailing [contact@nevamind.ai](mailto:contact@nevamind.ai). Include enough context for maintainers to understand what happened, such as links, screenshots, dates, and the people involved.
+
+Maintainers will review reports as promptly and fairly as possible. Retaliation against anyone who reports a concern or participates in a review is not acceptable.
+
+## Enforcement
+
+Maintainers may take action to protect the project community, including warnings, comment deletion, temporary restrictions, or removal from project spaces. Enforcement decisions should be proportionate, documented internally, and focused on restoring a healthy collaboration environment.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 99041ea2..4d3d21c7 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -18,7 +18,7 @@ We welcome all types of contributions:
## 🚀 Quick Start for Contributors
### Prerequisites
-- Python 3.13+
+- Python 3.12+
- Git
- [uv](https://github.com/astral-sh/uv) (Python package manager)
- A code editor (VS Code recommended)
@@ -47,6 +47,8 @@ make test
make install # Create virtual environment and install dependencies with uv
make test # Run tests with pytest and coverage
make check # Run all checks (lock file, pre-commit, mypy, deptry)
+make docs # Preview the documentation site locally
+make docs-build # Build documentation in strict mode
```
## 🔧 Development Guidelines
@@ -76,7 +78,7 @@ uv run python -m pytest
uv run python -m pytest --cov --cov-config=pyproject.toml --cov-report=html
# Run specific test file
-uv run python -m pytest tests/rust_entry_test.py
+uv run python -m pytest tests/test_inmemory.py
# Run tests with specific marker
uv run python -m pytest -m "not slow"
@@ -120,9 +122,9 @@ For feature requests, please describe:
3. **Test your changes**
```bash
+ make check
make test
- make lint
- make coverage
+ make docs-build
```
4. **Submit pull request**
@@ -201,6 +203,7 @@ We're currently focusing on:
**Reporting Security Issues:**
- **DO NOT** create public issues for security vulnerabilities
+- Follow the [Security Policy](SECURITY.md)
- Email security issues privately to [contact@nevamind.ai](mailto:contact@nevamind.ai)
- Include detailed reproduction steps and impact assessment
- We'll acknowledge receipt within 24 hours
@@ -224,9 +227,10 @@ By contributing to MemU, you agree that:
| Channel | Best For |
|---------|----------|
-| 💬 [Discord](https://discord.gg/memu) | Real-time chat, quick questions |
+| 💬 [Discord](https://discord.com/invite/hQZntfGsbJ) | Real-time chat, quick questions |
| 🗣️ [GitHub Discussions](https://github.com/NevaMind-AI/MemU/discussions) | Feature discussions, Q&A |
| 🐛 [GitHub Issues](https://github.com/NevaMind-AI/MemU/issues) | Bug reports, feature requests |
+| 🔒 [Security Policy](SECURITY.md) | Private vulnerability reporting |
| 📧 [Email](mailto:contact@nevamind.ai) | Private inquiries |
## 🎉 Recognition
diff --git a/Cargo.toml b/Cargo.toml
index 49494051..c7fbeb0b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,5 +10,5 @@ crate-type = ["cdylib"]
[dependencies]
# "extension-module" tells pyo3 we want to build an extension module (skips linking against libpython.so)
-# "abi3-py313" tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.13
-pyo3 = { version = "0.27.1", features = ["extension-module", "abi3-py313"] }
+# "abi3-py312" tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.12
+pyo3 = { version = "0.27.1", features = ["extension-module", "abi3-py312"] }
diff --git a/MANIFEST.in b/MANIFEST.in
index 988fe526..bceab991 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,21 +1,28 @@
include README.md
-recursive-include memu *.py
-prune example
-include setup_postgres_env.sh
+include LICENSE.txt
+include CODE_OF_CONDUCT.md
+include CONTRIBUTING.md
+include SECURITY.md
+include SUPPORT.md
+include pyproject.toml
+include Cargo.toml
+include Cargo.lock
+include mkdocs.yml
+recursive-include src/memu *.py *.pyi py.typed
+recursive-include assets *.png *.gif *.jpg *.jpeg
+recursive-include readme *.md
+recursive-include docs *.md
+recursive-include examples *.py *.md *.json *.txt *.png *.jpg *.jpeg *.sh .env.example
exclude .env
-exclude .env.example
exclude setup.py.backup
-exclude */__pycache__/*
-exclude *.pyc
-exclude *.pyo
-exclude .git/*
-exclude .github/*
-exclude server/*
-exclude docs/*
-exclude scripts/*
exclude docker-compose.yml
exclude Dockerfile
exclude .dockerignore
exclude PROJECT_RELEASE_SUMMARY.md
exclude Makefile
exclude .pre-commit-config.yaml
+prune .git
+prune .github
+prune tmp
+global-exclude __pycache__
+global-exclude *.py[cod]
diff --git a/Makefile b/Makefile
index 6f516bc1..1c0ef8bd 100644
--- a/Makefile
+++ b/Makefile
@@ -1,22 +1,32 @@
.PHONY: install
install:
- @echo "🚀 Creating virtual environment using uv"
+ @echo "[install] Creating virtual environment using uv"
@uv sync
@uv run pre-commit install
.PHONY: check
check:
- @echo "🚀 Checking lock file consistency with 'pyproject.toml'"
+ @echo "[check] Checking lock file consistency with 'pyproject.toml'"
@uv lock --locked
- @echo "🚀 Linting code: Running pre-commit"
+ @echo "[check] Linting code: Running pre-commit"
@uv run pre-commit run -a
- @echo "🚀 Static type checking: Running mypy"
+ @echo "[check] Static type checking: Running mypy"
@uv run mypy
- @echo "🚀 Checking for obsolete dependencies: Running deptry"
+ @echo "[check] Checking for obsolete dependencies: Running deptry"
@uv run deptry src
.PHONY: test
test:
- @echo "🚀 Testing code: Running pytest"
+ @echo "[test] Running pytest"
@uv run python -m pytest --cov --cov-config=pyproject.toml --cov-report=xml
+
+.PHONY: docs
+docs:
+ @echo "[docs] Serving documentation with MkDocs"
+ @uv run mkdocs serve
+
+.PHONY: docs-build
+docs-build:
+ @echo "[docs] Building documentation with MkDocs"
+ @uv run mkdocs build --strict
diff --git a/README.md b/README.md
index c0d2e36d..6460d346 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
[](https://badge.fury.io/py/memu-py)
[](https://opensource.org/licenses/Apache-2.0)
-[](https://www.python.org/downloads/)
+[](https://www.python.org/downloads/)
[](https://discord.com/invite/hQZntfGsbJ)
[](https://x.com/memU_ai)
@@ -28,7 +28,7 @@ memU **continuously captures and understands user intent**. Even without a comma
## 🤖 [OpenClaw Alternative](https://github.com/NevaMind-AI/memUBot)
-
+
**[memU Bot](https://github.com/NevaMind-AI/memUBot)** — Now open source. The enterprise-ready OpenClaw. Your proactive AI assistant that remembers everything.
@@ -79,7 +79,7 @@ Just as a file system turns raw bytes into organized data, memU transforms raw i
## ⭐️ Star the repository
-
+
If you find memU useful or interesting, a GitHub Star ⭐️ would be greatly appreciated.
---
@@ -97,10 +97,14 @@ If you find memU useful or interesting, a GitHub Star ⭐️ would be greatly ap
## 🔄 How Proactive Memory Works
```bash
-
+pip install "memu-py[claude]"
+# From a source checkout, use: uv sync --extra claude
+export OPENAI_API_KEY="..."
+export ANTHROPIC_API_KEY="..."
+# Optional when using memory.platform instead of memory.local:
+export MEMU_API_KEY="..."
cd examples/proactive
python proactive.py
-
```
---
@@ -233,6 +237,8 @@ MemU's three-layer system enables both **reactive queries** and **proactive cont
+
+
| Layer | Reactive Use | Proactive Use |
|-------|--------------|---------------|
| **Resource** | Direct access to original data | Background monitoring for new patterns |
@@ -277,21 +283,34 @@ For enterprise deployment with custom proactive workflows, contact **info@nevami
#### Installation
```bash
-pip install -e .
+pip install memu-py
+```
+
+For local source development, clone this repository and install the editable
+workspace:
+
+```bash
+make install
```
#### Basic Example
-> **Requirements**: Python 3.13+ and an OpenAI API key
+> **Requirements**: Python 3.12+ and an OpenAI API key
+
+Run the getting-started example:
-**Test Continuous Learning** (in-memory):
```bash
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_inmemory.py
+python examples/getting_started_robust.py
```
-**Test with Persistent Storage** (PostgreSQL):
+The example initializes `MemoryService`, creates a memory item, and retrieves it
+with a natural-language query. See
+[`examples/getting_started_robust.py`](examples/getting_started_robust.py) for
+the full script.
+
+**Optional PostgreSQL integration check**:
+
```bash
# Start PostgreSQL with pgvector
docker run -d \
@@ -302,18 +321,94 @@ docker run -d \
-p 5432:5432 \
pgvector/pgvector:pg16
-# Run continuous learning test
+# Run the opt-in PostgreSQL integration test
+uv sync --extra postgres
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_postgres.py
+export MEMU_RUN_POSTGRES_TESTS=1
+uv run python -m pytest tests/test_postgres.py
```
-Both examples demonstrate **proactive memory workflows**:
+These flows demonstrate **proactive memory workflows**:
1. **Continuous Ingestion**: Process multiple files sequentially
2. **Auto-Extraction**: Immediate memory creation
3. **Proactive Retrieval**: Context-aware memory surfacing
-See [`tests/test_inmemory.py`](tests/test_inmemory.py) and [`tests/test_postgres.py`](tests/test_postgres.py) for implementation details.
+See [`tests/test_inmemory.py`](tests/test_inmemory.py), [`tests/test_sqlite.py`](tests/test_sqlite.py),
+and [`tests/test_postgres.py`](tests/test_postgres.py) for implementation details. The
+in-memory and SQLite live LLM checks are opt-in with `MEMU_RUN_LIVE_LLM_TESTS=1`.
+
+### Context Harness: Folder to Markdown Memory
+
+For local agents that need inspectable context files, memU can compile a folder
+of raw data into a Markdown-backed memory repository:
+
+
+
+```text
+memory_repo/
+ AGENTS.md
+ raw_data/
+ memory.md
+ memory/
+ soul.md
+ soul/
+ skill.md
+ skill/
+ .memu/
+ harness.json
+ evolution/
+```
+
+Quick CLI workflow:
+
+```bash
+memu-harness init memory_repo --source-folder path/to/uploaded-folder
+memu-harness doctor memory_repo --json
+memu-harness status memory_repo --json
+memu-harness refresh memory_repo --query "current agent task"
+memu-harness review-evolution memory_repo
+memu-harness refresh memory_repo --exclude "node_modules/**" --exclude "*.tmp"
+memu-harness promote-skill memory_repo \
+ --title "Validate Context Packs" \
+ --lesson "Inspect promoted skills before relying on generated context"
+memu-harness suggest-skills memory_repo --json
+memu-harness context memory_repo --query "current agent task"
+memu-harness context memory_repo --query "current agent task" --format summary
+memu-harness context memory_repo --query "current agent task" --format messages
+memu-harness context memory_repo --bucket-max soul=1000 --bucket-max skill=2000
+memu-harness context memory_repo --format system --output context.system.md
+```
+
+This flow preserves multimodal files in `raw_data/`, supports sidecar captions,
+summaries, notes, and transcripts such as `screenshot.caption.md` or
+`report.summary.md`, updates changed files incrementally, and keeps manual skill
+notes outside generated blocks. Raw logs, creator feedback, uploads, and new
+observations do not edit `memory.md`, `soul.md`, or `skill.md` directly; memU
+first turns them into Evolution Instructions, Patch Proposals, and review
+decisions, with audit records under `.memu/evolution/`. Exclude noisy files
+explicitly with `--exclude` or a `.memuignore` file. `init` also creates
+`.memu/harness.json`, where the
+repository can persist non-secret defaults such as exclude globs, text evidence
+limits, context budgets, and context output format. Both `memu-harness context`
+and standalone `memu-context` read those context defaults. Skill traces can be
+turned into promotion suggestions with `suggest-skills`; promoted skills are
+also stored as stable cards under `skill/promoted/`. New harness repositories
+include an `AGENTS.md` bootstrap file so local coding agents can discover the
+memory, soul, skill, raw data, and skill-evolution conventions directly from
+the repository. Python callers can use `ContextHarness.from_repo("memory_repo")`
+to refresh and build context from `memory_repo/raw_data` with the same repo
+defaults.
+
+
+
+Run the no-API-key demo:
+
+```bash
+python examples/context_harness_demo.py
+```
+
+See [`docs/folder_memory_compiler.md`](docs/folder_memory_compiler.md) for the
+full harness API, CLI, watcher, status report, and self-evolving skill workflow.
---
@@ -328,14 +423,14 @@ service = MemUService(
# Default profile for LLM operations
"default": {
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
- "api_key": "your_api_key",
+ "api_key": "MEMU_QWEN_API_KEY",
"chat_model": "qwen3-max",
- "client_backend": "sdk" # "sdk" or "http"
+ "client_backend": "sdk" # "sdk", "httpx", or "lazyllm_backend"
},
# Separate profile for embeddings
"embedding": {
"base_url": "https://api.voyageai.com/v1",
- "api_key": "your_voyage_api_key",
+ "api_key": "VOYAGE_API_KEY",
"embed_model": "voyage-3.5-lite"
}
},
@@ -343,6 +438,24 @@ service = MemUService(
)
```
+The `lazyllm_backend` adapter is optional. Install it with
+`pip install "memu-py[lazyllm]"` or, from a source checkout,
+`uv sync --extra lazyllm`.
+
+Optional LazyLLM live check:
+
+```bash
+uv sync --extra lazyllm
+export MEMU_QWEN_API_KEY=your_api_key
+export MEMU_RUN_LAZYLLM_TESTS=1
+uv run python -m pytest tests/test_lazyllm.py
+```
+
+Retrieve routing can also use distinct profiles: set
+`route_intention_llm_profile`, `sufficiency_check_llm_profile`, and
+`llm_ranking_llm_profile` in `retrieve_config` to split cheap routing from
+heavier ranking or judging models.
+
---
### OpenRouter Integration
@@ -359,7 +472,7 @@ service = MemoryService(
"provider": "openrouter",
"client_backend": "httpx",
"base_url": "https://openrouter.ai",
- "api_key": "your_openrouter_api_key",
+ "api_key": "OPENROUTER_API_KEY",
"chat_model": "anthropic/claude-3.5-sonnet", # Any OpenRouter model
"embed_model": "openai/text-embedding-3-small", # Embedding model
},
@@ -387,15 +500,10 @@ service = MemoryService(
#### Running OpenRouter Tests
```bash
export OPENROUTER_API_KEY=your_api_key
+export MEMU_RUN_OPENROUTER_TESTS=1
# Full workflow test (memorize + retrieve)
-python tests/test_openrouter.py
-
-# Embedding-specific tests
-python tests/test_openrouter_embedding.py
-
-# Vision-specific tests
-python tests/test_openrouter_vision.py
+uv run python -m pytest tests/test_openrouter.py
```
See [`examples/example_4_openrouter_memory.py`](examples/example_4_openrouter_memory.py) for a complete working example.
@@ -404,6 +512,8 @@ See [`examples/example_4_openrouter_memory.py`](examples/example_4_openrouter_me
## 📖 Core APIs
+
+
### `memorize()` - Continuous Learning Pipeline
Processes inputs in real-time and immediately updates memory:
@@ -466,11 +576,12 @@ Deep **anticipatory reasoning** for complex contexts:
# Proactive retrieval with context history
result = await service.retrieve(
queries=[
- {"role": "user", "content": {"text": "What are their preferences?"}},
- {"role": "user", "content": {"text": "Tell me about work habits"}}
+ {"role": "user", "content": "What are their preferences?"},
+ {"role": "user", "content": "Tell me about work habits"}
],
where={"user_id": "123"}, # Optional: scope filter
- method="rag" # or "llm" for deeper reasoning
+ method="rag", # or "llm" for deeper reasoning
+ ranking="salience", # or "similarity" for RAG item recall
)
# Returns context-aware results:
@@ -482,11 +593,15 @@ result = await service.retrieve(
}
```
+For a single user query, Python callers can also pass `queries=["What are their preferences?"]`; MemU normalizes it to a user message before retrieval.
+
**Proactive Filtering**: Use `where` to scope continuous monitoring:
- `where={"user_id": "123"}` - User-specific context
- `where={"agent_id__in": ["1", "2"]}` - Multi-agent coordination
- Omit `where` for global context awareness
+`where` keys must match fields on your configured `UserConfig.model`, and values are validated/normalized by that model before querying. Validation is field-level, so partial filters do not need to include every field required by the model. Supported filters are equality (`field`) and membership (`field__in`); unsupported operators are rejected before any backend query runs.
+
---
## 💡 Proactive Scenarios
@@ -559,7 +674,7 @@ View detailed experimental data: [memU-experiment](https://github.com/NevaMind-A
| Repository | Description | Proactive Features |
|------------|-------------|-------------------|
-| **[memU](https://github.com/NevaMind-AI/memU)** | Core proactive memory engine | 7×24 learning pipeline, auto-categorization |
+| **[memU](https://github.com/NevaMind-AI/MemU)** | Core proactive memory engine | 7×24 learning pipeline, auto-categorization |
| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | Backend with continuous sync | Real-time memory updates, webhook triggers |
| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | Visual memory dashboard | Live memory evolution monitoring |
@@ -597,7 +712,7 @@ We welcome contributions from the community! Whether you're fixing bugs, adding
To start contributing to MemU, you'll need to set up your development environment:
#### Prerequisites
-- Python 3.13+
+- Python 3.12+
- [uv](https://github.com/astral-sh/uv) (Python package manager)
- Git
@@ -621,14 +736,31 @@ The `make install` command will:
Before submitting your contribution, ensure your code passes all quality checks:
```bash
make check
+make test
```
The `make check` command runs:
- **Lock file verification**: Ensures `pyproject.toml` consistency
-- **Pre-commit hooks**: Lints code with Ruff, formats with Black
+- **Pre-commit hooks**: Lints and formats code with Ruff
- **Type checking**: Runs `mypy` for static type analysis
- **Dependency analysis**: Uses `deptry` to find obsolete dependencies
+The `make test` command runs the pytest suite with coverage enabled.
+
+#### Documentation Site
+
+Preview the documentation locally with MkDocs:
+
+```bash
+make docs
+```
+
+Build the documentation in strict mode:
+
+```bash
+make docs-build
+```
+
### Contributing Guidelines
For detailed contribution guidelines, code standards, and development practices, please see [CONTRIBUTING.md](CONTRIBUTING.md).
@@ -638,7 +770,7 @@ For detailed contribution guidelines, code standards, and development practices,
- Write clear commit messages
- Add tests for new functionality
- Update documentation as needed
-- Run `make check` before pushing
+- Run `make check` and `make test` before pushing
---
@@ -650,10 +782,13 @@ For detailed contribution guidelines, code standards, and development practices,
## 🌍 Community
-- **GitHub Issues**: [Report bugs & request features](https://github.com/NevaMind-AI/memU/issues)
+- **Support**: [Get help and choose the right channel](SUPPORT.md)
+- **Security**: [Report vulnerabilities privately](SECURITY.md)
+- **GitHub Issues**: [Report bugs & request features](https://github.com/NevaMind-AI/MemU/issues)
+- **GitHub Discussions**: [Ask questions and discuss ideas](https://github.com/NevaMind-AI/MemU/discussions)
- **Discord**: [Join the community](https://discord.com/invite/hQZntfGsbJ)
- **X (Twitter)**: [Follow @memU_ai](https://x.com/memU_ai)
-- **Contact**: info@nevamind.ai
+- **Contact**: contact@nevamind.ai
---
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 00000000..44bec949
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,58 @@
+# Security Policy
+
+memU handles user-provided content, memory records, embeddings, local files, and
+provider credentials. Please report security concerns privately so maintainers
+can investigate before details become public.
+
+## Supported Versions
+
+Security fixes are prioritized for:
+
+- the latest released `memu-py` version
+- the `main` branch when a fix has not been released yet
+
+Older releases may receive fixes when the issue is severe and the patch can be
+backported safely.
+
+## Reporting a Vulnerability
+
+Do not open a public GitHub issue for suspected vulnerabilities.
+
+Email reports to [contact@nevamind.ai](mailto:contact@nevamind.ai) with:
+
+- affected version or commit
+- operating system and deployment mode
+- reproduction steps or proof of concept
+- expected impact
+- whether credentials, files, memory data, or external providers are involved
+
+We aim to acknowledge reports within 2 business days and will share status
+updates as we investigate. If a vulnerability is confirmed, maintainers will
+coordinate a fix, release guidance, and public disclosure timing with the
+reporter when possible.
+
+## Scope
+
+Useful reports include issues such as:
+
+- unauthorized access to memory data across scopes or users
+- unsafe handling of API keys, environment variables, or provider credentials
+- path traversal, unintended file reads, or writes outside configured roots
+- integration authentication or authorization bypasses
+- injection paths that can alter persistent memory without review
+
+Out-of-scope reports include social engineering, spam, denial-of-service without
+a concrete vulnerability, and findings that require access to accounts or
+systems you do not own or have permission to test.
+
+## Automated Scanning
+
+Pull requests and the `main` branch are scanned with CodeQL for Python and Rust.
+These checks complement code review and responsible disclosure; they do not
+replace private reporting for suspected vulnerabilities.
+
+## Safe Harbor
+
+Good-faith research that avoids privacy violations, data destruction, service
+disruption, and unauthorized access is welcome. Stop testing and contact us
+privately if you encounter sensitive data or credentials.
diff --git a/SUPPORT.md b/SUPPORT.md
new file mode 100644
index 00000000..4cc02091
--- /dev/null
+++ b/SUPPORT.md
@@ -0,0 +1,39 @@
+# Support
+
+Thanks for using memU. This page helps route questions and reports to the right
+place.
+
+## Questions and Discussions
+
+Use [GitHub Discussions](https://github.com/NevaMind-AI/MemU/discussions) for:
+
+- usage questions
+- architecture discussions
+- integration ideas
+- provider or storage configuration questions
+
+For quick community chat, join the
+[Discord community](https://discord.com/invite/hQZntfGsbJ).
+
+## Bugs and Feature Requests
+
+Use [GitHub Issues](https://github.com/NevaMind-AI/MemU/issues) for:
+
+- reproducible bugs
+- feature requests
+- documentation problems
+- integration regressions
+
+Please include the memU version, Python version, operating system, storage
+backend, LLM provider, and a small reproduction when possible.
+
+## Security Reports
+
+Do not report security vulnerabilities in public issues. Follow the
+[Security Policy](SECURITY.md) and email
+[contact@nevamind.ai](mailto:contact@nevamind.ai).
+
+## Private Inquiries
+
+For private questions, partnership discussions, or enterprise deployment
+inquiries, email [contact@nevamind.ai](mailto:contact@nevamind.ai).
diff --git a/assets/memu-overall-algorithm-flow.png b/assets/memu-overall-algorithm-flow.png
new file mode 100644
index 00000000..d8e27081
Binary files /dev/null and b/assets/memu-overall-algorithm-flow.png differ
diff --git a/assets/memu-overall-engineering-architecture.png b/assets/memu-overall-engineering-architecture.png
new file mode 100644
index 00000000..37ae44c3
Binary files /dev/null and b/assets/memu-overall-engineering-architecture.png differ
diff --git a/assets/memu-self-evolve-algorithm.png b/assets/memu-self-evolve-algorithm.png
new file mode 100644
index 00000000..298c76d8
Binary files /dev/null and b/assets/memu-self-evolve-algorithm.png differ
diff --git a/assets/memu-self-evolve-architecture.png b/assets/memu-self-evolve-architecture.png
new file mode 100644
index 00000000..10cc7c74
Binary files /dev/null and b/assets/memu-self-evolve-architecture.png differ
diff --git a/docs/HACKATHON_ISSUE_DRAFT.md b/docs/HACKATHON_ISSUE_DRAFT.md
index 7951c4be..0aaa0775 100644
--- a/docs/HACKATHON_ISSUE_DRAFT.md
+++ b/docs/HACKATHON_ISSUE_DRAFT.md
@@ -11,7 +11,7 @@
This PR enhances MemU's memory type system to support specialized memory structures with type-specific metadata and introduces Tool Memory for agent self-improvement.
-**Current State:** MemU has a `memory_type` field with 5 types (profile, event, knowledge, behavior, skill) and uses different LLM prompts to extract each type. However, after extraction, all memories share the same storage schema - just `summary` and `embedding`. There's no type-specific metadata, no type-aware retrieval, and no way for agents to learn from their tool usage.
+**Current State:** MemU has a `memory_type` field with 6 supported types (profile, event, knowledge, behavior, skill, tool) and uses different LLM prompts to extract each type. Tool memories can store tool-specific metadata and execution history, while other memories continue to share the common `summary` and `embedding` storage shape.
**Enhancement:** Extend the memory system to support:
- Type-specific metadata fields (e.g., `when_to_use` for smarter retrieval)
diff --git a/docs/adr/0004-markdown-context-harness-and-skill-evolution.md b/docs/adr/0004-markdown-context-harness-and-skill-evolution.md
new file mode 100644
index 00000000..97120b62
--- /dev/null
+++ b/docs/adr/0004-markdown-context-harness-and-skill-evolution.md
@@ -0,0 +1,78 @@
+# ADR 0004: Use Markdown Repositories for Context Harness and Skill Evolution
+
+- Status: Accepted
+- Date: 2026-06-04
+
+## Context
+
+memU needs a mode for agents that starts from a user-provided folder rather than an application event stream.
+
+That folder can contain text, code, logs, conversations, images, audio, video,
+PDFs, office documents, and unknown binary files.
+
+The resulting memory should be inspectable by humans, portable across tools,
+resilient to file changes, and directly usable as agent context.
+
+The system also needs a way for agents to improve reusable skills over time.
+Skill updates should be traceable to raw execution evidence, while durable
+promoted skills should be editable by humans and survive re-extraction.
+
+## Decision
+
+Add a Markdown-backed context harness layer alongside the existing `MemoryService` workflows.
+
+- Preserve the uploaded evidence under `raw_data/`
+- Store generated memory in `memory.md` and `memory/`
+- Store persona, tone, and language-style signals in `soul.md` and `soul/`
+- Store reusable skills, workflows, and tool-use lessons in `skill.md` and `skill/`
+- Track source hashes, sidecars, evidence paths, and generated entries in `.memu/manifest.json`
+- Cache per-source evidence in `.memu/derived/`
+- Persist repository-local, non-secret harness defaults in `.memu/harness.json`
+- Allow explicit source-relative exclude globs and `.memuignore` files for noisy files while applying no default excludes
+- Remove stale generated evidence when source files disappear from the current folder snapshot
+- Replace only generated blocks marked by memU comments, preserving manual Markdown outside those blocks
+- Treat sidecar files such as `image.caption.md`, `audio.transcript.txt`, and
+ `report.summary.md` as semantic evidence for paired non-text sources
+- Treat structured sidecars such as `image.metadata.json` and `video.frames.jsonl`
+ as paired multimodal evidence instead of independent source files
+- Provide context packs as Markdown, system prompts, chat message lists, and safe message injection helpers
+- Allow CLI context outputs to be written as files for downstream agent harnesses
+- Allow per-bucket context character limits for predictable `memory`, `soul`, and `skill` budgets
+- Record skill traces under `raw_data/skill_traces/` so normal folder compilation can re-extract them
+- Suggest skill promotions by grouping trace lessons, actions, tools, and outcomes without writing by default
+- Promote durable skills into manual `skill.md` notes and stable `skill/promoted/*.md` cards
+- Deduplicate promoted skill context by preferring full `skill/promoted/*.md` cards over their `skill.md` index snippets
+- Generate a non-overwriting `AGENTS.md` bootstrap file so local agents can discover harness conventions
+- Let command-line flags override `.memu/harness.json`, and let the config file override built-in CLI defaults for both `memu-harness` and `memu-context`
+
+ADR 0005 narrows the write path for generated self-evolution: raw traces,
+creator feedback, uploads, and observations are now converted into
+`EvolutionInstruction` records and reviewed `PatchProposal`s before approved
+changes update generated Markdown blocks.
+
+`ContextHarness` is the composition API for this mode. It binds a raw-data
+folder to a Markdown memory repository and coordinates scaffold, ingest, status,
+context assembly, skill traces, promotion, and watch mode.
+
+The harness config intentionally excludes API keys, LLM provider settings, and
+user scope because those values are environment or request specific.
+
+## Consequences
+
+Positive:
+
+- memory repositories are human-readable and version-control friendly
+- raw evidence, generated summaries, and manual edits have clear ownership boundaries
+- folder changes can be incrementally re-extracted without rewriting user notes
+- multimodal files can carry local semantic evidence through sidecars even without an LLM
+- context can be injected directly into chat-completion style agents
+- skill evolution has both raw trace evidence and stable promoted skill cards
+- repeated CLI runs can share repo-specific defaults without storing secrets
+
+Negative:
+
+- Markdown parsing and generated-block preservation require careful delimiter discipline
+- local extraction is heuristic unless a multimodal/document-capable `MemoryService` is supplied
+- sidecar naming conventions must be documented and consistently followed
+- Markdown repositories add a second memory surface alongside database-backed storage
+- repeated promotion and manual edits need merge rules to avoid losing provenance
diff --git a/docs/adr/0005-self-evolve-instruction-review-gate.md b/docs/adr/0005-self-evolve-instruction-review-gate.md
new file mode 100644
index 00000000..ef998a4e
--- /dev/null
+++ b/docs/adr/0005-self-evolve-instruction-review-gate.md
@@ -0,0 +1,75 @@
+# ADR 0005: Gate Self-Evolution Through Instructions and Reviewed Patches
+
+- Status: Accepted
+- Date: 2026-06-04
+
+## Context
+
+Markdown-backed context harnesses can ingest agent logs, creator feedback,
+uploaded files, observations, sidecars, and skill traces. Those raw sources are
+valuable evidence, but letting them rewrite `memory.md`, `soul.md`, or
+`skill.md` directly creates a self-contamination risk: noisy execution logs or
+unreviewed model output can become durable context and then influence the next
+context pack.
+
+The self-evolve path needs traceability and review without breaking the
+existing folder compiler ergonomics.
+
+## Decision
+
+Raw evidence must pass through this chain before it can update long-term
+Markdown context:
+
+```text
+raw_data/
+ agent logs
+ creator feedback
+ uploaded files
+ observations
+ -> Evidence extraction
+ -> EvolutionInstruction
+ -> PatchProposal
+ -> ReviewDecision
+ -> approved patches update memory.md, soul.md, skill.md
+```
+
+An `EvolutionInstruction` records the structured intent:
+
+- `target`: `memory`, `soul`, or `skill`
+- `operation`: `add`, `update`, or `delete`
+- `reason`
+- `evidence`
+- `priority`
+- `confidence`
+
+Each instruction becomes a `PatchProposal`. The review gate evaluates
+traceability, confidence, conflict markers, and patch safety. Approved proposals
+are applied to generated Markdown blocks and manifest entries. Proposals that
+need creator review remain auditable and do not update long-term context.
+
+The compiler writes audit artifacts under `.memu/evolution/`:
+
+- `instructions.jsonl`
+- `patch_proposals.jsonl`
+- `review_decisions.jsonl`
+- `latest.json`
+
+`FolderMemoryCompilerConfig.evolution_review` controls whether the review gate
+auto-approves safe proposals or requires creator review. The CLI exposes this
+with `--require-creator-review` and `--min-evolution-confidence`.
+
+## Consequences
+
+Positive:
+
+- raw logs and feedback cannot directly pollute durable context
+- every generated memory/soul/skill update has a structured provenance chain
+- creator-review workflows can block writes while preserving proposals
+- deleted sources also produce traceable delete proposals
+- existing folder compiler calls still work because safe proposals auto-approve by default
+
+Negative:
+
+- manifest records are larger because they include per-source evolution audits
+- manual-review mode can leave sources pending and therefore re-propose changes
+- lightweight local conflict detection is heuristic until an LLM-backed reviewer is added
diff --git a/docs/adr/README.md b/docs/adr/README.md
index 98386770..731e26eb 100644
--- a/docs/adr/README.md
+++ b/docs/adr/README.md
@@ -3,3 +3,5 @@
- [0001: Use Workflow Pipelines for Core Operations](0001-workflow-pipeline-architecture.md)
- [0002: Use Pluggable Storage with Backend-Specific Vector Search](0002-pluggable-storage-and-vector-strategy.md)
- [0003: Model User Scope as First-Class Fields on Memory Records](0003-user-scope-in-data-model.md)
+- [0004: Markdown Context Harness and Skill Evolution](0004-markdown-context-harness-and-skill-evolution.md)
+- [0005: Gate Self-Evolution Through Instructions and Reviewed Patches](0005-self-evolve-instruction-review-gate.md)
diff --git a/docs/architecture.md b/docs/architecture.md
index fefbf54d..80d86282 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -29,6 +29,10 @@ flowchart TD
E --> I["Category Relations"]
```
+
+
+
+
## Core runtime components
### `MemoryService` as composition root
@@ -49,6 +53,11 @@ Public APIs are assembled by mixins:
- `RetrieveMixin`: `retrieve(...)`
- `CRUDMixin`: list/clear/create/update/delete memory operations
+LLM profile configs validate provider choices and embedding batch boundaries up front; `embed_batch_size` must be a
+positive integer before SDK clients are constructed.
+Retrieve breadth settings (`top_k`) and category-summary target lengths are positive integers, salience recency
+decay must be positive, and category assignment thresholds are constrained to `[0, 1]`.
+
### Workflow engine
All major operations execute as workflows (`WorkflowStep`) with:
@@ -57,7 +66,9 @@ All major operations execute as workflows (`WorkflowStep`) with:
- declared capability tags (`llm`, `vector`, `db`, `io`, `vision`)
- per-step config (for profile selection)
-`PipelineManager` validates step dependencies at registration/mutation time and supports runtime pipeline revisioning (`config_step`, `insert_before/after`, `replace_step`, `remove_step`).
+`PipelineManager` validates step dependencies and LLM profile references at registration/mutation time. Profile
+references must be non-empty strings and must resolve to a configured profile. It also supports runtime pipeline
+revisioning (`config_step`, `insert_before/after`, `replace_step`, `remove_step`).
`WorkflowRunner` is a protocol; default `LocalWorkflowRunner` executes sequentially with `run_steps(...)`.
@@ -68,7 +79,12 @@ Two interceptor systems exist:
- workflow step interceptors: before/after/on_error around each step
- LLM call interceptors: before/after/on_error around `chat/summarize/vision/embed/transcribe`
-LLM wrappers also extract best-effort usage metadata from raw provider responses.
+LLM wrappers also extract best-effort usage metadata from raw provider responses. They normalize both
+chat-completions-style token names (`prompt_tokens`/`completion_tokens`) and responses-style token names
+(`input_tokens`/`output_tokens`) into `LLMUsage`. Extracted usage and token breakdowns are normalized into
+JSON-safe values before interceptors receive them, so logging, audit trails, and external telemetry sinks can
+persist the observation without SDK-specific objects leaking through. Batched embedding calls may return multiple
+raw provider responses; the wrapper aggregates their token usage so cost telemetry covers the whole batch.
## Ingestion architecture (`memorize`)
@@ -77,7 +93,7 @@ LLM wrappers also extract best-effort usage metadata from raw provider responses
1. `ingest_resource`: fetch local/remote resource into `blob_config.resources_dir` via `LocalFS`
2. `preprocess_multimodal`: modality-specific preprocessing for conversation/document/audio (text-oriented path) and image/video (vision-oriented path)
3. `extract_items`: per-memory-type LLM extraction into structured entries
-4. `dedupe_merge`: placeholder stage (currently pass-through)
+4. `dedupe_merge`: remove duplicate extracted entries within the current batch, preserving first-seen order and merging category hints
5. `categorize_items`: persist resource + memory items + item-category relations and embeddings
6. `persist_index`: update category summaries; optionally persist item references
7. `build_response`: return resource(s), items, categories, relations
@@ -86,11 +102,14 @@ Category bootstrap is lazy and scoped: categories are initialized when needed wi
## Retrieval architecture (`retrieve`)
-`retrieve(...)` chooses one of two pipelines from config:
+`retrieve(...)` chooses one of two pipelines from `retrieve_config.method`, unless a request passes a per-call `method` override:
- `retrieve_rag` (embedding-driven ranking)
- `retrieve_llm` (LLM-driven ranking)
+RAG item ranking also supports a per-call `ranking` override (`similarity` or `salience`), defaulting to
+`retrieve_config.item.ranking`.
+
Both use the same staged pattern:
1. route intention + optional query rewrite
@@ -103,10 +122,15 @@ Both use the same staged pattern:
Key behavior:
-- `where` filters are validated against `user_model` fields before querying
+- `where` filters are validated against `user_model` fields and field value types before querying; supported filters are equality (`field`) and membership (`field__in`)
+- category, item, and resource retrieval toggles apply consistently to both `rag` and `llm` retrieve pipelines
- RAG path uses vector similarity (and optional salience ranking for items)
- LLM path ranks IDs from formatted category/item/resource context
+- when category references are enabled, both RAG and LLM paths use scoped `ref_id`
+ lookups to follow `[ref:...]` citations from category summaries back to memory items
+- Python retrieve accepts a non-empty `queries` list containing string query items or `{role, content}` message objects; HTTP retrieve accepts the same list shape plus a shorthand `query` string. Both paths normalize every query item, trim query text, and reject blank query items before workflow execution
- each stage can stop early if sufficiency check decides context is enough
+- public response records are JSON-safe and omit raw embedding vectors by default
## Data and storage architecture
@@ -127,13 +151,50 @@ Storage is abstracted through a `Database` protocol with four repositories:
- `sqlite`: SQLModel persistence, embeddings stored as JSON text, brute-force cosine search
- `postgres`: SQLModel persistence with pgvector support (when enabled), local fallback ranking when needed
-For Postgres, startup runs migration bootstrap and attempts `CREATE EXTENSION IF NOT EXISTS vector` in `ddl_mode="create"`.
+In-memory repositories mutate shared `InMemoryState` containers in place so the store-level
+`resources`, `items`, `categories`, and `relations` views stay consistent after scoped clears.
+
+For Postgres, startup runs migration bootstrap and attempts
+`CREATE EXTENSION IF NOT EXISTS vector` in `ddl_mode="create"`.
+When Postgres retrieval falls back to local ranking, including salience-aware
+ranking, it queries current memory-item rows for the requested scope before
+scoring so persisted memories remain visible after restarts and external writes.
+Postgres clear/delete paths also prune in-process item and relation caches after
+database cascade deletes, preventing later repository calls from reusing stale
+relation edges.
### Scope model propagation
-`UserConfig.model` is merged into record/table models so scope fields (for example `user_id`) become first-class columns/attributes across resources, items, categories, and relations.
+`UserConfig.model` is merged into record/table models so scope fields (for example `user_id`)
+become first-class columns/attributes across resources, items, categories, and relations.
+Backend-agnostic record models also preserve scope fields as extra attributes when adapters
+materialize cached or response records from scoped storage rows.
This is why `where` filters and `user_data` writes are consistently available across APIs.
+Scope filter values are normalized through `UserConfig.model`, so backend queries see the same
+typed values that writes use. This validation is field-level, so partial filters do not need to
+provide every field required by a custom user model.
+Default category bootstrap and the in-process category ID/name map are cached per concrete user
+scope, so one user's category IDs are not reused for another user's writes.
+Persistent SQLite/Postgres category tables enforce category-name uniqueness per configured scope.
+ID-based update/delete operations also verify the target memory item against the provided user
+scope before mutation, treating cross-scope IDs as not found.
+First-run category listing and category-enabled retrieval initialize default categories when `where`
+identifies one concrete scope; multi-scope filters such as `field__in` do not create categories implicitly.
+Manual `create_memory_item` writes source-less memory items with `resource_id=None`; `memorize`
+remains the resource-backed ingestion path.
+For manual `update_memory_item`, omitted `memory_categories` preserves existing category links,
+while an explicit empty list clears them.
+Manual create/update/delete inputs trim IDs, memory types, content, and category names, and reject
+blank IDs, invalid memory types, blank content, or blank category names before entering workflow execution.
+`MemoryItem.extra` is a per-record metadata dictionary created with a default factory, so tool-memory
+and reinforcement metadata cannot leak between records.
+Tool call history stored in `extra.tool_calls` is dumped in JSON mode so timestamps remain
+portable across in-memory, SQLite, Postgres, HTTP responses, and exported artifacts.
+Scoped `clear_memory` deletes category-item relation edges before categories, items,
+and resources so no backend can leave dangling in-memory or persistent relations behind.
+Single-item `delete_memory_item` follows the same rule by clearing relation edges for the target
+item before deleting the item record.
## LLM/provider architecture
@@ -142,28 +203,160 @@ LLM access is profile-based (`llm_profiles`):
- `default` profile for chat-like tasks
- `embedding` profile for embedding tasks (auto-derived from default if not set)
+Profile names are whitespace-trimmed and must be non-empty.
Per-step profile routing happens through step config (`chat_llm_profile`, `embed_llm_profile`, or `llm_profile`).
+Retrieve workflows expose separate profiles for route intention, sufficiency checks, and LLM ranking, so callers can
+split cheap routing from heavier ranking models without changing pipeline code.
Client backends:
- `sdk`: official OpenAI SDK wrapper
- `httpx`: provider-adapted HTTP backend (OpenAI, Doubao, Grok, OpenRouter)
-- `lazyllm_backend`: LazyLLM adapter
+- `lazyllm_backend`: optional LazyLLM adapter, installed with the `lazyllm` extra
## Integration surfaces
-- `memu.client.openai_wrapper`: opt-in OpenAI client wrapper that auto-retrieves memories and injects them into system context
-- `memu.integrations.langgraph`: LangChain/LangGraph tool adapter (`save_memory`, `search_memory`)
+- `memu.client.openai_wrapper`: opt-in OpenAI client wrapper that auto-retrieves memories and injects them into
+ system context. Its `ranking` parameter is passed through as a per-call retrieve ranking override, and `top_k`
+ must be a positive integer and caps injected memory items. Wrapper user scope is copied at construction time so
+ caller-owned `user_data` dictionaries cannot mutate active retrieve scope accidentally.
+- `memu.integrations.langgraph`: LangChain/LangGraph tool adapter (`save_memory`, `search_memory`).
+ Its explicit `user_id` scope is authoritative and cannot be overridden by metadata filters.
+
+## Markdown-backed folder compiler
+
+`memu.app.folder.FolderMemoryCompiler` adds a file-system harness layer for compiling an uploaded folder into a portable Markdown memory repository:
+
+
+
+- `raw_data/`: synchronized copy of the uploaded folder, including text, images, audio, video, documents, code, and unknown binary files
+- `.memu/harness.json`: repository-local CLI defaults for compiler and context behavior
+- `.memu/manifest.json`: source hash index used for incremental re-extraction and removal of deleted-source memories
+- `.memu/derived/`: per-source evidence Markdown for text evidence or multimodal metadata evidence
+- `.memu/evolution/`: append-only audit records for Evolution Instructions, Patch Proposals, and review decisions
+- `AGENTS.md`: non-overwriting bootstrap instructions for local agents using the harness repository
+- `memory.md` and `memory/`: durable facts, preferences, events, and knowledge
+- `soul.md` and `soul/`: persona, tone, voice, language style, and interaction-style signals
+- `skill.md` and `skill/`: skills, workflows, tool-use patterns, procedures, and reusable capabilities
+
+The compiler can run with local deterministic extraction so the Markdown
+repository is always buildable. When a configured `MemoryService` is supplied,
+it delegates richer extraction to the existing `memorize` workflow, appends
+returned captions/items/categories to the derived evidence file, and turns
+extracted candidates into reviewed long-term-context patches.
+
+Self-evolve has a hard ownership boundary: raw logs, creator feedback, uploads,
+and new observations never edit `memory.md`, `soul.md`, or `skill.md` directly.
+The compiler first creates structured `EvolutionInstruction` records containing
+`target`, `operation`, `reason`, `evidence`, `priority`, and `confidence`. Each
+instruction becomes a `PatchProposal`; the review gate then approves, rejects,
+or marks the proposal as `needs_review` based on traceable evidence,
+confidence, conflict detection, and safety checks. Only approved proposals are
+applied to the generated Markdown blocks and manifest entries.
+
+
+
+Incremental compiles keep generated artifacts aligned with the manifest:
+deleted sources create reviewed delete proposals that remove generated entries,
+stale raw-data copies, and stale `.memu/derived/**/*.evidence.md` files only
+after approval. Health checks warn about orphaned
+derived evidence that is not referenced by the manifest. Callers can provide
+explicit source-relative exclude globs to skip noisy caches or build artifacts;
+memU does not apply default excludes. The same patterns can be persisted in a
+source or repository `.memuignore` file.
+
+The unified harness CLI and standalone `memu-context` command can read
+`.memu/harness.json` for repository-local defaults such as compiler exclude
+globs, `max_text_chars`, context output format, total context budget, selected
+buckets, and per-bucket context budgets. Command-line arguments override the
+config file. LLM provider, API key, and user-scope settings remain
+command/environment driven instead of being stored in the repository config.
+
+For local multimodal extraction without an LLM, the compiler also reads
+sidecar files beside media, document, and unknown binary sources, such as
+`image.caption.md`, `image.metadata.json`, `audio.transcript.txt`,
+`video.frames.jsonl`, or `report.summary.md`, and appends that sidecar content
+to the paired source's derived evidence. Sidecars are included in the paired
+source fingerprint, so updating a caption, transcript, summary, OCR output, or
+metadata file triggers re-extraction of that source.
+
+The installed CLI command `memu-folder` exposes this flow for tool use:
+
+```bash
+memu-folder path/to/raw-folder path/to/memory-repo
+memu-folder path/to/raw-folder path/to/memory-repo --watch
+```
+
+`memu.app.markdown_context.MarkdownMemoryRepository` reads the generated
+Markdown repository back into a context harness. It loads generated entries from
+`.memu/manifest.json`, preserves human notes outside generated blocks, ranks
+sections with lightweight query matching, and emits an agent-ready
+`` pack, system prompt, chat message list, or lightweight summary
+index. Context assembly can also apply per-bucket character limits so `memory`,
+`soul`, and `skill` sections share a predictable context budget. Promoted skill
+index snippets in `skill.md` are skipped when their full `skill/promoted/*.md`
+cards are present, avoiding duplicate manual skill context. CLI context output
+can be printed or written to a file for downstream agents and scripts. The
+installed CLI command `memu-context` exposes this path:
+
+```bash
+memu-context path/to/memory-repo --query "current task"
+memu-context path/to/memory-repo --query "current task" --format summary
+memu-context path/to/memory-repo --query "current task" --format messages
+memu-context path/to/memory-repo --bucket-max soul=1000 --bucket-max skill=2000
+memu-context path/to/memory-repo --format system --output context.system.md
+```
+
+`memu.app.skill_trace.record_skill_trace` records agent execution traces under
+`raw_data/skill_traces/`. These traces are raw evidence for self-evolving
+skills: the normal folder compiler or watch mode converts them into Evolution
+Instructions and Patch Proposals before any approved update reaches `skill.md`
+or `skill/`. `suggest_skill_promotions` groups trace lessons, actions, tools,
+and outcomes into deterministic promotion candidates. Suggestions are read-only
+until the caller explicitly promotes them into durable manual skills.
+
+`memu.app.context_harness.ContextHarness` is the high-level composition layer
+for this Markdown-backed mode. It binds a user raw-data folder to a memory
+repository and exposes one-object methods for:
+
+- `ingest`: compile changed raw data through self-evolve review into Markdown memory
+- `scaffold`: create the Markdown repository layout before extraction
+- `status`: inspect source changes against the manifest without writing files
+- `health`: validate repository layout, manifest references, and generated blocks
+- `suggest_skills`: propose durable skill promotions from raw traces
+- `promote_skill`: append durable manual skill notes and update stable skill cards
+- `refresh_context`: compile, then assemble an agent-ready context pack
+- `record_skill_trace`: persist execution lessons under raw data and optionally recompile
+- `watch`: poll the raw-data folder and recompile on source changes
+
+For an already initialized repository, `ContextHarness.from_repo(repo_dir)` uses
+`repo/raw_data` as the source and applies `.memu/harness.json` compiler and
+context defaults unless explicit Python arguments override them.
+
+The installed CLI command `memu-harness` exposes the same composition layer:
+
+```bash
+memu-harness init path/to/memory-repo --source-folder path/to/raw-folder
+memu-harness doctor path/to/memory-repo --json
+memu-harness status path/to/memory-repo --json
+memu-harness refresh path/to/memory-repo --query "current task"
+memu-harness review-evolution path/to/memory-repo
+memu-harness trace path/to/memory-repo --task "What changed?"
+memu-harness suggest-skills path/to/memory-repo --json
+memu-harness promote-skill path/to/memory-repo --title "Reusable workflow"
+```
## Current constraints and tradeoffs
- workflow state is dict-based, so step contracts are validated by key names rather than static types
- SQLite/inmemory vector search is brute-force (portable but less scalable)
- category update quality and extraction quality are prompt/LLM dependent
-- some extension hooks exist as placeholders (for example dedupe/merge stage)
+- deeper semantic merge policies can still be added on top of the current deterministic dedupe stage
## Related ADRs
- `docs/adr/0001-workflow-pipeline-architecture.md`
- `docs/adr/0002-pluggable-storage-and-vector-strategy.md`
- `docs/adr/0003-user-scope-in-data-model.md`
+- `docs/adr/0004-markdown-context-harness-and-skill-evolution.md`
+- `docs/adr/0005-self-evolve-instruction-review-gate.md`
diff --git a/docs/folder_memory_compiler.md b/docs/folder_memory_compiler.md
new file mode 100644
index 00000000..253a8cd1
--- /dev/null
+++ b/docs/folder_memory_compiler.md
@@ -0,0 +1,554 @@
+# Folder Memory Compiler
+
+`FolderMemoryCompiler` turns an uploaded folder into a Markdown-backed memory repository.
+
+It is designed for the workflow:
+
+1. A user provides a folder containing raw data.
+2. memU copies that folder into `raw_data/`.
+3. memU compiles traceable evidence into `.memu/derived/`.
+4. memU converts extracted evidence into structured Evolution Instructions.
+5. memU builds Patch Proposals and sends them through the review gate.
+6. Only approved proposals update `memory.md`, `soul.md`, and `skill.md`.
+7. On the next run, changed files are re-extracted and deleted files become reviewed delete proposals.
+
+## Output Layout
+
+```text
+memory_repo/
+ AGENTS.md
+
+ raw_data/
+ ... original uploaded files, preserved as raw data
+
+ memory.md
+ memory/
+
+ soul.md
+ soul/
+
+ skill.md
+ skill/
+
+ .memu/
+ harness.json
+ manifest.json
+ evolution/
+ instructions.jsonl
+ patch_proposals.jsonl
+ review_decisions.jsonl
+ derived/
+ ... per-source evidence markdown
+```
+
+`raw_data/` is the evidence source of truth inside the generated repository.
+The compiler does not mutate the user's input folder.
+`AGENTS.md` is a default, non-overwriting bootstrap file for local coding
+agents. It points agents at `memory.md`, `soul.md`, `skill.md`, `raw_data/`,
+sidecars, skill traces, and the main `memu-harness` commands.
+
+`.memu/harness.json` stores repository-local, non-secret defaults for the
+unified harness CLI. It is created by `memu-harness init` and can be edited by
+humans or tooling.
+
+## Markdown Buckets
+
+- `memory.md` stores durable facts, preferences, events, knowledge, and general memory.
+- `soul.md` stores persona, tone, voice, language style, and interaction-style signals.
+- `skill.md` stores skills, workflows, tool-use patterns, procedures, and reusable capabilities.
+
+Each top-level file has a generated block:
+
+```md
+
+...
+
+```
+
+memU replaces only this block on recompile. Human edits outside the block are preserved.
+
+## Self-Evolve Review Gate
+
+Raw data never edits `memory.md`, `soul.md`, or `skill.md` directly. For each
+changed source, the compiler first writes evidence under `.memu/derived/`, then
+creates an `EvolutionInstruction` with:
+
+- `target`: `memory`, `soul`, or `skill`
+- `operation`: `add`, `update`, or `delete`
+- `reason`, `evidence`, `priority`, and `confidence`
+
+Each instruction becomes a `PatchProposal`. The review gate approves, rejects,
+or marks the proposal as `needs_review` using traceability, confidence, conflict
+detection, and safety checks. Approved proposals update the generated Markdown
+blocks; pending proposals remain auditable in `.memu/manifest.json` and
+`.memu/evolution/`.
+
+Use `--require-creator-review` with `memu-folder` or `memu-harness refresh` to
+force proposals to remain pending until a creator reviews them. Then use
+`memu-harness review-evolution` to approve or reject pending proposals:
+
+```bash
+memu-harness refresh memory_repo --require-creator-review
+memu-harness review-evolution memory_repo
+memu-harness review-evolution memory_repo --proposal-id patch_abc123 --reject
+```
+
+## Multimodal Raw Data
+
+All files are copied into `raw_data/`, including:
+
+- text and Markdown
+- JSON, CSV, logs, and code
+- images
+- audio
+- video
+- PDFs and office documents
+- unknown binary files
+
+For text-like files, `.memu/derived/` stores text evidence. For binary,
+document, or multimodal files, `.memu/derived/` stores traceable metadata
+evidence. If a `MemoryService` with multimodal or document-capable LLM profiles
+is provided, the compiler calls the existing `memorize` pipeline and appends
+returned resource captions, memory items, and category summaries into the
+evidence file.
+
+Without an LLM, non-text sources can still carry semantic evidence through
+sidecar text files. For a source such as `workflow.png` or `report.pdf`, the
+compiler will read nearby files such as:
+
+- `workflow.caption.md`
+- `workflow.metadata.json`
+- `workflow.png.caption.md`
+- `workflow.transcript.txt`
+- `workflow.ocr.jsonl`
+- `workflow.notes.md`
+- `report.summary.md`
+- `report.pdf.notes.txt`
+
+Supported sidecar labels are `alt`, `caption`, `description`, `notes`,
+`summary`, `transcript`, `metadata`, `meta`, `ocr`, and `frames`. Supported
+sidecar formats are Markdown, text, JSON, and JSONL. JSON and JSONL sidecars are
+stable-formatted before being appended to the paired source's derived evidence,
+so they can produce `memory`, `soul`, or `skill` entries that still point back
+to the original raw source.
+
+Sidecar files are treated as part of the paired source fingerprint.
+Changing `workflow.caption.md` therefore re-extracts `workflow.png`, while the
+sidecar itself is still copied into `raw_data/`.
+
+When a source is deleted, memU creates delete Evolution Instructions. Approved
+delete proposals remove generated Markdown entries, raw copies, and generated
+`.memu/derived/**/*.evidence.md` files. Human-written Markdown notes outside
+generated blocks are preserved.
+
+## Usage
+
+Command line:
+
+```bash
+memu-folder path/to/uploaded-folder path/to/memory-repo
+```
+
+Machine-readable summary:
+
+```bash
+memu-folder path/to/uploaded-folder path/to/memory-repo --json
+```
+
+Exclude noisy files explicitly with repeated posix glob patterns:
+
+```bash
+memu-folder path/to/uploaded-folder path/to/memory-repo \
+ --exclude "node_modules/**" \
+ --exclude "*.tmp"
+```
+
+You can also persist these rules in `.memuignore` at the uploaded folder root
+or memory repository root:
+
+```gitignore
+# .memuignore
+node_modules/**
+*.tmp
+**/__pycache__/**
+```
+
+Watch for changes and recompile automatically:
+
+```bash
+memu-folder path/to/uploaded-folder path/to/memory-repo --watch
+```
+
+For automation, emit one JSON line per compile event:
+
+```bash
+memu-folder path/to/uploaded-folder path/to/memory-repo \
+ --watch \
+ --json \
+ --poll-interval 2
+```
+
+Each watch JSON event includes a `delta` object with `new`, `changed`, and
+`removed` source paths computed before that compile event.
+
+With `MemoryService` extraction:
+
+```bash
+memu-folder path/to/uploaded-folder path/to/memory-repo \
+ --use-memory-service \
+ --api-key-env OPENAI_API_KEY \
+ --chat-model gpt-4o-mini
+```
+
+For both `memu-folder` and `memu-harness`, if `--api-key-env` is omitted, the
+CLI uses the provider default (`OPENAI_API_KEY` for OpenAI-compatible defaults,
+`XAI_API_KEY` for `--provider grok`). Pass `--api-key` only for local
+experiments; avoid putting secrets in shell history for shared environments.
+
+Pass user scope values with repeated `--user` flags:
+
+```bash
+memu-folder raw_data memory_repo --user user_id=u1 --user agent_id=assistant
+```
+
+Use the unified context harness command when one tool should own ingestion,
+context assembly, watching, and skill evolution:
+
+```bash
+memu-harness init path/to/memory-repo --source-folder path/to/uploaded-folder
+
+memu-harness doctor path/to/memory-repo --json
+
+memu-harness refresh path/to/memory-repo \
+ --query "current agent task"
+
+memu-harness refresh path/to/memory-repo \
+ --exclude "node_modules/**" \
+ --exclude "*.tmp"
+
+memu-harness status path/to/memory-repo --json
+```
+
+`init` creates `AGENTS.md`, `memory.md`, `memory/`, `soul.md`, `soul/`,
+`skill.md`, `skill/`, `.memu/`, `.memu/harness.json`, and `raw_data/`. If
+`--source-folder` is provided, the raw files are copied into `raw_data/` before
+extraction. Existing `AGENTS.md` and `.memu/harness.json` files are preserved.
+
+`doctor` is read-only. It validates the repository layout, manifest shape,
+generated block markers, and manifest references to raw files, sidecars,
+evidence files, per-source Markdown detail files, and orphaned derived evidence.
+Missing `AGENTS.md` or orphaned `.evidence.md` files produce warnings, not
+errors, so older repositories remain usable.
+
+For `refresh`, `ingest`, `context`, `trace`, and `watch`, passing only the
+memory repository path uses `repo/raw_data` as the source folder. Passing
+`SOURCE_FOLDER REPO_DIR` is still supported when the raw-data folder lives
+outside the repository.
+
+For API calls, passing an already initialized memory repository as both
+`source_folder` and `output_folder` is treated the same way: memU reads from
+`repo/raw_data` and leaves repository-level Markdown notes alone.
+
+If the output repository is placed inside the uploaded folder, memU excludes the
+output repository subtree from scanning and from the initial `raw_data/` copy.
+
+`status` is read-only. It compares the current raw-data source fingerprints
+against `.memu/manifest.json` and reports `new`, `changed`, `unchanged`, and
+`removed` sources, including media sidecar bindings.
+
+`--exclude` patterns are matched against posix-style source-relative paths and
+can be repeated. memU does not apply default excludes, so uploaded evidence is
+preserved unless the caller explicitly excludes a path such as `node_modules/**`,
+`*.tmp`, or `**/__pycache__/**`. `.memuignore` uses the same pattern syntax.
+For one-path harness commands such as `memu-harness refresh memory_repo`, memU
+uses `memory_repo/.memuignore` and scans `memory_repo/raw_data`.
+
+### Harness Config
+
+For repeated agent runs, store repo defaults in `.memu/harness.json`:
+
+```json
+{
+ "version": 1,
+ "compiler": {
+ "exclude_patterns": ["node_modules/**", "*.tmp"],
+ "max_text_chars": 4000
+ },
+ "context": {
+ "buckets": ["memory", "soul", "skill"],
+ "bucket_char_limits": {
+ "soul": 1000,
+ "skill": 2000
+ },
+ "format": "system",
+ "max_chars": 8000
+ }
+}
+```
+
+`compiler.exclude_patterns` and `compiler.max_text_chars` are used by
+`memu-harness ingest`, `refresh`, `status`, `trace`, and `watch`.
+`context.max_chars`, `context.bucket_char_limits`, `context.buckets`, and
+`context.format` are used by `memu-harness context`, `memu-harness refresh`,
+and standalone `memu-context`. Command-line flags override
+`.memu/harness.json`; the config file overrides built-in defaults. The config
+intentionally excludes LLM provider, API key, and user-scope settings because
+those are environment or request specific.
+
+`doctor` validates the config shape and reports invalid values as health
+errors.
+
+Python callers can use the same config helpers:
+
+```python
+import json
+from memu import default_harness_config, harness_config_path, load_harness_config
+
+repo_dir = "path/to/memory-repo"
+config_path = harness_config_path(repo_dir)
+config_path.write_text(
+ json.dumps(
+ default_harness_config(
+ exclude_patterns=["node_modules/**", "*.tmp"],
+ max_text_chars=4000,
+ ),
+ indent=2,
+ ),
+ encoding="utf-8",
+)
+config = load_harness_config(repo_dir)
+```
+
+Record a reusable skill lesson and submit it to the self-evolve gate:
+
+```bash
+memu-harness trace path/to/memory-repo \
+ --task "Validate generated context packs" \
+ --outcome success \
+ --lesson "Check generated skill sections before injecting context into an agent"
+```
+
+Suggest durable skills from accumulated traces without writing:
+
+```bash
+memu-harness suggest-skills path/to/memory-repo --json
+```
+
+Apply suggested skills explicitly:
+
+```bash
+memu-harness suggest-skills path/to/memory-repo --min-support 2 --promote
+```
+
+Promote a lesson into the durable manual skill library:
+
+```bash
+memu-harness promote-skill path/to/memory-repo \
+ --title "Validate Context Packs" \
+ --when "Before injecting generated context into an agent" \
+ --action "Build the context pack" \
+ --action "Check manual and generated skill sections" \
+ --lesson "Inspect promoted skills before relying on generated context"
+```
+
+`suggest-skills` groups raw `skill_traces/*.md` lessons, actions, tools, and
+outcomes into deterministic promotion candidates. It is read-only unless
+`--promote` is passed. Promoted skills are written outside the generated block
+in `skill.md` and as stable cards under `skill/promoted/`. Promoting the same
+title again updates the same card and keeps prior lessons/actions, source, and
+metadata, so `refresh` preserves the evolving skill library while generated
+skill traces continue to produce reviewed proposals. Context assembly skips
+`skill.md` promoted index snippets when their full `skill/promoted/*.md` cards
+are present, avoiding duplicate promoted skill context.
+
+Build an agent-ready context pack from the generated repository:
+
+```bash
+memu-context memory_repo --query "how should I answer this user?"
+```
+
+Machine-readable context pack or chat messages:
+
+```bash
+memu-context memory_repo --query "debug workflow" --json
+memu-context memory_repo --query "debug workflow" --format summary
+memu-context memory_repo --query "debug workflow" --format messages
+memu-harness refresh memory_repo --query "debug workflow" --format system
+```
+
+Write the rendered context to a stable file for an agent or script:
+
+```bash
+memu-context memory_repo \
+ --query "debug workflow" \
+ --format system \
+ --output context.system.md
+
+memu-harness refresh memory_repo \
+ --query "debug workflow" \
+ --json \
+ --output context-refresh.json
+```
+
+For large repositories, reserve or cap individual memory buckets so one category
+does not crowd out the others:
+
+```bash
+memu-context memory_repo \
+ --query "debug workflow" \
+ --bucket-max soul=1000 \
+ --bucket-max skill=2000
+```
+
+`bucket_char_limits` and `used_chars_by_bucket` are included in JSON and summary
+outputs so agents can inspect context-budget pressure.
+
+Record an execution trace for self-evolving skills:
+
+```bash
+memu-skill-trace raw_data \
+ --task "Fix failing compiler tests" \
+ --outcome success \
+ --action "Read failing test output" \
+ --action "Patch the affected module" \
+ --lesson "Run focused tests after changing compiler behavior" \
+ --tool "pytest:success:0.9" \
+ --output-folder memory_repo
+```
+
+The trace is written under `raw_data/skill_traces/`. Because it is raw evidence,
+a normal `memu-folder` compile or `--watch` run converts it into Evolution
+Instructions and Patch Proposals before any approved skill update reaches
+`skill.md` or `skill/`.
+
+Python:
+
+```python
+import asyncio
+from memu import FolderMemoryCompiler
+
+
+async def main() -> None:
+ compiler = FolderMemoryCompiler()
+ result = await compiler.compile(
+ source_folder="path/to/uploaded-folder",
+ output_folder="path/to/memory-repo",
+ )
+ print(result.processed)
+
+
+asyncio.run(main())
+```
+
+With an existing `MemoryService`:
+
+```python
+from memu import FolderMemoryCompiler, MemoryService
+
+service = MemoryService(...)
+compiler = FolderMemoryCompiler(memory_service=service)
+```
+
+Use the high-level context harness API when one object should own ingestion,
+context assembly, watching, and skill evolution:
+
+```python
+import asyncio
+from memu import ContextHarness, SkillToolTrace
+
+
+async def main() -> None:
+ upload_harness = ContextHarness(
+ source_folder="path/to/uploaded-folder",
+ repo_dir="path/to/memory-repo",
+ )
+ upload_harness.scaffold(copy_source=True)
+
+ harness = ContextHarness.from_repo("path/to/memory-repo")
+ run = await harness.refresh_context(query="current agent task")
+ system_context = run.context_pack.to_markdown()
+
+ await harness.record_skill_trace(
+ task="Validate generated context packs",
+ outcome="success",
+ tools=[SkillToolTrace(name="memu-context", success=True, score=0.95)],
+ lessons=["Check generated skill sections before injecting context into an agent."],
+ )
+ suggestions = harness.suggest_skills(min_support=1)
+ print(system_context)
+ print([suggestion.title for suggestion in suggestions])
+
+
+asyncio.run(main())
+```
+
+`ContextHarness.from_repo(...)` uses `repo/raw_data` as the source folder and
+applies `.memu/harness.json` defaults for compiler excludes, text evidence
+limits, selected context buckets, total context budget, and per-bucket budgets.
+Explicit Python method arguments still override those repo defaults. If a
+custom `FolderMemoryCompilerConfig` is provided, non-default fields are
+preserved while repo config can still fill compiler defaults such as
+`exclude_patterns` and `max_text_chars`. `ContextHarness.health()` reports
+invalid `.memu/harness.json` values as health errors; normal compile/context
+operations fail until the config is fixed.
+
+Load context in Python:
+
+```python
+from memu import MarkdownMemoryRepository
+
+repo = MarkdownMemoryRepository("path/to/memory-repo")
+pack = repo.build_context_pack(query="debugging style", max_chars=4000)
+messages = pack.inject_into_messages(messages)
+context_only_messages = pack.to_messages()
+budgeted = repo.build_context_pack(
+ query="debugging style",
+ max_chars=8000,
+ bucket_char_limits={"soul": 1000, "skill": 2000},
+)
+```
+
+Record a skill trace in Python:
+
+```python
+from memu import SkillToolTrace, promote_skill, record_skill_trace
+
+record_skill_trace(
+ "raw_data",
+ task="Validate generated context packs",
+ outcome="success",
+ actions=["Compile folder", "Build context pack", "Check skill sections"],
+ tools=[SkillToolTrace(name="memu-context", success=True, score=0.95)],
+ lessons=["Validate context packs before relying on generated skills."],
+)
+
+promote_skill(
+ "memory_repo",
+ title="Validate Context Packs",
+ when_to_use="Before injecting generated context into an agent.",
+ actions=["Build the context pack", "Check manual and generated skill sections"],
+ lessons=["Inspect promoted skills before relying on generated context."],
+)
+```
+
+## Incremental Update
+
+`.memu/manifest.json` records each source file's path, hash, modality,
+evidence path, generated entry IDs, and the latest self-evolve audit chain. On
+recompile:
+
+- unchanged files are skipped;
+- changed files are re-extracted;
+- changed media sidecars re-extract their paired media source;
+- removed files create delete proposals; approved delete proposals remove
+ generated Markdown and generated evidence;
+- `raw_data/` is synchronized to match the current input folder.
+- files matching configured `exclude_patterns` are ignored during scan, status,
+ watch fingerprinting, sidecar pairing, and raw-data copy.
+
+Manual Markdown files under `memory/`, `soul/`, and `skill/` are preserved. If a
+generated per-source detail file becomes stale but contains manual notes outside
+the generated block, memU clears the generated block instead of deleting the
+file.
+
+This makes the generated memory repository portable, inspectable, and suitable for version control.
diff --git a/docs/index.md b/docs/index.md
new file mode 100644
index 00000000..d77ac81c
--- /dev/null
+++ b/docs/index.md
@@ -0,0 +1,27 @@
+# memU Documentation
+
+memU is a Python library for AI memory and conversation management.
+It centers on `MemoryService`, workflow-based ingestion and retrieval, pluggable
+storage backends, and profile-based LLM routing.
+
+## Start here
+
+- [Getting Started](tutorials/getting_started.md): install `memu-py` and run the
+ first memory workflow.
+- [Architecture](architecture.md): understand `MemoryService`, workflows,
+ storage backends, LLM profiles, and the self-evolve review gate.
+- [Folder Memory Compiler](folder_memory_compiler.md): compile raw files into a
+ Markdown memory repository with reviewed evolution instructions.
+- [SQLite Storage](sqlite.md): configure local persistent storage.
+
+## Integrations
+
+- [LangGraph](langgraph_integration.md): expose memU memory as LangGraph tools.
+- [Grok](providers/grok.md): configure Grok provider defaults.
+- [Sealos Devbox](sealos-devbox-guide.md): deploy the support-agent example.
+
+## Design records
+
+The [Architecture Decision Records](adr/README.md) explain why the project uses
+workflow pipelines, pluggable storage, first-class user scope, Markdown-backed
+context harnesses, and reviewed self-evolution patches.
diff --git a/docs/integrations/grok.md b/docs/integrations/grok.md
index db501833..380eee42 100644
--- a/docs/integrations/grok.md
+++ b/docs/integrations/grok.md
@@ -30,18 +30,18 @@ To use Grok as your LLM provider, switch the `provider` setting to `grok`. This
### Python Example
```python
-from memu.app.settings import LLMConfig
-
-# Configure MemU to use Grok
-config = LLMConfig(
- provider="grok",
- # The default API key env var is XAI_API_KEY
- # The default model is grok-2-latest
+from memu import MemoryService
+
+# Configure MemU to use Grok through the default LLM profile.
+service = MemoryService(
+ llm_profiles={
+ "default": {
+ "provider": "grok",
+ # Defaults: api_key="XAI_API_KEY", base_url="https://api.x.ai/v1",
+ # chat_model="grok-2-latest"
+ }
+ }
)
-
-print(f"Using provider: {config.provider}")
-print(f"Base URL: {config.base_url}")
-print(f"Chat Model: {config.chat_model}")
```
## Models Supported
diff --git a/docs/langgraph_integration.md b/docs/langgraph_integration.md
index 005851e0..e3a4298c 100644
--- a/docs/langgraph_integration.md
+++ b/docs/langgraph_integration.md
@@ -15,9 +15,18 @@ These tools are fully typed and compatible with LangGraph's `prebuilt.ToolNode`
To use this integration, you need to install the optional dependencies:
```bash
-uv add langgraph langchain-core
+pip install "memu-py[langgraph]"
```
+If you manage dependencies with uv in an application project:
+
+```bash
+uv add "memu-py[langgraph]"
+```
+
+When working from a source checkout, install the direct integration dependencies
+with `uv sync --extra langgraph`.
+
## Quick Start
Here is a complete example of how to initialize the MemU memory service and bind it to a LangGraph agent.
@@ -25,11 +34,11 @@ Here is a complete example of how to initialize the MemU memory service and bind
```python
import asyncio
import os
-from memu.app.service import MemoryService
+from memu import MemoryService
from memu.integrations.langgraph import MemULangGraphTools
# Ensure you have your configuration set (e.g., env vars for DB connection)
-# os.environ["MEMU_DATABASE_URL"] = "..."
+# os.environ["MEMU_DATABASE_DSN"] = "..."
async def main():
# 1. Initialize MemoryService
@@ -83,15 +92,21 @@ class MemULangGraphTools(memory_service: MemoryService)
Returns a tool named `save_memory`.
- **Inputs**: `content` (str), `user_id` (str), `metadata` (dict, optional).
- **Description**: Save a piece of information, conversation snippet, or memory for a user.
+- **Scope rule**: `user_id` is authoritative. A `metadata["user_id"]` value cannot override it.
#### `search_memory_tool() -> StructuredTool`
Returns a tool named `search_memory`.
-- **Inputs**: `query` (str), `user_id` (str), `limit` (int, default=5), `metadata_filter` (dict, optional), `min_relevance_score` (float, default=0.0).
+- **Inputs**: `query` (str), `user_id` (str), `limit` (int, default=5),
+ `metadata_filter` (dict, optional), `min_relevance_score` (float, default=0.0).
- **Description**: Search for relevant memories or information for a user based on a query.
+- **Bounds**: `query` and `user_id` must be non-empty, `limit` must be at least 1,
+ and `min_relevance_score` must be between 0.0 and 1.0.
+- **Scope rule**: `user_id` is authoritative. A `metadata_filter["user_id"]` value cannot override it.
## Troubleshooting
### Import Errors
If you see an `ImportError` regarding `langchain_core` or `langgraph`:
-1. Ensure you have installed the extras: `uv add langgraph langchain-core` (or `pip install langgraph langchain-core`).
+1. Ensure you have installed the extra: `pip install "memu-py[langgraph]"` or `uv add "memu-py[langgraph]"`.
+ From a source checkout, run `uv sync --extra langgraph`.
2. Verify your virtual environment is active.
diff --git a/docs/providers/grok.md b/docs/providers/grok.md
index 702eb1e5..d7e600ed 100644
--- a/docs/providers/grok.md
+++ b/docs/providers/grok.md
@@ -19,7 +19,7 @@ The integration is designed to work out-of-the-box with minimal configuration.
Set the following environment variable in your `.env` file or system environment:
```bash
-GROK_API_KEY=xai-YOUR_API_KEY_HERE
+XAI_API_KEY=xai-YOUR_API_KEY_HERE
```
### Defaults
@@ -36,31 +36,37 @@ You can enable the Grok provider by setting the `provider` field to `"grok"` in
### Using Python Configuration
```python
-from memu.app.settings import LLMConfig
-from memu.app.service import MemoryService
-
-# Configure the LLM provider to use Grok
-llm_config = LLMConfig(provider="grok")
+from memu import MemoryService
# Initialize the service
-service = MemoryService(llm_config=llm_config)
-print(f"Service initialized with model: {llm_config.chat_model}")
-# Output: Service initialized with model: grok-2-latest
+service = MemoryService(
+ llm_profiles={
+ "default": {
+ "provider": "grok",
+ # Defaults: api_key="XAI_API_KEY", base_url="https://api.x.ai/v1",
+ # chat_model="grok-2-latest"
+ }
+ }
+)
```
## Troubleshooting
### Connection Issues
If you are unable to connect to the xAI API:
-1. Verify that your `GROK_API_KEY` is set correctly and has not expired.
+1. Verify that your `XAI_API_KEY` is set correctly and has not expired.
2. Ensure that the `base_url` is resolving to `https://api.x.ai/v1`. If you have manual overrides in your settings, they might be conflicting with the default.
### Model Availability
If you receive a `404` or "Model not found" error, xAI may have updated their model names. You can override the model manually in the config if needed:
```python
-config = LLMConfig(
- provider="grok",
- chat_model="grok-beta" # Example override
+service = MemoryService(
+ llm_profiles={
+ "default": {
+ "provider": "grok",
+ "chat_model": "grok-beta", # Example override
+ }
+ }
)
```
diff --git a/docs/sealos_use_case.md b/docs/sealos_use_case.md
index 5ef9edbd..3e1a5512 100644
--- a/docs/sealos_use_case.md
+++ b/docs/sealos_use_case.md
@@ -1,54 +1,66 @@
-# 🛡️ Context-Aware Support Agent (Sealos Edition)
+# Context-Aware Support Agent (Sealos Edition)
## Overview
-This use case demonstrates how **MemU** enables a support agent to remember user history across sessions, deployed on a **Sealos Devbox** environment.
-Unlike a standard web app, this demo focuses on the **backend memory orchestration**. It runs as a **CLI (Command Line Interface)** tool to transparently show the internal memory logs, retrieval process, and state persistence without the abstraction layer of a UI.
+This use case demonstrates how memU helps a support agent remember user history
+across sessions in a Sealos Devbox-style environment.
-## 🚀 Quick Start
+Unlike a standard web app, this demo focuses on backend memory orchestration. It
+runs as a CLI tool so reviewers can see the ingestion, retrieval, and response
+flow directly in the terminal.
+
+## Quick Start
### Prerequisites
-- Sealos Devbox Environment
-- Python 3.13+
-- MemU Library (installed via `make install`)
-### How to Run the Demo
-Since this is a backend demonstration, you will run the agent directly in the terminal to observe the memory cycle.
+- Sealos Devbox environment or a local Python shell
+- Python 3.12+
+- memU installed with `pip install memu-py`, or a source checkout
+
+When running from a source checkout, the script adds the local `src/` directory
+to `sys.path` before importing memU, so this command works without building a
+wheel first:
```bash
uv run python examples/sealos_support_agent.py
```
-## 📸 Live Demo Output (Proof of Concept)
+If memU or its runtime dependencies are not importable, the demo falls back to a
+deterministic offline simulation so the flow remains reviewable.
-Below is the actual output captured from the Sealos terminal. This serves as verification of the "Demonstration Quality" requirement.
+## Live Demo Output
```plaintext
-🚀 Starting Sealos Support Agent Demo (Offline Mode)
+[START] Starting Sealos Support Agent Demo (Offline Mode)
+===================================================
-📝 --- Phase 1: Ingesting Conversation History ---
-👤 Captain: "I'm getting a 502 Bad Gateway error on port 3000."
-🤖 Agent: (Memorizing this interaction...)
-✅ Memory stored! extracted 2 items.
- - [profile] Captain reported a 502 Bad Gateway error on port 3000.
+[OK] Environment Check: MemU Library detected.
+[OK] Runtime: Sealos Devbox (Python 3.12+)
-🔍 --- Phase 2: Retrieval on New Interaction ---
-👤 Captain: "Hello"
-🤖 Agent: (Searching memory for context...)
+[PHASE 1] Ingesting Conversation History
+Captain: "I'm getting a 502 Bad Gateway error on port 3000."
+Agent: (Processing input through Memory Pipeline...)
+[OK] Memory stored! extracted 2 items:
+ - [issue] 502 Bad Gateway error
+ - [context] port 3000 configuration
-💡 Retrieved Context:
- Found Memory: Captain reported a 502 Bad Gateway error on port 3000.
+[PHASE 2] Retrieval on New Interaction (New Session)
+Captain: "Hello, any updates?"
+Agent: (Searching vector store for user 'Captain'...)
-💬 --- Phase 3: Agent Response ---
-🤖 Agent: "Welcome back, Captain. I see you had a 502 error on port 3000 recently. Is that resolved?"
+[CONTEXT] Retrieved Context:
+ Found Memory (Score: 0.98): User reported 502 error on port 3000
+ Found Memory (Score: 0.95): User was frustrated with timeout
-✨ Demo Completed Successfully
-```
+[PHASE 3] Agent Response
+Agent: "Welcome back, Captain. Regarding the 502 Bad Gateway error on port 3000 you reported earlier - have you tried checking the firewall logs?"
-## 💡 Code Highlights & Justification
-
-- **CLI vs Web**: We chose a CLI implementation to provide clear visibility into the memory ingestion and retrieval logs, which are often hidden in web implementations.
+[DONE] Demo Completed Successfully
+===================================================
+```
-- **MockLLM**: Includes a MockLLM class to ensure the demo is 100% reproducible by reviewers without needing external API keys.
+## Code Highlights
-- **Sealos Native**: Optimized to run within the ephemeral Sealos Devbox container lifecycle.
+- CLI-first: keeps the memory flow visible without a web UI.
+- Offline-safe: reviewers can run the demo even before configuring API keys.
+- Source-checkout friendly: local `src/` is used before installed packages.
diff --git a/docs/sqlite.md b/docs/sqlite.md
index 176b7ad2..d3dbe1c7 100644
--- a/docs/sqlite.md
+++ b/docs/sqlite.md
@@ -11,12 +11,16 @@ MemU supports SQLite as a lightweight, file-based database backend for memory st
### Basic Configuration
+Set `OPENAI_API_KEY` in your environment before running these examples. The
+`api_key` value below is the environment variable name that memU resolves at
+runtime, not a literal secret to paste into source code.
+
```python
-from memu.app import MemoryService
+from memu import MemoryService
# Using default SQLite file (memu.db in current directory)
service = MemoryService(
- llm_profiles={"default": {"api_key": "your-api-key"}},
+ llm_profiles={"default": {"api_key": "OPENAI_API_KEY"}},
database_config={
"metadata_store": {
"provider": "sqlite",
@@ -26,7 +30,7 @@ service = MemoryService(
# Or specify a custom database path
service = MemoryService(
- llm_profiles={"default": {"api_key": "your-api-key"}},
+ llm_profiles={"default": {"api_key": "OPENAI_API_KEY"}},
database_config={
"metadata_store": {
"provider": "sqlite",
@@ -42,7 +46,7 @@ For testing or temporary storage, you can use an in-memory SQLite database:
```python
service = MemoryService(
- llm_profiles={"default": {"api_key": "your-api-key"}},
+ llm_profiles={"default": {"api_key": "OPENAI_API_KEY"}},
database_config={
"metadata_store": {
"provider": "sqlite",
@@ -73,7 +77,7 @@ SQLite doesn't have native vector support like PostgreSQL's pgvector. MemU uses
```python
service = MemoryService(
- llm_profiles={"default": {"api_key": "your-api-key"}},
+ llm_profiles={"default": {"api_key": "OPENAI_API_KEY"}},
database_config={
"metadata_store": {
"provider": "sqlite",
@@ -114,13 +118,23 @@ shutil.copy("memu.db", "memu_backup.db")
### Import from SQLite to PostgreSQL
-To migrate data from SQLite to PostgreSQL:
+To migrate data from SQLite to PostgreSQL, install the optional PostgreSQL
+dependencies first:
+
+```bash
+pip install "memu-py[postgres]"
+# From a source checkout:
+uv sync --extra postgres
+```
+
+Then connect both backends with explicit DSNs. This migration snippet intentionally uses
+low-level repository builders so it can copy existing rows backend-to-backend; normal application code should continue to use `MemoryService(database_config=...)`.
```python
import json
from memu.database.sqlite import build_sqlite_database
from memu.database.postgres import build_postgres_database
-from memu.app.settings import DatabaseConfig
+from memu import DatabaseConfig
from pydantic import BaseModel
class UserScope(BaseModel):
@@ -135,7 +149,10 @@ sqlite_db.load_existing()
# Connect to PostgreSQL
postgres_config = DatabaseConfig(
- metadata_store={"provider": "postgres", "dsn": "postgresql://..."}
+ metadata_store={
+ "provider": "postgres",
+ "dsn": "postgresql+psycopg://user:password@host:5432/memu",
+ }
)
postgres_db = build_postgres_database(config=postgres_config, user_model=UserScope)
@@ -167,12 +184,12 @@ for res_id, resource in sqlite_db.resources.items():
```python
import asyncio
-from memu.app import MemoryService
+from memu import MemoryService
async def main():
# Initialize with SQLite
service = MemoryService(
- llm_profiles={"default": {"api_key": "your-api-key"}},
+ llm_profiles={"default": {"api_key": "OPENAI_API_KEY"}},
database_config={
"metadata_store": {
"provider": "sqlite",
@@ -192,7 +209,7 @@ async def main():
# Retrieve relevant memories
memories = await service.retrieve(
queries=[
- {"role": "user", "content": {"text": "What are my preferences?"}}
+ {"role": "user", "content": "What are my preferences?"}
],
where={"user_id": "alice"},
)
diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md
index f198664a..a2fae28d 100644
--- a/docs/tutorials/getting_started.md
+++ b/docs/tutorials/getting_started.md
@@ -6,7 +6,7 @@ Welcome to MemU! This guide will help you add robust long-term memory capabiliti
Before we begin, ensure you have the following:
-- **Python 3.13+**: MemU takes advantage of modern Python features.
+- **Python 3.12+**: MemU takes advantage of modern Python features.
- **OpenAI API Key**: This quickstart uses OpenAI's models (`gpt-4o-mini`). You will need a valid API key.
## Step-by-Step Guide
@@ -16,11 +16,13 @@ Before we begin, ensure you have the following:
Install MemU using `pip` or `uv`:
```bash
-pip install memu
+pip install memu-py
# OR
-uv add memu
+uv add memu-py
```
+The package installs the `memu` Python import namespace.
+
### 2. Configuration
MemU requires an LLM backend to function. By default, it looks for the `OPENAI_API_KEY` environment variable.
@@ -61,7 +63,7 @@ import logging
import os
import sys
-from memu.app import MemoryService
+from memu import MemoryService
# Configure logging to show info but suppress noisy libraries
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
@@ -110,6 +112,7 @@ async def main() -> None:
# We manually inject a memory into the system.
# This is useful for bootstrapping a user profile or adding explicit knowledge.
print("[*] Injecting memory...")
+ user_scope = {"user_id": "demo_user"}
memory_content = "The user is a senior Python architect who loves clean code and type hints."
# We use 'create_memory_item' to insert a single memory record.
@@ -118,6 +121,7 @@ async def main() -> None:
memory_type="profile",
memory_content=memory_content,
memory_categories=["User Facts"],
+ user=user_scope,
)
print(f"[OK] Memory created! ID: {result.get('memory_item', {}).get('id')}\n")
@@ -127,7 +131,8 @@ async def main() -> None:
print(f"[*] Querying: '{query_text}'")
search_results = await service.retrieve(
- queries=[{"role": "user", "content": query_text}]
+ queries=[{"role": "user", "content": query_text}],
+ where=user_scope,
)
# 5. Display Results
@@ -153,8 +158,8 @@ if __name__ == "__main__":
### Understanding the Code
1. **Initialization**: We configure `MemoryService` with specific `llm_profiles`. This tells MemU which model to use. We also define a `memorize_config` with a "User Facts" category. Categories help the LLM organize and retrieve information more effectively.
-2. **Memory Injection**: `create_memory_item` is used to explicitly add a piece of knowledge. We tag it with `memory_type="profile"` to semantically indicate this is a user attribute.
-3. **Retrieval**: We use `retrieve` with a natural language query. MemU's internal workflow ("RAG" or "LLM" based) will determine the best way to find relevant memories.
+2. **Memory Injection**: `create_memory_item` is used to explicitly add a piece of knowledge. We tag it with `memory_type="profile"` and `user=user_scope` to semantically indicate this is a scoped user attribute.
+3. **Retrieval**: We use `retrieve` with a natural language query and the same `where=user_scope` filter. MemU's internal workflow ("RAG" or "LLM" based) will determine the best way to find relevant memories within that user scope.
## Troubleshooting
diff --git a/examples/context_harness_demo.py b/examples/context_harness_demo.py
new file mode 100644
index 00000000..4c035f34
--- /dev/null
+++ b/examples/context_harness_demo.py
@@ -0,0 +1,79 @@
+"""
+Folder-backed context harness demo.
+
+Run from the repository root:
+
+ python examples/context_harness_demo.py
+
+This demo does not require an API key. It uses deterministic local extraction,
+sidecar evidence for a fake screenshot, and a promoted manual skill note.
+"""
+
+from __future__ import annotations
+
+import shutil
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+
+from memu import ContextHarness, FolderMemoryCompilerConfig, SkillToolTrace
+
+
+def main() -> None:
+ base_dir = ROOT / "examples" / "output" / "context_harness_demo"
+ upload_dir = base_dir / "upload"
+ repo_dir = base_dir / "memory_repo"
+
+ if base_dir.exists():
+ shutil.rmtree(base_dir)
+ upload_dir.mkdir(parents=True)
+
+ (upload_dir / "profile.txt").write_text(
+ "The user prefers calm, concise answers. "
+ "Skill: validate generated context before relying on it.",
+ encoding="utf-8",
+ )
+ (upload_dir / "workflow.png").write_bytes(b"\x89PNG\r\n\x1a\n")
+ (upload_dir / "workflow.caption.md").write_text(
+ "Skill: inspect screenshots and compare them against acceptance criteria.",
+ encoding="utf-8",
+ )
+
+ config = FolderMemoryCompilerConfig(use_memory_service=False)
+ upload_harness = ContextHarness(upload_dir, repo_dir, compiler_config=config)
+ upload_harness.scaffold(copy_source=True)
+
+ harness = ContextHarness.from_repo(repo_dir, compiler_config=config)
+ run = harness.refresh_context_sync(query="context validation workflow")
+
+ harness.record_skill_trace_sync(
+ task="Validate generated context packs",
+ outcome="success",
+ summary="Compiled raw data, built a context pack, and checked skill sections.",
+ actions=["Compile raw data", "Build context pack", "Inspect skill sections"],
+ tools=[SkillToolTrace(name="memu-harness", success=True, score=0.95)],
+ lessons=["Inspect generated and promoted skill sections before using context."],
+ )
+ harness.promote_skill(
+ title="Validate Context Packs",
+ when_to_use="Before injecting generated context into an agent.",
+ actions=["Build the context pack", "Check generated and promoted skill sections"],
+ lessons=["Inspect promoted skills before relying on generated context."],
+ tags=["context", "validation"],
+ )
+
+ context = harness.build_context_markdown(query="context validation workflow", max_chars=4000)
+ print("memU context harness demo complete")
+ print(f" repo: {repo_dir}")
+ print(f" processed: {run.compile_result.processed}")
+ print(f" memory: {repo_dir / 'memory.md'}")
+ print(f" soul: {repo_dir / 'soul.md'}")
+ print(f" skill: {repo_dir / 'skill.md'}")
+ print("\n--- Context Pack Preview ---")
+ print(context)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/example_1_conversation_memory.py b/examples/example_1_conversation_memory.py
index e023fae3..fdfb68ea 100644
--- a/examples/example_1_conversation_memory.py
+++ b/examples/example_1_conversation_memory.py
@@ -12,12 +12,16 @@
import asyncio
import os
import sys
+from pathlib import Path
-from memu.app import MemoryService
+ROOT = Path(__file__).resolve().parents[1]
-# Add src to sys.path
-src_path = os.path.abspath("src")
-sys.path.insert(0, src_path)
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
+
+from memu import MemoryService
async def generate_memory_md(categories, output_dir):
@@ -80,9 +84,9 @@ async def main():
# Conversation files to process
conversation_files = [
- "examples/resources/conversations/conv1.json",
- "examples/resources/conversations/conv2.json",
- "examples/resources/conversations/conv3.json",
+ ROOT / "examples" / "resources" / "conversations" / "conv1.json",
+ ROOT / "examples" / "resources" / "conversations" / "conv2.json",
+ ROOT / "examples" / "resources" / "conversations" / "conv3.json",
]
# Process each conversation
@@ -90,11 +94,11 @@ async def main():
total_items = 0
categories = []
for conv_file in conversation_files:
- if not os.path.exists(conv_file):
+ if not conv_file.exists():
continue
try:
- result = await service.memorize(resource_url=conv_file, modality="conversation")
+ result = await service.memorize(resource_url=str(conv_file), modality="conversation")
total_items += len(result.get("items", []))
# Categories are returned in the result and updated after each memorize call
categories = result.get("categories", [])
@@ -102,15 +106,15 @@ async def main():
print(f"Error: {e}")
# Write to output files
- output_dir = "examples/output/conversation_example"
+ output_dir = ROOT / "examples" / "output" / "conversation_example"
os.makedirs(output_dir, exist_ok=True)
# 1. Generate individual Markdown files for each category
await generate_memory_md(categories, output_dir)
- print(f"\n✓ Processed {len(conversation_files)} files, extracted {total_items} items")
- print(f"✓ Generated {len(categories)} categories")
- print(f"✓ Output: {output_dir}/")
+ print(f"\n[OK] Processed {len(conversation_files)} files, extracted {total_items} items")
+ print(f"[OK] Generated {len(categories)} categories")
+ print(f"[OK] Output: {output_dir}/")
if __name__ == "__main__":
diff --git a/examples/example_2_skill_extraction.py b/examples/example_2_skill_extraction.py
index 3ca75804..3861f174 100644
--- a/examples/example_2_skill_extraction.py
+++ b/examples/example_2_skill_extraction.py
@@ -12,14 +12,16 @@
import asyncio
import os
import sys
+from pathlib import Path
-from openai import AsyncOpenAI
+ROOT = Path(__file__).resolve().parents[1]
-from memu.app import MemoryService
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
-# Add src to sys.path
-src_path = os.path.abspath("src")
-sys.path.insert(0, src_path)
+from memu import MemoryService
async def generate_skill_md(
@@ -107,23 +109,17 @@ async def generate_skill_md(
Generate the complete markdown document now:"""
- client = AsyncOpenAI(api_key=service.llm_config.api_key)
-
- response = await client.chat.completions.create(
- model=service.llm_config.chat_model,
- messages=[
- {
- "role": "system",
- "content": "You are an expert technical writer creating concise, production-grade deployment guides from real experiences.",
- },
- {"role": "user", "content": prompt},
- ],
+ system_prompt = (
+ "You are an expert technical writer creating concise, "
+ "production-grade deployment guides from real experiences."
+ )
+ generated_content = await service.llm_client.chat(
+ prompt,
+ system_prompt=system_prompt,
temperature=0.7,
max_tokens=3000,
)
- generated_content = response.choices[0].message.content
-
# Write to file
with open(output_file, "w", encoding="utf-8") as f:
f.write(generated_content)
@@ -157,7 +153,7 @@ async def main():
For each significant action or phase:
1. **Action/Phase**: What was being attempted?
- 2. **Status**: SUCCESS ✅ or FAILURE ❌
+ 2. **Status**: SUCCESS or FAILURE
3. **What Happened**: What was executed
4. **Outcome**: What worked/failed, metrics
5. **Root Cause** (for failures): Why did it fail?
@@ -177,7 +173,10 @@ async def main():
# Define custom categories
skill_categories = [
- {"name": "deployment_execution", "description": "Deployment actions, traffic shifting, environment management"},
+ {
+ "name": "deployment_execution",
+ "description": "Deployment actions, traffic shifting, environment management",
+ },
{
"name": "pre_deployment_validation",
"description": "Capacity validation, configuration checks, readiness verification",
@@ -216,9 +215,9 @@ async def main():
# Resources to process
resources = [
- ("examples/resources/logs/log1.txt", "document"),
- ("examples/resources/logs/log2.txt", "document"),
- ("examples/resources/logs/log3.txt", "document"),
+ (ROOT / "examples" / "resources" / "logs" / "log1.txt", "document"),
+ (ROOT / "examples" / "resources" / "logs" / "log2.txt", "document"),
+ (ROOT / "examples" / "resources" / "logs" / "log3.txt", "document"),
]
# Process each resource sequentially
@@ -227,16 +226,16 @@ async def main():
categories = []
for idx, (resource_file, modality) in enumerate(resources, 1):
- if not os.path.exists(resource_file):
+ if not resource_file.exists():
continue
try:
- result = await service.memorize(resource_url=resource_file, modality=modality)
+ result = await service.memorize(resource_url=str(resource_file), modality=modality)
# Extract skill items
for item in result.get("items", []):
if item.get("memory_type") == "skill":
- all_skills.append({"skill": item.get("summary", ""), "source": os.path.basename(resource_file)})
+ all_skills.append({"skill": item.get("summary", ""), "source": resource_file.name})
# Categories are returned in the result and updated after each memorize call
categories = result.get("categories", [])
@@ -245,7 +244,7 @@ async def main():
await generate_skill_md(
all_skills=all_skills,
service=service,
- output_file=f"examples/output/skill_example/log_{idx}.md",
+ output_file=ROOT / "examples" / "output" / "skill_example" / f"log_{idx}.md",
attempt_number=idx,
total_attempts=len(resources),
categories=categories,
@@ -258,16 +257,16 @@ async def main():
await generate_skill_md(
all_skills=all_skills,
service=service,
- output_file="examples/output/skill_example/skill.md",
+ output_file=ROOT / "examples" / "output" / "skill_example" / "skill.md",
attempt_number=len(resources),
total_attempts=len(resources),
categories=categories,
is_final=True,
)
- print(f"\n✓ Processed {len(resources)} files, extracted {len(all_skills)} skills")
- print(f"✓ Generated {len(categories)} categories")
- print("✓ Output: examples/output/skill_example/")
+ print(f"\n[OK] Processed {len(resources)} files, extracted {len(all_skills)} skills")
+ print(f"[OK] Generated {len(categories)} categories")
+ print(f"[OK] Output: {ROOT / 'examples' / 'output' / 'skill_example'}/")
if __name__ == "__main__":
diff --git a/examples/example_3_multimodal_memory.py b/examples/example_3_multimodal_memory.py
index 83aba74a..3a5c6a58 100644
--- a/examples/example_3_multimodal_memory.py
+++ b/examples/example_3_multimodal_memory.py
@@ -12,12 +12,16 @@
import asyncio
import os
import sys
+from pathlib import Path
-from memu.app import MemoryService
+ROOT = Path(__file__).resolve().parents[1]
-# Add src to sys.path
-src_path = os.path.abspath("src")
-sys.path.insert(0, src_path)
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
+
+from memu import MemoryService
async def generate_memory_md(categories, output_dir):
@@ -100,9 +104,9 @@ async def main():
# Resources to process (file_path, modality)
resources = [
- ("examples/resources/docs/doc1.txt", "document"),
- ("examples/resources/docs/doc2.txt", "document"),
- ("examples/resources/images/image1.png", "image"),
+ (ROOT / "examples" / "resources" / "docs" / "doc1.txt", "document"),
+ (ROOT / "examples" / "resources" / "docs" / "doc2.txt", "document"),
+ (ROOT / "examples" / "resources" / "images" / "image1.png", "image"),
]
# Process each resource
@@ -110,11 +114,11 @@ async def main():
total_items = 0
categories = []
for resource_file, modality in resources:
- if not os.path.exists(resource_file):
+ if not resource_file.exists():
continue
try:
- result = await service.memorize(resource_url=resource_file, modality=modality)
+ result = await service.memorize(resource_url=str(resource_file), modality=modality)
total_items += len(result.get("items", []))
# Categories are returned in the result and updated after each memorize call
categories = result.get("categories", [])
@@ -122,15 +126,15 @@ async def main():
print(f"Error: {e}")
# Write to output files
- output_dir = "examples/output/multimodal_example"
+ output_dir = ROOT / "examples" / "output" / "multimodal_example"
os.makedirs(output_dir, exist_ok=True)
# 1. Generate individual Markdown files for each category
await generate_memory_md(categories, output_dir)
- print(f"\n✓ Processed {len(resources)} files, extracted {total_items} items")
- print(f"✓ Generated {len(categories)} categories")
- print(f"✓ Output: {output_dir}/")
+ print(f"\n[OK] Processed {len(resources)} files, extracted {total_items} items")
+ print(f"[OK] Generated {len(categories)} categories")
+ print(f"[OK] Output: {output_dir}/")
if __name__ == "__main__":
diff --git a/examples/example_4_openrouter_memory.py b/examples/example_4_openrouter_memory.py
index f5a8daa2..bc8539bf 100644
--- a/examples/example_4_openrouter_memory.py
+++ b/examples/example_4_openrouter_memory.py
@@ -12,11 +12,16 @@
import asyncio
import os
import sys
+from pathlib import Path
-from memu.app import MemoryService
+ROOT = Path(__file__).resolve().parents[1]
-src_path = os.path.abspath("src")
-sys.path.insert(0, src_path)
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
+
+from memu import MemoryService
async def generate_memory_md(categories, output_dir):
@@ -76,9 +81,9 @@ async def main():
)
conversation_files = [
- "examples/resources/conversations/conv1.json",
- "examples/resources/conversations/conv2.json",
- "examples/resources/conversations/conv3.json",
+ ROOT / "examples" / "resources" / "conversations" / "conv1.json",
+ ROOT / "examples" / "resources" / "conversations" / "conv2.json",
+ ROOT / "examples" / "resources" / "conversations" / "conv3.json",
]
print("\nProcessing conversations...")
@@ -86,19 +91,19 @@ async def main():
categories = []
for conv_file in conversation_files:
- if not os.path.exists(conv_file):
+ if not conv_file.exists():
print(f"Skipped: {conv_file} not found")
continue
try:
print(f"Processing: {conv_file}")
- result = await service.memorize(resource_url=conv_file, modality="conversation")
+ result = await service.memorize(resource_url=str(conv_file), modality="conversation")
total_items += len(result.get("items", []))
categories = result.get("categories", [])
except Exception as e:
print(f"Error processing {conv_file}: {e}")
- output_dir = "examples/output/openrouter_example"
+ output_dir = ROOT / "examples" / "output" / "openrouter_example"
os.makedirs(output_dir, exist_ok=True)
await generate_memory_md(categories, output_dir)
diff --git a/examples/example_5_with_lazyllm_client.py b/examples/example_5_with_lazyllm_client.py
index 3b300298..672eadd8 100644
--- a/examples/example_5_with_lazyllm_client.py
+++ b/examples/example_5_with_lazyllm_client.py
@@ -23,13 +23,14 @@
import sys
from pathlib import Path
-# Add src to sys.path FIRST before importing memu
-project_root = Path(__file__).parent.parent
+# Add src to sys.path before importing memu from a source checkout.
+project_root = Path(__file__).resolve().parents[1]
+examples_dir = project_root / "examples"
src_path = str(project_root / "src")
if src_path not in sys.path:
sys.path.insert(0, src_path)
-from memu.app import MemoryService
+from memu import MemoryService
# ==========================================
# PART 1: Conversation Memory Processing
@@ -42,33 +43,33 @@ async def run_conversation_memory_demo(service):
print("=" * 60)
conversation_files = [
- "examples/resources/conversations/conv1.json",
- "examples/resources/conversations/conv2.json",
- "examples/resources/conversations/conv3.json",
+ examples_dir / "resources" / "conversations" / "conv1.json",
+ examples_dir / "resources" / "conversations" / "conv2.json",
+ examples_dir / "resources" / "conversations" / "conv3.json",
]
total_items = 0
categories = []
for conv_file in conversation_files:
- if not os.path.exists(conv_file):
- print(f"⚠ File not found: {conv_file}")
+ if not conv_file.exists():
+ print(f"[WARN] File not found: {conv_file}")
continue
try:
print(f" Processing: {conv_file}")
- result = await service.memorize(resource_url=conv_file, modality="conversation")
+ result = await service.memorize(resource_url=str(conv_file), modality="conversation")
total_items += len(result.get("items", []))
categories = result.get("categories", [])
- print(f" ✓ Extracted {len(result.get('items', []))} items")
+ print(f" [OK] Extracted {len(result.get('items', []))} items")
except Exception as e:
- print(f" ✗ Error processing {conv_file}: {e}")
+ print(f" [ERROR] Error processing {conv_file}: {e}")
# Output generation
- output_dir = "examples/output/lazyllm_example/conversation"
+ output_dir = examples_dir / "output" / "lazyllm_example" / "conversation"
os.makedirs(output_dir, exist_ok=True)
await generate_markdown_output(categories, output_dir)
- print(f"✓ Conversation processing complete. Output: {output_dir}")
+ print(f"[OK] Conversation processing complete. Output: {output_dir}")
# ==========================================
@@ -106,28 +107,32 @@ async def run_skill_extraction_demo(service):
service.memorize_config.memory_types = ["skill"]
service.memorize_config.memory_type_prompts = {"skill": skill_prompt}
- logs = ["examples/resources/logs/log1.txt", "examples/resources/logs/log2.txt", "examples/resources/logs/log3.txt"]
+ logs = [
+ examples_dir / "resources" / "logs" / "log1.txt",
+ examples_dir / "resources" / "logs" / "log2.txt",
+ examples_dir / "resources" / "logs" / "log3.txt",
+ ]
all_skills = []
for log_file in logs:
- if not os.path.exists(log_file):
+ if not log_file.exists():
continue
print(f" Processing log: {log_file}")
try:
- result = await service.memorize(resource_url=log_file, modality="document")
+ result = await service.memorize(resource_url=str(log_file), modality="document")
for item in result.get("items", []):
if item.get("memory_type") == "skill":
all_skills.append(item.get("summary", ""))
- print(f" ✓ Extracted {len(result.get('items', []))} skills")
+ print(f" [OK] Extracted {len(result.get('items', []))} skills")
except Exception as e:
- print(f" ✗ Error: {e}")
+ print(f" [ERROR] Error: {e}")
# Generate summary guide
if all_skills:
- output_file = "examples/output/lazyllm_example/skills/skill_guide.md"
+ output_file = examples_dir / "output" / "lazyllm_example" / "skills" / "skill_guide.md"
await generate_skill_guide(all_skills, service, output_file)
- print(f"✓ Skill guide generated: {output_file}")
+ print(f"[OK] Skill guide generated: {output_file}")
# ==========================================
@@ -159,27 +164,27 @@ async def run_multimodal_demo(service):
service.memorize_config.memory_type_prompts = {"knowledge": xml_prompt}
resources = [
- ("examples/resources/docs/doc1.txt", "document"),
- ("examples/resources/images/image1.png", "image"),
+ (examples_dir / "resources" / "docs" / "doc1.txt", "document"),
+ (examples_dir / "resources" / "images" / "image1.png", "image"),
]
categories = []
for res_file, modality in resources:
- if not os.path.exists(res_file):
+ if not res_file.exists():
continue
print(f" Processing {modality}: {res_file}")
try:
- result = await service.memorize(resource_url=res_file, modality=modality)
+ result = await service.memorize(resource_url=str(res_file), modality=modality)
categories = result.get("categories", [])
- print(f" ✓ Extracted {len(result.get('items', []))} items")
+ print(f" [OK] Extracted {len(result.get('items', []))} items")
except Exception as e:
- print(f" ✗ Error: {e}")
+ print(f" [ERROR] Error: {e}")
- output_dir = "examples/output/lazyllm_example/multimodal"
+ output_dir = examples_dir / "output" / "lazyllm_example" / "multimodal"
os.makedirs(output_dir, exist_ok=True)
await generate_markdown_output(categories, output_dir)
- print(f"✓ Multimodal processing complete. Output: {output_dir}")
+ print(f"[OK] Multimodal processing complete. Output: {output_dir}")
# ==========================================
diff --git a/examples/getting_started_robust.py b/examples/getting_started_robust.py
index 883af997..d10cf9ab 100644
--- a/examples/getting_started_robust.py
+++ b/examples/getting_started_robust.py
@@ -16,11 +16,14 @@
import logging
import os
import sys
+from pathlib import Path
-# Ensure src is in the path for local usage if custom installing
-sys.path.insert(0, os.path.abspath("src"))
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(Path(__file__).resolve().parents[1] / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
-from memu.app import MemoryService
+from memu import MemoryService
# Configure logging to show info but suppress noisy libraries
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
@@ -69,6 +72,7 @@ async def main() -> None:
# We manually inject a memory into the system.
# This is useful for bootstrapping a user profile or adding explicit knowledge.
print("[*] Injecting memory...")
+ user_scope = {"user_id": "demo_user"}
memory_content = "The user is a senior Python architect who loves clean code and type hints."
# We use 'create_memory_item' to insert a single memory record.
@@ -77,6 +81,7 @@ async def main() -> None:
memory_type="profile",
memory_content=memory_content,
memory_categories=["User Facts"],
+ user=user_scope,
)
print(f"[OK] Memory created! ID: {result.get('memory_item', {}).get('id')}\n")
@@ -85,7 +90,10 @@ async def main() -> None:
query_text = "What kind of code does the user like?"
print(f"[*] Querying: '{query_text}'")
- search_results = await service.retrieve(queries=[{"role": "user", "content": query_text}])
+ search_results = await service.retrieve(
+ queries=[{"role": "user", "content": query_text}],
+ where=user_scope,
+ )
# 5. Display Results
items = search_results.get("items", [])
diff --git a/examples/langgraph_demo.py b/examples/langgraph_demo.py
index 4107452f..5a90e7dc 100644
--- a/examples/langgraph_demo.py
+++ b/examples/langgraph_demo.py
@@ -4,18 +4,32 @@
import logging
import os
import sys
+from pathlib import Path
-# Try imports and fail proactively if missing
+INSTALL_HINT = (
+ "Missing LangGraph dependencies. Install them with "
+ "pip install 'memu-py[langgraph]', or run uv sync --extra langgraph "
+ "from a source checkout."
+)
+
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(Path(__file__).resolve().parents[1] / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
+
+# Try optional integration imports first and fail proactively if missing.
try:
import langgraph # noqa: F401
from langchain_core.tools import BaseTool
-
- from memu.app.service import MemoryService
- from memu.integrations.langgraph import MemULangGraphTools
-except ImportError:
- print("Missing dependencies. Please run: uv sync --extra langgraph")
+except ModuleNotFoundError as exc:
+ if exc.name not in {"langgraph", "langchain_core"}:
+ raise
+ print(INSTALL_HINT)
sys.exit(1)
+from memu import MemoryService
+from memu.integrations.langgraph import MemULangGraphTools
+
# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("langgraph_demo")
diff --git a/examples/proactive/memory/__init__.py b/examples/proactive/memory/__init__.py
new file mode 100644
index 00000000..5b2a5286
--- /dev/null
+++ b/examples/proactive/memory/__init__.py
@@ -0,0 +1 @@
+"""Shared helpers for the proactive memory examples."""
diff --git a/examples/proactive/memory/claude_sdk.py b/examples/proactive/memory/claude_sdk.py
new file mode 100644
index 00000000..0b0b40b6
--- /dev/null
+++ b/examples/proactive/memory/claude_sdk.py
@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+INSTALL_HINT = (
+ "The proactive Claude example requires claude-agent-sdk. "
+ "Install the optional extra with `pip install 'memu-py[claude]'`, "
+ "or run `uv sync --extra claude` from a source checkout."
+)
+
+try:
+ from claude_agent_sdk import (
+ AssistantMessage,
+ ClaudeAgentOptions,
+ ClaudeSDKClient,
+ ResultMessage,
+ TextBlock,
+ create_sdk_mcp_server,
+ tool,
+ )
+except ModuleNotFoundError as exc:
+ if exc.name != "claude_agent_sdk":
+ raise
+ raise SystemExit(INSTALL_HINT) from exc
+
+__all__ = [
+ "AssistantMessage",
+ "ClaudeAgentOptions",
+ "ClaudeSDKClient",
+ "ResultMessage",
+ "TextBlock",
+ "create_sdk_mcp_server",
+ "tool",
+]
diff --git a/examples/proactive/memory/config.py b/examples/proactive/memory/config.py
index 622d5e11..0b138fb1 100644
--- a/examples/proactive/memory/config.py
+++ b/examples/proactive/memory/config.py
@@ -27,7 +27,7 @@
"name": "todo",
"description": "This file traces the latest status of the task. All records should be included in this file.",
"target_length": None,
- "custom_prompt": {
+ "summary_prompt": {
"objective": {
"ordinal": 10,
"prompt": "# Task Objective\nYou are a specialist in task management. You should update the markdown file to reflect the latest status of the task.",
diff --git a/examples/proactive/memory/local/common.py b/examples/proactive/memory/local/common.py
index 8f6394cd..249efb03 100644
--- a/examples/proactive/memory/local/common.py
+++ b/examples/proactive/memory/local/common.py
@@ -1,6 +1,6 @@
import os
-from memu.app import MemoryService
+from memu import MemoryService
from ..config import memorize_config, retrieve_config
diff --git a/examples/proactive/memory/local/tools.py b/examples/proactive/memory/local/tools.py
index 7062c2c5..f204bb77 100644
--- a/examples/proactive/memory/local/tools.py
+++ b/examples/proactive/memory/local/tools.py
@@ -1,7 +1,6 @@
from typing import Any
-from claude_agent_sdk import create_sdk_mcp_server, tool
-
+from ..claude_sdk import create_sdk_mcp_server, tool
from .common import get_memory_service
USER_ID = "claude_user"
@@ -14,7 +13,7 @@ async def get_memory(args: dict[str, Any]) -> dict[str, Any]:
memory_service = get_memory_service()
- result = await memory_service.retrieve(query, where={"user_id": USER_ID})
+ result = await memory_service.retrieve(queries=[query], where={"user_id": USER_ID})
return {"content": [{"type": "text", "text": str(result)}]}
diff --git a/examples/proactive/memory/platform/common.py b/examples/proactive/memory/platform/common.py
new file mode 100644
index 00000000..ae868a0e
--- /dev/null
+++ b/examples/proactive/memory/platform/common.py
@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class PlatformMemoryConfig:
+ base_url: str
+ api_key: str
+ user_id: str
+ agent_id: str
+
+
+def get_platform_memory_config() -> PlatformMemoryConfig:
+ api_key = os.getenv("MEMU_API_KEY", "").strip()
+ if not api_key:
+ msg = "Please set MEMU_API_KEY for the platform proactive example"
+ raise ValueError(msg)
+
+ base_url = os.getenv("MEMU_BASE_URL", "https://api.memu.so").strip().rstrip("/")
+ user_id = os.getenv("MEMU_USER_ID", "claude_user").strip() or "claude_user"
+ agent_id = os.getenv("MEMU_AGENT_ID", "claude_agent").strip() or "claude_agent"
+
+ return PlatformMemoryConfig(
+ base_url=base_url or "https://api.memu.so",
+ api_key=api_key,
+ user_id=user_id,
+ agent_id=agent_id,
+ )
diff --git a/examples/proactive/memory/platform/memorize.py b/examples/proactive/memory/platform/memorize.py
index 9fc1722f..3c436190 100644
--- a/examples/proactive/memory/platform/memorize.py
+++ b/examples/proactive/memory/platform/memorize.py
@@ -1,31 +1,26 @@
from typing import Any
-import aiohttp
+import httpx
from ..config import memorize_config
-
-BASE_URL = "https://api.memu.so"
-API_KEY = "your memu api key"
-USER_ID = "claude_user"
-AGENT_ID = "claude_agent"
+from .common import get_platform_memory_config
async def memorize(conversation_messages: list[dict[str, Any]]) -> str | None:
+ config = get_platform_memory_config()
payload = {
"conversation": conversation_messages,
- "user_id": USER_ID,
- "agent_id": AGENT_ID,
+ "user_id": config.user_id,
+ "agent_id": config.agent_id,
"override_config": memorize_config,
}
- async with (
- aiohttp.ClientSession() as session,
- session.post(
- f"{BASE_URL}/api/v3/memory/memorize",
- headers={"Authorization": f"Bearer {API_KEY}"},
+ async with httpx.AsyncClient(timeout=30.0) as client:
+ response = await client.post(
+ f"{config.base_url}/api/v3/memory/memorize",
+ headers={"Authorization": f"Bearer {config.api_key}"},
json=payload,
- ) as response,
- ):
- result = await response.json()
- task_id = result["task_id"]
- return task_id
+ )
+ response.raise_for_status()
+ result = response.json()
+ return result["task_id"]
diff --git a/examples/proactive/memory/platform/tools.py b/examples/proactive/memory/platform/tools.py
index 5ed859de..9b345c0d 100644
--- a/examples/proactive/memory/platform/tools.py
+++ b/examples/proactive/memory/platform/tools.py
@@ -1,37 +1,40 @@
from typing import Any
-import aiohttp
-from claude_agent_sdk import create_sdk_mcp_server, tool
+import httpx
-BASE_URL = "https://api.memu.so"
-API_KEY = "your memu api key"
-USER_ID = "claude_user"
-AGENT_ID = "claude_agent"
+from ..claude_sdk import create_sdk_mcp_server, tool
+from .common import get_platform_memory_config
@tool("memu_memory", "Retrieve memory based on a query", {"query": str})
async def get_memory(args: dict[str, Any]) -> dict[str, Any]:
"""Retrieve memory from the memory API based on the provided query."""
query = args["query"]
- url = f"{BASE_URL}/api/v3/memory/retrieve"
- headers = {"Authorization": f"Bearer {API_KEY}"}
- data = {"user_id": USER_ID, "agent_id": AGENT_ID, "query": query}
+ config = get_platform_memory_config()
+ url = f"{config.base_url}/api/v3/memory/retrieve"
+ headers = {"Authorization": f"Bearer {config.api_key}"}
+ data = {"user_id": config.user_id, "agent_id": config.agent_id, "query": query}
- async with aiohttp.ClientSession() as session, session.post(url, headers=headers, json=data) as response:
- result = await response.json()
+ async with httpx.AsyncClient(timeout=30.0) as client:
+ response = await client.post(url, headers=headers, json=data)
+ response.raise_for_status()
+ result = response.json()
return {"content": [{"type": "text", "text": str(result)}]}
async def _get_todos() -> str:
- url = f"{BASE_URL}/api/v3/memory/categories"
- headers = {"Authorization": f"Bearer {API_KEY}"}
+ config = get_platform_memory_config()
+ url = f"{config.base_url}/api/v3/memory/categories"
+ headers = {"Authorization": f"Bearer {config.api_key}"}
data = {
- "user_id": USER_ID,
- "agent_id": AGENT_ID,
+ "user_id": config.user_id,
+ "agent_id": config.agent_id,
}
- async with aiohttp.ClientSession() as session, session.post(url, headers=headers, json=data) as response:
- result = await response.json()
+ async with httpx.AsyncClient(timeout=30.0) as client:
+ response = await client.post(url, headers=headers, json=data)
+ response.raise_for_status()
+ result = response.json()
categories = result["categories"]
todos = ""
diff --git a/examples/proactive/proactive.py b/examples/proactive/proactive.py
index 7ffe10c1..2b53fa80 100644
--- a/examples/proactive/proactive.py
+++ b/examples/proactive/proactive.py
@@ -1,6 +1,14 @@
import asyncio
+import sys
+from pathlib import Path
+from typing import Any
-from claude_agent_sdk import (
+# Add src to sys.path before memory.local imports memu from a source checkout.
+src_path = str(Path(__file__).resolve().parents[2] / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
+
+from memory.claude_sdk import (
AssistantMessage,
ClaudeAgentOptions,
ClaudeSDKClient,
@@ -17,7 +25,7 @@
RUNNING_MEMORIZATION: asyncio.Task | None = None
-async def trigger_memorize(messages: list[dict[str, any]]) -> bool:
+async def trigger_memorize(messages: list[dict[str, Any]]) -> bool:
"""Create a background task to memorize conversation messages.
Returns True if the task was successfully created and registered.
@@ -94,7 +102,7 @@ async def process_response(client: ClaudeSDKClient) -> list[str]:
return assistant_text_parts
-async def check_and_memorize(conversation_messages: list[dict[str, any]]) -> None:
+async def check_and_memorize(conversation_messages: list[dict[str, Any]]) -> None:
"""Check if memorization threshold is reached and trigger if needed.
Skips triggering if a previous memorization task is still running.
@@ -122,9 +130,9 @@ async def check_and_memorize(conversation_messages: list[dict[str, any]]) -> Non
conversation_messages.clear()
-async def run_conversation_loop(client: ClaudeSDKClient) -> list[dict[str, any]]:
+async def run_conversation_loop(client: ClaudeSDKClient) -> list[dict[str, Any]]:
"""Run the main conversation loop."""
- conversation_messages: list[dict[str, any]] = []
+ conversation_messages: list[dict[str, Any]] = []
iteration = 0
while True:
@@ -152,7 +160,7 @@ async def run_conversation_loop(client: ClaudeSDKClient) -> list[dict[str, any]]
return conversation_messages
-async def main():
+async def main() -> None:
options = ClaudeAgentOptions(
mcp_servers={"memu": memu_server},
allowed_tools=[
diff --git a/examples/resources/docs/doc1.txt b/examples/resources/docs/doc1.txt
index ff280176..1c28136f 100644
--- a/examples/resources/docs/doc1.txt
+++ b/examples/resources/docs/doc1.txt
@@ -255,40 +255,43 @@ API Reference
Basic Usage Example:
```python
-from memu.app import MemoryService
+from memu import MemoryService
# Initialize service
service = MemoryService(
- llm_config={
- "api_key": "your-api-key",
- "chat_model": "gpt-4o-mini"
+ llm_profiles={
+ "default": {
+ "api_key": "OPENAI_API_KEY",
+ "chat_model": "gpt-4o-mini"
+ }
}
)
# Store a memory
result = await service.memorize(
resource_url="conversation.json",
- modality="conversation"
+ modality="conversation",
+ user={"user_id": "alex"}
)
# Retrieve memories
memories = await service.retrieve(
- query="What programming languages does Alex know?",
- top_k=5
+ queries=["What programming languages does Alex know?"],
+ where={"user_id": "alex"}
)
# Access categories
-categories = service.store.categories
+categories = await service.list_memory_categories(where={"user_id": "alex"})
```
Advanced Configuration:
```python
-# Custom memory types
+# Memory extraction configuration
memorize_config = {
- "memory_types": ["profile", "knowledge", "custom"],
+ "memory_types": ["profile", "knowledge", "skill"],
"memory_type_prompts": {
- "custom": "Extract specific information: {resource}"
+ "skill": "Extract actionable skills and workflows: {resource}"
},
"memory_categories": [
{"name": "technical_skills", "description": "Programming and technical abilities"},
@@ -297,7 +300,7 @@ memorize_config = {
}
service = MemoryService(
- llm_config=llm_config,
+ llm_profiles={"default": {"api_key": "OPENAI_API_KEY"}},
memorize_config=memorize_config
)
```
diff --git a/examples/sealos_support_agent.py b/examples/sealos_support_agent.py
index 710ef009..2eeb2f44 100644
--- a/examples/sealos_support_agent.py
+++ b/examples/sealos_support_agent.py
@@ -1,19 +1,34 @@
import sys
import time
+from pathlib import Path
-# Intentamos importar la librería instalada por uv
+
+def configure_output_encoding() -> None:
+ """Keep demo output usable on Windows terminals with legacy encodings."""
+ for stream in (sys.stdout, sys.stderr):
+ if hasattr(stream, "reconfigure"):
+ stream.reconfigure(encoding="utf-8", errors="replace")
+
+
+configure_output_encoding()
+
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(Path(__file__).resolve().parents[1] / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
+
+# Detect whether the package import namespace is available.
try:
- from memu import Memory # noqa: F401
+ import memu # noqa: F401
MEMU_INSTALLED = True
except ImportError as e:
- # Si falla, guardamos el error para debug
MEMU_INSTALLED = False
IMPORT_ERROR = str(e)
-def print_slow(text, delay=0.02):
- """Typing effect for realism"""
+def print_slow(text: str, delay: float = 0.02) -> None:
+ """Typing effect for realism."""
for char in text:
sys.stdout.write(char)
sys.stdout.flush()
@@ -21,48 +36,49 @@ def print_slow(text, delay=0.02):
print()
-def run_rigorous_demo():
- print("\n🚀 Starting Sealos Support Agent Demo (Offline Mode)")
+def run_rigorous_demo() -> None:
+ print("\n[START] Starting Sealos Support Agent Demo (Offline Mode)")
print("===================================================\n")
# 1. ENVIRONMENT CHECK
if MEMU_INSTALLED:
- print("✅ Environment Check: MemU Library detected (Installed via uv).")
- print("✅ Runtime: Sealos Devbox (Python 3.13+)")
+ print("[OK] Environment Check: MemU Library detected.")
+ print("[OK] Runtime: Sealos Devbox (Python 3.12+)")
else:
- # En caso de error, mostramos advertencia pero permitimos la captura
- print("⚠️ Warning: MemU library not detected. Running in Simulation Mode.")
- if "IMPORT_ERROR" in globals():
- print(f" Debug Error: {IMPORT_ERROR}")
+ print("[WARN] memU runtime is not fully importable. Running in Simulation Mode.")
+ print(" Run uv sync or pip install memu-py to use the live package mode.")
time.sleep(0.5)
# 2. MEMORY INGESTION (PHASE 1)
- print("\n📝 --- Phase 1: Ingesting Conversation History ---")
- print('👤 Captain: "I\'m getting a 502 Bad Gateway error on port 3000."')
- print_slow("🤖 Agent: (Processing input through Memory Pipeline...)", delay=0.01)
+ print("\n[PHASE 1] Ingesting Conversation History")
+ print('Captain: "I\'m getting a 502 Bad Gateway error on port 3000."')
+ print_slow("Agent: (Processing input through Memory Pipeline...)", delay=0.01)
time.sleep(1.0)
- print("✅ Memory stored! extracted 2 items:")
+ print("[OK] Memory stored! extracted 2 items:")
print(" - [issue] 502 Bad Gateway error")
print(" - [context] port 3000 configuration")
# 3. CONTEXT RETRIEVAL (PHASE 2)
- print("\n🔍 --- Phase 2: Retrieval on New Interaction (New Session) ---")
- print('👤 Captain: "Hello, any updates?"')
- print_slow("🤖 Agent: (Searching vector store for user 'Captain'...)", delay=0.01)
+ print("\n[PHASE 2] Retrieval on New Interaction (New Session)")
+ print('Captain: "Hello, any updates?"')
+ print_slow("Agent: (Searching vector store for user 'Captain'...)", delay=0.01)
time.sleep(1.0)
- print("\n💡 Retrieved Context:")
+ print("\n[CONTEXT] Retrieved Context:")
print(" Found Memory (Score: 0.98): User reported 502 error on port 3000")
print(" Found Memory (Score: 0.95): User was frustrated with timeout")
# 4. AGENT RESPONSE (PHASE 3)
- print("\n💬 --- Phase 3: Agent Response ---")
- response = '🤖 Agent: "Welcome back, Captain. Regarding the 502 Bad Gateway error on port 3000 you reported earlier - have you tried checking the firewall logs?"'
+ print("\n[PHASE 3] Agent Response")
+ response = (
+ 'Agent: "Welcome back, Captain. Regarding the 502 Bad Gateway error on '
+ 'port 3000 you reported earlier - have you tried checking the firewall logs?"'
+ )
print_slow(response)
- print("\n✨ Demo Completed Successfully")
+ print("\n[DONE] Demo Completed Successfully")
print("===================================================")
diff --git a/examples/test_nebius_provider.py b/examples/test_nebius_provider.py
index 5df5e59a..d2a44dca 100644
--- a/examples/test_nebius_provider.py
+++ b/examples/test_nebius_provider.py
@@ -19,10 +19,12 @@
import asyncio
import os
import sys
+from pathlib import Path
-# Add src to path for local development
-src_path = os.path.abspath("src")
-sys.path.insert(0, src_path)
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(Path(__file__).resolve().parents[1] / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
# Nebius configuration
NEBIUS_BASE_URL = "https://api.tokenfactory.nebius.com/v1/"
@@ -68,10 +70,10 @@ async def test_nebius_chat():
# Truncate long responses for display
display = content[:100] + "..." if len(content) > 100 else content
print(f" Response: {display}")
- print(" ✓ Chat API works!")
+ print(" [OK] Chat API works!")
return True
except Exception as e:
- print(f" ✗ Chat API failed: {e}")
+ print(f" [ERROR] Chat API failed: {e}")
return False
@@ -97,16 +99,16 @@ async def test_nebius_embeddings():
)
print(f" Embedding dimensions: {len(response.data[0].embedding)}")
print(f" Number of embeddings: {len(response.data)}")
- print(" ✓ Embeddings API works!")
+ print(" [OK] Embeddings API works!")
return True
except Exception as e:
- print(f" ✗ Embeddings API failed: {e}")
+ print(f" [ERROR] Embeddings API failed: {e}")
return False
async def test_memu_with_nebius():
"""Test MemU with Nebius as the LLM provider."""
- from memu.app import MemoryService
+ from memu import MemoryService
api_key = os.environ.get("NEBIUS_API_KEY")
if not api_key:
@@ -136,7 +138,7 @@ async def test_memu_with_nebius():
try:
# Create MemU service with Nebius
service = MemoryService(llm_profiles=llm_profiles)
- print(" ✓ MemoryService initialized with Nebius!")
+ print(" [OK] MemoryService initialized with Nebius!")
# Test memorize with a file (create temp file)
print("\n Testing memorize...")
@@ -149,11 +151,11 @@ async def test_memu_with_nebius():
try:
result = await service.memorize(
resource_url=temp_file,
- modality="text",
+ modality="document",
)
items_count = len(result.get("items", []))
categories_count = len(result.get("categories", []))
- print(f" ✓ Memorized! Items: {items_count}, Categories: {categories_count}")
+ print(f" [OK] Memorized! Items: {items_count}, Categories: {categories_count}")
# Show what was extracted
for item in result.get("items", [])[:3]:
@@ -167,7 +169,7 @@ async def test_memu_with_nebius():
retrieve_result = await service.retrieve(
queries=[{"role": "user", "content": "What programming language does the user like?"}]
)
- print(f" ✓ Retrieved! Needs retrieval: {retrieve_result.get('needs_retrieval')}")
+ print(f" [OK] Retrieved! Needs retrieval: {retrieve_result.get('needs_retrieval')}")
items = retrieve_result.get("items", [])
if items:
@@ -180,12 +182,12 @@ async def test_memu_with_nebius():
print(f" - {summary}...")
print("\n" + "=" * 60)
- print("✓ SUCCESS: MemU works with Nebius!")
+ print("[OK] SUCCESS: MemU works with Nebius!")
print("=" * 60)
return True
except Exception as e:
- print(f" ✗ MemU with Nebius failed: {e}")
+ print(f" [ERROR] MemU with Nebius failed: {e}")
import traceback
traceback.print_exc()
@@ -220,7 +222,7 @@ async def main():
await test_memu_with_nebius()
else:
print("\n" + "=" * 60)
- print("✗ FAILED: Basic API tests failed, skipping MemU test")
+ print("[ERROR] FAILED: Basic API tests failed, skipping MemU test")
print("=" * 60)
diff --git a/mkdocs.yml b/mkdocs.yml
new file mode 100644
index 00000000..e29b1926
--- /dev/null
+++ b/mkdocs.yml
@@ -0,0 +1,50 @@
+site_name: memU
+site_description: AI memory and conversation management framework
+site_url: https://nevamind-ai.github.io/MemU/
+repo_url: https://github.com/NevaMind-AI/MemU
+repo_name: NevaMind-AI/MemU
+
+theme:
+ name: material
+ features:
+ - navigation.sections
+ - navigation.top
+ - search.highlight
+ - content.code.copy
+
+plugins:
+ - search
+ - mkdocstrings:
+ handlers:
+ python:
+ paths:
+ - src
+
+markdown_extensions:
+ - admonition
+ - attr_list
+ - md_in_html
+ - pymdownx.details
+ - pymdownx.superfences
+
+nav:
+ - Home: index.md
+ - Getting Started: tutorials/getting_started.md
+ - Architecture: architecture.md
+ - Folder Memory Compiler: folder_memory_compiler.md
+ - Storage:
+ - SQLite: sqlite.md
+ - Providers:
+ - Grok: providers/grok.md
+ - Integrations:
+ - LangGraph: langgraph_integration.md
+ - Grok: integrations/grok.md
+ - Sealos Devbox: sealos-devbox-guide.md
+ - Sealos Use Case: sealos_use_case.md
+ - ADRs:
+ - Overview: adr/README.md
+ - Workflow Pipelines: adr/0001-workflow-pipeline-architecture.md
+ - Pluggable Storage: adr/0002-pluggable-storage-and-vector-strategy.md
+ - User Scope: adr/0003-user-scope-in-data-model.md
+ - Markdown Context Harness: adr/0004-markdown-context-harness-and-skill-evolution.md
+ - Self-Evolve Review Gate: adr/0005-self-evolve-instruction-review-gate.md
diff --git a/pyproject.toml b/pyproject.toml
index 82574482..332e5e9f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -6,15 +6,18 @@ authors = [
]
description = "AI Memory and Conversation Management Framework - Simple as mem0, Powerful as MemU"
readme = "README.md"
-# license = {file = "LICENSE"}
-requires-python = ">=3.13"
+license = {file = "LICENSE.txt"}
+requires-python = ">=3.12"
classifiers = [
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
+ "License :: OSI Approved :: Apache Software License",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
+ "Typing :: Typed",
]
keywords = ["ai", "memory", "conversation", "llm", "chatbot", "agent"]
dependencies = [
@@ -26,8 +29,6 @@ dependencies = [
"sqlmodel>=0.0.27",
"alembic>=1.14.0",
"pendulum>=3.1.0",
- "langchain-core>=1.2.7",
- "lazyllm>=0.7.3",
]
[build-system]
@@ -38,6 +39,7 @@ build-backend = "maturin"
module-name = "memu._core"
python-packages = ["memu"]
python-source = "src"
+include = ["memu/py.typed"]
[dependency-groups]
dev = [
@@ -68,16 +70,22 @@ test = [
[project.optional-dependencies]
postgres = ["pgvector>=0.3.4", "sqlalchemy[postgresql-psycopgbinary]>=2.0.36"]
-langgraph = ["langgraph>=0.0.10", "langchain-core>=0.1.0"]
+langgraph = ["langgraph>=1.0.6", "langchain-core>=1.2.7"]
+lazyllm = ["lazyllm>=0.7.3"]
claude = ["claude-agent-sdk>=0.1.24"]
[project.urls]
"Homepage" = "https://github.com/NevaMind-AI/MemU"
"Bug Tracker" = "https://github.com/NevaMind-AI/MemU/issues"
"Documentation" = "https://github.com/NevaMind-AI/MemU#readme"
+"Changelog" = "https://github.com/NevaMind-AI/MemU/blob/main/CHANGELOG.md"
+"Source" = "https://github.com/NevaMind-AI/MemU"
[project.scripts]
-memu-server = "memu.server.cli:main"
+memu-context = "memu.app.context_cli:main"
+memu-folder = "memu.app.folder_cli:main"
+memu-harness = "memu.app.context_harness_cli:main"
+memu-skill-trace = "memu.app.skill_trace_cli:main"
[tool.deptry.per_rule_ignores]
# Optional dependencies used in examples/
@@ -85,7 +93,7 @@ DEP002 = ["claude-agent-sdk"]
[tool.mypy]
files = ["src", "tests"]
-python_version = "3.13"
+python_version = "3.12"
disallow_untyped_defs = true
disallow_any_unimported = true
no_implicit_optional = true
@@ -115,7 +123,7 @@ disallow_untyped_defs = false
warn_return_any = false
[tool.ruff]
-target-version = "py313"
+target-version = "py312"
line-length = 120
fix = true
diff --git a/readme/README_en.md b/readme/README_en.md
index 1cb36490..cbfecf3f 100644
--- a/readme/README_en.md
+++ b/readme/README_en.md
@@ -8,7 +8,7 @@
[](https://badge.fury.io/py/memu-py)
[](https://opensource.org/licenses/Apache-2.0)
-[](https://www.python.org/downloads/)
+[](https://www.python.org/downloads/)
[](https://discord.gg/memu)
[](https://x.com/memU_ai)
@@ -28,7 +28,7 @@ memU **continuously captures and understands user intent**. Even without a comma
## 🤖 [OpenClaw (Moltbot, Clawdbot) Alternative](https://memu.bot)
-
+
- **Download-and-use and simple** to get started.
- Builds long-term memory to **understand user intent** and act proactively.
@@ -77,7 +77,7 @@ Just as a file system turns raw bytes into organized data, memU transforms raw i
## ⭐️ Star the repository
-
+
If you find memU useful or interesting, a GitHub Star ⭐️ would be greatly appreciated.
---
@@ -95,10 +95,14 @@ If you find memU useful or interesting, a GitHub Star ⭐️ would be greatly ap
## 🔄 How Proactive Memory Works
```bash
-
+pip install "memu-py[claude]"
+# From a source checkout, use: uv sync --extra claude
+export OPENAI_API_KEY="..."
+export ANTHROPIC_API_KEY="..."
+# Optional when using memory.platform instead of memory.local:
+export MEMU_API_KEY="..."
cd examples/proactive
python proactive.py
-
```
---
@@ -275,18 +279,17 @@ For enterprise deployment with custom proactive workflows, contact **info@nevami
#### Installation
```bash
-pip install -e .
+pip install memu-py
```
#### Basic Example
-> **Requirements**: Python 3.13+ and an OpenAI API key
+> **Requirements**: Python 3.12+ and an OpenAI API key
**Test Continuous Learning** (in-memory):
```bash
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_inmemory.py
+python examples/getting_started_robust.py
```
**Test with Persistent Storage** (PostgreSQL):
@@ -301,9 +304,10 @@ docker run -d \
pgvector/pgvector:pg16
# Run continuous learning test
+uv sync --extra postgres
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_postgres.py
+export MEMU_RUN_POSTGRES_TESTS=1
+uv run python -m pytest tests/test_postgres.py
```
Both examples demonstrate **proactive memory workflows**:
@@ -311,9 +315,9 @@ Both examples demonstrate **proactive memory workflows**:
2. **Auto-Extraction**: Immediate memory creation
3. **Proactive Retrieval**: Context-aware memory surfacing
-See [`tests/test_inmemory.py`](../tests/test_inmemory.py) and [`tests/test_postgres.py`](../tests/test_postgres.py) for implementation details.
-
----
+See [`tests/test_inmemory.py`](../tests/test_inmemory.py), [`tests/test_sqlite.py`](../tests/test_sqlite.py),
+and [`tests/test_postgres.py`](../tests/test_postgres.py) for implementation details. The in-memory and SQLite
+live LLM checks are opt-in with `MEMU_RUN_LIVE_LLM_TESTS=1`.
### Custom LLM and Embedding Providers
@@ -326,14 +330,14 @@ service = MemUService(
# Default profile for LLM operations
"default": {
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
- "api_key": "your_api_key",
+ "api_key": "MEMU_QWEN_API_KEY",
"chat_model": "qwen3-max",
- "client_backend": "sdk" # "sdk" or "http"
+ "client_backend": "sdk" # "sdk", "httpx", or "lazyllm_backend"
},
# Separate profile for embeddings
"embedding": {
"base_url": "https://api.voyageai.com/v1",
- "api_key": "your_voyage_api_key",
+ "api_key": "VOYAGE_API_KEY",
"embed_model": "voyage-3.5-lite"
}
},
@@ -341,6 +345,19 @@ service = MemUService(
)
```
+The `lazyllm_backend` adapter is optional. Install it with
+`pip install "memu-py[lazyllm]"` or, from a source checkout,
+`uv sync --extra lazyllm`.
+
+Optional LazyLLM live check:
+
+```bash
+uv sync --extra lazyllm
+export MEMU_QWEN_API_KEY=your_api_key
+export MEMU_RUN_LAZYLLM_TESTS=1
+uv run python -m pytest tests/test_lazyllm.py
+```
+
---
### OpenRouter Integration
@@ -357,7 +374,7 @@ service = MemoryService(
"provider": "openrouter",
"client_backend": "httpx",
"base_url": "https://openrouter.ai",
- "api_key": "your_openrouter_api_key",
+ "api_key": "OPENROUTER_API_KEY",
"chat_model": "anthropic/claude-3.5-sonnet", # Any OpenRouter model
"embed_model": "openai/text-embedding-3-small", # Embedding model
},
@@ -385,15 +402,10 @@ service = MemoryService(
#### Running OpenRouter Tests
```bash
export OPENROUTER_API_KEY=your_api_key
+export MEMU_RUN_OPENROUTER_TESTS=1
# Full workflow test (memorize + retrieve)
-python tests/test_openrouter.py
-
-# Embedding-specific tests
-python tests/test_openrouter_embedding.py
-
-# Vision-specific tests
-python tests/test_openrouter_vision.py
+uv run python -m pytest tests/test_openrouter.py
```
See [`examples/example_4_openrouter_memory.py`](../examples/example_4_openrouter_memory.py) for a complete working example.
@@ -464,8 +476,8 @@ Deep **anticipatory reasoning** for complex contexts:
# Proactive retrieval with context history
result = await service.retrieve(
queries=[
- {"role": "user", "content": {"text": "What are their preferences?"}},
- {"role": "user", "content": {"text": "Tell me about work habits"}}
+ {"role": "user", "content": "What are their preferences?"},
+ {"role": "user", "content": "Tell me about work habits"}
],
where={"user_id": "123"}, # Optional: scope filter
method="rag" # or "llm" for deeper reasoning
@@ -480,12 +492,14 @@ result = await service.retrieve(
}
```
+For a single user query, Python callers can also pass `queries=["What are their preferences?"]`; MemU normalizes it to a user message before retrieval.
+
**Proactive Filtering**: Use `where` to scope continuous monitoring:
- `where={"user_id": "123"}` - User-specific context
- `where={"agent_id__in": ["1", "2"]}` - Multi-agent coordination
- Omit `where` for global context awareness
-> 📚 **For complete API documentation**, see [SERVICE_API.md](../docs/SERVICE_API.md) - includes proactive workflow patterns, pipeline configuration, and real-time update handling.
+> 📚 **For complete API documentation**, see [memu.pro/docs](https://memu.pro/docs) - includes proactive workflow patterns, pipeline configuration, and real-time update handling.
---
@@ -559,7 +573,7 @@ View detailed experimental data: [memU-experiment](https://github.com/NevaMind-A
| Repository | Description | Proactive Features |
|------------|-------------|-------------------|
-| **[memU](https://github.com/NevaMind-AI/memU)** | Core proactive memory engine | 7×24 learning pipeline, auto-categorization |
+| **[memU](https://github.com/NevaMind-AI/MemU)** | Core proactive memory engine | 7×24 learning pipeline, auto-categorization |
| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | Backend with continuous sync | Real-time memory updates, webhook triggers |
| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | Visual memory dashboard | Live memory evolution monitoring |
@@ -596,7 +610,7 @@ We welcome contributions from the community! Whether you're fixing bugs, adding
To start contributing to MemU, you'll need to set up your development environment:
#### Prerequisites
-- Python 3.13+
+- Python 3.12+
- [uv](https://github.com/astral-sh/uv) (Python package manager)
- Git
@@ -649,7 +663,7 @@ For detailed contribution guidelines, code standards, and development practices,
## 🌍 Community
-- **GitHub Issues**: [Report bugs & request features](https://github.com/NevaMind-AI/memU/issues)
+- **GitHub Issues**: [Report bugs & request features](https://github.com/NevaMind-AI/MemU/issues)
- **Discord**: [Join the community](https://discord.com/invite/hQZntfGsbJ)
- **X (Twitter)**: [Follow @memU_ai](https://x.com/memU_ai)
- **Contact**: info@nevamind.ai
diff --git a/readme/README_es.md b/readme/README_es.md
index a0edff7d..2d300873 100644
--- a/readme/README_es.md
+++ b/readme/README_es.md
@@ -8,7 +8,7 @@
[](https://badge.fury.io/py/memu-py)
[](https://opensource.org/licenses/Apache-2.0)
-[](https://www.python.org/downloads/)
+[](https://www.python.org/downloads/)
[](https://discord.gg/memu)
[](https://x.com/memU_ai)
@@ -28,7 +28,7 @@ memU **captura y comprende continuamente la intención del usuario**. Incluso si
## 🤖 [OpenClaw (Moltbot, Clawdbot) Alternative](https://memu.bot)
-
+
- **Download-and-use and simple** to get started.
- Builds long-term memory to **understand user intent** and act proactively.
@@ -77,7 +77,7 @@ Así como un sistema de archivos convierte bytes crudos en datos organizados, me
## ⭐️ Dale una estrella al repositorio
-
+
Si encuentras memU útil o interesante, te agradeceríamos mucho una estrella en GitHub ⭐️.
---
@@ -95,10 +95,14 @@ Si encuentras memU útil o interesante, te agradeceríamos mucho una estrella en
## 🔄 Cómo Funciona la Memoria Proactiva
```bash
-
+pip install "memu-py[claude]"
+# From a source checkout, use: uv sync --extra claude
+export OPENAI_API_KEY="..."
+export ANTHROPIC_API_KEY="..."
+# Optional when using memory.platform instead of memory.local:
+export MEMU_API_KEY="..."
cd examples/proactive
python proactive.py
-
```
---
@@ -275,18 +279,17 @@ Para despliegue empresarial con flujos de trabajo proactivos personalizados, con
#### Instalación
```bash
-pip install -e .
+pip install memu-py
```
#### Ejemplo Básico
-> **Requisitos**: Python 3.13+ y una clave API de OpenAI
+> **Requisitos**: Python 3.12+ y una clave API de OpenAI
**Probar Aprendizaje Continuo** (en memoria):
```bash
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_inmemory.py
+python examples/getting_started_robust.py
```
**Probar con Almacenamiento Persistente** (PostgreSQL):
@@ -301,9 +304,10 @@ docker run -d \
pgvector/pgvector:pg16
# Ejecutar prueba de aprendizaje continuo
+uv sync --extra postgres
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_postgres.py
+export MEMU_RUN_POSTGRES_TESTS=1
+uv run python -m pytest tests/test_postgres.py
```
Ambos ejemplos demuestran **flujos de trabajo de memoria proactiva**:
@@ -311,7 +315,9 @@ Ambos ejemplos demuestran **flujos de trabajo de memoria proactiva**:
2. **Auto-Extracción**: Creación inmediata de memoria
3. **Recuperación Proactiva**: Presentación de memoria consciente del contexto
-Ver [`tests/test_inmemory.py`](../tests/test_inmemory.py) y [`tests/test_postgres.py`](../tests/test_postgres.py) para detalles de implementación.
+Ver [`tests/test_inmemory.py`](../tests/test_inmemory.py), [`tests/test_sqlite.py`](../tests/test_sqlite.py)
+y [`tests/test_postgres.py`](../tests/test_postgres.py) para detalles de implementación. Las pruebas live LLM
+de in-memory y SQLite son opt-in con `MEMU_RUN_LIVE_LLM_TESTS=1`.
---
@@ -326,14 +332,14 @@ service = MemUService(
# Perfil predeterminado para operaciones LLM
"default": {
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
- "api_key": "your_api_key",
+ "api_key": "MEMU_QWEN_API_KEY",
"chat_model": "qwen3-max",
- "client_backend": "sdk" # "sdk" o "http"
+ "client_backend": "sdk" # "sdk", "httpx" o "lazyllm_backend"
},
# Perfil separado para embeddings
"embedding": {
"base_url": "https://api.voyageai.com/v1",
- "api_key": "your_voyage_api_key",
+ "api_key": "VOYAGE_API_KEY",
"embed_model": "voyage-3.5-lite"
}
},
@@ -341,6 +347,19 @@ service = MemUService(
)
```
+The `lazyllm_backend` adapter is optional. Install it with
+`pip install "memu-py[lazyllm]"` or, from a source checkout,
+`uv sync --extra lazyllm`.
+
+Optional LazyLLM live check:
+
+```bash
+uv sync --extra lazyllm
+export MEMU_QWEN_API_KEY=your_api_key
+export MEMU_RUN_LAZYLLM_TESTS=1
+uv run python -m pytest tests/test_lazyllm.py
+```
+
---
### Integración con OpenRouter
@@ -357,7 +376,7 @@ service = MemoryService(
"provider": "openrouter",
"client_backend": "httpx",
"base_url": "https://openrouter.ai",
- "api_key": "your_openrouter_api_key",
+ "api_key": "OPENROUTER_API_KEY",
"chat_model": "anthropic/claude-3.5-sonnet", # Cualquier modelo de OpenRouter
"embed_model": "openai/text-embedding-3-small", # Modelo de embedding
},
@@ -385,15 +404,10 @@ service = MemoryService(
#### Ejecutar Pruebas de OpenRouter
```bash
export OPENROUTER_API_KEY=your_api_key
+export MEMU_RUN_OPENROUTER_TESTS=1
# Prueba de flujo completo (memorize + retrieve)
-python tests/test_openrouter.py
-
-# Pruebas específicas de embedding
-python tests/test_openrouter_embedding.py
-
-# Pruebas específicas de visión
-python tests/test_openrouter_vision.py
+uv run python -m pytest tests/test_openrouter.py
```
Ver [`examples/example_4_openrouter_memory.py`](../examples/example_4_openrouter_memory.py) para un ejemplo completo funcional.
@@ -464,8 +478,8 @@ MemU soporta tanto **carga proactiva de contexto** como **consultas reactivas**:
# Recuperación proactiva con historial de contexto
result = await service.retrieve(
queries=[
- {"role": "user", "content": {"text": "¿Cuáles son sus preferencias?"}},
- {"role": "user", "content": {"text": "Cuéntame sobre los hábitos de trabajo"}}
+ {"role": "user", "content": "¿Cuáles son sus preferencias?"},
+ {"role": "user", "content": "Cuéntame sobre los hábitos de trabajo"}
],
where={"user_id": "123"}, # Opcional: filtro de alcance
method="rag" # o "llm" para razonamiento más profundo
@@ -485,7 +499,7 @@ result = await service.retrieve(
- `where={"agent_id__in": ["1", "2"]}` - Coordinación multi-agente
- Omitir `where` para conciencia de contexto global
-> 📚 **Para documentación completa de API**, ver [SERVICE_API.md](../docs/SERVICE_API.md) - incluye patrones de flujo de trabajo proactivo, configuración de pipeline y manejo de actualizaciones en tiempo real.
+> 📚 **Para documentación completa de API**, ver [memu.pro/docs](https://memu.pro/docs) - incluye patrones de flujo de trabajo proactivo, configuración de pipeline y manejo de actualizaciones en tiempo real.
---
@@ -559,7 +573,7 @@ Ver datos experimentales detallados: [memU-experiment](https://github.com/NevaMi
| Repositorio | Descripción | Características Proactivas |
|-------------|-------------|---------------------------|
-| **[memU](https://github.com/NevaMind-AI/memU)** | Motor principal de memoria proactiva | Pipeline de aprendizaje 7×24, auto-categorización |
+| **[memU](https://github.com/NevaMind-AI/MemU)** | Motor principal de memoria proactiva | Pipeline de aprendizaje 7×24, auto-categorización |
| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | Backend con sincronización continua | Actualizaciones de memoria en tiempo real, triggers de webhook |
| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | Dashboard visual de memoria | Monitoreo de evolución de memoria en vivo |
@@ -596,7 +610,7 @@ Ver datos experimentales detallados: [memU-experiment](https://github.com/NevaMi
Para empezar a contribuir a MemU, necesitarás configurar tu entorno de desarrollo:
#### Prerrequisitos
-- Python 3.13+
+- Python 3.12+
- [uv](https://github.com/astral-sh/uv) (gestor de paquetes Python)
- Git
@@ -649,7 +663,7 @@ Para guías detalladas de contribución, estándares de código y prácticas de
## 🌍 Comunidad
-- **GitHub Issues**: [Reportar bugs y solicitar características](https://github.com/NevaMind-AI/memU/issues)
+- **GitHub Issues**: [Reportar bugs y solicitar características](https://github.com/NevaMind-AI/MemU/issues)
- **Discord**: [Unirse a la comunidad](https://discord.com/invite/hQZntfGsbJ)
- **X (Twitter)**: [Seguir @memU_ai](https://x.com/memU_ai)
- **Contacto**: info@nevamind.ai
diff --git a/readme/README_fr.md b/readme/README_fr.md
index fd1e1308..cf5a0b84 100644
--- a/readme/README_fr.md
+++ b/readme/README_fr.md
@@ -8,7 +8,7 @@
[](https://badge.fury.io/py/memu-py)
[](https://opensource.org/licenses/Apache-2.0)
-[](https://www.python.org/downloads/)
+[](https://www.python.org/downloads/)
[](https://discord.gg/memu)
[](https://x.com/memU_ai)
@@ -28,7 +28,7 @@ memU **capture et comprend continuellement l'intention de l'utilisateur**. Même
## 🤖 [OpenClaw (Moltbot, Clawdbot) Alternative](https://memu.bot)
-
+
- **Download-and-use and simple** to get started.
- Builds long-term memory to **understand user intent** and act proactively.
@@ -77,7 +77,7 @@ Tout comme un système de fichiers transforme des octets bruts en données organ
## ⭐️ Mettez une étoile au dépôt
-
+
Si vous trouvez memU utile ou intéressant, une étoile GitHub ⭐️ serait grandement appréciée.
---
@@ -95,10 +95,14 @@ Si vous trouvez memU utile ou intéressant, une étoile GitHub ⭐️ serait gra
## 🔄 Comment Fonctionne la Mémoire Proactive
```bash
-
+pip install "memu-py[claude]"
+# From a source checkout, use: uv sync --extra claude
+export OPENAI_API_KEY="..."
+export ANTHROPIC_API_KEY="..."
+# Optional when using memory.platform instead of memory.local:
+export MEMU_API_KEY="..."
cd examples/proactive
python proactive.py
-
```
---
@@ -275,18 +279,17 @@ Pour un déploiement entreprise avec des workflows proactifs personnalisés, con
#### Installation
```bash
-pip install -e .
+pip install memu-py
```
#### Exemple de Base
-> **Prérequis**: Python 3.13+ et une clé API OpenAI
+> **Prérequis**: Python 3.12+ et une clé API OpenAI
**Tester l'Apprentissage Continu** (en mémoire):
```bash
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_inmemory.py
+python examples/getting_started_robust.py
```
**Tester avec Stockage Persistant** (PostgreSQL):
@@ -301,9 +304,10 @@ docker run -d \
pgvector/pgvector:pg16
# Exécuter le test d'apprentissage continu
+uv sync --extra postgres
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_postgres.py
+export MEMU_RUN_POSTGRES_TESTS=1
+uv run python -m pytest tests/test_postgres.py
```
Les deux exemples démontrent **les workflows de mémoire proactive**:
@@ -311,7 +315,9 @@ Les deux exemples démontrent **les workflows de mémoire proactive**:
2. **Auto-Extraction**: Création immédiate de mémoire
3. **Récupération Proactive**: Affichage de mémoire contextuel
-Voir [`tests/test_inmemory.py`](../tests/test_inmemory.py) et [`tests/test_postgres.py`](../tests/test_postgres.py) pour les détails d'implémentation.
+Voir [`tests/test_inmemory.py`](../tests/test_inmemory.py), [`tests/test_sqlite.py`](../tests/test_sqlite.py)
+et [`tests/test_postgres.py`](../tests/test_postgres.py) pour les détails d'implémentation. Les vérifications live LLM
+in-memory et SQLite sont opt-in avec `MEMU_RUN_LIVE_LLM_TESTS=1`.
---
@@ -326,14 +332,14 @@ service = MemUService(
# Profil par défaut pour les opérations LLM
"default": {
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
- "api_key": "your_api_key",
+ "api_key": "MEMU_QWEN_API_KEY",
"chat_model": "qwen3-max",
- "client_backend": "sdk" # "sdk" ou "http"
+ "client_backend": "sdk" # "sdk", "httpx" ou "lazyllm_backend"
},
# Profil séparé pour les embeddings
"embedding": {
"base_url": "https://api.voyageai.com/v1",
- "api_key": "your_voyage_api_key",
+ "api_key": "VOYAGE_API_KEY",
"embed_model": "voyage-3.5-lite"
}
},
@@ -341,6 +347,19 @@ service = MemUService(
)
```
+The `lazyllm_backend` adapter is optional. Install it with
+`pip install "memu-py[lazyllm]"` or, from a source checkout,
+`uv sync --extra lazyllm`.
+
+Optional LazyLLM live check:
+
+```bash
+uv sync --extra lazyllm
+export MEMU_QWEN_API_KEY=your_api_key
+export MEMU_RUN_LAZYLLM_TESTS=1
+uv run python -m pytest tests/test_lazyllm.py
+```
+
---
### Intégration OpenRouter
@@ -357,7 +376,7 @@ service = MemoryService(
"provider": "openrouter",
"client_backend": "httpx",
"base_url": "https://openrouter.ai",
- "api_key": "your_openrouter_api_key",
+ "api_key": "OPENROUTER_API_KEY",
"chat_model": "anthropic/claude-3.5-sonnet", # N'importe quel modèle OpenRouter
"embed_model": "openai/text-embedding-3-small", # Modèle d'embedding
},
@@ -385,15 +404,10 @@ service = MemoryService(
#### Exécuter les Tests OpenRouter
```bash
export OPENROUTER_API_KEY=your_api_key
+export MEMU_RUN_OPENROUTER_TESTS=1
# Test de workflow complet (memorize + retrieve)
-python tests/test_openrouter.py
-
-# Tests spécifiques aux embeddings
-python tests/test_openrouter_embedding.py
-
-# Tests spécifiques à la vision
-python tests/test_openrouter_vision.py
+uv run python -m pytest tests/test_openrouter.py
```
Voir [`examples/example_4_openrouter_memory.py`](../examples/example_4_openrouter_memory.py) pour un exemple complet fonctionnel.
@@ -464,8 +478,8 @@ MemU supporte à la fois **le chargement proactif de contexte** et **les requêt
# Récupération proactive avec historique de contexte
result = await service.retrieve(
queries=[
- {"role": "user", "content": {"text": "Quelles sont leurs préférences?"}},
- {"role": "user", "content": {"text": "Parle-moi des habitudes de travail"}}
+ {"role": "user", "content": "Quelles sont leurs préférences?"},
+ {"role": "user", "content": "Parle-moi des habitudes de travail"}
],
where={"user_id": "123"}, # Optionnel: filtre de portée
method="rag" # ou "llm" pour raisonnement plus profond
@@ -485,7 +499,7 @@ result = await service.retrieve(
- `where={"agent_id__in": ["1", "2"]}` - Coordination multi-agent
- Omettre `where` pour conscience de contexte globale
-> 📚 **Pour la documentation API complète**, voir [SERVICE_API.md](../docs/SERVICE_API.md) - inclut les patterns de workflow proactif, configuration de pipeline et gestion des mises à jour en temps réel.
+> 📚 **Pour la documentation API complète**, voir [memu.pro/docs](https://memu.pro/docs) - inclut les patterns de workflow proactif, configuration de pipeline et gestion des mises à jour en temps réel.
---
@@ -559,7 +573,7 @@ Voir les données expérimentales détaillées: [memU-experiment](https://github
| Dépôt | Description | Fonctionnalités Proactives |
|-------|-------------|---------------------------|
-| **[memU](https://github.com/NevaMind-AI/memU)** | Moteur principal de mémoire proactive | Pipeline d'apprentissage 7×24, auto-catégorisation |
+| **[memU](https://github.com/NevaMind-AI/MemU)** | Moteur principal de mémoire proactive | Pipeline d'apprentissage 7×24, auto-catégorisation |
| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | Backend avec synchronisation continue | Mises à jour de mémoire en temps réel, déclencheurs webhook |
| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | Dashboard visuel de mémoire | Surveillance de l'évolution de la mémoire en direct |
@@ -596,7 +610,7 @@ Nous accueillons les contributions de la communauté! Que vous corrigiez des bug
Pour commencer à contribuer à MemU, vous devrez configurer votre environnement de développement:
#### Prérequis
-- Python 3.13+
+- Python 3.12+
- [uv](https://github.com/astral-sh/uv) (gestionnaire de paquets Python)
- Git
@@ -649,7 +663,7 @@ Pour des directives de contribution détaillées, standards de code et pratiques
## 🌍 Communauté
-- **GitHub Issues**: [Signaler des bugs & demander des fonctionnalités](https://github.com/NevaMind-AI/memU/issues)
+- **GitHub Issues**: [Signaler des bugs & demander des fonctionnalités](https://github.com/NevaMind-AI/MemU/issues)
- **Discord**: [Rejoindre la communauté](https://discord.com/invite/hQZntfGsbJ)
- **X (Twitter)**: [Suivre @memU_ai](https://x.com/memU_ai)
- **Contact**: info@nevamind.ai
diff --git a/readme/README_ja.md b/readme/README_ja.md
index 24aa97b5..fab93e5d 100644
--- a/readme/README_ja.md
+++ b/readme/README_ja.md
@@ -8,7 +8,7 @@
[](https://badge.fury.io/py/memu-py)
[](https://opensource.org/licenses/Apache-2.0)
-[](https://www.python.org/downloads/)
+[](https://www.python.org/downloads/)
[](https://discord.gg/memu)
[](https://x.com/memU_ai)
@@ -28,7 +28,7 @@ memUは**ユーザーの意図を継続的にキャプチャして理解**しま
## 🤖 [OpenClaw (Moltbot, Clawdbot) Alternative](https://memu.bot)
-
+
- **Download-and-use and simple** to get started.
- Builds long-term memory to **understand user intent** and act proactively.
@@ -77,7 +77,7 @@ memory/
## ⭐️ リポジトリにスターを
-
+
memUが役立つまたは興味深いと思われた場合は、GitHub Star ⭐️をいただけると大変嬉しいです。
---
@@ -95,10 +95,14 @@ memUが役立つまたは興味深いと思われた場合は、GitHub Star ⭐
## 🔄 プロアクティブメモリの仕組み
```bash
-
+pip install "memu-py[claude]"
+# From a source checkout, use: uv sync --extra claude
+export OPENAI_API_KEY="..."
+export ANTHROPIC_API_KEY="..."
+# Optional when using memory.platform instead of memory.local:
+export MEMU_API_KEY="..."
cd examples/proactive
python proactive.py
-
```
---
@@ -275,18 +279,17 @@ MemUの3層システムは、**リアクティブクエリ**と**プロアクテ
#### インストール
```bash
-pip install -e .
+pip install memu-py
```
#### 基本例
-> **要件**：Python 3.13+ と OpenAI APIキー
+> **要件**：Python 3.12+ と OpenAI APIキー
**継続学習をテスト**（インメモリ）：
```bash
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_inmemory.py
+python examples/getting_started_robust.py
```
**永続ストレージでテスト**（PostgreSQL）：
@@ -301,9 +304,10 @@ docker run -d \
pgvector/pgvector:pg16
# 継続学習テストを実行
+uv sync --extra postgres
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_postgres.py
+export MEMU_RUN_POSTGRES_TESTS=1
+uv run python -m pytest tests/test_postgres.py
```
両方の例は**プロアクティブメモリワークフロー**を示しています：
@@ -311,7 +315,9 @@ python test_postgres.py
2. **自動抽出**：即座のメモリ作成
3. **プロアクティブ検索**：コンテキストに応じたメモリ表示
-実装の詳細については [`tests/test_inmemory.py`](../tests/test_inmemory.py) と [`tests/test_postgres.py`](../tests/test_postgres.py) を参照してください。
+実装の詳細については [`tests/test_inmemory.py`](../tests/test_inmemory.py)、[`tests/test_sqlite.py`](../tests/test_sqlite.py)、
+および [`tests/test_postgres.py`](../tests/test_postgres.py) を参照してください。in-memory と SQLite の live LLM チェックは
+`MEMU_RUN_LIVE_LLM_TESTS=1` による opt-in です。
---
@@ -326,14 +332,14 @@ service = MemUService(
# LLM操作のデフォルトプロファイル
"default": {
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
- "api_key": "your_api_key",
+ "api_key": "MEMU_QWEN_API_KEY",
"chat_model": "qwen3-max",
- "client_backend": "sdk" # "sdk" または "http"
+ "client_backend": "sdk" # "sdk"、"httpx"、または "lazyllm_backend"
},
# 埋め込み用の別プロファイル
"embedding": {
"base_url": "https://api.voyageai.com/v1",
- "api_key": "your_voyage_api_key",
+ "api_key": "VOYAGE_API_KEY",
"embed_model": "voyage-3.5-lite"
}
},
@@ -341,6 +347,19 @@ service = MemUService(
)
```
+The `lazyllm_backend` adapter is optional. Install it with
+`pip install "memu-py[lazyllm]"` or, from a source checkout,
+`uv sync --extra lazyllm`.
+
+Optional LazyLLM live check:
+
+```bash
+uv sync --extra lazyllm
+export MEMU_QWEN_API_KEY=your_api_key
+export MEMU_RUN_LAZYLLM_TESTS=1
+uv run python -m pytest tests/test_lazyllm.py
+```
+
---
### OpenRouter統合
@@ -357,7 +376,7 @@ service = MemoryService(
"provider": "openrouter",
"client_backend": "httpx",
"base_url": "https://openrouter.ai",
- "api_key": "your_openrouter_api_key",
+ "api_key": "OPENROUTER_API_KEY",
"chat_model": "anthropic/claude-3.5-sonnet", # 任意のOpenRouterモデル
"embed_model": "openai/text-embedding-3-small", # 埋め込みモデル
},
@@ -385,15 +404,10 @@ service = MemoryService(
#### OpenRouterテストの実行
```bash
export OPENROUTER_API_KEY=your_api_key
+export MEMU_RUN_OPENROUTER_TESTS=1
# フルワークフローテスト（メモリ化 + 検索）
-python tests/test_openrouter.py
-
-# 埋め込み固有のテスト
-python tests/test_openrouter_embedding.py
-
-# ビジョン固有のテスト
-python tests/test_openrouter_vision.py
+uv run python -m pytest tests/test_openrouter.py
```
完全な動作例については [`examples/example_4_openrouter_memory.py`](../examples/example_4_openrouter_memory.py) を参照してください。
@@ -464,8 +478,8 @@ MemUは**プロアクティブコンテキストロード**と**リアクティ
# コンテキスト履歴を含むプロアクティブ検索
result = await service.retrieve(
queries=[
- {"role": "user", "content": {"text": "彼らの好みは何ですか？"}},
- {"role": "user", "content": {"text": "仕事の習慣について教えて"}}
+ {"role": "user", "content": "彼らの好みは何ですか？"},
+ {"role": "user", "content": "仕事の習慣について教えて"}
],
where={"user_id": "123"}, # オプション：スコープフィルター
method="rag" # または "llm" でより深い推論
@@ -485,7 +499,7 @@ result = await service.retrieve(
- `where={"agent_id__in": ["1", "2"]}` - マルチエージェント調整
- `where`を省略してグローバルコンテキスト認識
-> 📚 **完全なAPIドキュメント**については、[SERVICE_API.md](../docs/SERVICE_API.md) を参照 - プロアクティブワークフローパターン、パイプライン設定、リアルタイム更新処理を含む。
+> 📚 **完全なAPIドキュメント**については、[memu.pro/docs](https://memu.pro/docs) を参照 - プロアクティブワークフローパターン、パイプライン設定、リアルタイム更新処理を含む。
---
@@ -559,7 +573,7 @@ MemUは、すべての推論タスクでLocomoベンチマークで**92.09%の
| リポジトリ | 説明 | プロアクティブ機能 |
|-----------|------|------------------|
-| **[memU](https://github.com/NevaMind-AI/memU)** | コアプロアクティブメモリエンジン | 7×24学習パイプライン、自動分類 |
+| **[memU](https://github.com/NevaMind-AI/MemU)** | コアプロアクティブメモリエンジン | 7×24学習パイプライン、自動分類 |
| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | 継続同期を備えたバックエンド | リアルタイムメモリ更新、webhookトリガー |
| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | ビジュアルメモリダッシュボード | ライブメモリ進化モニタリング |
@@ -596,7 +610,7 @@ MemUは、すべての推論タスクでLocomoベンチマークで**92.09%の
MemUへのコントリビュートを開始するには、開発環境をセットアップする必要があります：
#### 前提条件
-- Python 3.13+
+- Python 3.12+
- [uv](https://github.com/astral-sh/uv)（Pythonパッケージマネージャー）
- Git
@@ -649,7 +663,7 @@ make check
## 🌍 コミュニティ
-- **GitHub Issues**：[バグを報告 & 機能をリクエスト](https://github.com/NevaMind-AI/memU/issues)
+- **GitHub Issues**：[バグを報告 & 機能をリクエスト](https://github.com/NevaMind-AI/MemU/issues)
- **Discord**：[コミュニティに参加](https://discord.com/invite/hQZntfGsbJ)
- **X (Twitter)**：[@memU_ai をフォロー](https://x.com/memU_ai)
- **お問い合わせ**：info@nevamind.ai
diff --git a/readme/README_ko.md b/readme/README_ko.md
index d114897f..24ca00fa 100644
--- a/readme/README_ko.md
+++ b/readme/README_ko.md
@@ -8,7 +8,7 @@
[](https://badge.fury.io/py/memu-py)
[](https://opensource.org/licenses/Apache-2.0)
-[](https://www.python.org/downloads/)
+[](https://www.python.org/downloads/)
[](https://discord.gg/memu)
[](https://x.com/memU_ai)
@@ -28,7 +28,7 @@ memU는 **사용자 의도를 지속적으로 캡처하고 이해**합니다.
## 🤖 [OpenClaw (Moltbot, Clawdbot) Alternative](https://memu.bot)
-
+
- **Download-and-use and simple** to get started.
- Builds long-term memory to **understand user intent** and act proactively.
@@ -77,7 +77,7 @@ memory/
## ⭐️ 리포지토리에 스타를
-
+
MemU가 유용하거나 흥미롭다면, GitHub Star ⭐️를 눌러주시면 큰 힘이 됩니다.
---
@@ -95,10 +95,14 @@ MemU가 유용하거나 흥미롭다면, GitHub Star ⭐️를 눌러주시면
## 🔄 프로액티브 메모리 작동 방식
```bash
-
+pip install "memu-py[claude]"
+# From a source checkout, use: uv sync --extra claude
+export OPENAI_API_KEY="..."
+export ANTHROPIC_API_KEY="..."
+# Optional when using memory.platform instead of memory.local:
+export MEMU_API_KEY="..."
cd examples/proactive
python proactive.py
-
```
---
@@ -275,18 +279,17 @@ MemU의 3계층 시스템은 **반응적 쿼리**와 **프로액티브 컨텍스
#### 설치
```bash
-pip install -e .
+pip install memu-py
```
#### 기본 예제
-> **요구사항**: Python 3.13+ 및 OpenAI API 키
+> **요구사항**: Python 3.12+ 및 OpenAI API 키
**지속 학습 테스트** (인메모리):
```bash
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_inmemory.py
+python examples/getting_started_robust.py
```
**영구 저장소로 테스트** (PostgreSQL):
@@ -301,9 +304,10 @@ docker run -d \
pgvector/pgvector:pg16
# 지속 학습 테스트 실행
+uv sync --extra postgres
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_postgres.py
+export MEMU_RUN_POSTGRES_TESTS=1
+uv run python -m pytest tests/test_postgres.py
```
두 예제 모두 **프로액티브 메모리 워크플로우**를 보여줍니다:
@@ -311,7 +315,9 @@ python test_postgres.py
2. **자동 추출**: 즉각적인 메모리 생성
3. **프로액티브 검색**: 컨텍스트 인식 메모리 표시
-구현 세부사항은 [`tests/test_inmemory.py`](../tests/test_inmemory.py)와 [`tests/test_postgres.py`](../tests/test_postgres.py)를 참조하세요.
+구현 세부사항은 [`tests/test_inmemory.py`](../tests/test_inmemory.py), [`tests/test_sqlite.py`](../tests/test_sqlite.py),
+그리고 [`tests/test_postgres.py`](../tests/test_postgres.py)를 참조하세요. in-memory 및 SQLite live LLM 검사는
+`MEMU_RUN_LIVE_LLM_TESTS=1`로 opt-in해야 합니다.
---
@@ -326,14 +332,14 @@ service = MemUService(
# LLM 작업용 기본 프로필
"default": {
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
- "api_key": "your_api_key",
+ "api_key": "MEMU_QWEN_API_KEY",
"chat_model": "qwen3-max",
- "client_backend": "sdk" # "sdk" 또는 "http"
+ "client_backend": "sdk" # "sdk", "httpx" 또는 "lazyllm_backend"
},
# 임베딩용 별도 프로필
"embedding": {
"base_url": "https://api.voyageai.com/v1",
- "api_key": "your_voyage_api_key",
+ "api_key": "VOYAGE_API_KEY",
"embed_model": "voyage-3.5-lite"
}
},
@@ -341,6 +347,19 @@ service = MemUService(
)
```
+The `lazyllm_backend` adapter is optional. Install it with
+`pip install "memu-py[lazyllm]"` or, from a source checkout,
+`uv sync --extra lazyllm`.
+
+Optional LazyLLM live check:
+
+```bash
+uv sync --extra lazyllm
+export MEMU_QWEN_API_KEY=your_api_key
+export MEMU_RUN_LAZYLLM_TESTS=1
+uv run python -m pytest tests/test_lazyllm.py
+```
+
---
### OpenRouter 통합
@@ -357,7 +376,7 @@ service = MemoryService(
"provider": "openrouter",
"client_backend": "httpx",
"base_url": "https://openrouter.ai",
- "api_key": "your_openrouter_api_key",
+ "api_key": "OPENROUTER_API_KEY",
"chat_model": "anthropic/claude-3.5-sonnet", # 모든 OpenRouter 모델
"embed_model": "openai/text-embedding-3-small", # 임베딩 모델
},
@@ -385,15 +404,10 @@ service = MemoryService(
#### OpenRouter 테스트 실행
```bash
export OPENROUTER_API_KEY=your_api_key
+export MEMU_RUN_OPENROUTER_TESTS=1
# 전체 워크플로우 테스트 (메모라이즈 + 검색)
-python tests/test_openrouter.py
-
-# 임베딩 특화 테스트
-python tests/test_openrouter_embedding.py
-
-# 비전 특화 테스트
-python tests/test_openrouter_vision.py
+uv run python -m pytest tests/test_openrouter.py
```
완전한 작동 예제는 [`examples/example_4_openrouter_memory.py`](../examples/example_4_openrouter_memory.py)를 참조하세요.
@@ -464,8 +478,8 @@ MemU는 **프로액티브 컨텍스트 로딩**과 **반응적 쿼리**를 모
# 컨텍스트 히스토리를 포함한 프로액티브 검색
result = await service.retrieve(
queries=[
- {"role": "user", "content": {"text": "그들의 선호도가 무엇입니까?"}},
- {"role": "user", "content": {"text": "업무 습관에 대해 알려주세요"}}
+ {"role": "user", "content": "그들의 선호도가 무엇입니까?"},
+ {"role": "user", "content": "업무 습관에 대해 알려주세요"}
],
where={"user_id": "123"}, # 선택: 범위 필터
method="rag" # 또는 "llm"으로 더 깊은 추론
@@ -485,7 +499,7 @@ result = await service.retrieve(
- `where={"agent_id__in": ["1", "2"]}` - 다중 에이전트 조정
- `where` 생략으로 전역 컨텍스트 인식
-> 📚 **전체 API 문서**는 [SERVICE_API.md](../docs/SERVICE_API.md) 참조 - 프로액티브 워크플로우 패턴, 파이프라인 구성, 실시간 업데이트 처리 포함.
+> 📚 **전체 API 문서**는 [memu.pro/docs](https://memu.pro/docs) 참조 - 프로액티브 워크플로우 패턴, 파이프라인 구성, 실시간 업데이트 처리 포함.
---
@@ -559,7 +573,7 @@ MemU는 모든 추론 작업에서 Locomo 벤치마크에서 **92.09% 평균 정
| 리포지토리 | 설명 | 프로액티브 기능 |
|-----------|------|----------------|
-| **[memU](https://github.com/NevaMind-AI/memU)** | 핵심 프로액티브 메모리 엔진 | 7×24 학습 파이프라인, 자동 분류 |
+| **[memU](https://github.com/NevaMind-AI/MemU)** | 핵심 프로액티브 메모리 엔진 | 7×24 학습 파이프라인, 자동 분류 |
| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | 지속 동기화가 포함된 백엔드 | 실시간 메모리 업데이트, 웹훅 트리거 |
| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | 시각적 메모리 대시보드 | 라이브 메모리 진화 모니터링 |
@@ -596,7 +610,7 @@ MemU는 모든 추론 작업에서 Locomo 벤치마크에서 **92.09% 평균 정
MemU에 기여하려면 개발 환경을 설정해야 합니다:
#### 사전 요구사항
-- Python 3.13+
+- Python 3.12+
- [uv](https://github.com/astral-sh/uv) (Python 패키지 관리자)
- Git
@@ -649,7 +663,7 @@ make check
## 🌍 커뮤니티
-- **GitHub Issues**: [버그 보고 및 기능 요청](https://github.com/NevaMind-AI/memU/issues)
+- **GitHub Issues**: [버그 보고 및 기능 요청](https://github.com/NevaMind-AI/MemU/issues)
- **Discord**: [커뮤니티 참여](https://discord.com/invite/hQZntfGsbJ)
- **X (Twitter)**: [@memU_ai 팔로우](https://x.com/memU_ai)
- **연락처**: info@nevamind.ai
diff --git a/readme/README_zh.md b/readme/README_zh.md
index d6b3db1b..d86b9207 100644
--- a/readme/README_zh.md
+++ b/readme/README_zh.md
@@ -8,7 +8,7 @@
[](https://badge.fury.io/py/memu-py)
[](https://opensource.org/licenses/Apache-2.0)
-[](https://www.python.org/downloads/)
+[](https://www.python.org/downloads/)
[](https://discord.gg/memu)
[](https://x.com/memU_ai)
@@ -28,7 +28,7 @@ memU **持续捕获并理解用户意图**。即使没有明确指令，智能
## 🤖 [OpenClaw (Moltbot, Clawdbot) Alternative](https://memu.bot)
-
+
- **Download-and-use and simple** to get started.
- Builds long-term memory to **understand user intent** and act proactively.
@@ -77,7 +77,7 @@ memory/
## ⭐️ 给项目点个星
-
+
如果你觉得 memU 有用或有趣，请给项目点个星 ⭐️，这将是对我们最大的支持！
---
@@ -95,10 +95,14 @@ memory/
## 🔄 主动记忆工作原理
```bash
-
+pip install "memu-py[claude]"
+# From a source checkout, use: uv sync --extra claude
+export OPENAI_API_KEY="..."
+export ANTHROPIC_API_KEY="..."
+# Optional when using memory.platform instead of memory.local:
+export MEMU_API_KEY="..."
cd examples/proactive
python proactive.py
-
```
---
@@ -273,18 +277,17 @@ MemU 的三层系统同时支持**响应式查询**和**主动上下文加载**
#### 安装
```bash
-pip install -e .
+pip install memu-py
```
#### 基础示例
-> **要求**：Python 3.13+ 和 OpenAI API 密钥
+> **要求**：Python 3.12+ 和 OpenAI API 密钥
**测试持续学习**（内存模式）：
```bash
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_inmemory.py
+python examples/getting_started_robust.py
```
**测试持久化存储**（PostgreSQL）：
@@ -299,9 +302,10 @@ docker run -d \
pgvector/pgvector:pg16
# 运行持续学习测试
+uv sync --extra postgres
export OPENAI_API_KEY=your_api_key
-cd tests
-python test_postgres.py
+export MEMU_RUN_POSTGRES_TESTS=1
+uv run python -m pytest tests/test_postgres.py
```
两个示例都演示了**主动记忆工作流**：
@@ -309,7 +313,9 @@ python test_postgres.py
2. **自动提取**：即时创建记忆
3. **主动检索**：上下文感知的记忆呈现
-查看 [`tests/test_inmemory.py`](../tests/test_inmemory.py) 和 [`tests/test_postgres.py`](../tests/test_postgres.py) 了解实现细节。
+查看 [`tests/test_inmemory.py`](../tests/test_inmemory.py)、[`tests/test_sqlite.py`](../tests/test_sqlite.py)
+和 [`tests/test_postgres.py`](../tests/test_postgres.py) 了解实现细节。in-memory 和 SQLite live LLM 检查需要通过
+`MEMU_RUN_LIVE_LLM_TESTS=1` 显式 opt-in。
---
@@ -324,14 +330,14 @@ service = MemUService(
# LLM 操作的默认配置
"default": {
"base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
- "api_key": "your_api_key",
+ "api_key": "MEMU_QWEN_API_KEY",
"chat_model": "qwen3-max",
- "client_backend": "sdk" # "sdk" 或 "http"
+ "client_backend": "sdk" # "sdk"、"httpx" 或 "lazyllm_backend"
},
# 嵌入的单独配置
"embedding": {
"base_url": "https://api.voyageai.com/v1",
- "api_key": "your_voyage_api_key",
+ "api_key": "VOYAGE_API_KEY",
"embed_model": "voyage-3.5-lite"
}
},
@@ -339,6 +345,19 @@ service = MemUService(
)
```
+The `lazyllm_backend` adapter is optional. Install it with
+`pip install "memu-py[lazyllm]"` or, from a source checkout,
+`uv sync --extra lazyllm`.
+
+Optional LazyLLM live check:
+
+```bash
+uv sync --extra lazyllm
+export MEMU_QWEN_API_KEY=your_api_key
+export MEMU_RUN_LAZYLLM_TESTS=1
+uv run python -m pytest tests/test_lazyllm.py
+```
+
---
### OpenRouter 集成
@@ -355,7 +374,7 @@ service = MemoryService(
"provider": "openrouter",
"client_backend": "httpx",
"base_url": "https://openrouter.ai",
- "api_key": "your_openrouter_api_key",
+ "api_key": "OPENROUTER_API_KEY",
"chat_model": "anthropic/claude-3.5-sonnet", # 任何 OpenRouter 模型
"embed_model": "openai/text-embedding-3-small", # 嵌入模型
},
@@ -383,15 +402,10 @@ service = MemoryService(
#### 运行 OpenRouter 测试
```bash
export OPENROUTER_API_KEY=your_api_key
+export MEMU_RUN_OPENROUTER_TESTS=1
# 完整工作流测试（记忆 + 检索）
-python tests/test_openrouter.py
-
-# 嵌入专项测试
-python tests/test_openrouter_embedding.py
-
-# 视觉专项测试
-python tests/test_openrouter_vision.py
+uv run python -m pytest tests/test_openrouter.py
```
查看 [`examples/example_4_openrouter_memory.py`](../examples/example_4_openrouter_memory.py) 获取完整示例。
@@ -462,8 +476,8 @@ MemU 同时支持**主动上下文加载**和**响应式查询**：
# 带上下文历史的主动检索
result = await service.retrieve(
queries=[
- {"role": "user", "content": {"text": "他们的偏好是什么？"}},
- {"role": "user", "content": {"text": "告诉我工作习惯"}}
+ {"role": "user", "content": "他们的偏好是什么？"},
+ {"role": "user", "content": "告诉我工作习惯"}
],
where={"user_id": "123"}, # 可选：范围过滤
method="rag" # 或 "llm" 用于更深入的推理
@@ -483,7 +497,7 @@ result = await service.retrieve(
- `where={"agent_id__in": ["1", "2"]}` - 多智能体协调
- 省略 `where` 以获取全局上下文感知
-> 📚 **完整 API 文档**，请参阅 [SERVICE_API.md](../docs/SERVICE_API.md) - 包含主动工作流模式、管道配置和实时更新处理。
+> 📚 **完整 API 文档**，请参阅 [memu.pro/docs](https://memu.pro/docs) - 包含主动工作流模式、管道配置和实时更新处理。
---
@@ -557,7 +571,7 @@ MemU 在 Locomo 基准测试中，在所有推理任务上实现了 **92.09% 的
| 仓库 | 描述 | 主动功能 |
|------|------|----------|
-| **[memU](https://github.com/NevaMind-AI/memU)** | 核心主动记忆引擎 | 7×24 学习管道、自动分类 |
+| **[memU](https://github.com/NevaMind-AI/MemU)** | 核心主动记忆引擎 | 7×24 学习管道、自动分类 |
| **[memU-server](https://github.com/NevaMind-AI/memU-server)** | 带持续同步的后端 | 实时记忆更新、webhook 触发 |
| **[memU-ui](https://github.com/NevaMind-AI/memU-ui)** | 可视化记忆仪表板 | 实时记忆演化监控 |
@@ -594,7 +608,7 @@ MemU 在 Locomo 基准测试中，在所有推理任务上实现了 **92.09% 的
要开始为 MemU 做贡献，您需要设置开发环境：
#### 先决条件
-- Python 3.13+
+- Python 3.12+
- [uv](https://github.com/astral-sh/uv)（Python 包管理器）
- Git
@@ -647,7 +661,7 @@ make check
## 🌍 社区
-- **GitHub Issues**：[报告错误和请求功能](https://github.com/NevaMind-AI/memU/issues)
+- **GitHub Issues**：[报告错误和请求功能](https://github.com/NevaMind-AI/MemU/issues)
- **Discord**：[加入社区](https://discord.com/invite/hQZntfGsbJ)
- **X (Twitter)**：[关注 @memU_ai](https://x.com/memU_ai)
- **联系方式**：info@nevamind.ai
diff --git a/src/memu/__init__.py b/src/memu/__init__.py
index 38772b8b..b58010ab 100644
--- a/src/memu/__init__.py
+++ b/src/memu/__init__.py
@@ -1,9 +1,215 @@
-from memu._core import hello_from_bin
-from memu.app.service import MemoryService
+from __future__ import annotations
-# Public alias used in documentation examples
-MemUService = MemoryService
+from typing import TYPE_CHECKING, Any
+
+from memu._version import __version__
+
+try:
+ from memu._core import hello_from_bin
+except ModuleNotFoundError as exc:
+ if exc.name != "memu._core":
+ raise
+
+ # Source-tree imports during local tests may run before the Rust extension is built.
+ def hello_from_bin() -> str:
+ return "Hello from memu!"
+
+
+from memu.app.context_harness import (
+ ContextHarness,
+ ContextHarnessRun,
+ ContextHarnessSkillEvolutionResult,
+ ContextHarnessSkillTraceResult,
+)
+from memu.app.folder import (
+ EvolutionReviewApplyResult,
+ FolderCompileResult,
+ FolderHealthIssue,
+ FolderHealthResult,
+ FolderHealthSeverity,
+ FolderMemoryCompiler,
+ FolderMemoryCompilerConfig,
+ FolderScaffoldResult,
+ FolderSourceState,
+ FolderSourceStatus,
+ FolderStatusResult,
+ FolderWatchEvent,
+ MarkdownMemoryEntry,
+ compile_folder_to_markdown,
+ compile_folder_to_markdown_sync,
+ inspect_folder_memory_health,
+ inspect_folder_memory_status,
+ review_folder_evolution,
+ scaffold_folder_memory_repository,
+ watch_folder_to_markdown,
+ watch_folder_to_markdown_sync,
+)
+from memu.app.harness_config import (
+ DEFAULT_CONTEXT_MAX_CHARS,
+ DEFAULT_MAX_TEXT_CHARS,
+ HARNESS_CONFIG_NAME,
+ HARNESS_CONFIG_VERSION,
+ default_harness_config,
+ harness_config_path,
+ load_harness_config,
+ validate_harness_config,
+)
+from memu.app.markdown_context import (
+ MarkdownContextPack,
+ MarkdownContextSection,
+ MarkdownMemoryRepository,
+ build_markdown_context_pack,
+ inject_context_messages,
+)
+from memu.app.self_evolve import (
+ EvidenceRecord,
+ EvolutionInstruction,
+ EvolutionReviewBundle,
+ EvolutionReviewConfig,
+ PatchProposal,
+ ReviewDecision,
+ ReviewStatus,
+)
+from memu.app.skill_trace import (
+ SkillEvolutionProposal,
+ SkillPromotionRecord,
+ SkillToolTrace,
+ SkillTrace,
+ SkillTraceRecord,
+ promote_skill,
+ record_skill_trace,
+ suggest_skill_promotions,
+)
+
+if TYPE_CHECKING:
+ from memu.app.service import MemoryService
+ from memu.app.settings import (
+ BlobConfig,
+ DatabaseConfig,
+ DefaultUserModel,
+ LLMConfig,
+ LLMProfilesConfig,
+ MemorizeConfig,
+ RetrieveConfig,
+ UserConfig,
+ )
+ from memu.workflow.runner import (
+ LocalWorkflowRunner,
+ WorkflowRunner,
+ register_workflow_runner,
+ resolve_workflow_runner,
+ )
+
+ MemUService = MemoryService
+
+
+_LAZY_EXPORTS = {
+ "BlobConfig": ("memu.app.settings", "BlobConfig"),
+ "DatabaseConfig": ("memu.app.settings", "DatabaseConfig"),
+ "DefaultUserModel": ("memu.app.settings", "DefaultUserModel"),
+ "LLMConfig": ("memu.app.settings", "LLMConfig"),
+ "LLMProfilesConfig": ("memu.app.settings", "LLMProfilesConfig"),
+ "LocalWorkflowRunner": ("memu.workflow.runner", "LocalWorkflowRunner"),
+ "MemUService": ("memu.app.service", "MemoryService"),
+ "MemorizeConfig": ("memu.app.settings", "MemorizeConfig"),
+ "MemoryService": ("memu.app.service", "MemoryService"),
+ "RetrieveConfig": ("memu.app.settings", "RetrieveConfig"),
+ "UserConfig": ("memu.app.settings", "UserConfig"),
+ "WorkflowRunner": ("memu.workflow.runner", "WorkflowRunner"),
+ "register_workflow_runner": ("memu.workflow.runner", "register_workflow_runner"),
+ "resolve_workflow_runner": ("memu.workflow.runner", "resolve_workflow_runner"),
+}
def _rust_entry() -> str:
return hello_from_bin()
+
+
+__all__ = [
+ "BlobConfig",
+ "DEFAULT_CONTEXT_MAX_CHARS",
+ "DEFAULT_MAX_TEXT_CHARS",
+ "DatabaseConfig",
+ "DefaultUserModel",
+ "FolderCompileResult",
+ "FolderHealthIssue",
+ "FolderHealthResult",
+ "FolderHealthSeverity",
+ "FolderMemoryCompiler",
+ "FolderMemoryCompilerConfig",
+ "FolderScaffoldResult",
+ "FolderSourceState",
+ "FolderSourceStatus",
+ "FolderStatusResult",
+ "FolderWatchEvent",
+ "HARNESS_CONFIG_NAME",
+ "HARNESS_CONFIG_VERSION",
+ "ContextHarness",
+ "ContextHarnessRun",
+ "ContextHarnessSkillEvolutionResult",
+ "ContextHarnessSkillTraceResult",
+ "LLMConfig",
+ "LLMProfilesConfig",
+ "LocalWorkflowRunner",
+ "MarkdownContextPack",
+ "MarkdownContextSection",
+ "MarkdownMemoryEntry",
+ "MarkdownMemoryRepository",
+ "EvidenceRecord",
+ "EvolutionInstruction",
+ "EvolutionReviewBundle",
+ "EvolutionReviewConfig",
+ "EvolutionReviewApplyResult",
+ "MemUService",
+ "MemorizeConfig",
+ "MemoryService",
+ "PatchProposal",
+ "ReviewDecision",
+ "ReviewStatus",
+ "RetrieveConfig",
+ "SkillEvolutionProposal",
+ "SkillPromotionRecord",
+ "SkillToolTrace",
+ "SkillTrace",
+ "SkillTraceRecord",
+ "UserConfig",
+ "WorkflowRunner",
+ "build_markdown_context_pack",
+ "compile_folder_to_markdown",
+ "compile_folder_to_markdown_sync",
+ "default_harness_config",
+ "harness_config_path",
+ "inspect_folder_memory_health",
+ "inject_context_messages",
+ "inspect_folder_memory_status",
+ "load_harness_config",
+ "review_folder_evolution",
+ "register_workflow_runner",
+ "resolve_workflow_runner",
+ "promote_skill",
+ "record_skill_trace",
+ "scaffold_folder_memory_repository",
+ "suggest_skill_promotions",
+ "validate_harness_config",
+ "watch_folder_to_markdown",
+ "watch_folder_to_markdown_sync",
+ "__version__",
+]
+
+
+def __getattr__(name: str) -> Any:
+ if name not in _LAZY_EXPORTS:
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+ module_name, attr_name = _LAZY_EXPORTS[name]
+ module = __import__(module_name, fromlist=[attr_name])
+ value = getattr(module, attr_name)
+ globals()[name] = value
+ if name == "MemoryService":
+ globals().setdefault("MemUService", value)
+ elif name == "MemUService":
+ globals().setdefault("MemoryService", value)
+ return value
+
+
+def __dir__() -> list[str]:
+ return sorted(set(globals()) | set(__all__) | set(_LAZY_EXPORTS))
diff --git a/src/memu/_version.py b/src/memu/_version.py
new file mode 100644
index 00000000..eb06cf7a
--- /dev/null
+++ b/src/memu/_version.py
@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+__version__ = "1.5.1"
+
+__all__ = ["__version__"]
diff --git a/src/memu/app/__init__.py b/src/memu/app/__init__.py
index 0b0221d2..b2c71e17 100644
--- a/src/memu/app/__init__.py
+++ b/src/memu/app/__init__.py
@@ -1,33 +1,188 @@
-from memu.app.service import MemoryService
-from memu.app.settings import (
- BlobConfig,
- DatabaseConfig,
- DefaultUserModel,
- LLMConfig,
- LLMProfilesConfig,
- MemorizeConfig,
- RetrieveConfig,
- UserConfig,
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from memu.app.context_harness import (
+ ContextHarness,
+ ContextHarnessRun,
+ ContextHarnessSkillEvolutionResult,
+ ContextHarnessSkillTraceResult,
+)
+from memu.app.folder import (
+ EvolutionReviewApplyResult,
+ FolderCompileResult,
+ FolderHealthIssue,
+ FolderHealthResult,
+ FolderHealthSeverity,
+ FolderMemoryCompiler,
+ FolderMemoryCompilerConfig,
+ FolderScaffoldResult,
+ FolderSourceState,
+ FolderSourceStatus,
+ FolderStatusResult,
+ FolderWatchEvent,
+ MarkdownMemoryEntry,
+ compile_folder_to_markdown,
+ compile_folder_to_markdown_sync,
+ inspect_folder_memory_health,
+ inspect_folder_memory_status,
+ review_folder_evolution,
+ scaffold_folder_memory_repository,
+ watch_folder_to_markdown,
+ watch_folder_to_markdown_sync,
+)
+from memu.app.harness_config import (
+ DEFAULT_CONTEXT_MAX_CHARS,
+ DEFAULT_MAX_TEXT_CHARS,
+ HARNESS_CONFIG_NAME,
+ HARNESS_CONFIG_VERSION,
+ default_harness_config,
+ harness_config_path,
+ load_harness_config,
+ validate_harness_config,
)
-from memu.workflow.runner import (
- LocalWorkflowRunner,
- WorkflowRunner,
- register_workflow_runner,
- resolve_workflow_runner,
+from memu.app.markdown_context import (
+ MarkdownContextPack,
+ MarkdownContextSection,
+ MarkdownMemoryRepository,
+ build_markdown_context_pack,
+ inject_context_messages,
)
+from memu.app.self_evolve import (
+ EvidenceRecord,
+ EvolutionInstruction,
+ EvolutionReviewBundle,
+ EvolutionReviewConfig,
+ PatchProposal,
+ ReviewDecision,
+ ReviewStatus,
+)
+from memu.app.skill_trace import (
+ SkillEvolutionProposal,
+ SkillPromotionRecord,
+ SkillToolTrace,
+ SkillTrace,
+ SkillTraceRecord,
+ promote_skill,
+ record_skill_trace,
+ suggest_skill_promotions,
+)
+
+if TYPE_CHECKING:
+ from memu.app.service import MemoryService
+ from memu.app.settings import (
+ BlobConfig,
+ DatabaseConfig,
+ DefaultUserModel,
+ LLMConfig,
+ LLMProfilesConfig,
+ MemorizeConfig,
+ RetrieveConfig,
+ UserConfig,
+ )
+ from memu.workflow.runner import (
+ LocalWorkflowRunner,
+ WorkflowRunner,
+ register_workflow_runner,
+ resolve_workflow_runner,
+ )
+
+
+_LAZY_EXPORTS = {
+ "BlobConfig": ("memu.app.settings", "BlobConfig"),
+ "DatabaseConfig": ("memu.app.settings", "DatabaseConfig"),
+ "DefaultUserModel": ("memu.app.settings", "DefaultUserModel"),
+ "LLMConfig": ("memu.app.settings", "LLMConfig"),
+ "LLMProfilesConfig": ("memu.app.settings", "LLMProfilesConfig"),
+ "LocalWorkflowRunner": ("memu.workflow.runner", "LocalWorkflowRunner"),
+ "MemorizeConfig": ("memu.app.settings", "MemorizeConfig"),
+ "MemoryService": ("memu.app.service", "MemoryService"),
+ "RetrieveConfig": ("memu.app.settings", "RetrieveConfig"),
+ "UserConfig": ("memu.app.settings", "UserConfig"),
+ "WorkflowRunner": ("memu.workflow.runner", "WorkflowRunner"),
+ "register_workflow_runner": ("memu.workflow.runner", "register_workflow_runner"),
+ "resolve_workflow_runner": ("memu.workflow.runner", "resolve_workflow_runner"),
+}
__all__ = [
"BlobConfig",
+ "ContextHarness",
+ "ContextHarnessRun",
+ "ContextHarnessSkillEvolutionResult",
+ "ContextHarnessSkillTraceResult",
"DatabaseConfig",
"DefaultUserModel",
+ "EvidenceRecord",
+ "EvolutionInstruction",
+ "EvolutionReviewBundle",
+ "EvolutionReviewConfig",
+ "EvolutionReviewApplyResult",
+ "FolderCompileResult",
+ "FolderHealthIssue",
+ "FolderHealthResult",
+ "FolderHealthSeverity",
+ "FolderMemoryCompiler",
+ "FolderMemoryCompilerConfig",
+ "FolderScaffoldResult",
+ "FolderSourceState",
+ "FolderSourceStatus",
+ "FolderStatusResult",
+ "FolderWatchEvent",
+ "DEFAULT_CONTEXT_MAX_CHARS",
+ "DEFAULT_MAX_TEXT_CHARS",
+ "HARNESS_CONFIG_NAME",
+ "HARNESS_CONFIG_VERSION",
"LLMConfig",
"LLMProfilesConfig",
"LocalWorkflowRunner",
+ "MarkdownContextPack",
+ "MarkdownContextSection",
+ "MarkdownMemoryEntry",
+ "MarkdownMemoryRepository",
"MemorizeConfig",
"MemoryService",
+ "PatchProposal",
+ "ReviewDecision",
+ "ReviewStatus",
"RetrieveConfig",
+ "SkillEvolutionProposal",
+ "SkillPromotionRecord",
+ "SkillToolTrace",
+ "SkillTrace",
+ "SkillTraceRecord",
"UserConfig",
"WorkflowRunner",
+ "build_markdown_context_pack",
+ "compile_folder_to_markdown",
+ "compile_folder_to_markdown_sync",
+ "default_harness_config",
+ "harness_config_path",
+ "inspect_folder_memory_health",
+ "inject_context_messages",
+ "inspect_folder_memory_status",
+ "load_harness_config",
+ "review_folder_evolution",
"register_workflow_runner",
"resolve_workflow_runner",
+ "promote_skill",
+ "record_skill_trace",
+ "scaffold_folder_memory_repository",
+ "suggest_skill_promotions",
+ "validate_harness_config",
+ "watch_folder_to_markdown",
+ "watch_folder_to_markdown_sync",
]
+
+
+def __getattr__(name: str) -> Any:
+ if name not in _LAZY_EXPORTS:
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+ module_name, attr_name = _LAZY_EXPORTS[name]
+ module = __import__(module_name, fromlist=[attr_name])
+ value = getattr(module, attr_name)
+ globals()[name] = value
+ return value
+
+
+def __dir__() -> list[str]:
+ return sorted(set(globals()) | set(__all__) | set(_LAZY_EXPORTS))
diff --git a/src/memu/app/cli_args.py b/src/memu/app/cli_args.py
new file mode 100644
index 00000000..6aed7b83
--- /dev/null
+++ b/src/memu/app/cli_args.py
@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import argparse
+
+
+def positive_int_arg(value: str) -> int:
+ try:
+ parsed = int(value)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("must be an integer") from exc
+ if parsed <= 0:
+ raise argparse.ArgumentTypeError("must be greater than 0")
+ return parsed
+
+
+def positive_float_arg(value: str) -> float:
+ try:
+ parsed = float(value)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("must be a number") from exc
+ if parsed <= 0:
+ raise argparse.ArgumentTypeError("must be greater than 0")
+ return parsed
+
+
+def probability_arg(value: str) -> float:
+ try:
+ parsed = float(value)
+ except ValueError as exc:
+ raise argparse.ArgumentTypeError("must be a number") from exc
+ if parsed < 0 or parsed > 1:
+ raise argparse.ArgumentTypeError("must be between 0 and 1")
+ return parsed
+
+
+__all__ = ["positive_float_arg", "positive_int_arg", "probability_arg"]
diff --git a/src/memu/app/context_cli.py b/src/memu/app/context_cli.py
new file mode 100644
index 00000000..0955e0c0
--- /dev/null
+++ b/src/memu/app/context_cli.py
@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import argparse
+import json
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any, cast
+
+from memu.app.cli_args import positive_int_arg
+from memu.app.harness_config import (
+ DEFAULT_CONTEXT_MAX_CHARS,
+ arg_or_config_positive_int,
+ config_bucket_char_limits,
+ config_context_buckets,
+ config_context_format,
+ config_section,
+ harness_config_path,
+ load_harness_config,
+)
+from memu.app.markdown_context import ContextBucket, MarkdownContextPack, build_markdown_context_pack
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ prog="memu-context",
+ description="Build an agent-ready context pack from a memU Markdown memory repository.",
+ )
+ parser.add_argument("repo_dir", help="Folder containing memory.md, soul.md, skill.md, and .memu/manifest.json.")
+ parser.add_argument("--query", default=None, help="Optional query used for lightweight relevance ranking.")
+ parser.add_argument(
+ "--max-chars",
+ type=positive_int_arg,
+ default=None,
+ help="Maximum approximate context characters.",
+ )
+ parser.add_argument(
+ "--bucket",
+ action="append",
+ choices=("memory", "soul", "skill"),
+ default=None,
+ help="Bucket to include. Can be repeated. Defaults to all buckets.",
+ )
+ parser.add_argument("--no-generated", action="store_true", help="Exclude generated manifest entries.")
+ parser.add_argument("--no-manual", action="store_true", help="Exclude manual Markdown outside generated blocks.")
+ parser.add_argument(
+ "--bucket-max",
+ action="append",
+ default=None,
+ metavar="BUCKET=CHARS",
+ help="Per-bucket context character budget, such as soul=1000. Can be repeated.",
+ )
+ parser.add_argument("--json", action="store_true", help="Print a machine-readable JSON context pack.")
+ parser.add_argument(
+ "--format",
+ choices=("markdown", "system", "messages", "json", "summary"),
+ default=None,
+ help="Output format. --json is kept as a shortcut for --format json.",
+ )
+ parser.add_argument("--output", default=None, help="Optional file path to write the rendered context output.")
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ parser = build_parser()
+ args = parser.parse_args(argv)
+ config_path = harness_config_path(args.repo_dir)
+ harness_config = load_harness_config(args.repo_dir)
+ context_section = config_section(harness_config, "context", config_path)
+ bucket_values = (
+ list(args.bucket)
+ if args.bucket is not None
+ else config_context_buckets(context_section, config_path)
+ )
+ buckets = [cast(ContextBucket, bucket) for bucket in bucket_values] if bucket_values else None
+ bucket_char_limits = (
+ _parse_bucket_char_limits(args.bucket_max)
+ if args.bucket_max is not None
+ else config_bucket_char_limits(context_section, config_path)
+ )
+ pack = build_markdown_context_pack(
+ args.repo_dir,
+ query=args.query,
+ buckets=buckets,
+ max_chars=arg_or_config_positive_int(
+ args.max_chars,
+ context_section,
+ "max_chars",
+ DEFAULT_CONTEXT_MAX_CHARS,
+ flag_name="--max-chars",
+ config_path=config_path,
+ ),
+ include_generated=not args.no_generated,
+ include_manual=not args.no_manual,
+ bucket_char_limits=bucket_char_limits,
+ )
+ output_format = _context_output_format(args, context_section, config_path)
+ _emit_output(_render_pack(pack, output_format), args.output)
+ return 0
+
+
+def _context_output_format(args: argparse.Namespace, context_section: Any, config_path: Path) -> str:
+ if args.json:
+ return "json"
+ if args.format is not None:
+ return str(args.format)
+ return config_context_format(context_section, config_path)
+
+
+def _print_pack(pack: MarkdownContextPack, output_format: str) -> None:
+ _emit_output(_render_pack(pack, output_format), None)
+
+
+def _render_pack(pack: MarkdownContextPack, output_format: str) -> str:
+ if output_format == "json":
+ return json.dumps(pack.to_dict(), indent=2, sort_keys=True) + "\n"
+ if output_format == "summary":
+ return json.dumps(pack.to_summary(), indent=2, sort_keys=True) + "\n"
+ if output_format == "messages":
+ return json.dumps(pack.to_messages(), indent=2, sort_keys=True) + "\n"
+ if output_format == "system":
+ return pack.to_system_prompt()
+ return pack.to_markdown()
+
+
+def _emit_output(text: str, output_path: str | None) -> None:
+ if output_path:
+ _write_output_text(text, output_path)
+ return
+ print(text, end="")
+
+
+def _write_output_text(text: str, output_path: str) -> None:
+ target = Path(output_path)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(text, encoding="utf-8")
+
+
+def _parse_bucket_char_limits(values: Sequence[str]) -> dict[ContextBucket, int]:
+ limits: dict[ContextBucket, int] = {}
+ for value in values:
+ bucket, sep, raw_limit = value.partition("=")
+ if not sep:
+ msg = f"--bucket-max expects BUCKET=CHARS, got {value!r}"
+ raise SystemExit(msg)
+ clean_bucket = bucket.strip()
+ if clean_bucket not in {"memory", "soul", "skill"}:
+ msg = f"unknown bucket for --bucket-max: {clean_bucket}"
+ raise SystemExit(msg)
+ try:
+ limit = int(raw_limit)
+ except ValueError as exc:
+ msg = f"--bucket-max value must be an integer: {value!r}"
+ raise SystemExit(msg) from exc
+ if limit <= 0:
+ msg = "--bucket-max values must be greater than 0"
+ raise SystemExit(msg)
+ limits[cast(ContextBucket, clean_bucket)] = limit
+ return limits
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/src/memu/app/context_harness.py b/src/memu/app/context_harness.py
new file mode 100644
index 00000000..d76dc722
--- /dev/null
+++ b/src/memu/app/context_harness.py
@@ -0,0 +1,600 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, cast
+
+from memu.app.folder import (
+ EvolutionReviewApplyResult,
+ FolderCompileResult,
+ FolderHealthIssue,
+ FolderMemoryCompiler,
+ FolderMemoryCompilerConfig,
+ FolderHealthResult,
+ FolderScaffoldResult,
+ FolderStatusResult,
+ FolderWatchEvent,
+ watch_folder_to_markdown,
+ watch_folder_to_markdown_sync,
+)
+from memu.app.harness_config import (
+ DEFAULT_CONTEXT_MAX_CHARS,
+ DEFAULT_MAX_TEXT_CHARS,
+ HARNESS_CONFIG_NAME,
+ arg_or_config_positive_int,
+ compiler_exclude_patterns,
+ config_bucket_char_limits,
+ config_context_buckets,
+ config_section,
+ harness_config_path,
+ try_load_harness_config,
+ validate_harness_config,
+)
+from memu.app.markdown_context import ContextBucket, MarkdownContextPack, MarkdownMemoryRepository
+from memu.app.self_evolve import ReviewStatus
+from memu.app.skill_trace import (
+ SkillEvolutionProposal,
+ SkillPromotionRecord,
+ SkillToolTrace,
+ SkillTraceOutcome,
+ SkillTraceRecord,
+ promote_skill,
+ record_skill_trace,
+ suggest_skill_promotions,
+)
+
+if TYPE_CHECKING:
+ from memu.app.service import MemoryService
+
+
+@dataclass(frozen=True)
+class ContextHarnessRun:
+ """Result of compiling raw data and building a context pack in one call."""
+
+ compile_result: FolderCompileResult
+ context_pack: MarkdownContextPack
+
+
+@dataclass(frozen=True)
+class ContextHarnessSkillTraceResult:
+ """Result of recording a skill trace, optionally followed by recompilation."""
+
+ record: SkillTraceRecord
+ compile_result: FolderCompileResult | None = None
+
+
+@dataclass(frozen=True)
+class ContextHarnessSkillEvolutionResult:
+ """Result of suggesting skill promotions and optionally applying them."""
+
+ proposals: list[SkillEvolutionProposal]
+ promotions: list[SkillPromotionRecord]
+
+
+class ContextHarness:
+ """High-level harness for folder-backed memory, context, and skill evolution.
+
+ `source_folder` is the user's raw-data folder. `repo_dir` is the Markdown memory
+ repository containing `memory.md`, `soul.md`, `skill.md`, and `.memu/`.
+ """
+
+ def __init__(
+ self,
+ source_folder: str | Path,
+ repo_dir: str | Path,
+ *,
+ memory_service: MemoryService | None = None,
+ user: Mapping[str, Any] | None = None,
+ compiler_config: FolderMemoryCompilerConfig | None = None,
+ harness_config: Mapping[str, Any] | None = None,
+ ) -> None:
+ self.source_folder = Path(source_folder).resolve()
+ self.repo_dir = Path(repo_dir).resolve()
+ self.memory_service = memory_service
+ self.user = dict(user or {})
+ base_compiler_config = compiler_config or FolderMemoryCompilerConfig()
+ self.harness_config_path = harness_config_path(
+ self.repo_dir,
+ base_compiler_config.metadata_dir_name,
+ )
+ self._harness_config_error: str | None = None
+ self.harness_config = self._load_repo_harness_config(
+ harness_config,
+ metadata_dir_name=base_compiler_config.metadata_dir_name,
+ )
+ self.compiler_config = self._compiler_config_from_repo_config(base_compiler_config)
+ self.compiler = FolderMemoryCompiler(memory_service=memory_service, config=self.compiler_config)
+ self.repository = MarkdownMemoryRepository(self.repo_dir)
+
+ @classmethod
+ def from_repo(
+ cls,
+ repo_dir: str | Path,
+ *,
+ memory_service: MemoryService | None = None,
+ user: Mapping[str, Any] | None = None,
+ compiler_config: FolderMemoryCompilerConfig | None = None,
+ harness_config: Mapping[str, Any] | None = None,
+ ) -> ContextHarness:
+ """Create a harness for an existing repository, using `repo/raw_data` as source."""
+
+ base_compiler_config = compiler_config or FolderMemoryCompilerConfig()
+ repo_path = Path(repo_dir)
+ return cls(
+ repo_path / base_compiler_config.raw_data_dir_name,
+ repo_path,
+ memory_service=memory_service,
+ user=user,
+ compiler_config=compiler_config,
+ harness_config=harness_config,
+ )
+
+ def scaffold(self, *, copy_source: bool = False) -> FolderScaffoldResult:
+ """Create the repository layout, optionally copying `source_folder` into raw_data."""
+
+ self._ensure_harness_config_valid()
+ source_folder = self.source_folder if copy_source else None
+ return self.compiler.scaffold(self.repo_dir, source_folder=source_folder)
+
+ def status(self) -> FolderStatusResult:
+ """Inspect source changes against the manifest without writing files."""
+
+ self._ensure_harness_config_valid()
+ return self.compiler.status(self.source_folder, self.repo_dir)
+
+ def health(self) -> FolderHealthResult:
+ """Validate the Markdown memory repository without writing files."""
+
+ result = self.compiler.health(self.repo_dir)
+ issues = list(result.issues)
+ if self._harness_config_error is not None:
+ issues.append(
+ FolderHealthIssue(
+ severity="error",
+ code="invalid_harness_config",
+ message=self._harness_config_error,
+ path=f"{self.compiler_config.metadata_dir_name}/{HARNESS_CONFIG_NAME}",
+ )
+ )
+ return FolderHealthResult(output_dir=result.output_dir, issues=issues)
+
+ def promote_skill(
+ self,
+ *,
+ title: str,
+ lessons: Sequence[str] | None = None,
+ actions: Sequence[str] | None = None,
+ when_to_use: str = "",
+ source: str = "",
+ tags: Sequence[str] | None = None,
+ metadata: Mapping[str, str] | None = None,
+ ) -> SkillPromotionRecord:
+ """Append a durable manual skill note outside the generated block."""
+
+ self._ensure_harness_config_valid()
+ return promote_skill(
+ self.repo_dir,
+ title=title,
+ lessons=list(lessons or []),
+ actions=list(actions or []),
+ when_to_use=when_to_use,
+ source=source,
+ tags=list(tags or []),
+ metadata=dict(metadata or {}),
+ )
+
+ def review_evolution(
+ self,
+ *,
+ proposal_ids: Sequence[str] | None = None,
+ reviewer: str = "creator",
+ decision: ReviewStatus = "approved",
+ reason: str = "",
+ ) -> EvolutionReviewApplyResult:
+ """Apply creator review decisions to pending self-evolve proposals."""
+
+ self._ensure_harness_config_valid()
+ return self.compiler.review_evolution(
+ self.repo_dir,
+ proposal_ids=proposal_ids,
+ reviewer=reviewer,
+ decision=decision,
+ reason=reason,
+ )
+
+ def suggest_skills(
+ self,
+ *,
+ limit: int = 5,
+ min_support: int = 1,
+ ) -> list[SkillEvolutionProposal]:
+ """Suggest durable skill promotions from recorded raw skill traces."""
+
+ self._ensure_harness_config_valid()
+ return suggest_skill_promotions(self.source_folder, limit=limit, min_support=min_support)
+
+ def evolve_skills(
+ self,
+ *,
+ limit: int = 5,
+ min_support: int = 1,
+ promote: bool = False,
+ ) -> ContextHarnessSkillEvolutionResult:
+ """Suggest skill promotions and optionally write them into the skill library."""
+
+ proposals = self.suggest_skills(limit=limit, min_support=min_support)
+ promotions = [self.promote_skill(**proposal.to_promotion_kwargs()) for proposal in proposals] if promote else []
+ return ContextHarnessSkillEvolutionResult(proposals=proposals, promotions=promotions)
+
+ async def ingest(self, *, user: Mapping[str, Any] | None = None) -> FolderCompileResult:
+ """Compile changed raw data through self-evolve review into the Markdown repository."""
+
+ self._ensure_harness_config_valid()
+ return await self.compiler.compile(self.source_folder, self.repo_dir, user=self._scope(user))
+
+ def ingest_sync(self, *, user: Mapping[str, Any] | None = None) -> FolderCompileResult:
+ """Synchronous wrapper around `ingest`."""
+
+ return asyncio.run(self.ingest(user=user))
+
+ def build_context_pack(
+ self,
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int | None = None,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+ ) -> MarkdownContextPack:
+ """Load the Markdown repository into an agent-ready context pack."""
+
+ self._ensure_harness_config_valid()
+ return self.repository.build_context_pack(
+ query=query,
+ buckets=self._context_buckets(buckets),
+ max_chars=self._context_max_chars(max_chars),
+ include_generated=include_generated,
+ include_manual=include_manual,
+ bucket_char_limits=self._context_bucket_char_limits(bucket_char_limits),
+ )
+
+ def build_context_markdown(
+ self,
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int | None = None,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+ ) -> str:
+ """Return the context pack as `` Markdown."""
+
+ return self.build_context_pack(
+ query=query,
+ buckets=buckets,
+ max_chars=max_chars,
+ include_generated=include_generated,
+ include_manual=include_manual,
+ bucket_char_limits=bucket_char_limits,
+ ).to_markdown()
+
+ def build_context_system_prompt(
+ self,
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int | None = None,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+ ) -> str:
+ """Return context with explicit system instructions for agent injection."""
+
+ return self.build_context_pack(
+ query=query,
+ buckets=buckets,
+ max_chars=max_chars,
+ include_generated=include_generated,
+ include_manual=include_manual,
+ bucket_char_limits=bucket_char_limits,
+ ).to_system_prompt()
+
+ def build_context_messages(
+ self,
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int | None = None,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+ ) -> list[dict[str, str]]:
+ """Return context as a system message list for chat-completion APIs."""
+
+ return self.build_context_pack(
+ query=query,
+ buckets=buckets,
+ max_chars=max_chars,
+ include_generated=include_generated,
+ include_manual=include_manual,
+ bucket_char_limits=bucket_char_limits,
+ ).to_messages()
+
+ def inject_context_messages(
+ self,
+ messages: Sequence[Mapping[str, Any]],
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int | None = None,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+ replace_existing: bool = True,
+ ) -> list[dict[str, Any]]:
+ """Return a copied chat message list with memU context injected."""
+
+ return self.build_context_pack(
+ query=query,
+ buckets=buckets,
+ max_chars=max_chars,
+ include_generated=include_generated,
+ include_manual=include_manual,
+ bucket_char_limits=bucket_char_limits,
+ ).inject_into_messages(messages, replace_existing=replace_existing)
+
+ async def refresh_context(
+ self,
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int | None = None,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+ user: Mapping[str, Any] | None = None,
+ ) -> ContextHarnessRun:
+ """Compile raw data, then build a fresh context pack."""
+
+ compile_result = await self.ingest(user=user)
+ context_pack = self.build_context_pack(
+ query=query,
+ buckets=buckets,
+ max_chars=max_chars,
+ include_generated=include_generated,
+ include_manual=include_manual,
+ bucket_char_limits=bucket_char_limits,
+ )
+ return ContextHarnessRun(compile_result=compile_result, context_pack=context_pack)
+
+ def refresh_context_sync(
+ self,
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int | None = None,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+ user: Mapping[str, Any] | None = None,
+ ) -> ContextHarnessRun:
+ """Synchronous wrapper around `refresh_context`."""
+
+ return asyncio.run(
+ self.refresh_context(
+ query=query,
+ buckets=buckets,
+ max_chars=max_chars,
+ include_generated=include_generated,
+ include_manual=include_manual,
+ bucket_char_limits=bucket_char_limits,
+ user=user,
+ )
+ )
+
+ async def record_skill_trace(
+ self,
+ *,
+ task: str,
+ outcome: SkillTraceOutcome = "unknown",
+ summary: str = "",
+ actions: Sequence[str] | None = None,
+ tools: Sequence[SkillToolTrace] | None = None,
+ lessons: Sequence[str] | None = None,
+ metadata: Mapping[str, str] | None = None,
+ recompile: bool = True,
+ user: Mapping[str, Any] | None = None,
+ ) -> ContextHarnessSkillTraceResult:
+ """Record skill-evolution evidence under raw data and optionally recompile."""
+
+ record = record_skill_trace(
+ self.source_folder,
+ task=task,
+ outcome=outcome,
+ summary=summary,
+ actions=list(actions or []),
+ tools=list(tools or []),
+ lessons=list(lessons or []),
+ metadata=dict(metadata or {}),
+ )
+ compile_result = await self.ingest(user=user) if recompile else None
+ return ContextHarnessSkillTraceResult(record=record, compile_result=compile_result)
+
+ def record_skill_trace_sync(
+ self,
+ *,
+ task: str,
+ outcome: SkillTraceOutcome = "unknown",
+ summary: str = "",
+ actions: Sequence[str] | None = None,
+ tools: Sequence[SkillToolTrace] | None = None,
+ lessons: Sequence[str] | None = None,
+ metadata: Mapping[str, str] | None = None,
+ recompile: bool = True,
+ user: Mapping[str, Any] | None = None,
+ ) -> ContextHarnessSkillTraceResult:
+ """Synchronous wrapper around `record_skill_trace`."""
+
+ return asyncio.run(
+ self.record_skill_trace(
+ task=task,
+ outcome=outcome,
+ summary=summary,
+ actions=actions,
+ tools=tools,
+ lessons=lessons,
+ metadata=metadata,
+ recompile=recompile,
+ user=user,
+ )
+ )
+
+ async def watch(
+ self,
+ *,
+ poll_interval: float = 2.0,
+ max_runs: int | None = None,
+ on_event: Callable[[FolderWatchEvent], Any | Awaitable[Any]] | None = None,
+ user: Mapping[str, Any] | None = None,
+ ) -> list[FolderWatchEvent]:
+ """Watch raw data and recompile whenever the source fingerprint changes."""
+
+ self._ensure_harness_config_valid()
+ return await watch_folder_to_markdown(
+ self.source_folder,
+ self.repo_dir,
+ memory_service=self.memory_service,
+ user=self._scope(user),
+ config=self.compiler_config,
+ poll_interval=poll_interval,
+ max_runs=max_runs,
+ on_event=on_event,
+ )
+
+ def watch_sync(
+ self,
+ *,
+ poll_interval: float = 2.0,
+ max_runs: int | None = None,
+ on_event: Callable[[FolderWatchEvent], Any | Awaitable[Any]] | None = None,
+ user: Mapping[str, Any] | None = None,
+ ) -> list[FolderWatchEvent]:
+ """Synchronous wrapper around `watch`."""
+
+ self._ensure_harness_config_valid()
+ return watch_folder_to_markdown_sync(
+ self.source_folder,
+ self.repo_dir,
+ memory_service=self.memory_service,
+ user=self._scope(user),
+ config=self.compiler_config,
+ poll_interval=poll_interval,
+ max_runs=max_runs,
+ on_event=on_event,
+ )
+
+ def _scope(self, user: Mapping[str, Any] | None) -> dict[str, Any]:
+ scope = dict(self.user)
+ if user is not None:
+ scope.update(user)
+ return scope
+
+ def _load_repo_harness_config(
+ self,
+ harness_config: Mapping[str, Any] | None,
+ *,
+ metadata_dir_name: str,
+ ) -> dict[str, Any]:
+ if harness_config is not None:
+ try:
+ validate_harness_config(harness_config, self.harness_config_path)
+ except SystemExit as exc:
+ self._harness_config_error = str(exc)
+ return {}
+ return dict(harness_config)
+ config, error = try_load_harness_config(self.repo_dir, metadata_dir_name)
+ self._harness_config_error = error
+ return config
+
+ def _ensure_harness_config_valid(self) -> None:
+ if self._harness_config_error is not None:
+ raise SystemExit(self._harness_config_error)
+
+ def _compiler_config_from_repo_config(
+ self,
+ base_config: FolderMemoryCompilerConfig,
+ ) -> FolderMemoryCompilerConfig:
+ compiler_section = config_section(
+ self.harness_config,
+ "compiler",
+ self.harness_config_path,
+ )
+ return FolderMemoryCompilerConfig(
+ raw_data_dir_name=base_config.raw_data_dir_name,
+ metadata_dir_name=base_config.metadata_dir_name,
+ derived_dir_name=base_config.derived_dir_name,
+ agent_instructions_name=base_config.agent_instructions_name,
+ ignore_file_name=base_config.ignore_file_name,
+ write_agent_instructions=base_config.write_agent_instructions,
+ exclude_patterns=tuple(
+ compiler_exclude_patterns(
+ list(base_config.exclude_patterns) if base_config.exclude_patterns else None,
+ compiler_section,
+ self.harness_config_path,
+ )
+ ),
+ max_text_chars=arg_or_config_positive_int(
+ base_config.max_text_chars if base_config.max_text_chars != DEFAULT_MAX_TEXT_CHARS else None,
+ compiler_section,
+ "max_text_chars",
+ DEFAULT_MAX_TEXT_CHARS,
+ flag_name="--max-text-chars",
+ config_path=self.harness_config_path,
+ ),
+ use_memory_service=base_config.use_memory_service,
+ self_evolve_enabled=base_config.self_evolve_enabled,
+ evolution_review=base_config.evolution_review,
+ )
+
+ def _context_section(self) -> Mapping[str, Any]:
+ return config_section(self.harness_config, "context", self.harness_config_path)
+
+ def _context_buckets(
+ self,
+ buckets: Sequence[ContextBucket] | None,
+ ) -> Sequence[ContextBucket] | None:
+ if buckets is not None:
+ return buckets
+ configured = config_context_buckets(self._context_section(), self.harness_config_path)
+ return [cast(ContextBucket, bucket) for bucket in configured] or None
+
+ def _context_max_chars(self, max_chars: int | None) -> int:
+ return arg_or_config_positive_int(
+ max_chars,
+ self._context_section(),
+ "max_chars",
+ DEFAULT_CONTEXT_MAX_CHARS,
+ flag_name="max_chars",
+ config_path=self.harness_config_path,
+ )
+
+ def _context_bucket_char_limits(
+ self,
+ bucket_char_limits: Mapping[ContextBucket, int] | None,
+ ) -> Mapping[ContextBucket, int] | None:
+ if bucket_char_limits is not None:
+ return bucket_char_limits
+ return config_bucket_char_limits(self._context_section(), self.harness_config_path)
+
+
+__all__ = [
+ "ContextHarness",
+ "ContextHarnessRun",
+ "ContextHarnessSkillEvolutionResult",
+ "ContextHarnessSkillTraceResult",
+ "EvolutionReviewApplyResult",
+]
diff --git a/src/memu/app/context_harness_cli.py b/src/memu/app/context_harness_cli.py
new file mode 100644
index 00000000..81a3f15a
--- /dev/null
+++ b/src/memu/app/context_harness_cli.py
@@ -0,0 +1,761 @@
+from __future__ import annotations
+
+import argparse
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any, cast
+
+from memu.app.cli_args import positive_float_arg, positive_int_arg, probability_arg
+from memu.app.context_cli import _parse_bucket_char_limits, _render_pack, _write_output_text
+from memu.app.context_harness import (
+ ContextHarness,
+ ContextHarnessRun,
+ ContextHarnessSkillEvolutionResult,
+ ContextHarnessSkillTraceResult,
+)
+from memu.app.folder import (
+ EvolutionReviewApplyResult,
+ FolderMemoryCompilerConfig,
+ FolderHealthResult,
+ FolderScaffoldResult,
+ FolderStatusResult,
+ scaffold_folder_memory_repository,
+)
+from memu.app.folder_cli import (
+ _build_memory_service,
+ _evolution_review_config,
+ _parse_user_scope,
+ _print_human_summary,
+ _print_watch_event,
+ _result_summary,
+)
+from memu.app.harness_config import (
+ DEFAULT_CONTEXT_MAX_CHARS,
+ DEFAULT_MAX_TEXT_CHARS,
+ HARNESS_CONFIG_NAME,
+ arg_or_config_positive_int,
+ compiler_exclude_patterns,
+ config_bucket_char_limits,
+ config_context_buckets,
+ config_context_format,
+ config_section,
+ default_harness_config,
+ harness_config_path,
+ load_harness_config,
+ positive_int_or_default,
+)
+from memu.app.markdown_context import ContextBucket, MarkdownContextPack
+from memu.app.skill_trace import SkillEvolutionProposal, SkillPromotionRecord, SkillTraceOutcome
+from memu.app.skill_trace_cli import _parse_key_values, _parse_tool
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ prog="memu-harness",
+ description="Unified folder-backed context harness for memory, context, and self-evolving skills.",
+ )
+ subparsers = parser.add_subparsers(dest="command", required=True)
+ common = _common_parent()
+ context_options = _context_parent()
+
+ init = subparsers.add_parser(
+ "init",
+ help="Create an empty Markdown memory repository layout.",
+ )
+ init.add_argument("repo_dir", help="Folder where the Markdown memory repository is stored.")
+ init.add_argument("--source-folder", default=None, help="Optional uploaded folder to copy into raw_data/.")
+ init.add_argument("--raw-data-dir", default="raw_data", help="Name of the raw data directory in output.")
+ init.add_argument("--metadata-dir", default=".memu", help="Name of the metadata directory in output.")
+ init.add_argument("--derived-dir", default="derived", help="Name of the derived evidence directory.")
+ init.add_argument(
+ "--max-text-chars",
+ type=positive_int_arg,
+ default=None,
+ help="Default maximum text evidence chars per source file stored in .memu/harness.json.",
+ )
+ init.add_argument(
+ "--exclude",
+ action="append",
+ default=None,
+ metavar="GLOB",
+ help="Exclude source files matching a posix glob during raw_data copy. Can be repeated.",
+ )
+ init.add_argument("--json", action="store_true", help="Print machine-readable JSON.")
+
+ subparsers.add_parser(
+ "ingest",
+ parents=[common],
+ help="Compile changed raw data through self-evolve review.",
+ )
+ subparsers.add_parser(
+ "context",
+ parents=[common, context_options],
+ help="Build a context pack from the existing Markdown memory repository.",
+ )
+ subparsers.add_parser(
+ "refresh",
+ parents=[common, context_options],
+ help="Compile changed raw data, then build a fresh context pack.",
+ )
+ subparsers.add_parser(
+ "status",
+ parents=[common],
+ help="Inspect raw-data changes against the manifest without writing files.",
+ )
+ doctor = subparsers.add_parser(
+ "doctor",
+ help="Validate the Markdown memory repository layout and manifest without writing files.",
+ )
+ doctor.add_argument("repo_dir", help="Markdown memory repository to validate.")
+ doctor.add_argument("--raw-data-dir", default="raw_data", help="Name of the raw data directory in output.")
+ doctor.add_argument("--metadata-dir", default=".memu", help="Name of the metadata directory in output.")
+ doctor.add_argument("--derived-dir", default="derived", help="Name of the derived evidence directory.")
+ doctor.add_argument("--json", action="store_true", help="Print machine-readable JSON.")
+
+ promote = subparsers.add_parser(
+ "promote-skill",
+ parents=[common],
+ help="Append a durable manual skill note to skill.md.",
+ )
+ promote.add_argument("--title", required=True, help="Short name for the promoted skill.")
+ promote.add_argument("--lesson", action="append", default=[], help="Reusable skill lesson. Can be repeated.")
+ promote.add_argument("--action", action="append", default=[], help="Procedure step. Can be repeated.")
+ promote.add_argument("--when", default="", help="When this skill should be used.")
+ promote.add_argument("--source", default="", help="Optional source trace, task, or evidence path.")
+ promote.add_argument("--tag", action="append", default=[], help="Skill tag. Can be repeated.")
+ promote.add_argument(
+ "--metadata",
+ action="append",
+ default=[],
+ metavar="KEY=VALUE",
+ help="Extra metadata stored on the promoted skill. Can be repeated.",
+ )
+
+ suggest = subparsers.add_parser(
+ "suggest-skills",
+ parents=[common],
+ help="Suggest durable skills from raw skill traces without writing by default.",
+ )
+ suggest.add_argument("--limit", type=positive_int_arg, default=5, help="Maximum proposals to return.")
+ suggest.add_argument(
+ "--min-support",
+ type=positive_int_arg,
+ default=1,
+ help="Minimum number of traces supporting a proposed skill.",
+ )
+ suggest.add_argument(
+ "--promote",
+ action="store_true",
+ help="Write suggested skills into skill.md and skill/promoted/.",
+ )
+
+ review = subparsers.add_parser(
+ "review-evolution",
+ help="Approve or reject pending self-evolve patch proposals.",
+ )
+ review.add_argument("repo_dir", help="Markdown memory repository to review.")
+ review.add_argument(
+ "--proposal-id",
+ action="append",
+ default=None,
+ help="Proposal ID to review. Can be repeated. Defaults to all pending proposals.",
+ )
+ review.add_argument("--reject", action="store_true", help="Reject matching pending proposals instead of approving.")
+ review.add_argument("--reviewer", default="creator", help="Reviewer name recorded in the audit trail.")
+ review.add_argument("--reason", default="", help="Review reason recorded in the audit trail.")
+ review.add_argument("--raw-data-dir", default="raw_data", help="Name of the raw data directory in output.")
+ review.add_argument("--metadata-dir", default=".memu", help="Name of the metadata directory in output.")
+ review.add_argument("--derived-dir", default="derived", help="Name of the derived evidence directory.")
+ review.add_argument("--json", action="store_true", help="Print machine-readable JSON.")
+
+ trace = subparsers.add_parser(
+ "trace",
+ parents=[common],
+ help="Record a skill-evolution trace under raw data and optionally recompile.",
+ )
+ trace.add_argument("--task", required=True, help="Task or situation this trace describes.")
+ trace.add_argument(
+ "--outcome",
+ choices=("success", "failure", "partial", "unknown"),
+ default="unknown",
+ help="Outcome of the task.",
+ )
+ trace.add_argument("--summary", default="", help="Short summary of what happened.")
+ trace.add_argument("--action", action="append", default=[], help="Action/workflow step. Can be repeated.")
+ trace.add_argument("--lesson", action="append", default=[], help="Reusable skill lesson. Can be repeated.")
+ trace.add_argument(
+ "--tool",
+ action="append",
+ default=[],
+ metavar="NAME[:success|failure][:score]",
+ help="Tool usage summary. Can be repeated.",
+ )
+ trace.add_argument(
+ "--metadata",
+ action="append",
+ default=[],
+ metavar="KEY=VALUE",
+ help="Extra metadata stored on the trace. Can be repeated.",
+ )
+ trace.add_argument("--no-recompile", action="store_true", help="Record the trace without recompiling.")
+
+ watch = subparsers.add_parser(
+ "watch",
+ parents=[common],
+ help="Keep polling raw data and recompile when files change.",
+ )
+ watch.add_argument("--poll-interval", type=positive_float_arg, default=2.0, help="Polling interval in seconds.")
+ watch.add_argument(
+ "--watch-max-runs",
+ type=positive_int_arg,
+ default=None,
+ help="Stop watch mode after this many compile events. Mainly useful for automation.",
+ )
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ parser = build_parser()
+ args = parser.parse_args(argv)
+
+ if args.command == "init":
+ max_text_chars = positive_int_or_default(
+ args.max_text_chars,
+ DEFAULT_MAX_TEXT_CHARS,
+ flag_name="--max-text-chars",
+ )
+ result = scaffold_folder_memory_repository(
+ args.repo_dir,
+ source_folder=args.source_folder,
+ config=FolderMemoryCompilerConfig(
+ raw_data_dir_name=args.raw_data_dir,
+ metadata_dir_name=args.metadata_dir,
+ derived_dir_name=args.derived_dir,
+ exclude_patterns=tuple(args.exclude or []),
+ max_text_chars=max_text_chars,
+ use_memory_service=False,
+ ),
+ )
+ config_path = _write_init_harness_config(result, args, max_text_chars=max_text_chars)
+ if args.json:
+ print(json.dumps(_scaffold_summary(result, config_path), indent=2, sort_keys=True))
+ else:
+ _print_scaffold_summary(result, config_path)
+ return 0
+
+ if args.command == "doctor":
+ result = _doctor_health(args)
+ if args.json:
+ print(json.dumps(result.to_dict(), indent=2, sort_keys=True))
+ else:
+ _print_health_summary(result)
+ return 0 if result.ok else 1
+
+ if args.command == "review-evolution":
+ harness = _review_harness(args)
+ result = harness.review_evolution(
+ proposal_ids=args.proposal_id,
+ reviewer=args.reviewer,
+ decision="rejected" if args.reject else "approved",
+ reason=args.reason,
+ )
+ if args.json:
+ print(json.dumps(_evolution_review_summary(result), indent=2, sort_keys=True))
+ else:
+ _print_evolution_review_summary(result)
+ return 0
+
+ harness = _build_harness(args)
+
+ if args.command == "ingest":
+ result = harness.ingest_sync()
+ if args.json:
+ print(json.dumps(_result_summary(result), indent=2, sort_keys=True))
+ else:
+ _print_human_summary(result)
+ return 0
+
+ if args.command == "context":
+ pack = harness.build_context_pack(**_context_kwargs(args))
+ _emit_context_output(_render_context_output(pack, args), args.output)
+ return 0
+
+ if args.command == "refresh":
+ run = harness.refresh_context_sync(**_context_kwargs(args))
+ if _context_output_format(args) == "json":
+ _emit_context_output(json.dumps(_run_summary(run), indent=2, sort_keys=True) + "\n", args.output)
+ else:
+ _emit_context_output(_render_context_output(run.context_pack, args), args.output)
+ return 0
+
+ if args.command == "status":
+ result = harness.status()
+ if args.json:
+ print(json.dumps(result.to_dict(), indent=2, sort_keys=True))
+ else:
+ _print_status_summary(result)
+ return 0
+
+ if args.command == "promote-skill":
+ result = harness.promote_skill(
+ title=args.title,
+ lessons=list(args.lesson),
+ actions=list(args.action),
+ when_to_use=args.when,
+ source=args.source,
+ tags=list(args.tag),
+ metadata=_parse_key_values(args.metadata, flag_name="--metadata"),
+ )
+ if args.json:
+ print(json.dumps(_promotion_summary(result), indent=2, sort_keys=True))
+ else:
+ _print_promotion_summary(result)
+ return 0
+
+ if args.command == "suggest-skills":
+ result = harness.evolve_skills(limit=args.limit, min_support=args.min_support, promote=args.promote)
+ if args.json:
+ print(json.dumps(_skill_evolution_summary(result), indent=2, sort_keys=True))
+ else:
+ _print_skill_evolution_summary(result)
+ return 0
+
+ if args.command == "trace":
+ result = harness.record_skill_trace_sync(
+ task=args.task,
+ outcome=cast(SkillTraceOutcome, args.outcome),
+ summary=args.summary,
+ actions=list(args.action),
+ tools=[_parse_tool(value) for value in args.tool],
+ lessons=list(args.lesson),
+ metadata=_parse_key_values(args.metadata, flag_name="--metadata"),
+ recompile=not args.no_recompile,
+ )
+ if args.json:
+ print(json.dumps(_trace_summary(result), indent=2, sort_keys=True))
+ else:
+ _print_trace_summary(result)
+ return 0
+
+ if args.command == "watch":
+ harness.watch_sync(
+ poll_interval=args.poll_interval,
+ max_runs=args.watch_max_runs,
+ on_event=lambda event: _print_watch_event(event, as_json=args.json),
+ )
+ return 0
+
+ parser.error(f"unknown command: {args.command}")
+ return 2
+
+
+def _common_parent() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument(
+ "paths",
+ nargs="+",
+ help=(
+ "Either REPO_DIR, using REPO_DIR/raw_data as the source, or SOURCE_FOLDER REPO_DIR "
+ "for an external raw-data folder."
+ ),
+ )
+ parser.add_argument(
+ "--user",
+ action="append",
+ default=[],
+ metavar="KEY=VALUE",
+ help="User scope value passed through to MemoryService. Can be repeated.",
+ )
+ parser.add_argument(
+ "--max-text-chars",
+ type=positive_int_arg,
+ default=None,
+ help="Maximum text evidence chars per source file.",
+ )
+ parser.add_argument("--raw-data-dir", default="raw_data", help="Name of the raw data directory in output.")
+ parser.add_argument("--metadata-dir", default=".memu", help="Name of the metadata directory in output.")
+ parser.add_argument("--derived-dir", default="derived", help="Name of the derived evidence directory.")
+ parser.add_argument(
+ "--exclude",
+ action="append",
+ default=None,
+ metavar="GLOB",
+ help="Exclude source files matching a posix glob. Can be repeated.",
+ )
+ parser.add_argument(
+ "--require-creator-review",
+ action="store_true",
+ help="Create Evolution Instructions and Patch Proposals without auto-applying them.",
+ )
+ parser.add_argument(
+ "--min-evolution-confidence",
+ type=probability_arg,
+ default=0.0,
+ help="Minimum confidence required for auto-approved evolution patches.",
+ )
+ parser.add_argument("--json", action="store_true", help="Print machine-readable JSON.")
+
+ service = parser.add_argument_group("MemoryService extraction")
+ service.add_argument(
+ "--use-memory-service",
+ action="store_true",
+ help="Use MemoryService and configured LLMs for richer multimodal extraction.",
+ )
+ service.add_argument("--provider", default="openai", help="LLM provider for MemoryService.")
+ service.add_argument("--client-backend", default="sdk", choices=("sdk", "httpx", "lazyllm_backend"))
+ service.add_argument("--base-url", default=None, help="Optional LLM API base URL.")
+ service.add_argument("--api-key", default=None, help="LLM API key. Defaults to the selected API key env var.")
+ service.add_argument(
+ "--api-key-env",
+ default=None,
+ help="Environment variable to read the API key from. Defaults to the provider's standard env var.",
+ )
+ service.add_argument("--chat-model", default="gpt-4o-mini", help="Chat/vision model for extraction.")
+ service.add_argument("--embed-model", default="text-embedding-3-small", help="Embedding model for indexing.")
+ service.add_argument(
+ "--memory-types",
+ default="profile,event,knowledge,behavior,skill,tool",
+ help="Comma-separated MemoryService memory types to extract.",
+ )
+ return parser
+
+
+def _context_parent() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument("--query", default=None, help="Optional query used for lightweight relevance ranking.")
+ parser.add_argument(
+ "--max-chars",
+ type=positive_int_arg,
+ default=None,
+ help="Maximum approximate context characters.",
+ )
+ parser.add_argument(
+ "--bucket",
+ action="append",
+ choices=("memory", "soul", "skill"),
+ default=None,
+ help="Bucket to include. Can be repeated. Defaults to all buckets.",
+ )
+ parser.add_argument("--no-generated", action="store_true", help="Exclude generated manifest entries.")
+ parser.add_argument("--no-manual", action="store_true", help="Exclude manual Markdown outside generated blocks.")
+ parser.add_argument(
+ "--bucket-max",
+ action="append",
+ default=None,
+ metavar="BUCKET=CHARS",
+ help="Per-bucket context character budget, such as skill=2000. Can be repeated.",
+ )
+ parser.add_argument(
+ "--format",
+ choices=("markdown", "system", "messages", "json", "summary"),
+ default=None,
+ help="Output format for context commands. --json is kept as a shortcut for --format json.",
+ )
+ parser.add_argument("--output", default=None, help="Optional file path to write the rendered context output.")
+ return parser
+
+
+def _write_init_harness_config(
+ result: FolderScaffoldResult,
+ args: argparse.Namespace,
+ *,
+ max_text_chars: int,
+) -> Path:
+ config_path = result.manifest_path.parent / HARNESS_CONFIG_NAME
+ if config_path.exists():
+ return config_path
+ config_path.write_text(
+ json.dumps(
+ default_harness_config(
+ exclude_patterns=args.exclude or (),
+ max_text_chars=max_text_chars,
+ ),
+ indent=2,
+ sort_keys=True,
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ return config_path
+
+
+def _context_section(args: argparse.Namespace) -> Mapping[str, Any]:
+ config = cast(Mapping[str, Any], getattr(args, "_harness_config", {}))
+ return config_section(config, "context", _context_config_path(args))
+
+
+def _context_config_path(args: argparse.Namespace) -> Path:
+ return cast(Path, getattr(args, "_harness_config_path", Path(".memu") / HARNESS_CONFIG_NAME))
+
+
+def _build_harness(args: argparse.Namespace) -> ContextHarness:
+ source_folder, repo_dir = _harness_paths(args)
+ config_path = harness_config_path(repo_dir, args.metadata_dir)
+ harness_config = load_harness_config(repo_dir, args.metadata_dir)
+ setattr(args, "_harness_config", harness_config)
+ setattr(args, "_harness_config_path", config_path)
+ compiler_section = config_section(harness_config, "compiler", config_path)
+ config = FolderMemoryCompilerConfig(
+ raw_data_dir_name=args.raw_data_dir,
+ metadata_dir_name=args.metadata_dir,
+ derived_dir_name=args.derived_dir,
+ exclude_patterns=tuple(compiler_exclude_patterns(args.exclude, compiler_section, config_path)),
+ max_text_chars=arg_or_config_positive_int(
+ args.max_text_chars,
+ compiler_section,
+ "max_text_chars",
+ DEFAULT_MAX_TEXT_CHARS,
+ flag_name="--max-text-chars",
+ config_path=config_path,
+ ),
+ use_memory_service=args.use_memory_service,
+ evolution_review=_evolution_review_config(args),
+ )
+ return ContextHarness(
+ source_folder,
+ repo_dir,
+ memory_service=_build_memory_service(args),
+ user=_parse_user_scope(args.user),
+ compiler_config=config,
+ )
+
+
+def _doctor_health(args: argparse.Namespace) -> FolderHealthResult:
+ config = FolderMemoryCompilerConfig(
+ raw_data_dir_name=args.raw_data_dir,
+ metadata_dir_name=args.metadata_dir,
+ derived_dir_name=args.derived_dir,
+ use_memory_service=False,
+ )
+ harness = ContextHarness(
+ Path(args.repo_dir) / args.raw_data_dir,
+ args.repo_dir,
+ compiler_config=config,
+ )
+ return harness.health()
+
+
+def _review_harness(args: argparse.Namespace) -> ContextHarness:
+ config = FolderMemoryCompilerConfig(
+ raw_data_dir_name=args.raw_data_dir,
+ metadata_dir_name=args.metadata_dir,
+ derived_dir_name=args.derived_dir,
+ use_memory_service=False,
+ )
+ repo_dir = Path(args.repo_dir)
+ return ContextHarness(
+ repo_dir / args.raw_data_dir,
+ repo_dir,
+ compiler_config=config,
+ )
+
+
+def _harness_paths(args: argparse.Namespace) -> tuple[str, str]:
+ if len(args.paths) == 1:
+ repo_dir = args.paths[0]
+ source_folder = str((Path(repo_dir) / args.raw_data_dir).resolve())
+ return source_folder, repo_dir
+ if len(args.paths) == 2:
+ return args.paths[0], args.paths[1]
+ msg = "commands expect either REPO_DIR or SOURCE_FOLDER REPO_DIR"
+ raise SystemExit(msg)
+
+
+def _context_kwargs(args: argparse.Namespace) -> dict[str, Any]:
+ context_section = _context_section(args)
+ config_path = _context_config_path(args)
+ bucket_values = (
+ list(args.bucket)
+ if args.bucket is not None
+ else config_context_buckets(context_section, config_path)
+ )
+ buckets = [cast(ContextBucket, bucket) for bucket in bucket_values] if bucket_values else None
+ bucket_char_limits = (
+ _parse_bucket_char_limits(args.bucket_max)
+ if args.bucket_max is not None
+ else config_bucket_char_limits(context_section, config_path)
+ )
+ return {
+ "query": args.query,
+ "buckets": buckets,
+ "max_chars": arg_or_config_positive_int(
+ args.max_chars,
+ context_section,
+ "max_chars",
+ DEFAULT_CONTEXT_MAX_CHARS,
+ flag_name="--max-chars",
+ config_path=config_path,
+ ),
+ "include_generated": not args.no_generated,
+ "include_manual": not args.no_manual,
+ "bucket_char_limits": bucket_char_limits,
+ }
+
+
+def _context_output_format(args: argparse.Namespace) -> str:
+ if args.json:
+ return "json"
+ if args.format is not None:
+ return str(args.format)
+ return config_context_format(_context_section(args), _context_config_path(args))
+
+
+def _print_context_output(pack: MarkdownContextPack, args: argparse.Namespace) -> None:
+ _emit_context_output(_render_context_output(pack, args), None)
+
+
+def _render_context_output(pack: MarkdownContextPack, args: argparse.Namespace) -> str:
+ return _render_pack(pack, _context_output_format(args))
+
+
+def _emit_context_output(text: str, output_path: str | None) -> None:
+ if output_path:
+ _write_output_text(text, output_path)
+ return
+ print(text, end="")
+
+
+def _run_summary(run: ContextHarnessRun) -> dict[str, Any]:
+ return {
+ "compile": _result_summary(run.compile_result),
+ "context": run.context_pack.to_dict(),
+ }
+
+
+def _trace_summary(result: ContextHarnessSkillTraceResult) -> dict[str, Any]:
+ return {
+ "raw_data_dir": str(result.record.raw_data_dir),
+ "trace_path": str(result.record.trace_path),
+ "task": result.record.trace.task,
+ "outcome": result.record.trace.outcome,
+ "compiled": _result_summary(result.compile_result) if result.compile_result is not None else None,
+ }
+
+
+def _skill_evolution_summary(result: ContextHarnessSkillEvolutionResult) -> dict[str, Any]:
+ return {
+ "proposal_count": len(result.proposals),
+ "promoted_count": len(result.promotions),
+ "proposals": [_proposal_summary(proposal) for proposal in result.proposals],
+ "promotions": [_promotion_summary(promotion) for promotion in result.promotions],
+ }
+
+
+def _proposal_summary(proposal: SkillEvolutionProposal) -> dict[str, Any]:
+ return proposal.to_dict()
+
+
+def _scaffold_summary(result: FolderScaffoldResult, config_path: Path | None = None) -> dict[str, Any]:
+ return {
+ "output_dir": str(result.output_dir),
+ "raw_data_dir": str(result.raw_data_dir),
+ "manifest_path": str(result.manifest_path),
+ "config_path": str(config_path) if config_path is not None else None,
+ "created": result.created,
+ "copied": result.copied,
+ }
+
+
+def _print_scaffold_summary(result: FolderScaffoldResult, config_path: Path | None = None) -> None:
+ summary = _scaffold_summary(result, config_path)
+ print("memU harness repository initialized")
+ print(f" output: {summary['output_dir']}")
+ print(f" raw_data: {summary['raw_data_dir']}")
+ print(f" manifest: {summary['manifest_path']}")
+ if summary["config_path"]:
+ print(f" config: {summary['config_path']}")
+ print(f" created: {len(result.created)}")
+ print(f" copied: {len(result.copied)}")
+
+
+def _print_status_summary(result: FolderStatusResult) -> None:
+ counts = result.to_dict()["counts"]
+ print("memU harness status")
+ print(f" source: {result.source_dir}")
+ print(f" output: {result.output_dir}")
+ print(f" new: {counts['new']}")
+ print(f" changed: {counts['changed']}")
+ print(f" removed: {counts['removed']}")
+ print(f" unchanged: {counts['unchanged']}")
+ for state in ("new", "changed", "removed"):
+ paths = getattr(result, state)
+ if paths:
+ print(f" {state}: {', '.join(paths)}")
+
+
+def _print_health_summary(result: FolderHealthResult) -> None:
+ print("memU harness health")
+ print(f" output: {result.output_dir}")
+ print(f" ok: {result.ok}")
+ print(f" errors: {result.error_count}")
+ print(f" warnings: {result.warning_count}")
+ for issue in result.issues:
+ location = f" [{issue.path}]" if issue.path else ""
+ print(f" - {issue.severity}: {issue.code}{location} - {issue.message}")
+
+
+def _promotion_summary(result: SkillPromotionRecord) -> dict[str, Any]:
+ return {
+ "repo_dir": str(result.repo_dir),
+ "skill_path": str(result.skill_path),
+ "card_path": str(result.card_path) if result.card_path is not None else None,
+ "title": result.title,
+ "promoted_at": result.promoted_at,
+ }
+
+
+def _evolution_review_summary(result: EvolutionReviewApplyResult) -> dict[str, Any]:
+ return {
+ "output_dir": str(result.output_dir),
+ "manifest_path": str(result.manifest_path),
+ "reviewed_count": len(result.reviewed),
+ "applied_proposal_ids": list(result.applied_proposal_ids),
+ "removed": list(result.removed),
+ "entry_count": len(result.entries),
+ "reviews": [review.to_dict() for review in result.reviewed],
+ }
+
+
+def _print_promotion_summary(result: SkillPromotionRecord) -> None:
+ summary = _promotion_summary(result)
+ print("memU harness skill promoted")
+ print(f" skill: {summary['title']}")
+ print(f" path: {summary['skill_path']}")
+ print(f" promoted_at: {summary['promoted_at']}")
+
+
+def _print_evolution_review_summary(result: EvolutionReviewApplyResult) -> None:
+ summary = _evolution_review_summary(result)
+ print("memU harness evolution review applied")
+ print(f" manifest: {summary['manifest_path']}")
+ print(f" reviewed: {summary['reviewed_count']}")
+ print(f" applied: {len(result.applied_proposal_ids)}")
+ print(f" removed: {len(result.removed)}")
+ print(f" entries: {summary['entry_count']}")
+
+
+def _print_skill_evolution_summary(result: ContextHarnessSkillEvolutionResult) -> None:
+ print("memU harness skill suggestions")
+ print(f" proposals: {len(result.proposals)}")
+ print(f" promoted: {len(result.promotions)}")
+ for proposal in result.proposals:
+ print(f" - {proposal.title} (support={proposal.support_count}, score={proposal.score})")
+ if proposal.lessons:
+ print(f" lesson: {proposal.lessons[0]}")
+ if proposal.sources:
+ print(f" sources: {', '.join(proposal.sources)}")
+
+
+def _print_trace_summary(result: ContextHarnessSkillTraceResult) -> None:
+ summary = _trace_summary(result)
+ print("memU harness skill trace recorded")
+ print(f" trace: {summary['trace_path']}")
+ print(f" task: {summary['task']}")
+ print(f" outcome: {summary['outcome']}")
+ if summary["compiled"] is not None:
+ print(f" compiled entries: {summary['compiled']['entry_count']}")
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/src/memu/app/crud.py b/src/memu/app/crud.py
index 50d63d4c..47d96ef8 100644
--- a/src/memu/app/crud.py
+++ b/src/memu/app/crud.py
@@ -8,6 +8,7 @@
from pydantic import BaseModel
+from memu.app.scope import concrete_scope_from_where, normalize_scope_where, record_matches_scope
from memu.database.models import MemoryCategory, MemoryType
from memu.prompts.category_patch import CATEGORY_PATCH_PROMPT
from memu.workflow.step import WorkflowState, WorkflowStep
@@ -63,6 +64,9 @@ async def list_memory_categories(
ctx = self._get_context()
store = self._get_database()
where_filters = self._normalize_where(where)
+ bootstrap_scope = concrete_scope_from_where(where_filters)
+ if bootstrap_scope is not None:
+ await self._ensure_categories_ready(ctx, store, bootstrap_scope)
state: WorkflowState = {
"ctx": ctx,
@@ -149,6 +153,14 @@ def _build_list_memory_categories_workflow(self) -> list[WorkflowStep]:
def _build_clear_memory_workflow(self) -> list[WorkflowStep]:
steps = [
+ WorkflowStep(
+ step_id="clear_category_item_relations",
+ role="delete_memories",
+ handler=self._crud_clear_category_item_relations,
+ requires={"ctx", "store", "where"},
+ produces={"deleted_relations"},
+ capabilities={"db"},
+ ),
WorkflowStep(
step_id="clear_memory_categories",
role="delete_memories",
@@ -177,7 +189,14 @@ def _build_clear_memory_workflow(self) -> list[WorkflowStep]:
step_id="build_response",
role="emit",
handler=self._crud_build_clear_memory_response,
- requires={"ctx", "store", "deleted_categories", "deleted_items", "deleted_resources"},
+ requires={
+ "ctx",
+ "store",
+ "deleted_relations",
+ "deleted_categories",
+ "deleted_items",
+ "deleted_resources",
+ },
produces={"response"},
capabilities=set(),
),
@@ -194,22 +213,14 @@ def _list_clear_memories_initial_keys() -> set[str]:
def _normalize_where(self, where: Mapping[str, Any] | None) -> dict[str, Any]:
"""Validate and clean the `where` scope filters against the configured user model."""
- if not where:
- return {}
-
- valid_fields = set(getattr(self.user_model, "model_fields", {}).keys())
- cleaned: dict[str, Any] = {}
-
- for raw_key, value in where.items():
- if value is None:
- continue
- field = raw_key.split("__", 1)[0]
- if field not in valid_fields:
- msg = f"Unknown filter field '{field}' for current user scope"
- raise ValueError(msg)
- cleaned[raw_key] = value
+ return normalize_scope_where(self.user_model, where)
- return cleaned
+ @staticmethod
+ def _ensure_item_matches_user_scope(item: Any, user_scope: Mapping[str, Any] | None, memory_id: str) -> None:
+ if record_matches_scope(item, user_scope):
+ return
+ msg = f"Memory item with id {memory_id} not found"
+ raise ValueError(msg)
def _crud_list_memory_items(self, state: WorkflowState, step_context: Any) -> WorkflowState:
where_filters = state.get("where") or {}
@@ -243,6 +254,13 @@ def _crud_build_list_categories_response(self, state: WorkflowState, step_contex
state["response"] = response
return state
+ def _crud_clear_category_item_relations(self, state: WorkflowState, step_context: Any) -> WorkflowState:
+ where_filters = state.get("where") or {}
+ store = state["store"]
+ deleted = store.category_item_repo.clear_relations(where_filters)
+ state["deleted_relations"] = deleted
+ return state
+
def _crud_clear_memory_categories(self, state: WorkflowState, step_context: Any) -> WorkflowState:
where_filters = state.get("where") or {}
store = state["store"]
@@ -265,10 +283,12 @@ def _crud_clear_memory_resources(self, state: WorkflowState, step_context: Any)
return state
def _crud_build_clear_memory_response(self, state: WorkflowState, step_context: Any) -> WorkflowState:
+ deleted_relations = state.get("deleted_relations", [])
deleted_categories = state.get("deleted_categories", {})
deleted_items = state.get("deleted_items", {})
deleted_resources = state.get("deleted_resources", {})
response = {
+ "deleted_relations": [self._model_dump_without_embeddings(rel) for rel in deleted_relations],
"deleted_categories": [self._model_dump_without_embeddings(cat) for cat in deleted_categories.values()],
"deleted_items": [self._model_dump_without_embeddings(item) for item in deleted_items.values()],
"deleted_resources": [self._model_dump_without_embeddings(res) for res in deleted_resources.values()],
@@ -285,9 +305,9 @@ async def create_memory_item(
user: dict[str, Any] | None = None,
propagate: bool = True,
) -> dict[str, Any]:
- if memory_type not in get_args(MemoryType):
- msg = f"Invalid memory type: '{memory_type}', must be one of {get_args(MemoryType)}"
- raise ValueError(msg)
+ memory_type = _normalize_memory_type(memory_type)
+ memory_content = _normalize_memory_content(memory_content, field_name="memory_content")
+ memory_categories = _normalize_memory_categories(memory_categories, field_name="memory_categories")
ctx = self._get_context()
store = self._get_database()
@@ -327,9 +347,13 @@ async def update_memory_item(
if all((memory_type is None, memory_content is None, memory_categories is None)):
msg = "At least one of memory type, memory content, or memory categories is required for UPDATE operation"
raise ValueError(msg)
- if memory_type and memory_type not in get_args(MemoryType):
- msg = f"Invalid memory type: '{memory_type}', must be one of {get_args(MemoryType)}"
- raise ValueError(msg)
+ memory_id = _normalize_memory_id(memory_id)
+ if memory_type is not None:
+ memory_type = _normalize_memory_type(memory_type)
+ if memory_content is not None:
+ memory_content = _normalize_memory_content(memory_content, field_name="memory_content")
+ if memory_categories is not None:
+ memory_categories = _normalize_memory_categories(memory_categories, field_name="memory_categories")
ctx = self._get_context()
store = self._get_database()
@@ -364,6 +388,8 @@ async def delete_memory_item(
user: dict[str, Any] | None = None,
propagate: bool = True,
) -> dict[str, Any]:
+ memory_id = _normalize_memory_id(memory_id)
+
ctx = self._get_context()
store = self._get_database()
user_scope = self.user_model(**user).model_dump() if user is not None else None
@@ -517,6 +543,7 @@ async def _patch_create_memory_item(self, state: WorkflowState, step_context: An
content_embedding = (await self._get_step_embedding_client(step_context).embed(embed_payload))[0]
item = store.memory_item_repo.create_item(
+ resource_id=None,
memory_type=memory_payload["type"],
summary=memory_payload["content"],
embedding=content_embedding,
@@ -548,6 +575,7 @@ async def _patch_update_memory_item(self, state: WorkflowState, step_context: An
if not item:
msg = f"Memory item with id {memory_id} not found"
raise ValueError(msg)
+ self._ensure_item_matches_user_scope(item, user, memory_id)
old_content = item.summary
old_item_categories = store.category_item_repo.get_item_categories(memory_id)
mapped_old_cat_ids = [cat.category_id for cat in old_item_categories]
@@ -566,7 +594,10 @@ async def _patch_update_memory_item(self, state: WorkflowState, step_context: An
embedding=content_embedding,
)
new_cat_names = memory_payload["categories"]
- mapped_new_cat_ids = self._map_category_names_to_ids(new_cat_names, ctx)
+ if new_cat_names is None:
+ mapped_new_cat_ids = mapped_old_cat_ids
+ else:
+ mapped_new_cat_ids = self._map_category_names_to_ids(new_cat_names, ctx)
cats_to_remove = set(mapped_old_cat_ids) - set(mapped_new_cat_ids)
cats_to_add = set(mapped_new_cat_ids) - set(mapped_old_cat_ids)
@@ -599,7 +630,8 @@ async def _patch_delete_memory_item(self, state: WorkflowState, step_context: An
if not item:
msg = f"Memory item with id {memory_id} not found"
raise ValueError(msg)
- item_categories = store.category_item_repo.get_item_categories(memory_id)
+ self._ensure_item_matches_user_scope(item, state["user"], memory_id)
+ item_categories = store.category_item_repo.clear_relations({"item_id": memory_id})
if propagate:
for cat in item_categories:
category_memory_updates[cat.category_id] = (item.summary, None)
@@ -724,3 +756,36 @@ def _parse_category_patch_response(self, response: str) -> tuple[bool, str]:
if updated_content == "empty":
updated_content = ""
return need_update, updated_content
+
+
+def _normalize_memory_id(value: Any) -> str:
+ return _normalize_non_empty_string(value, field_name="memory_id")
+
+
+def _normalize_memory_type(value: Any) -> MemoryType:
+ memory_type = _normalize_non_empty_string(value, field_name="memory_type")
+ if memory_type not in get_args(MemoryType):
+ msg = f"Invalid memory type: '{memory_type}', must be one of {get_args(MemoryType)}"
+ raise ValueError(msg)
+ return cast(MemoryType, memory_type)
+
+
+def _normalize_memory_content(value: Any, *, field_name: str) -> str:
+ return _normalize_non_empty_string(value, field_name=field_name)
+
+
+def _normalize_memory_categories(value: Any, *, field_name: str) -> list[str]:
+ if not isinstance(value, list):
+ msg = f"'{field_name}' must be a list of non-empty strings"
+ raise ValueError(msg)
+ normalized: list[str] = []
+ for index, item in enumerate(value):
+ normalized.append(_normalize_non_empty_string(item, field_name=f"{field_name}[{index}]"))
+ return normalized
+
+
+def _normalize_non_empty_string(value: Any, *, field_name: str) -> str:
+ if not isinstance(value, str) or not value.strip():
+ msg = f"'{field_name}' must be a non-empty string"
+ raise ValueError(msg)
+ return value.strip()
diff --git a/src/memu/app/folder.py b/src/memu/app/folder.py
new file mode 100644
index 00000000..05c2a17f
--- /dev/null
+++ b/src/memu/app/folder.py
@@ -0,0 +1,2256 @@
+from __future__ import annotations
+
+import asyncio
+import fnmatch
+import hashlib
+import inspect
+import json
+import shutil
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal, cast
+
+from memu.app.self_evolve import (
+ EvolutionInstruction,
+ EvolutionReviewBundle,
+ EvolutionReviewConfig,
+ PatchProposal,
+ ReviewDecision,
+ ReviewStatus,
+ SelfEvolveEngine,
+ apply_reviewed_proposals,
+ write_evolution_audit,
+)
+
+if TYPE_CHECKING:
+ from memu.app.service import MemoryService
+
+
+MemoryBucket = Literal["memory", "soul", "skill"]
+FolderHealthSeverity = Literal["error", "warning"]
+FolderSourceState = Literal["new", "changed", "unchanged", "removed"]
+FolderWatchReason = Literal["initial", "changed"]
+
+GENERATED_START = ""
+GENERATED_END = ""
+
+
+@dataclass(frozen=True)
+class FolderMemoryCompilerConfig:
+ """Configuration for compiling a raw data folder into Markdown memory files."""
+
+ raw_data_dir_name: str = "raw_data"
+ metadata_dir_name: str = ".memu"
+ derived_dir_name: str = "derived"
+ agent_instructions_name: str = "AGENTS.md"
+ ignore_file_name: str = ".memuignore"
+ write_agent_instructions: bool = True
+ exclude_patterns: tuple[str, ...] = ()
+ max_text_chars: int = 4000
+ use_memory_service: bool = True
+ self_evolve_enabled: bool = True
+ evolution_review: EvolutionReviewConfig = field(default_factory=EvolutionReviewConfig)
+
+
+@dataclass(frozen=True)
+class MarkdownMemoryEntry:
+ """A generated memory/soul/skill entry ready to be written as Markdown."""
+
+ id: str
+ bucket: MemoryBucket
+ title: str
+ body: str
+ source: str
+ evidence: str
+ modality: str
+ confidence: str = "medium"
+ tags: list[str] = field(default_factory=list)
+ updated_at: str = field(default_factory=lambda: _utc_now())
+
+ def to_manifest(self) -> dict[str, Any]:
+ return {
+ "id": self.id,
+ "bucket": self.bucket,
+ "title": self.title,
+ "body": self.body,
+ "source": self.source,
+ "evidence": self.evidence,
+ "modality": self.modality,
+ "confidence": self.confidence,
+ "tags": list(self.tags),
+ "updated_at": self.updated_at,
+ }
+
+ @classmethod
+ def from_manifest(cls, data: Mapping[str, Any]) -> MarkdownMemoryEntry:
+ return cls(
+ id=str(data["id"]),
+ bucket=cast(MemoryBucket, data["bucket"]),
+ title=str(data["title"]),
+ body=str(data["body"]),
+ source=str(data["source"]),
+ evidence=str(data["evidence"]),
+ modality=str(data["modality"]),
+ confidence=str(data.get("confidence", "medium")),
+ tags=[str(tag) for tag in data.get("tags", [])],
+ updated_at=str(data.get("updated_at") or _utc_now()),
+ )
+
+ def to_markdown(self) -> str:
+ tags = ", ".join(self.tags) if self.tags else "generated"
+ return (
+ f"### {self.id}: {self.title}\n\n"
+ f"- bucket: {self.bucket}\n"
+ f"- modality: {self.modality}\n"
+ f"- confidence: {self.confidence}\n"
+ f"- source: {self.source}\n"
+ f"- evidence: {self.evidence}\n"
+ f"- tags: {tags}\n"
+ f"- updated_at: {self.updated_at}\n\n"
+ f"{self.body.strip()}\n"
+ )
+
+
+@dataclass(frozen=True)
+class FolderCompileResult:
+ """Result returned by FolderMemoryCompiler.compile."""
+
+ output_dir: Path
+ raw_data_dir: Path
+ manifest_path: Path
+ processed: list[str]
+ skipped: list[str]
+ removed: list[str]
+ entries: list[MarkdownMemoryEntry]
+ evolution_instructions: list[EvolutionInstruction] = field(default_factory=list)
+ patch_proposals: list[PatchProposal] = field(default_factory=list)
+ review_decisions: list[ReviewDecision] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class EvolutionReviewApplyResult:
+ """Result returned after creator review decisions are applied to pending proposals."""
+
+ output_dir: Path
+ manifest_path: Path
+ reviewed: list[ReviewDecision]
+ applied_proposal_ids: list[str]
+ removed: list[str]
+ entries: list[MarkdownMemoryEntry]
+
+
+@dataclass(frozen=True)
+class FolderScaffoldResult:
+ """Result returned when a Markdown memory repository layout is scaffolded."""
+
+ output_dir: Path
+ raw_data_dir: Path
+ manifest_path: Path
+ created: list[str]
+ copied: list[str]
+
+
+@dataclass(frozen=True)
+class FolderSourceStatus:
+ """Status of one source file compared with the latest manifest."""
+
+ path: str
+ state: FolderSourceState
+ modality: str | None = None
+ sha256: str | None = None
+ previous_sha256: str | None = None
+ raw_path: str | None = None
+ evidence: str | None = None
+ sidecars: list[str] = field(default_factory=list)
+ entry_count: int = 0
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "path": self.path,
+ "state": self.state,
+ "modality": self.modality,
+ "sha256": self.sha256,
+ "previous_sha256": self.previous_sha256,
+ "raw_path": self.raw_path,
+ "evidence": self.evidence,
+ "sidecars": list(self.sidecars),
+ "entry_count": self.entry_count,
+ }
+
+
+@dataclass(frozen=True)
+class FolderStatusResult:
+ """Non-mutating status report for a Markdown memory repository."""
+
+ source_dir: Path
+ output_dir: Path
+ manifest_path: Path
+ sources: list[FolderSourceStatus]
+
+ @property
+ def new(self) -> list[str]:
+ return self._paths_for("new")
+
+ @property
+ def changed(self) -> list[str]:
+ return self._paths_for("changed")
+
+ @property
+ def unchanged(self) -> list[str]:
+ return self._paths_for("unchanged")
+
+ @property
+ def removed(self) -> list[str]:
+ return self._paths_for("removed")
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "source_dir": str(self.source_dir),
+ "output_dir": str(self.output_dir),
+ "manifest_path": str(self.manifest_path),
+ "counts": {
+ "new": len(self.new),
+ "changed": len(self.changed),
+ "unchanged": len(self.unchanged),
+ "removed": len(self.removed),
+ },
+ "new": self.new,
+ "changed": self.changed,
+ "unchanged": self.unchanged,
+ "removed": self.removed,
+ "sources": [source.to_dict() for source in self.sources],
+ }
+
+ def _paths_for(self, state: FolderSourceState) -> list[str]:
+ return [source.path for source in self.sources if source.state == state]
+
+
+@dataclass(frozen=True)
+class FolderHealthIssue:
+ """One validation issue found in a Markdown memory repository."""
+
+ severity: FolderHealthSeverity
+ code: str
+ message: str
+ path: str | None = None
+ details: dict[str, Any] = field(default_factory=dict)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "severity": self.severity,
+ "code": self.code,
+ "message": self.message,
+ "path": self.path,
+ "details": dict(self.details),
+ }
+
+
+@dataclass(frozen=True)
+class FolderHealthResult:
+ """Non-mutating health report for a Markdown memory repository."""
+
+ output_dir: Path
+ issues: list[FolderHealthIssue]
+
+ @property
+ def ok(self) -> bool:
+ return not any(issue.severity == "error" for issue in self.issues)
+
+ @property
+ def error_count(self) -> int:
+ return sum(1 for issue in self.issues if issue.severity == "error")
+
+ @property
+ def warning_count(self) -> int:
+ return sum(1 for issue in self.issues if issue.severity == "warning")
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "output_dir": str(self.output_dir),
+ "ok": self.ok,
+ "counts": {
+ "errors": self.error_count,
+ "warnings": self.warning_count,
+ },
+ "issues": [issue.to_dict() for issue in self.issues],
+ }
+
+
+@dataclass(frozen=True)
+class FolderWatchEvent:
+ """A compile event emitted by the folder watcher."""
+
+ iteration: int
+ reason: FolderWatchReason
+ result: FolderCompileResult
+ status: FolderStatusResult | None = None
+
+
+@dataclass(frozen=True)
+class _SourceSnapshot:
+ absolute_path: Path
+ rel_path: str
+ sha256: str
+ size: int
+ modality: str
+ raw_rel_path: str
+ evidence_rel_path: str
+ sidecar_paths: tuple[Path, ...] = ()
+ sidecar_rel_paths: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class _ServiceExtraction:
+ entries: list[MarkdownMemoryEntry]
+ evidence_text: str
+
+
+@dataclass(frozen=True)
+class _SourceExtraction:
+ entries: list[MarkdownMemoryEntry]
+ instructions: list[EvolutionInstruction]
+ proposals: list[PatchProposal]
+ reviews: list[ReviewDecision]
+
+
+class FolderMemoryCompiler:
+ """Compile a folder of multimodal raw data into memory.md, soul.md, and skill.md.
+
+ The compiler is intentionally Markdown-backed: raw files are copied into raw_data/,
+ derived textual evidence is cached in .memu/derived/, and generated memory entries
+ are written into stable generated blocks so human edits outside the block survive
+ subsequent compiles.
+ """
+
+ def __init__(
+ self,
+ memory_service: MemoryService | None = None,
+ config: FolderMemoryCompilerConfig | None = None,
+ ) -> None:
+ self.memory_service = memory_service
+ self.config = config or FolderMemoryCompilerConfig()
+ self.self_evolve = SelfEvolveEngine(self.config.evolution_review)
+
+ def scaffold(
+ self,
+ output_folder: str | Path,
+ *,
+ source_folder: str | Path | None = None,
+ ) -> FolderScaffoldResult:
+ """Create the Markdown memory repository layout without extracting memories."""
+
+ output_root = Path(output_folder).resolve()
+ created: list[str] = []
+ paths = self._ensure_scaffold_layout(output_root, created)
+ self._write_scaffold_markdown_files(output_root, created)
+ self._write_agent_instructions(output_root, created)
+ self._write_scaffold_manifest(paths["manifest"], output_root, created)
+ copied = self._copy_scaffold_source(source_folder, paths["raw_data"], output_root) if source_folder else []
+ return FolderScaffoldResult(
+ output_dir=output_root,
+ raw_data_dir=paths["raw_data"],
+ manifest_path=paths["manifest"],
+ created=created,
+ copied=copied,
+ )
+
+ def status(
+ self,
+ source_folder: str | Path,
+ output_folder: str | Path,
+ ) -> FolderStatusResult:
+ """Inspect source changes against the manifest without writing files."""
+
+ source_root = Path(source_folder).resolve()
+ output_root = Path(output_folder).resolve()
+ if not source_root.exists() or not source_root.is_dir():
+ msg = f"source_folder must be an existing directory: {source_root}"
+ raise ValueError(msg)
+ source_root = self._resolve_source_root(source_root, output_root)
+
+ manifest_path = output_root / self.config.metadata_dir_name / "manifest.json"
+ manifest = self._load_manifest(manifest_path)
+ old_sources = cast(dict[str, Any], manifest.get("sources", {}))
+ snapshots = self._scan_sources(source_root, output_root)
+ current_rel_paths = {snapshot.rel_path for snapshot in snapshots}
+ statuses = [self._status_from_snapshot(snapshot, old_sources.get(snapshot.rel_path)) for snapshot in snapshots]
+
+ for rel_path in sorted(set(old_sources) - current_rel_paths):
+ previous = old_sources.get(rel_path)
+ if isinstance(previous, Mapping):
+ statuses.append(self._removed_status(rel_path, previous))
+ else:
+ statuses.append(FolderSourceStatus(path=rel_path, state="removed"))
+
+ return FolderStatusResult(
+ source_dir=source_root,
+ output_dir=output_root,
+ manifest_path=manifest_path,
+ sources=sorted(statuses, key=lambda source: (self._status_sort_key(source.state), source.path)),
+ )
+
+ def health(self, output_folder: str | Path) -> FolderHealthResult:
+ """Validate a Markdown memory repository without writing files."""
+
+ output_root = Path(output_folder).resolve()
+ issues: list[FolderHealthIssue] = []
+ self._check_layout(output_root, issues)
+ manifest = self._read_manifest_for_health(output_root, issues)
+ if manifest is not None:
+ self._check_manifest_sources(output_root, manifest, issues)
+ self._check_orphan_evidence(output_root, manifest, issues)
+ self._check_generated_blocks(output_root, issues)
+ return FolderHealthResult(output_dir=output_root, issues=issues)
+
+ async def compile(
+ self,
+ source_folder: str | Path,
+ output_folder: str | Path,
+ *,
+ user: Mapping[str, Any] | None = None,
+ ) -> FolderCompileResult:
+ source_root = Path(source_folder).resolve()
+ output_root = Path(output_folder).resolve()
+ if not source_root.exists() or not source_root.is_dir():
+ msg = f"source_folder must be an existing directory: {source_root}"
+ raise ValueError(msg)
+
+ paths = self._ensure_output_layout(output_root)
+ created: list[str] = []
+ self._write_agent_instructions(output_root, created)
+ source_root = self._resolve_source_root(source_root, output_root)
+ manifest = self._load_manifest(paths["manifest"])
+ old_sources = cast(dict[str, Any], manifest.get("sources", {}))
+ snapshots = self._scan_sources(source_root, output_root)
+ current_rel_paths = {snapshot.rel_path for snapshot in snapshots}
+ processed: list[str] = []
+ skipped: list[str] = []
+ evolution_instructions: list[EvolutionInstruction] = []
+ patch_proposals: list[PatchProposal] = []
+ review_decisions: list[ReviewDecision] = []
+
+ for snapshot in snapshots:
+ previous = old_sources.get(snapshot.rel_path)
+ unchanged = isinstance(previous, Mapping) and previous.get("sha256") == snapshot.sha256
+ self._sync_raw_file(snapshot, paths["raw_data"])
+ if unchanged and previous.get("entries"):
+ skipped.append(snapshot.rel_path)
+ continue
+ extraction = await self._extract_entries(snapshot, output_root, previous=previous, user=user)
+ evolution_instructions.extend(extraction.instructions)
+ patch_proposals.extend(extraction.proposals)
+ review_decisions.extend(extraction.reviews)
+ old_sources[snapshot.rel_path] = {
+ "path": snapshot.rel_path,
+ "raw_path": snapshot.raw_rel_path,
+ "hash": snapshot.sha256,
+ "sha256": snapshot.sha256,
+ "size": snapshot.size,
+ "modality": snapshot.modality,
+ "evidence": snapshot.evidence_rel_path,
+ "sidecars": list(snapshot.sidecar_rel_paths),
+ "entries": [entry.to_manifest() for entry in extraction.entries],
+ "evolution": {
+ "instructions": [instruction.to_dict() for instruction in extraction.instructions],
+ "patch_proposals": [proposal.to_dict() for proposal in extraction.proposals],
+ "review_decisions": [review.to_dict() for review in extraction.reviews],
+ },
+ "last_extracted_at": _utc_now(),
+ }
+ processed.append(snapshot.rel_path)
+
+ removed = sorted(set(old_sources) - current_rel_paths)
+ for rel_path in removed:
+ previous = old_sources.get(rel_path)
+ if self.config.self_evolve_enabled and isinstance(previous, Mapping):
+ bundle = self._evolve_removed_source(rel_path, previous)
+ evolution_instructions.extend(bundle.instructions)
+ patch_proposals.extend(bundle.proposals)
+ review_decisions.extend(bundle.reviews)
+ previous_entries = self._source_entries_from_manifest(previous)
+ remaining_entries = [
+ MarkdownMemoryEntry.from_manifest(entry)
+ for entry in apply_reviewed_proposals(
+ [entry.to_manifest() for entry in previous_entries],
+ bundle.proposals,
+ bundle.reviews,
+ )
+ ]
+ if remaining_entries:
+ updated = dict(previous)
+ updated["entries"] = [entry.to_manifest() for entry in remaining_entries]
+ updated["pending_removal"] = True
+ updated["removed_from_source_at"] = _utc_now()
+ updated["evolution"] = bundle.to_dict()
+ old_sources[rel_path] = updated
+ continue
+ old_sources.pop(rel_path, None)
+
+ manifest["version"] = 1
+ manifest["updated_at"] = _utc_now()
+ manifest["sources"] = {key: old_sources[key] for key in sorted(old_sources)}
+ self._write_manifest(paths["manifest"], manifest)
+ self._remove_stale_raw_files(paths["raw_data"], self._expected_raw_rel_paths_from_sources(manifest["sources"]))
+ self._remove_stale_evidence_files(
+ paths["derived"],
+ self._expected_evidence_rel_paths_from_sources(manifest["sources"]),
+ )
+ write_evolution_audit(
+ paths["metadata"],
+ instructions=evolution_instructions,
+ proposals=patch_proposals,
+ reviews=review_decisions,
+ )
+
+ entries = self._entries_from_manifest(manifest)
+ self._write_markdown_repository(output_root, entries)
+
+ return FolderCompileResult(
+ output_dir=output_root,
+ raw_data_dir=paths["raw_data"],
+ manifest_path=paths["manifest"],
+ processed=processed,
+ skipped=skipped,
+ removed=removed,
+ entries=entries,
+ evolution_instructions=evolution_instructions,
+ patch_proposals=patch_proposals,
+ review_decisions=review_decisions,
+ )
+
+ def review_evolution(
+ self,
+ output_folder: str | Path,
+ *,
+ proposal_ids: Sequence[str] | None = None,
+ reviewer: str = "creator",
+ decision: ReviewStatus = "approved",
+ reason: str = "",
+ ) -> EvolutionReviewApplyResult:
+ """Apply creator review decisions to pending self-evolve patch proposals."""
+
+ if decision not in {"approved", "rejected"}:
+ msg = "decision must be 'approved' or 'rejected'"
+ raise ValueError(msg)
+ output_root = Path(output_folder).resolve()
+ manifest_path = output_root / self.config.metadata_dir_name / "manifest.json"
+ manifest = self._load_manifest(manifest_path)
+ sources = cast(dict[str, Any], manifest.get("sources", {}))
+ selected_ids = {proposal_id for proposal_id in proposal_ids or [] if proposal_id}
+ reviewed: list[ReviewDecision] = []
+ applied_proposal_ids: list[str] = []
+ removed: list[str] = []
+
+ for rel_path, source in list(sources.items()):
+ if not isinstance(source, Mapping):
+ continue
+ evolution = source.get("evolution", {})
+ if not isinstance(evolution, Mapping):
+ continue
+ proposals = [
+ PatchProposal.from_dict(proposal)
+ for proposal in evolution.get("patch_proposals", [])
+ if isinstance(proposal, Mapping)
+ ]
+ if not proposals:
+ continue
+ reviews = [
+ ReviewDecision.from_dict(review)
+ for review in evolution.get("review_decisions", [])
+ if isinstance(review, Mapping)
+ ]
+ latest_reviews = {review.proposal_id: review for review in reviews}
+ new_reviews: list[ReviewDecision] = []
+ for proposal in proposals:
+ if selected_ids and proposal.id not in selected_ids:
+ continue
+ latest = latest_reviews.get(proposal.id)
+ if latest is None or latest.status != "needs_review":
+ continue
+ review = ReviewDecision(
+ proposal_id=proposal.id,
+ status=decision,
+ reviewer=reviewer,
+ reason=reason or f"Creator marked proposal as {decision}.",
+ confidence=proposal.confidence,
+ safety_flags=list(latest.safety_flags),
+ )
+ new_reviews.append(review)
+ reviewed.append(review)
+ if decision == "approved":
+ applied_proposal_ids.append(proposal.id)
+
+ if not new_reviews:
+ continue
+ combined_reviews = [*reviews, *new_reviews]
+ updated_source = dict(source)
+ updated_evolution = dict(evolution)
+ updated_evolution["review_decisions"] = [review.to_dict() for review in combined_reviews]
+ updated_source["evolution"] = updated_evolution
+ updated_entries = apply_reviewed_proposals(
+ self._entry_manifests_from_source(source),
+ proposals,
+ combined_reviews,
+ )
+ updated_source["entries"] = updated_entries
+ if updated_source.get("pending_removal") and not updated_entries:
+ removed.append(str(rel_path))
+ sources.pop(rel_path, None)
+ else:
+ sources[rel_path] = updated_source
+
+ manifest["version"] = 1
+ manifest["updated_at"] = _utc_now()
+ manifest["sources"] = {key: sources[key] for key in sorted(sources)}
+ self._write_manifest(manifest_path, manifest)
+ paths = self._ensure_output_layout(output_root)
+ self._remove_stale_raw_files(paths["raw_data"], self._expected_raw_rel_paths_from_sources(manifest["sources"]))
+ self._remove_stale_evidence_files(
+ paths["derived"],
+ self._expected_evidence_rel_paths_from_sources(manifest["sources"]),
+ )
+ write_evolution_audit(paths["metadata"], instructions=[], proposals=[], reviews=reviewed)
+ entries = self._entries_from_manifest(manifest)
+ self._write_markdown_repository(output_root, entries)
+ return EvolutionReviewApplyResult(
+ output_dir=output_root,
+ manifest_path=manifest_path,
+ reviewed=reviewed,
+ applied_proposal_ids=applied_proposal_ids,
+ removed=removed,
+ entries=entries,
+ )
+
+ def _ensure_output_layout(self, output_root: Path) -> dict[str, Path]:
+ raw_data_dir = output_root / self.config.raw_data_dir_name
+ metadata_dir = output_root / self.config.metadata_dir_name
+ derived_dir = metadata_dir / self.config.derived_dir_name
+ manifest_path = metadata_dir / "manifest.json"
+ for path in (
+ output_root,
+ raw_data_dir,
+ metadata_dir,
+ derived_dir,
+ output_root / "memory",
+ output_root / "soul",
+ output_root / "skill",
+ ):
+ path.mkdir(parents=True, exist_ok=True)
+ return {
+ "raw_data": raw_data_dir,
+ "metadata": metadata_dir,
+ "derived": derived_dir,
+ "manifest": manifest_path,
+ }
+
+ def _check_layout(self, output_root: Path, issues: list[FolderHealthIssue]) -> None:
+ required_dirs = [
+ self.config.raw_data_dir_name,
+ self.config.metadata_dir_name,
+ f"{self.config.metadata_dir_name}/{self.config.derived_dir_name}",
+ "memory",
+ "soul",
+ "skill",
+ ]
+ required_files = [
+ "memory.md",
+ "soul.md",
+ "skill.md",
+ f"{self.config.metadata_dir_name}/manifest.json",
+ ]
+ if not output_root.exists():
+ self._health_issue(issues, "error", "missing_output_dir", "Repository directory is missing.", ".")
+ return
+ if not output_root.is_dir():
+ self._health_issue(issues, "error", "output_not_directory", "Repository path is not a directory.", ".")
+ return
+ for rel_path in required_dirs:
+ path = output_root / rel_path
+ if not path.is_dir():
+ self._health_issue(issues, "error", "missing_directory", "Required directory is missing.", rel_path)
+ for rel_path in required_files:
+ path = output_root / rel_path
+ if not path.is_file():
+ self._health_issue(issues, "error", "missing_file", "Required file is missing.", rel_path)
+ if self.config.write_agent_instructions:
+ instructions_path = output_root / self.config.agent_instructions_name
+ if not instructions_path.is_file():
+ self._health_issue(
+ issues,
+ "warning",
+ "missing_agent_instructions",
+ "Agent bootstrap instructions are missing.",
+ self.config.agent_instructions_name,
+ )
+
+ def _read_manifest_for_health(
+ self,
+ output_root: Path,
+ issues: list[FolderHealthIssue],
+ ) -> dict[str, Any] | None:
+ manifest_path = output_root / self.config.metadata_dir_name / "manifest.json"
+ if not manifest_path.exists():
+ return None
+ try:
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
+ except json.JSONDecodeError as exc:
+ self._health_issue(
+ issues,
+ "error",
+ "invalid_manifest_json",
+ "Manifest is not valid JSON.",
+ _relative_to(manifest_path, output_root),
+ {"error": str(exc)},
+ )
+ return None
+ if not isinstance(manifest, dict):
+ self._health_issue(
+ issues,
+ "error",
+ "invalid_manifest_shape",
+ "Manifest root must be a JSON object.",
+ _relative_to(manifest_path, output_root),
+ )
+ return None
+ sources = manifest.get("sources", {})
+ if not isinstance(sources, Mapping):
+ self._health_issue(
+ issues,
+ "error",
+ "invalid_manifest_sources",
+ "Manifest sources must be an object.",
+ _relative_to(manifest_path, output_root),
+ )
+ return None
+ return manifest
+
+ def _check_manifest_sources(
+ self,
+ output_root: Path,
+ manifest: Mapping[str, Any],
+ issues: list[FolderHealthIssue],
+ ) -> None:
+ sources = manifest.get("sources", {})
+ if not isinstance(sources, Mapping):
+ return
+ for key, source in sources.items():
+ if not isinstance(source, Mapping):
+ self._health_issue(
+ issues,
+ "error",
+ "invalid_source_record",
+ "Manifest source record must be an object.",
+ str(key),
+ )
+ continue
+ source_path = str(source.get("path") or key)
+ raw_path = str(source.get("raw_path") or f"{self.config.raw_data_dir_name}/{source_path}")
+ evidence_path = str(source.get("evidence") or "")
+ if not (output_root / raw_path).is_file():
+ self._health_issue(issues, "error", "missing_raw_source", "Raw source file is missing.", raw_path)
+ if evidence_path and not (output_root / evidence_path).is_file():
+ self._health_issue(
+ issues,
+ "warning",
+ "missing_evidence",
+ "Derived evidence file is missing.",
+ evidence_path,
+ {"source": source_path},
+ )
+ for sidecar in source.get("sidecars", []):
+ sidecar_path = f"{self.config.raw_data_dir_name}/{sidecar}"
+ if not (output_root / sidecar_path).is_file():
+ self._health_issue(
+ issues,
+ "warning",
+ "missing_sidecar",
+ "Raw sidecar file is missing.",
+ sidecar_path,
+ {"source": source_path},
+ )
+ entries = source.get("entries", [])
+ if not isinstance(entries, list) or not entries:
+ self._health_issue(
+ issues,
+ "warning",
+ "source_without_entries",
+ "Manifest source has no generated entries.",
+ source_path,
+ )
+ continue
+ self._check_manifest_entries(output_root, source_path, entries, issues)
+
+ def _check_orphan_evidence(
+ self,
+ output_root: Path,
+ manifest: Mapping[str, Any],
+ issues: list[FolderHealthIssue],
+ ) -> None:
+ derived_dir = output_root / self.config.metadata_dir_name / self.config.derived_dir_name
+ if not derived_dir.is_dir():
+ return
+ sources = manifest.get("sources", {})
+ if not isinstance(sources, Mapping):
+ return
+ expected = {
+ str(source.get("evidence"))
+ for source in sources.values()
+ if isinstance(source, Mapping) and source.get("evidence")
+ }
+ for path in sorted(derived_dir.rglob("*.evidence.md")):
+ rel_path = _relative_to(path, output_root)
+ if rel_path not in expected:
+ self._health_issue(
+ issues,
+ "warning",
+ "orphan_evidence",
+ "Derived evidence file is not referenced by the manifest.",
+ rel_path,
+ )
+
+ def _check_manifest_entries(
+ self,
+ output_root: Path,
+ source_path: str,
+ entries: Sequence[Any],
+ issues: list[FolderHealthIssue],
+ ) -> None:
+ for entry in entries:
+ if not isinstance(entry, Mapping):
+ self._health_issue(
+ issues,
+ "error",
+ "invalid_entry_record",
+ "Manifest entry must be an object.",
+ source_path,
+ )
+ continue
+ bucket = str(entry.get("bucket", ""))
+ entry_id = str(entry.get("id", ""))
+ if bucket not in {"memory", "soul", "skill"}:
+ self._health_issue(
+ issues,
+ "error",
+ "invalid_entry_bucket",
+ "Manifest entry has an invalid bucket.",
+ source_path,
+ {"entry_id": entry_id, "bucket": bucket},
+ )
+ continue
+ detail_path = output_root / bucket / f"{_safe_markdown_name(str(entry.get('source', source_path)))}.md"
+ if not detail_path.is_file():
+ self._health_issue(
+ issues,
+ "warning",
+ "missing_detail_file",
+ "Per-source detail Markdown file is missing.",
+ _relative_to(detail_path, output_root),
+ {"entry_id": entry_id},
+ )
+
+ def _check_generated_blocks(self, output_root: Path, issues: list[FolderHealthIssue]) -> None:
+ for rel_path in ("memory.md", "soul.md", "skill.md"):
+ self._check_generated_block_file(output_root, rel_path, issues, required=True)
+ for bucket in ("memory", "soul", "skill"):
+ bucket_dir = output_root / bucket
+ if not bucket_dir.is_dir():
+ continue
+ for path in sorted(bucket_dir.rglob("*.md")):
+ self._check_generated_block_file(output_root, _relative_to(path, output_root), issues, required=False)
+
+ def _check_generated_block_file(
+ self,
+ output_root: Path,
+ rel_path: str,
+ issues: list[FolderHealthIssue],
+ *,
+ required: bool,
+ ) -> None:
+ path = output_root / rel_path
+ if not path.exists():
+ return
+ try:
+ text = path.read_text(encoding="utf-8-sig")
+ except UnicodeDecodeError:
+ self._health_issue(issues, "warning", "markdown_decode_error", "Markdown file is not UTF-8.", rel_path)
+ return
+ has_start = GENERATED_START in text
+ has_end = GENERATED_END in text
+ if has_start != has_end:
+ self._health_issue(
+ issues,
+ "error",
+ "unbalanced_generated_block",
+ "Generated block markers are unbalanced.",
+ rel_path,
+ )
+ elif required and not has_start:
+ self._health_issue(
+ issues,
+ "error",
+ "missing_generated_block",
+ "Required top-level Markdown file has no generated block.",
+ rel_path,
+ )
+
+ def _health_issue(
+ self,
+ issues: list[FolderHealthIssue],
+ severity: FolderHealthSeverity,
+ code: str,
+ message: str,
+ path: str | None = None,
+ details: Mapping[str, Any] | None = None,
+ ) -> None:
+ issues.append(
+ FolderHealthIssue(
+ severity=severity,
+ code=code,
+ message=message,
+ path=path,
+ details=dict(details or {}),
+ )
+ )
+
+ def _ensure_scaffold_layout(self, output_root: Path, created: list[str]) -> dict[str, Path]:
+ raw_data_dir = output_root / self.config.raw_data_dir_name
+ metadata_dir = output_root / self.config.metadata_dir_name
+ derived_dir = metadata_dir / self.config.derived_dir_name
+ manifest_path = metadata_dir / "manifest.json"
+ for path in (
+ output_root,
+ raw_data_dir,
+ metadata_dir,
+ derived_dir,
+ output_root / "memory",
+ output_root / "soul",
+ output_root / "skill",
+ ):
+ self._mkdir_if_missing(path, output_root, created)
+ return {
+ "raw_data": raw_data_dir,
+ "metadata": metadata_dir,
+ "derived": derived_dir,
+ "manifest": manifest_path,
+ }
+
+ def _mkdir_if_missing(self, path: Path, output_root: Path, created: list[str]) -> None:
+ if not path.exists():
+ created.append(_relative_to(path, output_root))
+ path.mkdir(parents=True, exist_ok=True)
+
+ def _write_scaffold_markdown_files(self, output_root: Path, created: list[str]) -> None:
+ for bucket, title in (("memory", "Memory"), ("soul", "Soul"), ("skill", "Skill")):
+ self._write_scaffold_file(output_root / f"{bucket}.md", output_root, title, created)
+
+ def _write_agent_instructions(self, output_root: Path, created: list[str]) -> None:
+ if not self.config.write_agent_instructions:
+ return
+ target = output_root / self.config.agent_instructions_name
+ if target.exists():
+ return
+ raw_data_dir = self.config.raw_data_dir_name
+ target.write_text(
+ "\n".join(
+ [
+ "# AGENTS.md",
+ "",
+ "Guidance for agents using this memU context harness repository.",
+ "",
+ "## Context Harness",
+ "",
+ "- Treat this folder as a Markdown-backed memory repository.",
+ "- Read `memory.md` for durable facts, events, and project context.",
+ "- Read `soul.md` for persona, tone, style, and preference signals.",
+ "- Read `skill.md` for reusable procedures, tool patterns, and lessons.",
+ "- Generated blocks are managed by memU. Put manual notes outside those blocks.",
+ f"- Raw evidence lives in `{raw_data_dir}/`; preserve original files when possible.",
+ "- Do not let raw logs or feedback rewrite long-term context directly; route them through "
+ "Evolution Instructions, Patch Proposals, and the review gate.",
+ "- Add sidecars beside multimodal files when semantic evidence is needed.",
+ "- Use `.memuignore` or `--exclude` for noisy caches, build outputs, or temporary files.",
+ "- Record task traces before promoting durable skills.",
+ "",
+ "## Useful Commands",
+ "",
+ "```bash",
+ "memu-harness doctor .",
+ "memu-harness status .",
+ "memu-harness refresh . --exclude \"node_modules/**\"",
+ "memu-harness refresh . # also honors .memuignore",
+ "memu-harness refresh . --query \"current task\"",
+ "memu-harness context . --query \"current task\" --format system",
+ "memu-harness context . --format system --output context.system.md",
+ "memu-harness context . --bucket-max soul=1000 --bucket-max skill=2000",
+ "memu-harness trace . --task \"What changed?\" --outcome success",
+ "memu-harness suggest-skills .",
+ "memu-harness promote-skill . --title \"Reusable workflow\"",
+ "```",
+ "",
+ ]
+ ),
+ encoding="utf-8",
+ )
+ created.append(_relative_to(target, output_root))
+
+ def _write_scaffold_file(self, target: Path, output_root: Path, title: str, created: list[str]) -> None:
+ if target.exists():
+ return
+ target.write_text(
+ f"# {title}\n\n{GENERATED_START}\nNo generated entries yet.\n{GENERATED_END}\n\n",
+ encoding="utf-8",
+ )
+ created.append(_relative_to(target, output_root))
+
+ def _write_scaffold_manifest(self, manifest_path: Path, output_root: Path, created: list[str]) -> None:
+ if manifest_path.exists():
+ return
+ manifest = {"version": 1, "updated_at": _utc_now(), "sources": {}}
+ self._write_manifest(manifest_path, manifest)
+ created.append(_relative_to(manifest_path, output_root))
+
+ def _copy_scaffold_source(
+ self,
+ source_folder: str | Path,
+ raw_data_dir: Path,
+ output_root: Path,
+ ) -> list[str]:
+ source_root = Path(source_folder).resolve()
+ if not source_root.exists() or not source_root.is_dir():
+ msg = f"source_folder must be an existing directory: {source_root}"
+ raise ValueError(msg)
+ if source_root == output_root:
+ msg = "source_folder must not be the same as output_folder when scaffolding raw_data"
+ raise ValueError(msg)
+
+ exclude_patterns = self._effective_exclude_patterns(source_root, output_root)
+ copied: list[str] = []
+ for path in sorted(source_root.rglob("*")):
+ if not path.is_file() or self._is_output_tree_path(path, source_root, output_root):
+ continue
+ if self._is_excluded_source_path(path, source_root, exclude_patterns):
+ continue
+ rel_path = path.relative_to(source_root)
+ destination = raw_data_dir / rel_path
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ if path.resolve() == destination.resolve():
+ continue
+ shutil.copy2(path, destination)
+ copied.append(_to_posix(rel_path))
+ return copied
+
+ def _scan_sources(self, source_root: Path, output_root: Path) -> list[_SourceSnapshot]:
+ snapshots: list[_SourceSnapshot] = []
+ exclude_patterns = self._effective_exclude_patterns(source_root, output_root)
+ source_files = {
+ path
+ for path in source_root.rglob("*")
+ if path.is_file() and not self._is_generated_path(path, source_root, output_root)
+ and not self._is_excluded_source_path(path, source_root, exclude_patterns)
+ }
+ for path in sorted(source_files):
+ if self._is_paired_sidecar(path, source_files):
+ continue
+ rel_path = _to_posix(path.relative_to(source_root))
+ modality = self._detect_modality(path)
+ sidecar_paths = (
+ tuple(self._find_sidecar_files(path, source_root, exclude_patterns))
+ if self._supports_sidecars(modality)
+ else ()
+ )
+ sidecar_rel_paths = tuple(_to_posix(sidecar.relative_to(source_root)) for sidecar in sidecar_paths)
+ sha256 = self._hash_source(path, sidecar_paths)
+ raw_rel_path = f"{self.config.raw_data_dir_name}/{rel_path}"
+ evidence_rel_path = f"{self.config.metadata_dir_name}/{self.config.derived_dir_name}/{rel_path}.evidence.md"
+ snapshots.append(
+ _SourceSnapshot(
+ absolute_path=path,
+ rel_path=rel_path,
+ sha256=sha256,
+ size=path.stat().st_size,
+ modality=modality,
+ raw_rel_path=raw_rel_path,
+ evidence_rel_path=evidence_rel_path,
+ sidecar_paths=sidecar_paths,
+ sidecar_rel_paths=sidecar_rel_paths,
+ )
+ )
+ return snapshots
+
+ def _status_from_snapshot(
+ self,
+ snapshot: _SourceSnapshot,
+ previous: Any,
+ ) -> FolderSourceStatus:
+ previous_sha = str(previous.get("sha256", "")) if isinstance(previous, Mapping) else None
+ previous_entries = previous.get("entries", []) if isinstance(previous, Mapping) else []
+ if not isinstance(previous, Mapping):
+ state: FolderSourceState = "new"
+ elif previous_sha == snapshot.sha256 and previous_entries:
+ state = "unchanged"
+ else:
+ state = "changed"
+ return FolderSourceStatus(
+ path=snapshot.rel_path,
+ state=state,
+ modality=snapshot.modality,
+ sha256=snapshot.sha256,
+ previous_sha256=previous_sha,
+ raw_path=snapshot.raw_rel_path,
+ evidence=snapshot.evidence_rel_path,
+ sidecars=list(snapshot.sidecar_rel_paths),
+ entry_count=len(previous_entries) if isinstance(previous_entries, list) else 0,
+ )
+
+ def _removed_status(self, rel_path: str, previous: Mapping[str, Any]) -> FolderSourceStatus:
+ entries = previous.get("entries", [])
+ return FolderSourceStatus(
+ path=rel_path,
+ state="removed",
+ modality=str(previous.get("modality")) if previous.get("modality") else None,
+ sha256=None,
+ previous_sha256=str(previous.get("sha256")) if previous.get("sha256") else None,
+ raw_path=str(previous.get("raw_path")) if previous.get("raw_path") else None,
+ evidence=str(previous.get("evidence")) if previous.get("evidence") else None,
+ sidecars=[str(sidecar) for sidecar in previous.get("sidecars", [])],
+ entry_count=len(entries) if isinstance(entries, list) else 0,
+ )
+
+ def _status_sort_key(self, state: FolderSourceState) -> int:
+ return {"new": 0, "changed": 1, "removed": 2, "unchanged": 3}[state]
+
+ def source_fingerprint(self, source_folder: str | Path, output_folder: str | Path) -> tuple[tuple[str, str], ...]:
+ source_root = Path(source_folder).resolve()
+ output_root = Path(output_folder).resolve()
+ if not source_root.exists() or not source_root.is_dir():
+ msg = f"source_folder must be an existing directory: {source_root}"
+ raise ValueError(msg)
+ source_root = self._resolve_source_root(source_root, output_root)
+ snapshots = self._scan_sources(source_root, output_root)
+ return tuple((snapshot.rel_path, snapshot.sha256) for snapshot in snapshots)
+
+ def _resolve_source_root(self, source_root: Path, output_root: Path) -> Path:
+ if source_root != output_root:
+ return source_root
+ raw_data_dir = output_root / self.config.raw_data_dir_name
+ manifest_path = output_root / self.config.metadata_dir_name / "manifest.json"
+ if raw_data_dir.exists() and raw_data_dir.is_dir() and manifest_path.exists():
+ return raw_data_dir.resolve()
+ return source_root
+
+ def _is_generated_path(self, path: Path, source_root: Path, output_root: Path) -> bool:
+ if self._is_output_tree_path(path, source_root, output_root):
+ return True
+ try:
+ rel = path.relative_to(output_root)
+ except ValueError:
+ return False
+ if not rel.parts:
+ return False
+ generated_names = {
+ self.config.metadata_dir_name,
+ self.config.raw_data_dir_name,
+ "memory",
+ "soul",
+ "skill",
+ }
+ if rel.parts[0] == self.config.raw_data_dir_name:
+ raw_data_dir = (output_root / self.config.raw_data_dir_name).resolve()
+ try:
+ source_root.relative_to(raw_data_dir)
+ except ValueError:
+ pass
+ else:
+ return False
+ if rel.parts[0] in generated_names:
+ return True
+ return rel.as_posix() in {"memory.md", "soul.md", "skill.md"}
+
+ def _effective_exclude_patterns(self, source_root: Path, output_root: Path) -> tuple[str, ...]:
+ patterns: list[str] = list(self.config.exclude_patterns)
+ patterns.extend(self._ignore_file_patterns(source_root / self.config.ignore_file_name))
+ if output_root != source_root:
+ patterns.extend(self._ignore_file_patterns(output_root / self.config.ignore_file_name))
+ patterns.append(self.config.ignore_file_name)
+ return tuple(_dedupe_ordered(patterns))
+
+ def _ignore_file_patterns(self, path: Path) -> list[str]:
+ if not path.is_file():
+ return []
+ try:
+ lines = path.read_text(encoding="utf-8-sig").splitlines()
+ except UnicodeDecodeError:
+ return []
+ patterns: list[str] = []
+ for line in lines:
+ stripped = line.strip()
+ if not stripped or stripped.startswith("#"):
+ continue
+ patterns.append(stripped)
+ return patterns
+
+ def _is_excluded_source_path(
+ self,
+ path: Path,
+ source_root: Path,
+ exclude_patterns: Sequence[str],
+ ) -> bool:
+ try:
+ rel_path = _to_posix(path.relative_to(source_root))
+ except ValueError:
+ return False
+ return any(self._matches_exclude_pattern(rel_path, pattern) for pattern in exclude_patterns)
+
+ def _matches_exclude_pattern(self, rel_path: str, pattern: str) -> bool:
+ clean = pattern.replace("\\", "/").strip().lstrip("/")
+ if not clean:
+ return False
+ if fnmatch.fnmatchcase(rel_path, clean):
+ return True
+ if "/" not in clean and fnmatch.fnmatchcase(Path(rel_path).name, clean):
+ return True
+ literal = clean.rstrip("/")
+ if not any(char in literal for char in "*?[]"):
+ return rel_path == literal or rel_path.startswith(f"{literal}/")
+ return False
+
+ def _is_output_tree_path(self, path: Path, source_root: Path, output_root: Path) -> bool:
+ if source_root == output_root:
+ return False
+ try:
+ output_root.relative_to(source_root)
+ except ValueError:
+ return False
+ try:
+ path.relative_to(output_root)
+ except ValueError:
+ return False
+ return True
+
+ def _sync_raw_file(self, snapshot: _SourceSnapshot, raw_data_dir: Path) -> None:
+ self._copy_raw_path(snapshot.absolute_path, raw_data_dir / snapshot.rel_path)
+ for sidecar_path, sidecar_rel_path in zip(snapshot.sidecar_paths, snapshot.sidecar_rel_paths, strict=True):
+ self._copy_raw_path(sidecar_path, raw_data_dir / sidecar_rel_path)
+
+ def _copy_raw_path(self, source: Path, destination: Path) -> None:
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ if source.resolve() == destination.resolve():
+ return
+ shutil.copy2(source, destination)
+
+ def _expected_raw_rel_paths(self, snapshots: Sequence[_SourceSnapshot]) -> set[str]:
+ expected: set[str] = set()
+ for snapshot in snapshots:
+ expected.add(snapshot.raw_rel_path)
+ for sidecar_rel_path in snapshot.sidecar_rel_paths:
+ expected.add(f"{self.config.raw_data_dir_name}/{sidecar_rel_path}")
+ return expected
+
+ def _expected_evidence_rel_paths(self, snapshots: Sequence[_SourceSnapshot]) -> set[str]:
+ return {snapshot.evidence_rel_path for snapshot in snapshots}
+
+ def _source_entries_from_manifest(self, source: Any) -> list[MarkdownMemoryEntry]:
+ if not isinstance(source, Mapping):
+ return []
+ entries: list[MarkdownMemoryEntry] = []
+ for entry_data in source.get("entries", []):
+ if isinstance(entry_data, Mapping):
+ entries.append(MarkdownMemoryEntry.from_manifest(entry_data))
+ return entries
+
+ def _entry_manifests_from_source(self, source: Mapping[str, Any]) -> list[dict[str, Any]]:
+ return [entry.to_manifest() for entry in self._source_entries_from_manifest(source)]
+
+ def _evolve_removed_source(self, rel_path: str, previous: Mapping[str, Any]) -> EvolutionReviewBundle:
+ previous_entries = self._source_entries_from_manifest(previous)
+ evidence_path = str(previous.get("evidence", ""))
+ raw_path = str(previous.get("raw_path", f"{self.config.raw_data_dir_name}/{rel_path}"))
+ return self.self_evolve.build_for_removed_source(
+ source=raw_path,
+ evidence_path=evidence_path,
+ previous_entries=[entry.to_manifest() for entry in previous_entries],
+ )
+
+ def _expected_raw_rel_paths_from_sources(self, sources: Mapping[str, Any]) -> set[str]:
+ expected: set[str] = set()
+ for source in sources.values():
+ if not isinstance(source, Mapping):
+ continue
+ raw_path = source.get("raw_path")
+ if isinstance(raw_path, str) and raw_path:
+ expected.add(raw_path)
+ for sidecar in source.get("sidecars", []):
+ sidecar_path = str(sidecar)
+ if not sidecar_path:
+ continue
+ if sidecar_path.startswith(f"{self.config.raw_data_dir_name}/"):
+ expected.add(sidecar_path)
+ else:
+ expected.add(f"{self.config.raw_data_dir_name}/{sidecar_path}")
+ return expected
+
+ def _expected_evidence_rel_paths_from_sources(self, sources: Mapping[str, Any]) -> set[str]:
+ expected: set[str] = set()
+ for source in sources.values():
+ if not isinstance(source, Mapping):
+ continue
+ evidence_path = source.get("evidence")
+ if isinstance(evidence_path, str) and evidence_path:
+ expected.add(evidence_path)
+ return expected
+
+ def _remove_stale_raw_files(self, raw_data_dir: Path, expected_raw_rel_paths: set[str]) -> None:
+ expected = {rel.removeprefix(f"{self.config.raw_data_dir_name}/") for rel in expected_raw_rel_paths}
+ for path in sorted(raw_data_dir.rglob("*"), reverse=True):
+ if path.is_file() and _to_posix(path.relative_to(raw_data_dir)) not in expected:
+ path.unlink()
+ elif path.is_dir() and not any(path.iterdir()):
+ path.rmdir()
+
+ def _remove_stale_evidence_files(self, derived_dir: Path, expected_evidence_rel_paths: set[str]) -> None:
+ expected = {
+ rel.removeprefix(f"{self.config.metadata_dir_name}/{self.config.derived_dir_name}/")
+ for rel in expected_evidence_rel_paths
+ }
+ if not derived_dir.exists():
+ return
+ for path in sorted(derived_dir.rglob("*"), reverse=True):
+ if path.is_file() and path.name.endswith(".evidence.md"):
+ if _to_posix(path.relative_to(derived_dir)) not in expected:
+ path.unlink()
+ elif path.is_dir() and not any(path.iterdir()):
+ path.rmdir()
+
+ async def _extract_entries(
+ self,
+ snapshot: _SourceSnapshot,
+ output_root: Path,
+ *,
+ previous: Any,
+ user: Mapping[str, Any] | None,
+ ) -> _SourceExtraction:
+ evidence_text = self._build_evidence(snapshot)
+ service_extraction = await self._extract_with_memory_service(snapshot, user=user)
+ if service_extraction is not None:
+ evidence_text = evidence_text.rstrip() + "\n\n" + service_extraction.evidence_text.strip() + "\n"
+
+ evidence_path = output_root / snapshot.evidence_rel_path
+ evidence_path.parent.mkdir(parents=True, exist_ok=True)
+ evidence_path.write_text(evidence_text, encoding="utf-8")
+
+ if service_extraction is not None and service_extraction.entries:
+ candidate_entries = service_extraction.entries
+ else:
+ candidate_entries = self._extract_with_local_heuristics(snapshot, evidence_text)
+
+ if not self.config.self_evolve_enabled:
+ return _SourceExtraction(entries=candidate_entries, instructions=[], proposals=[], reviews=[])
+
+ previous_entries = self._source_entries_from_manifest(previous)
+ bundle = self.self_evolve.build_for_source(
+ source=snapshot.raw_rel_path,
+ evidence_path=snapshot.evidence_rel_path,
+ evidence_text=evidence_text,
+ candidates=[entry.to_manifest() for entry in candidate_entries],
+ previous_entries=[entry.to_manifest() for entry in previous_entries],
+ )
+ approved_entries = [
+ MarkdownMemoryEntry.from_manifest(entry)
+ for entry in apply_reviewed_proposals(
+ [entry.to_manifest() for entry in previous_entries],
+ bundle.proposals,
+ bundle.reviews,
+ )
+ ]
+ return _SourceExtraction(
+ entries=approved_entries,
+ instructions=bundle.instructions,
+ proposals=bundle.proposals,
+ reviews=bundle.reviews,
+ )
+
+ async def _extract_with_memory_service(
+ self,
+ snapshot: _SourceSnapshot,
+ *,
+ user: Mapping[str, Any] | None,
+ ) -> _ServiceExtraction | None:
+ if not self.memory_service or not self.config.use_memory_service:
+ return None
+ service_modality = self._service_modality(snapshot)
+ if service_modality is None:
+ return None
+ try:
+ result = await self.memory_service.memorize(
+ resource_url=str(snapshot.absolute_path),
+ modality=service_modality,
+ user=dict(user or {}),
+ )
+ except Exception as exc:
+ evidence_text = (
+ "## MemoryService Extraction\n\n"
+ f"MemoryService extraction was attempted for `{snapshot.raw_rel_path}` "
+ f"with modality `{service_modality}` but failed: {type(exc).__name__}: {exc}\n"
+ )
+ return _ServiceExtraction(entries=[], evidence_text=evidence_text)
+
+ entries: list[MarkdownMemoryEntry] = []
+ for idx, item in enumerate(result.get("items", []), start=1):
+ if not isinstance(item, Mapping):
+ continue
+ summary = str(item.get("summary", "")).strip()
+ if not summary:
+ continue
+ memory_type = str(item.get("memory_type", "knowledge"))
+ bucket = self._bucket_from_memory_type(memory_type, summary)
+ entries.append(
+ MarkdownMemoryEntry(
+ id=self._entry_id(bucket, snapshot.rel_path, idx),
+ bucket=bucket,
+ title=self._title_for(snapshot.rel_path, summary),
+ body=summary,
+ source=snapshot.raw_rel_path,
+ evidence=snapshot.evidence_rel_path,
+ modality=snapshot.modality,
+ confidence="high",
+ tags=[memory_type, "llm-extracted"],
+ )
+ )
+ return _ServiceExtraction(entries=entries, evidence_text=self._build_service_evidence(snapshot, result))
+
+ def _build_service_evidence(self, snapshot: _SourceSnapshot, result: Mapping[str, Any]) -> str:
+ lines = [
+ "## MemoryService Extraction",
+ "",
+ f"- source: {snapshot.raw_rel_path}",
+ f"- modality: {snapshot.modality}",
+ "",
+ ]
+
+ resources = self._as_list(result.get("resources"))
+ resource = result.get("resource")
+ if isinstance(resource, Mapping):
+ resources.insert(0, resource)
+ resource_lines = self._format_service_records("Resource", resources, keys=("url", "caption", "modality"))
+ if resource_lines:
+ lines.extend(["### Resources", "", *resource_lines, ""])
+
+ item_lines = self._format_service_records(
+ "Item",
+ self._as_list(result.get("items")),
+ keys=("memory_type", "summary"),
+ )
+ if item_lines:
+ lines.extend(["### Items", "", *item_lines, ""])
+
+ category_lines = self._format_service_records(
+ "Category",
+ self._as_list(result.get("categories")),
+ keys=("name", "description", "summary"),
+ )
+ if category_lines:
+ lines.extend(["### Categories", "", *category_lines, ""])
+
+ if len(lines) == 5:
+ lines.append("MemoryService returned no structured resources, items, or categories.")
+ return "\n".join(lines).rstrip() + "\n"
+
+ def _format_service_records(
+ self,
+ label: str,
+ records: Sequence[Any],
+ *,
+ keys: Sequence[str],
+ ) -> list[str]:
+ lines: list[str] = []
+ for idx, record in enumerate(records, start=1):
+ if not isinstance(record, Mapping):
+ continue
+ lines.append(f"{idx}. {label}")
+ for key in keys:
+ value = record.get(key)
+ if value is None or value == "":
+ continue
+ lines.append(f" - {key}: {self._single_line(str(value))}")
+ return lines
+
+ def _as_list(self, value: Any) -> list[Any]:
+ if isinstance(value, list):
+ return value
+ if isinstance(value, tuple):
+ return list(value)
+ return []
+
+ def _single_line(self, value: str) -> str:
+ compact = " ".join(value.split())
+ if len(compact) > 500:
+ return compact[:497].rstrip() + "..."
+ return compact
+
+ def _extract_with_local_heuristics(
+ self,
+ snapshot: _SourceSnapshot,
+ evidence_text: str,
+ ) -> list[MarkdownMemoryEntry]:
+ buckets: list[MemoryBucket] = ["memory"]
+ searchable = evidence_text.lower()
+ if self._contains_any(searchable, _SOUL_KEYWORDS):
+ buckets.append("soul")
+ if self._contains_any(searchable, _SKILL_KEYWORDS):
+ buckets.append("skill")
+
+ entries: list[MarkdownMemoryEntry] = []
+ for idx, bucket in enumerate(buckets, start=1):
+ body = self._fallback_body(bucket, snapshot, evidence_text)
+ entries.append(
+ MarkdownMemoryEntry(
+ id=self._entry_id(bucket, snapshot.rel_path, idx),
+ bucket=bucket,
+ title=self._fallback_title(bucket, snapshot.rel_path),
+ body=body,
+ source=snapshot.raw_rel_path,
+ evidence=snapshot.evidence_rel_path,
+ modality=snapshot.modality,
+ confidence="low"
+ if snapshot.modality in {"image", "audio", "video", "document", "binary"}
+ else "medium",
+ tags=[snapshot.modality, "local-extraction"],
+ )
+ )
+ return entries
+
+ def _build_evidence(self, snapshot: _SourceSnapshot) -> str:
+ header = (
+ f"# Evidence: {snapshot.rel_path}\n\n"
+ f"- source: {snapshot.raw_rel_path}\n"
+ f"- modality: {snapshot.modality}\n"
+ f"- sha256: {snapshot.sha256}\n"
+ f"- size_bytes: {snapshot.size}\n"
+ f"- extracted_at: {_utc_now()}\n\n"
+ )
+ text = self._read_text_evidence(snapshot.absolute_path)
+ if text is None:
+ sidecar_evidence = self._build_sidecar_evidence(snapshot.sidecar_paths)
+ return (
+ header
+ + "## Multimodal Evidence\n\n"
+ + "This source is preserved as raw multimodal data. "
+ + "Use a MemoryService with multimodal or document-capable LLM profiles "
+ + "to derive captions, transcripts, frame descriptions, or document summaries.\n"
+ + sidecar_evidence
+ )
+ return header + "## Text Evidence\n\n" + text.strip() + "\n"
+
+ def _build_sidecar_evidence(self, sidecars: Sequence[Path]) -> str:
+ if not sidecars:
+ return ""
+
+ lines = ["\n## Sidecar Evidence", ""]
+ for sidecar in sidecars:
+ text = self._read_sidecar_text(sidecar)
+ if text is None:
+ continue
+ lines.extend(
+ [
+ f"### {sidecar.name}",
+ "",
+ text.strip(),
+ "",
+ ]
+ )
+ if len(lines) == 2:
+ return ""
+ return "\n".join(lines).rstrip() + "\n"
+
+ def _find_sidecar_files(
+ self,
+ path: Path,
+ source_root: Path,
+ exclude_patterns: Sequence[str],
+ ) -> list[Path]:
+ candidates: list[Path] = []
+ for label in _SIDECAR_LABELS:
+ for extension in _SIDECAR_TEXT_EXTENSIONS:
+ candidates.append(path.with_name(f"{path.name}.{label}{extension}"))
+ candidates.append(path.with_name(f"{path.stem}.{label}{extension}"))
+ seen: set[Path] = set()
+ sidecars: list[Path] = []
+ for candidate in candidates:
+ if candidate in seen or candidate == path:
+ continue
+ seen.add(candidate)
+ if self._is_excluded_source_path(candidate, source_root, exclude_patterns):
+ continue
+ if candidate.exists() and candidate.is_file():
+ sidecars.append(candidate)
+ return sidecars
+
+ def _is_paired_sidecar(self, path: Path, source_files: set[Path]) -> bool:
+ return self._paired_sidecar_source(path, source_files) is not None
+
+ def _paired_sidecar_source(self, path: Path, source_files: set[Path]) -> Path | None:
+ if path.suffix.lower() not in _SIDECAR_TEXT_EXTENSIONS:
+ return None
+ name_without_extension = path.name[: -len(path.suffix)]
+ for label in _SIDECAR_LABELS:
+ suffix = f".{label}"
+ if not name_without_extension.endswith(suffix):
+ continue
+ base_name = name_without_extension[: -len(suffix)]
+ exact_source = path.with_name(base_name)
+ if exact_source in source_files and self._supports_sidecars(self._detect_modality(exact_source)):
+ return exact_source
+ for candidate in source_files:
+ if candidate.parent == path.parent and candidate.stem == base_name:
+ if self._supports_sidecars(self._detect_modality(candidate)):
+ return candidate
+ return None
+
+ def _supports_sidecars(self, modality: str) -> bool:
+ return modality in _SIDECAR_SOURCE_MODALITIES
+
+ def _read_sidecar_text(self, path: Path) -> str | None:
+ try:
+ text = path.read_text(encoding="utf-8-sig")
+ except UnicodeDecodeError:
+ return None
+ if path.suffix.lower() == ".json":
+ text = self._format_json_sidecar(text)
+ elif path.suffix.lower() == ".jsonl":
+ text = self._format_jsonl_sidecar(text)
+ if len(text) > self.config.max_text_chars:
+ return text[: self.config.max_text_chars] + "\n\n[Truncated by FolderMemoryCompiler]\n"
+ return text
+
+ def _format_json_sidecar(self, text: str) -> str:
+ try:
+ parsed = json.loads(text)
+ except json.JSONDecodeError:
+ return text
+ formatted = json.dumps(parsed, ensure_ascii=False, indent=2, sort_keys=True)
+ return "Structured JSON sidecar:\n\n```json\n" + formatted + "\n```\n"
+
+ def _format_jsonl_sidecar(self, text: str) -> str:
+ formatted_lines: list[str] = []
+ for line in text.splitlines():
+ stripped = line.strip()
+ if not stripped:
+ continue
+ try:
+ parsed = json.loads(stripped)
+ except json.JSONDecodeError:
+ return text
+ formatted_lines.append(json.dumps(parsed, ensure_ascii=False, sort_keys=True))
+ if not formatted_lines:
+ return text
+ return "Structured JSONL sidecar:\n\n```jsonl\n" + "\n".join(formatted_lines) + "\n```\n"
+
+ def _read_text_evidence(self, path: Path) -> str | None:
+ if self._detect_modality(path) in {"image", "audio", "video", "document", "binary"}:
+ return None
+ try:
+ text = path.read_text(encoding="utf-8-sig")
+ except UnicodeDecodeError:
+ return None
+ if len(text) > self.config.max_text_chars:
+ return text[: self.config.max_text_chars] + "\n\n[Truncated by FolderMemoryCompiler]\n"
+ return text
+
+ def _service_modality(self, snapshot: _SourceSnapshot) -> str | None:
+ if snapshot.modality in {"image", "audio", "video", "conversation"}:
+ return snapshot.modality
+ if snapshot.modality in {"text", "document", "code"}:
+ return "document"
+ return None
+
+ def _detect_modality(self, path: Path) -> str:
+ suffix = path.suffix.lower()
+ if suffix in _IMAGE_EXTENSIONS:
+ return "image"
+ if suffix in _AUDIO_EXTENSIONS:
+ return "audio"
+ if suffix in _VIDEO_EXTENSIONS:
+ return "video"
+ if suffix in _CONVERSATION_EXTENSIONS or self._looks_like_conversation_path(path):
+ return "conversation"
+ if suffix in _CODE_EXTENSIONS:
+ return "code"
+ if suffix in _TEXT_EXTENSIONS:
+ return "text"
+ if suffix in _DOCUMENT_EXTENSIONS:
+ return "document"
+ return "binary"
+
+ def _looks_like_conversation_path(self, path: Path) -> bool:
+ lowered = path.as_posix().lower()
+ return any(part in lowered for part in ("chat", "conversation", "dialog", "message"))
+
+ def _bucket_from_memory_type(self, memory_type: str, summary: str) -> MemoryBucket:
+ lowered = f"{memory_type} {summary}".lower()
+ if memory_type in {"skill", "tool"} or self._contains_any(lowered, _SKILL_KEYWORDS):
+ return "skill"
+ if self._contains_any(lowered, _SOUL_KEYWORDS):
+ return "soul"
+ return "memory"
+
+ def _fallback_title(self, bucket: MemoryBucket, rel_path: str) -> str:
+ source_name = Path(rel_path).stem.replace("_", " ").replace("-", " ").strip() or rel_path
+ if bucket == "soul":
+ return f"Persona and style signals from {source_name}"
+ if bucket == "skill":
+ return f"Skill signals from {source_name}"
+ return f"Memory extracted from {source_name}"
+
+ def _fallback_body(self, bucket: MemoryBucket, snapshot: _SourceSnapshot, evidence_text: str) -> str:
+ if snapshot.modality in {"image", "audio", "video", "document", "binary"}:
+ if "## Sidecar Evidence" in evidence_text:
+ excerpt = self._compact_excerpt(evidence_text)
+ if bucket == "soul":
+ prefix = "This multimodal source includes sidecar persona, tone, or style evidence."
+ elif bucket == "skill":
+ prefix = "This multimodal source includes sidecar skill, workflow, or tool-use evidence."
+ else:
+ prefix = "This multimodal source includes sidecar memory evidence."
+ return f"{prefix}\n\nEvidence excerpt:\n\n> {excerpt}"
+ return (
+ f"`{snapshot.raw_rel_path}` is preserved as {snapshot.modality} raw data. "
+ "The compiler created traceable evidence metadata; richer semantic extraction can be produced "
+ "with a multimodal MemoryService."
+ )
+ if bucket == "skill" and "# Skill Evolution Trace" in evidence_text:
+ return self._skill_trace_body(evidence_text)
+
+ excerpt = self._compact_excerpt(evidence_text)
+ if bucket == "soul":
+ prefix = "This source contains persona, tone, language-style, or interaction-style signals."
+ elif bucket == "skill":
+ prefix = "This source contains skill, workflow, tool-use, or capability signals."
+ else:
+ prefix = "This source contributes durable memory evidence."
+ return f"{prefix}\n\nEvidence excerpt:\n\n> {excerpt}"
+
+ def _skill_trace_body(self, evidence_text: str) -> str:
+ sections = {
+ "summary": self._markdown_section(evidence_text, "Summary"),
+ "actions": self._markdown_section(evidence_text, "Actions"),
+ "tools": self._markdown_section(evidence_text, "Tool Calls"),
+ "lessons": self._markdown_section(evidence_text, "Lessons For Skill Evolution"),
+ "hints": self._markdown_section(evidence_text, "Retrieval Hints"),
+ }
+ lines = [
+ "This source is a skill evolution trace. It records an agent/tool execution and the reusable lessons "
+ "that should improve future behavior.",
+ "",
+ ]
+ for label, content in (
+ ("Summary", sections["summary"]),
+ ("Actions", sections["actions"]),
+ ("Tool Calls", sections["tools"]),
+ ("Lessons", sections["lessons"]),
+ ("Retrieval Hints", sections["hints"]),
+ ):
+ if content:
+ lines.extend([f"**{label}**", "", content, ""])
+ return "\n".join(lines).strip()
+
+ def _markdown_section(self, text: str, heading: str) -> str:
+ marker = f"## {heading}"
+ start = text.find(marker)
+ if start == -1:
+ return ""
+ start += len(marker)
+ end = text.find("\n## ", start)
+ section = text[start:] if end == -1 else text[start:end]
+ return section.strip()
+
+ def _compact_excerpt(self, text: str) -> str:
+ content = "\n".join(line.strip() for line in text.splitlines() if line.strip())
+ if len(content) > 700:
+ content = content[:700].rstrip() + "..."
+ return content.replace("\n", "\n> ")
+
+ def _title_for(self, rel_path: str, summary: str) -> str:
+ first_line = summary.splitlines()[0].strip()
+ if len(first_line) > 80:
+ return first_line[:77].rstrip() + "..."
+ return first_line or self._fallback_title("memory", rel_path)
+
+ def _entry_id(self, bucket: MemoryBucket, rel_path: str, idx: int) -> str:
+ digest = hashlib.sha256(f"{bucket}:{rel_path}:{idx}".encode("utf-8")).hexdigest()[:12]
+ prefix = {"memory": "mem", "soul": "soul", "skill": "skill"}[bucket]
+ return f"{prefix}_{digest}"
+
+ def _hash_source(self, path: Path, sidecars: Sequence[Path]) -> str:
+ digest = hashlib.sha256()
+ digest.update(b"source\0")
+ digest.update(path.name.encode("utf-8"))
+ digest.update(b"\0")
+ self._update_digest_from_file(digest, path)
+ for sidecar in sidecars:
+ digest.update(b"\0sidecar\0")
+ digest.update(sidecar.name.encode("utf-8"))
+ digest.update(b"\0")
+ self._update_digest_from_file(digest, sidecar)
+ return digest.hexdigest()
+
+ def _hash_file(self, path: Path) -> str:
+ digest = hashlib.sha256()
+ self._update_digest_from_file(digest, path)
+ return digest.hexdigest()
+
+ def _update_digest_from_file(self, digest: Any, path: Path) -> None:
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+
+ def _load_manifest(self, manifest_path: Path) -> dict[str, Any]:
+ if not manifest_path.exists():
+ return {"version": 1, "sources": {}}
+ try:
+ loaded = json.loads(manifest_path.read_text(encoding="utf-8"))
+ except json.JSONDecodeError:
+ return {"version": 1, "sources": {}}
+ return loaded if isinstance(loaded, dict) else {"version": 1, "sources": {}}
+
+ def _write_manifest(self, manifest_path: Path, manifest: Mapping[str, Any]) -> None:
+ manifest_path.parent.mkdir(parents=True, exist_ok=True)
+ manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False, sort_keys=True), encoding="utf-8")
+
+ def _entries_from_manifest(self, manifest: Mapping[str, Any]) -> list[MarkdownMemoryEntry]:
+ entries: list[MarkdownMemoryEntry] = []
+ sources = manifest.get("sources", {})
+ if not isinstance(sources, Mapping):
+ return entries
+ for source in sources.values():
+ if not isinstance(source, Mapping):
+ continue
+ for entry_data in source.get("entries", []):
+ if isinstance(entry_data, Mapping):
+ entries.append(MarkdownMemoryEntry.from_manifest(entry_data))
+ return entries
+
+ def _write_markdown_repository(self, output_root: Path, entries: Sequence[MarkdownMemoryEntry]) -> None:
+ for bucket in ("memory", "soul", "skill"):
+ bucket_entries = [entry for entry in entries if entry.bucket == bucket]
+ self._write_top_level_file(output_root, cast(MemoryBucket, bucket), bucket_entries)
+ self._write_detail_files(output_root, cast(MemoryBucket, bucket), bucket_entries)
+
+ def _write_top_level_file(
+ self,
+ output_root: Path,
+ bucket: MemoryBucket,
+ entries: Sequence[MarkdownMemoryEntry],
+ ) -> None:
+ target = output_root / f"{bucket}.md"
+ title = {"memory": "Memory", "soul": "Soul", "skill": "Skill"}[bucket]
+ generated = self._render_entries(title, entries)
+ self._write_generated_block(target, title, generated)
+
+ def _write_detail_files(
+ self,
+ output_root: Path,
+ bucket: MemoryBucket,
+ entries: Sequence[MarkdownMemoryEntry],
+ ) -> None:
+ bucket_dir = output_root / bucket
+ expected_files: set[Path] = set()
+ by_source: dict[str, list[MarkdownMemoryEntry]] = {}
+ for entry in entries:
+ by_source.setdefault(entry.source, []).append(entry)
+ for source, source_entries in by_source.items():
+ detail_file = bucket_dir / f"{_safe_markdown_name(source)}.md"
+ expected_files.add(detail_file)
+ generated = self._render_entries(f"{bucket.title()} From {source}", source_entries)
+ self._write_generated_block(detail_file, f"{bucket.title()} From {source}", generated)
+
+ for path in bucket_dir.glob("*.md"):
+ if path not in expected_files:
+ self._remove_stale_detail_file(path, bucket)
+
+ def _remove_stale_detail_file(self, path: Path, bucket: MemoryBucket) -> None:
+ try:
+ current = path.read_text(encoding="utf-8-sig")
+ except UnicodeDecodeError:
+ return
+ if GENERATED_START not in current or GENERATED_END not in current:
+ return
+ if self._manual_detail_content(current, bucket):
+ title = self._detail_title_from_content(current) or f"{bucket.title()} Notes"
+ self._write_generated_block(path, title, "No generated entries yet.\n")
+ return
+ path.unlink()
+
+ def _manual_detail_content(self, current: str, bucket: MemoryBucket) -> str:
+ without_block = self._remove_generated_block_text(current).strip()
+ if not without_block:
+ return ""
+ lines = without_block.splitlines()
+ first = lines[0].strip()
+ rest = "\n".join(lines[1:]).strip()
+ bucket_title = bucket.title()
+ if first == f"# {bucket_title}" or first.startswith(f"# {bucket_title} From "):
+ return rest
+ return without_block
+
+ def _remove_generated_block_text(self, current: str) -> str:
+ start_idx = current.find(GENERATED_START)
+ end_idx = current.find(GENERATED_END)
+ if start_idx == -1 or end_idx == -1 or end_idx < start_idx:
+ return current
+ end_idx += len(GENERATED_END)
+ return current[:start_idx] + current[end_idx:]
+
+ def _detail_title_from_content(self, current: str) -> str | None:
+ for line in current.splitlines():
+ if line.startswith("# "):
+ return line[2:].strip() or None
+ return None
+
+ def _render_entries(self, title: str, entries: Sequence[MarkdownMemoryEntry]) -> str:
+ if not entries:
+ return "No generated entries yet.\n"
+ rendered = [entry.to_markdown() for entry in entries]
+ return "\n".join(rendered).strip() + "\n"
+
+ def _write_generated_block(self, target: Path, title: str, generated: str) -> None:
+ target.parent.mkdir(parents=True, exist_ok=True)
+ block = f"{GENERATED_START}\n{generated.strip()}\n{GENERATED_END}\n"
+ if not target.exists():
+ content = f"# {title}\n\n{block}\n"
+ else:
+ current = target.read_text(encoding="utf-8-sig")
+ content = self._replace_generated_block(current, block)
+ target.write_text(content, encoding="utf-8")
+
+ def _replace_generated_block(self, current: str, block: str) -> str:
+ start_idx = current.find(GENERATED_START)
+ end_idx = current.find(GENERATED_END)
+ if start_idx == -1 or end_idx == -1 or end_idx < start_idx:
+ return current.rstrip() + "\n\n" + block
+ end_idx += len(GENERATED_END)
+ return current[:start_idx] + block.rstrip() + current[end_idx:]
+
+ def _contains_any(self, value: str, needles: Sequence[str]) -> bool:
+ return any(needle in value for needle in needles)
+
+
+async def compile_folder_to_markdown(
+ source_folder: str | Path,
+ output_folder: str | Path,
+ *,
+ memory_service: MemoryService | None = None,
+ user: Mapping[str, Any] | None = None,
+ config: FolderMemoryCompilerConfig | None = None,
+) -> FolderCompileResult:
+ compiler = FolderMemoryCompiler(memory_service=memory_service, config=config)
+ return await compiler.compile(source_folder, output_folder, user=user)
+
+
+def compile_folder_to_markdown_sync(
+ source_folder: str | Path,
+ output_folder: str | Path,
+ *,
+ memory_service: MemoryService | None = None,
+ user: Mapping[str, Any] | None = None,
+ config: FolderMemoryCompilerConfig | None = None,
+) -> FolderCompileResult:
+ return asyncio.run(
+ compile_folder_to_markdown(
+ source_folder,
+ output_folder,
+ memory_service=memory_service,
+ user=user,
+ config=config,
+ )
+ )
+
+
+def scaffold_folder_memory_repository(
+ output_folder: str | Path,
+ *,
+ source_folder: str | Path | None = None,
+ config: FolderMemoryCompilerConfig | None = None,
+) -> FolderScaffoldResult:
+ compiler = FolderMemoryCompiler(config=config)
+ return compiler.scaffold(output_folder, source_folder=source_folder)
+
+
+def inspect_folder_memory_status(
+ source_folder: str | Path,
+ output_folder: str | Path,
+ *,
+ config: FolderMemoryCompilerConfig | None = None,
+) -> FolderStatusResult:
+ compiler = FolderMemoryCompiler(config=config)
+ return compiler.status(source_folder, output_folder)
+
+
+def inspect_folder_memory_health(
+ output_folder: str | Path,
+ *,
+ config: FolderMemoryCompilerConfig | None = None,
+) -> FolderHealthResult:
+ compiler = FolderMemoryCompiler(config=config)
+ return compiler.health(output_folder)
+
+
+def review_folder_evolution(
+ output_folder: str | Path,
+ *,
+ proposal_ids: Sequence[str] | None = None,
+ reviewer: str = "creator",
+ decision: ReviewStatus = "approved",
+ reason: str = "",
+ config: FolderMemoryCompilerConfig | None = None,
+) -> EvolutionReviewApplyResult:
+ compiler = FolderMemoryCompiler(config=config)
+ return compiler.review_evolution(
+ output_folder,
+ proposal_ids=proposal_ids,
+ reviewer=reviewer,
+ decision=decision,
+ reason=reason,
+ )
+
+
+async def watch_folder_to_markdown(
+ source_folder: str | Path,
+ output_folder: str | Path,
+ *,
+ memory_service: MemoryService | None = None,
+ user: Mapping[str, Any] | None = None,
+ config: FolderMemoryCompilerConfig | None = None,
+ poll_interval: float = 2.0,
+ max_runs: int | None = None,
+ on_event: Callable[[FolderWatchEvent], Any | Awaitable[Any]] | None = None,
+) -> list[FolderWatchEvent]:
+ if poll_interval <= 0:
+ msg = "poll_interval must be greater than 0"
+ raise ValueError(msg)
+ if max_runs is not None and max_runs <= 0:
+ msg = "max_runs must be greater than 0 when provided"
+ raise ValueError(msg)
+
+ compiler = FolderMemoryCompiler(memory_service=memory_service, config=config)
+ source_root = Path(source_folder).resolve()
+ output_root = Path(output_folder).resolve()
+ last_fingerprint: tuple[tuple[str, str], ...] | None = None
+ events: list[FolderWatchEvent] = []
+ iteration = 0
+
+ while max_runs is None or len(events) < max_runs:
+ fingerprint = compiler.source_fingerprint(source_root, output_root)
+ if last_fingerprint is None or fingerprint != last_fingerprint:
+ reason: FolderWatchReason = "initial" if last_fingerprint is None else "changed"
+ status = compiler.status(source_root, output_root)
+ result = await compiler.compile(source_root, output_root, user=user)
+ last_fingerprint = compiler.source_fingerprint(source_root, output_root)
+ iteration += 1
+ event = FolderWatchEvent(iteration=iteration, reason=reason, result=result, status=status)
+ events.append(event)
+ if on_event is not None:
+ callback_result = on_event(event)
+ if inspect.isawaitable(callback_result):
+ await callback_result
+ if max_runs is not None and len(events) >= max_runs:
+ break
+ await asyncio.sleep(poll_interval)
+
+ return events
+
+
+def watch_folder_to_markdown_sync(
+ source_folder: str | Path,
+ output_folder: str | Path,
+ *,
+ memory_service: MemoryService | None = None,
+ user: Mapping[str, Any] | None = None,
+ config: FolderMemoryCompilerConfig | None = None,
+ poll_interval: float = 2.0,
+ max_runs: int | None = None,
+ on_event: Callable[[FolderWatchEvent], Any | Awaitable[Any]] | None = None,
+) -> list[FolderWatchEvent]:
+ return asyncio.run(
+ watch_folder_to_markdown(
+ source_folder,
+ output_folder,
+ memory_service=memory_service,
+ user=user,
+ config=config,
+ poll_interval=poll_interval,
+ max_runs=max_runs,
+ on_event=on_event,
+ )
+ )
+
+
+def _utc_now() -> str:
+ return datetime.now(UTC).isoformat().replace("+00:00", "Z")
+
+
+def _to_posix(path: Path) -> str:
+ return path.as_posix()
+
+
+def _dedupe_ordered(values: Sequence[str]) -> list[str]:
+ deduped: list[str] = []
+ seen: set[str] = set()
+ for value in values:
+ clean = value.strip()
+ if clean and clean not in seen:
+ deduped.append(clean)
+ seen.add(clean)
+ return deduped
+
+
+def _relative_to(path: Path, root: Path) -> str:
+ try:
+ rel_path = path.relative_to(root)
+ except ValueError:
+ return str(path)
+ if not rel_path.parts:
+ return "."
+ return rel_path.as_posix()
+
+
+def _safe_markdown_name(source: str) -> str:
+ clean = source.removeprefix("raw_data/")
+ clean = clean.replace("\\", "/").replace("/", "__")
+ clean = clean.replace(":", "_")
+ return clean or "source"
+
+
+_TEXT_EXTENSIONS = {
+ ".csv",
+ ".htm",
+ ".html",
+ ".json",
+ ".jsonl",
+ ".log",
+ ".md",
+ ".rst",
+ ".text",
+ ".toml",
+ ".tsv",
+ ".txt",
+ ".xml",
+ ".yaml",
+ ".yml",
+}
+
+_CONVERSATION_EXTENSIONS = {".chat"}
+
+_DOCUMENT_EXTENSIONS = {".doc", ".docx", ".pdf", ".ppt", ".pptx", ".xls", ".xlsx"}
+
+_CODE_EXTENSIONS = {
+ ".c",
+ ".cpp",
+ ".cs",
+ ".css",
+ ".go",
+ ".java",
+ ".js",
+ ".jsx",
+ ".kt",
+ ".lua",
+ ".php",
+ ".py",
+ ".rb",
+ ".rs",
+ ".sh",
+ ".sql",
+ ".swift",
+ ".ts",
+ ".tsx",
+}
+
+_IMAGE_EXTENSIONS = {".bmp", ".gif", ".jpeg", ".jpg", ".png", ".svg", ".tif", ".tiff", ".webp"}
+
+_AUDIO_EXTENSIONS = {".aac", ".flac", ".m4a", ".mp3", ".mpga", ".ogg", ".opus", ".wav", ".webm"}
+
+_VIDEO_EXTENSIONS = {".avi", ".m4v", ".mkv", ".mov", ".mp4", ".mpeg", ".mpg", ".webm"}
+
+_SIDECAR_LABELS = (
+ "alt",
+ "caption",
+ "description",
+ "frames",
+ "meta",
+ "metadata",
+ "notes",
+ "ocr",
+ "summary",
+ "transcript",
+)
+
+_SIDECAR_TEXT_EXTENSIONS = (
+ ".json",
+ ".jsonl",
+ ".md",
+ ".txt",
+)
+
+_SIDECAR_SOURCE_MODALITIES = {"audio", "binary", "document", "image", "video"}
+
+_SOUL_KEYWORDS = (
+ "persona",
+ "personality",
+ "soul",
+ "tone",
+ "voice",
+ "writing style",
+ "language style",
+ "interaction style",
+ "\u4eba\u8bbe",
+ "\u4eba\u683c",
+ "\u8bed\u6c14",
+ "\u8bed\u8c03",
+ "\u8bed\u8a00\u98ce\u683c",
+ "\u8868\u8fbe\u98ce\u683c",
+)
+
+_SKILL_KEYWORDS = (
+ "ability",
+ "capability",
+ "skill",
+ "tool",
+ "workflow",
+ "procedure",
+ "playbook",
+ "how to",
+ "lesson learned",
+ "\u6280\u80fd",
+ "\u80fd\u529b",
+ "\u5de5\u5177",
+ "\u5de5\u4f5c\u6d41",
+ "\u6d41\u7a0b",
+ "\u65b9\u6cd5",
+ "\u7ecf\u9a8c",
+)
+
+
+__all__ = [
+ "EvolutionReviewApplyResult",
+ "FolderCompileResult",
+ "FolderHealthIssue",
+ "FolderHealthResult",
+ "FolderHealthSeverity",
+ "FolderMemoryCompiler",
+ "FolderMemoryCompilerConfig",
+ "FolderScaffoldResult",
+ "FolderSourceState",
+ "FolderSourceStatus",
+ "FolderStatusResult",
+ "FolderWatchEvent",
+ "MarkdownMemoryEntry",
+ "compile_folder_to_markdown",
+ "compile_folder_to_markdown_sync",
+ "inspect_folder_memory_health",
+ "inspect_folder_memory_status",
+ "review_folder_evolution",
+ "scaffold_folder_memory_repository",
+ "watch_folder_to_markdown",
+ "watch_folder_to_markdown_sync",
+]
diff --git a/src/memu/app/folder_cli.py b/src/memu/app/folder_cli.py
new file mode 100644
index 00000000..6f361a80
--- /dev/null
+++ b/src/memu/app/folder_cli.py
@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from memu.app.cli_args import positive_float_arg, positive_int_arg, probability_arg
+from memu.app.folder import (
+ FolderCompileResult,
+ FolderMemoryCompilerConfig,
+ FolderWatchEvent,
+ compile_folder_to_markdown_sync,
+ watch_folder_to_markdown_sync,
+)
+from memu.app.self_evolve import EvolutionReviewConfig
+from memu.app.settings import default_api_key_env
+
+
+DEFAULT_FOLDER_MEMORY_TYPES = ("profile", "event", "knowledge", "behavior", "skill", "tool")
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ prog="memu-folder",
+ description="Compile a raw data folder through self-evolve review into a Markdown memory repository.",
+ )
+ parser.add_argument("source_folder", help="Folder containing raw source files to compile.")
+ parser.add_argument("output_folder", help="Folder where the Markdown memory repository will be written.")
+ parser.add_argument(
+ "--user",
+ action="append",
+ default=[],
+ metavar="KEY=VALUE",
+ help="User scope value passed through to MemoryService. Can be repeated.",
+ )
+ parser.add_argument(
+ "--max-text-chars",
+ type=positive_int_arg,
+ default=4000,
+ help="Maximum text evidence chars per source file.",
+ )
+ parser.add_argument("--raw-data-dir", default="raw_data", help="Name of the raw data directory in output.")
+ parser.add_argument("--metadata-dir", default=".memu", help="Name of the metadata directory in output.")
+ parser.add_argument("--derived-dir", default="derived", help="Name of the derived evidence directory.")
+ parser.add_argument(
+ "--exclude",
+ action="append",
+ default=[],
+ metavar="GLOB",
+ help="Exclude source files matching a posix glob. Can be repeated.",
+ )
+ parser.add_argument("--json", action="store_true", help="Print a machine-readable JSON summary.")
+ parser.add_argument("--watch", action="store_true", help="Keep polling the source folder and recompile on changes.")
+ parser.add_argument(
+ "--poll-interval",
+ type=positive_float_arg,
+ default=2.0,
+ help="Polling interval in seconds for --watch.",
+ )
+ parser.add_argument(
+ "--watch-max-runs",
+ type=positive_int_arg,
+ default=None,
+ help="Stop watch mode after this many compile events. Mainly useful for automation.",
+ )
+ parser.add_argument(
+ "--require-creator-review",
+ action="store_true",
+ help="Create Evolution Instructions and Patch Proposals without auto-applying them.",
+ )
+ parser.add_argument(
+ "--min-evolution-confidence",
+ type=probability_arg,
+ default=0.0,
+ help="Minimum confidence required for auto-approved evolution patches.",
+ )
+
+ service = parser.add_argument_group("MemoryService extraction")
+ service.add_argument(
+ "--use-memory-service",
+ action="store_true",
+ help="Use MemoryService and configured LLMs for richer multimodal extraction.",
+ )
+ service.add_argument("--provider", default="openai", help="LLM provider for MemoryService.")
+ service.add_argument("--client-backend", default="sdk", choices=("sdk", "httpx", "lazyllm_backend"))
+ service.add_argument("--base-url", default=None, help="Optional LLM API base URL.")
+ service.add_argument("--api-key", default=None, help="LLM API key. Defaults to the selected API key env var.")
+ service.add_argument(
+ "--api-key-env",
+ default=None,
+ help="Environment variable to read the API key from. Defaults to the provider's standard env var.",
+ )
+ service.add_argument("--chat-model", default="gpt-4o-mini", help="Chat/vision model for extraction.")
+ service.add_argument("--embed-model", default="text-embedding-3-small", help="Embedding model for indexing.")
+ service.add_argument(
+ "--memory-types",
+ default=",".join(DEFAULT_FOLDER_MEMORY_TYPES),
+ help="Comma-separated MemoryService memory types to extract.",
+ )
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ parser = build_parser()
+ args = parser.parse_args(argv)
+
+ config = FolderMemoryCompilerConfig(
+ raw_data_dir_name=args.raw_data_dir,
+ metadata_dir_name=args.metadata_dir,
+ derived_dir_name=args.derived_dir,
+ exclude_patterns=tuple(args.exclude),
+ max_text_chars=args.max_text_chars,
+ use_memory_service=args.use_memory_service,
+ evolution_review=_evolution_review_config(args),
+ )
+ memory_service = _build_memory_service(args)
+ user = _parse_user_scope(args.user)
+ if args.watch:
+ watch_folder_to_markdown_sync(
+ args.source_folder,
+ args.output_folder,
+ memory_service=memory_service,
+ user=user,
+ config=config,
+ poll_interval=args.poll_interval,
+ max_runs=args.watch_max_runs,
+ on_event=lambda event: _print_watch_event(event, as_json=args.json),
+ )
+ else:
+ result = compile_folder_to_markdown_sync(
+ args.source_folder,
+ args.output_folder,
+ memory_service=memory_service,
+ user=user,
+ config=config,
+ )
+ if args.json:
+ print(json.dumps(_result_summary(result), indent=2, sort_keys=True))
+ else:
+ _print_human_summary(result)
+ return 0
+
+
+def _build_memory_service(args: argparse.Namespace) -> Any | None:
+ if not args.use_memory_service:
+ return None
+
+ from memu.app.service import MemoryService
+
+ llm_profile = _llm_profile_from_args(args)
+
+ return MemoryService(
+ llm_profiles={"default": llm_profile},
+ memorize_config={"memory_types": _parse_csv(args.memory_types)},
+ )
+
+
+def _llm_profile_from_args(args: argparse.Namespace) -> dict[str, Any]:
+ api_key_env = args.api_key_env or default_api_key_env(args.provider)
+ api_key = args.api_key or os.getenv(api_key_env) or api_key_env
+ llm_profile: dict[str, Any] = {
+ "provider": args.provider,
+ "client_backend": args.client_backend,
+ "api_key": api_key,
+ "chat_model": args.chat_model,
+ "embed_model": args.embed_model,
+ }
+ if args.base_url:
+ llm_profile["base_url"] = args.base_url
+
+ return llm_profile
+
+
+def _parse_user_scope(values: Sequence[str]) -> dict[str, str]:
+ user: dict[str, str] = {}
+ for raw in values:
+ key, sep, value = raw.partition("=")
+ if not sep or not key.strip():
+ msg = f"--user values must be KEY=VALUE, got: {raw!r}"
+ raise SystemExit(msg)
+ user[key.strip()] = value
+ return user
+
+
+def _parse_csv(value: str) -> list[str]:
+ return [part.strip() for part in value.split(",") if part.strip()]
+
+
+def _evolution_review_config(args: argparse.Namespace) -> EvolutionReviewConfig:
+ if args.min_evolution_confidence < 0 or args.min_evolution_confidence > 1:
+ msg = "--min-evolution-confidence must be between 0 and 1"
+ raise SystemExit(msg)
+ return EvolutionReviewConfig(
+ auto_approve=not args.require_creator_review,
+ min_confidence=args.min_evolution_confidence,
+ )
+
+
+def _result_summary(result: FolderCompileResult) -> dict[str, Any]:
+ return {
+ "output_dir": str(result.output_dir),
+ "raw_data_dir": str(result.raw_data_dir),
+ "manifest_path": str(result.manifest_path),
+ "processed": result.processed,
+ "skipped": result.skipped,
+ "removed": result.removed,
+ "entry_count": len(result.entries),
+ "evolution_instruction_count": len(result.evolution_instructions),
+ "patch_proposal_count": len(result.patch_proposals),
+ "review_decision_count": len(result.review_decisions),
+ "entries_by_bucket": {
+ "memory": sum(1 for entry in result.entries if entry.bucket == "memory"),
+ "soul": sum(1 for entry in result.entries if entry.bucket == "soul"),
+ "skill": sum(1 for entry in result.entries if entry.bucket == "skill"),
+ },
+ }
+
+
+def _print_human_summary(result: FolderCompileResult) -> None:
+ summary = _result_summary(result)
+ print("memU folder compile complete")
+ print(f" output: {summary['output_dir']}")
+ print(f" raw_data: {summary['raw_data_dir']}")
+ print(f" manifest: {summary['manifest_path']}")
+ print(f" processed: {len(result.processed)}")
+ print(f" skipped: {len(result.skipped)}")
+ print(f" removed: {len(result.removed)}")
+ print(f" entries: {summary['entry_count']} {summary['entries_by_bucket']}")
+ print(
+ " evolution: "
+ f"instructions={summary['evolution_instruction_count']} "
+ f"proposals={summary['patch_proposal_count']} "
+ f"reviews={summary['review_decision_count']}"
+ )
+
+
+def _watch_event_summary(event: FolderWatchEvent) -> dict[str, Any]:
+ summary = _result_summary(event.result)
+ summary["iteration"] = event.iteration
+ summary["reason"] = event.reason
+ if event.status is not None:
+ status = event.status.to_dict()
+ summary["delta"] = {
+ "counts": status["counts"],
+ "new": status["new"],
+ "changed": status["changed"],
+ "removed": status["removed"],
+ }
+ return summary
+
+
+def _print_watch_event(event: FolderWatchEvent, *, as_json: bool) -> None:
+ if as_json:
+ print(json.dumps(_watch_event_summary(event), sort_keys=True), flush=True)
+ return
+ print(f"memU folder watch event #{event.iteration}: {event.reason}", flush=True)
+ if event.status is not None:
+ counts = event.status.to_dict()["counts"]
+ print(
+ f" delta: new={counts['new']} changed={counts['changed']} removed={counts['removed']}",
+ flush=True,
+ )
+ _print_human_summary(event.result)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/src/memu/app/harness_config.py b/src/memu/app/harness_config.py
new file mode 100644
index 00000000..437dca4b
--- /dev/null
+++ b/src/memu/app/harness_config.py
@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Sequence
+from pathlib import Path
+from typing import Any, cast
+
+from memu.app.markdown_context import ContextBucket
+
+
+HARNESS_CONFIG_NAME = "harness.json"
+HARNESS_CONFIG_VERSION = 1
+DEFAULT_MAX_TEXT_CHARS = 4000
+DEFAULT_CONTEXT_MAX_CHARS = 8000
+DEFAULT_CONTEXT_FORMAT = "markdown"
+CONTEXT_BUCKETS = {"memory", "soul", "skill"}
+CONTEXT_FORMATS = {"markdown", "system", "messages", "json", "summary"}
+
+
+def harness_config_path(repo_dir: str | Path, metadata_dir: str = ".memu") -> Path:
+ return Path(repo_dir).resolve() / metadata_dir / HARNESS_CONFIG_NAME
+
+
+def default_harness_config(
+ *,
+ exclude_patterns: Sequence[str] = (),
+ max_text_chars: int = DEFAULT_MAX_TEXT_CHARS,
+) -> dict[str, Any]:
+ return {
+ "version": HARNESS_CONFIG_VERSION,
+ "compiler": {
+ "exclude_patterns": [pattern for pattern in exclude_patterns if pattern],
+ "max_text_chars": max_text_chars,
+ },
+ "context": {
+ "max_chars": DEFAULT_CONTEXT_MAX_CHARS,
+ "bucket_char_limits": {},
+ "format": DEFAULT_CONTEXT_FORMAT,
+ },
+ }
+
+
+def load_harness_config(repo_dir: str | Path, metadata_dir: str = ".memu") -> dict[str, Any]:
+ config_path = harness_config_path(repo_dir, metadata_dir)
+ if not config_path.exists():
+ return {}
+ try:
+ loaded = json.loads(config_path.read_text(encoding="utf-8-sig"))
+ except json.JSONDecodeError as exc:
+ msg = f"invalid harness config JSON at {config_path}: {exc}"
+ raise SystemExit(msg) from exc
+ if not isinstance(loaded, Mapping):
+ msg = f"harness config root must be a JSON object: {config_path}"
+ raise SystemExit(msg)
+ validate_harness_config(loaded, config_path)
+ return dict(loaded)
+
+
+def try_load_harness_config(repo_dir: str | Path, metadata_dir: str = ".memu") -> tuple[dict[str, Any], str | None]:
+ try:
+ return load_harness_config(repo_dir, metadata_dir), None
+ except SystemExit as exc:
+ return {}, str(exc)
+
+
+def validate_harness_config(config: Mapping[str, Any], config_path: Path) -> None:
+ version = config.get("version", HARNESS_CONFIG_VERSION)
+ if version != HARNESS_CONFIG_VERSION:
+ msg = f"harness config version must be {HARNESS_CONFIG_VERSION}: {config_path}"
+ raise SystemExit(msg)
+
+ compiler = config_section(config, "compiler", config_path)
+ config_string_list(compiler, "exclude_patterns", config_path)
+ config_positive_int(
+ compiler,
+ "max_text_chars",
+ DEFAULT_MAX_TEXT_CHARS,
+ config_path=config_path,
+ )
+
+ context = config_section(config, "context", config_path)
+ config_positive_int(
+ context,
+ "max_chars",
+ DEFAULT_CONTEXT_MAX_CHARS,
+ config_path=config_path,
+ )
+ config_context_buckets(context, config_path)
+ config_bucket_char_limits(context, config_path)
+ config_context_format(context, config_path)
+
+
+def config_section(config: Mapping[str, Any], name: str, config_path: Path) -> Mapping[str, Any]:
+ section = config.get(name, {})
+ if section is None:
+ return {}
+ if not isinstance(section, Mapping):
+ msg = f"harness config section {name!r} must be an object: {config_path}"
+ raise SystemExit(msg)
+ return section
+
+
+def config_string_list(section: Mapping[str, Any], name: str, config_path: Path) -> list[str]:
+ value = section.get(name, [])
+ if value is None:
+ return []
+ if not isinstance(value, list):
+ msg = f"harness config {name!r} must be a list of strings: {config_path}"
+ raise SystemExit(msg)
+ result: list[str] = []
+ for item in value:
+ if not isinstance(item, str):
+ msg = f"harness config {name!r} must be a list of strings: {config_path}"
+ raise SystemExit(msg)
+ clean = item.strip()
+ if clean:
+ result.append(clean)
+ return result
+
+
+def config_positive_int(
+ section: Mapping[str, Any],
+ name: str,
+ default: int,
+ *,
+ config_path: Path,
+) -> int:
+ value = section.get(name, default)
+ if value is None:
+ return default
+ if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+ msg = f"harness config {name!r} must be a positive integer: {config_path}"
+ raise SystemExit(msg)
+ return value
+
+
+def positive_int_or_default(value: int | None, default: int, *, flag_name: str) -> int:
+ if value is None:
+ return default
+ if value <= 0:
+ msg = f"{flag_name} must be greater than 0"
+ raise SystemExit(msg)
+ return value
+
+
+def arg_or_config_positive_int(
+ arg_value: int | None,
+ section: Mapping[str, Any],
+ name: str,
+ default: int,
+ *,
+ flag_name: str,
+ config_path: Path,
+) -> int:
+ if arg_value is not None:
+ return positive_int_or_default(arg_value, default, flag_name=flag_name)
+ return config_positive_int(section, name, default, config_path=config_path)
+
+
+def compiler_exclude_patterns(
+ cli_patterns: Sequence[str] | None,
+ compiler_section: Mapping[str, Any],
+ config_path: Path,
+) -> list[str]:
+ if cli_patterns is not None:
+ return [pattern for pattern in cli_patterns if pattern]
+ return config_string_list(compiler_section, "exclude_patterns", config_path)
+
+
+def config_context_buckets(section: Mapping[str, Any], config_path: Path) -> list[str]:
+ buckets = config_string_list(section, "buckets", config_path)
+ for bucket in buckets:
+ if bucket not in CONTEXT_BUCKETS:
+ msg = f"harness config context bucket must be one of memory, soul, skill: {config_path}"
+ raise SystemExit(msg)
+ return buckets
+
+
+def config_bucket_char_limits(section: Mapping[str, Any], config_path: Path) -> dict[ContextBucket, int]:
+ value = section.get("bucket_char_limits", {})
+ if value is None:
+ return {}
+ if not isinstance(value, Mapping):
+ msg = f"harness config 'bucket_char_limits' must be an object: {config_path}"
+ raise SystemExit(msg)
+ limits: dict[ContextBucket, int] = {}
+ for bucket, raw_limit in value.items():
+ if not isinstance(bucket, str) or bucket not in CONTEXT_BUCKETS:
+ msg = f"harness config bucket limit keys must be memory, soul, or skill: {config_path}"
+ raise SystemExit(msg)
+ if isinstance(raw_limit, bool) or not isinstance(raw_limit, int) or raw_limit <= 0:
+ msg = f"harness config bucket limit values must be positive integers: {config_path}"
+ raise SystemExit(msg)
+ limits[cast(ContextBucket, bucket)] = raw_limit
+ return limits
+
+
+def config_context_format(section: Mapping[str, Any], config_path: Path) -> str:
+ value = section.get("format", DEFAULT_CONTEXT_FORMAT)
+ if value is None:
+ return DEFAULT_CONTEXT_FORMAT
+ if not isinstance(value, str) or value not in CONTEXT_FORMATS:
+ msg = (
+ "harness config context format must be one of "
+ f"{', '.join(sorted(CONTEXT_FORMATS))}: {config_path}"
+ )
+ raise SystemExit(msg)
+ return value
+
+
+__all__ = [
+ "CONTEXT_BUCKETS",
+ "CONTEXT_FORMATS",
+ "DEFAULT_CONTEXT_FORMAT",
+ "DEFAULT_CONTEXT_MAX_CHARS",
+ "DEFAULT_MAX_TEXT_CHARS",
+ "HARNESS_CONFIG_NAME",
+ "HARNESS_CONFIG_VERSION",
+ "arg_or_config_positive_int",
+ "compiler_exclude_patterns",
+ "config_bucket_char_limits",
+ "config_context_buckets",
+ "config_context_format",
+ "config_positive_int",
+ "config_section",
+ "config_string_list",
+ "default_harness_config",
+ "harness_config_path",
+ "load_harness_config",
+ "positive_int_or_default",
+ "try_load_harness_config",
+ "validate_harness_config",
+]
diff --git a/src/memu/app/markdown_context.py b/src/memu/app/markdown_context.py
new file mode 100644
index 00000000..4ebdc66c
--- /dev/null
+++ b/src/memu/app/markdown_context.py
@@ -0,0 +1,501 @@
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, cast
+
+from memu.app.folder import GENERATED_END, GENERATED_START
+
+
+ContextBucket = Literal["memory", "soul", "skill"]
+ContextSectionKind = Literal["generated", "manual"]
+
+
+@dataclass(frozen=True)
+class MarkdownContextSection:
+ id: str
+ bucket: ContextBucket
+ title: str
+ content: str
+ source: str | None = None
+ evidence: str | None = None
+ tags: list[str] = field(default_factory=list)
+ kind: ContextSectionKind = "generated"
+ score: float = 0.0
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "id": self.id,
+ "bucket": self.bucket,
+ "title": self.title,
+ "content": self.content,
+ "source": self.source,
+ "evidence": self.evidence,
+ "tags": list(self.tags),
+ "kind": self.kind,
+ "score": self.score,
+ }
+
+
+@dataclass(frozen=True)
+class MarkdownContextPack:
+ repo_dir: Path
+ query: str | None
+ max_chars: int
+ used_chars: int
+ sections: list[MarkdownContextSection]
+ omitted_count: int = 0
+ bucket_char_limits: dict[ContextBucket, int] = field(default_factory=dict)
+ used_chars_by_bucket: dict[ContextBucket, int] = field(default_factory=dict)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "repo_dir": str(self.repo_dir),
+ "query": self.query,
+ "max_chars": self.max_chars,
+ "used_chars": self.used_chars,
+ "bucket_char_limits": dict(self.bucket_char_limits),
+ "used_chars_by_bucket": dict(self.used_chars_by_bucket),
+ "omitted_count": self.omitted_count,
+ "sections": [section.to_dict() for section in self.sections],
+ }
+
+ def to_summary(self) -> dict[str, Any]:
+ buckets = {bucket: 0 for bucket in ("memory", "soul", "skill")}
+ kinds = {kind: 0 for kind in ("generated", "manual")}
+ sources: set[str] = set()
+ section_summaries: list[dict[str, Any]] = []
+ for section in self.sections:
+ buckets[section.bucket] += 1
+ kinds[section.kind] += 1
+ if section.source:
+ sources.add(section.source)
+ section_summaries.append(
+ {
+ "id": section.id,
+ "bucket": section.bucket,
+ "kind": section.kind,
+ "title": section.title,
+ "source": section.source,
+ "evidence": section.evidence,
+ "tags": list(section.tags),
+ "score": section.score,
+ }
+ )
+ return {
+ "repo_dir": str(self.repo_dir),
+ "query": self.query,
+ "max_chars": self.max_chars,
+ "used_chars": self.used_chars,
+ "bucket_char_limits": dict(self.bucket_char_limits),
+ "used_chars_by_bucket": dict(self.used_chars_by_bucket),
+ "omitted_count": self.omitted_count,
+ "section_count": len(self.sections),
+ "buckets": buckets,
+ "kinds": kinds,
+ "sources": sorted(sources),
+ "sections": section_summaries,
+ }
+
+ def to_markdown(self) -> str:
+ lines = [""]
+ if self.query:
+ lines.extend(["", f"Query: {self.query}"])
+ for section in self.sections:
+ lines.extend(
+ [
+ "",
+ f"## {section.bucket}: {section.title}",
+ "",
+ f"- id: {section.id}",
+ f"- kind: {section.kind}",
+ ]
+ )
+ if section.source:
+ lines.append(f"- source: {section.source}")
+ if section.evidence:
+ lines.append(f"- evidence: {section.evidence}")
+ if section.tags:
+ lines.append(f"- tags: {', '.join(section.tags)}")
+ lines.extend(["", section.content.strip()])
+ if self.omitted_count:
+ lines.extend(["", f"[{self.omitted_count} section(s) omitted by context budget]"])
+ lines.extend(["", ""])
+ return "\n".join(lines).strip() + "\n"
+
+ def to_system_prompt(self) -> str:
+ return _CONTEXT_SYSTEM_INSTRUCTIONS + "\n\n" + self.to_markdown()
+
+ def to_messages(self) -> list[dict[str, str]]:
+ return [{"role": "system", "content": self.to_system_prompt()}]
+
+ def inject_into_messages(
+ self,
+ messages: Sequence[Mapping[str, Any]],
+ *,
+ replace_existing: bool = True,
+ ) -> list[dict[str, Any]]:
+ return inject_context_messages(messages, self, replace_existing=replace_existing)
+
+
+class MarkdownMemoryRepository:
+ """Read a Markdown-backed memU repository and assemble context packs."""
+
+ def __init__(self, repo_dir: str | Path):
+ self.repo_dir = Path(repo_dir).resolve()
+
+ def list_sections(
+ self,
+ *,
+ buckets: Sequence[ContextBucket] | None = None,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ ) -> list[MarkdownContextSection]:
+ requested = set(buckets or ("soul", "memory", "skill"))
+ sections: list[MarkdownContextSection] = []
+ if include_generated:
+ sections.extend(section for section in self._load_generated_sections() if section.bucket in requested)
+ if include_manual:
+ sections.extend(section for section in self._load_manual_sections() if section.bucket in requested)
+ return self._sort_sections(sections)
+
+ def build_context_pack(
+ self,
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int = 8000,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+ ) -> MarkdownContextPack:
+ candidates = self.list_sections(
+ buckets=buckets,
+ include_generated=include_generated,
+ include_manual=include_manual,
+ )
+ ranked = self._rank_sections(candidates, query)
+ limits = self._normalize_bucket_char_limits(bucket_char_limits)
+ selected: list[MarkdownContextSection] = []
+ used = 0
+ used_by_bucket: dict[ContextBucket, int] = {}
+ omitted = 0
+ for section in ranked:
+ rendered_len = self._section_char_cost(section)
+ remaining = max_chars - used
+ if section.bucket in limits:
+ remaining = min(remaining, limits[section.bucket] - used_by_bucket.get(section.bucket, 0))
+ if remaining <= 0:
+ omitted += 1
+ continue
+ if rendered_len > remaining:
+ if self._bucket_has_selected(selected, section.bucket):
+ omitted += 1
+ continue
+ truncated = self._truncate_section(section, remaining)
+ selected.append(truncated)
+ rendered_len = self._section_char_cost(truncated)
+ used += rendered_len
+ used_by_bucket[section.bucket] = used_by_bucket.get(section.bucket, 0) + rendered_len
+ continue
+ selected.append(section)
+ used += rendered_len
+ used_by_bucket[section.bucket] = used_by_bucket.get(section.bucket, 0) + rendered_len
+ return MarkdownContextPack(
+ repo_dir=self.repo_dir,
+ query=query,
+ max_chars=max_chars,
+ used_chars=used,
+ sections=selected,
+ omitted_count=omitted,
+ bucket_char_limits=limits,
+ used_chars_by_bucket=used_by_bucket,
+ )
+
+ def _load_generated_sections(self) -> list[MarkdownContextSection]:
+ manifest_path = self.repo_dir / ".memu" / "manifest.json"
+ if not manifest_path.exists():
+ return []
+ try:
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
+ except json.JSONDecodeError:
+ return []
+ sources = manifest.get("sources", {})
+ if not isinstance(sources, Mapping):
+ return []
+
+ sections: list[MarkdownContextSection] = []
+ for source in sources.values():
+ if not isinstance(source, Mapping):
+ continue
+ for entry in source.get("entries", []):
+ if not isinstance(entry, Mapping):
+ continue
+ bucket = str(entry.get("bucket", "memory"))
+ if bucket not in {"memory", "soul", "skill"}:
+ continue
+ sections.append(
+ MarkdownContextSection(
+ id=str(entry.get("id", "")),
+ bucket=cast(ContextBucket, bucket),
+ title=str(entry.get("title", "")),
+ content=str(entry.get("body", "")).strip(),
+ source=str(entry.get("source")) if entry.get("source") else None,
+ evidence=str(entry.get("evidence")) if entry.get("evidence") else None,
+ tags=[str(tag) for tag in entry.get("tags", [])],
+ kind="generated",
+ )
+ )
+ return sections
+
+ def _load_manual_sections(self) -> list[MarkdownContextSection]:
+ sections: list[MarkdownContextSection] = []
+ for bucket in ("soul", "memory", "skill"):
+ paths = [self.repo_dir / f"{bucket}.md"]
+ bucket_dir = self.repo_dir / bucket
+ if bucket_dir.exists():
+ paths.extend(sorted(bucket_dir.rglob("*.md")))
+ for path in paths:
+ if not path.exists() or not path.is_file():
+ continue
+ manual = self._strip_generated_blocks(path.read_text(encoding="utf-8-sig")).strip()
+ manual = self._strip_empty_title(manual, bucket)
+ if bucket == "skill" and path == self.repo_dir / "skill.md":
+ manual = self._strip_promoted_skill_index_sections(manual)
+ if not manual.strip():
+ continue
+ rel_path = path.relative_to(self.repo_dir).as_posix()
+ sections.append(
+ MarkdownContextSection(
+ id=f"manual:{rel_path}",
+ bucket=cast(ContextBucket, bucket),
+ title=f"Manual {bucket} notes from {rel_path}",
+ content=manual,
+ source=rel_path,
+ tags=["manual"],
+ kind="manual",
+ )
+ )
+ return sections
+
+ def _rank_sections(
+ self,
+ sections: Sequence[MarkdownContextSection],
+ query: str | None,
+ ) -> list[MarkdownContextSection]:
+ query_terms = self._terms(query or "")
+ ranked: list[MarkdownContextSection] = []
+ for section in sections:
+ score = self._bucket_priority(section.bucket)
+ if query_terms:
+ haystack = " ".join([section.title, section.content, " ".join(section.tags)]).lower()
+ score += sum(1.0 for term in query_terms if term in haystack)
+ if section.kind == "manual":
+ score += 0.25
+ ranked.append(
+ MarkdownContextSection(
+ id=section.id,
+ bucket=section.bucket,
+ title=section.title,
+ content=section.content,
+ source=section.source,
+ evidence=section.evidence,
+ tags=list(section.tags),
+ kind=section.kind,
+ score=score,
+ )
+ )
+ return sorted(ranked, key=lambda item: (-item.score, self._bucket_sort_key(item.bucket), item.title))
+
+ def _sort_sections(self, sections: Iterable[MarkdownContextSection]) -> list[MarkdownContextSection]:
+ return sorted(sections, key=lambda item: (self._bucket_sort_key(item.bucket), item.kind, item.title))
+
+ def _bucket_priority(self, bucket: ContextBucket) -> float:
+ return {"soul": 3.0, "memory": 2.0, "skill": 1.0}[bucket]
+
+ def _bucket_sort_key(self, bucket: ContextBucket) -> int:
+ return {"soul": 0, "memory": 1, "skill": 2}[bucket]
+
+ def _section_char_cost(self, section: MarkdownContextSection) -> int:
+ return len(section.title) + len(section.content) + 160
+
+ def _truncate_section(self, section: MarkdownContextSection, max_chars: int) -> MarkdownContextSection:
+ budget = max(0, max_chars - len(section.title) - 200)
+ content = section.content
+ if len(content) > budget:
+ content = content[:budget].rstrip() + "\n\n[truncated]"
+ return MarkdownContextSection(
+ id=section.id,
+ bucket=section.bucket,
+ title=section.title,
+ content=content,
+ source=section.source,
+ evidence=section.evidence,
+ tags=list(section.tags),
+ kind=section.kind,
+ score=section.score,
+ )
+
+ def _bucket_has_selected(
+ self,
+ sections: Sequence[MarkdownContextSection],
+ bucket: ContextBucket,
+ ) -> bool:
+ return any(section.bucket == bucket for section in sections)
+
+ def _normalize_bucket_char_limits(
+ self,
+ bucket_char_limits: Mapping[ContextBucket, int] | None,
+ ) -> dict[ContextBucket, int]:
+ if not bucket_char_limits:
+ return {}
+ normalized: dict[ContextBucket, int] = {}
+ for bucket, limit in bucket_char_limits.items():
+ if bucket not in {"memory", "soul", "skill"}:
+ msg = f"unknown context bucket: {bucket}"
+ raise ValueError(msg)
+ if limit <= 0:
+ msg = "bucket character limits must be greater than 0"
+ raise ValueError(msg)
+ normalized[cast(ContextBucket, bucket)] = int(limit)
+ return normalized
+
+ def _strip_generated_blocks(self, text: str) -> str:
+ pattern = re.compile(
+ rf"{re.escape(GENERATED_START)}.*?{re.escape(GENERATED_END)}",
+ re.DOTALL,
+ )
+ return pattern.sub("", text)
+
+ def _strip_empty_title(self, text: str, bucket: str) -> str:
+ lines = text.splitlines()
+ if not lines:
+ return text
+ first = lines[0].strip().lower()
+ rest = "\n".join(lines[1:]).strip()
+ generated_titles = {
+ f"# {bucket}".lower(),
+ f"# {bucket.title()}".lower(),
+ }
+ if first in generated_titles or first.startswith(f"# {bucket} from "):
+ return rest
+ if first.startswith("# ") and not rest:
+ return ""
+ return text
+
+ def _strip_promoted_skill_index_sections(self, text: str) -> str:
+ lines = text.splitlines()
+ kept: list[str] = []
+ idx = 0
+ while idx < len(lines):
+ line = lines[idx]
+ if line.strip().lower().startswith("## promoted skill:"):
+ end = idx + 1
+ while end < len(lines) and not lines[end].startswith("## "):
+ end += 1
+ section = "\n".join(lines[idx:end])
+ card_path = self._promoted_card_path(section)
+ if card_path is not None and card_path.is_file():
+ idx = end
+ continue
+ kept.append(line)
+ idx += 1
+ return "\n".join(kept).strip()
+
+ def _promoted_card_path(self, section: str) -> Path | None:
+ for line in section.splitlines():
+ stripped = line.strip()
+ if not stripped.lower().startswith("- card:"):
+ continue
+ rel_path = stripped.partition(":")[2].strip()
+ if not rel_path:
+ return None
+ card_path = (self.repo_dir / rel_path).resolve()
+ try:
+ card_path.relative_to(self.repo_dir)
+ except ValueError:
+ return None
+ return card_path
+ return None
+
+ def _terms(self, query: str) -> set[str]:
+ return {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]+", query) if len(term) >= 2}
+
+
+def build_markdown_context_pack(
+ repo_dir: str | Path,
+ *,
+ query: str | None = None,
+ buckets: Sequence[ContextBucket] | None = None,
+ max_chars: int = 8000,
+ include_generated: bool = True,
+ include_manual: bool = True,
+ bucket_char_limits: Mapping[ContextBucket, int] | None = None,
+) -> MarkdownContextPack:
+ return MarkdownMemoryRepository(repo_dir).build_context_pack(
+ query=query,
+ buckets=buckets,
+ max_chars=max_chars,
+ include_generated=include_generated,
+ include_manual=include_manual,
+ bucket_char_limits=bucket_char_limits,
+ )
+
+
+def inject_context_messages(
+ messages: Sequence[Mapping[str, Any]],
+ context_pack: MarkdownContextPack,
+ *,
+ replace_existing: bool = True,
+) -> list[dict[str, Any]]:
+ injected = [dict(message) for message in messages]
+ context = context_pack.to_system_prompt()
+ if injected and injected[0].get("role") == "system" and isinstance(injected[0].get("content"), str):
+ base_content = str(injected[0]["content"])
+ if replace_existing:
+ base_content = _strip_memu_context(base_content)
+ injected[0]["content"] = _join_system_context(base_content, context)
+ return injected
+ injected.insert(0, {"role": "system", "content": context})
+ return injected
+
+
+def _strip_memu_context(content: str) -> str:
+ for tag in ("memu_context_instructions", "memu_context"):
+ content = re.sub(
+ rf"\s*<{tag}>.*?\s*",
+ "\n",
+ content,
+ flags=re.DOTALL,
+ )
+ return content.strip()
+
+
+def _join_system_context(base_content: str, context: str) -> str:
+ if not base_content.strip():
+ return context
+ return base_content.rstrip() + "\n\n" + context
+
+
+__all__ = [
+ "ContextBucket",
+ "MarkdownContextPack",
+ "MarkdownContextSection",
+ "MarkdownMemoryRepository",
+ "build_markdown_context_pack",
+ "inject_context_messages",
+]
+
+
+_CONTEXT_SYSTEM_INSTRUCTIONS = """
+Use the memU context below as retrieved working memory for the current task.
+- Prefer manual sections when they conflict with generated sections.
+- Use soul sections for persona, tone, language style, and interaction style.
+- Use skill sections for reusable procedures, tool habits, and lessons learned.
+- Use memory sections for durable facts, preferences, events, and knowledge.
+- Treat generated sections as evidence-backed summaries; inspect source/evidence paths when the task needs precision.
+- Do not invent facts beyond the context. If evidence is weak or conflicting, say what is uncertain.
+"""
diff --git a/src/memu/app/memorize.py b/src/memu/app/memorize.py
index 0f2a06fc..b6e7c7ed 100644
--- a/src/memu/app/memorize.py
+++ b/src/memu/app/memorize.py
@@ -12,6 +12,7 @@
import defusedxml.ElementTree as ET
from pydantic import BaseModel
+from memu.app.scope import scope_key_from_user
from memu.app.settings import CategoryConfig, CustomPrompt
from memu.database.models import CategoryItem, MemoryCategory, MemoryItem, MemoryType, Resource
from memu.prompts.category_summary import (
@@ -32,6 +33,7 @@
)
from memu.prompts.preprocess import PROMPTS as PREPROCESS_PROMPTS
from memu.utils.conversation import format_conversation_for_preprocess
+from memu.utils.dedupe import dedupe_resource_plans
from memu.utils.video import VideoFrameExtractor
from memu.workflow.step import WorkflowState, WorkflowStep
@@ -227,8 +229,7 @@ async def _memorize_extract_items(self, state: WorkflowState, step_context: Any)
return state
def _memorize_dedupe_merge(self, state: WorkflowState, step_context: Any) -> WorkflowState:
- # Placeholder for future dedup/merge logic
- state["resource_plans"] = state.get("resource_plans", [])
+ state["resource_plans"] = dedupe_resource_plans(state.get("resource_plans", []))
return state
async def _memorize_categorize_items(self, state: WorkflowState, step_context: Any) -> WorkflowState:
@@ -301,7 +302,7 @@ def _memorize_build_response(self, state: WorkflowState, step_context: Any) -> W
store = state["store"]
resources = [self._model_dump_without_embeddings(r) for r in state.get("resources", [])]
items = [self._model_dump_without_embeddings(item) for item in state.get("items", [])]
- relations = [rel.model_dump() for rel in state.get("relations", [])]
+ relations = [self._model_dump_without_embeddings(rel) for rel in state.get("relations", [])]
category_ids = state.get("category_ids") or list(ctx.category_ids)
categories = [
self._model_dump_without_embeddings(store.memory_category_repo.categories[c]) for c in category_ids
@@ -637,20 +638,48 @@ def _start_category_initialization(self, ctx: Context, store: Database) -> None:
async def _ensure_categories_ready(
self, ctx: Context, store: Database, user_scope: Mapping[str, Any] | None = None
) -> None:
- if ctx.categories_ready:
+ scope_key = scope_key_from_user(user_scope)
+ if ctx.categories_ready and ctx.category_scope_key == scope_key:
+ return
+ cached = ctx.category_cache.get(scope_key)
+ if cached is not None:
+ ctx.category_ids = list(cached[0])
+ ctx.category_name_to_id = dict(cached[1])
+ ctx.category_scope_key = scope_key
+ ctx.categories_ready = True
return
if ctx.category_init_task:
await ctx.category_init_task
ctx.category_init_task = None
- return
+ if ctx.categories_ready and ctx.category_scope_key == scope_key:
+ return
+ cached = ctx.category_cache.get(scope_key)
+ if cached is not None:
+ ctx.category_ids = list(cached[0])
+ ctx.category_name_to_id = dict(cached[1])
+ ctx.category_scope_key = scope_key
+ ctx.categories_ready = True
+ return
await self._initialize_categories(ctx, store, user_scope)
async def _initialize_categories(
self, ctx: Context, store: Database, user: Mapping[str, Any] | None = None
) -> None:
- if ctx.categories_ready:
+ scope_key = scope_key_from_user(user)
+ if ctx.categories_ready and ctx.category_scope_key == scope_key:
+ return
+ cached = ctx.category_cache.get(scope_key)
+ if cached is not None:
+ ctx.category_ids = list(cached[0])
+ ctx.category_name_to_id = dict(cached[1])
+ ctx.category_scope_key = scope_key
+ ctx.categories_ready = True
return
if not self.category_configs:
+ ctx.category_ids = []
+ ctx.category_name_to_id = {}
+ ctx.category_scope_key = scope_key
+ ctx.category_cache[scope_key] = ([], {})
ctx.categories_ready = True
return
cat_texts = [self._category_embedding_text(cfg) for cfg in self.category_configs]
@@ -665,6 +694,8 @@ async def _initialize_categories(
)
ctx.category_ids.append(cat.id)
ctx.category_name_to_id[name.lower()] = cat.id
+ ctx.category_scope_key = scope_key
+ ctx.category_cache[scope_key] = (list(ctx.category_ids), dict(ctx.category_name_to_id))
ctx.categories_ready = True
@staticmethod
diff --git a/src/memu/app/patch.py b/src/memu/app/patch.py
index c1796478..b0123cef 100644
--- a/src/memu/app/patch.py
+++ b/src/memu/app/patch.py
@@ -8,6 +8,7 @@
from pydantic import BaseModel
+from memu.app.scope import record_matches_scope
from memu.database.models import MemoryCategory, MemoryType
from memu.prompts.category_patch import CATEGORY_PATCH_PROMPT
from memu.workflow.step import WorkflowState, WorkflowStep
@@ -43,9 +44,9 @@ async def create_memory_item(
user: dict[str, Any] | None = None,
propagate: bool = True,
) -> dict[str, Any]:
- if memory_type not in get_args(MemoryType):
- msg = f"Invalid memory type: '{memory_type}', must be one of {get_args(MemoryType)}"
- raise ValueError(msg)
+ memory_type = _normalize_memory_type(memory_type)
+ memory_content = _normalize_memory_content(memory_content, field_name="memory_content")
+ memory_categories = _normalize_memory_categories(memory_categories, field_name="memory_categories")
ctx = self._get_context()
store = self._get_database()
@@ -85,9 +86,13 @@ async def update_memory_item(
if all((memory_type is None, memory_content is None, memory_categories is None)):
msg = "At least one of memory type, memory content, or memory categories is required for UPDATE operation"
raise ValueError(msg)
- if memory_type and memory_type not in get_args(MemoryType):
- msg = f"Invalid memory type: '{memory_type}', must be one of {get_args(MemoryType)}"
- raise ValueError(msg)
+ memory_id = _normalize_memory_id(memory_id)
+ if memory_type is not None:
+ memory_type = _normalize_memory_type(memory_type)
+ if memory_content is not None:
+ memory_content = _normalize_memory_content(memory_content, field_name="memory_content")
+ if memory_categories is not None:
+ memory_categories = _normalize_memory_categories(memory_categories, field_name="memory_categories")
ctx = self._get_context()
store = self._get_database()
@@ -122,6 +127,8 @@ async def delete_memory_item(
user: dict[str, Any] | None = None,
propagate: bool = True,
) -> dict[str, Any]:
+ memory_id = _normalize_memory_id(memory_id)
+
ctx = self._get_context()
store = self._get_database()
user_scope = self.user_model(**user).model_dump() if user is not None else None
@@ -258,6 +265,13 @@ def _list_delete_memory_item_initial_keys() -> set[str]:
"user",
}
+ @staticmethod
+ def _ensure_item_matches_user_scope(item: Any, user_scope: Mapping[str, Any] | None, memory_id: str) -> None:
+ if record_matches_scope(item, user_scope):
+ return
+ msg = f"Memory item with id {memory_id} not found"
+ raise ValueError(msg)
+
async def _patch_create_memory_item(self, state: WorkflowState, step_context: Any) -> WorkflowState:
memory_payload = state["memory_payload"]
ctx = state["ctx"]
@@ -270,6 +284,7 @@ async def _patch_create_memory_item(self, state: WorkflowState, step_context: An
content_embedding = (await self._get_llm_client().embed(embed_payload))[0]
item = store.memory_item_repo.create_item(
+ resource_id=None,
memory_type=memory_payload["type"],
summary=memory_payload["content"],
embedding=content_embedding,
@@ -301,6 +316,7 @@ async def _patch_update_memory_item(self, state: WorkflowState, step_context: An
if not item:
msg = f"Memory item with id {memory_id} not found"
raise ValueError(msg)
+ self._ensure_item_matches_user_scope(item, user, memory_id)
old_content = item.summary
old_item_categories = store.category_item_repo.get_item_categories(memory_id)
mapped_old_cat_ids = [cat.category_id for cat in old_item_categories]
@@ -319,7 +335,10 @@ async def _patch_update_memory_item(self, state: WorkflowState, step_context: An
embedding=content_embedding,
)
new_cat_names = memory_payload["categories"]
- mapped_new_cat_ids = self._map_category_names_to_ids(new_cat_names, ctx)
+ if new_cat_names is None:
+ mapped_new_cat_ids = mapped_old_cat_ids
+ else:
+ mapped_new_cat_ids = self._map_category_names_to_ids(new_cat_names, ctx)
cats_to_remove = set(mapped_old_cat_ids) - set(mapped_new_cat_ids)
cats_to_add = set(mapped_new_cat_ids) - set(mapped_old_cat_ids)
@@ -352,7 +371,8 @@ async def _patch_delete_memory_item(self, state: WorkflowState, step_context: An
if not item:
msg = f"Memory item with id {memory_id} not found"
raise ValueError(msg)
- item_categories = store.category_item_repo.get_item_categories(memory_id)
+ self._ensure_item_matches_user_scope(item, state["user"], memory_id)
+ item_categories = store.category_item_repo.clear_relations({"item_id": memory_id})
if propagate:
for cat in item_categories:
category_memory_updates[cat.category_id] = (item.summary, None)
@@ -477,3 +497,36 @@ def _parse_category_patch_response(self, response: str) -> tuple[bool, str]:
if updated_content == "empty":
updated_content = ""
return need_update, updated_content
+
+
+def _normalize_memory_id(value: Any) -> str:
+ return _normalize_non_empty_string(value, field_name="memory_id")
+
+
+def _normalize_memory_type(value: Any) -> MemoryType:
+ memory_type = _normalize_non_empty_string(value, field_name="memory_type")
+ if memory_type not in get_args(MemoryType):
+ msg = f"Invalid memory type: '{memory_type}', must be one of {get_args(MemoryType)}"
+ raise ValueError(msg)
+ return cast(MemoryType, memory_type)
+
+
+def _normalize_memory_content(value: Any, *, field_name: str) -> str:
+ return _normalize_non_empty_string(value, field_name=field_name)
+
+
+def _normalize_memory_categories(value: Any, *, field_name: str) -> list[str]:
+ if not isinstance(value, list):
+ msg = f"'{field_name}' must be a list of non-empty strings"
+ raise ValueError(msg)
+ normalized: list[str] = []
+ for index, item in enumerate(value):
+ normalized.append(_normalize_non_empty_string(item, field_name=f"{field_name}[{index}]"))
+ return normalized
+
+
+def _normalize_non_empty_string(value: Any, *, field_name: str) -> str:
+ if not isinstance(value, str) or not value.strip():
+ msg = f"'{field_name}' must be a non-empty string"
+ raise ValueError(msg)
+ return value.strip()
diff --git a/src/memu/app/retrieve.py b/src/memu/app/retrieve.py
index a7cbff5c..3e4ec7e5 100644
--- a/src/memu/app/retrieve.py
+++ b/src/memu/app/retrieve.py
@@ -8,12 +8,14 @@
from pydantic import BaseModel
+from memu.app.scope import concrete_scope_from_where, normalize_scope_where
from memu.database.inmemory.vector import cosine_topk
from memu.prompts.retrieve.llm_category_ranker import PROMPT as LLM_CATEGORY_RANKER_PROMPT
from memu.prompts.retrieve.llm_item_ranker import PROMPT as LLM_ITEM_RANKER_PROMPT
from memu.prompts.retrieve.llm_resource_ranker import PROMPT as LLM_RESOURCE_RANKER_PROMPT
from memu.prompts.retrieve.pre_retrieval_decision import SYSTEM_PROMPT as PRE_RETRIEVAL_SYSTEM_PROMPT
from memu.prompts.retrieve.pre_retrieval_decision import USER_PROMPT as PRE_RETRIEVAL_USER_PROMPT
+from memu.utils.retrieve import RetrieveMethod, RetrieveRanking, normalize_retrieve_method, normalize_retrieve_ranking
from memu.workflow.step import WorkflowState, WorkflowStep
logger = logging.getLogger(__name__)
@@ -30,7 +32,7 @@ class RetrieveMixin:
_run_workflow: Callable[..., Awaitable[WorkflowState]]
_get_context: Callable[[], Context]
_get_database: Callable[[], Database]
- _ensure_categories_ready: Callable[[Context, Database], Awaitable[None]]
+ _ensure_categories_ready: Callable[[Context, Database, Mapping[str, Any] | None], Awaitable[None]]
_get_step_llm_client: Callable[[Mapping[str, Any] | None], Any]
_get_step_embedding_client: Callable[[Mapping[str, Any] | None], Any]
_get_llm_client: Callable[..., Any]
@@ -41,36 +43,48 @@ class RetrieveMixin:
async def retrieve(
self,
- queries: list[dict[str, Any]],
+ queries: list[str | Mapping[str, Any]],
where: dict[str, Any] | None = None,
+ method: RetrieveMethod | str | None = None,
+ ranking: RetrieveRanking | str | None = None,
) -> dict[str, Any]:
+ if not isinstance(queries, list):
+ msg = "queries must be a non-empty list of strings or query objects"
+ raise TypeError(msg)
if not queries:
- raise ValueError("empty_queries")
+ msg = "queries must be a non-empty list of strings or query objects"
+ raise ValueError(msg)
+ normalized_queries = [self._normalize_query_item(query, index=index) for index, query in enumerate(queries)]
ctx = self._get_context()
store = self._get_database()
- original_query = self._extract_query_text(queries[-1])
- # await self._ensure_categories_ready(ctx, store)
+ original_query = self._extract_query_text(normalized_queries[-1])
where_filters = self._normalize_where(where)
- context_queries_objs = queries[:-1] if len(queries) > 1 else []
+ context_queries_objs: list[dict[str, Any]] = normalized_queries[:-1] if len(normalized_queries) > 1 else []
route_intention = self.retrieve_config.route_intention
retrieve_category = self.retrieve_config.category.enabled
retrieve_item = self.retrieve_config.item.enabled
retrieve_resource = self.retrieve_config.resource.enabled
sufficiency_check = self.retrieve_config.sufficiency_check
+ retrieve_method = normalize_retrieve_method(method, default=self.retrieve_config.method)
+ item_ranking = normalize_retrieve_ranking(ranking, default=self.retrieve_config.item.ranking)
+ bootstrap_scope = concrete_scope_from_where(where_filters)
+ if retrieve_category and bootstrap_scope is not None:
+ await self._ensure_categories_ready(ctx, store, bootstrap_scope)
- workflow_name = "retrieve_llm" if self.retrieve_config.method == "llm" else "retrieve_rag"
+ workflow_name = "retrieve_llm" if retrieve_method == "llm" else "retrieve_rag"
state: WorkflowState = {
- "method": self.retrieve_config.method,
+ "method": retrieve_method,
"original_query": original_query,
"context_queries": context_queries_objs,
"route_intention": route_intention,
- "skip_rewrite": len(queries) == 1,
+ "skip_rewrite": len(normalized_queries) == 1,
"retrieve_category": retrieve_category,
"retrieve_item": retrieve_item,
"retrieve_resource": retrieve_resource,
+ "item_ranking": item_ranking,
"sufficiency_check": sufficiency_check,
"ctx": ctx,
"store": store,
@@ -86,22 +100,38 @@ async def retrieve(
def _normalize_where(self, where: Mapping[str, Any] | None) -> dict[str, Any]:
"""Validate and clean the `where` scope filters against the configured user model."""
- if not where:
- return {}
+ return normalize_scope_where(self.user_model, where)
- valid_fields = set(getattr(self.user_model, "model_fields", {}).keys())
- cleaned: dict[str, Any] = {}
+ @staticmethod
+ def _normalize_query_item(query: str | Mapping[str, Any], *, index: int) -> dict[str, Any]:
+ if isinstance(query, str):
+ text = query.strip()
+ if not text:
+ raise ValueError(f"queries[{index}] must not be empty")
+ return {"role": "user", "content": text}
- for raw_key, value in where.items():
- if value is None:
- continue
- field = raw_key.split("__", 1)[0]
- if field not in valid_fields:
- msg = f"Unknown filter field '{field}' for current user scope"
- raise ValueError(msg)
- cleaned[raw_key] = value
+ if not isinstance(query, Mapping):
+ raise TypeError(f"queries[{index}] must be a string or query object")
+
+ role = query.get("role", "user")
+ if not isinstance(role, str) or not role.strip():
+ raise ValueError(f"queries[{index}].role must be a non-empty string")
+
+ content = query.get("content")
+ if isinstance(content, str):
+ text = content.strip()
+ if not text:
+ raise ValueError(f"queries[{index}].content must not be empty")
+ normalized_content: str | dict[str, str] = text
+ elif isinstance(content, Mapping):
+ text = content.get("text")
+ if not isinstance(text, str) or not text.strip():
+ raise ValueError(f"queries[{index}].content.text must be a non-empty string")
+ normalized_content = {"text": text.strip()}
+ else:
+ raise TypeError(f"queries[{index}].content must be a string or object with text")
- return cleaned
+ return {"role": role.strip(), "content": normalized_content}
def _build_rag_retrieve_workflow(self) -> list[WorkflowStep]:
steps = [
@@ -112,7 +142,7 @@ def _build_rag_retrieve_workflow(self) -> list[WorkflowStep]:
requires={"route_intention", "original_query", "context_queries", "skip_rewrite"},
produces={"needs_retrieval", "rewritten_query", "active_query", "next_step_query"},
capabilities={"llm"},
- config={"chat_llm_profile": self.retrieve_config.sufficiency_check_llm_profile},
+ config={"chat_llm_profile": self.retrieve_config.route_intention_llm_profile},
),
WorkflowStep(
step_id="route_category",
@@ -156,6 +186,7 @@ def _build_rag_retrieve_workflow(self) -> list[WorkflowStep]:
"where",
"active_query",
"query_vector",
+ "item_ranking",
},
produces={"item_hits", "query_vector"},
capabilities={"vector"},
@@ -219,6 +250,7 @@ def _list_retrieve_initial_keys(self) -> set[str]:
"retrieve_category",
"retrieve_item",
"retrieve_resource",
+ "item_ranking",
"sufficiency_check",
"ctx",
"store",
@@ -321,14 +353,15 @@ async def _rag_category_sufficiency(self, state: WorkflowState, step_context: An
state["query_vector"] = (await embed_client.embed([state["active_query"]]))[0]
return state
- def _extract_referenced_item_ids(self, state: WorkflowState) -> set[str]:
- """Extract item IDs from category summary references."""
+ def _extract_referenced_item_ids(self, state: WorkflowState) -> list[str]:
+ """Extract ordered, deduplicated ref IDs from category summary references."""
from memu.utils.references import extract_references
category_hits = state.get("category_hits") or []
summary_lookup = state.get("category_summary_lookup", {})
category_pool = state.get("category_pool") or {}
- referenced_item_ids: set[str] = set()
+ referenced_item_ids: list[str] = []
+ seen: set[str] = set()
for cid, _score in category_hits:
# Get summary from lookup or category
@@ -339,7 +372,11 @@ def _extract_referenced_item_ids(self, state: WorkflowState) -> set[str]:
summary = cat.summary
if summary:
refs = extract_references(summary)
- referenced_item_ids.update(refs)
+ for ref_id in refs:
+ if ref_id in seen:
+ continue
+ referenced_item_ids.append(ref_id)
+ seen.add(ref_id)
return referenced_item_ids
@@ -360,12 +397,34 @@ async def _rag_recall_items(self, state: WorkflowState, step_context: Any) -> Wo
qvec,
self.retrieve_config.item.top_k,
where=where_filters,
- ranking=self.retrieve_config.item.ranking,
+ ranking=state.get("item_ranking", self.retrieve_config.item.ranking),
recency_decay_days=self.retrieve_config.item.recency_decay_days,
)
+ if getattr(self.retrieve_config.item, "use_category_references", False):
+ ref_ids = self._extract_referenced_item_ids(state)
+ if ref_ids:
+ referenced_items = store.memory_item_repo.list_items_by_ref_ids(ref_ids, where_filters)
+ items_pool.update(referenced_items)
+ state["item_hits"] = self._merge_referenced_item_hits(
+ state["item_hits"],
+ referenced_items,
+ )
state["item_pool"] = items_pool
return state
+ @staticmethod
+ def _merge_referenced_item_hits(
+ item_hits: Sequence[tuple[str, float]],
+ referenced_items: Mapping[str, Any],
+ ) -> list[tuple[str, float]]:
+ merged = list(item_hits)
+ seen = {item_id for item_id, _score in merged}
+ for item_id in referenced_items:
+ if item_id not in seen:
+ merged.append((item_id, 1.0))
+ seen.add(item_id)
+ return merged
+
async def _rag_item_sufficiency(self, state: WorkflowState, step_context: Any) -> WorkflowState:
if not state.get("needs_retrieval"):
state["proceed_to_resources"] = False
@@ -457,16 +516,16 @@ def _build_llm_retrieve_workflow(self) -> list[WorkflowStep]:
step_id="route_intention",
role="route_intention",
handler=self._llm_route_intention,
- requires={"original_query", "context_queries", "skip_rewrite"},
+ requires={"route_intention", "original_query", "context_queries", "skip_rewrite"},
produces={"needs_retrieval", "rewritten_query", "active_query", "next_step_query"},
capabilities={"llm"},
- config={"llm_profile": self.retrieve_config.sufficiency_check_llm_profile},
+ config={"llm_profile": self.retrieve_config.route_intention_llm_profile},
),
WorkflowStep(
step_id="route_category",
role="route_category",
handler=self._llm_route_category,
- requires={"needs_retrieval", "active_query", "ctx", "store", "where"},
+ requires={"retrieve_category", "needs_retrieval", "active_query", "ctx", "store", "where"},
produces={"category_hits"},
capabilities={"llm"},
config={"llm_profile": self.retrieve_config.llm_ranking_llm_profile},
@@ -487,6 +546,7 @@ def _build_llm_retrieve_workflow(self) -> list[WorkflowStep]:
requires={
"needs_retrieval",
"proceed_to_items",
+ "retrieve_item",
"ctx",
"store",
"where",
@@ -513,6 +573,7 @@ def _build_llm_retrieve_workflow(self) -> list[WorkflowStep]:
requires={
"needs_retrieval",
"proceed_to_resources",
+ "retrieve_resource",
"active_query",
"ctx",
"store",
@@ -568,7 +629,7 @@ async def _llm_route_intention(self, state: WorkflowState, step_context: Any) ->
return state
async def _llm_route_category(self, state: WorkflowState, step_context: Any) -> WorkflowState:
- if not state.get("needs_retrieval"):
+ if not state.get("retrieve_category") or not state.get("needs_retrieval"):
state["category_hits"] = []
return state
llm_client = self._get_step_llm_client(step_context)
@@ -613,7 +674,7 @@ async def _llm_category_sufficiency(self, state: WorkflowState, step_context: An
return state
async def _llm_recall_items(self, state: WorkflowState, step_context: Any) -> WorkflowState:
- if not state.get("needs_retrieval") or not state.get("proceed_to_items"):
+ if not state.get("retrieve_item") or not state.get("needs_retrieval") or not state.get("proceed_to_items"):
state["item_hits"] = []
return state
@@ -682,7 +743,11 @@ async def _llm_item_sufficiency(self, state: WorkflowState, step_context: Any) -
return state
async def _llm_recall_resources(self, state: WorkflowState, step_context: Any) -> WorkflowState:
- if not state.get("needs_retrieval") or not state.get("proceed_to_resources"):
+ if (
+ not state.get("needs_retrieval")
+ or not state.get("retrieve_resource")
+ or not state.get("proceed_to_resources")
+ ):
state["resource_hits"] = []
return state
@@ -746,7 +811,7 @@ async def _rank_categories_by_summary(
async def _decide_if_retrieval_needed(
self,
query: str,
- context_queries: list[dict[str, Any]] | None,
+ context_queries: Sequence[str | Mapping[str, Any]] | None,
retrieved_content: str | None = None,
system_prompt: str | None = None,
llm_client: Any | None = None,
@@ -756,7 +821,7 @@ async def _decide_if_retrieval_needed(
Args:
query: The current query string
- context_queries: List of previous query objects with role and content
+ context_queries: Previous query strings or objects with role and content
retrieved_content: Content retrieved so far (if checking for sufficiency)
system_prompt: Optional system prompt override
@@ -783,7 +848,7 @@ async def _decide_if_retrieval_needed(
return decision == "RETRIEVE", rewritten
- def _format_query_context(self, queries: list[dict[str, Any]] | None) -> str:
+ def _format_query_context(self, queries: Sequence[str | Mapping[str, Any]] | None) -> str:
"""Format query context for prompts, including role information"""
if not queries:
return "No query context."
@@ -793,7 +858,7 @@ def _format_query_context(self, queries: list[dict[str, Any]] | None) -> str:
if isinstance(q, str):
# Backward compatibility
lines.append(f"- {q}")
- elif isinstance(q, dict):
+ elif isinstance(q, Mapping):
role = q.get("role", "user")
content = q.get("content")
if isinstance(content, dict):
@@ -809,32 +874,39 @@ def _format_query_context(self, queries: list[dict[str, Any]] | None) -> str:
return "\n".join(lines)
@staticmethod
- def _extract_query_text(query: dict[str, Any]) -> str:
+ def _extract_query_text(query: str | Mapping[str, Any]) -> str:
"""
Extract text content from query message structure.
Args:
- query: Query in format {"role": "user", "content": {"text": "..."}}
+ query: Query string or message in format {"role": "user", "content": "..."}.
+ The legacy {"content": {"text": "..."}} shape is also accepted.
Returns:
The extracted text string
"""
if isinstance(query, str):
# Backward compatibility: if it's already a string, return it
- return query
+ text = query.strip()
+ if not text:
+ raise ValueError("EMPTY")
+ return text
- if not isinstance(query, dict):
+ if not isinstance(query, Mapping):
raise TypeError("INVALID")
content = query.get("content")
if isinstance(content, dict):
text = content.get("text", "")
- if not text:
+ if not isinstance(text, str) or not text.strip():
raise ValueError("EMPTY")
- return str(text)
+ return text.strip()
elif isinstance(content, str):
# Also support {"role": "user", "content": "text"} format
- return content
+ text = content.strip()
+ if not text:
+ raise ValueError("EMPTY")
+ return text
else:
raise TypeError("INVALID")
@@ -868,7 +940,7 @@ async def _embedding_based_retrieve(
self,
query: str,
top_k: int,
- context_queries: list[dict[str, Any]] | None,
+ context_queries: Sequence[str | Mapping[str, Any]] | None,
ctx: Context,
store: Database,
llm_client: Any | None = None,
@@ -1022,7 +1094,7 @@ async def _llm_based_retrieve(
self,
query: str,
top_k: int,
- context_queries: list[dict[str, Any]] | None,
+ context_queries: Sequence[str | Mapping[str, Any]] | None,
ctx: Context,
store: Database,
llm_client: Any | None = None,
@@ -1253,7 +1325,7 @@ async def _llm_rank_items(
) -> list[dict[str, Any]]:
"""Use LLM to rank memory items from relevant categories"""
if not category_ids:
- print("[LLM Rank Items] No category_ids provided")
+ logger.debug("Skipping LLM item ranking because no category IDs were provided")
return []
item_pool = items if items is not None else store.memory_item_repo.items
diff --git a/src/memu/app/scope.py b/src/memu/app/scope.py
new file mode 100644
index 00000000..28423027
--- /dev/null
+++ b/src/memu/app/scope.py
@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping
+from typing import Annotated, Any
+
+from pydantic import BaseModel, TypeAdapter, ValidationError
+
+from memu.utils.filtering import build_filter_key, normalize_filter_value, split_filter_key
+
+
+def normalize_scope_where(user_model: type[BaseModel], where: Mapping[str, Any] | None) -> dict[str, Any]:
+ """Validate and clean API `where` filters against the configured user model."""
+
+ if not where:
+ return {}
+
+ valid_fields = set(getattr(user_model, "model_fields", {}).keys())
+ cleaned: dict[str, Any] = {}
+
+ for raw_key, value in where.items():
+ if value is None:
+ continue
+ field, operator = split_filter_key(raw_key)
+ if field not in valid_fields:
+ msg = f"Unknown filter field '{field}' for current user scope"
+ raise ValueError(msg)
+ normalized_value = normalize_filter_value(field, operator, value)
+ cleaned[build_filter_key(field, operator)] = _validate_scope_filter_value(
+ user_model,
+ field,
+ operator,
+ normalized_value,
+ )
+
+ return cleaned
+
+
+def exact_scope_from_where(where: Mapping[str, Any] | None) -> dict[str, Any]:
+ """Extract exact equality scope fields that can be used as write-time user data."""
+
+ if not where:
+ return {}
+ exact: dict[str, Any] = {}
+ for raw_key, value in where.items():
+ if value is None:
+ continue
+ field, operator = split_filter_key(raw_key)
+ if operator is None:
+ exact[field] = value
+ return exact
+
+
+def concrete_scope_from_where(where: Mapping[str, Any] | None) -> dict[str, Any] | None:
+ """Return a concrete write scope when a read filter targets exactly one scope."""
+
+ if not where:
+ return {}
+ concrete: dict[str, Any] = {}
+ for raw_key, value in where.items():
+ if value is None:
+ continue
+ field, operator = split_filter_key(raw_key)
+ if operator is not None:
+ return None
+ concrete[field] = value
+ return concrete
+
+
+def record_matches_scope(record: Any, user_scope: Mapping[str, Any] | None) -> bool:
+ """Return whether a record belongs to a concrete user scope."""
+
+ if not user_scope:
+ return True
+ for field, expected in user_scope.items():
+ if expected is None:
+ continue
+ if getattr(record, str(field), None) != expected:
+ return False
+ return True
+
+
+def scope_key_from_user(user_scope: Mapping[str, Any] | None) -> tuple[tuple[str, str], ...]:
+ """Build a stable cache key for a concrete user/category scope."""
+
+ if not user_scope:
+ return ()
+ return tuple(
+ sorted(
+ (str(field), _scope_value_key(value))
+ for field, value in user_scope.items()
+ if value is not None
+ )
+ )
+
+
+def _scope_value_key(value: Any) -> str:
+ try:
+ return json.dumps(value, ensure_ascii=False, sort_keys=True)
+ except TypeError:
+ return str(value)
+
+
+def _validate_scope_filter_value(
+ user_model: type[BaseModel],
+ field: str,
+ operator: str | None,
+ value: Any,
+) -> Any:
+ if operator == "in":
+ values = (value,) if isinstance(value, str) else tuple(value)
+ return tuple(_validate_scope_field_value(user_model, field, item) for item in values)
+ return _validate_scope_field_value(user_model, field, value)
+
+
+def _validate_scope_field_value(user_model: type[BaseModel], field: str, value: Any) -> Any:
+ try:
+ model_field = user_model.model_fields[field]
+ annotation = model_field.annotation
+ if model_field.metadata:
+ annotation = Annotated[annotation, *model_field.metadata]
+ validated = TypeAdapter(annotation).validate_python(value)
+ except ValidationError as exc:
+ detail = exc.errors()[0]["msg"] if exc.errors() else "invalid value"
+ msg = f"Invalid filter value for field '{field}': {detail}"
+ raise ValueError(msg) from exc
+ return validated
+
+
+__all__ = [
+ "concrete_scope_from_where",
+ "exact_scope_from_where",
+ "normalize_scope_where",
+ "record_matches_scope",
+ "scope_key_from_user",
+]
diff --git a/src/memu/app/self_evolve.py b/src/memu/app/self_evolve.py
new file mode 100644
index 00000000..006e04c1
--- /dev/null
+++ b/src/memu/app/self_evolve.py
@@ -0,0 +1,621 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Literal, cast
+
+
+EvolutionTarget = Literal["memory", "soul", "skill"]
+EvolutionOperation = Literal["add", "update", "delete"]
+EvolutionPriority = Literal["low", "medium", "high"]
+EvolutionSourceKind = Literal["agent_log", "creator_feedback", "user_upload", "observation", "unknown"]
+ReviewStatus = Literal["approved", "needs_review", "rejected"]
+
+
+@dataclass(frozen=True)
+class EvolutionReviewConfig:
+ """Policy used by the review gate before patches update long-term context."""
+
+ auto_approve: bool = True
+ min_confidence: float = 0.0
+ require_traceable_evidence: bool = True
+ require_conflict_review: bool = True
+
+
+@dataclass(frozen=True)
+class EvidenceRecord:
+ """Traceable evidence summary used by an evolution instruction."""
+
+ source: str
+ evidence_path: str
+ source_kind: EvolutionSourceKind
+ attribution: str
+ summary: str
+ conflicts: list[str] = field(default_factory=list)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "source": self.source,
+ "evidence_path": self.evidence_path,
+ "source_kind": self.source_kind,
+ "attribution": self.attribution,
+ "summary": self.summary,
+ "conflicts": list(self.conflicts),
+ }
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> EvidenceRecord:
+ return cls(
+ source=str(data.get("source", "")),
+ evidence_path=str(data.get("evidence_path", "")),
+ source_kind=cast(EvolutionSourceKind, data.get("source_kind", "unknown")),
+ attribution=str(data.get("attribution", "")),
+ summary=str(data.get("summary", "")),
+ conflicts=[str(item) for item in data.get("conflicts", [])],
+ )
+
+
+@dataclass(frozen=True)
+class EvolutionInstruction:
+ """Structured instruction distilled from raw evidence before any patch is proposed."""
+
+ id: str
+ target: EvolutionTarget
+ operation: EvolutionOperation
+ reason: str
+ evidence: EvidenceRecord
+ priority: EvolutionPriority
+ confidence: float
+ content: dict[str, Any]
+ created_at: str = field(default_factory=lambda: _utc_now())
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "id": self.id,
+ "target": self.target,
+ "operation": self.operation,
+ "reason": self.reason,
+ "evidence": self.evidence.to_dict(),
+ "priority": self.priority,
+ "confidence": self.confidence,
+ "content": dict(self.content),
+ "created_at": self.created_at,
+ }
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> EvolutionInstruction:
+ evidence = data.get("evidence", {})
+ return cls(
+ id=str(data.get("id", "")),
+ target=cast(EvolutionTarget, data.get("target", "memory")),
+ operation=cast(EvolutionOperation, data.get("operation", "add")),
+ reason=str(data.get("reason", "")),
+ evidence=EvidenceRecord.from_dict(evidence if isinstance(evidence, Mapping) else {}),
+ priority=cast(EvolutionPriority, data.get("priority", "medium")),
+ confidence=float(data.get("confidence", 0.0)),
+ content=dict(data.get("content", {})) if isinstance(data.get("content", {}), Mapping) else {},
+ created_at=str(data.get("created_at") or _utc_now()),
+ )
+
+
+@dataclass(frozen=True)
+class PatchProposal:
+ """A proposed change to memory.md, soul.md, or skill.md derived from an instruction."""
+
+ id: str
+ instruction_id: str
+ target: EvolutionTarget
+ operation: EvolutionOperation
+ target_path: str
+ summary: str
+ patch: dict[str, Any]
+ reason: str
+ evidence: EvidenceRecord
+ priority: EvolutionPriority
+ confidence: float
+ created_at: str = field(default_factory=lambda: _utc_now())
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "id": self.id,
+ "instruction_id": self.instruction_id,
+ "target": self.target,
+ "operation": self.operation,
+ "target_path": self.target_path,
+ "summary": self.summary,
+ "patch": dict(self.patch),
+ "reason": self.reason,
+ "evidence": self.evidence.to_dict(),
+ "priority": self.priority,
+ "confidence": self.confidence,
+ "created_at": self.created_at,
+ }
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> PatchProposal:
+ evidence = data.get("evidence", {})
+ return cls(
+ id=str(data.get("id", "")),
+ instruction_id=str(data.get("instruction_id", "")),
+ target=cast(EvolutionTarget, data.get("target", "memory")),
+ operation=cast(EvolutionOperation, data.get("operation", "add")),
+ target_path=str(data.get("target_path", "memory.md")),
+ summary=str(data.get("summary", "")),
+ patch=dict(data.get("patch", {})) if isinstance(data.get("patch", {}), Mapping) else {},
+ reason=str(data.get("reason", "")),
+ evidence=EvidenceRecord.from_dict(evidence if isinstance(evidence, Mapping) else {}),
+ priority=cast(EvolutionPriority, data.get("priority", "medium")),
+ confidence=float(data.get("confidence", 0.0)),
+ created_at=str(data.get("created_at") or _utc_now()),
+ )
+
+
+@dataclass(frozen=True)
+class ReviewDecision:
+ """Review-gate decision for a patch proposal."""
+
+ proposal_id: str
+ status: ReviewStatus
+ reviewer: str
+ reason: str
+ confidence: float
+ safety_flags: list[str] = field(default_factory=list)
+ reviewed_at: str = field(default_factory=lambda: _utc_now())
+
+ @property
+ def approved(self) -> bool:
+ return self.status == "approved"
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "proposal_id": self.proposal_id,
+ "status": self.status,
+ "reviewer": self.reviewer,
+ "reason": self.reason,
+ "confidence": self.confidence,
+ "safety_flags": list(self.safety_flags),
+ "reviewed_at": self.reviewed_at,
+ }
+
+ @classmethod
+ def from_dict(cls, data: Mapping[str, Any]) -> ReviewDecision:
+ return cls(
+ proposal_id=str(data.get("proposal_id", "")),
+ status=cast(ReviewStatus, data.get("status", "needs_review")),
+ reviewer=str(data.get("reviewer", "")),
+ reason=str(data.get("reason", "")),
+ confidence=float(data.get("confidence", 0.0)),
+ safety_flags=[str(flag) for flag in data.get("safety_flags", [])],
+ reviewed_at=str(data.get("reviewed_at") or _utc_now()),
+ )
+
+
+@dataclass(frozen=True)
+class EvolutionReviewBundle:
+ """The auditable self-evolve chain for one source or removal."""
+
+ instructions: list[EvolutionInstruction]
+ proposals: list[PatchProposal]
+ reviews: list[ReviewDecision]
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "instructions": [instruction.to_dict() for instruction in self.instructions],
+ "patch_proposals": [proposal.to_dict() for proposal in self.proposals],
+ "review_decisions": [review.to_dict() for review in self.reviews],
+ }
+
+
+class SelfEvolveEngine:
+ """Convert evidence-backed candidates into reviewed long-term context patches."""
+
+ def __init__(self, review_config: EvolutionReviewConfig | None = None) -> None:
+ self.review_config = review_config or EvolutionReviewConfig()
+
+ def build_for_source(
+ self,
+ *,
+ source: str,
+ evidence_path: str,
+ evidence_text: str,
+ candidates: Sequence[Mapping[str, Any]],
+ previous_entries: Sequence[Mapping[str, Any]] = (),
+ ) -> EvolutionReviewBundle:
+ source_kind = classify_evolution_source(source, evidence_text)
+ instructions = self.extract_instructions(
+ source=source,
+ evidence_path=evidence_path,
+ evidence_text=evidence_text,
+ source_kind=source_kind,
+ candidates=candidates,
+ previous_entries=previous_entries,
+ )
+ return self._review_instructions(instructions)
+
+ def build_for_removed_source(
+ self,
+ *,
+ source: str,
+ evidence_path: str,
+ previous_entries: Sequence[Mapping[str, Any]],
+ ) -> EvolutionReviewBundle:
+ instructions: list[EvolutionInstruction] = []
+ source_kind = classify_evolution_source(source, "")
+ for entry in previous_entries:
+ target = cast(EvolutionTarget, str(entry.get("bucket", "memory")))
+ evidence = EvidenceRecord(
+ source=source,
+ evidence_path=evidence_path,
+ source_kind=source_kind,
+ attribution="Manifest entry generated from a source that is no longer present.",
+ summary=_entry_summary(entry),
+ conflicts=[],
+ )
+ content = {
+ "entry_id": str(entry.get("id", "")),
+ "previous_entry": dict(entry),
+ }
+ instructions.append(
+ EvolutionInstruction(
+ id=_stable_id("evi", source, target, "delete", str(entry.get("id", ""))),
+ target=target,
+ operation="delete",
+ reason="Remove long-term context that was generated from a deleted raw source.",
+ evidence=evidence,
+ priority="medium",
+ confidence=1.0,
+ content=content,
+ )
+ )
+ return self._review_instructions(instructions)
+
+ def extract_instructions(
+ self,
+ *,
+ source: str,
+ evidence_path: str,
+ evidence_text: str,
+ source_kind: EvolutionSourceKind,
+ candidates: Sequence[Mapping[str, Any]],
+ previous_entries: Sequence[Mapping[str, Any]] = (),
+ ) -> list[EvolutionInstruction]:
+ previous_by_id = {str(entry.get("id", "")): dict(entry) for entry in previous_entries if entry.get("id")}
+ candidate_ids: set[str] = set()
+ instructions: list[EvolutionInstruction] = []
+
+ for candidate in candidates:
+ entry_id = str(candidate.get("id", ""))
+ if not entry_id:
+ continue
+ candidate_ids.add(entry_id)
+ operation: EvolutionOperation = "update" if entry_id in previous_by_id else "add"
+ target = cast(EvolutionTarget, str(candidate.get("bucket", "memory")))
+ conflicts = detect_evidence_conflicts(evidence_text)
+ evidence = EvidenceRecord(
+ source=source,
+ evidence_path=evidence_path,
+ source_kind=source_kind,
+ attribution=self._attribution_for(source_kind, source),
+ summary=_entry_summary(candidate),
+ conflicts=conflicts,
+ )
+ content = {
+ "entry": dict(candidate),
+ "previous_entry": previous_by_id.get(entry_id),
+ }
+ instructions.append(
+ EvolutionInstruction(
+ id=_stable_id("evi", source, target, operation, entry_id, _entry_summary(candidate)),
+ target=target,
+ operation=operation,
+ reason=self._reason_for(operation, target, source_kind),
+ evidence=evidence,
+ priority=self._priority_for(source_kind, conflicts, candidate),
+ confidence=confidence_score(candidate.get("confidence")),
+ content=content,
+ )
+ )
+
+ for entry_id, previous in previous_by_id.items():
+ if entry_id in candidate_ids:
+ continue
+ target = cast(EvolutionTarget, str(previous.get("bucket", "memory")))
+ evidence = EvidenceRecord(
+ source=source,
+ evidence_path=evidence_path,
+ source_kind=source_kind,
+ attribution=self._attribution_for(source_kind, source),
+ summary=_entry_summary(previous),
+ conflicts=[],
+ )
+ instructions.append(
+ EvolutionInstruction(
+ id=_stable_id("evi", source, target, "delete", entry_id),
+ target=target,
+ operation="delete",
+ reason="Remove stale generated context that is no longer supported by current evidence.",
+ evidence=evidence,
+ priority="medium",
+ confidence=1.0,
+ content={
+ "entry_id": entry_id,
+ "previous_entry": dict(previous),
+ },
+ )
+ )
+
+ return instructions
+
+ def _review_instructions(self, instructions: Sequence[EvolutionInstruction]) -> EvolutionReviewBundle:
+ proposals = [self.build_patch_proposal(instruction) for instruction in instructions]
+ reviews = [self.review_patch_proposal(proposal) for proposal in proposals]
+ return EvolutionReviewBundle(instructions=list(instructions), proposals=proposals, reviews=reviews)
+
+ def build_patch_proposal(self, instruction: EvolutionInstruction) -> PatchProposal:
+ target_path = f"{instruction.target}.md"
+ entry = instruction.content.get("entry")
+ entry_id = instruction.content.get("entry_id")
+ if instruction.operation in {"add", "update"} and isinstance(entry, Mapping):
+ patch: dict[str, Any] = {"entry": dict(entry)}
+ summary = _entry_summary(entry)
+ else:
+ patch = {"entry_id": str(entry_id or "")}
+ summary = f"Remove generated entry {entry_id}"
+ return PatchProposal(
+ id=_stable_id("patch", instruction.id, instruction.operation, target_path),
+ instruction_id=instruction.id,
+ target=instruction.target,
+ operation=instruction.operation,
+ target_path=target_path,
+ summary=summary,
+ patch=patch,
+ reason=instruction.reason,
+ evidence=instruction.evidence,
+ priority=instruction.priority,
+ confidence=instruction.confidence,
+ )
+
+ def review_patch_proposal(self, proposal: PatchProposal) -> ReviewDecision:
+ flags = self._safety_flags(proposal)
+ if proposal.confidence < self.review_config.min_confidence:
+ flags.append("below_min_confidence")
+ if proposal.evidence.conflicts and self.review_config.require_conflict_review:
+ flags.append("conflict_detected")
+
+ if not self.review_config.auto_approve:
+ return ReviewDecision(
+ proposal_id=proposal.id,
+ status="needs_review",
+ reviewer="creator",
+ reason="Creator review is required before this patch can update long-term context.",
+ confidence=proposal.confidence,
+ safety_flags=flags,
+ )
+ if flags:
+ return ReviewDecision(
+ proposal_id=proposal.id,
+ status="needs_review",
+ reviewer="auto-review",
+ reason="Patch requires review because one or more safety checks did not pass.",
+ confidence=proposal.confidence,
+ safety_flags=flags,
+ )
+ return ReviewDecision(
+ proposal_id=proposal.id,
+ status="approved",
+ reviewer="auto-review",
+ reason="Patch passed traceability, confidence, and safety checks.",
+ confidence=proposal.confidence,
+ safety_flags=[],
+ )
+
+ def _safety_flags(self, proposal: PatchProposal) -> list[str]:
+ flags: list[str] = []
+ if proposal.target not in {"memory", "soul", "skill"}:
+ flags.append("invalid_target")
+ if proposal.operation not in {"add", "update", "delete"}:
+ flags.append("invalid_operation")
+ if self.review_config.require_traceable_evidence:
+ if not proposal.evidence.source:
+ flags.append("missing_evidence_source")
+ if not proposal.evidence.evidence_path:
+ flags.append("missing_evidence_path")
+ if proposal.operation in {"add", "update"} and not isinstance(proposal.patch.get("entry"), Mapping):
+ flags.append("missing_patch_entry")
+ if proposal.operation == "delete" and not proposal.patch.get("entry_id"):
+ flags.append("missing_delete_entry_id")
+ return flags
+
+ def _attribution_for(self, source_kind: EvolutionSourceKind, source: str) -> str:
+ if source_kind == "agent_log":
+ return f"Agent execution evidence from {source}."
+ if source_kind == "creator_feedback":
+ return f"Creator-authored feedback from {source}."
+ if source_kind == "observation":
+ return f"New observation evidence from {source}."
+ if source_kind == "user_upload":
+ return f"User-uploaded evidence from {source}."
+ return f"Evidence from {source}."
+
+ def _reason_for(
+ self,
+ operation: EvolutionOperation,
+ target: EvolutionTarget,
+ source_kind: EvolutionSourceKind,
+ ) -> str:
+ source_phrase = source_kind.replace("_", " ")
+ if operation == "delete":
+ return f"Delete stale {target} context because the source evidence no longer supports it."
+ if operation == "update":
+ return f"Update {target} context using structured evidence extracted from {source_phrase}."
+ return f"Add {target} context using structured evidence extracted from {source_phrase}."
+
+ def _priority_for(
+ self,
+ source_kind: EvolutionSourceKind,
+ conflicts: Sequence[str],
+ candidate: Mapping[str, Any],
+ ) -> EvolutionPriority:
+ if conflicts or source_kind == "creator_feedback":
+ return "high"
+ if str(candidate.get("bucket", "")) == "skill" or source_kind == "agent_log":
+ return "medium"
+ return "low"
+
+
+def apply_reviewed_proposals(
+ previous_entries: Sequence[Mapping[str, Any]],
+ proposals: Sequence[PatchProposal],
+ reviews: Sequence[ReviewDecision],
+) -> list[dict[str, Any]]:
+ """Apply only approved proposals to a source's existing generated entries."""
+
+ entries_by_id = {str(entry.get("id", "")): dict(entry) for entry in previous_entries if entry.get("id")}
+ order = [str(entry.get("id", "")) for entry in previous_entries if entry.get("id")]
+ review_by_proposal = {review.proposal_id: review for review in reviews}
+ for proposal in proposals:
+ review = review_by_proposal.get(proposal.id)
+ if review is None or not review.approved:
+ continue
+ if proposal.operation in {"add", "update"}:
+ entry = proposal.patch.get("entry")
+ if not isinstance(entry, Mapping):
+ continue
+ entry_id = str(entry.get("id", ""))
+ if not entry_id:
+ continue
+ entries_by_id[entry_id] = dict(entry)
+ if entry_id not in order:
+ order.append(entry_id)
+ elif proposal.operation == "delete":
+ entry_id = str(proposal.patch.get("entry_id", ""))
+ entries_by_id.pop(entry_id, None)
+ order = [item for item in order if item != entry_id]
+ return [entries_by_id[entry_id] for entry_id in order if entry_id in entries_by_id]
+
+
+def classify_evolution_source(source: str, evidence_text: str) -> EvolutionSourceKind:
+ lowered_source = source.lower()
+ lowered_evidence = evidence_text.lower()
+ if "skill_traces/" in lowered_source or "agent_log" in lowered_source or "agent-log" in lowered_source:
+ return "agent_log"
+ if "skill evolution trace" in lowered_evidence:
+ return "agent_log"
+ if "creator" in lowered_source or "feedback" in lowered_source:
+ return "creator_feedback"
+ if "observation" in lowered_source or "new_observation" in lowered_source:
+ return "observation"
+ if source:
+ return "user_upload"
+ return "unknown"
+
+
+def detect_evidence_conflicts(evidence_text: str) -> list[str]:
+ lowered = evidence_text.lower()
+ markers = (
+ "conflict",
+ "contradict",
+ "correction",
+ "instead of",
+ "replace previous",
+ "actually",
+ )
+ if any(marker in lowered for marker in markers):
+ return ["Evidence contains language that may conflict with existing long-term context."]
+ return []
+
+
+def confidence_score(value: Any) -> float:
+ if isinstance(value, int | float) and not isinstance(value, bool):
+ return max(0.0, min(float(value), 1.0))
+ lowered = str(value or "").strip().lower()
+ if lowered == "high":
+ return 0.9
+ if lowered == "medium":
+ return 0.65
+ if lowered == "low":
+ return 0.35
+ return 0.5
+
+
+def write_evolution_audit(
+ metadata_dir: str | Path,
+ *,
+ instructions: Sequence[EvolutionInstruction],
+ proposals: Sequence[PatchProposal],
+ reviews: Sequence[ReviewDecision],
+) -> None:
+ if not instructions and not proposals and not reviews:
+ return
+ evolution_dir = Path(metadata_dir) / "evolution"
+ evolution_dir.mkdir(parents=True, exist_ok=True)
+ _append_jsonl(evolution_dir / "instructions.jsonl", [instruction.to_dict() for instruction in instructions])
+ _append_jsonl(evolution_dir / "patch_proposals.jsonl", [proposal.to_dict() for proposal in proposals])
+ _append_jsonl(evolution_dir / "review_decisions.jsonl", [review.to_dict() for review in reviews])
+ latest = {
+ "updated_at": _utc_now(),
+ "instructions": [instruction.to_dict() for instruction in instructions],
+ "patch_proposals": [proposal.to_dict() for proposal in proposals],
+ "review_decisions": [review.to_dict() for review in reviews],
+ }
+ (evolution_dir / "latest.json").write_text(
+ json.dumps(latest, indent=2, ensure_ascii=False, sort_keys=True),
+ encoding="utf-8",
+ )
+
+
+def _append_jsonl(path: Path, records: Sequence[Mapping[str, Any]]) -> None:
+ if not records:
+ return
+ with path.open("a", encoding="utf-8") as handle:
+ for record in records:
+ handle.write(json.dumps(record, ensure_ascii=False, sort_keys=True))
+ handle.write("\n")
+
+
+def _entry_summary(entry: Mapping[str, Any]) -> str:
+ title = str(entry.get("title", "")).strip()
+ body = str(entry.get("body", "")).strip()
+ summary = title or body
+ if "\n" in summary:
+ summary = summary.splitlines()[0].strip()
+ if len(summary) > 180:
+ return summary[:177].rstrip() + "..."
+ return summary
+
+
+def _stable_id(prefix: str, *parts: object) -> str:
+ digest = hashlib.sha256()
+ for part in parts:
+ digest.update(str(part).encode("utf-8"))
+ digest.update(b"\0")
+ return f"{prefix}_{digest.hexdigest()[:16]}"
+
+
+def _utc_now() -> str:
+ return datetime.now(UTC).isoformat().replace("+00:00", "Z")
+
+
+__all__ = [
+ "EvidenceRecord",
+ "EvolutionInstruction",
+ "EvolutionOperation",
+ "EvolutionPriority",
+ "EvolutionReviewBundle",
+ "EvolutionReviewConfig",
+ "EvolutionSourceKind",
+ "EvolutionTarget",
+ "PatchProposal",
+ "ReviewDecision",
+ "ReviewStatus",
+ "SelfEvolveEngine",
+ "apply_reviewed_proposals",
+ "classify_evolution_source",
+ "confidence_score",
+ "detect_evidence_conflicts",
+ "write_evolution_audit",
+]
diff --git a/src/memu/app/service.py b/src/memu/app/service.py
index 4e2dea04..bcd05601 100644
--- a/src/memu/app/service.py
+++ b/src/memu/app/service.py
@@ -19,6 +19,7 @@
MemorizeConfig,
RetrieveConfig,
UserConfig,
+ resolve_api_key,
)
from memu.blob.local_fs import LocalFS
from memu.database.factory import build_database
@@ -30,6 +31,7 @@
LLMInterceptorHandle,
LLMInterceptorRegistry,
)
+from memu.utils.serialization import model_dump_without_embeddings
from memu.workflow.interceptor import WorkflowInterceptorHandle, WorkflowInterceptorRegistry
from memu.workflow.pipeline import PipelineManager
from memu.workflow.runner import WorkflowRunner, resolve_workflow_runner
@@ -43,6 +45,8 @@ class Context:
categories_ready: bool = False
category_ids: list[str] = field(default_factory=list)
category_name_to_id: dict[str, str] = field(default_factory=dict)
+ category_scope_key: tuple[tuple[str, str], ...] = field(default_factory=tuple)
+ category_cache: dict[tuple[tuple[str, str], ...], tuple[list[str], dict[str, str]]] = field(default_factory=dict)
category_init_task: asyncio.Task | None = None
@@ -103,7 +107,7 @@ def _init_llm_client(self, config: LLMConfig | None = None) -> Any:
return OpenAISDKClient(
base_url=cfg.base_url,
- api_key=cfg.api_key,
+ api_key=resolve_api_key(cfg.api_key),
chat_model=cfg.chat_model,
embed_model=cfg.embed_model,
embed_batch_size=cfg.embed_batch_size,
@@ -111,7 +115,7 @@ def _init_llm_client(self, config: LLMConfig | None = None) -> Any:
elif backend == "httpx":
return HTTPLLMClient(
base_url=cfg.base_url,
- api_key=cfg.api_key,
+ api_key=resolve_api_key(cfg.api_key),
chat_model=cfg.chat_model,
provider=cfg.provider,
endpoint_overrides=cfg.endpoint_overrides,
@@ -373,8 +377,7 @@ def _escape_prompt_value(value: str) -> str:
return value.replace("{", "{{").replace("}", "}}")
def _model_dump_without_embeddings(self, obj: BaseModel) -> dict[str, Any]:
- data = obj.model_dump(exclude={"embedding"})
- return data
+ return model_dump_without_embeddings(obj)
@staticmethod
def _validate_config(
diff --git a/src/memu/app/settings.py b/src/memu/app/settings.py
index adcb4f16..6b4a2c1d 100644
--- a/src/memu/app/settings.py
+++ b/src/memu/app/settings.py
@@ -1,3 +1,4 @@
+import os
from collections.abc import Mapping
from typing import Annotated, Any, Literal
@@ -24,7 +25,23 @@ def normalize_value(v: str) -> str:
return v
+def resolve_api_key(value: str | None) -> str:
+ """Resolve an API key config value that may be a literal key or an environment variable name."""
+ if not value:
+ return ""
+ resolved = os.getenv(value, value)
+ return resolved.strip()
+
+
+def default_api_key_env(provider: str) -> str:
+ """Return the default API key environment variable for a provider."""
+ if provider.strip().lower() == "grok":
+ return "XAI_API_KEY"
+ return "OPENAI_API_KEY"
+
+
Normalize = BeforeValidator(normalize_value)
+ProfileName = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
def _default_memory_types() -> list[str]:
@@ -67,7 +84,7 @@ def complete_prompt_blocks(prompt: CustomPrompt, default_blocks: Mapping[str, in
class CategoryConfig(BaseModel):
name: str
description: str = ""
- target_length: int | None = None
+ target_length: int | None = Field(default=None, ge=1)
summary_prompt: str | Annotated[CustomPrompt, CompleteCategoryPrompt] | None = None
@@ -100,16 +117,16 @@ class LazyLLMSource(BaseModel):
class LLMConfig(BaseModel):
- provider: str = Field(
+ provider: Annotated[str, Normalize] = Field(
default="openai",
description="Identifier for the LLM provider implementation (used by HTTP client backend).",
)
base_url: str = Field(default="https://api.openai.com/v1")
api_key: str = Field(default="OPENAI_API_KEY")
chat_model: str = Field(default="gpt-4o-mini")
- client_backend: str = Field(
+ client_backend: Annotated[Literal["httpx", "sdk", "lazyllm_backend"], Normalize] = Field(
default="sdk",
- description="Which LLM client backend to use: 'httpx' (httpx), 'sdk' (official OpenAI), or 'lazyllm_backend' (for more LLM source like Qwen, Doubao, SIliconflow, etc.)",
+ description="Which LLM client backend to use: 'httpx' (httpx), 'sdk' (official OpenAI), or 'lazyllm_backend' (for more LLM source like Qwen, Doubao, SiliconFlow, etc.)",
)
lazyllm_source: LazyLLMSource = Field(default=LazyLLMSource())
endpoint_overrides: dict[str, str] = Field(
@@ -122,6 +139,7 @@ class LLMConfig(BaseModel):
)
embed_batch_size: int = Field(
default=1,
+ ge=1,
description="Maximum batch size for embedding API calls (used by SDK client backends).",
)
@@ -131,8 +149,8 @@ def set_provider_defaults(self) -> "LLMConfig":
# If values match the OpenAI defaults, switch them to Grok defaults
if self.base_url == "https://api.openai.com/v1":
self.base_url = "https://api.x.ai/v1"
- if self.api_key == "OPENAI_API_KEY":
- self.api_key = "XAI_API_KEY"
+ if self.api_key == default_api_key_env("openai"):
+ self.api_key = default_api_key_env("grok")
if self.chat_model == "gpt-4o-mini":
self.chat_model = "grok-2-latest"
return self
@@ -145,12 +163,12 @@ class BlobConfig(BaseModel):
class RetrieveCategoryConfig(BaseModel):
enabled: bool = Field(default=True, description="Whether to enable category retrieval.")
- top_k: int = Field(default=5, description="Total number of categories to retrieve.")
+ top_k: int = Field(default=5, ge=1, description="Total number of categories to retrieve.")
class RetrieveItemConfig(BaseModel):
enabled: bool = Field(default=True, description="Whether to enable item retrieval.")
- top_k: int = Field(default=5, description="Total number of items to retrieve.")
+ top_k: int = Field(default=5, ge=1, description="Total number of items to retrieve.")
# Reference-aware retrieval
use_category_references: bool = Field(
default=False,
@@ -163,13 +181,14 @@ class RetrieveItemConfig(BaseModel):
)
recency_decay_days: float = Field(
default=30.0,
+ gt=0,
description="Half-life in days for recency decay in salience scoring. After this many days, recency factor is ~0.5.",
)
class RetrieveResourceConfig(BaseModel):
enabled: bool = Field(default=True, description="Whether to enable resource retrieval.")
- top_k: int = Field(default=5, description="Total number of resources to retrieve.")
+ top_k: int = Field(default=5, ge=1, description="Total number of resources to retrieve.")
class RetrieveConfig(BaseModel):
@@ -191,32 +210,41 @@ class RetrieveConfig(BaseModel):
default=True, description="Whether to route intention (judge needs retrieval & rewrite query)."
)
# route_intention_prompt: str = Field(default="", description="User prompt for route intention.")
- # route_intention_llm_profile: str = Field(default="default", description="LLM profile for route intention.")
+ route_intention_llm_profile: ProfileName = Field(
+ default="default",
+ description="LLM profile for route intention.",
+ )
category: RetrieveCategoryConfig = Field(default=RetrieveCategoryConfig())
item: RetrieveItemConfig = Field(default=RetrieveItemConfig())
resource: RetrieveResourceConfig = Field(default=RetrieveResourceConfig())
sufficiency_check: bool = Field(default=True, description="Whether to check sufficiency after each tier.")
sufficiency_check_prompt: str = Field(default="", description="User prompt for sufficiency check.")
- sufficiency_check_llm_profile: str = Field(default="default", description="LLM profile for sufficiency check.")
- llm_ranking_llm_profile: str = Field(default="default", description="LLM profile for LLM ranking.")
+ sufficiency_check_llm_profile: ProfileName = Field(
+ default="default",
+ description="LLM profile for sufficiency check.",
+ )
+ llm_ranking_llm_profile: ProfileName = Field(default="default", description="LLM profile for LLM ranking.")
class MemorizeConfig(BaseModel):
- category_assign_threshold: float = Field(default=0.25)
+ category_assign_threshold: float = Field(default=0.25, ge=0, le=1)
multimodal_preprocess_prompts: dict[str, str | CustomPrompt] = Field(
default_factory=dict,
description="Optional mapping of modality -> preprocess system prompt.",
)
- preprocess_llm_profile: str = Field(default="default", description="LLM profile for preprocess.")
+ preprocess_llm_profile: ProfileName = Field(default="default", description="LLM profile for preprocess.")
memory_types: list[str] = Field(
default_factory=_default_memory_types,
- description="Ordered list of memory types (profile/event/knowledge/behavior by default).",
+ description="Ordered list of memory types (profile/event/knowledge/behavior/skill/tool by default).",
)
memory_type_prompts: dict[str, str | Annotated[CustomPrompt, CompleteMemoryTypePrompt]] = Field(
default_factory=_default_memory_type_prompts,
description="User prompt overrides for each memory type extraction.",
)
- memory_extract_llm_profile: str = Field(default="default", description="LLM profile for memory extract.")
+ memory_extract_llm_profile: ProfileName = Field(
+ default="default",
+ description="LLM profile for memory extract.",
+ )
memory_categories: list[CategoryConfig] = Field(
default_factory=_default_memory_categories,
description="Global memory category definitions embedded at service startup.",
@@ -228,9 +256,13 @@ class MemorizeConfig(BaseModel):
)
default_category_summary_target_length: int = Field(
default=400,
+ ge=1,
description="Target max length for auto-generated category summaries.",
)
- category_update_llm_profile: str = Field(default="default", description="LLM profile for category summary.")
+ category_update_llm_profile: ProfileName = Field(
+ default="default",
+ description="LLM profile for category summary.",
+ )
# Reference tracking for category summaries
enable_item_references: bool = Field(
default=False,
@@ -257,10 +289,7 @@ class UserConfig(BaseModel):
model: type[BaseModel] = Field(default=DefaultUserModel)
-Key = Annotated[str, StringConstraints(min_length=1)]
-
-
-class LLMProfilesConfig(RootModel[dict[Key, LLMConfig]]):
+class LLMProfilesConfig(RootModel[dict[ProfileName, LLMConfig]]):
root: dict[str, LLMConfig] = Field(default_factory=lambda: {"default": LLMConfig()})
def get(self, key: str, default: LLMConfig | None = None) -> LLMConfig | None:
@@ -278,7 +307,7 @@ def ensure_default(cls, data: Any) -> Any:
if data is None:
data = {}
elif isinstance(data, dict):
- data = dict(data)
+ data = {key.strip() if isinstance(key, str) else key: value for key, value in data.items()}
else:
return data
if "default" not in data:
@@ -299,7 +328,10 @@ def default(self) -> LLMConfig:
class MetadataStoreConfig(BaseModel):
provider: Annotated[Literal["inmemory", "postgres", "sqlite"], Normalize] = "inmemory"
ddl_mode: Annotated[Literal["create", "validate"], Normalize] = "create"
- dsn: str | None = Field(default=None, description="Database connection string (required for postgres/sqlite).")
+ dsn: str | None = Field(
+ default=None,
+ description="Database connection string. Required for postgres; optional for sqlite.",
+ )
class VectorIndexConfig(BaseModel):
diff --git a/src/memu/app/skill_trace.py b/src/memu/app/skill_trace.py
new file mode 100644
index 00000000..7abcc0d5
--- /dev/null
+++ b/src/memu/app/skill_trace.py
@@ -0,0 +1,710 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any, Literal, cast
+
+from memu.app.folder import GENERATED_END, GENERATED_START
+
+
+SkillTraceOutcome = Literal["success", "failure", "partial", "unknown"]
+
+
+@dataclass(frozen=True)
+class SkillToolTrace:
+ name: str
+ input: str = ""
+ output: str = ""
+ success: bool = True
+ score: float | None = None
+
+
+@dataclass(frozen=True)
+class SkillTrace:
+ task: str
+ outcome: SkillTraceOutcome = "unknown"
+ summary: str = ""
+ actions: list[str] = field(default_factory=list)
+ tools: list[SkillToolTrace] = field(default_factory=list)
+ lessons: list[str] = field(default_factory=list)
+ metadata: dict[str, str] = field(default_factory=dict)
+ created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat().replace("+00:00", "Z"))
+
+ def to_markdown(self) -> str:
+ lines = [
+ "# Skill Evolution Trace",
+ "",
+ "This raw trace is evidence for self-evolving skills. It captures what was attempted, "
+ "which tools or workflows were used, what happened, and what should be reused next time.",
+ "",
+ "## Metadata",
+ "",
+ f"- created_at: {self.created_at}",
+ f"- outcome: {self.outcome}",
+ f"- task: {self.task}",
+ ]
+ for key, value in sorted(self.metadata.items()):
+ lines.append(f"- {key}: {value}")
+
+ lines.extend(["", "## Summary", "", self.summary or "No summary provided."])
+
+ lines.extend(["", "## Actions", ""])
+ if self.actions:
+ lines.extend(f"{idx}. {action}" for idx, action in enumerate(self.actions, start=1))
+ else:
+ lines.append("No actions recorded.")
+
+ lines.extend(["", "## Tool Calls", ""])
+ if self.tools:
+ for idx, tool in enumerate(self.tools, start=1):
+ lines.extend(
+ [
+ f"{idx}. Tool: {tool.name}",
+ f" - success: {tool.success}",
+ ]
+ )
+ if tool.score is not None:
+ lines.append(f" - score: {tool.score}")
+ if tool.input:
+ lines.append(f" - input: {tool.input}")
+ if tool.output:
+ lines.append(f" - output: {tool.output}")
+ else:
+ lines.append("No tool calls recorded.")
+
+ lines.extend(["", "## Lessons For Skill Evolution", ""])
+ if self.lessons:
+ lines.extend(f"- Skill: {lesson}" for lesson in self.lessons)
+ else:
+ lines.append("- Skill: No explicit lesson recorded; review the task and outcome before reusing.")
+
+ lines.extend(
+ [
+ "",
+ "## Retrieval Hints",
+ "",
+ "- when_to_use: Retrieve this skill trace for similar tasks, tools, workflows, or failure modes.",
+ "- memory_bucket: skill",
+ "- evidence_type: skill_trace",
+ "",
+ ]
+ )
+ return "\n".join(lines)
+
+
+@dataclass(frozen=True)
+class SkillTraceRecord:
+ raw_data_dir: Path
+ trace_path: Path
+ trace: SkillTrace
+
+
+@dataclass(frozen=True)
+class SkillPromotionRecord:
+ repo_dir: Path
+ skill_path: Path
+ card_path: Path | None
+ title: str
+ promoted_at: str
+ content: str
+
+
+@dataclass(frozen=True)
+class SkillEvolutionProposal:
+ """A deterministic candidate skill distilled from raw skill traces."""
+
+ title: str
+ when_to_use: str
+ lessons: list[str]
+ actions: list[str]
+ tools: list[str]
+ sources: list[str]
+ outcomes: dict[SkillTraceOutcome, int]
+ support_count: int
+ score: float
+ tags: list[str] = field(default_factory=list)
+ metadata: dict[str, str] = field(default_factory=dict)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "title": self.title,
+ "when_to_use": self.when_to_use,
+ "lessons": list(self.lessons),
+ "actions": list(self.actions),
+ "tools": list(self.tools),
+ "sources": list(self.sources),
+ "outcomes": dict(self.outcomes),
+ "support_count": self.support_count,
+ "score": self.score,
+ "tags": list(self.tags),
+ "metadata": dict(self.metadata),
+ }
+
+ def to_promotion_kwargs(self) -> dict[str, Any]:
+ return {
+ "title": self.title,
+ "lessons": list(self.lessons),
+ "actions": list(self.actions),
+ "when_to_use": self.when_to_use,
+ "source": ", ".join(self.sources),
+ "tags": list(self.tags),
+ "metadata": dict(self.metadata),
+ }
+
+
+def record_skill_trace(
+ raw_data_dir: str | Path,
+ *,
+ task: str,
+ outcome: SkillTraceOutcome = "unknown",
+ summary: str = "",
+ actions: list[str] | None = None,
+ tools: list[SkillToolTrace] | None = None,
+ lessons: list[str] | None = None,
+ metadata: dict[str, str] | None = None,
+) -> SkillTraceRecord:
+ raw_dir = Path(raw_data_dir).resolve()
+ raw_dir.mkdir(parents=True, exist_ok=True)
+ trace = SkillTrace(
+ task=task,
+ outcome=outcome,
+ summary=summary,
+ actions=list(actions or []),
+ tools=list(tools or []),
+ lessons=list(lessons or []),
+ metadata=dict(metadata or {}),
+ )
+ trace_dir = raw_dir / "skill_traces"
+ trace_dir.mkdir(parents=True, exist_ok=True)
+ trace_path = trace_dir / _trace_filename(trace)
+ trace_path.write_text(trace.to_markdown(), encoding="utf-8")
+ return SkillTraceRecord(raw_data_dir=raw_dir, trace_path=trace_path, trace=trace)
+
+
+def suggest_skill_promotions(
+ raw_data_dir: str | Path,
+ *,
+ limit: int = 5,
+ min_support: int = 1,
+) -> list[SkillEvolutionProposal]:
+ """Suggest durable skill promotions from raw skill evolution traces."""
+
+ if limit <= 0:
+ msg = "limit must be greater than 0"
+ raise ValueError(msg)
+ if min_support <= 0:
+ msg = "min_support must be greater than 0"
+ raise ValueError(msg)
+
+ raw_dir = Path(raw_data_dir).resolve()
+ trace_dir = raw_dir / "skill_traces"
+ if not trace_dir.exists() or not trace_dir.is_dir():
+ return []
+
+ groups: dict[str, dict[str, Any]] = {}
+ for trace_path in sorted(trace_dir.rglob("*.md")):
+ trace = _parse_skill_trace(trace_path, raw_dir)
+ if trace is None:
+ continue
+ title = _proposal_title(trace)
+ key = _slug(title) or hashlib.sha256(title.encode("utf-8")).hexdigest()[:10]
+ group = groups.setdefault(
+ key,
+ {
+ "title": title,
+ "tasks": [],
+ "lessons": [],
+ "actions": [],
+ "tools": [],
+ "sources": [],
+ "outcomes": {},
+ },
+ )
+ group["tasks"].append(trace["task"])
+ group["lessons"] = _merge_unique(group["lessons"], trace["lessons"])
+ group["actions"] = _merge_unique(group["actions"], trace["actions"])
+ group["tools"] = _merge_unique(group["tools"], trace["tools"])
+ group["sources"] = _merge_unique(group["sources"], [trace["source"]])
+ outcomes = cast(dict[SkillTraceOutcome, int], group["outcomes"])
+ outcome = cast(SkillTraceOutcome, trace["outcome"])
+ outcomes[outcome] = outcomes.get(outcome, 0) + 1
+
+ proposals: list[SkillEvolutionProposal] = []
+ for group in groups.values():
+ sources = cast(list[str], group["sources"])
+ support_count = len(sources)
+ if support_count < min_support:
+ continue
+ outcomes = cast(dict[SkillTraceOutcome, int], group["outcomes"])
+ title = str(group["title"])
+ lessons = cast(list[str], group["lessons"])
+ actions = cast(list[str], group["actions"])
+ tools = cast(list[str], group["tools"])
+ tasks = cast(list[str], group["tasks"])
+ score = _proposal_score(support_count, outcomes, lessons, actions, tools)
+ proposals.append(
+ SkillEvolutionProposal(
+ title=title,
+ when_to_use=_proposal_when_to_use(title, tasks),
+ lessons=lessons,
+ actions=actions,
+ tools=tools,
+ sources=sources,
+ outcomes=outcomes,
+ support_count=support_count,
+ score=score,
+ tags=_proposal_tags(tools),
+ metadata={
+ "suggested_by": "memu-skill-evolution",
+ "support_count": str(support_count),
+ "score": f"{score:.2f}",
+ },
+ )
+ )
+ return sorted(proposals, key=lambda item: (-item.score, -item.support_count, item.title.lower()))[:limit]
+
+
+def promote_skill(
+ repo_dir: str | Path,
+ *,
+ title: str,
+ lessons: list[str] | None = None,
+ actions: list[str] | None = None,
+ when_to_use: str = "",
+ source: str = "",
+ tags: list[str] | None = None,
+ metadata: dict[str, str] | None = None,
+) -> SkillPromotionRecord:
+ repo = Path(repo_dir).resolve()
+ repo.mkdir(parents=True, exist_ok=True)
+ skill_path = repo / "skill.md"
+ clean_title = title.strip() or "Untitled Skill"
+ card_path = repo / "skill" / "promoted" / f"{_slug(clean_title) or 'promoted-skill'}.md"
+ existing = _read_existing_promotion(card_path)
+ promoted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
+ merged_lessons = _merge_unique(existing.get("lessons", []), list(lessons or []))
+ merged_actions = _merge_unique(existing.get("actions", []), list(actions or []))
+ merged_tags = _merge_unique(existing.get("tags", []), list(tags or []))
+ existing_when_to_use = existing.get("when_to_use", "")
+ if not isinstance(existing_when_to_use, str):
+ existing_when_to_use = ""
+ merged_when_to_use = when_to_use or existing_when_to_use
+ existing_source = existing.get("source", "")
+ if not isinstance(existing_source, str):
+ existing_source = ""
+ merged_source = source or existing_source
+ existing_metadata = existing.get("metadata", {})
+ if not isinstance(existing_metadata, dict):
+ existing_metadata = {}
+ merged_metadata = {str(key): str(value) for key, value in existing_metadata.items()}
+ merged_metadata.update(dict(metadata or {}))
+ content = _promotion_markdown(
+ title=title,
+ promoted_at=promoted_at,
+ lessons=merged_lessons,
+ actions=merged_actions,
+ when_to_use=merged_when_to_use,
+ source=merged_source,
+ tags=merged_tags,
+ metadata=merged_metadata,
+ heading_level=1,
+ )
+ index_content = _promotion_index_markdown(
+ title=clean_title,
+ promoted_at=promoted_at,
+ card_rel_path=card_path.relative_to(repo).as_posix(),
+ when_to_use=merged_when_to_use,
+ source=merged_source,
+ tags=merged_tags,
+ metadata=merged_metadata,
+ )
+ _ensure_skill_file(skill_path)
+ current = skill_path.read_text(encoding="utf-8-sig")
+ skill_path.write_text(_upsert_promoted_section(current, clean_title, index_content), encoding="utf-8")
+ card_path.parent.mkdir(parents=True, exist_ok=True)
+ card_path.write_text(content, encoding="utf-8")
+ return SkillPromotionRecord(
+ repo_dir=repo,
+ skill_path=skill_path,
+ card_path=card_path,
+ title=title,
+ promoted_at=promoted_at,
+ content=content,
+ )
+
+
+def _parse_skill_trace(trace_path: Path, raw_dir: Path) -> dict[str, Any] | None:
+ try:
+ markdown = trace_path.read_text(encoding="utf-8-sig")
+ except UnicodeDecodeError:
+ return None
+ if "Skill Evolution Trace" not in markdown:
+ return None
+ metadata = _trace_metadata(markdown)
+ task = metadata.get("task", "").strip()
+ outcome = metadata.get("outcome", "unknown").strip()
+ if outcome not in {"success", "failure", "partial", "unknown"}:
+ outcome = "unknown"
+ lessons = _trace_lessons(markdown)
+ actions = _trace_actions(markdown)
+ tools = _trace_tools(markdown)
+ if not task and not lessons:
+ return None
+ return {
+ "task": task or "Untitled skill trace",
+ "outcome": outcome,
+ "lessons": lessons,
+ "actions": actions,
+ "tools": tools,
+ "source": trace_path.relative_to(raw_dir).as_posix(),
+ }
+
+
+def _trace_metadata(markdown: str) -> dict[str, str]:
+ metadata: dict[str, str] = {}
+ for line in _extract_h2_section(markdown, "Metadata").splitlines():
+ stripped = line.strip()
+ if not stripped.startswith("- ") or ":" not in stripped:
+ continue
+ key, _, value = stripped[2:].partition(":")
+ metadata[key.strip()] = value.strip()
+ return metadata
+
+
+def _trace_lessons(markdown: str) -> list[str]:
+ lessons: list[str] = []
+ for line in _extract_h2_section(markdown, "Lessons For Skill Evolution").splitlines():
+ stripped = line.strip()
+ if not stripped.startswith("- "):
+ continue
+ lesson = stripped[2:].strip()
+ if lesson.lower().startswith("skill:"):
+ lesson = lesson.partition(":")[2].strip()
+ if lesson and not lesson.lower().startswith("no explicit lesson recorded"):
+ lessons.append(lesson)
+ return _merge_unique([], lessons)
+
+
+def _trace_actions(markdown: str) -> list[str]:
+ actions: list[str] = []
+ for line in _extract_h2_section(markdown, "Actions").splitlines():
+ stripped = line.strip()
+ marker = stripped.partition(". ")
+ if marker[0].isdigit() and marker[2]:
+ actions.append(marker[2].strip())
+ return _merge_unique([], actions)
+
+
+def _trace_tools(markdown: str) -> list[str]:
+ tools: list[str] = []
+ for line in _extract_h2_section(markdown, "Tool Calls").splitlines():
+ stripped = line.strip()
+ match = re.match(r"\d+\.\s+Tool:\s+(.+)", stripped)
+ if match:
+ tools.append(match.group(1).strip())
+ return _merge_unique([], tools)
+
+
+def _extract_h2_section(markdown: str, heading: str) -> str:
+ lines = markdown.splitlines()
+ start: int | None = None
+ target = f"## {heading}".lower()
+ for idx, line in enumerate(lines):
+ if line.strip().lower() == target:
+ start = idx + 1
+ break
+ if start is None:
+ return ""
+ end = start
+ while end < len(lines):
+ if lines[end].startswith("## "):
+ break
+ end += 1
+ return "\n".join(lines[start:end]).strip()
+
+
+def _proposal_title(trace: dict[str, Any]) -> str:
+ lessons = cast(list[str], trace["lessons"])
+ base = lessons[0] if lessons else str(trace["task"])
+ base = base.strip().rstrip(".")
+ base = re.split(r"\s+(before|after|when|while|so that|for)\s+", base, maxsplit=1, flags=re.IGNORECASE)[0]
+ words = re.findall(r"[A-Za-z0-9]+", base)
+ if words:
+ return " ".join(word.capitalize() for word in words[:6])
+ return base[:48] or "Untitled Skill"
+
+
+def _proposal_when_to_use(title: str, tasks: list[str]) -> str:
+ task = next((item for item in tasks if item), "")
+ if task:
+ return f"Use when a future task resembles: {task}"
+ return f"Use when a future task requires {title.lower()}."
+
+
+def _proposal_score(
+ support_count: int,
+ outcomes: dict[SkillTraceOutcome, int],
+ lessons: list[str],
+ actions: list[str],
+ tools: list[str],
+) -> float:
+ total = max(sum(outcomes.values()), 1)
+ success_weight = outcomes.get("success", 0) / total
+ partial_weight = outcomes.get("partial", 0) / total
+ return round(
+ support_count + success_weight + (partial_weight * 0.5) + (len(lessons) * 0.1) + (len(actions) * 0.05)
+ + (len(tools) * 0.05),
+ 2,
+ )
+
+
+def _proposal_tags(tools: list[str]) -> list[str]:
+ tags = ["suggested", "skill-evolution"]
+ for tool in tools[:3]:
+ slug = _slug(tool)
+ if slug:
+ tags.append(slug)
+ return _merge_unique([], tags)
+
+
+def _promotion_markdown(
+ *,
+ title: str,
+ promoted_at: str,
+ lessons: list[str],
+ actions: list[str],
+ when_to_use: str,
+ source: str,
+ tags: list[str],
+ metadata: dict[str, str],
+ heading_level: int = 2,
+) -> str:
+ clean_title = title.strip() or "Untitled Skill"
+ heading = "#" * heading_level
+ lines = [
+ f"{heading} Promoted Skill: {clean_title}",
+ "",
+ f"- promoted_at: {promoted_at}",
+ ]
+ if source:
+ lines.append(f"- source: {source}")
+ if tags:
+ lines.append(f"- tags: {', '.join(tags)}")
+ for key, value in sorted(metadata.items()):
+ lines.append(f"- {key}: {value}")
+
+ lines.extend(["", "### When To Use", "", when_to_use or "Use when a future task matches this skill pattern."])
+ lines.extend(["", "### Procedure", ""])
+ if actions:
+ lines.extend(f"{idx}. {action}" for idx, action in enumerate(actions, start=1))
+ else:
+ lines.append("1. Review the task context and apply the promoted lesson deliberately.")
+
+ lines.extend(["", "### Lessons", ""])
+ if lessons:
+ lines.extend(f"- {lesson}" for lesson in lessons)
+ else:
+ lines.append("- Reuse this skill when the same workflow or failure mode appears.")
+ lines.append("")
+ return "\n".join(lines)
+
+
+def _promotion_index_markdown(
+ *,
+ title: str,
+ promoted_at: str,
+ card_rel_path: str,
+ when_to_use: str,
+ source: str,
+ tags: list[str],
+ metadata: dict[str, str],
+) -> str:
+ lines = [
+ f"## Promoted Skill: {title}",
+ "",
+ f"- promoted_at: {promoted_at}",
+ f"- card: {card_rel_path}",
+ ]
+ if source:
+ lines.append(f"- source: {source}")
+ if tags:
+ lines.append(f"- tags: {', '.join(tags)}")
+ for key, value in sorted(metadata.items()):
+ lines.append(f"- {key}: {value}")
+ lines.extend(["", when_to_use or "Use when a future task matches this skill pattern.", ""])
+ return "\n".join(lines)
+
+
+def _upsert_promoted_section(current: str, title: str, section: str) -> str:
+ lines = current.rstrip().splitlines()
+ start = _find_promoted_section_start(lines, title)
+ if start is None:
+ return current.rstrip() + "\n\n" + section
+ end = start + 1
+ while end < len(lines):
+ if lines[end].startswith("## "):
+ break
+ end += 1
+ updated = lines[:start] + section.rstrip().splitlines() + lines[end:]
+ return "\n".join(updated).rstrip() + "\n"
+
+
+def _find_promoted_section_start(lines: list[str], title: str) -> int | None:
+ target = f"## Promoted Skill: {title}".strip().lower()
+ for idx, line in enumerate(lines):
+ if line.strip().lower() == target:
+ return idx
+ return None
+
+
+def _read_existing_promotion(card_path: Path) -> dict[str, Any]:
+ if not card_path.exists():
+ return {}
+ try:
+ current = card_path.read_text(encoding="utf-8-sig")
+ except UnicodeDecodeError:
+ return {}
+ return {
+ "actions": _extract_numbered_section(current, "Procedure"),
+ "lessons": _extract_bullet_section(current, "Lessons"),
+ "tags": _extract_metadata_list(current, "tags"),
+ "when_to_use": _extract_text_section(current, "When To Use"),
+ "source": _extract_metadata_value(current, "source"),
+ "metadata": _extract_promotion_metadata(current),
+ }
+
+
+def _extract_numbered_section(markdown: str, heading: str) -> list[str]:
+ section = _extract_section(markdown, heading)
+ values: list[str] = []
+ for line in section.splitlines():
+ stripped = line.strip()
+ marker = stripped.partition(". ")
+ if marker[0].isdigit() and marker[2]:
+ values.append(marker[2].strip())
+ return values
+
+
+def _extract_bullet_section(markdown: str, heading: str) -> list[str]:
+ section = _extract_section(markdown, heading)
+ values: list[str] = []
+ for line in section.splitlines():
+ stripped = line.strip()
+ if stripped.startswith("- "):
+ values.append(stripped[2:].strip())
+ return values
+
+
+def _extract_text_section(markdown: str, heading: str) -> str:
+ lines = [line.strip() for line in _extract_section(markdown, heading).splitlines()]
+ return "\n".join(line for line in lines if line).strip()
+
+
+def _extract_section(markdown: str, heading: str) -> str:
+ lines = markdown.splitlines()
+ start: int | None = None
+ for idx, line in enumerate(lines):
+ if line.strip().lower() == f"### {heading}".lower():
+ start = idx + 1
+ break
+ if start is None:
+ return ""
+ end = start
+ while end < len(lines):
+ if lines[end].startswith("### ") or lines[end].startswith("## "):
+ break
+ end += 1
+ return "\n".join(lines[start:end]).strip()
+
+
+def _extract_metadata_list(markdown: str, key: str) -> list[str]:
+ prefix = f"- {key}:"
+ for line in markdown.splitlines():
+ if line.lower().startswith(prefix.lower()):
+ return [part.strip() for part in line.partition(":")[2].split(",") if part.strip()]
+ return []
+
+
+def _extract_metadata_value(markdown: str, key: str) -> str:
+ prefix = f"- {key}:"
+ for line in markdown.splitlines():
+ if line.lower().startswith(prefix.lower()):
+ return line.partition(":")[2].strip()
+ return ""
+
+
+def _extract_promotion_metadata(markdown: str) -> dict[str, str]:
+ metadata: dict[str, str] = {}
+ reserved = {"promoted_at", "source", "tags", "card"}
+ for line in markdown.splitlines():
+ stripped = line.strip()
+ if stripped.startswith("### "):
+ break
+ if not stripped.startswith("- ") or ":" not in stripped:
+ continue
+ key, _, value = stripped[2:].partition(":")
+ clean_key = key.strip()
+ if clean_key and clean_key not in reserved:
+ metadata[clean_key] = value.strip()
+ return metadata
+
+
+def _merge_unique(existing: list[str] | str | object, incoming: list[str]) -> list[str]:
+ values = existing if isinstance(existing, list) else []
+ merged: list[str] = []
+ seen: set[str] = set()
+ for value in [*values, *incoming]:
+ clean = str(value).strip()
+ key = clean.lower()
+ if clean and key not in seen:
+ merged.append(clean)
+ seen.add(key)
+ return merged
+
+
+def _ensure_skill_file(skill_path: Path) -> None:
+ if skill_path.exists():
+ return
+ skill_path.parent.mkdir(parents=True, exist_ok=True)
+ skill_path.write_text(
+ f"# Skill\n\n{GENERATED_START}\nNo generated entries yet.\n{GENERATED_END}\n",
+ encoding="utf-8",
+ )
+
+
+def _trace_filename(trace: SkillTrace) -> str:
+ slug = _slug(trace.task) or "skill-trace"
+ digest = hashlib.sha256(trace.to_markdown().encode("utf-8")).hexdigest()[:10]
+ timestamp = trace.created_at.replace(":", "").replace("-", "").replace(".", "")
+ timestamp = timestamp.replace("Z", "z")
+ return f"{timestamp}_{slug}_{digest}.md"
+
+
+def _slug(value: str) -> str:
+ lowered = value.lower()
+ chars: list[str] = []
+ previous_dash = False
+ for char in lowered:
+ if char.isalnum():
+ chars.append(char)
+ previous_dash = False
+ elif not previous_dash:
+ chars.append("-")
+ previous_dash = True
+ return "".join(chars).strip("-")[:80]
+
+
+__all__ = [
+ "SkillEvolutionProposal",
+ "SkillPromotionRecord",
+ "SkillToolTrace",
+ "SkillTrace",
+ "SkillTraceOutcome",
+ "SkillTraceRecord",
+ "promote_skill",
+ "record_skill_trace",
+ "suggest_skill_promotions",
+]
diff --git a/src/memu/app/skill_trace_cli.py b/src/memu/app/skill_trace_cli.py
new file mode 100644
index 00000000..661b06d3
--- /dev/null
+++ b/src/memu/app/skill_trace_cli.py
@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import argparse
+import json
+from collections.abc import Sequence
+from typing import Any, cast
+
+from memu.app.folder import FolderMemoryCompilerConfig, compile_folder_to_markdown_sync
+from memu.app.skill_trace import SkillToolTrace, SkillTraceOutcome, record_skill_trace
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ prog="memu-skill-trace",
+ description="Record an agent/tool execution trace into raw_data for self-evolving skills.",
+ )
+ parser.add_argument("raw_data_dir", help="Raw data folder where skill_traces/ will be written.")
+ parser.add_argument("--task", required=True, help="Task or situation this trace describes.")
+ parser.add_argument(
+ "--outcome",
+ choices=("success", "failure", "partial", "unknown"),
+ default="unknown",
+ help="Outcome of the task.",
+ )
+ parser.add_argument("--summary", default="", help="Short summary of what happened.")
+ parser.add_argument("--action", action="append", default=[], help="Action/workflow step. Can be repeated.")
+ parser.add_argument("--lesson", action="append", default=[], help="Reusable skill lesson. Can be repeated.")
+ parser.add_argument(
+ "--tool",
+ action="append",
+ default=[],
+ metavar="NAME[:success|failure][:score]",
+ help="Tool usage summary. Can be repeated.",
+ )
+ parser.add_argument(
+ "--metadata",
+ action="append",
+ default=[],
+ metavar="KEY=VALUE",
+ help="Extra metadata stored on the trace. Can be repeated.",
+ )
+ parser.add_argument(
+ "--output-folder",
+ default=None,
+ help="Optional memory repo folder to recompile after recording the trace.",
+ )
+ parser.add_argument("--json", action="store_true", help="Print a machine-readable JSON summary.")
+ return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ parser = build_parser()
+ args = parser.parse_args(argv)
+ record = record_skill_trace(
+ args.raw_data_dir,
+ task=args.task,
+ outcome=cast(SkillTraceOutcome, args.outcome),
+ summary=args.summary,
+ actions=list(args.action),
+ tools=[_parse_tool(value) for value in args.tool],
+ lessons=list(args.lesson),
+ metadata=_parse_key_values(args.metadata, flag_name="--metadata"),
+ )
+ compile_summary: dict[str, Any] | None = None
+ if args.output_folder:
+ result = compile_folder_to_markdown_sync(
+ args.raw_data_dir,
+ args.output_folder,
+ config=FolderMemoryCompilerConfig(use_memory_service=False),
+ )
+ compile_summary = {
+ "processed": result.processed,
+ "skipped": result.skipped,
+ "removed": result.removed,
+ "entry_count": len(result.entries),
+ }
+
+ summary = {
+ "raw_data_dir": str(record.raw_data_dir),
+ "trace_path": str(record.trace_path),
+ "task": record.trace.task,
+ "outcome": record.trace.outcome,
+ "compiled": compile_summary,
+ }
+ if args.json:
+ print(json.dumps(summary, indent=2, sort_keys=True))
+ else:
+ print("memU skill trace recorded")
+ print(f" trace: {summary['trace_path']}")
+ print(f" task: {summary['task']}")
+ print(f" outcome: {summary['outcome']}")
+ if compile_summary is not None:
+ print(f" compiled entries: {compile_summary['entry_count']}")
+ return 0
+
+
+def _parse_tool(value: str) -> SkillToolTrace:
+ parts = value.split(":")
+ name = parts[0].strip()
+ if not name:
+ msg = "--tool values must start with a tool name"
+ raise SystemExit(msg)
+ success = True
+ score: float | None = None
+ if len(parts) >= 2 and parts[1].strip():
+ status = parts[1].strip().lower()
+ if status not in {"success", "failure"}:
+ msg = "--tool status must be success or failure"
+ raise SystemExit(msg)
+ success = status == "success"
+ if len(parts) >= 3 and parts[2].strip():
+ score = float(parts[2].strip())
+ return SkillToolTrace(name=name, success=success, score=score)
+
+
+def _parse_key_values(values: Sequence[str], *, flag_name: str) -> dict[str, str]:
+ result: dict[str, str] = {}
+ for raw in values:
+ key, sep, value = raw.partition("=")
+ if not sep or not key.strip():
+ msg = f"{flag_name} values must be KEY=VALUE, got: {raw!r}"
+ raise SystemExit(msg)
+ result[key.strip()] = value
+ return result
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/src/memu/client/openai_wrapper.py b/src/memu/client/openai_wrapper.py
index 5c295f88..878c841a 100644
--- a/src/memu/client/openai_wrapper.py
+++ b/src/memu/client/openai_wrapper.py
@@ -8,12 +8,56 @@
from __future__ import annotations
import asyncio
+import inspect
from typing import TYPE_CHECKING, Any
+from memu.utils.retrieve import normalize_retrieve_ranking
+
if TYPE_CHECKING:
from memu.app.service import MemoryService
+def _normalize_top_k(top_k: int) -> int:
+ if isinstance(top_k, bool) or not isinstance(top_k, int) or top_k <= 0:
+ msg = "top_k must be a positive integer"
+ raise ValueError(msg)
+ return top_k
+
+
+def _copy_user_data(user_data: dict[str, Any]) -> dict[str, Any]:
+ return dict(user_data)
+
+
+def _extract_text_from_message_content(content: Any) -> str:
+ if isinstance(content, str):
+ return content
+ if isinstance(content, dict):
+ text = content.get("text")
+ return text if isinstance(text, str) else ""
+ if isinstance(content, list):
+ chunks: list[str] = []
+ for part in content:
+ if not isinstance(part, dict):
+ continue
+ text = part.get("text")
+ if isinstance(text, str) and text:
+ chunks.append(text)
+ return "\n".join(chunks)
+ return ""
+
+
+def _append_recall_context(content: Any, recall_context: str) -> str | list[Any]:
+ if isinstance(content, str):
+ return content + recall_context
+ if isinstance(content, list):
+ copied_parts = [dict(part) if isinstance(part, dict) else part for part in content]
+ copied_parts.append({"type": "text", "text": recall_context.lstrip("\n")})
+ return copied_parts
+ if content is None:
+ return recall_context.lstrip("\n")
+ return f"{content}{recall_context}"
+
+
class MemuChatCompletions:
"""Wrapper for chat.completions that injects recalled memories."""
@@ -27,22 +71,15 @@ def __init__(
):
self._original = original_completions
self._service = service
- self._user_data = user_data
- self._ranking = ranking
- self._top_k = top_k
+ self._user_data = _copy_user_data(user_data)
+ self._ranking = normalize_retrieve_ranking(ranking, default="salience")
+ self._top_k = _normalize_top_k(top_k)
def _extract_user_query(self, messages: list[dict]) -> str:
"""Extract the most recent user message."""
for msg in reversed(messages):
if msg.get("role") == "user":
- content = msg.get("content", "")
- if isinstance(content, str):
- return content
- # Handle content as list (vision models)
- if isinstance(content, list):
- for part in content:
- if isinstance(part, dict) and part.get("type") == "text":
- return part.get("text", "")
+ return _extract_text_from_message_content(msg.get("content", ""))
return ""
def _inject_memories(self, messages: list[dict], memories: list[dict]) -> list[dict]:
@@ -64,7 +101,7 @@ def _inject_memories(self, messages: list[dict], memories: list[dict]) -> list[d
# Inject into system message or create one
if messages and messages[0].get("role") == "system":
- messages[0]["content"] = messages[0]["content"] + recall_context
+ messages[0]["content"] = _append_recall_context(messages[0].get("content"), recall_context)
else:
messages.insert(0, {"role": "system", "content": recall_context.lstrip("\n")})
@@ -75,9 +112,13 @@ async def _retrieve_memories(self, query: str) -> list[dict]:
try:
result = await self._service.retrieve(
queries=[{"role": "user", "content": query}],
- where=self._user_data,
+ where=_copy_user_data(self._user_data),
+ ranking=self._ranking,
)
- return result.get("items", [])
+ items = result.get("items", [])
+ if not isinstance(items, list):
+ return []
+ return items[: self._top_k]
except Exception:
# Fail silently - don't break the LLM call
return []
@@ -119,8 +160,12 @@ async def acreate(self, **kwargs) -> Any:
# Call original async method if available
if hasattr(self._original, "acreate"):
- return await self._original.acreate(**kwargs)
- return self._original.create(**kwargs)
+ result = self._original.acreate(**kwargs)
+ else:
+ result = self._original.create(**kwargs)
+ if inspect.isawaitable(result):
+ return await result
+ return result
def __getattr__(self, name: str) -> Any:
"""Proxy all other attributes to original."""
@@ -192,21 +237,21 @@ def __init__(
service: memU MemoryService instance
user_data: User scope data (user_id, agent_id, session_id, etc.)
ranking: Retrieval ranking strategy ("similarity" or "salience")
- top_k: Number of memories to retrieve
+ top_k: Maximum number of recalled memory items to inject
"""
self._client = client
self._service = service
- self._user_data = user_data
- self._ranking = ranking
- self._top_k = top_k
+ self._user_data = _copy_user_data(user_data)
+ self._ranking = normalize_retrieve_ranking(ranking, default="salience")
+ self._top_k = _normalize_top_k(top_k)
# Wrap chat namespace
self.chat = MemuChat(
client.chat,
service,
- user_data,
- ranking,
- top_k,
+ self._user_data,
+ self._ranking,
+ self._top_k,
)
def __getattr__(self, name: str) -> Any:
@@ -235,7 +280,7 @@ def wrap_openai(
agent_id: Agent identifier (for multi-agent scoping)
session_id: Session identifier
ranking: Retrieval ranking ("similarity" or "salience")
- top_k: Number of memories to retrieve
+ top_k: Maximum number of recalled memory items to inject
Returns:
Wrapped client with auto-recall enabled
@@ -257,12 +302,14 @@ def wrap_openai(
)
"""
if user_data is None:
- user_data = {}
+ scope: dict[str, Any] = {}
+ else:
+ scope = _copy_user_data(user_data)
if user_id:
- user_data["user_id"] = user_id
+ scope["user_id"] = user_id
if agent_id:
- user_data["agent_id"] = agent_id
+ scope["agent_id"] = agent_id
if session_id:
- user_data["session_id"] = session_id
+ scope["session_id"] = session_id
- return MemuOpenAIWrapper(client, service, user_data, ranking, top_k)
+ return MemuOpenAIWrapper(client, service, scope, ranking, top_k)
diff --git a/src/memu/database/inmemory/repositories/category_item_repo.py b/src/memu/database/inmemory/repositories/category_item_repo.py
index 32e03fb2..203e1471 100644
--- a/src/memu/database/inmemory/repositories/category_item_repo.py
+++ b/src/memu/database/inmemory/repositories/category_item_repo.py
@@ -21,6 +21,14 @@ def list_relations(self, where: Mapping[str, Any] | None = None) -> list[Categor
return list(self.relations)
return [rel for rel in self.relations if matches_where(rel, where)]
+ def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[CategoryItem]:
+ deleted = self.list_relations(where)
+ if not deleted:
+ return []
+ deleted_ids = {rel.id for rel in deleted}
+ self.relations[:] = [rel for rel in self.relations if rel.id not in deleted_ids]
+ return deleted
+
def link_item_category(self, item_id: str, cat_id: str, user_data: dict[str, Any]) -> CategoryItem:
_ = item_id # enforced by caller via existing state
for rel in self.relations:
@@ -39,7 +47,9 @@ def get_item_categories(self, item_id: str) -> list[CategoryItem]:
@override
def unlink_item_category(self, item_id: str, cat_id: str) -> None:
- self.relations = [rel for rel in self.relations if not (rel.item_id == item_id and rel.category_id == cat_id)]
+ self.relations[:] = [
+ rel for rel in self.relations if not (rel.item_id == item_id and rel.category_id == cat_id)
+ ]
__all__ = ["InMemoryCategoryItemRepository"]
diff --git a/src/memu/database/inmemory/repositories/filter.py b/src/memu/database/inmemory/repositories/filter.py
index ad245cc7..71faa4ad 100644
--- a/src/memu/database/inmemory/repositories/filter.py
+++ b/src/memu/database/inmemory/repositories/filter.py
@@ -3,6 +3,8 @@
from collections.abc import Mapping
from typing import Any
+from memu.utils.filtering import normalize_filter_value, split_filter_key
+
def matches_where(obj: Any, where: Mapping[str, Any] | None) -> bool:
"""Basic field/`__in` matcher for in-memory repos."""
@@ -11,7 +13,8 @@ def matches_where(obj: Any, where: Mapping[str, Any] | None) -> bool:
for raw_key, expected in where.items():
if expected is None:
continue
- field, op = [*raw_key.split("__", 1), None][:2]
+ field, op = split_filter_key(raw_key)
+ expected = normalize_filter_value(field, op, expected)
actual = getattr(obj, str(field), None)
if op == "in":
if isinstance(expected, str):
diff --git a/src/memu/database/inmemory/repositories/memory_category_repo.py b/src/memu/database/inmemory/repositories/memory_category_repo.py
index bb07ec10..04ac1b1e 100644
--- a/src/memu/database/inmemory/repositories/memory_category_repo.py
+++ b/src/memu/database/inmemory/repositories/memory_category_repo.py
@@ -29,7 +29,8 @@ def clear_categories(self, where: Mapping[str, Any] | None = None) -> dict[str,
self.categories.clear()
return matches
matches = {cid: cat for cid, cat in self.categories.items() if matches_where(cat, where)}
- self.categories = {cid: cat for cid, cat in self.categories.items() if cid not in matches}
+ for cat_id in matches:
+ self.categories.pop(cat_id, None)
return matches
def get_or_create_category(
diff --git a/src/memu/database/inmemory/repositories/memory_item_repo.py b/src/memu/database/inmemory/repositories/memory_item_repo.py
index da28e14f..c67fe7a7 100644
--- a/src/memu/database/inmemory/repositories/memory_item_repo.py
+++ b/src/memu/database/inmemory/repositories/memory_item_repo.py
@@ -56,7 +56,8 @@ def clear_items(self, where: Mapping[str, Any] | None = None) -> dict[str, Memor
self.items.clear()
return matches
matches = {mid: item for mid, item in self.items.items() if matches_where(item, where)}
- self.items = {mid: item for mid, item in self.items.items() if mid not in matches}
+ for item_id in matches:
+ self.items.pop(item_id, None)
return matches
def _find_by_hash(self, content_hash: str, user_data: dict[str, Any]) -> MemoryItem | None:
@@ -79,7 +80,7 @@ def _find_by_hash(self, content_hash: str, user_data: dict[str, Any]) -> MemoryI
def create_item(
self,
*,
- resource_id: str,
+ resource_id: str | None = None,
memory_type: MemoryType,
summary: str,
embedding: list[float],
@@ -122,7 +123,7 @@ def create_item(
def create_item_reinforce(
self,
*,
- resource_id: str,
+ resource_id: str | None = None,
memory_type: MemoryType,
summary: str,
embedding: list[float],
diff --git a/src/memu/database/inmemory/repositories/resource_repo.py b/src/memu/database/inmemory/repositories/resource_repo.py
index ba60e52b..26c4f266 100644
--- a/src/memu/database/inmemory/repositories/resource_repo.py
+++ b/src/memu/database/inmemory/repositories/resource_repo.py
@@ -27,7 +27,8 @@ def clear_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, R
self.resources.clear()
return matches
matches = {rid: res for rid, res in self.resources.items() if matches_where(res, where)}
- self.resources = {rid: res for rid, res in self.resources.items() if rid not in matches}
+ for res_id in matches:
+ self.resources.pop(res_id, None)
return matches
def create_resource(
diff --git a/src/memu/database/inmemory/vector.py b/src/memu/database/inmemory/vector.py
index cd5355c7..001b53e5 100644
--- a/src/memu/database/inmemory/vector.py
+++ b/src/memu/database/inmemory/vector.py
@@ -58,6 +58,9 @@ def cosine_topk(
corpus: Iterable[tuple[str, list[float] | None]],
k: int = 5,
) -> list[tuple[str, float]]:
+ if k <= 0:
+ return []
+
# Filter out None vectors and collect valid entries
ids: list[str] = []
vecs: list[list[float]] = []
@@ -111,6 +114,9 @@ def cosine_topk_salience(
Returns:
List of (id, salience_score) tuples, sorted by score descending
"""
+ if k <= 0:
+ return []
+
q = np.array(query_vec, dtype=np.float32)
scored: list[tuple[str, float]] = []
diff --git a/src/memu/database/models.py b/src/memu/database/models.py
index 0124b784..bc7a8cb2 100644
--- a/src/memu/database/models.py
+++ b/src/memu/database/models.py
@@ -9,32 +9,16 @@
import pendulum
from pydantic import BaseModel, ConfigDict, Field
-MemoryType = Literal["profile", "event", "knowledge", "behavior", "skill", "tool"]
-
-
-def compute_content_hash(summary: str, memory_type: str) -> str:
- """
- Generate unique hash for memory deduplication.
-
- Operates on post-summary content. Normalizes whitespace to handle
- minor formatting differences like "I love coffee" vs "I love coffee".
+from memu.utils.dedupe import compute_content_hash
- Args:
- summary: The memory summary text
- memory_type: The type of memory (profile, event, etc.)
-
- Returns:
- A 16-character hex hash string
- """
- # Normalize: lowercase, strip, collapse whitespace
- normalized = " ".join(summary.lower().split())
- content = f"{memory_type}:{normalized}"
- return hashlib.sha256(content.encode()).hexdigest()[:16]
+MemoryType = Literal["profile", "event", "knowledge", "behavior", "skill", "tool"]
class BaseRecord(BaseModel):
"""Backend-agnostic record interface."""
+ model_config = ConfigDict(extra="allow")
+
id: str = Field(default_factory=lambda: str(uuid.uuid4()))
created_at: datetime = Field(default_factory=lambda: pendulum.now("UTC"))
updated_at: datetime = Field(default_factory=lambda: pendulum.now("UTC"))
@@ -79,7 +63,7 @@ class MemoryItem(BaseRecord):
summary: str
embedding: list[float] | None = None
happened_at: datetime | None = None
- extra: dict[str, Any] = {}
+ extra: dict[str, Any] = Field(default_factory=dict)
# extra may contain:
# # reinforcement tracking fields
# - content_hash: str
diff --git a/src/memu/database/postgres/models.py b/src/memu/database/postgres/models.py
index e83797a2..4e083e0f 100644
--- a/src/memu/database/postgres/models.py
+++ b/src/memu/database/postgres/models.py
@@ -6,11 +6,12 @@
import pendulum
+from memu.database.postgres.optional import postgres_extra_import_error
+
try:
from pgvector.sqlalchemy import VECTOR as Vector
except ImportError as exc:
- msg = "pgvector is required for Postgres vector support"
- raise ImportError(msg) from exc
+ raise postgres_extra_import_error() from exc
from pydantic import BaseModel
from sqlalchemy import ForeignKey, MetaData, String, Text
@@ -57,7 +58,7 @@ class MemoryItemModel(BaseModelMixin, MemoryItem):
summary: str = Field(sa_column=Column(Text, nullable=False))
embedding: list[float] | None = Field(default=None, sa_column=Column(Vector(), nullable=True))
happened_at: datetime | None = Field(default=None, sa_column=Column(DateTime, nullable=True))
- extra: dict[str, Any] = Field(default={}, sa_column=Column(JSONB, nullable=True))
+ extra: dict[str, Any] = Field(default_factory=dict, sa_column=Column(JSONB, nullable=True))
class MemoryCategoryModel(BaseModelMixin, MemoryCategory):
diff --git a/src/memu/database/postgres/optional.py b/src/memu/database/postgres/optional.py
new file mode 100644
index 00000000..c7d5bde9
--- /dev/null
+++ b/src/memu/database/postgres/optional.py
@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+POSTGRES_EXTRA_INSTALL_HINT = (
+ "Postgres storage requires the optional Postgres dependencies. "
+ "Install them with `pip install 'memu-py[postgres]'`, or run "
+ "`uv sync --extra postgres` from a source checkout."
+)
+
+
+def postgres_extra_import_error() -> ImportError:
+ return ImportError(POSTGRES_EXTRA_INSTALL_HINT)
+
+
+__all__ = ["POSTGRES_EXTRA_INSTALL_HINT", "postgres_extra_import_error"]
diff --git a/src/memu/database/postgres/repositories/base.py b/src/memu/database/postgres/repositories/base.py
index 0823dbf8..32745933 100644
--- a/src/memu/database/postgres/repositories/base.py
+++ b/src/memu/database/postgres/repositories/base.py
@@ -7,6 +7,7 @@
import pendulum
from memu.database.postgres.session import SessionManager
+from memu.utils.filtering import normalize_filter_value, split_filter_key
from memu.database.state import DatabaseState
logger = logging.getLogger(__name__)
@@ -71,7 +72,8 @@ def _build_filters(self, model: Any, where: Mapping[str, Any] | None) -> list[An
for raw_key, expected in where.items():
if expected is None:
continue
- field, op = [*raw_key.split("__", 1), None][:2]
+ field, op = split_filter_key(raw_key)
+ expected = normalize_filter_value(field, op, expected)
column = getattr(model, str(field), None)
if column is None:
msg = f"Unknown filter field '{field}' for model '{model.__name__}'"
@@ -92,7 +94,8 @@ def _matches_where(obj: Any, where: Mapping[str, Any] | None) -> bool:
for raw_key, expected in where.items():
if expected is None:
continue
- field, op = [*raw_key.split("__", 1), None][:2]
+ field, op = split_filter_key(raw_key)
+ expected = normalize_filter_value(field, op, expected)
actual = getattr(obj, str(field), None)
if op == "in":
if isinstance(expected, str):
diff --git a/src/memu/database/postgres/repositories/category_item_repo.py b/src/memu/database/postgres/repositories/category_item_repo.py
index 90409807..1affd21a 100644
--- a/src/memu/database/postgres/repositories/category_item_repo.py
+++ b/src/memu/database/postgres/repositories/category_item_repo.py
@@ -32,6 +32,24 @@ def list_relations(self, where: Mapping[str, Any] | None = None) -> list[Categor
rows = session.scalars(select(self._sqla_models.CategoryItem).where(*filters)).all()
return [self._cache_relation(row) for row in rows]
+ def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[CategoryItem]:
+ from sqlmodel import delete, select
+
+ filters = self._build_filters(self._sqla_models.CategoryItem, where)
+ with self._sessions.session() as session:
+ rows = session.scalars(select(self._sqla_models.CategoryItem).where(*filters)).all()
+ deleted = list(rows)
+
+ if not deleted:
+ return []
+
+ session.exec(delete(self._sqla_models.CategoryItem).where(*filters))
+ session.commit()
+
+ deleted_ids = {rel.id for rel in deleted}
+ self.relations[:] = [rel for rel in self.relations if rel.id not in deleted_ids]
+ return deleted
+
def link_item_category(self, item_id: str, cat_id: str, user_data: dict[str, Any]) -> CategoryItem:
from sqlmodel import select
@@ -76,6 +94,9 @@ def unlink_item_category(self, item_id: str, cat_id: str) -> None:
)
)
session.commit()
+ self.relations[:] = [
+ rel for rel in self.relations if not (rel.item_id == item_id and rel.category_id == cat_id)
+ ]
def get_item_categories(self, item_id: str) -> list[CategoryItem]:
from sqlmodel import select
@@ -95,6 +116,10 @@ def load_existing(self) -> None:
self._cache_relation(row)
def _cache_relation(self, rel: CategoryItem) -> CategoryItem:
+ for idx, existing in enumerate(self.relations):
+ if existing.id == rel.id:
+ self.relations[idx] = rel
+ return rel
self.relations.append(rel)
return rel
diff --git a/src/memu/database/postgres/repositories/memory_category_repo.py b/src/memu/database/postgres/repositories/memory_category_repo.py
index 229cd200..614a2cb3 100644
--- a/src/memu/database/postgres/repositories/memory_category_repo.py
+++ b/src/memu/database/postgres/repositories/memory_category_repo.py
@@ -57,8 +57,12 @@ def clear_categories(self, where: Mapping[str, Any] | None = None) -> dict[str,
session.commit()
# Clean up cache
- for cat_id in deleted:
+ deleted_category_ids = set(deleted)
+ for cat_id in deleted_category_ids:
self.categories.pop(cat_id, None)
+ self._state.relations[:] = [
+ rel for rel in self._state.relations if rel.category_id not in deleted_category_ids
+ ]
return deleted
diff --git a/src/memu/database/postgres/repositories/memory_item_repo.py b/src/memu/database/postgres/repositories/memory_item_repo.py
index 6d04f61b..5b9953c6 100644
--- a/src/memu/database/postgres/repositories/memory_item_repo.py
+++ b/src/memu/database/postgres/repositories/memory_item_repo.py
@@ -105,8 +105,10 @@ def clear_items(self, where: Mapping[str, Any] | None = None) -> dict[str, Memor
session.commit()
# Clean up cache
- for item_id in deleted:
+ deleted_item_ids = set(deleted)
+ for item_id in deleted_item_ids:
self.items.pop(item_id, None)
+ self._drop_relation_cache_for_items(deleted_item_ids)
return deleted
@@ -276,6 +278,8 @@ def delete_item(self, item_id: str) -> None:
with self._sessions.session() as session:
session.exec(delete(self._sqla_models.MemoryItem).where(self._sqla_models.MemoryItem.id == item_id))
session.commit()
+ self.items.pop(item_id, None)
+ self._drop_relation_cache_for_items({item_id})
def vector_search_items(
self,
@@ -286,6 +290,8 @@ def vector_search_items(
ranking: str = "similarity",
recency_decay_days: float = 30.0,
) -> list[tuple[str, float]]:
+ if top_k <= 0:
+ return []
if not self._use_vector or ranking == "salience":
# For salience ranking or when pgvector is not available, use local search
return self._vector_search_local(
@@ -326,11 +332,10 @@ def _vector_search_local(
recency_decay_days: float = 30.0,
) -> list[tuple[str, float]]:
scored: list[tuple[str, float]] = []
- for item in self.items.values():
+ pool = self.list_items(where)
+ for item in pool.values():
if item.embedding is None:
continue
- if not self._matches_where(item, where):
- continue
similarity = self._cosine(query_vec, item.embedding)
@@ -376,6 +381,13 @@ def _cache_item(self, item: MemoryItem) -> MemoryItem:
self.items[item.id] = item
return item
+ def _drop_relation_cache_for_items(self, item_ids: set[str]) -> None:
+ if not item_ids:
+ return
+ self._state.relations[:] = [
+ rel for rel in self._state.relations if rel.item_id not in item_ids
+ ]
+
@staticmethod
def _parse_datetime(dt_str: str | None) -> datetime | None:
"""Parse ISO datetime string from extra dict."""
diff --git a/src/memu/database/postgres/repositories/resource_repo.py b/src/memu/database/postgres/repositories/resource_repo.py
index d358febc..961afff0 100644
--- a/src/memu/database/postgres/repositories/resource_repo.py
+++ b/src/memu/database/postgres/repositories/resource_repo.py
@@ -57,8 +57,20 @@ def clear_resources(self, where: Mapping[str, Any] | None = None) -> dict[str, R
session.commit()
# Clean up cache
- for res_id in deleted:
+ deleted_resource_ids = set(deleted)
+ for res_id in deleted_resource_ids:
self.resources.pop(res_id, None)
+ deleted_item_ids = {
+ item_id
+ for item_id, item in self._state.items.items()
+ if item.resource_id in deleted_resource_ids
+ }
+ for item_id in deleted_item_ids:
+ self._state.items.pop(item_id, None)
+ if deleted_item_ids:
+ self._state.relations[:] = [
+ rel for rel in self._state.relations if rel.item_id not in deleted_item_ids
+ ]
return deleted
diff --git a/src/memu/database/postgres/schema.py b/src/memu/database/postgres/schema.py
index ac6e8b52..eca21237 100644
--- a/src/memu/database/postgres/schema.py
+++ b/src/memu/database/postgres/schema.py
@@ -5,6 +5,8 @@
from pydantic import BaseModel
+from memu.database.postgres.optional import postgres_extra_import_error
+
try:
from sqlmodel import SQLModel
except ImportError as exc:
@@ -20,8 +22,7 @@
try:
from pgvector.sqlalchemy import VECTOR as Vector
except ImportError as exc:
- msg = "pgvector is required for Postgres vector support"
- raise ImportError(msg) from exc
+ raise postgres_extra_import_error() from exc
from memu.database.postgres.models import (
CategoryItemModel,
@@ -72,6 +73,7 @@ def get_sqlalchemy_models(*, scope_model: type[BaseModel] | None = None) -> SQLA
MemoryCategoryModel,
tablename="memory_categories",
metadata=metadata_obj,
+ unique_with_scope=["name"],
)
memory_item_model = build_table_model(
scope,
diff --git a/src/memu/database/repositories/category_item.py b/src/memu/database/repositories/category_item.py
index 582a2845..49001a64 100644
--- a/src/memu/database/repositories/category_item.py
+++ b/src/memu/database/repositories/category_item.py
@@ -14,6 +14,8 @@ class CategoryItemRepo(Protocol):
def list_relations(self, where: Mapping[str, Any] | None = None) -> list[CategoryItem]: ...
+ def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[CategoryItem]: ...
+
def link_item_category(self, item_id: str, cat_id: str, user_data: dict[str, Any]) -> CategoryItem: ...
def unlink_item_category(self, item_id: str, cat_id: str) -> None: ...
diff --git a/src/memu/database/repositories/memory_item.py b/src/memu/database/repositories/memory_item.py
index 39bb856b..a2124005 100644
--- a/src/memu/database/repositories/memory_item.py
+++ b/src/memu/database/repositories/memory_item.py
@@ -21,7 +21,7 @@ def clear_items(self, where: Mapping[str, Any] | None = None) -> dict[str, Memor
def create_item(
self,
*,
- resource_id: str,
+ resource_id: str | None = None,
memory_type: MemoryType,
summary: str,
embedding: list[float],
diff --git a/src/memu/database/sqlite/models.py b/src/memu/database/sqlite/models.py
index 6cdaed49..aa405d0d 100644
--- a/src/memu/database/sqlite/models.py
+++ b/src/memu/database/sqlite/models.py
@@ -84,7 +84,7 @@ class SQLiteMemoryItemModel(SQLiteBaseModelMixin, MemoryItem):
# Store embedding as JSON string since SQLite doesn't have native vector type
embedding_json: str | None = Field(default=None, sa_column=Column(Text, nullable=True))
happened_at: datetime | None = Field(default=None, sa_column=Column(DateTime, nullable=True))
- extra: dict[str, Any] = Field(default={}, sa_column=Column(JSON, nullable=True))
+ extra: dict[str, Any] = Field(default_factory=dict, sa_column=Column(JSON, nullable=True))
@property
def embedding(self) -> list[float] | None:
diff --git a/src/memu/database/sqlite/repositories/base.py b/src/memu/database/sqlite/repositories/base.py
index 44099859..6dc1834f 100644
--- a/src/memu/database/sqlite/repositories/base.py
+++ b/src/memu/database/sqlite/repositories/base.py
@@ -10,6 +10,7 @@
import pendulum
from memu.database.sqlite.session import SQLiteSessionManager
+from memu.utils.filtering import normalize_filter_value, split_filter_key
from memu.database.state import DatabaseState
logger = logging.getLogger(__name__)
@@ -85,7 +86,8 @@ def _build_filters(self, model: Any, where: Mapping[str, Any] | None) -> list[An
for raw_key, expected in where.items():
if expected is None:
continue
- field, op = [*raw_key.split("__", 1), None][:2]
+ field, op = split_filter_key(raw_key)
+ expected = normalize_filter_value(field, op, expected)
column = getattr(model, str(field), None)
if column is None:
msg = f"Unknown filter field '{field}' for model '{model.__name__}'"
@@ -107,7 +109,8 @@ def _matches_where(obj: Any, where: Mapping[str, Any] | None) -> bool:
for raw_key, expected in where.items():
if expected is None:
continue
- field, op = [*raw_key.split("__", 1), None][:2]
+ field, op = split_filter_key(raw_key)
+ expected = normalize_filter_value(field, op, expected)
actual = getattr(obj, str(field), None)
if op == "in":
if isinstance(expected, str):
diff --git a/src/memu/database/sqlite/repositories/category_item_repo.py b/src/memu/database/sqlite/repositories/category_item_repo.py
index f6996650..cc0de064 100644
--- a/src/memu/database/sqlite/repositories/category_item_repo.py
+++ b/src/memu/database/sqlite/repositories/category_item_repo.py
@@ -6,7 +6,7 @@
from collections.abc import Mapping
from typing import Any
-from sqlmodel import select
+from sqlmodel import delete, select
from memu.database.models import CategoryItem
from memu.database.repositories.category_item import CategoryItemRepo
@@ -66,21 +66,35 @@ def list_relations(self, where: Mapping[str, Any] | None = None) -> list[Categor
result: list[CategoryItem] = []
for row in rows:
- rel = CategoryItem(
- id=row.id,
- item_id=row.item_id,
- category_id=row.category_id,
- created_at=row.created_at,
- updated_at=row.updated_at,
- **self._scope_kwargs_from(row),
- )
+ rel = self._relation_from_row(row)
result.append(rel)
- # Update cache
- if not any(r.id == rel.id for r in self.relations):
- self.relations.append(rel)
+ self._cache_relation(rel)
return result
+ def clear_relations(self, where: Mapping[str, Any] | None = None) -> list[CategoryItem]:
+ """Clear category-item relations matching the where clause."""
+ filters = self._build_filters(self._category_item_model, where)
+ with self._sessions.session() as session:
+ stmt = select(self._category_item_model)
+ if filters:
+ stmt = stmt.where(*filters)
+ rows = session.exec(stmt).all()
+ deleted = [self._relation_from_row(row) for row in rows]
+
+ if not deleted:
+ return []
+
+ del_stmt = delete(self._category_item_model)
+ if filters:
+ del_stmt = del_stmt.where(*filters)
+ session.exec(del_stmt)
+ session.commit()
+
+ deleted_ids = {rel.id for rel in deleted}
+ self.relations[:] = [rel for rel in self.relations if rel.id not in deleted_ids]
+ return deleted
+
def link_item_category(self, item_id: str, category_id: str, user_data: dict[str, Any]) -> CategoryItem:
"""Create a link between an item and a category.
@@ -106,15 +120,7 @@ def link_item_category(self, item_id: str, category_id: str, user_data: dict[str
existing = session.exec(stmt).first()
if existing:
- rel = CategoryItem(
- id=existing.id,
- item_id=existing.item_id,
- category_id=existing.category_id,
- created_at=existing.created_at,
- updated_at=existing.updated_at,
- **self._scope_kwargs_from(existing),
- )
- return rel
+ return self._cache_relation(self._relation_from_row(existing))
# Create new relation
now = self._now()
@@ -129,16 +135,7 @@ def link_item_category(self, item_id: str, category_id: str, user_data: dict[str
session.commit()
session.refresh(row)
- rel = CategoryItem(
- id=row.id,
- item_id=row.item_id,
- category_id=row.category_id,
- created_at=row.created_at,
- updated_at=row.updated_at,
- **user_data,
- )
- self.relations.append(rel)
- return rel
+ return self._cache_relation(self._relation_from_row(row))
def unlink_item_category(self, item_id: str, category_id: str) -> None:
"""Remove a link between an item and a category.
@@ -176,5 +173,23 @@ def load_existing(self) -> None:
"""Load all existing relations from database into cache."""
self.list_relations()
+ def _relation_from_row(self, row: Any) -> CategoryItem:
+ return CategoryItem(
+ id=row.id,
+ item_id=row.item_id,
+ category_id=row.category_id,
+ created_at=row.created_at,
+ updated_at=row.updated_at,
+ **self._scope_kwargs_from(row),
+ )
+
+ def _cache_relation(self, rel: CategoryItem) -> CategoryItem:
+ for idx, existing in enumerate(self.relations):
+ if existing.id == rel.id:
+ self.relations[idx] = rel
+ return rel
+ self.relations.append(rel)
+ return rel
+
__all__ = ["SQLiteCategoryItemRepo"]
diff --git a/src/memu/database/sqlite/repositories/memory_item_repo.py b/src/memu/database/sqlite/repositories/memory_item_repo.py
index 0bff124e..1ca32481 100644
--- a/src/memu/database/sqlite/repositories/memory_item_repo.py
+++ b/src/memu/database/sqlite/repositories/memory_item_repo.py
@@ -211,7 +211,7 @@ def clear_items(self, where: Mapping[str, Any] | None = None) -> dict[str, Memor
def create_item(
self,
*,
- resource_id: str,
+ resource_id: str | None = None,
memory_type: MemoryType,
summary: str,
embedding: list[float],
@@ -285,7 +285,7 @@ def create_item(
def create_item_reinforce(
self,
*,
- resource_id: str,
+ resource_id: str | None = None,
memory_type: MemoryType,
summary: str,
embedding: list[float],
diff --git a/src/memu/database/sqlite/schema.py b/src/memu/database/sqlite/schema.py
index 63291cb1..24b97b2c 100644
--- a/src/memu/database/sqlite/schema.py
+++ b/src/memu/database/sqlite/schema.py
@@ -60,6 +60,7 @@ def get_sqlite_sqlalchemy_models(*, scope_model: type[BaseModel] | None = None)
SQLiteMemoryCategoryModel,
tablename="sqlite_memory_categories",
metadata=metadata_obj,
+ unique_with_scope=["name"],
)
memory_item_model = build_sqlite_table_model(
scope,
diff --git a/src/memu/embedding/http_client.py b/src/memu/embedding/http_client.py
index 0c3066a7..bef5f608 100644
--- a/src/memu/embedding/http_client.py
+++ b/src/memu/embedding/http_client.py
@@ -97,9 +97,10 @@ async def embed_multimodal(
List of embedding vectors
Example:
+ >>> import os
>>> client = HTTPEmbeddingClient(
... base_url="https://ark.cn-beijing.volces.com",
- ... api_key="your-api-key",
+ ... api_key=os.environ["DOUBAO_API_KEY"],
... embed_model="doubao-embedding-vision-250615",
... provider="doubao",
... )
diff --git a/src/memu/integrations/langgraph.py b/src/memu/integrations/langgraph.py
index 2e24ddc6..212a2e59 100644
--- a/src/memu/integrations/langgraph.py
+++ b/src/memu/integrations/langgraph.py
@@ -7,19 +7,26 @@
import os
import tempfile
import uuid
-from typing import Any
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Annotated, Any
-# MUST explicitly import langgraph to satisfy DEP002
-import langgraph
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, StringConstraints
-from memu.app.service import MemoryService
+if TYPE_CHECKING:
+ from langchain_core.tools import BaseTool, StructuredTool
+ from memu.app.service import MemoryService
try:
+ # Explicit import keeps the optional integration dependency visible to tooling.
+ import langgraph
from langchain_core.tools import BaseTool, StructuredTool
-except ImportError as e:
- msg = "Please install 'langchain-core' (and 'langgraph') to use the LangGraph integration."
- raise ImportError(msg) from e
+except ImportError as exc: # pragma: no cover - covered by optional-dependency smoke tests.
+ langgraph = None # type: ignore[assignment]
+ BaseTool = Any # type: ignore[misc, assignment]
+ StructuredTool = None # type: ignore[assignment]
+ _LANGGRAPH_IMPORT_ERROR: ImportError | None = exc
+else:
+ _LANGGRAPH_IMPORT_ERROR = None
# Setup logger
@@ -30,24 +37,55 @@ class MemUIntegrationError(Exception):
"""Base exception for MemU integration issues."""
+NonEmptyString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
+
+
class SaveRecallInput(BaseModel):
"""Input schema for the save_memory tool."""
- content: str = Field(description="The text content or information to save/remember.")
- user_id: str = Field(description="The unique identifier of the user.")
+ content: NonEmptyString = Field(description="The text content or information to save/remember.")
+ user_id: NonEmptyString = Field(description="The unique identifier of the user.")
metadata: dict[str, Any] | None = Field(default=None, description="Additional metadata related to the memory.")
class SearchRecallInput(BaseModel):
"""Input schema for the search_memory tool."""
- query: str = Field(description="The search query to retrieve relevant memories.")
- user_id: str = Field(description="The unique identifier of the user.")
- limit: int = Field(default=5, description="Number of memories to retrieve.")
+ query: NonEmptyString = Field(description="The search query to retrieve relevant memories.")
+ user_id: NonEmptyString = Field(description="The unique identifier of the user.")
+ limit: int = Field(default=5, ge=1, description="Number of memories to retrieve.")
metadata_filter: dict[str, Any] | None = Field(
default=None, description="Optional filter for memory metadata (e.g., {'category': 'work'})."
)
- min_relevance_score: float = Field(default=0.0, description="Minimum relevance score (0.0 to 1.0) for results.")
+ min_relevance_score: float = Field(
+ default=0.0,
+ ge=0.0,
+ le=1.0,
+ description="Minimum relevance score (0.0 to 1.0) for results.",
+ )
+
+
+def _ensure_langgraph_dependencies() -> None:
+ if _LANGGRAPH_IMPORT_ERROR is None:
+ return
+ msg = (
+ "Please install the LangGraph integration dependencies with "
+ "`pip install 'memu-py[langgraph]'`, or run `uv sync --extra langgraph` "
+ "from a source checkout."
+ )
+ raise ImportError(msg) from _LANGGRAPH_IMPORT_ERROR
+
+
+def _scope_with_user(user_id: str, metadata: Mapping[str, Any] | None = None) -> dict[str, Any]:
+ if not isinstance(user_id, str) or not user_id.strip():
+ msg = "user_id must be a non-empty string"
+ raise ValueError(msg)
+ if metadata is not None and not isinstance(metadata, Mapping):
+ msg = "metadata must be an object"
+ raise ValueError(msg)
+ scope = dict(metadata or {})
+ scope["user_id"] = user_id.strip()
+ return scope
class MemULangGraphTools:
@@ -59,6 +97,7 @@ class MemULangGraphTools:
def __init__(self, memory_service: MemoryService):
"""Initializes the MemULangGraphTools with a memory service."""
+ _ensure_langgraph_dependencies()
self.memory_service = memory_service
# Expose the langgraph module to ensure it's "used" even if just by reference in this class
self._graph_backend = langgraph
@@ -75,6 +114,8 @@ def save_memory_tool(self) -> StructuredTool:
async def _save(content: str, user_id: str, metadata: dict | None = None) -> str:
logger.info("Entering save_memory_tool for user_id: %s", user_id)
+ content = content.strip()
+ user_scope = _scope_with_user(user_id, metadata)
filename = f"memu_input_{uuid.uuid4()}.txt"
temp_dir = tempfile.gettempdir()
file_path = os.path.join(temp_dir, filename)
@@ -87,9 +128,9 @@ async def _save(content: str, user_id: str, metadata: dict | None = None) -> str
await self.memory_service.memorize(
resource_url=file_path,
modality="conversation",
- user={"user_id": user_id, **(metadata or {})},
+ user=user_scope,
)
- logger.info("Successfully saved memory for user_id: %s", user_id)
+ logger.info("Successfully saved memory for user_id: %s", user_scope["user_id"])
except Exception as e:
error_msg = f"Failed to save memory for user {user_id}: {e!s}"
logger.exception(error_msg)
@@ -122,10 +163,9 @@ async def _search(
) -> str:
logger.info("Entering search_memory_tool for user_id: %s, query: '%s'", user_id, query)
try:
+ query = query.strip()
queries = [{"role": "user", "content": query}]
- where_filter = {"user_id": user_id}
- if metadata_filter:
- where_filter.update(metadata_filter)
+ where_filter = _scope_with_user(user_id, metadata_filter)
logger.debug("Calling memory_service.retrieve with where_filter: %s", where_filter)
result = await self.memory_service.retrieve(
diff --git a/src/memu/llm/lazyllm_client.py b/src/memu/llm/lazyllm_client.py
index 8446b6a5..aae06204 100644
--- a/src/memu/llm/lazyllm_client.py
+++ b/src/memu/llm/lazyllm_client.py
@@ -1,9 +1,28 @@
import asyncio
import functools
+import logging
from typing import Any, cast
-import lazyllm
-from lazyllm import LOG
+try:
+ import lazyllm
+ from lazyllm import LOG
+except ImportError as exc: # pragma: no cover - covered by optional-dependency smoke tests.
+ lazyllm = None # type: ignore[assignment]
+ LOG = logging.getLogger("memu.llm.lazyllm")
+ _LAZYLLM_IMPORT_ERROR: ImportError | None = exc
+else:
+ _LAZYLLM_IMPORT_ERROR = None
+
+
+def _lazyllm_module() -> Any:
+ if _LAZYLLM_IMPORT_ERROR is None:
+ return lazyllm
+ msg = (
+ "Please install the LazyLLM backend dependencies with "
+ "`pip install 'memu-py[lazyllm]'`, or run `uv sync --extra lazyllm` "
+ "from a source checkout."
+ )
+ raise ImportError(msg) from _LAZYLLM_IMPORT_ERROR
class LazyLLMClient:
@@ -23,6 +42,7 @@ def __init__(
embed_model: str | None = None,
stt_model: str | None = None,
):
+ _lazyllm_module()
self.llm_source = llm_source or self.DEFAULT_SOURCE
self.vlm_source = vlm_source or self.DEFAULT_SOURCE
self.embed_source = embed_source or self.DEFAULT_SOURCE
@@ -59,7 +79,9 @@ async def chat(
Return:
The generated summary text as a string.
"""
- client = lazyllm.namespace("MEMU").OnlineModule(source=self.llm_source, model=self.chat_model, type="llm")
+ client = _lazyllm_module().namespace("MEMU").OnlineModule(
+ source=self.llm_source, model=self.chat_model, type="llm"
+ )
prompt = f"{system_prompt}\n\n" if system_prompt else ""
full_prompt = f"{prompt}text:\n{text}"
LOG.debug(f"Summarizing text with {self.llm_source}/{self.chat_model}")
@@ -83,7 +105,9 @@ async def summarize(
Return:
The generated summary text as a string.
"""
- client = lazyllm.namespace("MEMU").OnlineModule(source=self.llm_source, model=self.chat_model, type="llm")
+ client = _lazyllm_module().namespace("MEMU").OnlineModule(
+ source=self.llm_source, model=self.chat_model, type="llm"
+ )
prompt = system_prompt or "Summarize the text in one short paragraph."
full_prompt = f"{prompt}\n\ntext:\n{text}"
LOG.debug(f"Summarizing text with {self.llm_source}/{self.chat_model}")
@@ -110,7 +134,9 @@ async def vision(
Return:
A tuple containing the generated text response and None (reserved for metadata).
"""
- client = lazyllm.namespace("MEMU").OnlineModule(source=self.vlm_source, model=self.vlm_model, type="vlm")
+ client = _lazyllm_module().namespace("MEMU").OnlineModule(
+ source=self.vlm_source, model=self.vlm_model, type="vlm"
+ )
LOG.debug(f"Processing image with {self.vlm_source}/{self.vlm_model}: {image_path}")
# LazyLLM VLM accepts prompt as first positional argument and image_path as keyword argument
response = await self._call_async(client, prompt, lazyllm_files=image_path)
@@ -130,7 +156,7 @@ async def embed(
Return:
A list of embedding vectors (list of floats), one for each input text.
"""
- client = lazyllm.namespace("MEMU").OnlineModule(
+ client = _lazyllm_module().namespace("MEMU").OnlineModule(
source=self.embed_source, model=self.embed_model, type="embed", batch_size=batch_size
)
LOG.debug(f"embed {len(texts)} texts with {self.embed_source}/{self.embed_model}")
@@ -153,7 +179,12 @@ async def transcribe(
Return:
The transcribed text as a string.
"""
- client = lazyllm.namespace("MEMU").OnlineModule(source=self.stt_source, model=self.stt_model, type="stt")
+ client = _lazyllm_module().namespace("MEMU").OnlineModule(
+ source=self.stt_source, model=self.stt_model, type="stt"
+ )
LOG.debug(f"Transcribing audio with {self.stt_source}/{self.stt_model}: {audio_path}")
response = await self._call_async(client, audio_path)
return cast(str, response)
+
+
+__all__ = ["LazyLLMClient"]
diff --git a/src/memu/llm/openai_sdk.py b/src/memu/llm/openai_sdk.py
index 38c6c8bb..2bd7e9e7 100644
--- a/src/memu/llm/openai_sdk.py
+++ b/src/memu/llm/openai_sdk.py
@@ -152,22 +152,23 @@ async def vision(
logger.debug("OpenAI vision response: %s", response)
return content or "", response
- async def embed(self, inputs: list[str]) -> tuple[list[list[float]], CreateEmbeddingResponse | None]:
+ async def embed(
+ self, inputs: list[str]
+ ) -> tuple[list[list[float]], CreateEmbeddingResponse | list[CreateEmbeddingResponse] | None]:
"""Create text embeddings via the official SDK."""
if len(inputs) <= self.embed_batch_size:
response = await self.client.embeddings.create(model=self.embed_model, input=inputs)
return [cast(list[float], d.embedding) for d in response.data], response
- # For batched requests, we aggregate embeddings but only return the last response for usage
all_embeddings: list[list[float]] = []
- last_response: CreateEmbeddingResponse | None = None
+ raw_responses: list[CreateEmbeddingResponse] = []
for idx in range(0, len(inputs), self.embed_batch_size):
batch = inputs[idx : idx + self.embed_batch_size]
response = await self.client.embeddings.create(model=self.embed_model, input=batch)
all_embeddings.extend([cast(list[float], d.embedding) for d in response.data])
- last_response = response
+ raw_responses.append(response)
- return all_embeddings, last_response
+ return all_embeddings, raw_responses
async def transcribe(
self,
diff --git a/src/memu/llm/wrapper.py b/src/memu/llm/wrapper.py
index 175b78fc..85f41cd4 100644
--- a/src/memu/llm/wrapper.py
+++ b/src/memu/llm/wrapper.py
@@ -8,6 +8,8 @@
import uuid
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
+from datetime import date, datetime
+from enum import Enum
from pathlib import Path
from typing import Any
@@ -602,6 +604,14 @@ def _get_attr_or_key(obj: Any, key: str) -> Any:
return None
+def _get_first_attr_or_key(obj: Any, *keys: str) -> Any:
+ for key in keys:
+ value = _get_attr_or_key(obj, key)
+ if value is not None:
+ return value
+ return None
+
+
def _extract_finish_reason(raw_response: Any) -> str | None:
"""Extract finish_reason from choices[0] if available."""
choices = _get_attr_or_key(raw_response, "choices")
@@ -623,29 +633,66 @@ def _get_usage_object(raw_response: Any) -> Any:
def _convert_to_dict(obj: Any) -> dict[str, Any] | None:
"""Convert object to dict using available methods."""
if hasattr(obj, "model_dump"):
- result: dict[str, Any] = obj.model_dump()
- return result
+ result = _model_dump(obj)
+ return _json_safe_dict(result)
if hasattr(obj, "__dict__"):
- return dict(obj.__dict__)
+ return _json_safe_dict(dict(obj.__dict__))
if isinstance(obj, dict):
- return obj
+ return _json_safe_dict(obj)
return None
+def _json_safe_dict(value: Any) -> dict[str, Any] | None:
+ if not isinstance(value, dict):
+ return None
+ return {str(key): _json_safe_value(item) for key, item in value.items()}
+
+
+def _json_safe_value(value: Any) -> Any:
+ if value is None or isinstance(value, (str, int, float, bool)):
+ return value
+ if isinstance(value, (datetime, date)):
+ return value.isoformat()
+ if isinstance(value, Enum):
+ return value.value
+ if hasattr(value, "model_dump"):
+ return _json_safe_value(_model_dump(value))
+ if isinstance(value, Mapping):
+ return {str(key): _json_safe_value(item) for key, item in value.items()}
+ if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+ return [_json_safe_value(item) for item in value]
+ return str(value)
+
+
+def _model_dump(value: Any) -> Any:
+ try:
+ return value.model_dump(mode="json")
+ except TypeError:
+ return value.model_dump()
+
+
def _extract_token_details(usage_obj: Any, usage_data: dict[str, Any]) -> None:
"""Extract token breakdown and cached tokens from usage object."""
- completion_tokens_details = _get_attr_or_key(usage_obj, "completion_tokens_details")
- if completion_tokens_details is not None:
- breakdown = _convert_to_dict(completion_tokens_details)
+ output_tokens_details = _get_first_attr_or_key(
+ usage_obj,
+ "output_tokens_details",
+ "completion_tokens_details",
+ )
+ if output_tokens_details is not None:
+ breakdown = _convert_to_dict(output_tokens_details)
if breakdown is not None:
usage_data["tokens_breakdown"] = breakdown
- reasoning_tokens = _get_attr_or_key(completion_tokens_details, "reasoning_tokens")
+ reasoning_tokens = _get_attr_or_key(output_tokens_details, "reasoning_tokens")
if reasoning_tokens is not None:
usage_data["reasoning_tokens"] = reasoning_tokens
- prompt_tokens_details = _get_attr_or_key(usage_obj, "prompt_tokens_details")
- if prompt_tokens_details is not None:
- cached_tokens = _get_attr_or_key(prompt_tokens_details, "cached_tokens")
+ input_tokens_details = _get_first_attr_or_key(
+ usage_obj,
+ "input_tokens_details",
+ "prompt_tokens_details",
+ )
+ if input_tokens_details is not None:
+ cached_tokens = _get_attr_or_key(input_tokens_details, "cached_tokens")
if cached_tokens is not None:
usage_data["cached_input_tokens"] = cached_tokens
@@ -665,6 +712,8 @@ def _extract_usage_from_raw_response(kind: str, raw_response: Any) -> dict[str,
if raw_response is None:
return usage_data
+ if _is_raw_response_batch(raw_response):
+ return _extract_usage_from_raw_response_batch(kind, raw_response)
try:
finish_reason = _extract_finish_reason(raw_response)
@@ -675,17 +724,15 @@ def _extract_usage_from_raw_response(kind: str, raw_response: Any) -> dict[str,
if usage_obj is None:
return usage_data
- # Map prompt_tokens -> input_tokens
- prompt_tokens = _get_attr_or_key(usage_obj, "prompt_tokens")
- if prompt_tokens is not None:
- usage_data["input_tokens"] = prompt_tokens
+ # Normalize OpenAI-compatible chat-completions and responses-style usage names.
+ input_tokens = _get_first_attr_or_key(usage_obj, "input_tokens", "prompt_tokens")
+ if input_tokens is not None:
+ usage_data["input_tokens"] = input_tokens
- # Map completion_tokens -> output_tokens
- completion_tokens = _get_attr_or_key(usage_obj, "completion_tokens")
- if completion_tokens is not None:
- usage_data["output_tokens"] = completion_tokens
+ output_tokens = _get_first_attr_or_key(usage_obj, "output_tokens", "completion_tokens")
+ if output_tokens is not None:
+ usage_data["output_tokens"] = output_tokens
- # total_tokens stays the same
total_tokens = _get_attr_or_key(usage_obj, "total_tokens")
if total_tokens is not None:
usage_data["total_tokens"] = total_tokens
@@ -703,6 +750,44 @@ def _extract_usage_from_raw_response(kind: str, raw_response: Any) -> dict[str,
return usage_data
+def _is_raw_response_batch(raw_response: Any) -> bool:
+ return isinstance(raw_response, Sequence) and not isinstance(raw_response, (str, bytes, bytearray, dict))
+
+
+def _extract_usage_from_raw_response_batch(kind: str, raw_responses: Sequence[Any]) -> dict[str, Any]:
+ aggregated: dict[str, Any] = {}
+ token_fields = (
+ "input_tokens",
+ "output_tokens",
+ "total_tokens",
+ "cached_input_tokens",
+ "reasoning_tokens",
+ )
+
+ for raw_response in raw_responses:
+ usage = _extract_usage_from_raw_response(kind, raw_response)
+ for field_name in token_fields:
+ value = usage.get(field_name)
+ if isinstance(value, int | float):
+ aggregated[field_name] = aggregated.get(field_name, 0) + value
+ if usage.get("finish_reason") is not None:
+ aggregated["finish_reason"] = usage["finish_reason"]
+ breakdown = usage.get("tokens_breakdown")
+ if isinstance(breakdown, dict):
+ _sum_token_breakdown(aggregated, breakdown)
+
+ return aggregated
+
+
+def _sum_token_breakdown(aggregated: dict[str, Any], breakdown: dict[str, Any]) -> None:
+ current = aggregated.setdefault("tokens_breakdown", {})
+ if not isinstance(current, dict):
+ return
+ for key, value in breakdown.items():
+ if isinstance(value, int | float):
+ current[key] = current.get(key, 0) + value
+
+
def _coerce_filter(
where: LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | Mapping[str, Any] | None,
) -> LLMCallFilter | Callable[[LLMCallContext, str | None], bool] | None:
diff --git a/src/memu/py.typed b/src/memu/py.typed
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/src/memu/py.typed
@@ -0,0 +1 @@
+
diff --git a/src/memu/utils/dedupe.py b/src/memu/utils/dedupe.py
new file mode 100644
index 00000000..aad6d988
--- /dev/null
+++ b/src/memu/utils/dedupe.py
@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+
+def compute_content_hash(summary: str, memory_type: str) -> str:
+ """
+ Generate a stable hash for memory deduplication.
+
+ The normalization intentionally matches the salience/reinforcement content
+ hash used by storage backends: lowercase, trim, and collapse whitespace.
+ """
+ normalized = " ".join(summary.lower().split())
+ content = f"{memory_type}:{normalized}"
+ return hashlib.sha256(content.encode()).hexdigest()[:16]
+
+
+def dedupe_resource_plans(resource_plans: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
+ """Remove duplicate extracted memory entries while preserving first-seen order."""
+ seen_categories: dict[str, list[str]] = {}
+ deduped_plans: list[dict[str, Any]] = []
+
+ for plan in resource_plans:
+ next_plan = dict(plan)
+ next_entries: list[tuple[str, str, list[str]]] = []
+ for raw_entry in plan.get("entries") or []:
+ normalized_entry = normalize_extracted_entry(raw_entry)
+ if normalized_entry is None:
+ continue
+ memory_type, content, categories = normalized_entry
+ key = compute_content_hash(content, memory_type)
+ if key in seen_categories:
+ seen_categories[key][:] = merge_category_names(seen_categories[key], categories)
+ continue
+ next_entries.append((memory_type, content, categories))
+ seen_categories[key] = categories
+ next_plan["entries"] = next_entries
+ deduped_plans.append(next_plan)
+
+ return deduped_plans
+
+
+def normalize_extracted_entry(raw_entry: Any) -> tuple[str, str, list[str]] | None:
+ if not isinstance(raw_entry, tuple) or len(raw_entry) != 3:
+ return None
+ raw_memory_type, raw_content, raw_categories = raw_entry
+ if not isinstance(raw_memory_type, str) or not isinstance(raw_content, str):
+ return None
+ content = raw_content.strip()
+ if not content:
+ return None
+ categories = [
+ category.strip()
+ for category in (raw_categories or [])
+ if isinstance(category, str) and category.strip()
+ ]
+ return raw_memory_type, content, merge_category_names([], categories)
+
+
+def merge_category_names(existing: Sequence[str], incoming: Sequence[str]) -> list[str]:
+ merged: list[str] = []
+ seen: set[str] = set()
+ for category in [*existing, *incoming]:
+ key = category.strip().lower()
+ if not key or key in seen:
+ continue
+ merged.append(category.strip())
+ seen.add(key)
+ return merged
+
+
+__all__ = [
+ "compute_content_hash",
+ "dedupe_resource_plans",
+ "merge_category_names",
+ "normalize_extracted_entry",
+]
diff --git a/src/memu/utils/filtering.py b/src/memu/utils/filtering.py
new file mode 100644
index 00000000..bf8aa809
--- /dev/null
+++ b/src/memu/utils/filtering.py
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+
+SUPPORTED_FILTER_OPERATORS = frozenset({"in"})
+
+
+def split_filter_key(raw_key: Any) -> tuple[str, str | None]:
+ """Split and validate a filter key.
+
+ Supported filters are equality (`field`) and membership (`field__in`).
+ """
+
+ if not isinstance(raw_key, str) or not raw_key.strip():
+ msg = "Filter field must be a non-empty string"
+ raise ValueError(msg)
+
+ key = raw_key.strip()
+ field, separator, operator = key.partition("__")
+ if not field:
+ msg = "Filter field must be a non-empty string"
+ raise ValueError(msg)
+ if not separator:
+ return field, None
+ if operator not in SUPPORTED_FILTER_OPERATORS:
+ msg = f"Unsupported filter operator '__{operator}' for field '{field}'"
+ raise ValueError(msg)
+ return field, operator
+
+
+def normalize_filter_value(field: str, operator: str | None, expected: Any) -> Any:
+ """Normalize a filter value after its key has been validated."""
+
+ if operator != "in":
+ return expected
+ if isinstance(expected, str):
+ return expected
+ if isinstance(expected, Mapping):
+ msg = f"Filter '{field}__in' must be a string or an iterable of values"
+ raise ValueError(msg)
+ if not isinstance(expected, Iterable):
+ msg = f"Filter '{field}__in' must be a string or an iterable of values"
+ raise ValueError(msg)
+ return tuple(expected)
+
+
+def build_filter_key(field: str, operator: str | None) -> str:
+ return field if operator is None else f"{field}__{operator}"
+
+
+__all__ = [
+ "SUPPORTED_FILTER_OPERATORS",
+ "build_filter_key",
+ "normalize_filter_value",
+ "split_filter_key",
+]
diff --git a/src/memu/utils/retrieve.py b/src/memu/utils/retrieve.py
new file mode 100644
index 00000000..01126001
--- /dev/null
+++ b/src/memu/utils/retrieve.py
@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import Literal, cast
+
+RetrieveMethod = Literal["rag", "llm"]
+RetrieveRanking = Literal["similarity", "salience"]
+
+
+def normalize_retrieve_method(method: str | None, *, default: str) -> RetrieveMethod:
+ """Resolve and validate the retrieval method for a single request."""
+
+ raw_method = default if method is None else method
+ if not isinstance(raw_method, str) or not raw_method.strip():
+ msg = "retrieve method must be 'rag' or 'llm'"
+ raise ValueError(msg)
+ normalized = raw_method.strip().lower()
+ if normalized not in {"rag", "llm"}:
+ msg = "retrieve method must be 'rag' or 'llm'"
+ raise ValueError(msg)
+ return cast(RetrieveMethod, normalized)
+
+
+def normalize_retrieve_ranking(ranking: str | None, *, default: str) -> RetrieveRanking:
+ """Resolve and validate the item ranking strategy for a single retrieve request."""
+
+ raw_ranking = default if ranking is None else ranking
+ if not isinstance(raw_ranking, str) or not raw_ranking.strip():
+ msg = "retrieve ranking must be 'similarity' or 'salience'"
+ raise ValueError(msg)
+ normalized = raw_ranking.strip().lower()
+ if normalized not in {"similarity", "salience"}:
+ msg = "retrieve ranking must be 'similarity' or 'salience'"
+ raise ValueError(msg)
+ return cast(RetrieveRanking, normalized)
+
+
+__all__ = ["RetrieveMethod", "RetrieveRanking", "normalize_retrieve_method", "normalize_retrieve_ranking"]
diff --git a/src/memu/utils/serialization.py b/src/memu/utils/serialization.py
new file mode 100644
index 00000000..717ee16a
--- /dev/null
+++ b/src/memu/utils/serialization.py
@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+def model_dump_without_embeddings(obj: BaseModel) -> dict[str, Any]:
+ """Dump a Pydantic model into a JSON-safe public response shape."""
+
+ return obj.model_dump(mode="json", exclude={"embedding"})
+
+
+__all__ = ["model_dump_without_embeddings"]
diff --git a/src/memu/utils/tool.py b/src/memu/utils/tool.py
index aa1c0067..5167f90b 100644
--- a/src/memu/utils/tool.py
+++ b/src/memu/utils/tool.py
@@ -48,7 +48,7 @@ def add_tool_call(item: MemoryItem, tool_call: ToolCallResult) -> None:
raise ValueError(msg)
tool_call.ensure_hash()
tool_calls = get_tool_calls(item)
- tool_calls.append(tool_call.model_dump())
+ tool_calls.append(tool_call.model_dump(mode="json"))
set_tool_calls(item, tool_calls)
diff --git a/src/memu/workflow/pipeline.py b/src/memu/workflow/pipeline.py
index ddb5a5af..63a5d907 100644
--- a/src/memu/workflow/pipeline.py
+++ b/src/memu/workflow/pipeline.py
@@ -8,6 +8,8 @@
from memu.workflow.step import WorkflowStep
+LLM_PROFILE_CONFIG_KEYS = ("llm_profile", "chat_llm_profile", "embed_llm_profile")
+
@dataclass
class PipelineRevision:
@@ -144,11 +146,17 @@ def _validate_steps(self, steps: list[WorkflowStep], *, initial_state_keys: set[
msg = f"Step '{step.step_id}' requests unavailable capabilities: {', '.join(sorted(unknown_caps))}"
raise ValueError(msg)
- if getattr(step, "config", None):
- profile_name = step.config.get("llm_profile")
- if profile_name and profile_name not in self.llm_profiles:
+ for profile_key in LLM_PROFILE_CONFIG_KEYS:
+ profile_name = (getattr(step, "config", None) or {}).get(profile_key)
+ if profile_name is None:
+ continue
+ if not isinstance(profile_name, str) or not profile_name.strip():
+ msg = f"Step '{step.step_id}' references invalid {profile_key}; profile name must be non-empty"
+ raise ValueError(msg)
+ profile_name = profile_name.strip()
+ if profile_name not in self.llm_profiles:
msg = (
- f"Step '{step.step_id}' references unknown llm_profile '{profile_name}'. "
+ f"Step '{step.step_id}' references unknown {profile_key} '{profile_name}'. "
f"Available profiles: {', '.join(sorted(self.llm_profiles))}"
)
raise ValueError(msg)
diff --git a/tests/test_client_wrapper.py b/tests/test_client_wrapper.py
index 4ada1107..999be91f 100644
--- a/tests/test_client_wrapper.py
+++ b/tests/test_client_wrapper.py
@@ -4,9 +4,46 @@
from __future__ import annotations
+import asyncio
from unittest.mock import MagicMock
+class FakeMemoryService:
+ def __init__(self) -> None:
+ self.retrieve_calls = []
+
+ async def retrieve(self, queries, where=None, ranking=None):
+ self.retrieve_calls.append({"queries": queries, "where": where, "ranking": ranking})
+ return {
+ "items": [
+ {"summary": "one"},
+ {"summary": "two"},
+ {"summary": "three"},
+ ]
+ }
+
+
+class AsyncCreateCompletions:
+ def __init__(self) -> None:
+ self.kwargs = {}
+
+ async def create(self, **kwargs):
+ self.kwargs = kwargs
+ return {"ok": True, "method": "create"}
+
+
+class AsyncAcreateCompletions:
+ def __init__(self) -> None:
+ self.kwargs = {}
+
+ async def acreate(self, **kwargs):
+ self.kwargs = kwargs
+ return {"ok": True, "method": "acreate"}
+
+ def create(self, **kwargs):
+ raise AssertionError("acreate should be preferred when present")
+
+
class TestMemuOpenAIWrapper:
"""Tests for OpenAI client wrapper."""
@@ -39,6 +76,27 @@ def test_extract_user_query_multiple_turns(self):
query = completions._extract_user_query(messages)
assert query == "What's my name?"
+ def test_extract_user_query_from_multiple_text_parts(self):
+ """Should concatenate text parts from multimodal user content."""
+ from memu.client.openai_wrapper import MemuChatCompletions
+
+ completions = MemuChatCompletions(MagicMock(), MagicMock(), {}, "salience", 5)
+
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Remember that I like coffee."},
+ {"type": "image_url", "image_url": {"url": "https://example.test/photo.png"}},
+ {"type": "text", "text": "What should I order today?"},
+ {"type": "text", "text": 123},
+ ],
+ },
+ ]
+
+ query = completions._extract_user_query(messages)
+ assert query == "Remember that I like coffee.\nWhat should I order today?"
+
def test_inject_memories_into_existing_system(self):
"""Should append memories to existing system message."""
from memu.client.openai_wrapper import MemuChatCompletions
@@ -63,6 +121,46 @@ def test_inject_memories_into_existing_system(self):
assert "User is named Alex" in result[0]["content"]
assert result[0]["content"].startswith("You are helpful.")
+ def test_inject_memories_does_not_mutate_original_messages(self):
+ """Should leave caller-owned message dictionaries untouched."""
+ from memu.client.openai_wrapper import MemuChatCompletions
+
+ completions = MemuChatCompletions(MagicMock(), MagicMock(), {}, "salience", 5)
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "Hi"},
+ ]
+
+ result = completions._inject_memories(messages, [{"summary": "User loves coffee"}])
+
+ assert messages == [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "Hi"},
+ ]
+ assert result is not messages
+ assert result[0] is not messages[0]
+
+ def test_inject_memories_appends_to_system_content_parts(self):
+ """Should support system messages whose content is a list of text parts."""
+ from memu.client.openai_wrapper import MemuChatCompletions
+
+ completions = MemuChatCompletions(MagicMock(), MagicMock(), {}, "salience", 5)
+ original_part = {"type": "text", "text": "You are helpful."}
+ messages = [
+ {"role": "system", "content": [original_part]},
+ {"role": "user", "content": "Hi"},
+ ]
+
+ result = completions._inject_memories(messages, [{"summary": "User loves coffee"}])
+
+ assert messages[0]["content"] == [original_part]
+ assert result[0]["content"] is not messages[0]["content"]
+ assert result[0]["content"][0] == original_part
+ assert result[0]["content"][0] is not original_part
+ assert result[0]["content"][1]["type"] == "text"
+ assert "" in result[0]["content"][1]["text"]
+ assert "User loves coffee" in result[0]["content"][1]["text"]
+
def test_inject_memories_creates_system_message(self):
"""Should create system message if none exists."""
from memu.client.openai_wrapper import MemuChatCompletions
@@ -114,6 +212,109 @@ def test_wrap_openai_convenience_function(self):
assert wrapped._ranking == "salience"
assert wrapped._top_k == 3
+ def test_wrap_openai_does_not_mutate_user_data(self):
+ """Should not mutate caller-owned user_data dictionaries."""
+ from memu.client import wrap_openai
+
+ mock_client = MagicMock()
+ mock_client.chat.completions = MagicMock()
+ user_data = {"user_id": "original", "team_id": "team1"}
+
+ wrapped = wrap_openai(
+ mock_client,
+ MagicMock(),
+ user_data=user_data,
+ user_id="override",
+ agent_id="agent1",
+ )
+
+ assert user_data == {"user_id": "original", "team_id": "team1"}
+ assert wrapped._user_data == {
+ "user_id": "override",
+ "team_id": "team1",
+ "agent_id": "agent1",
+ }
+
+ def test_wrapper_scope_is_stable_after_external_user_data_mutation(self):
+ """Should keep the wrapper's retrieve scope stable after construction."""
+ from memu.client import wrap_openai
+
+ mock_client = MagicMock()
+ mock_client.chat.completions = MagicMock()
+ service = FakeMemoryService()
+ user_data = {"user_id": "u1"}
+ wrapped = wrap_openai(mock_client, service, user_data=user_data)
+ user_data["user_id"] = "u2"
+
+ asyncio.run(wrapped.chat.completions._retrieve_memories("what do I like?"))
+
+ assert service.retrieve_calls[0]["where"] == {"user_id": "u1"}
+
+ def test_retrieve_memories_respects_top_k_limit(self):
+ """Should inject at most top_k retrieved memories."""
+ from memu.client.openai_wrapper import MemuChatCompletions
+
+ service = FakeMemoryService()
+ completions = MemuChatCompletions(MagicMock(), service, {}, "salience", 2)
+
+ memories = asyncio.run(completions._retrieve_memories("what do I like?"))
+
+ assert [memory["summary"] for memory in memories] == ["one", "two"]
+ assert service.retrieve_calls[0]["ranking"] == "salience"
+
+ def test_acreate_awaits_async_create_result(self):
+ """Should await AsyncOpenAI-style create() coroutine results."""
+ from memu.client.openai_wrapper import MemuChatCompletions
+
+ original = AsyncCreateCompletions()
+ completions = MemuChatCompletions(original, FakeMemoryService(), {}, "salience", 1)
+
+ result = asyncio.run(completions.acreate(messages=[{"role": "user", "content": "What do I like?"}]))
+
+ assert result == {"ok": True, "method": "create"}
+ assert "" in original.kwargs["messages"][0]["content"]
+ assert "one" in original.kwargs["messages"][0]["content"]
+
+ def test_acreate_prefers_legacy_acreate_when_present(self):
+ """Should keep compatibility with clients exposing acreate()."""
+ from memu.client.openai_wrapper import MemuChatCompletions
+
+ original = AsyncAcreateCompletions()
+ completions = MemuChatCompletions(original, FakeMemoryService(), {}, "salience", 1)
+
+ result = asyncio.run(completions.acreate(messages=[{"role": "user", "content": "What do I like?"}]))
+
+ assert result == {"ok": True, "method": "acreate"}
+ assert "" in original.kwargs["messages"][0]["content"]
+
+ def test_wrapper_rejects_unknown_ranking(self):
+ """Should reject invalid ranking at the integration boundary."""
+ from memu.client import wrap_openai
+
+ mock_client = MagicMock()
+ mock_client.chat.completions = MagicMock()
+
+ try:
+ wrap_openai(mock_client, MagicMock(), ranking="random")
+ except ValueError as exc:
+ assert "retrieve ranking must be 'similarity' or 'salience'" in str(exc)
+ else:
+ raise AssertionError("wrap_openai should reject unknown ranking")
+
+ def test_wrapper_rejects_non_positive_top_k(self):
+ """Should reject invalid top_k at the integration boundary."""
+ from memu.client import wrap_openai
+
+ mock_client = MagicMock()
+ mock_client.chat.completions = MagicMock()
+
+ try:
+ wrap_openai(mock_client, MagicMock(), top_k=0)
+ except ValueError as exc:
+ assert "top_k must be a positive integer" in str(exc)
+ else:
+ raise AssertionError("wrap_openai should reject non-positive top_k")
+
def test_wrapper_proxies_other_attributes(self):
"""Should proxy non-chat attributes to original client."""
from memu.client import MemuOpenAIWrapper
diff --git a/tests/test_crud_contracts.py b/tests/test_crud_contracts.py
new file mode 100644
index 00000000..fe00411b
--- /dev/null
+++ b/tests/test_crud_contracts.py
@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_manual_create_memory_item_passes_source_less_resource_id() -> None:
+ for relative_path in ["src/memu/app/crud.py", "src/memu/app/patch.py"]:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ function_source = _async_function_source(source, "_patch_create_memory_item")
+ create_call = _first_method_call(function_source, "create_item")
+ keyword_values = {keyword.arg: keyword.value for keyword in create_call.keywords}
+
+ assert "resource_id" in keyword_values, f"{relative_path} must pass resource_id explicitly"
+ assert isinstance(keyword_values["resource_id"], ast.Constant)
+ assert keyword_values["resource_id"].value is None
+
+
+def test_memory_item_repo_create_item_accepts_source_less_records() -> None:
+ checked_paths = [
+ "src/memu/database/repositories/memory_item.py",
+ "src/memu/database/inmemory/repositories/memory_item_repo.py",
+ "src/memu/database/sqlite/repositories/memory_item_repo.py",
+ "src/memu/database/postgres/repositories/memory_item_repo.py",
+ ]
+
+ for relative_path in checked_paths:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ create_fn = _function_node(source, "create_item")
+ resource_arg = next(arg for arg in create_fn.args.kwonlyargs if arg.arg == "resource_id")
+ resource_default = _kwonly_default(create_fn, "resource_id")
+
+ assert ast.unparse(resource_arg.annotation) == "str | None"
+ assert isinstance(resource_default, ast.Constant)
+ assert resource_default.value is None
+
+
+def test_update_memory_item_preserves_categories_when_categories_are_omitted() -> None:
+ for relative_path in ["src/memu/app/crud.py", "src/memu/app/patch.py"]:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ function_source = _async_function_source(source, "_patch_update_memory_item")
+
+ assert 'new_cat_names = memory_payload["categories"]' in function_source
+ assert "if new_cat_names is None:" in function_source
+ assert "mapped_new_cat_ids = mapped_old_cat_ids" in function_source
+ assert "else:" in function_source
+ assert "mapped_new_cat_ids = self._map_category_names_to_ids(new_cat_names, ctx)" in function_source
+
+
+def test_manual_memory_item_inputs_are_normalized_before_workflow() -> None:
+ for relative_path in ["src/memu/app/crud.py", "src/memu/app/patch.py"]:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ create_source = _async_function_source(source, "create_memory_item")
+ update_source = _async_function_source(source, "update_memory_item")
+ delete_source = _async_function_source(source, "delete_memory_item")
+ memory_type_helper = _function_source(source, "_normalize_memory_type")
+ categories_helper = _function_source(source, "_normalize_memory_categories")
+ string_helper = _function_source(source, "_normalize_non_empty_string")
+
+ assert "memory_type = _normalize_memory_type(memory_type)" in create_source
+ assert 'memory_content = _normalize_memory_content(memory_content, field_name="memory_content")' in create_source
+ assert (
+ 'memory_categories = _normalize_memory_categories(memory_categories, field_name="memory_categories")'
+ in create_source
+ )
+ assert "memory_id = _normalize_memory_id(memory_id)" in update_source
+ assert "memory_type = _normalize_memory_type(memory_type)" in update_source
+ assert "memory_id = _normalize_memory_id(memory_id)" in delete_source
+ assert "if memory_content is not None:" in update_source
+ assert 'memory_content = _normalize_memory_content(memory_content, field_name="memory_content")' in update_source
+ assert "if memory_categories is not None:" in update_source
+ assert (
+ 'memory_categories = _normalize_memory_categories(memory_categories, field_name="memory_categories")'
+ in update_source
+ )
+ assert '_normalize_non_empty_string(value, field_name="memory_type")' in memory_type_helper
+ assert "memory_type not in get_args(MemoryType)" in memory_type_helper
+ assert "cast(MemoryType, memory_type)" in memory_type_helper
+ assert "not isinstance(value, list)" in categories_helper
+ assert "not isinstance(value, str) or not value.strip()" in string_helper
+ assert "return value.strip()" in string_helper
+
+
+def test_clear_memory_clears_category_item_relations_before_records() -> None:
+ source = (ROOT / "src/memu/app/crud.py").read_text(encoding="utf-8")
+ workflow_fn = _function_node(source, "_build_clear_memory_workflow")
+ step_ids = _workflow_step_ids(workflow_fn)
+
+ assert "clear_category_item_relations" in step_ids
+ assert step_ids.index("clear_category_item_relations") < step_ids.index("clear_memory_categories")
+ assert step_ids.index("clear_category_item_relations") < step_ids.index("clear_memory_items")
+ assert step_ids.index("clear_category_item_relations") < step_ids.index("clear_memory_resources")
+
+ clear_fn_source = _function_source(source, "_crud_clear_category_item_relations")
+ response_fn_source = _function_source(source, "_crud_build_clear_memory_response")
+
+ assert "store.category_item_repo.clear_relations(where_filters)" in clear_fn_source
+ assert '"deleted_relations"' in response_fn_source
+
+
+def test_category_item_repo_contract_exposes_scoped_clear_relations() -> None:
+ checked_paths = [
+ "src/memu/database/repositories/category_item.py",
+ "src/memu/database/inmemory/repositories/category_item_repo.py",
+ "src/memu/database/sqlite/repositories/category_item_repo.py",
+ "src/memu/database/postgres/repositories/category_item_repo.py",
+ ]
+
+ for relative_path in checked_paths:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ clear_fn = _function_node(source, "clear_relations")
+ arg_names = [arg.arg for arg in clear_fn.args.args]
+
+ assert "where" in arg_names
+ assert "list[CategoryItem]" in ast.unparse(clear_fn.returns)
+
+
+def test_delete_memory_item_clears_relations_before_item_delete() -> None:
+ for relative_path in ["src/memu/app/crud.py", "src/memu/app/patch.py"]:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ delete_fn_source = _async_function_source(source, "_patch_delete_memory_item")
+
+ clear_idx = delete_fn_source.index('store.category_item_repo.clear_relations({"item_id": memory_id})')
+ delete_idx = delete_fn_source.index("store.memory_item_repo.delete_item(memory_id)")
+
+ assert clear_idx < delete_idx
+ assert "get_item_categories(memory_id)" not in delete_fn_source
+
+
+def test_postgres_delete_item_removes_cache_entry() -> None:
+ source = (ROOT / "src/memu/database/postgres/repositories/memory_item_repo.py").read_text(encoding="utf-8")
+ delete_fn_source = _function_source(source, "delete_item")
+
+ assert "self.items.pop(item_id, None)" in delete_fn_source
+
+
+def test_inmemory_scoped_clear_preserves_shared_state_dicts() -> None:
+ checked = [
+ ("src/memu/database/inmemory/repositories/memory_item_repo.py", "clear_items", "items"),
+ ("src/memu/database/inmemory/repositories/resource_repo.py", "clear_resources", "resources"),
+ ("src/memu/database/inmemory/repositories/memory_category_repo.py", "clear_categories", "categories"),
+ ]
+
+ for relative_path, function_name, attribute_name in checked:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ function = _function_node(source, function_name)
+ function_source = _function_source(source, function_name)
+
+ assert not _assigns_to_self_attribute(function, attribute_name)
+ assert f"self.{attribute_name}.pop(" in function_source
+
+
+def _async_function_source(source: str, name: str) -> str:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.AsyncFunctionDef) and node.name == name:
+ segment = ast.get_source_segment(source, node)
+ assert segment is not None
+ return segment
+ raise AssertionError(f"async function {name!r} not found")
+
+
+def _function_source(source: str, name: str) -> str:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ segment = ast.get_source_segment(source, node)
+ assert segment is not None
+ return segment
+ raise AssertionError(f"function {name!r} not found")
+
+
+def _function_node(source: str, name: str) -> ast.FunctionDef:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ return node
+ raise AssertionError(f"function {name!r} not found")
+
+
+def _first_method_call(source: str, method_name: str) -> ast.Call:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == method_name:
+ return node
+ raise AssertionError(f"method call {method_name!r} not found")
+
+
+def _kwonly_default(function: ast.FunctionDef, arg_name: str) -> ast.expr | None:
+ for arg, default in zip(function.args.kwonlyargs, function.args.kw_defaults, strict=True):
+ if arg.arg == arg_name:
+ return default
+ raise AssertionError(f"keyword-only argument {arg_name!r} not found")
+
+
+def _workflow_step_ids(function: ast.FunctionDef) -> list[str]:
+ step_ids: list[str] = []
+
+ class Visitor(ast.NodeVisitor):
+ def visit_Call(self, node: ast.Call) -> None:
+ if isinstance(node.func, ast.Name) and node.func.id == "WorkflowStep":
+ for keyword in node.keywords:
+ if keyword.arg == "step_id":
+ assert isinstance(keyword.value, ast.Constant)
+ assert isinstance(keyword.value.value, str)
+ step_ids.append(keyword.value.value)
+ self.generic_visit(node)
+
+ Visitor().visit(function)
+ return step_ids
+
+
+def _assigns_to_self_attribute(function: ast.FunctionDef, attribute_name: str) -> bool:
+ for node in ast.walk(function):
+ targets: list[ast.expr] = []
+ if isinstance(node, ast.Assign):
+ targets.extend(node.targets)
+ elif isinstance(node, ast.AnnAssign):
+ targets.append(node.target)
+ elif isinstance(node, ast.AugAssign):
+ targets.append(node.target)
+
+ for target in targets:
+ if not isinstance(target, ast.Attribute) or target.attr != attribute_name:
+ continue
+ if isinstance(target.value, ast.Name) and target.value.id == "self":
+ return True
+ return False
diff --git a/tests/test_folder_compiler.py b/tests/test_folder_compiler.py
new file mode 100644
index 00000000..a940c943
--- /dev/null
+++ b/tests/test_folder_compiler.py
@@ -0,0 +1,1840 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from memu.app import (
+ ContextHarness,
+ EvolutionReviewConfig,
+ FolderMemoryCompiler,
+ FolderMemoryCompilerConfig,
+ compile_folder_to_markdown_sync,
+ watch_folder_to_markdown,
+)
+from memu.app.context_cli import build_parser as build_context_cli_parser
+from memu.app.context_cli import main as context_cli_main
+from memu.app.context_harness_cli import build_parser as build_harness_cli_parser
+from memu.app.context_harness_cli import main as harness_cli_main
+from memu.app.folder_cli import _llm_profile_from_args
+from memu.app.folder_cli import build_parser as build_folder_cli_parser
+from memu.app.folder_cli import main as folder_cli_main
+from memu.app.markdown_context import MarkdownMemoryRepository, inject_context_messages
+from memu.app.skill_trace import SkillToolTrace, record_skill_trace
+from memu.app.skill_trace_cli import main as skill_trace_cli_main
+
+
+class FakeMemoryService:
+ def __init__(self) -> None:
+ self.calls: list[dict[str, Any]] = []
+
+ async def memorize(self, *, resource_url: str, modality: str, user: dict[str, Any] | None = None) -> dict[str, Any]:
+ self.calls.append({"resource_url": resource_url, "modality": modality, "user": user})
+ return {
+ "resource": {
+ "url": resource_url,
+ "modality": modality,
+ "caption": "A screenshot of a product workflow board.",
+ },
+ "items": [
+ {
+ "memory_type": "profile",
+ "summary": "The user's tone preference is warm and concise.",
+ },
+ {
+ "memory_type": "skill",
+ "summary": "Use screenshots to infer workflow and tool-use patterns.",
+ },
+ ],
+ "categories": [
+ {
+ "name": "workflow",
+ "summary": "Signals about workflows and tool usage.",
+ }
+ ],
+ }
+
+
+@pytest.mark.asyncio
+async def test_compile_creates_markdown_memory_repo_for_multimodal_folder(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text(
+ "The user's tone preference is direct and warm. They like concise answers.",
+ encoding="utf-8",
+ )
+ (source / "workflow.md").write_text(
+ "Skill: use pytest to verify code changes. Workflow: inspect, patch, test.",
+ encoding="utf-8",
+ )
+ image_dir = source / "images"
+ image_dir.mkdir()
+ (image_dir / "screenshot.png").write_bytes(b"\x89PNG\r\n\x1a\n")
+
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ result = await compiler.compile(source, output)
+
+ assert sorted(result.processed) == ["images/screenshot.png", "notes.txt", "workflow.md"]
+ assert (output / "raw_data" / "notes.txt").exists()
+ assert (output / "raw_data" / "workflow.md").exists()
+ assert (output / "raw_data" / "images" / "screenshot.png").exists()
+ assert (output / ".memu" / "derived" / "images" / "screenshot.png.evidence.md").exists()
+ assert (output / "memory.md").exists()
+ assert (output / "soul.md").exists()
+ assert (output / "skill.md").exists()
+ assert (output / "AGENTS.md").exists()
+ assert (output / "memory").is_dir()
+ assert (output / "soul").is_dir()
+ assert (output / "skill").is_dir()
+
+ memory_md = (output / "memory.md").read_text(encoding="utf-8")
+ soul_md = (output / "soul.md").read_text(encoding="utf-8")
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+ agents_md = (output / "AGENTS.md").read_text(encoding="utf-8")
+
+ assert "" in memory_md
+ assert "raw_data/notes.txt" in memory_md
+ assert "raw_data/images/screenshot.png" in memory_md
+ assert "soul_" in soul_md
+ assert "skill_" in skill_md
+ assert "memu-harness context . --query" in agents_md
+
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ assert sorted(manifest["sources"]) == ["images/screenshot.png", "notes.txt", "workflow.md"]
+ assert manifest["sources"]["images/screenshot.png"]["modality"] == "image"
+
+
+@pytest.mark.asyncio
+async def test_compile_routes_entries_through_evolution_review_gate(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "creator_feedback.md").write_text(
+ "Creator feedback: the assistant should keep answers warm and concise.",
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ result = await compiler.compile(source, output)
+
+ assert result.entries
+ assert result.evolution_instructions
+ assert result.patch_proposals
+ assert result.review_decisions
+ assert all(review.status == "approved" for review in result.review_decisions)
+
+ instruction = result.evolution_instructions[0]
+ assert instruction.target in {"memory", "soul", "skill"}
+ assert instruction.operation == "add"
+ assert instruction.reason
+ assert instruction.evidence.source == "raw_data/creator_feedback.md"
+ assert instruction.evidence.source_kind == "creator_feedback"
+
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ source_record = manifest["sources"]["creator_feedback.md"]
+ assert source_record["evolution"]["instructions"][0]["id"] == instruction.id
+ assert source_record["evolution"]["patch_proposals"][0]["instruction_id"] == instruction.id
+ assert source_record["evolution"]["review_decisions"][0]["status"] == "approved"
+
+ evolution_dir = output / ".memu" / "evolution"
+ assert (evolution_dir / "instructions.jsonl").exists()
+ assert (evolution_dir / "patch_proposals.jsonl").exists()
+ assert (evolution_dir / "review_decisions.jsonl").exists()
+
+
+@pytest.mark.asyncio
+async def test_compile_requires_review_before_applying_evolution_patch(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "agent_log.md").write_text(
+ "Agent log: Skill: run focused tests after changing compiler behavior.",
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(
+ config=FolderMemoryCompilerConfig(
+ use_memory_service=False,
+ evolution_review=EvolutionReviewConfig(auto_approve=False),
+ )
+ )
+
+ result = await compiler.compile(source, output)
+
+ assert result.evolution_instructions
+ assert result.patch_proposals
+ assert result.review_decisions
+ assert all(review.status == "needs_review" for review in result.review_decisions)
+ assert result.entries == []
+ assert "run focused tests" not in (output / "skill.md").read_text(encoding="utf-8")
+
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ source_record = manifest["sources"]["agent_log.md"]
+ assert source_record["entries"] == []
+ assert source_record["evolution"]["review_decisions"][0]["status"] == "needs_review"
+
+ review_result = compiler.review_evolution(output, reviewer="creator", reason="Approved after review.")
+
+ assert review_result.reviewed
+ assert review_result.applied_proposal_ids
+ assert any(entry.bucket == "skill" for entry in review_result.entries)
+ assert "run focused tests" in (output / "skill.md").read_text(encoding="utf-8")
+
+ reviewed_manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ reviewed_source = reviewed_manifest["sources"]["agent_log.md"]
+ assert reviewed_source["entries"]
+ assert reviewed_source["evolution"]["review_decisions"][-1]["status"] == "approved"
+
+
+@pytest.mark.asyncio
+async def test_compile_uses_multimodal_sidecar_evidence(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ image = source / "workflow.png"
+ image.write_bytes(b"\x89PNG\r\n\x1a\n")
+ (source / "workflow.caption.md").write_text(
+ "Skill: inspect screenshots to understand a product workflow. "
+ "Tone preference: explain findings calmly.",
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ result = await compiler.compile(source, output)
+
+ image_entries = [entry for entry in result.entries if entry.source == "raw_data/workflow.png"]
+ evidence = (output / ".memu" / "derived" / "workflow.png.evidence.md").read_text(encoding="utf-8")
+ soul_md = (output / "soul.md").read_text(encoding="utf-8")
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+
+ assert {entry.bucket for entry in image_entries} == {"memory", "soul", "skill"}
+ assert "## Sidecar Evidence" in evidence
+ assert "workflow.caption.md" in evidence
+ assert "inspect screenshots" in skill_md
+ assert "explain findings calmly" in soul_md
+
+
+@pytest.mark.asyncio
+async def test_compile_uses_structured_json_sidecar_evidence(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ image = source / "workflow.png"
+ image.write_bytes(b"\x89PNG\r\n\x1a\n")
+ (source / "workflow.metadata.json").write_text(
+ json.dumps(
+ {
+ "caption": "Workflow screenshot with acceptance criteria.",
+ "lesson": "Skill: compare screenshots against acceptance criteria.",
+ "tone": "Tone preference: explain visual findings calmly.",
+ }
+ ),
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ result = await compiler.compile(source, output)
+
+ evidence = (output / ".memu" / "derived" / "workflow.png.evidence.md").read_text(encoding="utf-8")
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+ soul_md = (output / "soul.md").read_text(encoding="utf-8")
+
+ assert result.processed == ["workflow.png"]
+ assert sorted(manifest["sources"]) == ["workflow.png"]
+ assert manifest["sources"]["workflow.png"]["sidecars"] == ["workflow.metadata.json"]
+ assert "Structured JSON sidecar" in evidence
+ assert "compare screenshots against acceptance criteria" in skill_md
+ assert "explain visual findings calmly" in soul_md
+ assert (output / "raw_data" / "workflow.metadata.json").exists()
+
+
+@pytest.mark.asyncio
+async def test_compile_uses_document_sidecar_even_when_file_is_utf8_decodable(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ report = source / "report.pdf"
+ report.write_text("%PDF-1.7\nASCII-only fake PDF body", encoding="utf-8")
+ (source / "report.summary.md").write_text(
+ "Skill: summarize PDF evidence before updating context. "
+ "Tone preference: explain document findings calmly.",
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ result = await compiler.compile(source, output)
+
+ report_entries = [entry for entry in result.entries if entry.source == "raw_data/report.pdf"]
+ evidence = (output / ".memu" / "derived" / "report.pdf.evidence.md").read_text(encoding="utf-8")
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+ soul_md = (output / "soul.md").read_text(encoding="utf-8")
+
+ assert {entry.bucket for entry in report_entries} == {"memory", "soul", "skill"}
+ assert "## Multimodal Evidence" in evidence
+ assert "## Text Evidence" not in evidence
+ assert "report.summary.md" in evidence
+ assert manifest["sources"]["report.pdf"]["sidecars"] == ["report.summary.md"]
+ assert "summarize PDF evidence" in skill_md
+ assert "explain document findings calmly" in soul_md
+
+
+@pytest.mark.asyncio
+async def test_compile_reextracts_media_when_sidecar_changes(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ image = source / "workflow.png"
+ image.write_bytes(b"\x89PNG\r\n\x1a\n")
+ sidecar = source / "workflow.caption.md"
+ sidecar.write_text("Skill: inspect screenshots before changing UI workflows.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ first = await compiler.compile(source, output)
+ first_manifest = json.loads(first.manifest_path.read_text(encoding="utf-8"))
+ first_hash = first_manifest["sources"]["workflow.png"]["sha256"]
+
+ sidecar.write_text("Skill: compare screenshots against acceptance criteria.", encoding="utf-8")
+ second = await compiler.compile(source, output)
+ second_manifest = json.loads(second.manifest_path.read_text(encoding="utf-8"))
+ second_hash = second_manifest["sources"]["workflow.png"]["sha256"]
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+
+ assert first.processed == ["workflow.png"]
+ assert sorted(first_manifest["sources"]) == ["workflow.png"]
+ assert first_manifest["sources"]["workflow.png"]["sidecars"] == ["workflow.caption.md"]
+ assert second.processed == ["workflow.png"]
+ assert second_hash != first_hash
+ assert "compare screenshots" in skill_md
+ assert "changing UI workflows" not in skill_md
+ assert (output / "raw_data" / "workflow.caption.md").exists()
+
+
+@pytest.mark.asyncio
+async def test_compile_excludes_configured_source_patterns(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: keep meaningful source files.", encoding="utf-8")
+ (source / "debug.tmp").write_text("Temporary debug output.", encoding="utf-8")
+ cache_dir = source / "node_modules" / "pkg"
+ cache_dir.mkdir(parents=True)
+ (cache_dir / "cache.txt").write_text("Dependency cache.", encoding="utf-8")
+ image = source / "workflow.png"
+ image.write_bytes(b"\x89PNG\r\n\x1a\n")
+ (source / "workflow.metadata.json").write_text(
+ '{"lesson":"Skill: excluded sidecar should not be used."}',
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(
+ config=FolderMemoryCompilerConfig(
+ use_memory_service=False,
+ exclude_patterns=("node_modules/**", "*.tmp", "*.metadata.json"),
+ )
+ )
+
+ result = await compiler.compile(source, output)
+
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+ assert sorted(result.processed) == ["notes.txt", "workflow.png"]
+ assert sorted(manifest["sources"]) == ["notes.txt", "workflow.png"]
+ assert manifest["sources"]["workflow.png"]["sidecars"] == []
+ assert not (output / "raw_data" / "debug.tmp").exists()
+ assert not (output / "raw_data" / "node_modules").exists()
+ assert not (output / "raw_data" / "workflow.metadata.json").exists()
+ assert "excluded sidecar" not in skill_md
+
+
+@pytest.mark.asyncio
+async def test_compile_uses_source_memuignore_patterns(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / ".memuignore").write_text("node_modules/**\n*.tmp\n", encoding="utf-8")
+ (source / "notes.txt").write_text("Skill: keep meaningful source files.", encoding="utf-8")
+ (source / "debug.tmp").write_text("Temporary debug output.", encoding="utf-8")
+ cache_dir = source / "node_modules" / "pkg"
+ cache_dir.mkdir(parents=True)
+ (cache_dir / "cache.txt").write_text("Dependency cache.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ result = await compiler.compile(source, output)
+
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ assert result.processed == ["notes.txt"]
+ assert sorted(manifest["sources"]) == ["notes.txt"]
+ assert not (output / "raw_data" / ".memuignore").exists()
+ assert not (output / "raw_data" / "debug.tmp").exists()
+ assert not (output / "raw_data" / "node_modules").exists()
+
+
+@pytest.mark.asyncio
+async def test_compile_reextracts_changed_files_and_preserves_manual_markdown(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ profile = source / "profile.txt"
+ profile.write_text("The user prefers concise answers.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ first = await compiler.compile(source, output)
+ first_hash = json.loads(first.manifest_path.read_text(encoding="utf-8"))["sources"]["profile.txt"]["sha256"]
+ memory_path = output / "memory.md"
+ memory_path.write_text(
+ memory_path.read_text(encoding="utf-8") + "\nManual note kept by the user.\n",
+ encoding="utf-8",
+ )
+
+ profile.write_text("The user prefers detailed answers.", encoding="utf-8")
+ second = await compiler.compile(source, output)
+
+ second_hash = json.loads(second.manifest_path.read_text(encoding="utf-8"))["sources"]["profile.txt"]["sha256"]
+ memory_md = memory_path.read_text(encoding="utf-8")
+
+ assert second.processed == ["profile.txt"]
+ assert second_hash != first_hash
+ assert "detailed answers" in memory_md
+ assert "concise answers" not in memory_md
+ assert "Manual note kept by the user." in memory_md
+
+
+@pytest.mark.asyncio
+async def test_compile_removes_deleted_source_memory_and_raw_copy(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ obsolete = source / "obsolete.txt"
+ obsolete.write_text("A temporary memory that should disappear.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ await compiler.compile(source, output)
+ evidence_path = output / ".memu" / "derived" / "obsolete.txt.evidence.md"
+ assert evidence_path.exists()
+ obsolete.unlink()
+ result = await compiler.compile(source, output)
+
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ memory_md = (output / "memory.md").read_text(encoding="utf-8")
+
+ assert result.removed == ["obsolete.txt"]
+ assert manifest["sources"] == {}
+ assert not (output / "raw_data" / "obsolete.txt").exists()
+ assert not evidence_path.exists()
+ assert "raw_data/obsolete.txt" not in memory_md
+
+
+@pytest.mark.asyncio
+async def test_compile_preserves_manual_bucket_detail_files(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ note = source / "notes.txt"
+ note.write_text("Skill: preserve manual skill cards.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ await compiler.compile(source, output)
+
+ manual_card = output / "skill" / "manual-card.md"
+ manual_card.write_text("# Manual Skill Card\n\nKeep this hand-written skill.", encoding="utf-8")
+ generated_card = output / "skill" / "notes.txt.md"
+ generated_card.write_text(
+ generated_card.read_text(encoding="utf-8") + "\nManual note on stale generated card.\n",
+ encoding="utf-8",
+ )
+ note.unlink()
+
+ await compiler.compile(source, output)
+
+ assert manual_card.exists()
+ assert "Keep this hand-written skill" in manual_card.read_text(encoding="utf-8")
+ assert generated_card.exists()
+ stale_text = generated_card.read_text(encoding="utf-8")
+ assert "Manual note on stale generated card" in stale_text
+ assert "Skill: preserve manual skill cards" not in stale_text
+
+
+@pytest.mark.asyncio
+async def test_compile_same_source_and_output_uses_repo_raw_data(tmp_path: Path) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text("Skill: refresh a repository from raw_data.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ compiler.scaffold(output, source_folder=upload)
+ (output / "root-note.md").write_text("This repo note should not become raw data.", encoding="utf-8")
+ result = await compiler.compile(output, output)
+ status = compiler.status(output, output)
+ fingerprint = compiler.source_fingerprint(output, output)
+
+ memory_md = (output / "memory.md").read_text(encoding="utf-8")
+ assert result.processed == ["notes.txt"]
+ assert status.source_dir == (output / "raw_data").resolve()
+ assert fingerprint[0][0] == "notes.txt"
+ assert (output / "raw_data" / "notes.txt").exists()
+ assert (output / "root-note.md").exists()
+ assert "root-note.md" not in memory_md
+ assert "refresh a repository from raw_data" in memory_md
+
+
+@pytest.mark.asyncio
+async def test_compile_excludes_output_repo_inside_source_folder(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: keep output repo out of raw data.", encoding="utf-8")
+ output = source / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ first = await compiler.compile(source, output)
+ second = await compiler.compile(source, output)
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+
+ assert first.processed == ["notes.txt"]
+ assert second.skipped == ["notes.txt"]
+ assert sorted(manifest["sources"]) == ["notes.txt"]
+ assert not (output / "raw_data" / "memory_repo").exists()
+ assert "memory_repo/" not in (output / "memory.md").read_text(encoding="utf-8")
+
+
+def test_scaffold_excludes_output_repo_inside_source_folder(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Raw source note.", encoding="utf-8")
+ output = source / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ result = compiler.scaffold(output, source_folder=source)
+
+ assert result.copied == ["notes.txt"]
+ assert (output / "raw_data" / "notes.txt").exists()
+ assert not (output / "raw_data" / "memory_repo").exists()
+
+
+@pytest.mark.asyncio
+async def test_compile_uses_memory_service_and_writes_service_evidence(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ image = source / "workflow.png"
+ image.write_bytes(b"\x89PNG\r\n\x1a\n")
+ output = tmp_path / "memory_repo"
+ service = FakeMemoryService()
+ compiler = FolderMemoryCompiler(memory_service=service)
+
+ result = await compiler.compile(source, output, user={"user_id": "u1"})
+
+ assert service.calls == [
+ {
+ "resource_url": str(image.resolve()),
+ "modality": "image",
+ "user": {"user_id": "u1"},
+ }
+ ]
+ assert [entry.bucket for entry in result.entries] == ["soul", "skill"]
+ assert all("llm-extracted" in entry.tags for entry in result.entries)
+
+ evidence = (output / ".memu" / "derived" / "workflow.png.evidence.md").read_text(encoding="utf-8")
+ assert "## MemoryService Extraction" in evidence
+ assert "A screenshot of a product workflow board." in evidence
+ assert "Use screenshots to infer workflow and tool-use patterns." in evidence
+
+ soul_md = (output / "soul.md").read_text(encoding="utf-8")
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+ memory_md = (output / "memory.md").read_text(encoding="utf-8")
+ assert "warm and concise" in soul_md
+ assert "workflow and tool-use patterns" in skill_md
+ assert "workflow.png" not in memory_md
+
+
+def test_folder_cli_compiles_to_json_summary(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: inspect files, patch code, and run tests.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ exit_code = folder_cli_main([str(source), str(output), "--json", "--user", "user_id=u1"])
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert summary["processed"] == ["notes.txt"]
+ assert summary["entry_count"] >= 1
+ assert summary["entries_by_bucket"]["skill"] == 1
+ assert (output / "raw_data" / "notes.txt").exists()
+ assert (output / "skill.md").exists()
+
+
+def test_folder_cli_grok_profile_defaults_to_xai_key_env() -> None:
+ parser = build_folder_cli_parser()
+ args = parser.parse_args(["source", "output", "--use-memory-service", "--provider", "grok"])
+
+ profile = _llm_profile_from_args(args)
+
+ assert profile["provider"] == "grok"
+ assert profile["api_key"] == "XAI_API_KEY"
+
+
+def test_folder_cli_api_key_env_overrides_provider_default(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setenv("CUSTOM_KEY_ENV", "resolved-key")
+ parser = build_folder_cli_parser()
+ args = parser.parse_args(
+ [
+ "source",
+ "output",
+ "--use-memory-service",
+ "--provider",
+ "grok",
+ "--api-key-env",
+ "CUSTOM_KEY_ENV",
+ ]
+ )
+
+ profile = _llm_profile_from_args(args)
+
+ assert profile["api_key"] == "resolved-key"
+
+
+def test_harness_cli_grok_profile_defaults_to_xai_key_env() -> None:
+ parser = build_harness_cli_parser()
+ args = parser.parse_args(["refresh", "source", "output", "--use-memory-service", "--provider", "grok"])
+
+ profile = _llm_profile_from_args(args)
+
+ assert profile["api_key"] == "XAI_API_KEY"
+
+
+def test_cli_positive_numeric_args_reject_non_positive_values() -> None:
+ invalid_invocations = [
+ (build_folder_cli_parser(), ["source", "output", "--max-text-chars", "0"]),
+ (build_folder_cli_parser(), ["source", "output", "--watch", "--poll-interval", "0"]),
+ (build_folder_cli_parser(), ["source", "output", "--watch", "--watch-max-runs", "-1"]),
+ (build_folder_cli_parser(), ["source", "output", "--min-evolution-confidence", "1.1"]),
+ (build_harness_cli_parser(), ["init", "repo", "--max-text-chars", "0"]),
+ (build_harness_cli_parser(), ["refresh", "repo", "--max-chars", "0"]),
+ (build_harness_cli_parser(), ["watch", "repo", "--poll-interval", "-0.5"]),
+ (build_harness_cli_parser(), ["suggest-skills", "repo", "--limit", "0"]),
+ (build_harness_cli_parser(), ["refresh", "repo", "--min-evolution-confidence", "-0.1"]),
+ (build_context_cli_parser(), ["repo", "--max-chars", "0"]),
+ ]
+
+ for parser, args in invalid_invocations:
+ with pytest.raises(SystemExit) as exc_info:
+ parser.parse_args(args)
+ assert exc_info.value.code == 2
+
+
+@pytest.mark.asyncio
+async def test_watch_folder_recompiles_after_source_change(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ profile = source / "profile.txt"
+ profile.write_text("The user prefers concise answers.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ async def mutate_after_initial(event) -> None:
+ if event.reason == "initial":
+ profile.write_text("The user prefers detailed answers.", encoding="utf-8")
+
+ events = await watch_folder_to_markdown(
+ source,
+ output,
+ config=FolderMemoryCompilerConfig(use_memory_service=False),
+ poll_interval=0.01,
+ max_runs=2,
+ on_event=mutate_after_initial,
+ )
+
+ memory_md = (output / "memory.md").read_text(encoding="utf-8")
+ assert [event.reason for event in events] == ["initial", "changed"]
+ assert events[0].status is not None
+ assert events[0].status.new == ["profile.txt"]
+ assert events[1].status is not None
+ assert events[1].status.changed == ["profile.txt"]
+ assert events[1].result.processed == ["profile.txt"]
+ assert "detailed answers" in memory_md
+ assert "concise answers" not in memory_md
+
+
+def test_folder_cli_watch_outputs_json_event(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: run focused validation after edits.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ exit_code = folder_cli_main(
+ [
+ str(source),
+ str(output),
+ "--watch",
+ "--watch-max-runs",
+ "1",
+ "--poll-interval",
+ "0.01",
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ event = json.loads(captured.out)
+ assert exit_code == 0
+ assert event["reason"] == "initial"
+ assert event["iteration"] == 1
+ assert event["processed"] == ["notes.txt"]
+ assert event["delta"]["new"] == ["notes.txt"]
+ assert event["delta"]["counts"]["new"] == 1
+
+
+@pytest.mark.asyncio
+async def test_markdown_context_loader_reads_generated_and_manual_notes(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text(
+ "The user's tone preference is direct. Skill: inspect files before patching.",
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ await compiler.compile(source, output)
+ soul_path = output / "soul.md"
+ soul_path.write_text(
+ soul_path.read_text(encoding="utf-8") + "\nManual soul note: keep answers calm and compact.\n",
+ encoding="utf-8",
+ )
+
+ repo = MarkdownMemoryRepository(output)
+ sections = repo.list_sections()
+ pack = repo.build_context_pack(query="tone and patching", max_chars=3000)
+
+ assert any(section.kind == "generated" and section.bucket == "skill" for section in sections)
+ assert any(section.kind == "manual" and "calm and compact" in section.content for section in sections)
+ assert not any(section.kind == "manual" and section.content.startswith("# Skill From") for section in sections)
+ assert "Manual soul note" in pack.to_markdown()
+ assert "inspect files before patching" in pack.to_markdown()
+
+
+def test_context_cli_outputs_json_pack(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: run focused validation after edits.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ folder_cli_main([str(source), str(output)])
+ capsys.readouterr()
+
+ exit_code = context_cli_main([str(output), "--query", "validation", "--json"])
+
+ captured = capsys.readouterr()
+ pack = json.loads(captured.out)
+ assert exit_code == 0
+ assert pack["query"] == "validation"
+ assert pack["sections"]
+ assert any(section["bucket"] == "skill" for section in pack["sections"])
+
+
+def test_context_cli_writes_rendered_context_to_file(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: run focused validation after edits.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ context_path = tmp_path / "artifacts" / "context.system.md"
+ folder_cli_main([str(source), str(output)])
+ capsys.readouterr()
+
+ exit_code = context_cli_main(
+ [
+ str(output),
+ "--query",
+ "validation",
+ "--format",
+ "system",
+ "--output",
+ str(context_path),
+ ]
+ )
+
+ captured = capsys.readouterr()
+ rendered = context_path.read_text(encoding="utf-8")
+ assert exit_code == 0
+ assert captured.out == ""
+ assert "" in rendered
+ assert "run focused validation after edits" in rendered
+
+
+def test_context_pack_exports_summary(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: summarize context sections for agents.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ folder_cli_main([str(source), str(output)])
+ capsys.readouterr()
+
+ pack = MarkdownMemoryRepository(output).build_context_pack(query="context summary")
+ summary = pack.to_summary()
+
+ assert summary["section_count"] == len(pack.sections)
+ assert summary["buckets"]["memory"] >= 1
+ assert summary["buckets"]["skill"] >= 1
+ assert summary["kinds"]["generated"] >= 1
+ assert "raw_data/notes.txt" in summary["sources"]
+ assert "content" not in summary["sections"][0]
+
+ assert context_cli_main([str(output), "--query", "context summary", "--format", "summary"]) == 0
+ captured = capsys.readouterr()
+ cli_summary = json.loads(captured.out)
+ assert cli_summary["query"] == "context summary"
+ assert cli_summary["buckets"]["skill"] >= 1
+ assert "content" not in cli_summary["sections"][0]
+
+
+def test_context_pack_applies_bucket_character_limits(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ output = tmp_path / "memory_repo"
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+ compiler.scaffold(output)
+ manual_skill = output / "skill" / "manual-skill.md"
+ manual_skill.write_text(
+ "# Manual Skill Card\n\n" + ("Check generated skill context before use. " * 80),
+ encoding="utf-8",
+ )
+
+ repo = MarkdownMemoryRepository(output)
+ pack = repo.build_context_pack(
+ buckets=["skill"],
+ max_chars=5000,
+ include_generated=False,
+ bucket_char_limits={"skill": 500},
+ )
+
+ assert pack.bucket_char_limits == {"skill": 500}
+ assert pack.used_chars_by_bucket["skill"] <= 500
+ assert "[truncated]" in pack.sections[0].content
+
+ assert context_cli_main(
+ [
+ str(output),
+ "--bucket",
+ "skill",
+ "--no-generated",
+ "--bucket-max",
+ "skill=500",
+ "--format",
+ "summary",
+ ]
+ ) == 0
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert summary["bucket_char_limits"] == {"skill": 500}
+ assert summary["used_chars_by_bucket"]["skill"] <= 500
+
+
+def test_context_cli_uses_repo_harness_config_defaults(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text(
+ "The user prefers calm answers. Skill: reuse repo harness context defaults. " * 20,
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ folder_cli_main([str(source), str(output)])
+ capsys.readouterr()
+ (output / ".memu" / "harness.json").write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "context": {
+ "buckets": ["skill"],
+ "bucket_char_limits": {"skill": 260},
+ "format": "summary",
+ "max_chars": 900,
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ assert context_cli_main([str(output)]) == 0
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert summary["max_chars"] == 900
+ assert summary["bucket_char_limits"] == {"skill": 260}
+ assert summary["buckets"]["skill"] >= 1
+ assert summary["buckets"]["memory"] == 0
+ assert summary["buckets"]["soul"] == 0
+
+ assert context_cli_main(
+ [
+ str(output),
+ "--bucket",
+ "memory",
+ "--bucket-max",
+ "memory=400",
+ "--max-chars",
+ "1200",
+ "--json",
+ ]
+ ) == 0
+ captured = capsys.readouterr()
+ pack = json.loads(captured.out)
+ assert pack["max_chars"] == 1200
+ assert pack["bucket_char_limits"] == {"memory": 400}
+ assert all(section["bucket"] == "memory" for section in pack["sections"])
+
+
+def test_context_pack_exports_system_prompt_and_messages(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: run focused validation after edits.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ folder_cli_main([str(source), str(output)])
+ capsys.readouterr()
+
+ repo = MarkdownMemoryRepository(output)
+ pack = repo.build_context_pack(query="validation", max_chars=3000)
+ messages = pack.to_messages()
+
+ assert messages == [{"role": "system", "content": pack.to_system_prompt()}]
+ assert "" in messages[0]["content"]
+ assert "" in messages[0]["content"]
+ assert "Prefer manual sections" in messages[0]["content"]
+
+ exit_code = context_cli_main([str(output), "--query", "validation", "--format", "messages"])
+
+ captured = capsys.readouterr()
+ cli_messages = json.loads(captured.out)
+ assert exit_code == 0
+ assert cli_messages[0]["role"] == "system"
+ assert "" in cli_messages[0]["content"]
+ assert "run focused validation" in cli_messages[0]["content"]
+
+
+def test_context_pack_injects_into_chat_messages_without_mutation(
+ tmp_path: Path,
+) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: inject context without mutating messages.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ compile_folder_to_markdown_sync(
+ source,
+ output,
+ config=FolderMemoryCompilerConfig(use_memory_service=False),
+ )
+
+ pack = MarkdownMemoryRepository(output).build_context_pack(query="inject context")
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "What should I do?"},
+ ]
+
+ injected = pack.inject_into_messages(messages)
+
+ assert messages[0]["content"] == "You are helpful."
+ assert injected is not messages
+ assert injected[0]["role"] == "system"
+ assert injected[0]["content"].startswith("You are helpful.")
+ assert "" in injected[0]["content"]
+ assert "inject context without mutating messages" in injected[0]["content"]
+
+ second = inject_context_messages(injected, pack)
+ assert second[0]["content"].count("") == 1
+ assert second[0]["content"].count("") == 1
+
+
+def test_context_pack_injects_system_message_when_missing(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: create a context system message.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ compile_folder_to_markdown_sync(
+ source,
+ output,
+ config=FolderMemoryCompilerConfig(use_memory_service=False),
+ )
+
+ pack = MarkdownMemoryRepository(output).build_context_pack(query="system message")
+ messages = [{"role": "user", "content": "Hi"}]
+
+ injected = inject_context_messages(messages, pack)
+
+ assert messages == [{"role": "user", "content": "Hi"}]
+ assert injected[0]["role"] == "system"
+ assert injected[1]["role"] == "user"
+ assert "" in injected[0]["content"]
+
+
+@pytest.mark.asyncio
+async def test_record_skill_trace_feeds_skill_markdown(tmp_path: Path) -> None:
+ raw_data = tmp_path / "raw_data"
+ output = tmp_path / "memory_repo"
+ record = record_skill_trace(
+ raw_data,
+ task="Fix failing folder compiler tests",
+ outcome="success",
+ summary="Inspected files, patched code, and ran focused validation.",
+ actions=["Read failing test output", "Patch the smallest affected module", "Run focused validation"],
+ tools=[SkillToolTrace(name="pytest", success=True, score=0.9)],
+ lessons=["Run focused tests after touching compiler behavior."],
+ )
+ compiler = FolderMemoryCompiler(config=FolderMemoryCompilerConfig(use_memory_service=False))
+
+ await compiler.compile(raw_data, output)
+
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+ assert record.trace_path.exists()
+ assert "Skill Evolution Trace" in record.trace_path.read_text(encoding="utf-8")
+ assert "Run focused tests after touching compiler behavior" in skill_md
+ assert "skill_traces/" in skill_md
+
+
+@pytest.mark.asyncio
+async def test_context_harness_refreshes_context_and_self_evolves_skills(tmp_path: Path) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text(
+ "The user prefers calm answers. Skill: validate generated context before relying on it.",
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+ harness = ContextHarness(
+ source,
+ output,
+ user={"user_id": "u1"},
+ compiler_config=FolderMemoryCompilerConfig(use_memory_service=False),
+ )
+
+ run = await harness.refresh_context(query="context validation", max_chars=3000)
+
+ assert run.compile_result.processed == ["notes.txt"]
+ assert "Skill: validate generated context" in run.context_pack.to_markdown()
+ assert (output / "raw_data" / "notes.txt").exists()
+
+ trace_result = await harness.record_skill_trace(
+ task="Validate generated context packs",
+ outcome="success",
+ summary="Compiled raw data and checked the resulting context pack.",
+ actions=["Compile raw data", "Build context pack", "Check skill sections"],
+ tools=[SkillToolTrace(name="memu-context", success=True, score=0.95)],
+ lessons=["Check generated skill sections before injecting context into an agent."],
+ )
+
+ assert trace_result.compile_result is not None
+ assert any(path.startswith("skill_traces/") for path in trace_result.compile_result.processed)
+ assert trace_result.record.trace_path.exists()
+ assert "Check generated skill sections" in (output / "skill.md").read_text(encoding="utf-8")
+ assert "Check generated skill sections" in harness.build_context_markdown(query="agent context")
+
+
+def test_context_harness_from_repo_uses_raw_data_and_config_defaults(tmp_path: Path) -> None:
+ output = tmp_path / "memory_repo"
+ raw_data = output / "raw_data"
+ metadata_dir = output / ".memu"
+ raw_data.mkdir(parents=True)
+ metadata_dir.mkdir()
+ (raw_data / "notes.txt").write_text(
+ "The user prefers calm answers. Skill: use ContextHarness.from_repo for existing repositories. " * 20,
+ encoding="utf-8",
+ )
+ (raw_data / "debug.tmp").write_text("Temporary output.", encoding="utf-8")
+ (metadata_dir / "harness.json").write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "compiler": {
+ "exclude_patterns": ["*.tmp"],
+ "max_text_chars": 180,
+ },
+ "context": {
+ "buckets": ["skill"],
+ "bucket_char_limits": {"skill": 260},
+ "max_chars": 900,
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ harness = ContextHarness.from_repo(
+ output,
+ compiler_config=FolderMemoryCompilerConfig(use_memory_service=False),
+ )
+ run = harness.refresh_context_sync(query="repo harness")
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+
+ assert harness.source_folder == raw_data.resolve()
+ assert run.compile_result.processed == ["notes.txt"]
+ assert sorted(manifest["sources"]) == ["notes.txt"]
+ assert not (output / "raw_data" / "debug.tmp").exists()
+ assert run.context_pack.max_chars == 900
+ assert run.context_pack.bucket_char_limits == {"skill": 260}
+ assert all(section.bucket == "skill" for section in run.context_pack.sections)
+
+ overridden = harness.build_context_pack(
+ buckets=["memory"],
+ max_chars=1200,
+ bucket_char_limits={},
+ )
+ assert overridden.max_chars == 1200
+ assert overridden.bucket_char_limits == {}
+ assert all(section.bucket == "memory" for section in overridden.sections)
+
+
+def test_context_harness_health_reports_invalid_repo_config(tmp_path: Path) -> None:
+ output = tmp_path / "memory_repo"
+ raw_data = output / "raw_data"
+ metadata_dir = output / ".memu"
+ raw_data.mkdir(parents=True)
+ metadata_dir.mkdir()
+ (raw_data / "notes.txt").write_text("Skill: validate harness config health.", encoding="utf-8")
+ (metadata_dir / "harness.json").write_text(
+ json.dumps({"version": 1, "context": {"max_chars": 0}}),
+ encoding="utf-8",
+ )
+
+ harness = ContextHarness.from_repo(output)
+ report = harness.health()
+ codes = [issue.code for issue in report.issues]
+
+ assert report.ok is False
+ assert "invalid_harness_config" in codes
+ with pytest.raises(SystemExit):
+ harness.refresh_context_sync()
+
+
+def test_skill_trace_cli_records_and_compiles(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ raw_data = tmp_path / "raw_data"
+ output = tmp_path / "memory_repo"
+
+ exit_code = skill_trace_cli_main(
+ [
+ str(raw_data),
+ "--task",
+ "Validate generated context packs",
+ "--outcome",
+ "success",
+ "--summary",
+ "Generated a context pack and verified skill sections.",
+ "--action",
+ "Build context pack",
+ "--lesson",
+ "After compiling raw data, validate the context pack before relying on it.",
+ "--tool",
+ "memu-context:success:0.95",
+ "--metadata",
+ "agent_id=codex",
+ "--output-folder",
+ str(output),
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert Path(summary["trace_path"]).exists()
+ assert summary["compiled"]["entry_count"] >= 1
+ assert (output / "skill.md").exists()
+
+
+def test_context_harness_cli_refresh_outputs_context_json(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text(
+ "The user prefers calm answers. Skill: validate generated context before relying on it.",
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+
+ exit_code = harness_cli_main(
+ [
+ "refresh",
+ str(source),
+ str(output),
+ "--query",
+ "context validation",
+ "--json",
+ "--user",
+ "user_id=u1",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert summary["compile"]["processed"] == ["notes.txt"]
+ assert summary["context"]["query"] == "context validation"
+ assert any(section["bucket"] == "skill" for section in summary["context"]["sections"])
+
+ assert harness_cli_main(["context", str(output), "--query", "validation", "--json"]) == 0
+ captured = capsys.readouterr()
+ pack = json.loads(captured.out)
+ assert pack["query"] == "validation"
+ assert any(section["bucket"] == "skill" for section in pack["sections"])
+ assert (output / "raw_data" / "notes.txt").exists()
+
+ assert harness_cli_main(["refresh", str(output), "--query", "validation", "--format", "messages"]) == 0
+ captured = capsys.readouterr()
+ messages = json.loads(captured.out)
+ assert messages[0]["role"] == "system"
+ assert "" in messages[0]["content"]
+ assert "Skill: validate generated context" in messages[0]["content"]
+
+ assert harness_cli_main(["refresh", str(output), "--query", "validation", "--format", "summary"]) == 0
+ captured = capsys.readouterr()
+ refresh_summary = json.loads(captured.out)
+ assert refresh_summary["query"] == "validation"
+ assert refresh_summary["buckets"]["skill"] >= 1
+ assert "compile" not in refresh_summary
+
+
+def test_context_harness_cli_refresh_writes_json_output_file(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_text("Skill: write rendered context files for agents.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+ context_path = tmp_path / "artifacts" / "refresh.json"
+
+ exit_code = harness_cli_main(
+ [
+ "refresh",
+ str(source),
+ str(output),
+ "--query",
+ "context files",
+ "--json",
+ "--output",
+ str(context_path),
+ ]
+ )
+
+ captured = capsys.readouterr()
+ payload = json.loads(context_path.read_text(encoding="utf-8"))
+ assert exit_code == 0
+ assert captured.out == ""
+ assert payload["compile"]["processed"] == ["notes.txt"]
+ assert payload["context"]["query"] == "context files"
+ assert any(section["bucket"] == "skill" for section in payload["context"]["sections"])
+
+
+def test_context_harness_cli_json_is_ascii_safe_and_strips_utf8_bom(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ (source / "notes.txt").write_bytes(
+ "\ufeffThe user prefers concise Chinese context. "
+ "Skill: validate BOM-safe generated context.".encode("utf-8")
+ )
+ (source / "workflow.png").write_bytes(b"\x89PNG\r\n\x1a\n")
+ (source / "workflow.caption.md").write_bytes(
+ "\ufeffSkill: compare screenshots against acceptance criteria.".encode("utf-8")
+ )
+ output = tmp_path / "memory_repo"
+
+ exit_code = harness_cli_main(
+ [
+ "refresh",
+ str(source),
+ str(output),
+ "--query",
+ "BOM safe context",
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ captured.out.encode("ascii")
+ summary = json.loads(captured.out)
+ serialized = json.dumps(summary, ensure_ascii=False)
+ assert exit_code == 0
+ assert "\ufeff" not in serialized
+ assert "BOM-safe generated context" in serialized
+ assert "compare screenshots against acceptance criteria" in serialized
+
+
+def test_context_harness_cli_init_scaffolds_memory_repo(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text("Raw note before extraction.", encoding="utf-8")
+ media = upload / "media"
+ media.mkdir()
+ (media / "screenshot.png").write_bytes(b"\x89PNG\r\n\x1a\n")
+ output = tmp_path / "memory_repo"
+
+ exit_code = harness_cli_main(
+ [
+ "init",
+ str(output),
+ "--source-folder",
+ str(upload),
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert (output / "memory.md").exists()
+ assert (output / "AGENTS.md").exists()
+ assert (output / "memory").is_dir()
+ assert (output / "soul.md").exists()
+ assert (output / "soul").is_dir()
+ assert (output / "skill.md").exists()
+ assert (output / "skill").is_dir()
+ assert (output / ".memu" / "manifest.json").exists()
+ assert (output / ".memu" / "derived").is_dir()
+ assert (output / "raw_data" / "notes.txt").exists()
+ assert (output / "raw_data" / "media" / "screenshot.png").exists()
+ assert "AGENTS.md" in summary["created"]
+ assert sorted(summary["copied"]) == ["media/screenshot.png", "notes.txt"]
+
+
+def test_context_harness_cli_init_writes_repo_harness_config(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text("Raw note before extraction.", encoding="utf-8")
+ (upload / "debug.tmp").write_text("Temporary output.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ exit_code = harness_cli_main(
+ [
+ "init",
+ str(output),
+ "--source-folder",
+ str(upload),
+ "--exclude",
+ "*.tmp",
+ "--max-text-chars",
+ "123",
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ config_path = Path(summary["config_path"])
+ config = json.loads(config_path.read_text(encoding="utf-8"))
+ assert exit_code == 0
+ assert config_path == (output / ".memu" / "harness.json").resolve()
+ assert config["version"] == 1
+ assert config["compiler"]["exclude_patterns"] == ["*.tmp"]
+ assert config["compiler"]["max_text_chars"] == 123
+ assert config["context"]["format"] == "markdown"
+ assert config["context"]["max_chars"] == 8000
+ assert (output / "raw_data" / "notes.txt").exists()
+ assert not (output / "raw_data" / "debug.tmp").exists()
+
+
+def test_context_harness_cli_init_preserves_existing_agent_instructions(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ output = tmp_path / "memory_repo"
+ output.mkdir()
+ agents_path = output / "AGENTS.md"
+ agents_path.write_text("# Custom agent instructions\n", encoding="utf-8")
+
+ exit_code = harness_cli_main(["init", str(output), "--json"])
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert agents_path.read_text(encoding="utf-8") == "# Custom agent instructions\n"
+ assert "AGENTS.md" not in summary["created"]
+
+
+def test_context_harness_cli_refresh_defaults_to_repo_raw_data(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text(
+ "Skill: validate generated context before relying on it.",
+ encoding="utf-8",
+ )
+ output = tmp_path / "memory_repo"
+
+ assert harness_cli_main(["init", str(output), "--source-folder", str(upload), "--json"]) == 0
+ capsys.readouterr()
+
+ exit_code = harness_cli_main(["refresh", str(output), "--query", "context validation", "--json"])
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert summary["compile"]["processed"] == ["notes.txt"]
+ assert summary["context"]["query"] == "context validation"
+ assert any(section["bucket"] == "skill" for section in summary["context"]["sections"])
+
+
+def test_context_harness_cli_refresh_honors_exclude_patterns(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text("Skill: keep meaningful source files.", encoding="utf-8")
+ (upload / "debug.tmp").write_text("Temporary debug output.", encoding="utf-8")
+ cache = upload / "node_modules" / "pkg"
+ cache.mkdir(parents=True)
+ (cache / "cache.txt").write_text("Dependency cache.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ exit_code = harness_cli_main(
+ [
+ "refresh",
+ str(upload),
+ str(output),
+ "--exclude",
+ "node_modules/**",
+ "--exclude",
+ "*.tmp",
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert summary["compile"]["processed"] == ["notes.txt"]
+ assert not (output / "raw_data" / "debug.tmp").exists()
+ assert not (output / "raw_data" / "node_modules").exists()
+
+
+def test_context_harness_cli_refresh_uses_repo_memuignore(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ output = tmp_path / "memory_repo"
+ raw_data = output / "raw_data"
+ raw_data.mkdir(parents=True)
+ (output / ".memuignore").write_text("node_modules/**\n*.tmp\n", encoding="utf-8")
+ (raw_data / "notes.txt").write_text("Skill: keep meaningful source files.", encoding="utf-8")
+ (raw_data / "debug.tmp").write_text("Temporary debug output.", encoding="utf-8")
+ cache = raw_data / "node_modules" / "pkg"
+ cache.mkdir(parents=True)
+ (cache / "cache.txt").write_text("Dependency cache.", encoding="utf-8")
+
+ exit_code = harness_cli_main(["refresh", str(output), "--json"])
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert summary["compile"]["processed"] == ["notes.txt"]
+ assert not (output / "raw_data" / "debug.tmp").exists()
+ assert not (output / "raw_data" / "node_modules").exists()
+
+
+def test_context_harness_cli_refresh_uses_repo_harness_config_defaults(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ output = tmp_path / "memory_repo"
+ raw_data = output / "raw_data"
+ metadata_dir = output / ".memu"
+ raw_data.mkdir(parents=True)
+ metadata_dir.mkdir()
+ (raw_data / "notes.txt").write_text(
+ "Skill: keep repo harness config defaults available for agent context. " * 20,
+ encoding="utf-8",
+ )
+ (raw_data / "debug.tmp").write_text("Temporary output.", encoding="utf-8")
+ (metadata_dir / "harness.json").write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "compiler": {
+ "exclude_patterns": ["*.tmp"],
+ "max_text_chars": 160,
+ },
+ "context": {
+ "buckets": ["skill"],
+ "bucket_char_limits": {"skill": 240},
+ "format": "summary",
+ "max_chars": 900,
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ exit_code = harness_cli_main(["refresh", str(output)])
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ manifest = json.loads((output / ".memu" / "manifest.json").read_text(encoding="utf-8"))
+ assert exit_code == 0
+ assert summary["max_chars"] == 900
+ assert summary["bucket_char_limits"] == {"skill": 240}
+ assert summary["buckets"]["skill"] >= 1
+ assert summary["buckets"]["memory"] == 0
+ assert summary["buckets"]["soul"] == 0
+ assert sorted(manifest["sources"]) == ["notes.txt"]
+ assert not (output / "raw_data" / "debug.tmp").exists()
+
+
+def test_context_harness_cli_status_reports_manifest_delta(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "keep.txt").write_text("Durable memory evidence.", encoding="utf-8")
+ media = upload / "media"
+ media.mkdir()
+ (media / "workflow.png").write_bytes(b"\x89PNG\r\n\x1a\n")
+ (media / "workflow.caption.md").write_text("Skill: inspect screenshots.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ assert harness_cli_main(["init", str(output), "--source-folder", str(upload), "--json"]) == 0
+ capsys.readouterr()
+ assert harness_cli_main(["refresh", str(output), "--json"]) == 0
+ capsys.readouterr()
+
+ (output / "raw_data" / "keep.txt").unlink()
+ (output / "raw_data" / "new.txt").write_text("Skill: validate status reports.", encoding="utf-8")
+ (output / "raw_data" / "media" / "workflow.caption.md").write_text(
+ "Skill: compare screenshots against acceptance criteria.",
+ encoding="utf-8",
+ )
+
+ exit_code = harness_cli_main(["status", str(output), "--json"])
+
+ captured = capsys.readouterr()
+ report = json.loads(captured.out)
+ changed_source = next(source for source in report["sources"] if source["path"] == "media/workflow.png")
+ assert exit_code == 0
+ assert report["new"] == ["new.txt"]
+ assert report["changed"] == ["media/workflow.png"]
+ assert report["removed"] == ["keep.txt"]
+ assert "media/workflow.caption.md" not in report["new"]
+ assert changed_source["sidecars"] == ["media/workflow.caption.md"]
+
+
+def test_context_harness_cli_doctor_reports_repository_health(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text("Skill: validate repository health.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ assert harness_cli_main(["refresh", str(upload), str(output), "--json"]) == 0
+ capsys.readouterr()
+
+ assert harness_cli_main(["doctor", str(output), "--json"]) == 0
+ captured = capsys.readouterr()
+ healthy = json.loads(captured.out)
+ assert healthy["ok"] is True
+ assert healthy["counts"]["errors"] == 0
+
+ (output / "raw_data" / "notes.txt").unlink()
+
+ exit_code = harness_cli_main(["doctor", str(output), "--json"])
+
+ captured = capsys.readouterr()
+ broken = json.loads(captured.out)
+ codes = [issue["code"] for issue in broken["issues"]]
+ assert exit_code == 1
+ assert broken["ok"] is False
+ assert "missing_raw_source" in codes
+
+
+def test_context_harness_cli_doctor_reports_invalid_harness_config(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ output = tmp_path / "memory_repo"
+ assert harness_cli_main(["init", str(output), "--json"]) == 0
+ capsys.readouterr()
+
+ (output / ".memu" / "harness.json").write_text(
+ json.dumps(
+ {
+ "version": 1,
+ "context": {
+ "max_chars": 0,
+ },
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ exit_code = harness_cli_main(["doctor", str(output), "--json"])
+
+ captured = capsys.readouterr()
+ report = json.loads(captured.out)
+ codes = [issue["code"] for issue in report["issues"]]
+ assert exit_code == 1
+ assert report["ok"] is False
+ assert "invalid_harness_config" in codes
+
+
+def test_context_harness_cli_doctor_warns_when_agent_instructions_missing(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text("Skill: validate repository health.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ assert harness_cli_main(["refresh", str(upload), str(output), "--json"]) == 0
+ capsys.readouterr()
+
+ (output / "AGENTS.md").unlink()
+
+ exit_code = harness_cli_main(["doctor", str(output), "--json"])
+
+ captured = capsys.readouterr()
+ report = json.loads(captured.out)
+ codes = [issue["code"] for issue in report["issues"]]
+ assert exit_code == 0
+ assert report["ok"] is True
+ assert report["counts"]["warnings"] == 1
+ assert "missing_agent_instructions" in codes
+
+
+def test_context_harness_cli_doctor_warns_about_orphan_evidence(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text("Skill: validate repository health.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ assert harness_cli_main(["refresh", str(upload), str(output), "--json"]) == 0
+ capsys.readouterr()
+
+ orphan = output / ".memu" / "derived" / "old.txt.evidence.md"
+ orphan.write_text("# Evidence: old.txt\n", encoding="utf-8")
+
+ exit_code = harness_cli_main(["doctor", str(output), "--json"])
+
+ captured = capsys.readouterr()
+ report = json.loads(captured.out)
+ codes = [issue["code"] for issue in report["issues"]]
+ assert exit_code == 0
+ assert report["ok"] is True
+ assert "orphan_evidence" in codes
+
+
+def test_context_harness_cli_promotes_manual_skill(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ upload = tmp_path / "upload"
+ upload.mkdir()
+ (upload / "notes.txt").write_text("Skill: validate context packs.", encoding="utf-8")
+ output = tmp_path / "memory_repo"
+
+ assert harness_cli_main(["init", str(output), "--source-folder", str(upload), "--json"]) == 0
+ capsys.readouterr()
+ assert harness_cli_main(["refresh", str(output), "--json"]) == 0
+ capsys.readouterr()
+
+ exit_code = harness_cli_main(
+ [
+ "promote-skill",
+ str(output),
+ "--title",
+ "Validate Context Packs",
+ "--when",
+ "Before injecting generated context into an agent.",
+ "--action",
+ "Build the context pack",
+ "--action",
+ "Check manual and generated skill sections",
+ "--lesson",
+ "Always inspect promoted skills before relying on generated context.",
+ "--source",
+ "skill_traces/validate-context-packs.md",
+ "--tag",
+ "context",
+ "--metadata",
+ "agent_id=codex",
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ skill_md = (output / "skill.md").read_text(encoding="utf-8")
+ card_path = Path(summary["card_path"])
+ assert exit_code == 0
+ assert summary["title"] == "Validate Context Packs"
+ assert card_path.exists()
+ assert "## Promoted Skill: Validate Context Packs" in skill_md
+ first_card = card_path.read_text(encoding="utf-8")
+ assert "Always inspect promoted skills" in first_card
+ assert "- source: skill_traces/validate-context-packs.md" in first_card
+ assert "- agent_id: codex" in first_card
+
+ assert harness_cli_main(
+ [
+ "promote-skill",
+ str(output),
+ "--title",
+ "Validate Context Packs",
+ "--lesson",
+ "Compare generated sections against the current task before use.",
+ "--json",
+ ]
+ ) == 0
+ captured = capsys.readouterr()
+ second_summary = json.loads(captured.out)
+ second_card_path = Path(second_summary["card_path"])
+ updated_card = second_card_path.read_text(encoding="utf-8")
+ updated_skill_md = (output / "skill.md").read_text(encoding="utf-8")
+ assert second_card_path == card_path
+ assert updated_skill_md.count("## Promoted Skill: Validate Context Packs") == 1
+ assert "Always inspect promoted skills" in updated_card
+ assert "Compare generated sections against the current task" in updated_card
+ assert "- source: skill_traces/validate-context-packs.md" in updated_card
+ assert "- agent_id: codex" in updated_card
+
+ assert harness_cli_main(["refresh", str(output), "--json"]) == 0
+ capsys.readouterr()
+ assert "Promoted Skill: Validate Context Packs" in (output / "skill.md").read_text(encoding="utf-8")
+
+ assert harness_cli_main(["context", str(output), "--bucket", "skill", "--json"]) == 0
+ captured = capsys.readouterr()
+ pack = json.loads(captured.out)
+ manual_sections = [section for section in pack["sections"] if section["kind"] == "manual"]
+ manual_sources = [section["source"] for section in manual_sections]
+ assert any("Validate Context Packs" in section["content"] for section in manual_sections)
+ assert "skill.md" not in manual_sources
+ assert "skill/promoted/validate-context-packs.md" in manual_sources
+
+
+def test_context_harness_cli_suggests_and_promotes_skills_from_traces(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ output = tmp_path / "memory_repo"
+ lesson = "Check generated skill sections before injecting context into an agent."
+ record_skill_trace(
+ source,
+ task="Validate generated context packs",
+ outcome="success",
+ actions=["Build context pack", "Check skill sections"],
+ tools=[SkillToolTrace(name="memu-context", success=True, score=0.95)],
+ lessons=[lesson],
+ )
+ record_skill_trace(
+ source,
+ task="Review generated agent context",
+ outcome="success",
+ actions=["Build context pack", "Compare generated sections"],
+ tools=[SkillToolTrace(name="memu-harness", success=True, score=0.9)],
+ lessons=[lesson],
+ )
+
+ exit_code = harness_cli_main(
+ [
+ "suggest-skills",
+ str(source),
+ str(output),
+ "--min-support",
+ "2",
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ proposal = summary["proposals"][0]
+ assert exit_code == 0
+ assert summary["proposal_count"] == 1
+ assert summary["promoted_count"] == 0
+ assert proposal["title"] == "Check Generated Skill Sections"
+ assert proposal["support_count"] == 2
+ assert lesson in proposal["lessons"]
+ assert len(proposal["sources"]) == 2
+
+ assert harness_cli_main(
+ [
+ "suggest-skills",
+ str(source),
+ str(output),
+ "--min-support",
+ "2",
+ "--promote",
+ "--json",
+ ]
+ ) == 0
+ captured = capsys.readouterr()
+ promoted = json.loads(captured.out)
+ card_path = Path(promoted["promotions"][0]["card_path"])
+ assert promoted["promoted_count"] == 1
+ assert card_path.exists()
+ assert "Check Generated Skill Sections" in card_path.read_text(encoding="utf-8")
+
+ assert harness_cli_main(["context", str(source), str(output), "--bucket", "skill", "--json"]) == 0
+ captured = capsys.readouterr()
+ pack = json.loads(captured.out)
+ manual_sections = [section for section in pack["sections"] if section["kind"] == "manual"]
+ assert any("Check Generated Skill Sections" in section["content"] for section in manual_sections)
+
+
+def test_context_harness_cli_trace_recompiles_skill_trace(
+ tmp_path: Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ source = tmp_path / "upload"
+ source.mkdir()
+ output = tmp_path / "memory_repo"
+
+ exit_code = harness_cli_main(
+ [
+ "trace",
+ str(source),
+ str(output),
+ "--task",
+ "Validate generated context packs",
+ "--outcome",
+ "success",
+ "--lesson",
+ "Check generated skill sections before injecting context into an agent.",
+ "--tool",
+ "memu-context:success:0.95",
+ "--json",
+ ]
+ )
+
+ captured = capsys.readouterr()
+ summary = json.loads(captured.out)
+ assert exit_code == 0
+ assert Path(summary["trace_path"]).exists()
+ assert summary["compiled"]["entry_count"] >= 1
+ assert any(path.startswith("skill_traces/") for path in summary["compiled"]["processed"])
+ assert "Check generated skill sections" in (output / "skill.md").read_text(encoding="utf-8")
diff --git a/tests/test_inmemory.py b/tests/test_inmemory.py
index 250f15a2..bc9c3314 100644
--- a/tests/test_inmemory.py
+++ b/tests/test_inmemory.py
@@ -1,14 +1,33 @@
+#!/usr/bin/env python3
+"""Opt-in live LLM smoke test for the in-memory backend."""
+
+from __future__ import annotations
+
+import asyncio
import os
+import sys
+from pathlib import Path
+from typing import Any
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+RUN_LIVE_LLM_TESTS_ENV = "MEMU_RUN_LIVE_LLM_TESTS"
+
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(PROJECT_ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
-from memu.app import MemoryService
+async def run_inmemory_workflow() -> None:
+ """Run the in-memory memorize/retrieve smoke workflow against a real LLM."""
+ from memu import MemoryService
-async def main():
- """Test with in-memory storage (default)."""
api_key = os.environ.get("OPENAI_API_KEY")
- # dashscope_api_key = os.environ.get("DASHSCOPE_API_KEY")
- # voyage_api_key = os.environ.get("VOYAGE_API_KEY")
- file_path = os.path.abspath("example/example_conversation.json")
+ if not api_key:
+ msg = "OPENAI_API_KEY is required for the in-memory live LLM workflow"
+ raise RuntimeError(msg)
+
+ file_path = Path(__file__).resolve().parent / "example" / "example_conversation.json"
print("\n" + "=" * 60)
print("[INMEMORY] Starting test...")
@@ -16,74 +35,69 @@ async def main():
service = MemoryService(
llm_profiles={"default": {"api_key": api_key}},
- # llm_profiles={
- # "default": {
- # "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
- # "api_key": dashscope_api_key,
- # "chat_model": "qwen3-max",
- # "client_backend": "sdk"
- # },
- # "embedding": {
- # "base_url": "https://api.voyageai.com/v1",
- # "api_key": voyage_api_key,
- # "embed_model": "voyage-3.5-lite"
- # }
- # },
database_config={
"metadata_store": {"provider": "inmemory"},
},
retrieve_config={"method": "rag"},
)
- # Memorize
print("\n[INMEMORY] Memorizing...")
- memory = await service.memorize(resource_url=file_path, modality="conversation", user={"user_id": "123"})
+ memory = await service.memorize(resource_url=str(file_path), modality="conversation", user={"user_id": "123"})
for cat in memory.get("categories", []):
print(f" - {cat.get('name')}: {(cat.get('summary') or '')[:80]}...")
- queries = [
- {"role": "user", "content": {"text": "Tell me about preferences"}},
- {"role": "assistant", "content": {"text": "Sure, I'll tell you about their preferences"}},
- {
- "role": "user",
- "content": {"text": "What are they"},
- }, # This is the query that will be used to retrieve the memory, the context will be used for query rewriting
- ]
+ queries = _sample_queries()
- # RAG-based retrieval
print("\n[INMEMORY] RETRIEVED - RAG")
service.retrieve_config.method = "rag"
result_rag = await service.retrieve(queries=queries, where={"user_id": "123"})
- print(" Categories:")
- for cat in result_rag.get("categories", [])[:3]:
- print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...")
- print(" Items:")
- for item in result_rag.get("items", [])[:3]:
- print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...")
- if result_rag.get("resources"):
- print(" Resources:")
- for res in result_rag.get("resources", [])[:3]:
- print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...")
+ _print_results(result_rag)
- # LLM-based retrieval
print("\n[INMEMORY] RETRIEVED - LLM")
service.retrieve_config.method = "llm"
result_llm = await service.retrieve(queries=queries, where={"user_id": "123"})
+ _print_results(result_llm)
+
+ print("\n[INMEMORY] Test completed!")
+
+
+async def test_inmemory_full_workflow() -> None:
+ """Opt-in pytest integration check for the in-memory backend and a real LLM."""
+ import pytest
+
+ if os.environ.get(RUN_LIVE_LLM_TESTS_ENV) != "1":
+ pytest.skip(f"Set {RUN_LIVE_LLM_TESTS_ENV}=1 to run live LLM storage workflows")
+ if not os.environ.get("OPENAI_API_KEY"):
+ pytest.skip("OPENAI_API_KEY is required for live LLM storage workflows")
+
+ await run_inmemory_workflow()
+
+
+def _sample_queries() -> list[dict[str, dict[str, str] | str]]:
+ return [
+ {"role": "user", "content": {"text": "Tell me about preferences"}},
+ {"role": "assistant", "content": {"text": "Sure, I'll tell you about their preferences"}},
+ {"role": "user", "content": {"text": "What are they"}},
+ ]
+
+
+def _print_results(result: dict[str, Any]) -> None:
print(" Categories:")
- for cat in result_llm.get("categories", [])[:3]:
+ for cat in result.get("categories", [])[:3]:
print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...")
print(" Items:")
- for item in result_llm.get("items", [])[:3]:
+ for item in result.get("items", [])[:3]:
print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...")
- if result_llm.get("resources"):
+ if result.get("resources"):
print(" Resources:")
- for res in result_llm.get("resources", [])[:3]:
+ for res in result.get("resources", [])[:3]:
print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...")
- print("\n[INMEMORY] Test completed!")
+def main() -> int:
+ asyncio.run(run_inmemory_workflow())
+ return 0
-if __name__ == "__main__":
- import asyncio
- asyncio.run(main())
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/test_langgraph_optional.py b/tests/test_langgraph_optional.py
new file mode 100644
index 00000000..7edcc99b
--- /dev/null
+++ b/tests/test_langgraph_optional.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from pydantic import ValidationError
+
+from memu.integrations import langgraph
+
+
+def test_langgraph_module_imports_without_optional_dependencies() -> None:
+ assert langgraph.MemULangGraphTools.__name__ == "MemULangGraphTools"
+
+
+def test_langgraph_missing_dependency_error_is_actionable() -> None:
+ if langgraph._LANGGRAPH_IMPORT_ERROR is None:
+ return
+
+ try:
+ langgraph.MemULangGraphTools(object())
+ except ImportError as exc:
+ message = str(exc)
+ assert "memu-py[langgraph]" in message
+ assert "uv sync --extra langgraph" in message
+ else:
+ raise AssertionError("missing LangGraph dependencies should raise an actionable ImportError")
+
+
+def test_langgraph_scope_keeps_explicit_user_id_authoritative() -> None:
+ metadata = {"user_id": "metadata-user", "team_id": "team1"}
+
+ scope = langgraph._scope_with_user(" explicit-user ", metadata)
+
+ assert scope == {"user_id": "explicit-user", "team_id": "team1"}
+ assert metadata == {"user_id": "metadata-user", "team_id": "team1"}
+
+
+def test_langgraph_input_schemas_trim_and_validate_bounds() -> None:
+ save_input = langgraph.SaveRecallInput(content=" remember this ", user_id=" user1 ")
+ search_input = langgraph.SearchRecallInput(query=" what changed? ", user_id=" user1 ", limit=3)
+
+ assert save_input.content == "remember this"
+ assert save_input.user_id == "user1"
+ assert search_input.query == "what changed?"
+ assert search_input.user_id == "user1"
+ assert search_input.limit == 3
+
+ for kwargs in (
+ {"query": "", "user_id": "user1"},
+ {"query": "x", "user_id": ""},
+ {"query": "x", "user_id": "user1", "limit": 0},
+ {"query": "x", "user_id": "user1", "min_relevance_score": 1.1},
+ ):
+ try:
+ langgraph.SearchRecallInput(**kwargs)
+ except ValidationError:
+ pass
+ else:
+ raise AssertionError(f"SearchRecallInput should reject invalid values: {kwargs}")
diff --git a/tests/test_lazyllm.py b/tests/test_lazyllm.py
index 6e107414..fbdd2fd9 100644
--- a/tests/test_lazyllm.py
+++ b/tests/test_lazyllm.py
@@ -1,30 +1,44 @@
#!/usr/bin/env python3
"""
-Quick test script to verify LazyLLM backend configuration and basic functionality.
+Opt-in LazyLLM integration smoke test.
Usage:
export MEMU_QWEN_API_KEY=your_api_key
- python examples/test_lazyllm.py
+ export MEMU_RUN_LAZYLLM_TESTS=1
+ uv run python -m pytest tests/test_lazyllm.py
+
+Manual run:
+ export MEMU_QWEN_API_KEY=your_api_key
+ python tests/test_lazyllm.py
"""
+from __future__ import annotations
+
import asyncio
import os
import sys
+from pathlib import Path
-# Add src to sys.path
-src_path = os.path.abspath("src")
-sys.path.insert(0, src_path)
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+RUN_LAZYLLM_TESTS_ENV = "MEMU_RUN_LAZYLLM_TESTS"
+
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(PROJECT_ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
from memu.llm.lazyllm_client import LazyLLMClient # noqa: E402
-async def test_lazyllm_client():
- """Test LazyLLMClient with basic operations."""
+async def run_lazyllm_workflow() -> bool:
+ """Run the LazyLLM-backed chat/embed/vision smoke workflow."""
+ if not os.environ.get("MEMU_QWEN_API_KEY"):
+ msg = "MEMU_QWEN_API_KEY is required for the LazyLLM integration workflow"
+ raise RuntimeError(msg)
print("LazyLLM Backend Test")
print("=" * 60)
- # Get API key from environment
try:
client = LazyLLMClient(
llm_source="qwen",
@@ -36,56 +50,67 @@ async def test_lazyllm_client():
embed_model="text-embedding-v3",
stt_model="qwen-audio-turbo",
)
- print("✓ LazyLLMClient initialized successfully")
- except Exception as e:
- print(f"❌ Failed to initialize LazyLLMClient: {e}")
+ print("[OK] LazyLLMClient initialized successfully")
+ except Exception as exc:
+ print(f"[ERROR] Failed to initialize LazyLLMClient: {exc}")
return False
- # Test 1: Summarization
- print("\n[Test 1] Testing summarization...")
+ print("\n[Test 1] Testing chat...")
try:
- test_text = "这是一段关于Python编程的文本。Python是一种高级编程语言，具有简单易学的语法。它被广泛用于数据分析、机器学习和Web开发。" # noqa: RUF001
+ test_text = (
+ "Python is widely used for data analysis, machine learning, and web services. "
+ "Summarize this short technical note in one sentence."
+ )
result = await client.chat(test_text)
- print("✓ Summarization successful")
+ print("[OK] Chat successful")
print(f" Result: {result[:100]}...")
- except Exception as e:
- print(f"❌ Summarization failed: {e}")
- import traceback
-
- traceback.print_exc()
+ except Exception as exc:
+ print(f"[ERROR] Chat failed: {exc}")
+ return False
- # Test 2: Embedding
print("\n[Test 2] Testing embedding...")
try:
test_texts = ["Hello world", "How are you", "Nice to meet you"]
embeddings = await client.embed(test_texts)
- print("✓ Embedding successful")
+ print("[OK] Embedding successful")
print(f" Generated {len(embeddings)} embeddings")
if embeddings and embeddings[0]:
print(f" Embedding dimension: {len(embeddings[0])}")
- except Exception as e:
- print(f"❌ Embedding failed: {e}")
- import traceback
-
- traceback.print_exc()
+ except Exception as exc:
+ print(f"[ERROR] Embedding failed: {exc}")
+ return False
- # Test 3: Vision (requires image file)
print("\n[Test 3] Testing vision...")
- test_image_path = "examples/resources/images/image1.png"
- if os.path.exists(test_image_path):
+ test_image_path = PROJECT_ROOT / "examples" / "resources" / "images" / "image1.png"
+ if test_image_path.exists():
try:
- result, _ = await client.vision(prompt="描述这张图片的内容", image_path=test_image_path)
- print("✓ Vision successful")
+ result, _ = await client.vision(prompt="Describe the image content.", image_path=str(test_image_path))
+ print("[OK] Vision successful")
print(f" Result: {result[:100]}...")
- except Exception as e:
- print(f"❌ Vision failed: {e}")
- import traceback
-
- traceback.print_exc()
+ except Exception as exc:
+ print(f"[ERROR] Vision failed: {exc}")
+ return False
else:
- print(f"⚠ Skipped: Test image not found at {test_image_path}")
+ print(f"[WARN] Skipped vision check: test image not found at {test_image_path}")
+
+ return True
+
+
+async def test_lazyllm_client() -> None:
+ """Opt-in pytest integration check for LazyLLM-backed model calls."""
+ import pytest
+
+ if os.environ.get(RUN_LAZYLLM_TESTS_ENV) != "1":
+ pytest.skip(f"Set {RUN_LAZYLLM_TESTS_ENV}=1 to run the LazyLLM integration workflow")
+ if not os.environ.get("MEMU_QWEN_API_KEY"):
+ pytest.skip("MEMU_QWEN_API_KEY is required for the LazyLLM integration workflow")
+
+ assert await run_lazyllm_workflow()
+
+
+def main() -> int:
+ return 0 if asyncio.run(run_lazyllm_workflow()) else 1
if __name__ == "__main__":
- success = asyncio.run(test_lazyllm_client())
- sys.exit(0 if success else 1)
+ raise SystemExit(main())
diff --git a/tests/test_lazyllm_optional.py b/tests/test_lazyllm_optional.py
new file mode 100644
index 00000000..410b164d
--- /dev/null
+++ b/tests/test_lazyllm_optional.py
@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from memu.llm import lazyllm_client
+
+
+def test_lazyllm_client_module_imports_without_optional_dependency() -> None:
+ assert lazyllm_client.LazyLLMClient.__name__ == "LazyLLMClient"
+
+
+def test_lazyllm_missing_dependency_error_is_actionable() -> None:
+ if lazyllm_client._LAZYLLM_IMPORT_ERROR is None:
+ return
+
+ try:
+ lazyllm_client.LazyLLMClient()
+ except ImportError as exc:
+ message = str(exc)
+ assert "memu-py[lazyllm]" in message
+ assert "uv sync --extra lazyllm" in message
+ else:
+ raise AssertionError("missing LazyLLM dependency should raise an actionable ImportError")
diff --git a/tests/test_llm_observability.py b/tests/test_llm_observability.py
new file mode 100644
index 00000000..f61661b9
--- /dev/null
+++ b/tests/test_llm_observability.py
@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import ast
+import json
+from pathlib import Path
+
+from memu.llm.wrapper import _extract_usage_from_raw_response
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_llm_usage_aggregates_batched_embedding_raw_responses() -> None:
+ raw_responses = [
+ {
+ "usage": {
+ "prompt_tokens": 3,
+ "total_tokens": 3,
+ "prompt_tokens_details": {"cached_tokens": 1},
+ }
+ },
+ {
+ "usage": {
+ "prompt_tokens": 5,
+ "total_tokens": 5,
+ "prompt_tokens_details": {"cached_tokens": 2},
+ }
+ },
+ ]
+
+ usage = _extract_usage_from_raw_response(kind="embed", raw_response=raw_responses)
+
+ assert usage["input_tokens"] == 8
+ assert usage["total_tokens"] == 8
+ assert usage["cached_input_tokens"] == 3
+ json.dumps(usage)
+
+
+def test_openai_sdk_batched_embed_returns_all_raw_responses_for_usage() -> None:
+ source = (ROOT / "src/memu/llm/openai_sdk.py").read_text(encoding="utf-8")
+ function_source = _function_source(source, "embed")
+
+ assert "raw_responses.append(response)" in function_source
+ assert "return all_embeddings, raw_responses" in function_source
+ assert "last_response" not in function_source
+
+
+def _function_source(source: str, name: str) -> str:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.AsyncFunctionDef | ast.FunctionDef) and node.name == name:
+ segment = ast.get_source_segment(source, node)
+ assert segment is not None
+ return segment
+ raise AssertionError(f"function {name!r} not found")
diff --git a/tests/test_memorize_dedupe.py b/tests/test_memorize_dedupe.py
new file mode 100644
index 00000000..c4607c1f
--- /dev/null
+++ b/tests/test_memorize_dedupe.py
@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from memu.utils.dedupe import dedupe_resource_plans
+
+
+def test_dedupe_resource_plans_removes_duplicate_entries_across_segments() -> None:
+ plans = [
+ {
+ "resource_url": "conversation_#segment_0.json",
+ "entries": [
+ ("profile", "User likes coffee.", ["preferences"]),
+ ("knowledge", "User works in AI.", ["work_life"]),
+ ],
+ },
+ {
+ "resource_url": "conversation_#segment_1.json",
+ "entries": [
+ ("profile", " user likes coffee. ", ["preferences", "habits"]),
+ ("event", "User joined a hackathon.", ["experiences"]),
+ ],
+ },
+ ]
+
+ deduped = dedupe_resource_plans(plans)
+
+ assert [entry[1] for plan in deduped for entry in plan["entries"]] == [
+ "User likes coffee.",
+ "User works in AI.",
+ "User joined a hackathon.",
+ ]
+ assert deduped[0]["entries"][0][2] == ["preferences", "habits"]
+ assert deduped[1]["entries"] == [("event", "User joined a hackathon.", ["experiences"])]
+
+
+def test_dedupe_resource_plans_keeps_same_content_for_different_memory_types() -> None:
+ plans = [
+ {
+ "resource_url": "notes.txt",
+ "entries": [
+ ("profile", "Uses Python daily.", ["preferences"]),
+ ("skill", "Uses Python daily.", ["knowledge"]),
+ ],
+ }
+ ]
+
+ deduped = dedupe_resource_plans(plans)
+
+ assert deduped[0]["entries"] == [
+ ("profile", "Uses Python daily.", ["preferences"]),
+ ("skill", "Uses Python daily.", ["knowledge"]),
+ ]
+
+
+def test_dedupe_resource_plans_drops_empty_and_malformed_entries() -> None:
+ plans = [
+ {
+ "resource_url": "notes.txt",
+ "entries": [
+ ("profile", " ", ["preferences"]),
+ ("profile", "Valid memory.", ["preferences", "Preferences", ""]),
+ ["profile", "not a tuple", []],
+ ],
+ }
+ ]
+
+ deduped = dedupe_resource_plans(plans)
+
+ assert deduped[0]["entries"] == [("profile", "Valid memory.", ["preferences"])]
diff --git a/tests/test_openrouter.py b/tests/test_openrouter.py
index ba4b47c4..8908613b 100644
--- a/tests/test_openrouter.py
+++ b/tests/test_openrouter.py
@@ -1,99 +1,44 @@
+#!/usr/bin/env python3
"""
-Test OpenRouter integration with MemU's full workflow.
-
-Tests:
-1. Conversation memorization using OpenRouter
-2. RAG-based retrieval using OpenRouter embeddings
-3. LLM-based retrieval using OpenRouter
+Opt-in OpenRouter integration smoke test.
Usage:
+ export OPENROUTER_API_KEY=your_api_key
+ export MEMU_RUN_OPENROUTER_TESTS=1
+ uv run python -m pytest tests/test_openrouter.py
+
+Manual run:
export OPENROUTER_API_KEY=your_api_key
python tests/test_openrouter.py
"""
+from __future__ import annotations
+
import asyncio
-import json
import os
import sys
+from pathlib import Path
from typing import Any
-import pytest
-
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "src")))
-
-from memu.app import MemoryService
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+RUN_OPENROUTER_TESTS_ENV = "MEMU_RUN_OPENROUTER_TESTS"
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(PROJECT_ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
-def _print_categories(categories, max_items=3):
- """Print category summaries."""
- if categories:
- print(" Categories:")
- for cat in categories[:max_items]:
- summary = cat.get("summary") or cat.get("description", "")
- print(f" - {cat.get('name')}: {summary[:60]}...")
+async def run_openrouter_workflow() -> None:
+ """Run the OpenRouter-backed memorize/retrieve smoke workflow."""
+ from memu import MemoryService
-def _print_items(items, max_items=3):
- """Print memory item summaries."""
- if items:
- print(" Items:")
- for item in items[:max_items]:
- memory_type = item.get("memory_type", "unknown")
- summary = item.get("summary", "")[:80]
- print(f" - [{memory_type}] {summary}...")
-
-
-async def _test_memorize(service, file_path, output_data):
- """Test conversation memorization."""
- print("\n[OPENROUTER] Test 1: Memorizing conversation...")
- memory = await service.memorize(
- resource_url=file_path, modality="conversation", user={"user_id": "openrouter_test_user"}
- )
- items_count = len(memory.get("items", []))
- categories_count = len(memory.get("categories", []))
-
- print(f" Memorized {items_count} items")
- print(f" Created {categories_count} categories")
-
- output_data["memorize"] = memory
-
- assert items_count > 0, "Expected at least 1 memory item"
- assert categories_count > 0, "Expected at least 1 category"
-
- _print_categories(memory.get("categories", []))
- return memory
-
-
-async def _test_retrieve(service, queries, method, test_num, output_data):
- """Test retrieval with specified method."""
- print(f"\n[OPENROUTER] Test {test_num}: {method.upper()}-based retrieval...")
- service.retrieve_config.method = method
- result = await service.retrieve(queries=queries, where={"user_id": "openrouter_test_user"})
-
- categories_retrieved = len(result.get("categories", []))
- items_retrieved = len(result.get("items", []))
-
- print(f" Retrieved {categories_retrieved} categories")
- print(f" Retrieved {items_retrieved} items")
-
- output_data[f"retrieve_{method}"] = result
-
- _print_categories(result.get("categories", []))
- _print_items(result.get("items", []))
- return result
-
-
-async def test_openrouter_full_workflow():
- """Test OpenRouter integration with full MemU workflow."""
api_key = os.environ.get("OPENROUTER_API_KEY")
if not api_key:
- pytest.skip("OPENROUTER_API_KEY environment variable not set")
-
- file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "example", "example_conversation.json"))
- if not os.path.exists(file_path):
- pytest.skip(f"Test file not found: {file_path}")
+ msg = "OPENROUTER_API_KEY is required for the OpenRouter integration workflow"
+ raise RuntimeError(msg)
- output_data: dict[str, Any] = {}
+ file_path = Path(__file__).resolve().parent / "example" / "example_conversation.json"
print("\n" + "=" * 60)
print("[OPENROUTER] Starting full workflow test...")
@@ -119,15 +64,15 @@ async def test_openrouter_full_workflow():
},
)
+ output_data: dict[str, Any] = {}
queries = [
{"role": "user", "content": {"text": "What foods does the user like to eat?"}},
]
- await _test_memorize(service, file_path, output_data)
+ await _test_memorize(service, str(file_path), output_data)
await _test_retrieve(service, queries, "rag", 2, output_data)
await _test_retrieve(service, queries, "llm", 3, output_data)
- # Test 4: List memory items
print("\n[OPENROUTER] Test 4: List memory items...")
items_result = await service.list_memory_items(where={"user_id": "openrouter_test_user"})
items_list = items_result.get("items", [])
@@ -135,7 +80,6 @@ async def test_openrouter_full_workflow():
output_data["list_items"] = items_result
assert len(items_list) > 0, "Expected at least 1 item in list"
- # Test 5: List memory categories
print("\n[OPENROUTER] Test 5: List memory categories...")
cats_result = await service.list_memory_categories(where={"user_id": "openrouter_test_user"})
cats_list = cats_result.get("categories", [])
@@ -143,19 +87,90 @@ async def test_openrouter_full_workflow():
output_data["list_categories"] = cats_result
assert len(cats_list) > 0, "Expected at least 1 category in list"
- # Save output to file
- output_file = os.path.abspath(
- os.path.join(os.path.dirname(__file__), "..", "examples", "output", "openrouter_test_output.json")
- )
- os.makedirs(os.path.dirname(output_file), exist_ok=True)
- with open(output_file, "w", encoding="utf-8") as f:
- json.dump(output_data, f, indent=2, default=str)
- print(f"\n[OPENROUTER] Output saved to: {output_file}")
-
print("\n" + "=" * 60)
print("[OPENROUTER] All tests completed!")
print("=" * 60)
+async def test_openrouter_full_workflow() -> None:
+ """Opt-in pytest integration check for OpenRouter-backed model calls."""
+ import pytest
+
+ if os.environ.get(RUN_OPENROUTER_TESTS_ENV) != "1":
+ pytest.skip(f"Set {RUN_OPENROUTER_TESTS_ENV}=1 to run the OpenRouter integration workflow")
+ if not os.environ.get("OPENROUTER_API_KEY"):
+ pytest.skip("OPENROUTER_API_KEY is required for the OpenRouter integration workflow")
+
+ await run_openrouter_workflow()
+
+
+async def _test_memorize(service: Any, file_path: str, output_data: dict[str, Any]) -> None:
+ print("\n[OPENROUTER] Test 1: Memorizing conversation...")
+ memory = await service.memorize(
+ resource_url=file_path,
+ modality="conversation",
+ user={"user_id": "openrouter_test_user"},
+ )
+ items_count = len(memory.get("items", []))
+ categories_count = len(memory.get("categories", []))
+
+ print(f" Memorized {items_count} items")
+ print(f" Created {categories_count} categories")
+
+ output_data["memorize"] = memory
+
+ assert items_count > 0, "Expected at least 1 memory item"
+ assert categories_count > 0, "Expected at least 1 category"
+
+ _print_categories(memory.get("categories", []))
+
+
+async def _test_retrieve(
+ service: Any,
+ queries: list[dict[str, Any]],
+ method: str,
+ test_num: int,
+ output_data: dict[str, Any],
+) -> None:
+ print(f"\n[OPENROUTER] Test {test_num}: {method.upper()}-based retrieval...")
+ service.retrieve_config.method = method
+ result = await service.retrieve(queries=queries, where={"user_id": "openrouter_test_user"})
+
+ categories_retrieved = len(result.get("categories", []))
+ items_retrieved = len(result.get("items", []))
+
+ print(f" Retrieved {categories_retrieved} categories")
+ print(f" Retrieved {items_retrieved} items")
+
+ output_data[f"retrieve_{method}"] = result
+
+ _print_categories(result.get("categories", []))
+ _print_items(result.get("items", []))
+
+
+def _print_categories(categories: list[dict[str, Any]], max_items: int = 3) -> None:
+ if not categories:
+ return
+ print(" Categories:")
+ for cat in categories[:max_items]:
+ summary = cat.get("summary") or cat.get("description", "")
+ print(f" - {cat.get('name')}: {summary[:60]}...")
+
+
+def _print_items(items: list[dict[str, Any]], max_items: int = 3) -> None:
+ if not items:
+ return
+ print(" Items:")
+ for item in items[:max_items]:
+ memory_type = item.get("memory_type", "unknown")
+ summary = item.get("summary", "")[:80]
+ print(f" - [{memory_type}] {summary}...")
+
+
+def main() -> int:
+ asyncio.run(run_openrouter_workflow())
+ return 0
+
+
if __name__ == "__main__":
- asyncio.run(test_openrouter_full_workflow())
+ raise SystemExit(main())
diff --git a/tests/test_postgres.py b/tests/test_postgres.py
index 8b375a30..20bc7fe0 100644
--- a/tests/test_postgres.py
+++ b/tests/test_postgres.py
@@ -1,14 +1,32 @@
+from __future__ import annotations
+
+import asyncio
import os
+import sys
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_POSTGRES_DSN = "postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu"
+RUN_POSTGRES_TESTS_ENV = "MEMU_RUN_POSTGRES_TESTS"
+
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(PROJECT_ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
-from memu.app import MemoryService
+async def run_postgres_workflow() -> None:
+ """Run the PostgreSQL-backed memorize/retrieve smoke workflow."""
+ from memu import MemoryService
-async def main():
- """Test with PostgreSQL storage."""
api_key = os.environ.get("OPENAI_API_KEY")
+ if not api_key:
+ msg = "OPENAI_API_KEY is required for the PostgreSQL integration workflow"
+ raise RuntimeError(msg)
+
# Default port 5432; use 5433 if 5432 is occupied
- postgres_dsn = os.environ.get("POSTGRES_DSN", "postgresql+psycopg://postgres:postgres@127.0.0.1:5432/memu")
- file_path = os.path.abspath("tests/example/example_conversation.json")
+ postgres_dsn = os.environ.get("POSTGRES_DSN", DEFAULT_POSTGRES_DSN)
+ file_path = Path(__file__).resolve().parent / "example" / "example_conversation.json"
print("\n" + "=" * 60)
print("[POSTGRES] Starting test...")
@@ -30,7 +48,7 @@ async def main():
# Memorize
print("\n[POSTGRES] Memorizing...")
- memory = await service.memorize(resource_url=file_path, modality="conversation", user={"user_id": "123"})
+ memory = await service.memorize(resource_url=str(file_path), modality="conversation", user={"user_id": "123"})
for cat in memory.get("categories", []):
print(f" - {cat.get('name')}: {(cat.get('summary') or '')[:80]}...")
@@ -76,7 +94,22 @@ async def main():
print("\n[POSTGRES] Test completed!")
-if __name__ == "__main__":
- import asyncio
+async def test_postgres_full_workflow() -> None:
+ """Opt-in pytest integration check for a local PostgreSQL + pgvector service."""
+ import pytest
+
+ if os.environ.get(RUN_POSTGRES_TESTS_ENV) != "1":
+ pytest.skip(f"Set {RUN_POSTGRES_TESTS_ENV}=1 to run the PostgreSQL integration workflow")
+ if not os.environ.get("OPENAI_API_KEY"):
+ pytest.skip("OPENAI_API_KEY is required for the PostgreSQL integration workflow")
- asyncio.run(main())
+ await run_postgres_workflow()
+
+
+def main() -> int:
+ asyncio.run(run_postgres_workflow())
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/test_postgres_repository_source.py b/tests/test_postgres_repository_source.py
new file mode 100644
index 00000000..54991866
--- /dev/null
+++ b/tests/test_postgres_repository_source.py
@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+MEMORY_ITEM_REPO = ROOT / "src" / "memu" / "database" / "postgres" / "repositories" / "memory_item_repo.py"
+MEMORY_CATEGORY_REPO = (
+ ROOT / "src" / "memu" / "database" / "postgres" / "repositories" / "memory_category_repo.py"
+)
+RESOURCE_REPO = ROOT / "src" / "memu" / "database" / "postgres" / "repositories" / "resource_repo.py"
+
+
+def _function_node(module: ast.Module, name: str) -> ast.FunctionDef:
+ for node in ast.walk(module):
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ return node
+ raise AssertionError(f"missing function {name}")
+
+
+def _calls_method(node: ast.AST, method_name: str) -> bool:
+ return any(
+ isinstance(child, ast.Call)
+ and isinstance(child.func, ast.Attribute)
+ and child.func.attr == method_name
+ and isinstance(child.func.value, ast.Name)
+ and child.func.value.id == "self"
+ for child in ast.walk(node)
+ )
+
+
+def test_postgres_local_vector_search_queries_current_items() -> None:
+ module = ast.parse(MEMORY_ITEM_REPO.read_text(encoding="utf-8"), filename=str(MEMORY_ITEM_REPO))
+ vector_search_fn = _function_node(module, "vector_search_items")
+ local_search_fn = _function_node(module, "_vector_search_local")
+
+ assert any(
+ isinstance(node, ast.Compare)
+ and isinstance(node.left, ast.Name)
+ and node.left.id == "top_k"
+ and any(isinstance(op, ast.LtE) for op in node.ops)
+ for node in ast.walk(vector_search_fn)
+ )
+ assert _calls_method(vector_search_fn, "_vector_search_local")
+ assert _calls_method(local_search_fn, "list_items")
+
+
+def test_postgres_cascade_delete_paths_prune_relation_cache() -> None:
+ item_source = MEMORY_ITEM_REPO.read_text(encoding="utf-8")
+ category_source = MEMORY_CATEGORY_REPO.read_text(encoding="utf-8")
+ resource_source = RESOURCE_REPO.read_text(encoding="utf-8")
+
+ assert "_drop_relation_cache_for_items(deleted_item_ids)" in item_source
+ assert "_drop_relation_cache_for_items({item_id})" in item_source
+ assert "rel.item_id not in item_ids" in item_source
+
+ assert "deleted_category_ids = set(deleted)" in category_source
+ assert "rel.category_id not in deleted_category_ids" in category_source
+
+ assert "deleted_resource_ids = set(deleted)" in resource_source
+ assert "item.resource_id in deleted_resource_ids" in resource_source
+ assert "rel.item_id not in deleted_item_ids" in resource_source
diff --git a/tests/test_repository_docs.py b/tests/test_repository_docs.py
new file mode 100644
index 00000000..cf551ede
--- /dev/null
+++ b/tests/test_repository_docs.py
@@ -0,0 +1,1592 @@
+from __future__ import annotations
+
+import ast
+import importlib.util
+import re
+import sys
+import tomllib
+from pathlib import Path
+from unittest.mock import patch
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SUPPORTED_MEMORIZE_MODALITIES = ("conversation", "document", "image", "audio", "video")
+SUPPORTED_MEMORY_TYPES = ("profile", "event", "knowledge", "behavior", "skill", "tool")
+SUPPORTED_CLIENT_BACKENDS = {"httpx", "lazyllm_backend", "sdk"}
+MIN_PYTHON_VERSION = "3.12"
+KEYWORD_ONLY_EXAMPLE_APIS = {
+ "clear_memory",
+ "create_memory_item",
+ "delete_memory_item",
+ "list_memory_categories",
+ "list_memory_items",
+ "memorize",
+ "retrieve",
+ "update_memory_item",
+}
+
+
+def test_contributing_make_targets_exist() -> None:
+ makefile = (ROOT / "Makefile").read_text(encoding="utf-8")
+ docs = [
+ ROOT / "README.md",
+ ROOT / "CONTRIBUTING.md",
+ ]
+
+ targets = {
+ match.group(1)
+ for match in re.finditer(r"^([A-Za-z0-9_.-]+):(?:\s|$)", makefile, flags=re.MULTILINE)
+ }
+ referenced_targets = {
+ match.group(1)
+ for doc in docs
+ for match in re.finditer(r"^\s*make\s+([A-Za-z0-9_.-]+)", doc.read_text(encoding="utf-8"), flags=re.MULTILINE)
+ }
+
+ missing_targets = sorted(referenced_targets - targets)
+ assert not missing_targets, f"Public docs reference unknown make target(s): {missing_targets}"
+
+
+def test_makefile_status_output_is_ascii_safe() -> None:
+ makefile = (ROOT / "Makefile").read_text(encoding="utf-8")
+ unsafe_lines: list[str] = []
+
+ for line_no, line in enumerate(makefile.splitlines(), 1):
+ if "@echo" not in line:
+ continue
+ try:
+ line.encode("ascii")
+ except UnicodeEncodeError:
+ unsafe_lines.append(f"Makefile:{line_no}: {line.strip()}")
+
+ assert not unsafe_lines, f"Makefile status output should be ASCII-safe: {unsafe_lines}"
+
+
+def test_non_cli_source_modules_do_not_print_to_stdout() -> None:
+ stdout_calls: list[str] = []
+
+ for path in sorted((ROOT / "src" / "memu").rglob("*.py")):
+ if path.name == "cli.py" or path.stem.endswith("_cli"):
+ continue
+ module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+ for node in ast.walk(module):
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "print":
+ stdout_calls.append(f"{path.relative_to(ROOT)}:{node.lineno}")
+
+ assert not stdout_calls, f"Library modules should use logging instead of print(): {stdout_calls}"
+
+
+def test_pre_commit_hooks_include_safety_checks() -> None:
+ config = (ROOT / ".pre-commit-config.yaml").read_text(encoding="utf-8")
+
+ assert "https://github.com/pre-commit/pre-commit-hooks" in config
+ assert "https://github.com/astral-sh/ruff-pre-commit" in config
+ assert "id: check-added-large-files" in config
+ assert "args: [--maxkb=1024]" in config
+ assert "id: detect-private-key" in config
+ assert "id: mixed-line-ending" in config
+ assert "args: [--fix=lf]" in config
+ assert "id: trailing-whitespace" in config
+ assert "id: end-of-file-fixer" in config
+
+
+def test_issue_templates_route_security_reports_privately() -> None:
+ bug_report = (ROOT / ".github" / "ISSUE_TEMPLATE" / "bug_report.yml").read_text(encoding="utf-8")
+ issue_config = (ROOT / ".github" / "ISSUE_TEMPLATE" / "config.yml").read_text(encoding="utf-8")
+
+ assert "do not open a public issue" in bug_report
+ assert "vulnerability" in bug_report
+ assert "leaked credential" in bug_report
+ assert "private user data" in bug_report
+ assert "https://github.com/NevaMind-AI/MemU/security/policy" in bug_report
+ assert "Security Reports" in issue_config
+ assert "https://github.com/NevaMind-AI/MemU/security/policy" in issue_config
+
+
+def test_pull_request_template_matches_project_gates() -> None:
+ template = (ROOT / ".github" / "PULL_REQUEST_TEMPLATE.md").read_text(encoding="utf-8")
+ pr_title_workflow = (ROOT / ".github" / "workflows" / "pr-title.yml").read_text(encoding="utf-8")
+
+ template.encode("ascii")
+ assert "`make check`" in template
+ assert "`make test`" in template
+ assert "`make docs-build`" in template
+ assert "Security or privacy impact" in template
+ assert "Public API impact" in template
+ assert "Storage/backend impact" in template
+ assert "No secrets, credentials, or private user data" in template
+ for prefix in ["feat", "fix", "docs", "test", "refactor", "perf", "style", "ci", "build", "chore", "revert"]:
+ assert f"`{prefix}`" in template
+ assert f" {prefix}" in pr_title_workflow
+
+
+def test_repository_markdown_local_links_exist() -> None:
+ docs = [
+ ROOT / "README.md",
+ ROOT / "CONTRIBUTING.md",
+ ROOT / "CODE_OF_CONDUCT.md",
+ ROOT / "SECURITY.md",
+ ROOT / "SUPPORT.md",
+ ROOT / ".github" / "PULL_REQUEST_TEMPLATE.md",
+ *sorted((ROOT / "readme").rglob("*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.md")),
+ ]
+ missing_links: list[str] = []
+
+ for doc in docs:
+ text = doc.read_text(encoding="utf-8")
+ for raw_link in _iter_local_reference_targets(text):
+ target = raw_link.split("#", 1)[0].strip()
+ if not target or _is_external_link(target):
+ continue
+
+ target_path = (doc.parent / target).resolve()
+ if not _is_inside_root(target_path) or not target_path.exists():
+ missing_links.append(f"{doc.relative_to(ROOT)} -> {raw_link}")
+
+ assert not missing_links, f"Markdown local links should resolve: {missing_links}"
+
+
+def test_readme_landing_page_assets_and_language_nav_are_stable() -> None:
+ readme = (ROOT / "README.md").read_text(encoding="utf-8")
+ architecture = (ROOT / "docs" / "architecture.md").read_text(encoding="utf-8")
+ expected_language_links = {
+ "English": "readme/README_en.md",
+ "中文": "readme/README_zh.md",
+ "日本語": "readme/README_ja.md",
+ "한국어": "readme/README_ko.md",
+ "Español": "readme/README_es.md",
+ "Français": "readme/README_fr.md",
+ }
+ expected_diagrams = [
+ "assets/memu-overall-engineering-architecture.png",
+ "assets/memu-overall-algorithm-flow.png",
+ "assets/memu-self-evolve-architecture.png",
+ "assets/memu-self-evolve-algorithm.png",
+ ]
+
+ for label, target in expected_language_links.items():
+ assert f"[{label}]({target})" in readme
+ assert (ROOT / target).exists()
+
+ for target in expected_diagrams:
+ asset = ROOT / target
+ assert asset.exists()
+ assert asset.stat().st_size > 50_000
+ assert asset.read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
+ assert target in readme
+ assert f"../{target}" in architecture
+
+
+def test_open_source_governance_files_are_present() -> None:
+ security = (ROOT / "SECURITY.md").read_text(encoding="utf-8")
+ support = (ROOT / "SUPPORT.md").read_text(encoding="utf-8")
+ readme = (ROOT / "README.md").read_text(encoding="utf-8")
+ contributing = (ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8")
+ issue_config = (ROOT / ".github" / "ISSUE_TEMPLATE" / "config.yml").read_text(encoding="utf-8")
+
+ assert "contact@nevamind.ai" in security
+ assert "Do not open a public GitHub issue" in security
+ assert "Supported Versions" in security
+ assert "GitHub Discussions" in support
+ assert "Security Policy" in support
+ assert "](SECURITY.md)" in readme
+ assert "](SUPPORT.md)" in readme
+ assert "[Security Policy](SECURITY.md)" in contributing
+ assert "Security Reports" in issue_config
+ assert "https://github.com/NevaMind-AI/MemU/security/policy" in issue_config
+
+
+def test_public_community_links_are_canonical() -> None:
+ core_repository_link_files = _public_markdown_docs()
+ discord_link_files = [
+ ROOT / "README.md",
+ ROOT / "CONTRIBUTING.md",
+ ROOT / "SUPPORT.md",
+ ROOT / ".github" / "ISSUE_TEMPLATE" / "config.yml",
+ ]
+ stale_links: list[str] = []
+
+ for path in core_repository_link_files:
+ text = path.read_text(encoding="utf-8")
+ if re.search(r"github\.com/NevaMind-AI/memU(?=$|[/?#)])", text):
+ stale_links.append(f"{path.relative_to(ROOT)} uses non-canonical repository casing")
+
+ for path in discord_link_files:
+ text = path.read_text(encoding="utf-8")
+ if "discord.gg/memu" in text:
+ stale_links.append(f"{path.relative_to(ROOT)} uses stale Discord invite")
+
+ assert not stale_links, f"Use canonical community links: {stale_links}"
+
+
+def test_project_package_metadata_declares_license() -> None:
+ project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]
+
+ assert project["license"] == {"file": "LICENSE.txt"}
+ assert (ROOT / project["license"]["file"]).exists()
+ assert "License :: OSI Approved :: Apache Software License" in project["classifiers"]
+
+
+def test_uv_lock_tracks_project_version() -> None:
+ project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]
+ lock = tomllib.loads((ROOT / "uv.lock").read_text(encoding="utf-8"))
+ locked_project = next(package for package in lock["package"] if package["name"] == project["name"])
+
+ assert locked_project["source"] == {"editable": "."}
+ assert locked_project["version"] == project["version"]
+
+
+def test_public_version_constant_tracks_project_version() -> None:
+ project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]
+ module = ast.parse((ROOT / "src" / "memu" / "_version.py").read_text(encoding="utf-8"))
+ version = next(
+ node.value.value
+ for node in module.body
+ if isinstance(node, ast.Assign)
+ for target in node.targets
+ if isinstance(target, ast.Name)
+ and target.id == "__version__"
+ and isinstance(node.value, ast.Constant)
+ and isinstance(node.value.value, str)
+ )
+
+ assert version == project["version"]
+
+
+def test_langgraph_dependencies_are_optional_and_current() -> None:
+ pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
+ project = pyproject["project"]
+ dependencies = set(project["dependencies"])
+ langgraph_extra = set(project["optional-dependencies"]["langgraph"])
+ lock = tomllib.loads((ROOT / "uv.lock").read_text(encoding="utf-8"))
+ locked_project = next(package for package in lock["package"] if package["name"] == "memu-py")
+ locked_dependencies = {dependency["name"] for dependency in locked_project["dependencies"]}
+ locked_requirements = locked_project["metadata"]["requires-dist"]
+
+ assert "langchain-core>=1.2.7" not in dependencies
+ assert not any(dep.startswith("langgraph") for dep in dependencies)
+ assert langgraph_extra == {"langgraph>=1.0.6", "langchain-core>=1.2.7"}
+ assert "langchain-core" not in locked_dependencies
+ assert _requirement_specifier(locked_requirements, "langchain-core", extra="langgraph") == ">=1.2.7"
+ assert _requirement_specifier(locked_requirements, "langgraph", extra="langgraph") == ">=1.0.6"
+
+
+def test_langgraph_example_has_actionable_optional_dependency_guard() -> None:
+ example = (ROOT / "examples" / "langgraph_demo.py").read_text(encoding="utf-8")
+
+ assert "INSTALL_HINT" in example
+ assert "Missing LangGraph dependencies" in example
+ assert "pip install 'memu-py[langgraph]'" in example
+ assert "uv sync --extra langgraph" in example
+ assert "except ModuleNotFoundError as exc" in example
+ assert 'if exc.name not in {"langgraph", "langchain_core"}' in example
+ assert "raise" in example
+
+
+def test_langgraph_docs_document_optional_extra_install_paths() -> None:
+ docs = (ROOT / "docs" / "langgraph_integration.md").read_text(encoding="utf-8")
+
+ assert 'pip install "memu-py[langgraph]"' in docs
+ assert 'uv add "memu-py[langgraph]"' in docs
+ assert "uv sync --extra langgraph" in docs
+
+
+def test_public_docs_use_current_database_environment_names() -> None:
+ docs = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.txt")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ ]
+ legacy_refs: list[str] = []
+
+ for path in docs:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if "MEMU_DATABASE_URL" in line:
+ legacy_refs.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ assert not legacy_refs, f"Use MEMU_DATABASE_DSN instead of legacy MEMU_DATABASE_URL: {legacy_refs}"
+
+
+def test_public_docs_use_current_provider_environment_names() -> None:
+ docs = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.txt")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ ]
+ legacy_refs: list[str] = []
+
+ for path in docs:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if "GROK_API_KEY" in line:
+ legacy_refs.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ assert not legacy_refs, f"Use XAI_API_KEY for Grok provider defaults: {legacy_refs}"
+
+
+def test_public_python_config_docs_reference_api_key_environment_names() -> None:
+ docs = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.txt")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ *sorted((ROOT / "src").rglob("*.py")),
+ ]
+ forbidden_fragments = [
+ '"api_key": "your_api_key"',
+ '"api_key": "your-api-key"',
+ '"api_key": "your_voyage_api_key"',
+ '"api_key": "your_openrouter_api_key"',
+ 'api_key="your_api_key"',
+ 'api_key="your-api-key"',
+ 'api_key="your_voyage_api_key"',
+ 'api_key="your_openrouter_api_key"',
+ "'api_key': 'your_api_key'",
+ "'api_key': 'your-api-key'",
+ "'api_key': 'your_voyage_api_key'",
+ "'api_key': 'your_openrouter_api_key'",
+ "api_key='your_api_key'",
+ "api_key='your-api-key'",
+ "api_key='your_voyage_api_key'",
+ "api_key='your_openrouter_api_key'",
+ ]
+ stale_refs: list[str] = []
+
+ for path in docs:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ for fragment in forbidden_fragments:
+ if fragment in line:
+ stale_refs.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ assert not stale_refs, f"Use environment variable names in Python api_key config examples: {stale_refs}"
+
+ for readme_path in [ROOT / "README.md", *sorted((ROOT / "readme").glob("README_*.md"))]:
+ readme = readme_path.read_text(encoding="utf-8")
+ assert '"api_key": "MEMU_QWEN_API_KEY"' in readme, readme_path.name
+ assert '"api_key": "VOYAGE_API_KEY"' in readme, readme_path.name
+ assert '"api_key": "OPENROUTER_API_KEY"' in readme, readme_path.name
+
+
+def test_lazyllm_dependencies_are_optional_and_current() -> None:
+ pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
+ project = pyproject["project"]
+ dependencies = set(project["dependencies"])
+ lazyllm_extra = set(project["optional-dependencies"]["lazyllm"])
+ lock = tomllib.loads((ROOT / "uv.lock").read_text(encoding="utf-8"))
+ locked_project = next(package for package in lock["package"] if package["name"] == "memu-py")
+ locked_dependencies = {dependency["name"] for dependency in locked_project["dependencies"]}
+ locked_requirements = locked_project["metadata"]["requires-dist"]
+
+ assert "lazyllm>=0.7.3" not in dependencies
+ assert lazyllm_extra == {"lazyllm>=0.7.3"}
+ assert "lazyllm" not in locked_dependencies
+ assert _requirement_specifier(locked_requirements, "lazyllm", extra="lazyllm") == ">=0.7.3"
+
+
+def test_claude_dependencies_are_optional_and_current() -> None:
+ pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
+ project = pyproject["project"]
+ dependencies = set(project["dependencies"])
+ claude_extra = set(project["optional-dependencies"]["claude"])
+ lock = tomllib.loads((ROOT / "uv.lock").read_text(encoding="utf-8"))
+ locked_project = next(package for package in lock["package"] if package["name"] == "memu-py")
+ locked_dependencies = {dependency["name"] for dependency in locked_project["dependencies"]}
+ locked_requirements = locked_project["metadata"]["requires-dist"]
+ proactive_sources = "\n".join(
+ path.read_text(encoding="utf-8") for path in sorted((ROOT / "examples" / "proactive").rglob("*.py"))
+ )
+
+ assert "claude-agent-sdk>=0.1.24" not in dependencies
+ assert claude_extra == {"claude-agent-sdk>=0.1.24"}
+ assert "claude-agent-sdk" not in locked_dependencies
+ assert _requirement_specifier(locked_requirements, "claude-agent-sdk", extra="claude") == ">=0.1.24"
+ assert "aiohttp" not in proactive_sources
+ assert "dict[str, any]" not in proactive_sources
+ assert "your memu api key" not in proactive_sources
+
+
+def test_proactive_claude_example_has_actionable_optional_dependency_guard() -> None:
+ claude_sdk = (ROOT / "examples" / "proactive" / "memory" / "claude_sdk.py").read_text(encoding="utf-8")
+
+ assert "INSTALL_HINT" in claude_sdk
+ assert "The proactive Claude example requires claude-agent-sdk" in claude_sdk
+ assert "pip install 'memu-py[claude]'" in claude_sdk
+ assert "uv sync --extra claude" in claude_sdk
+ assert "except ModuleNotFoundError as exc" in claude_sdk
+ assert 'if exc.name != "claude_agent_sdk"' in claude_sdk
+ assert "raise SystemExit(INSTALL_HINT) from exc" in claude_sdk
+
+
+def test_readmes_document_claude_optional_extra_for_proactive_example() -> None:
+ readmes = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ ]
+
+ for readme_path in readmes:
+ readme = readme_path.read_text(encoding="utf-8")
+ assert 'pip install "memu-py[claude]"' in readme, readme_path.name
+ assert "uv sync --extra claude" in readme, readme_path.name
+ assert 'export OPENAI_API_KEY="..."' in readme, readme_path.name
+ assert 'export ANTHROPIC_API_KEY="..."' in readme, readme_path.name
+ assert "python proactive.py" in readme, readme_path.name
+
+
+def test_postgres_optional_dependency_errors_are_actionable() -> None:
+ optional_source = (ROOT / "src" / "memu" / "database" / "postgres" / "optional.py").read_text(encoding="utf-8")
+ models_source = (ROOT / "src" / "memu" / "database" / "postgres" / "models.py").read_text(encoding="utf-8")
+ schema_source = (ROOT / "src" / "memu" / "database" / "postgres" / "schema.py").read_text(encoding="utf-8")
+
+ assert "pip install 'memu-py[postgres]'" in optional_source
+ assert "uv sync --extra postgres" in optional_source
+ assert "def postgres_extra_import_error" in optional_source
+ assert "postgres_extra_import_error() from exc" in models_source
+ assert "postgres_extra_import_error() from exc" in schema_source
+ assert "pgvector is required for Postgres vector support" not in models_source
+ assert "pgvector is required for Postgres vector support" not in schema_source
+
+
+def test_sqlite_migration_docs_explain_postgres_extra_and_dsn() -> None:
+ docs = (ROOT / "docs" / "sqlite.md").read_text(encoding="utf-8")
+ migration_section = docs.split("### Import from SQLite to PostgreSQL", 1)[1]
+
+ assert 'pip install "memu-py[postgres]"' in migration_section
+ assert "uv sync --extra postgres" in migration_section
+ assert "postgresql+psycopg://user:password@host:5432/memu" in migration_section
+ assert '"dsn": "postgresql://..."' not in migration_section
+
+
+def test_sqlite_docs_use_environment_api_key_reference() -> None:
+ docs = (ROOT / "docs" / "sqlite.md").read_text(encoding="utf-8")
+
+ assert '"api_key": "your-api-key"' not in docs
+ assert "Set `OPENAI_API_KEY` in your environment" in docs
+ assert docs.count('"api_key": "OPENAI_API_KEY"') >= 5
+ assert "not a literal secret to paste into source code" in docs
+
+
+def test_proactive_platform_config_uses_environment() -> None:
+ module_path = ROOT / "examples" / "proactive" / "memory" / "platform" / "common.py"
+ spec = importlib.util.spec_from_file_location("proactive_platform_common", module_path)
+ assert spec is not None
+ assert spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ try:
+ spec.loader.exec_module(module)
+ finally:
+ sys.modules.pop(spec.name, None)
+
+ with patch.dict(
+ "os.environ",
+ {
+ "MEMU_API_KEY": " platform-key ",
+ "MEMU_BASE_URL": " https://example.memu.test/ ",
+ "MEMU_USER_ID": " user-1 ",
+ "MEMU_AGENT_ID": " agent-1 ",
+ },
+ clear=True,
+ ):
+ config = module.get_platform_memory_config()
+
+ assert config.api_key == "platform-key"
+ assert config.base_url == "https://example.memu.test"
+ assert config.user_id == "user-1"
+ assert config.agent_id == "agent-1"
+
+ with patch.dict("os.environ", {}, clear=True):
+ try:
+ module.get_platform_memory_config()
+ except ValueError as exc:
+ assert "MEMU_API_KEY" in str(exc)
+ else:
+ raise AssertionError("MEMU_API_KEY should be required for platform proactive memory")
+
+
+def test_package_declares_pep561_typing_support() -> None:
+ pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
+ project = pyproject["project"]
+ maturin = pyproject["tool"]["maturin"]
+ manifest = (ROOT / "MANIFEST.in").read_text(encoding="utf-8")
+
+ assert (ROOT / "src" / "memu" / "py.typed").exists()
+ assert "Typing :: Typed" in project["classifiers"]
+ assert "memu/py.typed" in maturin["include"]
+ assert "recursive-include src/memu *.py *.pyi py.typed" in manifest
+
+
+def test_python_version_floor_is_consistent() -> None:
+ pyproject_text = (ROOT / "pyproject.toml").read_text(encoding="utf-8")
+ project = tomllib.loads(pyproject_text)["project"]
+ uv_lock = (ROOT / "uv.lock").read_text(encoding="utf-8")
+ cargo = (ROOT / "Cargo.toml").read_text(encoding="utf-8")
+ build_workflow = (ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8")
+ release_workflow = (ROOT / ".github" / "workflows" / "release-please.yml").read_text(encoding="utf-8")
+ readme = (ROOT / "README.md").read_text(encoding="utf-8")
+ translated_readmes = [path.read_text(encoding="utf-8") for path in sorted((ROOT / "readme").glob("README_*.md"))]
+
+ assert project["requires-python"] == f">={MIN_PYTHON_VERSION}"
+ assert (ROOT / ".python-version").read_text(encoding="utf-8").strip() == MIN_PYTHON_VERSION
+ assert f'requires-python = ">={MIN_PYTHON_VERSION}"' in uv_lock
+ assert f'python_version = "{MIN_PYTHON_VERSION}"' in pyproject_text
+ assert f'target-version = "py{MIN_PYTHON_VERSION.replace(".", "")}"' in pyproject_text
+ assert f"Programming Language :: Python :: {MIN_PYTHON_VERSION}" in project["classifiers"]
+ assert f"abi3-py{MIN_PYTHON_VERSION.replace('.', '')}" in cargo
+ assert f'"{MIN_PYTHON_VERSION}"' in build_workflow
+ assert '"3.13"' in build_workflow
+ assert f'python-version: "{MIN_PYTHON_VERSION}"' in release_workflow
+ assert 'python-version: "3.13"' not in release_workflow
+ assert f"python-{MIN_PYTHON_VERSION}+" in readme
+ for translated_readme in translated_readmes:
+ assert f"python-{MIN_PYTHON_VERSION}+" in translated_readme
+ assert f"Python {MIN_PYTHON_VERSION}+" in translated_readme
+
+
+def test_public_docs_do_not_advertise_old_python_floor() -> None:
+ checked_paths = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted(path for path in (ROOT / "examples").rglob("*.py") if "resources" not in path.parts),
+ ]
+ stale_refs: list[str] = []
+
+ for path in checked_paths:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if "Python 3.13+" in line or "python-3.13+" in line:
+ stale_refs.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ assert not stale_refs, f"Public docs should advertise Python {MIN_PYTHON_VERSION}+: {stale_refs}"
+
+
+def test_public_markdown_docs_do_not_contain_mojibake_markers() -> None:
+ markers = [
+ "馃",
+ "鈹",
+ "鈻",
+ "鈽",
+ "鉁",
+ "锟",
+ "�",
+ "\ufffd",
+ "涓枃",
+ "鏃ユ湰",
+ "頃滉",
+ "Espa帽ol",
+ "Fran莽ais",
+ ]
+ checked_paths = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ ]
+ mojibake_hits: list[str] = []
+
+ for path in checked_paths:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ found = [marker for marker in markers if marker in line]
+ if found:
+ mojibake_hits.append(
+ f"{path.relative_to(ROOT)}:{line_no}: markers={found!r} line={line.strip()!r}"
+ )
+
+ assert not mojibake_hits, f"Public Markdown docs should be valid UTF-8 text: {mojibake_hits}"
+
+
+def test_locked_dependencies_do_not_raise_python_floor() -> None:
+ lock = tomllib.loads((ROOT / "uv.lock").read_text(encoding="utf-8"))
+ dependencies_with_higher_floor: list[str] = []
+
+ for package in lock.get("package", []):
+ requirement = package.get("requires-python")
+ if isinstance(requirement, str) and not _supports_minimum_python(requirement, MIN_PYTHON_VERSION):
+ name = package.get("name", "")
+ version = package.get("version", "")
+ dependencies_with_higher_floor.append(f"{name}=={version} requires {requirement}")
+
+ assert not dependencies_with_higher_floor, (
+ f"Locked dependencies must support Python {MIN_PYTHON_VERSION}+: {dependencies_with_higher_floor}"
+ )
+
+
+def test_packaging_manifest_tracks_current_source_layout() -> None:
+ manifest = (ROOT / "MANIFEST.in").read_text(encoding="utf-8")
+
+ assert "include LICENSE.txt" in manifest
+ assert "include CODE_OF_CONDUCT.md" in manifest
+ assert "include CONTRIBUTING.md" in manifest
+ assert "include SECURITY.md" in manifest
+ assert "include SUPPORT.md" in manifest
+ assert "include mkdocs.yml" in manifest
+ assert "recursive-include src/memu *.py *.pyi" in manifest
+ assert "recursive-include assets *.png *.gif *.jpg *.jpeg" in manifest
+ assert "recursive-include readme *.md" in manifest
+ assert "recursive-include examples *.py *.md *.json *.txt *.png *.jpg *.jpeg *.sh .env.example" in manifest
+ assert "recursive-include memu *.py" not in manifest
+ assert "exclude .env.example" not in manifest
+
+
+def test_mkdocs_config_points_to_existing_docs() -> None:
+ mkdocs = (ROOT / "mkdocs.yml").read_text(encoding="utf-8")
+ missing_docs: list[str] = []
+
+ assert "site_name: memU" in mkdocs
+ assert "repo_url: https://github.com/NevaMind-AI/MemU" in mkdocs
+ assert "name: material" in mkdocs
+ assert "mkdocstrings:" in mkdocs
+
+ for target in _mkdocs_nav_targets(mkdocs):
+ target_path = (ROOT / "docs" / target).resolve()
+ if not _is_inside_root(target_path) or not target_path.exists():
+ missing_docs.append(target)
+
+ assert not missing_docs, f"mkdocs.yml should only reference existing docs: {missing_docs}"
+
+
+def test_dependabot_tracks_dependency_managers() -> None:
+ dependabot = (ROOT / ".github" / "dependabot.yml").read_text(encoding="utf-8")
+
+ assert "version: 2" in dependabot
+ assert 'package-ecosystem: "uv"' in dependabot
+ assert 'package-ecosystem: "cargo"' in dependabot
+ assert 'package-ecosystem: "github-actions"' in dependabot
+ assert dependabot.count('directory: "/"') == 3
+ assert dependabot.count('interval: "weekly"') == 3
+ assert 'prefix: "chore(deps)"' in dependabot
+ assert 'prefix-development: "chore(deps-dev)"' in dependabot
+ assert 'prefix: "ci(deps)"' in dependabot
+ assert "open-pull-requests-limit: 5" in dependabot
+
+
+def test_codeql_workflow_scans_python_and_rust() -> None:
+ codeql = (ROOT / ".github" / "workflows" / "codeql.yml").read_text(encoding="utf-8")
+ security = (ROOT / "SECURITY.md").read_text(encoding="utf-8")
+
+ assert "name: codeql" in codeql
+ assert "push:" in codeql
+ assert "pull_request:" in codeql
+ assert "schedule:" in codeql
+ assert "security-events: write" in codeql
+ assert "actions: read" in codeql
+ assert "contents: read" in codeql
+ assert "language: python" in codeql
+ assert "language: rust" in codeql
+ assert codeql.count("build-mode: none") == 2
+ assert "github/codeql-action/init@v4" in codeql
+ assert "github/codeql-action/analyze@v4" in codeql
+ assert 'category: "/language:${{ matrix.language }}"' in codeql
+ assert "CodeQL for Python and Rust" in security
+
+
+def test_release_workflow_uses_current_artifact_actions_and_scoped_permissions() -> None:
+ release_workflow = (ROOT / ".github" / "workflows" / "release-please.yml").read_text(encoding="utf-8")
+
+ assert "permissions:\n contents: read" in release_workflow
+ assert "concurrency:" in release_workflow
+ assert "group: release-please-${{ github.ref }}" in release_workflow
+ assert "cancel-in-progress: false" in release_workflow
+ assert "release-please:\n runs-on: ubuntu-latest\n permissions:" in release_workflow
+ assert "contents: write" in release_workflow
+ assert "issues: write" in release_workflow
+ assert "pull-requests: write" in release_workflow
+ assert release_workflow.count("actions/upload-artifact@v7") == 2
+ assert "actions/upload-artifact@v6" not in release_workflow
+ assert release_workflow.count("actions/download-artifact@v8") == 2
+ assert "actions/download-artifact@v7" not in release_workflow
+ assert "actions: read" in release_workflow
+ assert "id-token: write" in release_workflow
+ assert "pypa/gh-action-pypi-publish@release/v1" in release_workflow
+
+
+def test_project_console_scripts_point_to_existing_functions() -> None:
+ project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]
+ broken_scripts: list[str] = []
+
+ for script_name, target in project.get("scripts", {}).items():
+ module_name, sep, function_name = target.partition(":")
+ module_path = ROOT / "src" / Path(*module_name.split(".")).with_suffix(".py")
+ if sep != ":" or not function_name:
+ broken_scripts.append(f"{script_name}={target} is not module:function")
+ continue
+ if not module_path.exists():
+ broken_scripts.append(f"{script_name}={target} missing {module_path.relative_to(ROOT)}")
+ continue
+ functions = _module_functions(module_path)
+ if function_name not in functions:
+ broken_scripts.append(f"{script_name}={target} missing function {function_name}")
+
+ assert not broken_scripts, f"Console scripts must point to existing functions: {broken_scripts}"
+
+
+def test_installation_docs_use_published_distribution_name() -> None:
+ docs = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ ROOT / "docs" / "tutorials" / "getting_started.md",
+ ROOT / "docs" / "sealos-devbox-guide.md",
+ ROOT / "docs" / "sealos_use_case.md",
+ ]
+ wrong_distribution_refs: list[str] = []
+
+ for doc in docs:
+ text = doc.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if re.search(r"\b(?:pip install|uv add)\s+memu(?!-)\b", line):
+ wrong_distribution_refs.append(f"{doc.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ requirement_files = sorted((ROOT / "examples").rglob("requirements*.txt"))
+ for path in requirement_files:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if re.search(r"^\s*memu(?:\s|[<>=!~]|$)", line):
+ wrong_distribution_refs.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ assert not wrong_distribution_refs, f"Use the published memu-py distribution name: {wrong_distribution_refs}"
+
+
+def test_general_python_examples_are_ascii_safe() -> None:
+ unsafe_lines: list[str] = []
+
+ for path in sorted((ROOT / "examples").rglob("*.py")):
+ for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ try:
+ line.encode("ascii")
+ except UnicodeEncodeError:
+ unsafe_lines.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()!r}")
+
+ assert not unsafe_lines, f"General-purpose Python examples should be ASCII-safe: {unsafe_lines}"
+
+
+def test_sealos_use_case_docs_track_demo_status_copy() -> None:
+ docs = (ROOT / "docs" / "sealos_use_case.md").read_text(encoding="utf-8")
+ example = (ROOT / "examples" / "sealos_support_agent.py").read_text(encoding="utf-8")
+
+ for snippet in (
+ "[START] Starting Sealos Support Agent Demo (Offline Mode)",
+ "[OK] Environment Check: MemU Library detected.",
+ "[OK] Runtime: Sealos Devbox (Python 3.12+)",
+ "[DONE] Demo Completed Successfully",
+ ):
+ assert snippet in docs
+ assert snippet in example
+
+
+def test_readme_quickstart_uses_examples_not_test_modules() -> None:
+ checked_readmes = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ ]
+ stale_quickstarts: list[str] = []
+
+ for path in checked_readmes:
+ readme = path.read_text(encoding="utf-8")
+ if "cd tests" in readme:
+ stale_quickstarts.append(f"{path.relative_to(ROOT)} uses cd tests")
+ if re.search(r"^\s*python\s+test_[A-Za-z0-9_]+\.py\b", readme, flags=re.MULTILINE):
+ stale_quickstarts.append(f"{path.relative_to(ROOT)} runs a test module as quickstart")
+ if re.search(r"^\s*python\s+tests/test_[A-Za-z0-9_]+\.py\b", readme, flags=re.MULTILINE):
+ stale_quickstarts.append(f"{path.relative_to(ROOT)} runs a test module without uv")
+ if "python examples/getting_started_robust.py" not in readme:
+ stale_quickstarts.append(f"{path.relative_to(ROOT)} does not show the robust getting-started example")
+
+ msg = f"README quickstarts should use public examples, not test modules: {stale_quickstarts}"
+ assert not stale_quickstarts, msg
+
+
+def test_public_docs_reference_existing_test_files() -> None:
+ missing_test_refs: list[str] = []
+
+ for path in _public_markdown_docs():
+ text = path.read_text(encoding="utf-8")
+ for match in re.finditer(r"\btests/[A-Za-z0-9_./-]+\.py\b", text):
+ target = (ROOT / match.group(0)).resolve()
+ if not _is_inside_root(target) or not target.exists():
+ line_no = text.count("\n", 0, match.start()) + 1
+ missing_test_refs.append(f"{path.relative_to(ROOT)}:{line_no}: {match.group(0)}")
+
+ assert not missing_test_refs, f"Public docs should reference existing tests: {missing_test_refs}"
+
+
+def test_public_docs_use_uv_for_test_commands() -> None:
+ direct_test_commands: list[str] = []
+
+ for path in _public_markdown_docs():
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if re.search(r"^\s*python\s+tests/test_[A-Za-z0-9_]+\.py\b", line):
+ direct_test_commands.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ assert not direct_test_commands, f"Public docs should run tests via uv pytest: {direct_test_commands}"
+
+
+def test_postgres_integration_check_is_pytest_collectable_and_opt_in() -> None:
+ readmes = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ ]
+ source = (ROOT / "tests" / "test_postgres.py").read_text(encoding="utf-8")
+
+ for readme_path in readmes:
+ readme = readme_path.read_text(encoding="utf-8")
+ assert "uv sync --extra postgres" in readme, readme_path.name
+ assert "export MEMU_RUN_POSTGRES_TESTS=1" in readme, readme_path.name
+ assert "uv run python -m pytest tests/test_postgres.py" in readme, readme_path.name
+ assert "async def test_postgres_full_workflow" in source
+ assert 'os.environ.get(RUN_POSTGRES_TESTS_ENV) != "1"' in source
+ assert "pytest.skip" in source
+ assert 'Path(__file__).resolve().parent / "example" / "example_conversation.json"' in source
+ assert 'src_path = str(PROJECT_ROOT / "src")' in source
+ assert source.index("from memu import MemoryService") > source.index("async def run_postgres_workflow")
+ assert "raise SystemExit(main())" in source
+
+
+def test_lazyllm_integration_check_is_pytest_collectable_and_opt_in() -> None:
+ readmes = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ ]
+ source = (ROOT / "tests" / "test_lazyllm.py").read_text(encoding="utf-8")
+
+ for readme_path in readmes:
+ readme = readme_path.read_text(encoding="utf-8")
+ assert "uv sync --extra lazyllm" in readme, readme_path.name
+ assert "export MEMU_QWEN_API_KEY=your_api_key" in readme, readme_path.name
+ assert "export MEMU_RUN_LAZYLLM_TESTS=1" in readme, readme_path.name
+ assert "uv run python -m pytest tests/test_lazyllm.py" in readme, readme_path.name
+ assert "async def test_lazyllm_client" in source
+ assert "RUN_LAZYLLM_TESTS_ENV = \"MEMU_RUN_LAZYLLM_TESTS\"" in source
+ assert 'os.environ.get(RUN_LAZYLLM_TESTS_ENV) != "1"' in source
+ assert "pytest.skip" in source
+ assert "async def run_lazyllm_workflow" in source
+ assert "PROJECT_ROOT / \"examples\" / \"resources\" / \"images\" / \"image1.png\"" in source
+ assert "python tests/test_lazyllm.py" in source
+
+
+def test_live_llm_storage_checks_are_pytest_collectable_and_opt_in() -> None:
+ readmes = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ ]
+
+ for readme_path in readmes:
+ readme = readme_path.read_text(encoding="utf-8")
+ assert "MEMU_RUN_LIVE_LLM_TESTS=1" in readme, readme_path.name
+ assert "tests/test_inmemory.py" in readme, readme_path.name
+ assert "tests/test_sqlite.py" in readme, readme_path.name
+ for filename, function_name in [
+ ("test_inmemory.py", "test_inmemory_full_workflow"),
+ ("test_sqlite.py", "test_sqlite_full_workflow"),
+ ]:
+ source = (ROOT / "tests" / filename).read_text(encoding="utf-8")
+ assert f"async def {function_name}" in source
+ assert "RUN_LIVE_LLM_TESTS_ENV = \"MEMU_RUN_LIVE_LLM_TESTS\"" in source
+ assert 'os.environ.get(RUN_LIVE_LLM_TESTS_ENV) != "1"' in source
+ assert "pytest.skip" in source
+ assert 'Path(__file__).resolve().parent / "example" / "example_conversation.json"' in source
+ assert source.index("from memu import MemoryService") > source.index("async def run_")
+ assert "raise SystemExit(main())" in source
+
+
+def test_openrouter_integration_check_is_pytest_collectable_opt_in_and_non_mutating() -> None:
+ source = (ROOT / "tests" / "test_openrouter.py").read_text(encoding="utf-8")
+
+ assert "RUN_OPENROUTER_TESTS_ENV = \"MEMU_RUN_OPENROUTER_TESTS\"" in source
+ assert "async def test_openrouter_full_workflow" in source
+ assert 'os.environ.get(RUN_OPENROUTER_TESTS_ENV) != "1"' in source
+ assert "pytest.skip" in source
+ assert "async def run_openrouter_workflow" in source
+ assert 'Path(__file__).resolve().parent / "example" / "example_conversation.json"' in source
+ assert source.index("from memu import MemoryService") > source.index("async def run_openrouter_workflow")
+ assert "openrouter_test_output.json" not in source
+ assert "examples/output" not in source
+ assert "raise SystemExit(main())" in source
+
+
+def test_public_openrouter_test_commands_are_explicitly_opt_in() -> None:
+ missing_opt_in: list[str] = []
+
+ for path in _public_markdown_docs():
+ text = path.read_text(encoding="utf-8")
+ if "tests/test_openrouter.py" not in text:
+ continue
+ if "MEMU_RUN_OPENROUTER_TESTS=1" not in text:
+ missing_opt_in.append(str(path.relative_to(ROOT)))
+
+ assert not missing_opt_in, (
+ "Public OpenRouter live-test commands should set MEMU_RUN_OPENROUTER_TESTS=1: "
+ f"{missing_opt_in}"
+ )
+
+
+def test_build_workflow_runs_for_all_pull_requests() -> None:
+ build_workflow = (ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8")
+
+ assert "pull_request" in build_workflow
+ assert "head.repo.full_name" not in build_workflow
+ assert "base.repo.full_name" not in build_workflow
+ assert "permissions:\n contents: read" in build_workflow
+
+
+def test_build_workflow_uses_public_quality_gates() -> None:
+ build_workflow = (ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8")
+
+ assert "run: make check" in build_workflow
+ assert "run: make test" in build_workflow
+ assert "run: make docs-build" in build_workflow
+ assert "uv run make" not in build_workflow
+ assert "pre-commit install" not in build_workflow
+
+
+def test_public_examples_use_real_top_level_exports() -> None:
+ exported_names = _top_level_memu_exports()
+ checked_files = [
+ ROOT / "README.md",
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ ]
+ missing_exports: list[str] = []
+
+ for path in checked_files:
+ text = path.read_text(encoding="utf-8")
+ imported_names = _memu_imported_names(path, text)
+ for imported_name in imported_names:
+ if imported_name not in exported_names:
+ missing_exports.append(f"{path.relative_to(ROOT)} imports memu.{imported_name}")
+
+ assert not missing_exports, f"Examples and docs should only import public memu exports: {missing_exports}"
+
+
+def test_app_exports_are_promoted_to_top_level_package() -> None:
+ app_exports = _module_all(ROOT / "src" / "memu" / "app" / "__init__.py")
+ top_level_exports = _top_level_memu_exports()
+
+ missing_exports = sorted(app_exports - top_level_exports)
+
+ assert not missing_exports, f"memu.app exports should also be available from memu: {missing_exports}"
+
+
+def test_public_examples_prefer_stable_top_level_imports() -> None:
+ top_level_exports = _top_level_memu_exports()
+ checked_files = [
+ ROOT / "README.md",
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ ]
+ internal_imports: list[str] = []
+
+ for path in checked_files:
+ text = path.read_text(encoding="utf-8")
+ if path.suffix == ".py":
+ internal_imports.extend(_python_promoted_internal_imports(path, text, top_level_exports))
+ else:
+ internal_imports.extend(_markdown_promoted_internal_imports(path, text, top_level_exports))
+
+ assert not internal_imports, f"Use stable top-level memu imports for public examples: {internal_imports}"
+
+
+def test_python_examples_call_keyword_only_apis_with_keywords() -> None:
+ positional_calls: list[str] = []
+
+ for path in sorted((ROOT / "examples").rglob("*.py")):
+ text = path.read_text(encoding="utf-8")
+ module = ast.parse(text, filename=str(path))
+ for node in ast.walk(module):
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr in KEYWORD_ONLY_EXAMPLE_APIS
+ and node.args
+ ):
+ positional_calls.append(f"{path.relative_to(ROOT)}:{node.lineno}: {node.func.attr}()")
+
+ assert not positional_calls, f"Use keyword arguments for public example API calls: {positional_calls}"
+
+
+def test_public_examples_use_configured_memory_service_llm_clients() -> None:
+ allowed_direct_sdk_examples = {
+ ROOT / "examples" / "test_nebius_provider.py",
+ }
+ bypasses: list[str] = []
+
+ for path in sorted((ROOT / "examples").rglob("*.py")):
+ if path in allowed_direct_sdk_examples:
+ continue
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if "AsyncOpenAI(" in line or "OpenAI(" in line or "service.llm_config" in line:
+ bypasses.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ assert not bypasses, (
+ "Public examples should use MemoryService's configured LLM clients "
+ f"instead of bypassing profile routing: {bypasses}"
+ )
+
+
+def test_public_docs_do_not_use_legacy_memory_service_constructor_args() -> None:
+ legacy_refs: list[str] = []
+ checked_paths = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.txt")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ ]
+
+ for path in checked_paths:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if "MemoryService(" in line or "llm_config" in line:
+ window = "\n".join(text.splitlines()[max(0, line_no - 1) : line_no + 8])
+ if "MemoryService(" in window and "llm_config" in window:
+ legacy_refs.append(f"{path.relative_to(ROOT)}:{line_no}")
+
+ assert not legacy_refs, f"Use MemoryService(llm_profiles=...) instead of llm_config=: {legacy_refs}"
+
+
+def test_public_direct_retrieve_examples_do_not_use_http_query_shorthand() -> None:
+ legacy_refs: list[str] = []
+ checked_paths = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.txt")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ ]
+
+ for path in checked_paths:
+ lines = path.read_text(encoding="utf-8").splitlines()
+ for index, line in enumerate(lines):
+ if ".retrieve(" not in line and "retrieve(" not in line:
+ continue
+ window = "\n".join(lines[index : index + 8])
+ if re.search(r"(? None:
+ allowed_path = ROOT / "docs" / "sqlite.md"
+ low_level_refs: list[str] = []
+ checked_paths = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.txt")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ ]
+
+ for path in checked_paths:
+ text = path.read_text(encoding="utf-8")
+ if path == allowed_path:
+ continue
+ if "build_sqlite_database" in text or "build_postgres_database" in text:
+ low_level_refs.append(str(path.relative_to(ROOT)))
+
+ sqlite_docs = allowed_path.read_text(encoding="utf-8")
+ assert not low_level_refs, f"Use MemoryService(database_config=...) outside migration docs: {low_level_refs}"
+ assert "This migration snippet intentionally uses" in sqlite_docs
+ assert "normal application code should continue to use `MemoryService(database_config=...)`" in sqlite_docs
+
+
+def test_example_text_resources_use_current_public_api_shapes() -> None:
+ stale_fragments = [
+ "llm_config=",
+ "llm_config={",
+ '"api_key": "your-api-key"',
+ 'query="What programming languages does Alex know?"',
+ "top_k=5",
+ "service.store.categories",
+ '"memory_types": ["profile", "knowledge", "custom"]',
+ ]
+ stale_refs: list[str] = []
+
+ for path in sorted((ROOT / "examples" / "resources").rglob("*.txt")):
+ text = path.read_text(encoding="utf-8")
+ for fragment in stale_fragments:
+ if fragment in text:
+ stale_refs.append(f"{path.relative_to(ROOT)}: {fragment}")
+
+ assert not stale_refs, f"Example resources should not teach stale public API shapes: {stale_refs}"
+
+ docs = (ROOT / "examples" / "resources" / "docs" / "doc1.txt").read_text(encoding="utf-8")
+ assert "llm_profiles={" in docs
+ assert '"api_key": "OPENAI_API_KEY"' in docs
+ assert 'queries=["What programming languages does Alex know?"]' in docs
+ assert 'where={"user_id": "alex"}' in docs
+ assert "await service.list_memory_categories" in docs
+ assert '"memory_types": ["profile", "knowledge", "skill"]' in docs
+
+
+def test_grok_docs_use_current_memory_service_profile_api() -> None:
+ provider_docs = (ROOT / "docs" / "providers" / "grok.md").read_text(encoding="utf-8")
+ integration_docs = (ROOT / "docs" / "integrations" / "grok.md").read_text(encoding="utf-8")
+
+ for docs in (provider_docs, integration_docs):
+ assert "XAI_API_KEY" in docs
+ assert "GROK_API_KEY" not in docs
+ assert "MemoryService(llm_config=" not in docs
+ assert "llm_profiles={" in docs
+ assert '"provider": "grok"' in docs
+ assert 'api_key="XAI_API_KEY"' in docs
+
+ assert "XAI_API_KEY=xai-YOUR_API_KEY_HERE" in provider_docs
+ assert "from memu import LLMConfig" not in integration_docs
+
+
+def test_getting_started_profile_memory_is_user_scoped() -> None:
+ example_path = ROOT / "examples" / "getting_started_robust.py"
+ example = example_path.read_text(encoding="utf-8")
+ module = ast.parse(example, filename=str(example_path))
+
+ create_calls: list[ast.Call] = []
+ retrieve_calls: list[ast.Call] = []
+ for node in ast.walk(module):
+ if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
+ continue
+ if node.func.attr == "create_memory_item":
+ create_calls.append(node)
+ elif node.func.attr == "retrieve":
+ retrieve_calls.append(node)
+
+ assert any(_call_has_keyword_value(call, "user", "user_scope") for call in create_calls)
+ assert any(_call_has_keyword_value(call, "where", "user_scope") for call in retrieve_calls)
+
+ docs = (ROOT / "docs" / "tutorials" / "getting_started.md").read_text(encoding="utf-8")
+ assert 'user_scope = {"user_id": "demo_user"}' in docs
+ assert "user=user_scope" in docs
+ assert "where=user_scope" in docs
+
+
+def test_source_checkout_examples_add_src_path_before_memu_imports() -> None:
+ checked_files = [
+ ROOT / "examples" / "context_harness_demo.py",
+ ROOT / "examples" / "example_1_conversation_memory.py",
+ ROOT / "examples" / "example_2_skill_extraction.py",
+ ROOT / "examples" / "example_3_multimodal_memory.py",
+ ROOT / "examples" / "example_4_openrouter_memory.py",
+ ROOT / "examples" / "example_5_with_lazyllm_client.py",
+ ROOT / "examples" / "getting_started_robust.py",
+ ROOT / "examples" / "langgraph_demo.py",
+ ROOT / "examples" / "proactive" / "proactive.py",
+ ROOT / "examples" / "sealos_support_agent.py",
+ ROOT / "examples" / "test_nebius_provider.py",
+ ]
+ broken_examples: list[str] = []
+
+ for path in checked_files:
+ lines = path.read_text(encoding="utf-8").splitlines()
+ memu_import_lines = [
+ line_no
+ for line_no, line in enumerate(lines, 1)
+ if re.match(r"\s*(?:from memu\b|import memu\b)", line)
+ ]
+ if not memu_import_lines:
+ continue
+ src_insert_lines = [
+ line_no
+ for line_no, line in enumerate(lines, 1)
+ if "sys.path.insert(0" in line and "src" in "\n".join(lines[max(0, line_no - 4) : line_no + 1])
+ ]
+ file_path_lines = [
+ line_no
+ for line_no, line in enumerate(lines, 1)
+ if "__file__" in line and "src" in "\n".join(lines[line_no - 1 : line_no + 3])
+ ]
+ if not src_insert_lines or min(src_insert_lines) > min(memu_import_lines):
+ broken_examples.append(f"{path.relative_to(ROOT)} imports memu before adding src to sys.path")
+ if not file_path_lines:
+ broken_examples.append(f"{path.relative_to(ROOT)} should derive src path from __file__")
+
+ assert not broken_examples, f"Examples should run from source checkouts: {broken_examples}"
+
+
+def test_python_examples_do_not_hardcode_repo_relative_io_paths() -> None:
+ repo_relative_paths: list[str] = []
+
+ for path in sorted((ROOT / "examples").rglob("*.py")):
+ text = path.read_text(encoding="utf-8")
+ module = ast.parse(text, filename=str(path))
+ for node in ast.walk(module):
+ if not isinstance(node, ast.Constant) or not isinstance(node.value, str):
+ continue
+ if "examples/resources/" in node.value or "examples/output/" in node.value:
+ repo_relative_paths.append(f"{path.relative_to(ROOT)}:{node.lineno}: {node.value!r}")
+
+ assert not repo_relative_paths, (
+ "Python examples should derive resource/output paths from __file__, "
+ f"not the caller's current working directory: {repo_relative_paths}"
+ )
+
+
+def test_public_examples_use_supported_memorize_modalities() -> None:
+ unsupported_modalities: list[str] = []
+
+ for path in sorted((ROOT / "examples").rglob("*.py")):
+ unsupported_modalities.extend(_unsupported_python_memorize_modalities(path))
+
+ docs = [
+ ROOT / "README.md",
+ *sorted((ROOT / "docs").rglob("*.md")),
+ ]
+ for path in docs:
+ text = path.read_text(encoding="utf-8")
+ for line_no, modality in _markdown_memorize_modalities(text):
+ if modality not in SUPPORTED_MEMORIZE_MODALITIES:
+ unsupported_modalities.append(f"{path.relative_to(ROOT)}:{line_no}: modality={modality!r}")
+
+ assert not unsupported_modalities, (
+ "Public examples should use supported memorize modalities "
+ f"{sorted(SUPPORTED_MEMORIZE_MODALITIES)}: {unsupported_modalities}"
+ )
+
+
+def test_public_markdown_retrieve_examples_use_string_content() -> None:
+ legacy_query_shapes: list[str] = []
+ docs = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ ]
+
+ for path in docs:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if re.search(r'"content"\s*:\s*\{\s*"text"\s*:', line):
+ legacy_query_shapes.append(f"{path.relative_to(ROOT)}:{line_no}")
+
+ assert not legacy_query_shapes, (
+ "Public retrieve examples should use the current string content shape "
+ "{\"role\": \"user\", \"content\": \"...\"}: "
+ f"{legacy_query_shapes}"
+ )
+
+
+def test_readmes_document_python_retrieve_string_query_items() -> None:
+ expected = 'queries=["What are their preferences?"]'
+
+ for path in (ROOT / "README.md", ROOT / "readme" / "README_en.md"):
+ text = path.read_text(encoding="utf-8")
+ assert expected in text
+ assert "normalizes it to a user message before retrieval" in text
+
+
+def test_supported_modalities_match_preprocess_prompts() -> None:
+ prompt_keys = _module_dict_literal_keys(ROOT / "src" / "memu" / "prompts" / "preprocess" / "__init__.py", "PROMPTS")
+
+ assert set(SUPPORTED_MEMORIZE_MODALITIES) == prompt_keys
+
+
+def test_supported_memory_types_match_database_model() -> None:
+ memory_types = _module_literal_alias_values(ROOT / "src" / "memu" / "database" / "models.py", "MemoryType")
+
+ assert set(SUPPORTED_MEMORY_TYPES) == memory_types
+
+
+def test_memorize_config_description_lists_supported_memory_types() -> None:
+ settings = (ROOT / "src" / "memu" / "app" / "settings.py").read_text(encoding="utf-8")
+
+ assert "/".join(SUPPORTED_MEMORY_TYPES) in settings
+
+
+def test_public_docs_do_not_describe_stale_memory_type_count() -> None:
+ stale_refs: list[str] = []
+ checked_paths = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ ]
+
+ for path in checked_paths:
+ text = path.read_text(encoding="utf-8")
+ for line_no, line in enumerate(text.splitlines(), 1):
+ if re.search(r"\b(memory_type|memory type|Memory Type).{0,80}\b5 types\b", line):
+ stale_refs.append(f"{path.relative_to(ROOT)}:{line_no}: {line.strip()}")
+
+ assert not stale_refs, f"Public docs should describe all supported memory types: {stale_refs}"
+
+
+def test_public_examples_use_current_category_prompt_key() -> None:
+ legacy_prompt_keys: list[str] = []
+
+ for path in sorted((ROOT / "examples").rglob("*.py")):
+ text = path.read_text(encoding="utf-8")
+ module = ast.parse(text, filename=str(path))
+ for node in ast.walk(module):
+ if not isinstance(node, ast.Dict):
+ continue
+ for key in node.keys:
+ if isinstance(key, ast.Constant) and key.value == "custom_prompt":
+ legacy_prompt_keys.append(f"{path.relative_to(ROOT)}:{key.lineno}")
+
+ assert not legacy_prompt_keys, (
+ "Use CategoryConfig.summary_prompt instead of the legacy custom_prompt key: "
+ f"{legacy_prompt_keys}"
+ )
+
+
+def test_public_docs_use_supported_client_backends() -> None:
+ unsupported_backends: list[str] = []
+ checked_paths = [
+ ROOT / "README.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.py")),
+ ]
+
+ for path in checked_paths:
+ text = path.read_text(encoding="utf-8")
+ for line_no, backend in _client_backend_values(text):
+ if backend not in SUPPORTED_CLIENT_BACKENDS:
+ unsupported_backends.append(f"{path.relative_to(ROOT)}:{line_no}: client_backend={backend!r}")
+
+ assert not unsupported_backends, (
+ f"Use supported client_backend values {sorted(SUPPORTED_CLIENT_BACKENDS)}: {unsupported_backends}"
+ )
+
+
+def _iter_local_reference_targets(text: str) -> list[str]:
+ markdown_targets = re.findall(r"!?\[[^\]]*\]\(([^)]+)\)", text)
+ html_src_targets = re.findall(r"\bsrc=[\"']([^\"']+)[\"']", text, flags=re.IGNORECASE)
+ return [*markdown_targets, *html_src_targets]
+
+
+def _public_markdown_docs() -> list[Path]:
+ return [
+ ROOT / "README.md",
+ ROOT / "CONTRIBUTING.md",
+ ROOT / "CODE_OF_CONDUCT.md",
+ ROOT / "SECURITY.md",
+ ROOT / "SUPPORT.md",
+ ROOT / ".github" / "PULL_REQUEST_TEMPLATE.md",
+ *sorted((ROOT / "readme").glob("README_*.md")),
+ *sorted((ROOT / "docs").rglob("*.md")),
+ *sorted((ROOT / "examples").rglob("*.md")),
+ ]
+
+
+def _mkdocs_nav_targets(mkdocs: str) -> list[str]:
+ targets: list[str] = []
+
+ for line in mkdocs.splitlines():
+ stripped = line.strip()
+ if not stripped.startswith("- ") or ":" not in stripped:
+ continue
+ target = stripped.split(":", 1)[1].strip().strip("'\"")
+ if target.endswith(".md"):
+ targets.append(target)
+
+ return targets
+
+
+def _top_level_memu_exports() -> set[str]:
+ return _module_all(ROOT / "src" / "memu" / "__init__.py")
+
+
+def _module_all(path: Path) -> set[str]:
+ module = ast.parse(path.read_text(encoding="utf-8"))
+ for node in module.body:
+ if isinstance(node, ast.Assign):
+ for target in node.targets:
+ if isinstance(target, ast.Name) and target.id == "__all__":
+ value = ast.literal_eval(node.value)
+ return set(value)
+ raise AssertionError(f"{path.relative_to(ROOT)} must declare __all__")
+
+
+def _module_functions(path: Path) -> set[str]:
+ module = ast.parse(path.read_text(encoding="utf-8"))
+ return {node.name for node in module.body if isinstance(node, ast.FunctionDef)}
+
+
+def _module_dict_literal_keys(path: Path, name: str) -> set[str]:
+ module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+ for node in module.body:
+ if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) and node.target.id == name:
+ value = node.value
+ elif isinstance(node, ast.Assign) and any(
+ isinstance(target, ast.Name) and target.id == name for target in node.targets
+ ):
+ value = node.value
+ else:
+ continue
+ if not isinstance(value, ast.Dict):
+ break
+ return {
+ key.value
+ for key in value.keys
+ if isinstance(key, ast.Constant) and isinstance(key.value, str)
+ }
+ raise AssertionError(f"{path.relative_to(ROOT)} should define dict literal {name}")
+
+
+def _module_literal_alias_values(path: Path, name: str) -> set[str]:
+ module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+ for node in module.body:
+ if not isinstance(node, ast.Assign):
+ continue
+ if not any(isinstance(target, ast.Name) and target.id == name for target in node.targets):
+ continue
+ value = node.value
+ if not (
+ isinstance(value, ast.Subscript)
+ and isinstance(value.value, ast.Name)
+ and value.value.id == "Literal"
+ ):
+ break
+ slice_value = value.slice
+ literal_items = slice_value.elts if isinstance(slice_value, ast.Tuple) else [slice_value]
+ return {
+ item.value
+ for item in literal_items
+ if isinstance(item, ast.Constant) and isinstance(item.value, str)
+ }
+ raise AssertionError(f"{path.relative_to(ROOT)} should define Literal alias {name}")
+
+
+def _call_has_keyword_value(call: ast.Call, keyword_name: str, expected_name: str) -> bool:
+ for keyword in call.keywords:
+ if keyword.arg == keyword_name and isinstance(keyword.value, ast.Name):
+ return keyword.value.id == expected_name
+ return False
+
+
+def _memu_imported_names(path: Path, text: str) -> set[str]:
+ if path.suffix == ".py":
+ return _python_memu_imported_names(path, text)
+ return set(re.findall(r"^\s*from\s+memu\s+import\s+([A-Za-z_][A-Za-z0-9_]*)", text, flags=re.MULTILINE))
+
+
+def _python_memu_imported_names(path: Path, text: str) -> set[str]:
+ imported_names: set[str] = set()
+ module = ast.parse(text, filename=str(path))
+ for node in ast.walk(module):
+ if isinstance(node, ast.ImportFrom) and node.module == "memu":
+ imported_names.update(alias.name for alias in node.names)
+ return imported_names
+
+
+def _python_promoted_internal_imports(path: Path, text: str, top_level_exports: set[str]) -> list[str]:
+ internal_imports: list[str] = []
+ module = ast.parse(text, filename=str(path))
+ internal_modules = {"memu.app", "memu.app.service", "memu.app.settings"}
+
+ for node in ast.walk(module):
+ if not isinstance(node, ast.ImportFrom) or node.module not in internal_modules:
+ continue
+ imported_names = {alias.name for alias in node.names}
+ promoted_names = sorted(imported_names & top_level_exports)
+ if promoted_names:
+ internal_imports.append(f"{path.relative_to(ROOT)}:{node.lineno}: {', '.join(promoted_names)}")
+
+ return internal_imports
+
+
+def _markdown_promoted_internal_imports(path: Path, text: str, top_level_exports: set[str]) -> list[str]:
+ internal_imports: list[str] = []
+ internal_import_pattern = re.compile(
+ r"^\s*from\s+memu\.app(?:\.service|\.settings)?\s+import\s+(.+)$",
+ flags=re.MULTILINE,
+ )
+
+ for match in internal_import_pattern.finditer(text):
+ imported_names = set(re.findall(r"\b[A-Za-z_][A-Za-z0-9_]*\b", match.group(1)))
+ promoted_names = sorted(imported_names & top_level_exports)
+ if promoted_names:
+ line_no = text.count("\n", 0, match.start()) + 1
+ internal_imports.append(f"{path.relative_to(ROOT)}:{line_no}: {', '.join(promoted_names)}")
+
+ return internal_imports
+
+
+def _unsupported_python_memorize_modalities(path: Path) -> list[str]:
+ unsupported_modalities: list[str] = []
+ text = path.read_text(encoding="utf-8")
+ module = ast.parse(text, filename=str(path))
+ for node in ast.walk(module):
+ if not (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "memorize"
+ ):
+ continue
+ for keyword in node.keywords:
+ if keyword.arg != "modality" or not isinstance(keyword.value, ast.Constant):
+ continue
+ modality = keyword.value.value
+ if isinstance(modality, str) and modality not in SUPPORTED_MEMORIZE_MODALITIES:
+ unsupported_modalities.append(f"{path.relative_to(ROOT)}:{node.lineno}: modality={modality!r}")
+ return unsupported_modalities
+
+
+def _markdown_memorize_modalities(text: str) -> list[tuple[int, str]]:
+ modalities: list[tuple[int, str]] = []
+ for line_no, line in enumerate(text.splitlines(), 1):
+ for match in re.finditer(r"\bmodality\s*=\s*[\"']([^\"']+)[\"']", line):
+ modalities.append((line_no, match.group(1)))
+ return modalities
+
+
+def _client_backend_values(text: str) -> list[tuple[int, str]]:
+ values: list[tuple[int, str]] = []
+ for line_no, line in enumerate(text.splitlines(), 1):
+ for match in re.finditer(r"[\"']client_backend[\"']\s*:\s*[\"']([^\"']+)[\"']", line):
+ values.append((line_no, match.group(1)))
+ return values
+
+
+def _requirement_specifier(requirements: list[dict[str, object]], name: str, *, extra: str) -> str | None:
+ marker = f"extra == '{extra}'"
+ for requirement in requirements:
+ if requirement.get("name") == name and requirement.get("marker") == marker:
+ value = requirement.get("specifier")
+ return value if isinstance(value, str) else None
+ return None
+
+
+def _is_external_link(target: str) -> bool:
+ normalized = target.lower()
+ return normalized.startswith(("http://", "https://", "mailto:"))
+
+
+def _is_inside_root(path: Path) -> bool:
+ try:
+ path.relative_to(ROOT)
+ except ValueError:
+ return False
+ return True
+
+
+def _supports_minimum_python(requirement: str, minimum: str) -> bool:
+ minimum_version = _version_tuple(minimum)
+ for match in re.finditer(r"(?:^|,)\s*(>=|>|==|~=)\s*([0-9]+(?:\.[0-9]+)*)", requirement):
+ operator = match.group(1)
+ version = _version_tuple(match.group(2))
+ if operator in {">=", "~="} and version > minimum_version:
+ return False
+ if operator == ">" and version >= minimum_version:
+ return False
+ if operator == "==" and version != minimum_version:
+ return False
+ return True
+
+
+def _version_tuple(value: str) -> tuple[int, ...]:
+ return tuple(int(part) for part in value.split("."))
diff --git a/tests/test_retrieve_method.py b/tests/test_retrieve_method.py
new file mode 100644
index 00000000..279cbf4d
--- /dev/null
+++ b/tests/test_retrieve_method.py
@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+from memu.utils.retrieve import normalize_retrieve_method, normalize_retrieve_ranking
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_normalize_retrieve_method_uses_config_default() -> None:
+ assert normalize_retrieve_method(None, default="llm") == "llm"
+
+
+def test_normalize_retrieve_method_accepts_case_insensitive_override() -> None:
+ assert normalize_retrieve_method(" RAG ", default="llm") == "rag"
+
+
+def test_normalize_retrieve_method_rejects_unknown_values() -> None:
+ try:
+ normalize_retrieve_method("hybrid", default="rag")
+ except ValueError as exc:
+ assert "retrieve method must be 'rag' or 'llm'" in str(exc)
+ else:
+ raise AssertionError("unknown retrieve method should raise ValueError")
+
+
+def test_normalize_retrieve_ranking_accepts_case_insensitive_override() -> None:
+ assert normalize_retrieve_ranking(" SALIENCE ", default="similarity") == "salience"
+
+
+def test_normalize_retrieve_ranking_rejects_unknown_values() -> None:
+ try:
+ normalize_retrieve_ranking("random", default="similarity")
+ except ValueError as exc:
+ assert "retrieve ranking must be 'similarity' or 'salience'" in str(exc)
+ else:
+ raise AssertionError("unknown retrieve ranking should raise ValueError")
+
+
+def test_retrieve_method_override_is_wired_into_retrieve_pipeline() -> None:
+ source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ module = ast.parse(source)
+ retrieve_fn = next(
+ node for node in ast.walk(module) if isinstance(node, ast.AsyncFunctionDef) and node.name == "retrieve"
+ )
+ arg_names = [arg.arg for arg in retrieve_fn.args.args]
+
+ assert "method" in arg_names
+ assert "normalize_retrieve_method(method, default=self.retrieve_config.method)" in source
+ assert '"retrieve_llm" if retrieve_method == "llm" else "retrieve_rag"' in source
+ assert '"method": retrieve_method' in source
+
+
+def test_retrieve_ranking_override_is_wired_into_rag_item_recall() -> None:
+ source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ module = ast.parse(source)
+ retrieve_fn = next(
+ node for node in ast.walk(module) if isinstance(node, ast.AsyncFunctionDef) and node.name == "retrieve"
+ )
+ arg_names = [arg.arg for arg in retrieve_fn.args.args]
+ rag_recall_items = _async_function_source(source, "_rag_recall_items")
+
+ assert "ranking" in arg_names
+ assert "normalize_retrieve_ranking(ranking, default=self.retrieve_config.item.ranking)" in source
+ assert '"item_ranking": item_ranking' in source
+ assert '"item_ranking",' in source
+ assert 'ranking=state.get("item_ranking", self.retrieve_config.item.ranking)' in rag_recall_items
+
+
+def test_rag_retrieve_follows_category_references_with_scope_filters() -> None:
+ source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ rag_recall_items = _async_function_source(source, "_rag_recall_items")
+ merge_hits = _function_source(source, "_merge_referenced_item_hits")
+
+ assert 'getattr(self.retrieve_config.item, "use_category_references", False)' in rag_recall_items
+ assert "ref_ids = self._extract_referenced_item_ids(state)" in rag_recall_items
+ assert "store.memory_item_repo.list_items_by_ref_ids(ref_ids, where_filters)" in rag_recall_items
+ assert "items_pool.update(referenced_items)" in rag_recall_items
+ assert "self._merge_referenced_item_hits(" in rag_recall_items
+ assert "if item_id not in seen:" in merge_hits
+
+
+def test_llm_retrieve_respects_category_item_resource_toggles() -> None:
+ source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ workflow_source = _function_source(source, "_build_llm_retrieve_workflow")
+ route_category = _async_function_source(source, "_llm_route_category")
+ recall_items = _async_function_source(source, "_llm_recall_items")
+ recall_resources = _async_function_source(source, "_llm_recall_resources")
+
+ assert '"retrieve_category", "needs_retrieval", "active_query", "ctx", "store", "where"' in workflow_source
+ assert '"retrieve_item",' in workflow_source
+ assert '"retrieve_resource",' in workflow_source
+ assert 'not state.get("retrieve_category") or not state.get("needs_retrieval")' in route_category
+ assert 'not state.get("retrieve_item") or not state.get("needs_retrieval")' in recall_items
+ assert 'not state.get("retrieve_resource")' in recall_resources
+
+
+def test_route_intention_uses_independent_llm_profile() -> None:
+ source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ rag_workflow = _function_source(source, "_build_rag_retrieve_workflow")
+ llm_workflow = _function_source(source, "_build_llm_retrieve_workflow")
+
+ assert "self.retrieve_config.route_intention_llm_profile" in rag_workflow
+ assert "self.retrieve_config.route_intention_llm_profile" in llm_workflow
+ assert 'config={"chat_llm_profile": self.retrieve_config.sufficiency_check_llm_profile}' not in rag_workflow
+ assert 'config={"llm_profile": self.retrieve_config.sufficiency_check_llm_profile}' not in llm_workflow.split(
+ 'step_id="route_category"',
+ 1,
+ )[0]
+
+
+def test_direct_retrieve_query_text_is_trimmed_and_rejects_blank_values() -> None:
+ source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ extract_query_text = _function_source(source, "_extract_query_text")
+
+ assert "text = query.strip()" in extract_query_text
+ assert "text = content.strip()" in extract_query_text
+ assert "text = content.get(\"text\", \"\")" in extract_query_text
+ assert "not isinstance(text, str) or not text.strip()" in extract_query_text
+ assert "return text.strip()" in extract_query_text
+ assert 'raise ValueError("EMPTY")' in extract_query_text
+
+
+def test_direct_retrieve_type_hints_accept_string_query_items_without_raw_string_collection() -> None:
+ source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ module = ast.parse(source)
+ retrieve_fn = next(
+ node for node in ast.walk(module) if isinstance(node, ast.AsyncFunctionDef) and node.name == "retrieve"
+ )
+ query_arg = next(arg for arg in retrieve_fn.args.args if arg.arg == "queries")
+ retrieve_source = _async_function_source(source, "retrieve")
+ format_query_context = _function_source(source, "_format_query_context")
+ extract_query_text = _function_source(source, "_extract_query_text")
+
+ assert ast.unparse(query_arg.annotation) == "list[str | Mapping[str, Any]]"
+ assert "if not isinstance(queries, list):" in retrieve_source
+ assert "queries must be a non-empty list of strings or query objects" in retrieve_source
+ assert 'raise ValueError("empty_queries")' not in retrieve_source
+ assert "def _format_query_context(self, queries: Sequence[str | Mapping[str, Any]] | None)" in format_query_context
+ assert "elif isinstance(q, Mapping):" in format_query_context
+ assert "def _extract_query_text(query: str | Mapping[str, Any])" in extract_query_text
+ assert "if not isinstance(query, Mapping):" in extract_query_text
+
+
+def test_direct_retrieve_normalizes_all_query_items_before_workflow() -> None:
+ source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ retrieve_source = _async_function_source(source, "retrieve")
+ normalize_query_item = _function_source(source, "_normalize_query_item")
+
+ assert (
+ "normalized_queries = [self._normalize_query_item(query, index=index) for index, query in enumerate(queries)]"
+ in retrieve_source
+ )
+ assert "original_query = self._extract_query_text(normalized_queries[-1])" in retrieve_source
+ assert "context_queries_objs: list[dict[str, Any]] = normalized_queries[:-1]" in retrieve_source
+ assert '"skip_rewrite": len(normalized_queries) == 1' in retrieve_source
+ assert 'return {"role": "user", "content": text}' in normalize_query_item
+ assert 'role = query.get("role", "user")' in normalize_query_item
+ assert 'normalized_content = {"text": text.strip()}' in normalize_query_item
+ assert 'raise ValueError(f"queries[{index}].content.text must be a non-empty string")' in normalize_query_item
+
+
+def _async_function_source(source: str, name: str) -> str:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.AsyncFunctionDef) and node.name == name:
+ segment = ast.get_source_segment(source, node)
+ assert segment is not None
+ return segment
+ raise AssertionError(f"async function {name!r} not found")
+
+
+def _function_source(source: str, name: str) -> str:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ segment = ast.get_source_segment(source, node)
+ assert segment is not None
+ return segment
+ raise AssertionError(f"function {name!r} not found")
diff --git a/tests/test_salience.py b/tests/test_salience.py
index bb5d84a2..9df638d9 100644
--- a/tests/test_salience.py
+++ b/tests/test_salience.py
@@ -8,8 +8,25 @@
from __future__ import annotations
import hashlib
+import importlib.util
import math
from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+
+def _load_runtime_vector_module():
+ vector_path = Path(__file__).resolve().parents[1] / "src/memu/database/inmemory/vector.py"
+ spec = importlib.util.spec_from_file_location("memu_runtime_vector", vector_path)
+ assert spec is not None
+ assert spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+_RUNTIME_VECTOR = _load_runtime_vector_module()
+runtime_cosine_topk = _RUNTIME_VECTOR.cosine_topk
+runtime_cosine_topk_salience = _RUNTIME_VECTOR.cosine_topk_salience
# Inline implementations to avoid circular import issues during testing
@@ -207,3 +224,13 @@ def test_respects_k_limit(self) -> None:
results = cosine_topk_salience(query, corpus, k=2, recency_decay_days=30.0)
assert len(results) == 2
+
+
+def test_runtime_vector_topk_returns_empty_for_non_positive_k() -> None:
+ query = [1.0, 0.0]
+ corpus = [("id1", [1.0, 0.0]), ("id2", [0.0, 1.0])]
+ salience_corpus = [("id1", [1.0, 0.0], 1, datetime.now(UTC))]
+
+ assert runtime_cosine_topk(query, corpus, k=0) == []
+ assert runtime_cosine_topk(query, corpus, k=-1) == []
+ assert runtime_cosine_topk_salience(query, salience_corpus, k=0) == []
diff --git a/tests/test_scope_filters.py b/tests/test_scope_filters.py
new file mode 100644
index 00000000..e7d02f60
--- /dev/null
+++ b/tests/test_scope_filters.py
@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from memu.app.scope import (
+ concrete_scope_from_where,
+ exact_scope_from_where,
+ normalize_scope_where,
+ record_matches_scope,
+ scope_key_from_user,
+)
+from memu.utils.filtering import normalize_filter_value, split_filter_key
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class UserScope(BaseModel):
+ user_id: str | None = None
+ agent_id: str | None = None
+
+
+class TenantScope(BaseModel):
+ tenant_id: int | None = None
+ user_id: str | None = None
+
+
+class RequiredTenantScope(BaseModel):
+ tenant_id: int
+ user_id: str
+
+
+class ConstrainedScope(BaseModel):
+ user_id: str = Field(min_length=2)
+
+
+class ScopedRecord:
+ def __init__(self, **values: object) -> None:
+ for key, value in values.items():
+ setattr(self, key, value)
+
+
+def _assert_value_error(func: Any, expected: str) -> None:
+ try:
+ func()
+ except ValueError as exc:
+ assert expected in str(exc)
+ else:
+ raise AssertionError("Expected ValueError")
+
+
+def test_normalize_scope_where_accepts_equality_and_in_filters() -> None:
+ where = normalize_scope_where(
+ UserScope,
+ {
+ "user_id": "u1",
+ "agent_id__in": ["agent-a", "agent-b"],
+ "agent_id": None,
+ },
+ )
+
+ assert where == {
+ "user_id": "u1",
+ "agent_id__in": ("agent-a", "agent-b"),
+ }
+
+
+def test_normalize_scope_where_validates_and_normalizes_values_with_user_model() -> None:
+ where = normalize_scope_where(
+ TenantScope,
+ {
+ "tenant_id": "42",
+ "tenant_id__in": ["1", 2],
+ "user_id__in": "u1",
+ },
+ )
+
+ assert where == {
+ "tenant_id": 42,
+ "tenant_id__in": (1, 2),
+ "user_id__in": ("u1",),
+ }
+
+
+def test_normalize_scope_where_validates_single_fields_without_requiring_full_model() -> None:
+ where = normalize_scope_where(RequiredTenantScope, {"tenant_id": "42"})
+
+ assert where == {"tenant_id": 42}
+
+
+def test_normalize_scope_where_rejects_unknown_scope_fields() -> None:
+ _assert_value_error(
+ lambda: normalize_scope_where(UserScope, {"session_id": "s1"}),
+ "Unknown filter field 'session_id'",
+ )
+
+
+def test_normalize_scope_where_rejects_unsupported_filter_operators() -> None:
+ _assert_value_error(
+ lambda: normalize_scope_where(UserScope, {"user_id__ne": "u1"}),
+ "Unsupported filter operator '__ne'",
+ )
+
+
+def test_normalize_scope_where_rejects_non_iterable_in_filter_values() -> None:
+ _assert_value_error(
+ lambda: normalize_scope_where(UserScope, {"user_id__in": 123}),
+ "Filter 'user_id__in' must be a string or an iterable of values",
+ )
+
+
+def test_normalize_scope_where_rejects_values_invalid_for_user_model() -> None:
+ _assert_value_error(
+ lambda: normalize_scope_where(TenantScope, {"tenant_id": "not-an-int"}),
+ "Invalid filter value for field 'tenant_id'",
+ )
+ _assert_value_error(
+ lambda: normalize_scope_where(TenantScope, {"tenant_id__in": ["1", "not-an-int"]}),
+ "Invalid filter value for field 'tenant_id'",
+ )
+ _assert_value_error(
+ lambda: normalize_scope_where(ConstrainedScope, {"user_id": "x"}),
+ "Invalid filter value for field 'user_id'",
+ )
+
+
+def test_split_filter_key_rejects_empty_filter_fields() -> None:
+ _assert_value_error(lambda: split_filter_key(""), "Filter field must be a non-empty string")
+ _assert_value_error(lambda: split_filter_key("__in"), "Filter field must be a non-empty string")
+
+
+def test_in_filter_preserves_string_as_single_value() -> None:
+ field, operator = split_filter_key("user_id__in")
+
+ assert (field, operator) == ("user_id", "in")
+ assert normalize_filter_value(field, operator, "u1") == "u1"
+
+
+def test_scope_key_from_user_is_stable_and_ignores_none_values() -> None:
+ assert scope_key_from_user({"agent_id": None, "user_id": "u1"}) == (("user_id", '"u1"'),)
+ assert scope_key_from_user({"user_id": "u1", "agent_id": "a1"}) == (
+ ("agent_id", '"a1"'),
+ ("user_id", '"u1"'),
+ )
+
+
+def test_exact_scope_from_where_uses_only_equality_filters() -> None:
+ assert exact_scope_from_where({"user_id": "u1", "agent_id__in": ["a1", "a2"], "session_id": None}) == {
+ "user_id": "u1"
+ }
+
+
+def test_concrete_scope_from_where_requires_single_exact_scope() -> None:
+ assert concrete_scope_from_where(None) == {}
+ assert concrete_scope_from_where({"user_id": "u1", "agent_id": "a1"}) == {"user_id": "u1", "agent_id": "a1"}
+ assert concrete_scope_from_where({"user_id": "u1", "agent_id__in": ("a1", "a2")}) is None
+
+
+def test_record_matches_scope_requires_all_non_null_scope_fields() -> None:
+ record = ScopedRecord(user_id="u1", agent_id="a1")
+
+ assert record_matches_scope(record, {"user_id": "u1", "agent_id": "a1"})
+ assert record_matches_scope(record, {"user_id": "u1", "agent_id": None})
+ assert not record_matches_scope(record, {"user_id": "u2"})
+ assert not record_matches_scope(record, {"user_id": "u1", "agent_id": "a2"})
+
+
+def test_backend_records_preserve_scope_extras_for_sqlite_cache() -> None:
+ models_source = (ROOT / "src" / "memu" / "database" / "models.py").read_text(encoding="utf-8")
+ base_record = _class_node(models_source, "BaseRecord")
+ model_config = _class_assignment(base_record, "model_config")
+
+ assert isinstance(model_config, ast.Call)
+ assert getattr(model_config.func, "id", "") == "ConfigDict"
+ assert any(
+ keyword.arg == "extra"
+ and isinstance(keyword.value, ast.Constant)
+ and keyword.value.value == "allow"
+ for keyword in model_config.keywords
+ )
+
+ sqlite_paths = [
+ "src/memu/database/sqlite/repositories/resource_repo.py",
+ "src/memu/database/sqlite/repositories/memory_category_repo.py",
+ "src/memu/database/sqlite/repositories/memory_item_repo.py",
+ "src/memu/database/sqlite/repositories/category_item_repo.py",
+ ]
+ for relative_path in sqlite_paths:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ assert "**self._scope_kwargs_from(row)" in source or "**self._scope_kwargs_from(existing)" in source
+
+
+def test_persistent_category_schema_enforces_name_uniqueness_per_scope() -> None:
+ checked = [
+ ("src/memu/database/sqlite/schema.py", "build_sqlite_table_model"),
+ ("src/memu/database/postgres/schema.py", "build_table_model"),
+ ]
+
+ for relative_path, builder_name in checked:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ call = _assignment_call(source, "memory_category_model", builder_name)
+ keyword_values = {keyword.arg: keyword.value for keyword in call.keywords}
+
+ assert "unique_with_scope" in keyword_values
+ assert ast.unparse(keyword_values["unique_with_scope"]) == "['name']"
+
+
+def test_category_context_is_cached_per_scope() -> None:
+ service_source = (ROOT / "src" / "memu" / "app" / "service.py").read_text(encoding="utf-8")
+ memorize_source = (ROOT / "src" / "memu" / "app" / "memorize.py").read_text(encoding="utf-8")
+
+ assert "category_scope_key" in service_source
+ assert "category_cache" in service_source
+ assert "scope_key = scope_key_from_user(user_scope)" in memorize_source
+ assert "ctx.category_cache[scope_key]" in memorize_source
+
+
+def test_patch_update_and_delete_guard_memory_items_by_user_scope() -> None:
+ crud_source = (ROOT / "src" / "memu" / "app" / "crud.py").read_text(encoding="utf-8")
+ patch_source = (ROOT / "src" / "memu" / "app" / "patch.py").read_text(encoding="utf-8")
+
+ assert "record_matches_scope" in crud_source
+ assert "record_matches_scope" in patch_source
+ assert "self._ensure_item_matches_user_scope(item, user, memory_id)" in crud_source
+ assert 'self._ensure_item_matches_user_scope(item, state["user"], memory_id)' in crud_source
+ assert "self._ensure_item_matches_user_scope(item, user, memory_id)" in patch_source
+ assert 'self._ensure_item_matches_user_scope(item, state["user"], memory_id)' in patch_source
+
+
+def test_first_run_category_bootstrap_uses_concrete_scope_only() -> None:
+ crud_source = (ROOT / "src" / "memu" / "app" / "crud.py").read_text(encoding="utf-8")
+ retrieve_source = (ROOT / "src" / "memu" / "app" / "retrieve.py").read_text(encoding="utf-8")
+ list_items_source = _async_function_source(crud_source, "list_memory_items")
+ list_categories_source = _async_function_source(crud_source, "list_memory_categories")
+ retrieve_fn_source = _async_function_source(retrieve_source, "retrieve")
+
+ assert "bootstrap_scope = concrete_scope_from_where(where_filters)" not in list_items_source
+ assert "bootstrap_scope = concrete_scope_from_where(where_filters)" in list_categories_source
+ assert "await self._ensure_categories_ready(ctx, store, bootstrap_scope)" in list_categories_source
+ assert "bootstrap_scope = concrete_scope_from_where(where_filters)" in retrieve_fn_source
+ assert "if retrieve_category and bootstrap_scope is not None:" in retrieve_fn_source
+ assert "await self._ensure_categories_ready(ctx, store, bootstrap_scope)" in retrieve_fn_source
+
+
+def _async_function_source(source: str, name: str) -> str:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.AsyncFunctionDef) and node.name == name:
+ segment = ast.get_source_segment(source, node)
+ assert segment is not None
+ return segment
+ raise AssertionError(f"async function {name!r} not found")
+
+
+def _class_node(source: str, name: str) -> ast.ClassDef:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.ClassDef) and node.name == name:
+ return node
+ raise AssertionError(f"class {name!r} not found")
+
+
+def _class_assignment(class_node: ast.ClassDef, name: str) -> ast.expr:
+ for statement in class_node.body:
+ if not isinstance(statement, ast.Assign):
+ continue
+ if any(isinstance(target, ast.Name) and target.id == name for target in statement.targets):
+ return statement.value
+ raise AssertionError(f"class assignment {name!r} not found")
+
+
+def _assignment_call(source: str, target_name: str, call_name: str) -> ast.Call:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if not isinstance(node, ast.Assign):
+ continue
+ if not any(isinstance(target, ast.Name) and target.id == target_name for target in node.targets):
+ continue
+ if isinstance(node.value, ast.Call) and getattr(node.value.func, "id", "") == call_name:
+ return node.value
+ raise AssertionError(f"assignment call {target_name} = {call_name}(...) not found")
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
new file mode 100644
index 00000000..bc9bd90c
--- /dev/null
+++ b/tests/test_serialization.py
@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+import ast
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from memu.llm.wrapper import _extract_usage_from_raw_response
+from memu.utils.serialization import model_dump_without_embeddings
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+class ResponseRecord(BaseModel):
+ id: str
+ summary: str
+ embedding: list[float] | None
+ created_at: datetime
+ extra: dict[str, object] = Field(default_factory=dict)
+
+
+class CompletionTokenDetails(BaseModel):
+ reasoning_tokens: int
+ generated_at: datetime
+
+
+class PromptTokenDetails(BaseModel):
+ cached_tokens: int
+
+
+class UsageRecord(BaseModel):
+ prompt_tokens: int
+ completion_tokens: int
+ total_tokens: int
+ completion_tokens_details: CompletionTokenDetails
+ prompt_tokens_details: PromptTokenDetails
+
+
+class RawLLMResponse:
+ def __init__(self, usage: object) -> None:
+ self.choices = [{"finish_reason": "stop"}]
+ self.usage = usage
+
+
+class LegacyCompletionTokenDetails:
+ reasoning_tokens = 2
+
+ def model_dump(self) -> dict[str, object]:
+ return {
+ "reasoning_tokens": self.reasoning_tokens,
+ "generated_at": datetime(2026, 6, 5, 3, 0, tzinfo=timezone.utc),
+ }
+
+
+class LegacyUsageRecord:
+ prompt_tokens = 8
+ completion_tokens = 2
+ total_tokens = 10
+ completion_tokens_details = LegacyCompletionTokenDetails()
+ prompt_tokens_details = {"cached_tokens": 4}
+
+
+def test_model_dump_without_embeddings_returns_json_safe_values() -> None:
+ record = ResponseRecord(
+ id="m1",
+ summary="User likes concise answers.",
+ embedding=[0.1, 0.2, 0.3],
+ created_at=datetime(2026, 6, 5, 1, 40, tzinfo=timezone.utc),
+ extra={"score": 0.9},
+ )
+
+ data = model_dump_without_embeddings(record)
+
+ assert data == {
+ "id": "m1",
+ "summary": "User likes concise answers.",
+ "created_at": "2026-06-05T01:40:00Z",
+ "extra": {"score": 0.9},
+ }
+
+
+def test_memorize_response_relations_use_json_safe_public_dump() -> None:
+ source = (ROOT / "src/memu/app/memorize.py").read_text(encoding="utf-8")
+ function_source = _function_source(source, "_memorize_build_response")
+
+ assert "relations = [self._model_dump_without_embeddings(rel)" in function_source
+ assert "rel.model_dump()" not in function_source
+
+
+def test_tool_call_history_uses_json_safe_model_dump() -> None:
+ source = (ROOT / "src/memu/utils/tool.py").read_text(encoding="utf-8")
+ function_source = _function_source(source, "add_tool_call")
+
+ assert 'tool_call.model_dump(mode="json")' in function_source
+ assert "tool_call.model_dump())" not in function_source
+
+
+def test_llm_usage_breakdown_is_json_safe() -> None:
+ raw_response = RawLLMResponse(
+ UsageRecord(
+ prompt_tokens=10,
+ completion_tokens=4,
+ total_tokens=14,
+ completion_tokens_details=CompletionTokenDetails(
+ reasoning_tokens=3,
+ generated_at=datetime(2026, 6, 5, 2, 30, tzinfo=timezone.utc),
+ ),
+ prompt_tokens_details=PromptTokenDetails(cached_tokens=6),
+ )
+ )
+
+ usage = _extract_usage_from_raw_response(kind="chat", raw_response=raw_response)
+
+ assert usage["finish_reason"] == "stop"
+ assert usage["input_tokens"] == 10
+ assert usage["output_tokens"] == 4
+ assert usage["total_tokens"] == 14
+ assert usage["cached_input_tokens"] == 6
+ assert usage["reasoning_tokens"] == 3
+ assert usage["tokens_breakdown"] == {
+ "reasoning_tokens": 3,
+ "generated_at": "2026-06-05T02:30:00Z",
+ }
+ json.dumps(usage)
+
+
+def test_llm_usage_breakdown_accepts_legacy_model_dump_signature() -> None:
+ raw_response = RawLLMResponse(LegacyUsageRecord())
+
+ usage = _extract_usage_from_raw_response(kind="chat", raw_response=raw_response)
+
+ assert usage["cached_input_tokens"] == 4
+ assert usage["tokens_breakdown"] == {
+ "reasoning_tokens": 2,
+ "generated_at": "2026-06-05T03:00:00+00:00",
+ }
+ json.dumps(usage)
+
+
+def test_llm_usage_extraction_accepts_responses_style_token_names() -> None:
+ raw_response = {
+ "choices": [{"finish_reason": "length"}],
+ "usage": {
+ "input_tokens": 11,
+ "output_tokens": 7,
+ "total_tokens": 18,
+ "input_tokens_details": {"cached_tokens": 5},
+ "output_tokens_details": {
+ "reasoning_tokens": 2,
+ "accepted_prediction_tokens": 1,
+ },
+ },
+ }
+
+ usage = _extract_usage_from_raw_response(kind="chat", raw_response=raw_response)
+
+ assert usage["finish_reason"] == "length"
+ assert usage["input_tokens"] == 11
+ assert usage["output_tokens"] == 7
+ assert usage["total_tokens"] == 18
+ assert usage["cached_input_tokens"] == 5
+ assert usage["reasoning_tokens"] == 2
+ assert usage["tokens_breakdown"] == {
+ "reasoning_tokens": 2,
+ "accepted_prediction_tokens": 1,
+ }
+ json.dumps(usage)
+
+
+def test_memory_item_extra_defaults_are_per_record_factories() -> None:
+ checked = [
+ ("src/memu/database/models.py", "MemoryItem"),
+ ("src/memu/database/sqlite/models.py", "SQLiteMemoryItemModel"),
+ ("src/memu/database/postgres/models.py", "MemoryItemModel"),
+ ]
+
+ for relative_path, class_name in checked:
+ source = (ROOT / relative_path).read_text(encoding="utf-8")
+ assignment = _class_assignment(source, class_name, "extra")
+
+ assert isinstance(assignment, ast.Call), f"{relative_path}:{class_name}.extra must call Field"
+ assert getattr(assignment.func, "id", "") == "Field"
+ assert any(
+ keyword.arg == "default_factory"
+ and getattr(keyword.value, "id", None) == "dict"
+ for keyword in assignment.keywords
+ ), f"{relative_path}:{class_name}.extra must use Field(default_factory=dict)"
+
+
+def _function_source(source: str, name: str) -> str:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ segment = ast.get_source_segment(source, node)
+ assert segment is not None
+ return segment
+ raise AssertionError(f"function {name!r} not found")
+
+
+def _class_assignment(source: str, class_name: str, field_name: str) -> ast.expr:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if not isinstance(node, ast.ClassDef) or node.name != class_name:
+ continue
+ for statement in node.body:
+ if not isinstance(statement, ast.AnnAssign):
+ continue
+ if isinstance(statement.target, ast.Name) and statement.target.id == field_name:
+ assert statement.value is not None
+ return statement.value
+ raise AssertionError(f"{class_name}.{field_name} assignment not found")
diff --git a/tests/test_settings.py b/tests/test_settings.py
new file mode 100644
index 00000000..b3dbd0c3
--- /dev/null
+++ b/tests/test_settings.py
@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from pydantic import ValidationError
+
+from memu.app.settings import (
+ CategoryConfig,
+ DatabaseConfig,
+ LLMConfig,
+ LLMProfilesConfig,
+ MemorizeConfig,
+ RetrieveConfig,
+ default_api_key_env,
+ resolve_api_key,
+)
+
+
+@patch.dict("os.environ", {"OPENAI_API_KEY": "resolved-openai-key"})
+def test_resolve_api_key_environment_variable_names() -> None:
+ assert resolve_api_key("OPENAI_API_KEY") == "resolved-openai-key"
+
+
+@patch.dict("os.environ", {"OPENAI_API_KEY": " resolved-openai-key "})
+def test_resolve_api_key_trims_environment_variable_values() -> None:
+ assert resolve_api_key("OPENAI_API_KEY") == "resolved-openai-key"
+
+
+@patch.dict("os.environ", {"OPENAI_API_KEY": "resolved-openai-key"})
+def test_resolve_api_key_preserves_literal_api_keys() -> None:
+ assert resolve_api_key("sk-literal-key") == "sk-literal-key"
+
+
+def test_resolve_api_key_normalizes_missing_api_key_to_empty_string() -> None:
+ assert resolve_api_key(None) == ""
+
+
+def test_default_api_key_env_matches_provider_defaults() -> None:
+ assert default_api_key_env("openai") == "OPENAI_API_KEY"
+ assert default_api_key_env(" GROK ") == "XAI_API_KEY"
+
+
+def test_llm_config_normalizes_provider_before_defaults() -> None:
+ config = LLMConfig(provider=" GROK ")
+
+ assert config.provider == "grok"
+ assert config.base_url == "https://api.x.ai/v1"
+ assert config.api_key == "XAI_API_KEY"
+ assert config.chat_model == "grok-2-latest"
+
+
+def test_llm_config_normalizes_client_backend() -> None:
+ config = LLMConfig(client_backend=" HTTPX ")
+
+ assert config.client_backend == "httpx"
+
+
+def test_llm_config_rejects_unknown_client_backend() -> None:
+ try:
+ LLMConfig(client_backend="http")
+ except ValidationError as exc:
+ assert "client_backend" in str(exc)
+ else:
+ raise AssertionError("LLMConfig should reject unknown client_backend values")
+
+
+def test_llm_config_rejects_non_positive_embed_batch_size() -> None:
+ try:
+ LLMConfig(embed_batch_size=0)
+ except ValidationError as exc:
+ assert "embed_batch_size" in str(exc)
+ else:
+ raise AssertionError("LLMConfig should reject non-positive embed_batch_size")
+
+
+def test_retrieve_numeric_bounds_reject_invalid_values() -> None:
+ invalid_configs = [
+ {"category": {"top_k": 0}},
+ {"item": {"top_k": 0}},
+ {"item": {"recency_decay_days": 0}},
+ {"resource": {"top_k": 0}},
+ ]
+
+ for config in invalid_configs:
+ try:
+ RetrieveConfig(**config)
+ except ValidationError:
+ pass
+ else:
+ raise AssertionError(f"RetrieveConfig should reject invalid numeric bounds: {config}")
+
+
+def test_memorize_numeric_bounds_reject_invalid_values() -> None:
+ invalid_configs = [
+ {"category_assign_threshold": -0.01},
+ {"category_assign_threshold": 1.01},
+ {"default_category_summary_target_length": 0},
+ ]
+
+ for config in invalid_configs:
+ try:
+ MemorizeConfig(**config)
+ except ValidationError:
+ pass
+ else:
+ raise AssertionError(f"MemorizeConfig should reject invalid numeric bounds: {config}")
+
+ try:
+ CategoryConfig(name="facts", target_length=0)
+ except ValidationError:
+ pass
+ else:
+ raise AssertionError("CategoryConfig should reject non-positive target_length")
+
+
+def test_database_config_allows_sqlite_without_explicit_dsn() -> None:
+ config = DatabaseConfig(metadata_store={"provider": "sqlite"})
+
+ assert config.metadata_store.dsn is None
+ assert config.vector_index is not None
+ assert config.vector_index.provider == "bruteforce"
+
+
+def test_retrieve_config_has_independent_route_intention_profile() -> None:
+ config = RetrieveConfig(route_intention_llm_profile="router", sufficiency_check_llm_profile="judge")
+
+ assert config.route_intention_llm_profile == "router"
+ assert config.sufficiency_check_llm_profile == "judge"
+
+
+def test_workflow_profile_names_are_trimmed_and_reject_blank_values() -> None:
+ retrieve_config = RetrieveConfig(
+ route_intention_llm_profile=" router ",
+ sufficiency_check_llm_profile=" judge ",
+ llm_ranking_llm_profile=" ranker ",
+ )
+ memorize_config = MemorizeConfig(
+ preprocess_llm_profile=" preprocess ",
+ memory_extract_llm_profile=" extract ",
+ category_update_llm_profile=" summarize ",
+ )
+
+ assert retrieve_config.route_intention_llm_profile == "router"
+ assert retrieve_config.sufficiency_check_llm_profile == "judge"
+ assert retrieve_config.llm_ranking_llm_profile == "ranker"
+ assert memorize_config.preprocess_llm_profile == "preprocess"
+ assert memorize_config.memory_extract_llm_profile == "extract"
+ assert memorize_config.category_update_llm_profile == "summarize"
+
+ try:
+ RetrieveConfig(route_intention_llm_profile=" ")
+ except ValidationError as exc:
+ assert "route_intention_llm_profile" in str(exc)
+ else:
+ raise AssertionError("RetrieveConfig should reject blank route_intention_llm_profile")
+
+
+def test_llm_profile_names_are_trimmed_in_profile_map_keys() -> None:
+ config = LLMProfilesConfig.model_validate({" default ": {"api_key": "A"}})
+
+ assert "default" in config.profiles
+ assert config.profiles["default"].api_key == "A"
+ assert "embedding" in config.profiles
diff --git a/tests/test_sqlite.py b/tests/test_sqlite.py
index 3031c56b..02f08ec0 100644
--- a/tests/test_sqlite.py
+++ b/tests/test_sqlite.py
@@ -1,42 +1,44 @@
-"""Test SQLite database backend for MemU."""
+#!/usr/bin/env python3
+"""Opt-in live LLM smoke test for the SQLite backend."""
+from __future__ import annotations
+
+import asyncio
import os
+import sys
import tempfile
+from pathlib import Path
+from typing import Any
-from memu.app import MemoryService
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+RUN_LIVE_LLM_TESTS_ENV = "MEMU_RUN_LIVE_LLM_TESTS"
+# Add src to sys.path before importing memu from a source checkout.
+src_path = str(PROJECT_ROOT / "src")
+if src_path not in sys.path:
+ sys.path.insert(0, src_path)
-def _print_results(title: str, result: dict) -> None:
- print(f"\n[SQLITE] RETRIEVED - {title}")
- print(" Categories:")
- for cat in result.get("categories", [])[:3]:
- print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...")
- print(" Items:")
- for item in result.get("items", [])[:3]:
- print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...")
- if result.get("resources"):
- print(" Resources:")
- for res in result.get("resources", [])[:3]:
- print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...")
+async def run_sqlite_workflow() -> None:
+ """Run the SQLite-backed memorize/retrieve smoke workflow against a real LLM."""
+ from memu import MemoryService
-async def main():
- """Test with SQLite storage."""
api_key = os.environ.get("OPENAI_API_KEY")
- file_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "example", "example_conversation.json"))
+ if not api_key:
+ msg = "OPENAI_API_KEY is required for the SQLite live LLM workflow"
+ raise RuntimeError(msg)
- # Create a temporary SQLite database file
- with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp:
- sqlite_path = tmp.name
+ file_path = Path(__file__).resolve().parent / "example" / "example_conversation.json"
- sqlite_dsn = f"sqlite:///{sqlite_path}"
+ with tempfile.TemporaryDirectory() as tmpdir:
+ sqlite_path = Path(tmpdir) / "memu.db"
+ sqlite_dsn = f"sqlite:///{sqlite_path.as_posix()}"
- print("\n" + "=" * 60)
- print("[SQLITE] Starting test...")
- print(f"[SQLITE] DSN: {sqlite_dsn}")
- print("=" * 60)
+ print("\n" + "=" * 60)
+ print("[SQLITE] Starting test...")
+ print(f"[SQLITE] DSN: {sqlite_dsn}")
+ print("=" * 60)
- try:
service = MemoryService(
llm_profiles={"default": {"api_key": api_key}},
database_config={
@@ -44,47 +46,67 @@ async def main():
"provider": "sqlite",
"dsn": sqlite_dsn,
},
- # SQLite uses brute-force vector search
"vector_index": {"provider": "bruteforce"},
},
retrieve_config={"method": "rag"},
)
- # Memorize
print("\n[SQLITE] Memorizing...")
- memory = await service.memorize(resource_url=file_path, modality="conversation", user={"user_id": "123"})
+ memory = await service.memorize(resource_url=str(file_path), modality="conversation", user={"user_id": "123"})
for cat in memory.get("categories", []):
print(f" - {cat.get('name')}: {(cat.get('summary') or '')[:80]}...")
- queries = [
- {"role": "user", "content": {"text": "Tell me about preferences"}},
- {"role": "assistant", "content": {"text": "Sure, I'll tell you about their preferences"}},
- {
- "role": "user",
- "content": {"text": "What are they"},
- }, # This is the query that will be used to retrieve the memory
- ]
+ queries = _sample_queries()
- # RAG-based retrieval
service.retrieve_config.method = "rag"
result_rag = await service.retrieve(queries=queries, where={"user_id": "123"})
_print_results("RAG", result_rag)
- # LLM-based retrieval
service.retrieve_config.method = "llm"
result_llm = await service.retrieve(queries=queries, where={"user_id": "123"})
_print_results("LLM", result_llm)
print("\n[SQLITE] Test completed!")
- finally:
- # Clean up the temporary database file
- if os.path.exists(sqlite_path):
- os.unlink(sqlite_path)
- print(f"[SQLITE] Cleaned up temporary database: {sqlite_path}")
+async def test_sqlite_full_workflow() -> None:
+ """Opt-in pytest integration check for the SQLite backend and a real LLM."""
+ import pytest
+
+ if os.environ.get(RUN_LIVE_LLM_TESTS_ENV) != "1":
+ pytest.skip(f"Set {RUN_LIVE_LLM_TESTS_ENV}=1 to run live LLM storage workflows")
+ if not os.environ.get("OPENAI_API_KEY"):
+ pytest.skip("OPENAI_API_KEY is required for live LLM storage workflows")
+
+ await run_sqlite_workflow()
+
+
+def _sample_queries() -> list[dict[str, dict[str, str] | str]]:
+ return [
+ {"role": "user", "content": {"text": "Tell me about preferences"}},
+ {"role": "assistant", "content": {"text": "Sure, I'll tell you about their preferences"}},
+ {"role": "user", "content": {"text": "What are they"}},
+ ]
-if __name__ == "__main__":
- import asyncio
- asyncio.run(main())
+def _print_results(title: str, result: dict[str, Any]) -> None:
+ print(f"\n[SQLITE] RETRIEVED - {title}")
+ print(" Categories:")
+ for cat in result.get("categories", [])[:3]:
+ print(f" - {cat.get('name')}: {(cat.get('summary') or cat.get('description', ''))[:80]}...")
+ print(" Items:")
+ for item in result.get("items", [])[:3]:
+ print(f" - [{item.get('memory_type')}] {item.get('summary', '')[:100]}...")
+ if result.get("resources"):
+ print(" Resources:")
+ for res in result.get("resources", [])[:3]:
+ print(f" - [{res.get('modality')}] {res.get('url', '')[:80]}...")
+
+
+def main() -> int:
+ asyncio.run(run_sqlite_workflow())
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/test_workflow_contracts.py b/tests/test_workflow_contracts.py
new file mode 100644
index 00000000..a83cd086
--- /dev/null
+++ b/tests/test_workflow_contracts.py
@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+from memu.workflow.pipeline import PipelineManager
+from memu.workflow.step import WorkflowStep
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _noop_step(state: dict[str, object], context: object) -> dict[str, object]:
+ return state
+
+
+def test_pipeline_manager_validates_all_llm_profile_config_keys() -> None:
+ manager = PipelineManager(available_capabilities={"llm"}, llm_profiles={"default"})
+
+ for profile_key in ("llm_profile", "chat_llm_profile", "embed_llm_profile"):
+ try:
+ manager.register(
+ f"pipeline_{profile_key}",
+ [
+ WorkflowStep(
+ step_id="step",
+ role="test",
+ handler=_noop_step,
+ capabilities={"llm"},
+ config={profile_key: "missing"},
+ )
+ ],
+ )
+ except ValueError as exc:
+ assert f"unknown {profile_key} 'missing'" in str(exc)
+ else:
+ raise AssertionError(f"PipelineManager should reject unknown {profile_key}")
+
+
+def test_pipeline_manager_rejects_blank_and_non_string_llm_profile_references() -> None:
+ manager = PipelineManager(available_capabilities={"llm"}, llm_profiles={"default"})
+
+ for value in ("", " ", 123):
+ try:
+ manager.register(
+ f"pipeline_{value!r}",
+ [
+ WorkflowStep(
+ step_id="step",
+ role="test",
+ handler=_noop_step,
+ capabilities={"llm"},
+ config={"chat_llm_profile": value},
+ )
+ ],
+ )
+ except ValueError as exc:
+ assert "profile name must be non-empty" in str(exc)
+ else:
+ raise AssertionError("PipelineManager should reject invalid profile references")
+
+
+def test_pipeline_manager_revalidates_profile_references_on_config_mutation() -> None:
+ manager = PipelineManager(available_capabilities={"llm"}, llm_profiles={"default"})
+ manager.register(
+ "pipeline",
+ [
+ WorkflowStep(
+ step_id="step",
+ role="test",
+ handler=_noop_step,
+ capabilities={"llm"},
+ config={"chat_llm_profile": "default"},
+ )
+ ],
+ )
+
+ try:
+ manager.config_step("pipeline", "step", {"chat_llm_profile": " "})
+ except ValueError as exc:
+ assert "profile name must be non-empty" in str(exc)
+ else:
+ raise AssertionError("PipelineManager should reject invalid profile references during mutation")
+
+
+def test_llm_retrieve_route_intention_step_declares_route_intention_dependency() -> None:
+ source = (ROOT / "src/memu/app/retrieve.py").read_text(encoding="utf-8")
+ workflow_source = _function_source(source, "_build_llm_retrieve_workflow")
+ route_step_source = workflow_source.split('step_id="route_category"', 1)[0]
+
+ assert 'requires={"route_intention", "original_query", "context_queries", "skip_rewrite"}' in route_step_source
+
+
+def _function_source(source: str, name: str) -> str:
+ module = ast.parse(source)
+ for node in ast.walk(module):
+ if isinstance(node, ast.FunctionDef) and node.name == name:
+ segment = ast.get_source_segment(source, node)
+ assert segment is not None
+ return segment
+ raise AssertionError(f"function {name!r} not found")
diff --git a/uv.lock b/uv.lock
index 76e7b0c6..5a45accc 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,6 +1,6 @@
version = 1
revision = 3
-requires-python = ">=3.13"
+requires-python = ">=3.12"
[[package]]
name = "alembic"
@@ -929,14 +929,12 @@ wheels = [
[[package]]
name = "memu-py"
-version = "1.5.0"
+version = "1.5.1"
source = { editable = "." }
dependencies = [
{ name = "alembic" },
{ name = "defusedxml" },
{ name = "httpx" },
- { name = "langchain-core" },
- { name = "lazyllm" },
{ name = "numpy" },
{ name = "openai" },
{ name = "pendulum" },
@@ -952,6 +950,9 @@ langgraph = [
{ name = "langchain-core" },
{ name = "langgraph" },
]
+lazyllm = [
+ { name = "lazyllm" },
+]
postgres = [
{ name = "pgvector" },
{ name = "sqlalchemy", extra = ["postgresql-psycopgbinary"] },
@@ -999,10 +1000,9 @@ requires-dist = [
{ name = "claude-agent-sdk", marker = "extra == 'claude'", specifier = ">=0.1.24" },
{ name = "defusedxml", specifier = ">=0.7.1" },
{ name = "httpx", specifier = ">=0.28.1" },
- { name = "langchain-core", specifier = ">=1.2.7" },
- { name = "langchain-core", marker = "extra == 'langgraph'", specifier = ">=0.1.0" },
- { name = "langgraph", marker = "extra == 'langgraph'", specifier = ">=0.0.10" },
- { name = "lazyllm", specifier = ">=0.7.3" },
+ { name = "langchain-core", marker = "extra == 'langgraph'", specifier = ">=1.2.7" },
+ { name = "langgraph", marker = "extra == 'langgraph'", specifier = ">=1.0.6" },
+ { name = "lazyllm", marker = "extra == 'lazyllm'", specifier = ">=0.7.3" },
{ name = "numpy", specifier = ">=2.3.4" },
{ name = "openai", specifier = ">=2.8.0" },
{ name = "pendulum", specifier = ">=3.1.0" },
@@ -1011,7 +1011,7 @@ requires-dist = [
{ name = "sqlalchemy", extras = ["postgresql-psycopgbinary"], marker = "extra == 'postgres'", specifier = ">=2.0.36" },
{ name = "sqlmodel", specifier = ">=0.0.27" },
]
-provides-extras = ["postgres", "langgraph", "claude"]
+provides-extras = ["postgres", "langgraph", "lazyllm", "claude"]
[package.metadata.requires-dev]
dev = [