diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..12db13b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,27 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + groups: + python-dependencies: + patterns: ["*"] + + - package-ecosystem: npm + directory: "/sdk/javascript" + schedule: + interval: weekly + groups: + javascript-dependencies: + patterns: ["*"] + + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + + - package-ecosystem: docker + directory: "/" + schedule: + interval: weekly diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..8a9d6d6 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,24 @@ +name: CodeQL + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: '31 4 * * 1' + +permissions: + contents: read + security-events: write + +jobs: + analyze: + name: Analyze Python and JavaScript + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: github/codeql-action/init@v4 + with: + languages: python,javascript-typescript + - uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b8e8bfe..d3db0e8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,15 +4,41 @@ on: push: pull_request: +permissions: + contents: read + jobs: python-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: - python-version: '3.12' + python-version: ${{ matrix.python-version }} cache: 'pip' - run: python -m pip install --upgrade pip - run: python -m pip install -r requirements.txt - run: python -m pytest -q + - run: python -m py_compile app/*.py verifier/*.py + + javascript-sdk: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: '20' + - run: node --check sdk/javascript/index.js + - run: npm pack --dry-run + working-directory: sdk/javascript + + docker-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Build LoopGrid image + run: docker build -t loopgrid-ci . diff --git a/CHANGELOG.md b/CHANGELOG.md index 45da551..6adc116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +# LoopGrid v0.8.1 — OTLP Interoperability + +## Added / changed + +- `/v1/traces` accepts OTLP/HTTP binary protobuf (`application/x-protobuf`) in addition to JSON. +- OTLP/HTTP gzip request bodies are supported with decoded-size enforcement. +- JSON and protobuf requests share one normalized trace-to-evidence mapping. +- OTLP `AnyValue` parsing covers scalar, array, key/value-list and byte values. +- Successful OTLP responses use the standard `ExportTraceServiceResponse` encoding and mirror the request content type. +- `X-LoopGrid-Accepted` exposes the accepted-span count without adding non-standard fields to the OTLP response body. +- Added malformed-body, unsupported-media-type, gzip, parity and protobuf regression tests. +- CI updated to current GitHub Actions and multiple supported Python versions. +- Added Dependabot and CodeQL workflow configuration. + +## Compatibility + +- Evidence Bundle v2, Evidence Profile `3.0-draft`, signing, checkpoints and the standalone verifier are unchanged. +- GitHub Marketplace verifier `v1.0.0` does not require an update for this server-ingestion change. +- Python and JavaScript SDK package APIs remain `0.8.0`; they are not republished unless their package code changes. + +--- + # LoopGrid v0.8.0 — Design Partner Release ## Focus diff --git a/README.md b/README.md index bb14753..7cd4fa5 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,14 @@ CAPTURE → SEAL → VERIFY → INVESTIGATE → REVIEW → REPLAY → PROVE The append-only decision lifecycle includes `decision_created`, `model_completed`, `policy_evaluated`, human review events, tool/action evidence, `outcome_observed` and optional later replay/adjudication events. LoopGrid derives the current state from that history and rejects impossible consequential-event ordering. +## v0.8.1 interoperability update + +- `/v1/traces` now accepts standard OTLP/HTTP binary protobuf and JSON trace payloads. +- `Content-Encoding: gzip` is supported for OTLP/HTTP trace requests. +- OTLP success responses now use the standard `ExportTraceServiceResponse` encoding and match the request `Content-Type`. +- Decoded OTLP request bodies are bounded by `LOOPGRID_MAX_REQUEST_BODY_BYTES` to limit decompression expansion. +- Existing evidence, signing, bundle and verifier formats are unchanged from v0.8.0. + ## v0.8 highlights - PostgreSQL-backed Docker deployment path. @@ -41,7 +49,7 @@ The append-only decision lifecycle includes `decision_created`, `model_completed - FULL / REDACTED / PROOF-ONLY privacy modes. - AES-256-GCM disclosure vault with erasable payloads. - Deterministic policy provenance and human-review evidence. -- REST, Python, TypeScript/JavaScript, OpenTelemetry/OTLP and MCP paths. +- REST, Python, TypeScript/JavaScript, OpenTelemetry/OTLP (HTTP protobuf + JSON) and MCP paths. - Optional live OpenAI/Anthropic replay/provider validation helpers. - Optional RFC3161 timestamp and AWS KMS adapter paths. @@ -146,6 +154,27 @@ Portable consistency proof and signer trust are separate concepts. A bundle can Encrypted disclosures can be erased later without rewriting the signed evidence chain. + +## OTLP/HTTP trace ingestion + +LoopGrid accepts OTLP trace exports on the standard `/v1/traces` path using either: + +- `Content-Type: application/x-protobuf` (recommended/default for many OpenTelemetry SDKs) +- `Content-Type: application/json` +- optional `Content-Encoding: gzip` for either encoding + +Example environment configuration: + +```bash +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:8000/v1/traces +export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf +export OTEL_EXPORTER_OTLP_TRACES_HEADERS="X-LoopGrid-Key=lg_live_...,X-LoopGrid-Workspace=default" +``` + +A successful OTLP export returns the standard empty `ExportTraceServiceResponse`. The response header `X-LoopGrid-Accepted` reports how many spans were mapped into LoopGrid decisions. LoopGrid's OTLP adapter preserves both current `gen_ai.provider.name` and the older `gen_ai.system` provider attribute when present. + +This ingestion compatibility does not change the LoopGrid evidence bundle, hash-chain, signing or offline-verifier contracts. + ## Auth model Service API-key scopes: @@ -166,7 +195,7 @@ In production mode v0.8 refuses startup when auth is disabled or bootstrap/platf ## Supported integrations -Validated/foundation paths include REST, Python SDK, TypeScript/JavaScript SDK, OpenAI helper, Anthropic helper, LangGraph helper, OpenTelemetry/OTLP and allow-listed MCP proxy/ingestion. +Validated/foundation paths include REST, Python SDK, TypeScript/JavaScript SDK, OpenAI helper, Anthropic helper, LangGraph helper, OpenTelemetry/OTLP HTTP (protobuf + JSON) and allow-listed MCP proxy/ingestion. The canonical LoopGrid evidence schema remains provider-neutral. Do not couple your historical evidence model to a single vendor's telemetry schema. diff --git a/RELEASE-CHECKLIST-v0.8.1.md b/RELEASE-CHECKLIST-v0.8.1.md new file mode 100644 index 0000000..4db4886 --- /dev/null +++ b/RELEASE-CHECKLIST-v0.8.1.md @@ -0,0 +1,45 @@ +# LoopGrid v0.8.1 — OTLP Interoperability Release Checklist + +This is a backward-compatible server-ingestion update from the public v0.8.0 Design Partner Release. +The evidence bundle, signing format, verifier contract, Python SDK API and JavaScript SDK API are intentionally unchanged. + +## Before changing GitHub + +- [ ] Keep the current public `main` as the rollback point. Do not delete or overwrite the existing `v0.8.0` tag/release if present. +- [ ] Upload the v0.8.1 changed/new files with their repository paths preserved. +- [ ] Confirm `app/version.py` reports `0.8.1-design-partner`. +- [ ] Confirm `sdk/python/pyproject.toml`, `sdk/python/loopgrid/__init__.py`, and `sdk/javascript/package.json` remain at `0.8.0` because the SDK package code did not change. +- [ ] Do not modify the separate `loopgridio/loopgrid-evidence-verify` Marketplace repository for this release. + +## Required CI checks + +- [ ] GitHub `tests` workflow is green for Python 3.10, 3.12 and 3.13. +- [ ] JavaScript SDK syntax / `npm pack --dry-run` job is green. +- [ ] Docker image build job is green. +- [ ] CodeQL workflow is green or has no blocking findings caused by this patch. +- [ ] Dependabot configuration is accepted by GitHub. + +## OTLP acceptance checks + +- [ ] OTLP/HTTP JSON request succeeds and returns `{}` with `Content-Type: application/json`. +- [ ] OTLP/HTTP protobuf request succeeds and returns an empty `ExportTraceServiceResponse` with `Content-Type: application/x-protobuf`. +- [ ] Gzipped protobuf request succeeds. +- [ ] Malformed protobuf is rejected cleanly. +- [ ] Unsupported content type/encoding is rejected cleanly. +- [ ] Decoded gzip body limit is enforced. +- [ ] JSON and protobuf inputs map equivalent semantic attributes into LoopGrid evidence. +- [ ] A standard OpenTelemetry Python OTLP/HTTP protobuf exporter can send a span successfully. +- [ ] Evidence generated from protobuf ingestion exports successfully and verifies with the existing LoopGrid offline verifier / GitHub Action v1.0.0 contract. + +## Release + +- [ ] Commit to `main` only after CI is green. +- [ ] Create a GitHub release/tag `v0.8.1` from the tested commit. +- [ ] Release note: OTLP/HTTP protobuf + JSON + gzip interoperability; no evidence-format change. +- [ ] Do **not** republish PyPI `loopgrid` or npm `@cybertechsoft/loopgrid` solely for this backend change. +- [ ] Rebuild any deployed/self-hosted Docker image from the new GitHub source. +- [ ] Update loopgrid.io integration/docs copy only after the public GitHub checks are green. + +## Rollback + +If GitHub CI or a deployment smoke test fails, revert the v0.8.1 commit(s) on `main` or redeploy the previously validated v0.8.0 source. Do not rewrite published tags. diff --git a/UPGRADE-INSTRUCTIONS-v0.8.1.md b/UPGRADE-INSTRUCTIONS-v0.8.1.md new file mode 100644 index 0000000..20d7657 --- /dev/null +++ b/UPGRADE-INSTRUCTIONS-v0.8.1.md @@ -0,0 +1,59 @@ +# Safe upgrade instructions — public LoopGrid v0.8.0 → v0.8.1 + +The patch was built from the direct GitHub `loopgrid-main.zip` baseline supplied on 2026-09-18. Do not delete the current repository or overwrite any published tag. + +## Files to add + +- `.github/dependabot.yml` +- `.github/workflows/codeql.yml` +- `app/otlp.py` +- `tests/test_otlp_http.py` +- `RELEASE-CHECKLIST-v0.8.1.md` +- `VALIDATION-RELEASE-RECORD-v0.8.1.md` +- `UPGRADE-INSTRUCTIONS-v0.8.1.md` + +## Existing files to replace + +- `.github/workflows/tests.yml` +- `CHANGELOG.md` +- `README.md` +- `app/main.py` +- `app/schemas.py` +- `app/static/app.js` +- `app/static/index.html` +- `app/version.py` +- `requirements.txt` +- `scripts/real_scenario_test.py` +- `tests/test_flow.py` + +## Files that must NOT be changed for this release + +- `verifier/loopgrid_verify.py` +- `sdk/python/pyproject.toml` +- `sdk/python/loopgrid/__init__.py` +- `sdk/python/loopgrid/client.py` +- `sdk/javascript/package.json` +- `sdk/javascript/index.js` +- `sdk/javascript/index.d.ts` +- evidence-profile and bundle-format documents/schemas unless a separate reviewed change is intended + +The Python and JavaScript SDK packages therefore stay at `0.8.0`. The server runtime becomes `0.8.1-design-partner`. + +## Safest GitHub update sequence + +1. Confirm the current public repository is still `cybertechsoft/loopgrid` and keep the downloaded `loopgrid-main.zip` as rollback backup. +2. Apply only the files listed above, preserving their exact repository paths. +3. Commit with a message such as `Add OTLP HTTP protobuf interoperability`. +4. Open **Actions** immediately after the commit. Do not create a release yet. +5. Wait for the `tests` workflow and `CodeQL` workflow to finish. +6. Confirm all Python matrix jobs, JavaScript SDK job and Docker build job are green. +7. If any job is red, stop and fix/revert before tagging. The existing public v0.8.0 release remains the rollback baseline. +8. When green, test `/health` and confirm the runtime reports `0.8.1-design-partner`. +9. Run one OTLP JSON request and one standard OTLP/HTTP protobuf exporter request against the deployment being evaluated. +10. Export evidence from the protobuf-created decision and verify it with the existing `loopgrid-evidence-verify@v1.0.0` Action or the unchanged standalone verifier. +11. Only then create the GitHub release/tag `v0.8.1`. +12. Do not publish new PyPI/npm versions unless SDK package code is intentionally changed later. + +## Website update timing + +Do not update the live website first. After GitHub v0.8.1 is green and validated, update the website integration/docs copy from `OTLP/HTTP JSON` to `OTLP/HTTP protobuf + JSON`, and document gzip support. No website claim should say every OpenTelemetry framework has been validated unless it has actually been tested. diff --git a/VALIDATION-RELEASE-RECORD-v0.8.1.md b/VALIDATION-RELEASE-RECORD-v0.8.1.md new file mode 100644 index 0000000..e906451 --- /dev/null +++ b/VALIDATION-RELEASE-RECORD-v0.8.1.md @@ -0,0 +1,47 @@ +# LoopGrid v0.8.1 — OTLP Interoperability Validation Record + +**Baseline:** public `cybertechsoft/loopgrid` download supplied on 2026-09-18 +**Runtime candidate:** `0.8.1-design-partner` +**Evidence profile:** `3.0-draft` +**Bundle contract:** `loopgrid/evidence-bundle/2` (unchanged) + +## Validation completed in the build workspace + +| Check | Result | Notes | +|---|---|---| +| Python test suite | PASS | 55 tests passed. | +| Python syntax compile | PASS | `app/*.py` and `verifier/*.py`. | +| JavaScript SDK syntax | PASS | `node --check sdk/javascript/index.js`. | +| UI JS syntax | PASS | `node --check app/static/app.js`. | +| npm dry-run pack | PASS | Existing SDK remains `@cybertechsoft/loopgrid@0.8.0`; 5 expected files only. | +| OTLP/HTTP JSON | PASS | Standard `{}` `ExportTraceServiceResponse` JSON body; accepted count in response header. | +| OTLP/HTTP protobuf | PASS | Binary `ExportTraceServiceRequest` accepted; binary `ExportTraceServiceResponse` returned. | +| OTLP gzip | PASS | Gzipped protobuf accepted. | +| Malformed protobuf | PASS | Clean 400 with protobuf `google.rpc.Status` body. | +| Unsupported media/encoding | PASS | Clean 415 responses. | +| Decompressed body limit | PASS | Oversized decoded gzip body rejected with 413. | +| JSON/protobuf mapping parity | PASS | Core decision/model/context/action attributes map equivalently. | +| Standard OpenTelemetry Python HTTP exporter | PASS | `opentelemetry-exporter-otlp-proto-http` successfully exported a real span to `/v1/traces`; LoopGrid recorded the expected provider, model, service and tool attributes. | +| Evidence after protobuf ingestion | PASS | Exported bundle verifies with the existing standalone verifier contract; Bundle v2 and Ed25519 unchanged. | +| Full external HTTP real-scenario validation | PASS | Complete decision lifecycle, policy, human review, action/outcome, chain verification, tamper detection, checkpoint, signer-pinned offline verification, withheld export and OTLP JSON ingestion all passed. | +| YAML parse | PASS | CI, Dependabot and CodeQL YAML parse successfully. | +| Docker build in this build workspace | NOT RUN | Docker CLI is not available in this environment. A Docker build job is included in GitHub CI and must be green before tagging `v0.8.1`. | +| CodeQL execution | NOT RUN LOCALLY | CodeQL workflow is included and must be green/triaged on GitHub before tagging. | + +## Compatibility boundary + +No changes were made to: + +- Evidence Bundle v2 structure +- Evidence Profile `3.0-draft` +- Ed25519 signing semantics +- SHA-256 workspace chain semantics +- checkpoint representation +- standalone verifier behavior +- Python SDK package source/API +- JavaScript SDK package source/API +- GitHub Marketplace verifier Action `v1.0.0` + +## Release gate + +Do not tag `v0.8.1` until the pushed GitHub commit has green Python, JavaScript SDK and Docker jobs, and the CodeQL run has no new blocking finding caused by this patch. diff --git a/app/main.py b/app/main.py index 5e9d074..ea4741a 100644 --- a/app/main.py +++ b/app/main.py @@ -14,7 +14,7 @@ from .db import Base, engine, get_db from .models import LedgerEvent, Workspace, ApiKey, AuditLog, PolicyVersion, PayloadBlob, CheckpointRecord, UserAccount, WorkspaceMembership, UserSession, UsageEvent -from .schemas import DecisionCreate, EventCreate, ReplayRequest, WorkspaceCreate, WorkspaceUpdate, ApiKeyCreate, ApiKeyRotateRequest, OTelIngest, PolicyCreate, PolicyEvaluate, ReviewRequest, MCPIngest, ErasePayloadRequest, HealthResponse, ReadyResponse, WorkspaceResponse, DashboardResponse, ApiKeyIssued, IntegrityResponse, DecisionResult, DecisionDetail, CheckpointResponse, RetentionResult, SystemInfoResponse, BootstrapUserRequest, LoginRequest, UserCreateRequest, MembershipRequest, AuthSessionResponse, OkResponse, HumanPrincipalResponse, MemberResponse, ApiKeyMetadataResponse, ApiKeyRotateResponse, UsageSummaryResponse, AuditEntryResponse, PolicyResponse, PolicyEvaluationResponse, DecisionSummaryResponse, LedgerEventResponse, ReviewItemResponse, ReviewResultResponse, ReplayResultResponse, MCPIngestResponse, OTLPAcceptResponse, PayloadStatusResponse, PayloadEraseResponse, IngestResult, PilotReadinessResponse +from .schemas import DecisionCreate, EventCreate, ReplayRequest, WorkspaceCreate, WorkspaceUpdate, ApiKeyCreate, ApiKeyRotateRequest, OTelIngest, PolicyCreate, PolicyEvaluate, ReviewRequest, MCPIngest, ErasePayloadRequest, HealthResponse, ReadyResponse, WorkspaceResponse, DashboardResponse, ApiKeyIssued, IntegrityResponse, DecisionResult, DecisionDetail, CheckpointResponse, RetentionResult, SystemInfoResponse, BootstrapUserRequest, LoginRequest, UserCreateRequest, MembershipRequest, AuthSessionResponse, OkResponse, HumanPrincipalResponse, MemberResponse, ApiKeyMetadataResponse, ApiKeyRotateResponse, UsageSummaryResponse, AuditEntryResponse, PolicyResponse, PolicyEvaluationResponse, DecisionSummaryResponse, LedgerEventResponse, ReviewItemResponse, ReviewResultResponse, ReplayResultResponse, MCPIngestResponse, PayloadStatusResponse, PayloadEraseResponse, IngestResult, PilotReadinessResponse from .service import create_decision, add_event, get_events, list_decisions, summarize_decision, dashboard, evidence_coverage from .ledger import verify_ledger from .evidence import build_bundle, render_report @@ -33,6 +33,15 @@ from .user_auth import bearer_token_context,current_principal,create_user,add_membership,issue_session,verify_password,authenticate_session,revoke_session from .usage import usage_summary, record_usage from .lifecycle import LifecycleError +from .otlp import ( + OTLPHTTPError, + OTLP_JSON, + OTLP_PROTOBUF, + decode_trace_request, + error_response_body, + media_type_from_header, + success_response_body, +) STARTED=time.time() assert_production_safe(settings) @@ -65,7 +74,7 @@ async def lifespan(app:FastAPI): allow_credentials=True, allow_methods=["GET","POST","PUT","PATCH","DELETE","OPTIONS"], allow_headers=["Authorization","Content-Type","X-LoopGrid-Key","X-LoopGrid-Workspace","X-Request-ID"], - expose_headers=["X-Request-ID"], + expose_headers=["X-Request-ID","X-LoopGrid-Accepted"], ) STATIC=Path(__file__).parent/"static"; app.mount("/static",StaticFiles(directory=STATIC),name="static") @@ -366,23 +375,59 @@ def ingest_otel(req:OTelIngest,x_loopgrid_key:str|None=Header(default=None),db:S for span in req.spans: a=span.attributes;d=create_decision(db,{"workspace_id":req.workspace_id,"decision_type":a.get("loopgrid.decision.type","genai_operation"),"service_name":a.get("service.name","otel-service"),"idempotency_key":f"otel:{span.trace_id}:{span.span_id}","agent":{"id":a.get("gen_ai.agent.name") or a.get("service.name","unknown-agent")},"model":{"provider":a.get("gen_ai.provider.name"),"name":a.get("gen_ai.request.model") or a.get("gen_ai.response.model")},"context":{"trace_id":span.trace_id,"span_id":span.span_id,"span_name":span.name},"input":{"prompt":a.get("gen_ai.prompt") or a.get("gen_ai.input.messages")},"proposed_action":{"tool":a.get("gen_ai.tool.name")},"metadata":{"source":"opentelemetry","capture_source":"otel-mapping","attributes":a}});created.append(d["decision_id"]) return {"accepted":len(created),"decision_ids":created,"format":"loopgrid-otel-json"} -def _otlp_value(v): - if not isinstance(v,dict):return v - for k in ("stringValue","boolValue","intValue","doubleValue"): - if k in v:return v[k] - if "arrayValue" in v:return [_otlp_value(x) for x in v.get("arrayValue",{}).get("values",[])] - return v -def _otlp_attrs(attrs):return {a.get("key"):_otlp_value(a.get("value",{})) for a in (attrs or []) if a.get("key")} -@app.post("/v1/traces",response_model=OTLPAcceptResponse,tags=["Ingestion"],summary="OTLP/HTTP JSON traces receiver") +@app.post( + "/v1/traces", + tags=["Ingestion"], + summary="OTLP/HTTP traces receiver (protobuf + JSON)", + response_class=Response, + responses={ + 200:{"description":"OTLP ExportTraceServiceResponse","content":{"application/x-protobuf":{},"application/json":{}}}, + 400:{"description":"Malformed OTLP request"}, + 413:{"description":"OTLP request body too large"}, + 415:{"description":"Unsupported OTLP content type or content encoding"}, + }, + openapi_extra={"requestBody":{"content":{"application/x-protobuf":{"schema":{"type":"string","format":"binary"}},"application/json":{"schema":{"type":"object"}}}}}, +) async def ingest_otlp_http(request:Request,x_loopgrid_key:str|None=Header(default=None),x_loopgrid_workspace:str|None=Header(default=None),db:Session=Depends(get_db)): - wid=x_loopgrid_workspace or "default";_auth(db,x_loopgrid_key,wid,scope="ingest");body=await request.json();created=[] - for rs in body.get("resourceSpans",[]): - resource=_otlp_attrs((rs.get("resource") or {}).get("attributes",[])) - for ss in rs.get("scopeSpans",[]): - for sp in ss.get("spans",[]): - attrs={**resource,**_otlp_attrs(sp.get("attributes",[]))};trace=sp.get("traceId") or "unknown";sid=sp.get("spanId") or sha256_hex(canonical_json(sp))[:16] - d=create_decision(db,{"workspace_id":wid,"decision_type":attrs.get("loopgrid.decision.type","genai_operation"),"service_name":attrs.get("service.name","otlp-service"),"idempotency_key":f"otlp:{trace}:{sid}","agent":{"id":attrs.get("gen_ai.agent.name") or attrs.get("service.name","unknown-agent")},"model":{"provider":attrs.get("gen_ai.provider.name") or attrs.get("gen_ai.system"),"name":attrs.get("gen_ai.request.model") or attrs.get("gen_ai.response.model")},"context":{"trace_id":trace,"span_id":sid,"span_name":sp.get("name"),"scope":(ss.get("scope") or {}).get("name")},"input":{},"proposed_action":{"tool":attrs.get("gen_ai.tool.name")},"metadata":{"source":"otlp-http-json","capture_source":"otlp-http-json","attributes":attrs}});created.append(d["decision_id"]) - return {"partialSuccess":{},"loopgrid":{"accepted":len(created),"decision_ids":created}} + wid=x_loopgrid_workspace or "default" + _auth(db,x_loopgrid_key,wid,scope="ingest") + request_media_type=media_type_from_header(request.headers.get("content-type")) + try: + media_type,spans=decode_trace_request( + await request.body(), + content_type=request.headers.get("content-type"), + content_encoding=request.headers.get("content-encoding"), + max_bytes=settings.max_request_body_bytes, + ) + except OTLPHTTPError as exc: + response_type=request_media_type if request_media_type in {OTLP_JSON,OTLP_PROTOBUF} else OTLP_JSON + return Response(content=error_response_body(response_type,exc.message),status_code=exc.status_code,media_type=response_type) + + created=[] + for sp in spans: + attrs=sp.attributes + trace=sp.trace_id or "unknown" + sid=sp.span_id or sha256_hex(canonical_json({"trace_id":trace,"name":sp.name,"attributes":attrs}))[:16] + d=create_decision(db,{ + "workspace_id":wid, + "decision_type":attrs.get("loopgrid.decision.type","genai_operation"), + "service_name":attrs.get("service.name","otlp-service"), + "idempotency_key":f"otlp:{trace}:{sid}", + "agent":{"id":attrs.get("gen_ai.agent.name") or attrs.get("service.name","unknown-agent")}, + "model":{"provider":attrs.get("gen_ai.provider.name") or attrs.get("gen_ai.system"),"name":attrs.get("gen_ai.request.model") or attrs.get("gen_ai.response.model")}, + "context":{"trace_id":trace,"span_id":sid,"span_name":sp.name,"scope":sp.scope_name}, + "input":{}, + "proposed_action":{"tool":attrs.get("gen_ai.tool.name")}, + "metadata":{"source":sp.source,"capture_source":sp.source,"otlp_encoding":"protobuf" if media_type==OTLP_PROTOBUF else "json","attributes":attrs}, + }) + created.append(d["decision_id"]) + + return Response( + content=success_response_body(media_type), + status_code=200, + media_type=media_type, + headers={"X-LoopGrid-Accepted":str(len(created))}, + ) @app.post("/api/v1/ingest/mcp",response_model=MCPIngestResponse,tags=["Ingestion"],summary="Capture MCP JSON-RPC tool evidence") def ingest_mcp(req:MCPIngest,x_loopgrid_key:str|None=Header(default=None),db:Session=Depends(get_db)): _auth(db,x_loopgrid_key,req.workspace_id,scope="ingest");method=req.request.get("method","unknown");params=req.request.get("params") or {};tool=params.get("name") if method=="tools/call" else method;idem=f"mcp:{req.trace_id or sha256_hex(canonical_json(req.request))[:20]}:{req.request.get('id','noid')}" diff --git a/app/otlp.py b/app/otlp.py new file mode 100644 index 0000000..2f137ac --- /dev/null +++ b/app/otlp.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import base64 +import gzip +import json +from dataclasses import dataclass +from io import BytesIO +from typing import Any + +from google.protobuf.message import DecodeError +from google.rpc.status_pb2 import Status +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ExportTraceServiceResponse, +) + +OTLP_JSON = "application/json" +OTLP_PROTOBUF = "application/x-protobuf" +SUPPORTED_CONTENT_TYPES = {OTLP_JSON, OTLP_PROTOBUF} +SUPPORTED_CONTENT_ENCODINGS = {"", "identity", "gzip"} + + +@dataclass(frozen=True) +class NormalizedOTLPSpan: + trace_id: str + span_id: str + name: str | None + scope_name: str | None + attributes: dict[str, Any] + source: str + + +class OTLPHTTPError(ValueError): + def __init__(self, status_code: int, message: str): + super().__init__(message) + self.status_code = status_code + self.message = message + + +def media_type_from_header(value: str | None) -> str: + return (value or "").split(";", 1)[0].strip().lower() + + +def _read_gzip_limited(raw: bytes, max_bytes: int) -> bytes: + """Decompress gzip while bounding the decoded body size.""" + try: + with gzip.GzipFile(fileobj=BytesIO(raw), mode="rb") as stream: + decoded = stream.read(max_bytes + 1) + except (OSError, EOFError) as exc: + raise OTLPHTTPError(400, "Invalid gzip-compressed OTLP request body") from exc + if len(decoded) > max_bytes: + raise OTLPHTTPError(413, f"Decoded OTLP request body exceeds {max_bytes} bytes") + return decoded + + +def decode_body(raw: bytes, *, content_encoding: str | None, max_bytes: int) -> bytes: + encoding = (content_encoding or "").strip().lower() + if encoding not in SUPPORTED_CONTENT_ENCODINGS: + raise OTLPHTTPError(415, f"Unsupported Content-Encoding: {encoding or ''}") + if len(raw) > max_bytes: + raise OTLPHTTPError(413, f"OTLP request body exceeds {max_bytes} bytes") + if encoding == "gzip": + return _read_gzip_limited(raw, max_bytes) + return raw + + +def _json_any_value(value: Any) -> Any: + if not isinstance(value, dict): + return value + if "stringValue" in value: + return value["stringValue"] + if "boolValue" in value: + return value["boolValue"] + if "intValue" in value: + raw = value["intValue"] + try: + return int(raw) + except (TypeError, ValueError): + return raw + if "doubleValue" in value: + return value["doubleValue"] + if "arrayValue" in value: + return [_json_any_value(item) for item in (value.get("arrayValue") or {}).get("values", [])] + if "kvlistValue" in value: + return _json_key_values((value.get("kvlistValue") or {}).get("values", [])) + if "bytesValue" in value: + # OTLP/JSON follows protobuf JSON mapping for bytes and therefore carries base64 text. + return value["bytesValue"] + return value + + +def _json_key_values(attributes: list[dict[str, Any]] | None) -> dict[str, Any]: + result: dict[str, Any] = {} + for attr in attributes or []: + key = attr.get("key") + if key: + result[key] = _json_any_value(attr.get("value", {})) + return result + + +def _protobuf_any_value(value: Any) -> Any: + kind = value.WhichOneof("value") + if kind is None: + return None + if kind == "array_value": + return [_protobuf_any_value(item) for item in value.array_value.values] + if kind == "kvlist_value": + return {item.key: _protobuf_any_value(item.value) for item in value.kvlist_value.values} + if kind == "bytes_value": + return base64.b64encode(value.bytes_value).decode("ascii") + return getattr(value, kind) + + +def _protobuf_key_values(attributes: Any) -> dict[str, Any]: + return {attr.key: _protobuf_any_value(attr.value) for attr in attributes if attr.key} + + +def parse_otlp_json(body: bytes) -> dict[str, Any]: + try: + parsed = json.loads(body.decode("utf-8")) if body else {} + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise OTLPHTTPError(400, "Invalid OTLP/HTTP JSON request body") from exc + if not isinstance(parsed, dict): + raise OTLPHTTPError(400, "OTLP/HTTP JSON request body must be an object") + return parsed + + +def parse_otlp_protobuf(body: bytes) -> ExportTraceServiceRequest: + message = ExportTraceServiceRequest() + try: + message.ParseFromString(body) + except DecodeError as exc: + raise OTLPHTTPError(400, "Invalid OTLP/HTTP protobuf request body") from exc + return message + + +def normalize_json_request(body: dict[str, Any]) -> list[NormalizedOTLPSpan]: + spans: list[NormalizedOTLPSpan] = [] + for resource_spans in body.get("resourceSpans", []) or []: + if not isinstance(resource_spans, dict): + continue + resource_attrs = _json_key_values((resource_spans.get("resource") or {}).get("attributes", [])) + # scopeSpans is current OTLP. instrumentationLibrarySpans is accepted for older senders. + scope_spans = resource_spans.get("scopeSpans") or resource_spans.get("instrumentationLibrarySpans") or [] + for scope_group in scope_spans: + if not isinstance(scope_group, dict): + continue + scope = scope_group.get("scope") or scope_group.get("instrumentationLibrary") or {} + scope_attrs = _json_key_values(scope.get("attributes", [])) if isinstance(scope, dict) else {} + scope_name = scope.get("name") if isinstance(scope, dict) else None + for span in scope_group.get("spans", []) or []: + if not isinstance(span, dict): + continue + attrs = {**resource_attrs, **scope_attrs, **_json_key_values(span.get("attributes", []))} + trace_id = str(span.get("traceId") or "unknown") + span_id = str(span.get("spanId") or "") + spans.append( + NormalizedOTLPSpan( + trace_id=trace_id, + span_id=span_id, + name=span.get("name"), + scope_name=scope_name, + attributes=attrs, + source="otlp-http-json", + ) + ) + return spans + + +def normalize_protobuf_request(message: ExportTraceServiceRequest) -> list[NormalizedOTLPSpan]: + spans: list[NormalizedOTLPSpan] = [] + for resource_spans in message.resource_spans: + resource_attrs = _protobuf_key_values(resource_spans.resource.attributes) + for scope_group in resource_spans.scope_spans: + scope_attrs = _protobuf_key_values(scope_group.scope.attributes) + for span in scope_group.spans: + attrs = {**resource_attrs, **scope_attrs, **_protobuf_key_values(span.attributes)} + spans.append( + NormalizedOTLPSpan( + trace_id=span.trace_id.hex() if span.trace_id else "unknown", + span_id=span.span_id.hex() if span.span_id else "", + name=span.name or None, + scope_name=scope_group.scope.name or None, + attributes=attrs, + source="otlp-http-protobuf", + ) + ) + return spans + + +def decode_trace_request( + raw: bytes, + *, + content_type: str | None, + content_encoding: str | None, + max_bytes: int, +) -> tuple[str, list[NormalizedOTLPSpan]]: + media_type = media_type_from_header(content_type) + if media_type not in SUPPORTED_CONTENT_TYPES: + raise OTLPHTTPError( + 415, + "Unsupported Content-Type for OTLP traces; use application/x-protobuf or application/json", + ) + body = decode_body(raw, content_encoding=content_encoding, max_bytes=max_bytes) + if media_type == OTLP_PROTOBUF: + return media_type, normalize_protobuf_request(parse_otlp_protobuf(body)) + return media_type, normalize_json_request(parse_otlp_json(body)) + + +def success_response_body(media_type: str) -> bytes: + response = ExportTraceServiceResponse() + if media_type == OTLP_PROTOBUF: + return response.SerializeToString() + # Successful OTLP responses leave partialSuccess unset, so JSON is simply {}. + return b"{}" + + +def error_response_body(media_type: str, message: str) -> bytes: + if media_type == OTLP_PROTOBUF: + return Status(message=message).SerializeToString() + return json.dumps({"message": message}, separators=(",", ":")).encode("utf-8") diff --git a/app/schemas.py b/app/schemas.py index d1213c8..603099a 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -143,10 +143,6 @@ class ReplayResultResponse(BaseModel): class MCPIngestResponse(BaseModel): decision_id:str;idempotent_replay:bool;protocol:str -class OTLPAcceptResponse(BaseModel): - model_config=ConfigDict(extra="allow") - partialSuccess:dict[str,Any]=Field(default_factory=dict);loopgrid:dict[str,Any] - class PayloadEventStatusResponse(BaseModel): event_id:str;event_type:str;privacy_mode:str;payload:dict[str,Any] class PayloadStatusResponse(BaseModel): diff --git a/app/static/app.js b/app/static/app.js index d3d29a7..a237957 100644 --- a/app/static/app.js +++ b/app/static/app.js @@ -319,7 +319,7 @@ async function loadTrust(){ if($('envSigner')) $('envSigner').textContent=sg.algorithm||sg.provider||'—'; if($('envTsa')) $('envTsa').textContent=ts.tsa_url_configured?'Configured':'Not set'; if($('envTsaSignal')) $('envTsaSignal').className='signal '+(ts.tsa_url_configured?'ok':'warn'); - if($('sidebarVersion')) $('sidebarVersion').textContent='v'+String(info.version||'0.8.0').split('-')[0]; + if($('sidebarVersion')) $('sidebarVersion').textContent='v'+String(info.version||'0.8.1').split('-')[0]; if($('trustSigner')) $('trustSigner').textContent=sg.provider==='aws_kms'?'AWS KMS signing boundary':'Local Ed25519 signer'; if($('trustSignerNote')) $('trustSignerNote').textContent=sg.hardware_backed?'Asymmetric signing is executed by the configured AWS KMS key.':'Zero-setup local signer for design-partner evaluation. Use KMS for a hardware-backed production boundary.'; if($('trustAlgorithm')) $('trustAlgorithm').textContent=sg.algorithm||'—'; @@ -411,7 +411,7 @@ async function bootLoopGridUI(){ }); window.__loopgridBooted=true; - window.LoopGridUI={version:'0.8.0',loadWorkspace,refresh,verifyLedger,loadTrust}; + window.LoopGridUI={version:'0.8.1',loadWorkspace,refresh,verifyLedger,loadTrust}; await Promise.all([refresh(),loadTrust()]); console.info('[LoopGrid UI] v0.8 Design Partner Release ready'); }catch(err){ diff --git a/app/static/index.html b/app/static/index.html index c1c1d60..750c03b 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -38,7 +38,7 @@
Chain scopeWorkspace
External TSANot set
-
v0.8.0external pilot
+
v0.8.1external pilot
@@ -557,7 +557,7 @@

Evidence for decisions
that matter.

if($('envSigner')) $('envSigner').textContent=sg.algorithm||sg.provider||'—'; if($('envTsa')) $('envTsa').textContent=ts.tsa_url_configured?'Configured':'Not set'; if($('envTsaSignal')) $('envTsaSignal').className='signal '+(ts.tsa_url_configured?'ok':'warn'); - if($('sidebarVersion')) $('sidebarVersion').textContent='v'+String(info.version||'0.8.0').split('-')[0]; + if($('sidebarVersion')) $('sidebarVersion').textContent='v'+String(info.version||'0.8.1').split('-')[0]; if($('trustSigner')) $('trustSigner').textContent=sg.provider==='aws_kms'?'AWS KMS signing boundary':'Local Ed25519 signer'; if($('trustSignerNote')) $('trustSignerNote').textContent=sg.hardware_backed?'Asymmetric signing is executed by the configured AWS KMS key.':'Zero-setup local signer for design-partner evaluation. Use KMS for a hardware-backed production boundary.'; if($('trustAlgorithm')) $('trustAlgorithm').textContent=sg.algorithm||'—'; @@ -649,9 +649,9 @@

Evidence for decisions
that matter.

}); window.__loopgridBooted=true; - window.LoopGridUI={version:'0.8.0',loadWorkspace,refresh,verifyLedger,loadTrust}; + window.LoopGridUI={version:'0.8.1',loadWorkspace,refresh,verifyLedger,loadTrust}; await Promise.all([refresh(),loadTrust()]); - console.info('[LoopGrid UI] v0.8.0 Evidence Gateway ready'); + console.info('[LoopGrid UI] v0.8.1 Evidence Gateway ready'); }catch(err){ window.__loopgridBooted=false; showBootError(err); diff --git a/app/version.py b/app/version.py index 00e1aea..d541651 100644 --- a/app/version.py +++ b/app/version.py @@ -1,4 +1,4 @@ -VERSION = "0.8.0-design-partner" +VERSION = "0.8.1-design-partner" EVIDENCE_PROFILE = "3.0-draft" API_VERSION = "v1" PRODUCT_STAGE = "design partner release" diff --git a/requirements.txt b/requirements.txt index f113375..c64ad6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,5 @@ psycopg[binary]>=3.2,<4 pytest>=8,<10 asn1crypto>=1.5,<2 boto3>=1.35,<2 +opentelemetry-proto>=1.42,<2 +googleapis-common-protos>=1.69,<2 diff --git a/scripts/real_scenario_test.py b/scripts/real_scenario_test.py index eb4342c..94bc5ef 100644 --- a/scripts/real_scenario_test.py +++ b/scripts/real_scenario_test.py @@ -532,10 +532,13 @@ def main() -> None: "POST", "/v1/traces", key=key, - headers={"X-LoopGrid-Workspace": workspace_id}, - json=otlp, - ).json() - log("OTLP/HTTP JSON ingestion", otr.get("loopgrid", {}).get("accepted") == 1) + headers={"X-LoopGrid-Workspace": workspace_id, "Content-Type": "application/json"}, + content=json.dumps(otlp).encode("utf-8"), + ) + log( + "OTLP/HTTP JSON ingestion", + otr.headers.get("X-LoopGrid-Accepted") == "1" and otr.json() == {}, + ) result = { "validation_run_id": run_id, diff --git a/tests/test_flow.py b/tests/test_flow.py index 8368aa3..61e0e07 100644 --- a/tests/test_flow.py +++ b/tests/test_flow.py @@ -192,7 +192,7 @@ def test_mcp_jsonrpc_ingestion(): def test_otlp_http_json_receiver(): reset(); otlp={'resourceSpans':[{'resource':{'attributes':[{'key':'service.name','value':{'stringValue':'agent-api'}}]},'scopeSpans':[{'scope':{'name':'openai.instrumentation'},'spans':[{'traceId':'abc','spanId':'def','name':'chat','attributes':[{'key':'gen_ai.provider.name','value':{'stringValue':'openai'}},{'key':'gen_ai.request.model','value':{'stringValue':'gpt-test'}}]}]}]}]} - r=client.post('/v1/traces',headers={'X-LoopGrid-Workspace':'default'},json=otlp); assert r.status_code==200 and r.json()['loopgrid']['accepted']==1 + r=client.post('/v1/traces',headers={'X-LoopGrid-Workspace':'default','Content-Type':'application/json'},content=__import__('json').dumps(otlp).encode()); assert r.status_code==200 and r.headers['x-loopgrid-accepted']=='1' and r.json()=={} d=client.get('/api/v1/decisions').json()[0]; assert d['model']['provider']=='openai' and d['model']['name']=='gpt-test' and d['metadata']['source']=='otlp-http-json' def test_audit_log_and_operational_endpoints(): @@ -212,7 +212,7 @@ def test_demo_reset_restores_baseline_policy(): def test_v07_system_info_and_openapi_contract(): info=client.get('/api/v1/system/info');assert info.status_code==200 - j=info.json();assert j['version'].startswith('0.8.0') and j['evidence_profile']=='3.0-draft' + j=info.json();assert j['version'].startswith('0.8.1') and j['evidence_profile']=='3.0-draft' assert j['payload_vault']['algorithm']=='AES-256-GCM' and j['payload_vault']['erasable_without_breaking_chain'] is True schema=client.get('/openapi.json').json();schemes=schema['components']['securitySchemes'] assert 'LoopGridApiKey' in schemes and 'LoopGridUserBearer' in schemes @@ -258,7 +258,7 @@ def test_checkpoint_is_persisted_and_exported(tmp_path): ev=client.get(f'/api/v1/decisions/{did}/evidence');p=tmp_path/'checkpoint.zip';p.write_bytes(ev.content) import zipfile,json with zipfile.ZipFile(p) as z:manifest=json.loads(z.read('manifest.json')) - assert manifest['checkpoint']['checkpoint_id']==cp['checkpoint_id'] and manifest['software_version'].startswith('0.8.0') and manifest['version']=='3.0-draft' + assert manifest['checkpoint']['checkpoint_id']==cp['checkpoint_id'] and manifest['software_version'].startswith('0.8.1') and manifest['version']=='3.0-draft' def test_human_rbac_bearer_sessions_and_roles(): @@ -332,7 +332,7 @@ def test_v07_dashboard_separates_completed_and_pending_reviews(): reset();client.post('/api/v1/demo/workspace');client.post('/api/v1/demo/pending-review');d=client.get('/api/v1/dashboard').json();assert d['completed_reviews']==1 and d['human_approvals']==1 and d['human_rejections']==0 and d['pending_reviews']==1 def test_v07_pilot_readiness_endpoint_discloses_gaps(): - r=client.get('/api/v1/pilot/readiness');assert r.status_code==200;j=r.json();assert j['version'].startswith('0.8.0') and j['controlled_pilot_ready'] is True + r=client.get('/api/v1/pilot/readiness');assert r.status_code==200;j=r.json();assert j['version'].startswith('0.8.1') and j['controlled_pilot_ready'] is True ids={x['id']:x for x in j['checks']};assert ids['database']['status']=='pass' and ids['signed_ledger']['status']=='pass' and ids['payload_vault']['status']=='pass';assert 'signer_boundary' in ids and 'external_timestamp' in ids and 'database_topology' in ids @@ -527,6 +527,6 @@ def test_v08_production_safety_guard_accepts_design_partner_config(): def test_v08_system_info_exposes_deployment_security_posture(): info=client.get('/api/v1/system/info').json() - assert info['version'].startswith('0.8.0') + assert info['version'].startswith('0.8.1') assert 'deployment_security' in info assert info['deployment_security']['strict_production_safety'] is True diff --git a/tests/test_otlp_http.py b/tests/test_otlp_http.py new file mode 100644 index 0000000..e9ada04 --- /dev/null +++ b/tests/test_otlp_http.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import gzip +import json +from dataclasses import replace + +from fastapi.testclient import TestClient +from google.rpc.status_pb2 import Status +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ExportTraceServiceResponse, +) + +import app.main as main_module +from app.main import app + + +client = TestClient(app) + + +def reset() -> None: + response = client.delete("/api/v1/demo/reset") + assert response.status_code == 200 + + +def _json_request() -> dict: + return { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "agent-api"}}, + ] + }, + "scopeSpans": [ + { + "scope": {"name": "openai.instrumentation"}, + "spans": [ + { + "traceId": "00112233445566778899aabbccddeeff", + "spanId": "0011223344556677", + "name": "chat", + "attributes": [ + {"key": "gen_ai.provider.name", "value": {"stringValue": "openai"}}, + {"key": "gen_ai.request.model", "value": {"stringValue": "gpt-test"}}, + {"key": "gen_ai.tool.name", "value": {"stringValue": "refund_customer"}}, + { + "key": "loopgrid.test.nested", + "value": { + "kvlistValue": { + "values": [ + {"key": "attempt", "value": {"intValue": "2"}}, + {"key": "ok", "value": {"boolValue": True}}, + ] + } + }, + }, + ], + } + ], + } + ], + } + ] + } + + +def _protobuf_request() -> ExportTraceServiceRequest: + request = ExportTraceServiceRequest() + resource_spans = request.resource_spans.add() + resource_attr = resource_spans.resource.attributes.add() + resource_attr.key = "service.name" + resource_attr.value.string_value = "agent-api" + + scope_spans = resource_spans.scope_spans.add() + scope_spans.scope.name = "openai.instrumentation" + span = scope_spans.spans.add() + span.trace_id = bytes.fromhex("00112233445566778899aabbccddeeff") + span.span_id = bytes.fromhex("0011223344556677") + span.name = "chat" + + provider = span.attributes.add() + provider.key = "gen_ai.provider.name" + provider.value.string_value = "openai" + + model = span.attributes.add() + model.key = "gen_ai.request.model" + model.value.string_value = "gpt-test" + + tool = span.attributes.add() + tool.key = "gen_ai.tool.name" + tool.value.string_value = "refund_customer" + + nested = span.attributes.add() + nested.key = "loopgrid.test.nested" + attempt = nested.value.kvlist_value.values.add() + attempt.key = "attempt" + attempt.value.int_value = 2 + ok = nested.value.kvlist_value.values.add() + ok.key = "ok" + ok.value.bool_value = True + return request + + +def _latest_decision() -> dict: + decisions = client.get("/api/v1/decisions").json() + assert decisions + return decisions[0] + + +def test_otlp_http_json_standard_success_response() -> None: + reset() + response = client.post( + "/v1/traces", + headers={"Content-Type": "application/json", "X-LoopGrid-Workspace": "default"}, + content=json.dumps(_json_request()).encode("utf-8"), + ) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/json") + assert response.headers["x-loopgrid-accepted"] == "1" + assert response.json() == {} + + decision = _latest_decision() + assert decision["model"] == {"provider": "openai", "name": "gpt-test"} + assert decision["proposed_action"]["tool"] == "refund_customer" + assert decision["metadata"]["source"] == "otlp-http-json" + assert decision["metadata"]["otlp_encoding"] == "json" + assert decision["metadata"]["attributes"]["loopgrid.test.nested"] == {"attempt": 2, "ok": True} + + +def test_otlp_http_binary_protobuf_success() -> None: + reset() + payload = _protobuf_request().SerializeToString() + response = client.post( + "/v1/traces", + headers={"Content-Type": "application/x-protobuf", "X-LoopGrid-Workspace": "default"}, + content=payload, + ) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("application/x-protobuf") + assert response.headers["x-loopgrid-accepted"] == "1" + parsed = ExportTraceServiceResponse() + parsed.ParseFromString(response.content) + assert not parsed.HasField("partial_success") + + decision = _latest_decision() + assert decision["model"] == {"provider": "openai", "name": "gpt-test"} + assert decision["context"]["trace_id"] == "00112233445566778899aabbccddeeff" + assert decision["context"]["span_id"] == "0011223344556677" + assert decision["metadata"]["source"] == "otlp-http-protobuf" + assert decision["metadata"]["otlp_encoding"] == "protobuf" + assert decision["metadata"]["attributes"]["loopgrid.test.nested"] == {"attempt": 2, "ok": True} + + +def test_otlp_json_and_protobuf_map_equivalently() -> None: + reset() + json_response = client.post( + "/v1/traces", + headers={"Content-Type": "application/json"}, + content=json.dumps(_json_request()).encode("utf-8"), + ) + assert json_response.status_code == 200 + json_decision = _latest_decision() + + reset() + protobuf_response = client.post( + "/v1/traces", + headers={"Content-Type": "application/x-protobuf"}, + content=_protobuf_request().SerializeToString(), + ) + assert protobuf_response.status_code == 200 + protobuf_decision = _latest_decision() + + for key in ("decision_type", "service_name", "agent", "model", "context", "proposed_action"): + assert json_decision[key] == protobuf_decision[key] + assert json_decision["metadata"]["attributes"] == protobuf_decision["metadata"]["attributes"] + + +def test_otlp_http_gzip_protobuf() -> None: + reset() + compressed = gzip.compress(_protobuf_request().SerializeToString()) + response = client.post( + "/v1/traces", + headers={ + "Content-Type": "application/x-protobuf", + "Content-Encoding": "gzip", + "X-LoopGrid-Workspace": "default", + }, + content=compressed, + ) + assert response.status_code == 200 + assert response.headers["x-loopgrid-accepted"] == "1" + assert _latest_decision()["metadata"]["source"] == "otlp-http-protobuf" + + +def test_otlp_rejects_malformed_protobuf_with_otlp_status() -> None: + reset() + response = client.post( + "/v1/traces", + headers={"Content-Type": "application/x-protobuf"}, + content=b"\x0a\xff\xff\xff", + ) + assert response.status_code == 400 + assert response.headers["content-type"].startswith("application/x-protobuf") + status = Status() + status.ParseFromString(response.content) + assert "Invalid OTLP/HTTP protobuf" in status.message + + +def test_otlp_rejects_unsupported_content_type() -> None: + reset() + response = client.post("/v1/traces", headers={"Content-Type": "text/plain"}, content=b"{}") + assert response.status_code == 415 + assert response.json()["message"].startswith("Unsupported Content-Type") + + +def test_otlp_rejects_unsupported_content_encoding() -> None: + reset() + response = client.post( + "/v1/traces", + headers={"Content-Type": "application/json", "Content-Encoding": "br"}, + content=b"{}", + ) + assert response.status_code == 415 + assert "Unsupported Content-Encoding" in response.json()["message"] + + +def test_otlp_limits_decoded_gzip_body(monkeypatch) -> None: + reset() + monkeypatch.setattr(main_module, "settings", replace(main_module.settings, max_request_body_bytes=256)) + oversized = json.dumps({"padding": "A" * 2048}).encode("utf-8") + compressed = gzip.compress(oversized) + assert len(compressed) < 256 + response = client.post( + "/v1/traces", + headers={"Content-Type": "application/json", "Content-Encoding": "gzip"}, + content=compressed, + ) + assert response.status_code == 413 + assert "Decoded OTLP request body exceeds" in response.json()["message"] + + +def test_protobuf_ingested_evidence_remains_v1_verifier_compatible(tmp_path) -> None: + from verifier.loopgrid_verify import verify_bundle + + reset() + response = client.post( + "/v1/traces", + headers={"Content-Type": "application/x-protobuf"}, + content=_protobuf_request().SerializeToString(), + ) + assert response.status_code == 200 + decision_id = _latest_decision()["decision_id"] + exported = client.get(f"/api/v1/decisions/{decision_id}/evidence") + assert exported.status_code == 200 + bundle = tmp_path / "otlp-protobuf-evidence.zip" + bundle.write_bytes(exported.content) + verified = verify_bundle(str(bundle)) + assert verified["valid"] is True + assert verified["bundle_schema"] == "loopgrid/evidence-bundle/2" + assert verified["signature_algorithm"] == "Ed25519"