chore: separate explorer from gatekeeper (#1209)

* chore: separate explorer from gatekeeper

* test: fix index export lint issues

* test: restore sync coverage

* chore: update generated OpenAPI docs

* test: restore search db coverage

* test: exclude test helpers from coverage

* test: exclude gatekeeper production db adapters from coverage

* ci: align docker test container uid

* chore: standardize search server env vars

* chore: silence dotenv info output

* chore: order receipt pages by columns

* fix: use signature time first

* fix: mongo check indexes on start

* fix: return after error status

* fix mongo index startup migration

* fix mongo replica set healthcheck

* harden redis startup and logging

---------

Co-authored-by: GitHub Action <action@github.com>

Author: Bushstar

.github/workflows/docker-build-test.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Grab logs
         if: always()
         run: |
-          docker compose logs --no-color keymaster cli gatekeeper redis > docker-compose-logs.txt
+          docker compose logs --no-color keymaster cli gatekeeper redis search-server > docker-compose-logs.txt
           cat docker-compose-logs.txt
       
 
@@ -73,4 +73,3 @@ jobs:
             name: E2E Artifacts
             path: tests/test-results/cli-tests.xml
 
-

doc/gatekeeper-api.json

@@ -978,6 +978,148 @@
         }
       }
     },
+    "/index/export": {
+      "post": {
+        "summary": "Export a paginated DID index snapshot or incremental change batch.",
+        "requestBody": {
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "mode"
+                ],
+                "properties": {
+                  "mode": {
+                    "type": "string",
+                    "enum": [
+                      "snapshot",
+                      "changes"
+                    ]
+                  },
+                  "cursor": {
+                    "type": "string",
+                    "nullable": true,
+                    "description": "Opaque snapshot cursor or incremental change cursor."
+                  },
+                  "checkpointCursor": {
+                    "type": "string",
+                    "nullable": true,
+                    "description": "Snapshot-only high-water change cursor returned by the first snapshot page and required on continuation pages."
+                  },
+                  "limit": {
+                    "type": "integer",
+                    "minimum": 1
+                  }
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Index export page.",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "oneOf": [
+                    {
+                      "type": "object",
+                      "required": [
+                        "mode",
+                        "cursor",
+                        "checkpointCursor",
+                        "hasMore",
+                        "dids",
+                        "blocks"
+                      ],
+                      "properties": {
+                        "mode": {
+                          "type": "string",
+                          "enum": [
+                            "snapshot"
+                          ]
+                        },
+                        "cursor": {
+                          "type": "string",
+                          "nullable": true,
+                          "description": "Opaque snapshot page cursor. Clients must store and pass it back without interpretation."
+                        },
+                        "checkpointCursor": {
+                          "type": "string",
+                          "nullable": true,
+                          "description": "Snapshot high-water change cursor returned on every snapshot page."
+                        },
+                        "hasMore": {
+                          "type": "boolean"
+                        },
+                        "dids": {
+                          "type": "array",
+                          "items": {
+                            "type": "object"
+                          }
+                        },
+                        "blocks": {
+                          "type": "array",
+                          "items": {
+                            "type": "object"
+                          }
+                        }
+                      }
+                    },
+                    {
+                      "type": "object",
+                      "required": [
+                        "mode",
+                        "cursor",
+                        "hasMore",
+                        "dids",
+                        "blocks"
+                      ],
+                      "properties": {
+                        "mode": {
+                          "type": "string",
+                          "enum": [
+                            "changes"
+                          ]
+                        },
+                        "cursor": {
+                          "type": "string",
+                          "nullable": true,
+                          "description": "Incremental change cursor."
+                        },
+                        "hasMore": {
+                          "type": "boolean"
+                        },
+                        "dids": {
+                          "type": "array",
+                          "items": {
+                            "type": "object"
+                          }
+                        },
+                        "blocks": {
+                          "type": "array",
+                          "items": {
+                            "type": "object"
+                          }
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+            }
+          },
+          "400": {
+            "description": "Invalid request body."
+          },
+          "500": {
+            "description": "Internal Server Error."
+          }
+        }
+      }
+    },
     "/dids/remove": {
       "post": {
         "summary": "Remove one or more DIDs",

docker-compose.yml

@@ -1,10 +1,17 @@
 services:
   mongodb:
     image: mongo:8.0
+    command: [ "mongod", "--replSet", "rs0", "--bind_ip_all" ]
     volumes:
       - ./data/mongodb:/data/db
     ports:
       - 127.0.0.1:27017:27017
+    healthcheck:
+      test: [ "CMD-SHELL", "mongosh --quiet --eval \"const host = 'mongodb:27017'; try { const cfg = rs.conf(); if (cfg._id !== 'rs0' || cfg.members.length !== 1 || cfg.members[0].host !== host) { cfg._id = 'rs0'; cfg.version = (cfg.version || 1) + 1; cfg.members = [{ _id: 0, host }]; rs.reconfig(cfg, { force: true }); } } catch (err) { try { rs.initiate({ _id: 'rs0', members: [{ _id: 0, host }] }); } catch (initErr) {} } try { quit(db.adminCommand({ hello: 1 }).isWritablePrimary ? 0 : 1); } catch (err) { quit(1); }\"" ]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+      start_period: 5s
 
   redis:
     image: redis:8.0.4-alpine
@@ -14,6 +21,12 @@ services:
       - ./data/redis:/data
     ports:
       - 127.0.0.1:6379:6379
+    healthcheck:
+      test: [ "CMD", "redis-cli", "ping" ]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+      start_period: 5s
 
   postgres:
     image: postgres:17-alpine
@@ -65,7 +78,7 @@ services:
       - KC_GATEKEEPER_RATE_LIMIT_WHITELIST=${KC_GATEKEEPER_RATE_LIMIT_WHITELIST}
       - KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS=${KC_GATEKEEPER_RATE_LIMIT_SKIP_PATHS}
       - KC_LOG_LEVEL=${KC_LOG_LEVEL}
-      - KC_MONGODB_URL=mongodb://mongodb:27017
+      - KC_MONGODB_URL=mongodb://mongodb:27017/?replicaSet=rs0
       - KC_REDIS_URL=redis://redis:6379
       - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
       - KC_IPFS_ENABLE=${KC_IPFS_ENABLE}
@@ -78,9 +91,12 @@ services:
     ports:
       - ${KC_GATEKEEPER_PORT}:4224
     depends_on:
-      - mongodb
-      - redis
-      - postgres
+      mongodb:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      postgres:
+        condition: service_started
 
   keymaster:
     build:
@@ -91,7 +107,7 @@ services:
       - KC_KEYMASTER_PORT=4226
       - KC_GATEKEEPER_URL=http://gatekeeper:4224
       - KC_DISABLE_SEARCH=${KC_DISABLE_SEARCH:-false}
-      - KC_SEARCH_URL=http://search-server:${SEARCH_SERVER_PORT:-4002}
+      - KC_SEARCH_URL=http://search-server:${KC_SEARCH_SERVER_PORT:-4002}
       - KC_NODE_ID=${KC_NODE_ID}
       - KC_KEYMASTER_DB=${KC_KEYMASTER_DB}
       - KC_ENCRYPTED_PASSPHRASE=${KC_ENCRYPTED_PASSPHRASE}
@@ -106,7 +122,7 @@ services:
       - KC_KEYMASTER_RATE_LIMIT_WHITELIST=${KC_KEYMASTER_RATE_LIMIT_WHITELIST}
       - KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS=${KC_KEYMASTER_RATE_LIMIT_SKIP_PATHS}
       - KC_LOG_LEVEL=${KC_LOG_LEVEL}
-      - KC_MONGODB_URL=mongodb://mongodb:27017
+      - KC_MONGODB_URL=mongodb://mongodb:27017/?replicaSet=rs0
       - KC_REDIS_URL=redis://redis:6379
       - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
     volumes:
@@ -115,11 +131,16 @@ services:
     ports:
       - ${KC_KEYMASTER_PORT}:4226
     depends_on:
-      - gatekeeper
-      - redis
-      - mongodb
-      - postgres
-      - search-server
+      gatekeeper:
+        condition: service_started
+      redis:
+        condition: service_healthy
+      mongodb:
+        condition: service_healthy
+      postgres:
+        condition: service_started
+      search-server:
+        condition: service_started
 
   hypr-mediator:
     build:
@@ -164,7 +185,7 @@ services:
     environment:
       - KC_GATEKEEPER_URL=http://gatekeeper:4224
       - KC_KEYMASTER_URL=http://keymaster:4226
-      - KC_MONGODB_URL=mongodb://mongodb:27017
+      - KC_MONGODB_URL=mongodb://mongodb:27017/?replicaSet=rs0
       - KC_REDIS_URL=redis://redis:6379
       - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
       - KC_NODE_ID=${KC_NODE_ID}
@@ -208,7 +229,7 @@ services:
     environment:
       - KC_GATEKEEPER_URL=http://gatekeeper:4224
       - KC_KEYMASTER_URL=http://keymaster:4226
-      - KC_MONGODB_URL=mongodb://mongodb:27017
+      - KC_MONGODB_URL=mongodb://mongodb:27017/?replicaSet=rs0
       - KC_REDIS_URL=redis://redis:6379
       - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
       - KC_NODE_ID=${KC_NODE_ID}
@@ -250,7 +271,7 @@ services:
     environment:
       - KC_GATEKEEPER_URL=http://gatekeeper:4224
       - KC_KEYMASTER_URL=http://keymaster:4226
-      - KC_MONGODB_URL=mongodb://mongodb:27017
+      - KC_MONGODB_URL=mongodb://mongodb:27017/?replicaSet=rs0
       - KC_REDIS_URL=redis://redis:6379
       - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
       - KC_NODE_ID=${KC_NODE_ID}
@@ -294,7 +315,7 @@ services:
     environment:
       - KC_GATEKEEPER_URL=http://gatekeeper:4224
       - KC_KEYMASTER_URL=http://keymaster:4226
-      - KC_MONGODB_URL=mongodb://mongodb:27017
+      - KC_MONGODB_URL=mongodb://mongodb:27017/?replicaSet=rs0
       - KC_REDIS_URL=redis://redis:6379
       - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
       - KC_NODE_ID=${KC_NODE_ID}
@@ -331,7 +352,7 @@ services:
     environment:
       - KC_GATEKEEPER_URL=http://gatekeeper:4224
       - KC_KEYMASTER_URL=http://keymaster:4226
-      - KC_MONGODB_URL=mongodb://mongodb:27017
+      - KC_MONGODB_URL=mongodb://mongodb:27017/?replicaSet=rs0
       - KC_REDIS_URL=redis://redis:6379
       - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
       - KC_NODE_ID=${KC_NODE_ID}
@@ -384,39 +405,39 @@ services:
     image: keychainmdip/explorer
     environment:
       - VITE_EXPLORER_PORT=4000
-      - VITE_GATEKEEPER_URL=http://localhost:4224
-      - VITE_SEARCH_SERVER=http://localhost:${SEARCH_SERVER_PORT:-4002}
+      - VITE_SEARCH_SERVER=http://localhost:${KC_SEARCH_SERVER_PORT:-4002}
       - VITE_OPERATION_NETWORKS=hyperswarm,local,TFTC,TBTC
       - KC_LOG_LEVEL=${KC_LOG_LEVEL}
     ports:
       - "4000:4000"
     depends_on:
-      - gatekeeper
+      - search-server
 
   search-server:
     build:
       context: .
       dockerfile: Dockerfile.search-server
     image: keychainmdip/search-server
     environment:
-      - SEARCH_SERVER_PORT=${SEARCH_SERVER_PORT:-4002}
-      - SEARCH_SERVER_GATEKEEPER_URL=${SEARCH_SERVER_GATEKEEPER_URL:-http://gatekeeper:4224}
-      - SEARCH_SERVER_REFRESH_INTERVAL_MS=${SEARCH_SERVER_REFRESH_INTERVAL_MS:-5000}
-      - SEARCH_SERVER_DB=${KC_SEARCH_SERVER_DB:-sqlite}
-      - SEARCH_SERVER_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
-      - SEARCH_SERVER_TRUST_PROXY=${SEARCH_SERVER_TRUST_PROXY:-false}
-      - SEARCH_SERVER_RATE_LIMIT_ENABLED=${SEARCH_SERVER_RATE_LIMIT_ENABLED:-false}
-      - SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=${SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE:-1}
-      - SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=${SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT:-minute}
-      - SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=${SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS:-600}
-      - SEARCH_SERVER_RATE_LIMIT_WHITELIST=${SEARCH_SERVER_RATE_LIMIT_WHITELIST:-}
-      - SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=${SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS:-/api/v1/ready}
+      - KC_SEARCH_SERVER_PORT=${KC_SEARCH_SERVER_PORT:-4002}
+      - KC_SEARCH_SERVER_GATEKEEPER_URL=${KC_SEARCH_SERVER_GATEKEEPER_URL:-http://gatekeeper:4224}
+      - KC_SEARCH_SERVER_REFRESH_INTERVAL_MS=${KC_SEARCH_SERVER_REFRESH_INTERVAL_MS:-5000}
+      - KC_SEARCH_SERVER_DB=${KC_SEARCH_SERVER_DB:-sqlite}
+      - KC_SEARCH_SERVER_POSTGRES_URL=${KC_SEARCH_SERVER_POSTGRES_URL:-}
+      - KC_POSTGRES_URL=postgresql://${KC_POSTGRES_USER:-mdip}:${KC_POSTGRES_PASSWORD:-mdip}@postgres:5432/${KC_POSTGRES_DB:-mdip}
+      - KC_SEARCH_SERVER_TRUST_PROXY=${KC_SEARCH_SERVER_TRUST_PROXY:-false}
+      - KC_SEARCH_SERVER_RATE_LIMIT_ENABLED=${KC_SEARCH_SERVER_RATE_LIMIT_ENABLED:-false}
+      - KC_SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE=${KC_SEARCH_SERVER_RATE_LIMIT_WINDOW_VALUE:-1}
+      - KC_SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT=${KC_SEARCH_SERVER_RATE_LIMIT_WINDOW_UNIT:-minute}
+      - KC_SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS=${KC_SEARCH_SERVER_RATE_LIMIT_MAX_REQUESTS:-600}
+      - KC_SEARCH_SERVER_RATE_LIMIT_WHITELIST=${KC_SEARCH_SERVER_RATE_LIMIT_WHITELIST:-}
+      - KC_SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS=${KC_SEARCH_SERVER_RATE_LIMIT_SKIP_PATHS:-/api/v1/ready,/api/v1/status}
       - KC_LOG_LEVEL=${KC_LOG_LEVEL}
     volumes:
       - ./data:/app/search-server/data
     user: "${KC_UID}:${KC_GID}"
     ports:
-      - "${SEARCH_SERVER_PORT:-4002}:${SEARCH_SERVER_PORT:-4002}"
+      - "${KC_SEARCH_SERVER_PORT:-4002}:${KC_SEARCH_SERVER_PORT:-4002}"
     depends_on:
       - gatekeeper
 
@@ -429,7 +450,7 @@ services:
       - VITE_PORT=${KC_REACT_WALLET_PORT:-4228}
       - VITE_GATEKEEPER_URL=http://gatekeeper:4224
       - VITE_KEYMASTER_URL=http://keymaster:4226
-      - VITE_SEARCH_SERVER=http://search-server:${SEARCH_SERVER_PORT:-4002}
+      - VITE_SEARCH_SERVER=http://search-server:${KC_SEARCH_SERVER_PORT:-4002}
     user: "${KC_UID}:${KC_GID}"
     ports:
       - "${KC_REACT_WALLET_PORT:-4228}:${KC_REACT_WALLET_PORT:-4228}"

jest.config.js

@@ -37,6 +37,11 @@ const config = {
         '^\\.\\/sync-persistence\\.js$': '<rootDir>/services/mediators/hyperswarm/src/sync-persistence.ts',
         '^\\.\\/windows\\.js$': '<rootDir>/services/mediators/hyperswarm/src/negentropy/windows.ts',
         '^\\.\\/abstract-json\\.js$': '<rootDir>/packages/gatekeeper/src/db/abstract-json.ts',
+        '^\\.\\/db-utils\\.js$': '<rootDir>/services/search-server/src/db/db-utils.ts',
+        '^\\.\\/did-resolver\\.js$': '<rootDir>/packages/gatekeeper/src/did-resolver.ts',
+        '^\\.\\/index-export\\.js$': '<rootDir>/packages/gatekeeper/src/db/index-export.ts',
+        '^\\.\\/index-helpers\\.js$': '<rootDir>/services/search-server/src/index-helpers.ts',
+        '^\\.\\/projections\\.js$': '<rootDir>/services/search-server/src/projections.ts',
         '^\\.\\/published-credentials\\.js$': '<rootDir>/services/search-server/src/published-credentials.ts',
         '^\\.\\/challenge-receipts\\.js$': '<rootDir>/services/search-server/src/challenge-receipts.ts',
         '^\\.\\/cipher-base\\.js$': '<rootDir>/packages/cipher/src/cipher-base.ts',
@@ -49,6 +54,11 @@ const config = {
     ],
     coveragePathIgnorePatterns: [
         "/node_modules/",
+        "/tests/",
+        "/packages/gatekeeper/src/db/mongo\\.ts$",
+        "/packages/gatekeeper/src/db/postgres\\.ts$",
+        "/packages/gatekeeper/src/db/redis\\.ts$",
+        "/packages/gatekeeper/src/db/sqlite\\.ts$",
         "/services/mediators/hyperswarm/src/negentropy/Negentropy\\.cjs$",
     ]
 };