Add gatekeeper DB liveness readiness

Bushstar · Bushstar · commit 0dc829091181 · 2026-06-08T14:12:52.000+01:00
diff --git a/jest.config.js b/jest.config.js
@@ -40,6 +40,7 @@ const config = {
         '^\\.\\/abstract-json\\.js$': '<rootDir>/packages/gatekeeper/src/db/abstract-json.ts',
         '^\\.\\/db-utils\\.js$': '<rootDir>/services/search-server/src/db/db-utils.ts',
         '^\\.\\/did-resolver\\.js$': '<rootDir>/packages/gatekeeper/src/did-resolver.ts',
+        '^\\.\\/health\\.js$': '<rootDir>/packages/gatekeeper/src/db/health.ts',
         '^\\.\\/index-export\\.js$': '<rootDir>/packages/gatekeeper/src/db/index-export.ts',
         '^\\.\\/index-helpers\\.js$': '<rootDir>/services/search-server/src/index-helpers.ts',
         '^\\.\\/projections\\.js$': '<rootDir>/services/search-server/src/projections.ts',
diff --git a/packages/gatekeeper/src/db/abstract-json.ts b/packages/gatekeeper/src/db/abstract-json.ts
@@ -66,6 +66,10 @@ export abstract class AbstractJson implements GatekeeperDb {
         return;
     }
 
+    async isReady(): Promise<boolean> {
+        return true;
+    }
+
     private splitSuffix(did: string): string {
         if (!did) {
             throw new InvalidDIDError();
diff --git a/packages/gatekeeper/src/db/health.ts b/packages/gatekeeper/src/db/health.ts
@@ -0,0 +1,23 @@
+export const DB_HEALTH_TIMEOUT_MS = 1_000;
+
+export async function withHealthCheckTimeout<T>(
+    check: Promise<T>,
+    message: string,
+    timeoutMs: number = DB_HEALTH_TIMEOUT_MS
+): Promise<T> {
+    let timeoutId: NodeJS.Timeout | undefined;
+
+    try {
+        return await Promise.race([
+            check,
+            new Promise<T>((_, reject) => {
+                timeoutId = setTimeout(() => reject(new Error(message)), timeoutMs);
+            }),
+        ]);
+    }
+    finally {
+        if (timeoutId) {
+            clearTimeout(timeoutId);
+        }
+    }
+}
diff --git a/packages/gatekeeper/src/db/mongo.ts b/packages/gatekeeper/src/db/mongo.ts
@@ -19,6 +19,7 @@ import {
     normalizeIndexExportLimit,
     parseIndexExportCursor
 } from './index-export.js';
+import { DB_HEALTH_TIMEOUT_MS, withHealthCheckTimeout } from './health.js';
 
 interface DidsDoc {
     id: string
@@ -105,6 +106,26 @@ export default class DbMongo implements GatekeeperDb {
         }
     }
 
+    async isReady(): Promise<boolean> {
+        if (!this.client || !this.db) {
+            return false;
+        }
+
+        try {
+            await withHealthCheckTimeout(
+                this.client.db('admin').command(
+                    { ping: 1 },
+                    { timeoutMS: DB_HEALTH_TIMEOUT_MS }
+                ),
+                'Mongo readiness check timed out'
+            );
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+
     async resetDb(): Promise<void> {
         if (!this.db) {
             throw new Error(MONGO_NOT_STARTED_ERROR)
diff --git a/packages/gatekeeper/src/db/postgres.ts b/packages/gatekeeper/src/db/postgres.ts
@@ -17,6 +17,7 @@ import {
     normalizeIndexExportLimit,
     parseIndexExportCursor
 } from './index-export.js';
+import { withHealthCheckTimeout } from './health.js';
 
 interface EventRow {
     event: GatekeeperEvent | string | null;
@@ -263,6 +264,23 @@ export default class DbPostgres implements GatekeeperDb {
         }
     }
 
+    async isReady(): Promise<boolean> {
+        if (!this.pool) {
+            return false;
+        }
+
+        try {
+            await withHealthCheckTimeout(
+                this.pool.query('SELECT 1'),
+                'Postgres readiness check timed out'
+            );
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+
     async resetDb(): Promise<void> {
         await this.withTx(async client => {
             await client.query('DELETE FROM gatekeeper_events WHERE namespace = $1', [this.dbName]);
diff --git a/packages/gatekeeper/src/db/redis.ts b/packages/gatekeeper/src/db/redis.ts
@@ -19,6 +19,7 @@ import {
     normalizeIndexExportLimit,
     parseIndexExportCursor
 } from './index-export.js';
+import { withHealthCheckTimeout } from './health.js';
 
 const REDIS_NOT_STARTED_ERROR = 'Redis not started. Call start() first.';
 const log = childLogger({ service: 'gatekeeper-db', module: 'redis' });
@@ -71,6 +72,26 @@ export default class DbRedis implements GatekeeperDb {
         }
     }
 
+    async isReady(): Promise<boolean> {
+        if (!this.redis) {
+            return false;
+        }
+        if (this.redis.status !== 'ready') {
+            return false;
+        }
+
+        try {
+            const response = await withHealthCheckTimeout(
+                this.redis.ping(),
+                'Redis readiness check timed out'
+            );
+            return response === 'PONG';
+        }
+        catch {
+            return false;
+        }
+    }
+
     private logRedisConnectionError(error: unknown): void {
         const now = Date.now();
         if (now - this.lastRedisErrorLogAt < 60_000) {
diff --git a/packages/gatekeeper/src/db/sqlite.ts b/packages/gatekeeper/src/db/sqlite.ts
@@ -149,6 +149,20 @@ export default class DbSqlite implements GatekeeperDb {
         }
     }
 
+    async isReady(): Promise<boolean> {
+        if (!this.db) {
+            return false;
+        }
+
+        try {
+            await this.db.get('SELECT 1');
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+
 
     async resetDb(): Promise<void> {
         if (!this.db) {
diff --git a/packages/gatekeeper/src/types.ts b/packages/gatekeeper/src/types.ts
@@ -191,6 +191,7 @@ export type IndexExportResponse = IndexExportSnapshotResponse | IndexExportChang
 export interface GatekeeperDb {
     start(): Promise<void>;
     stop(): Promise<void>;
+    isReady(): Promise<boolean>;
     resetDb(): Promise<void | number | JsonDbFile>;
     addEvent(did: string, event: GatekeeperEvent): Promise<void | number>;
     getEvents(did: string): Promise<GatekeeperEvent[]>;
diff --git a/services/gatekeeper/server/src/gatekeeper-api.ts b/services/gatekeeper/server/src/gatekeeper-api.ts
@@ -25,6 +25,7 @@ import {
     formatBytes,
     formatDuration,
     isRateLimitWhitelistedRequest,
+    isGatekeeperReady,
     logRequest,
     parseIndexExportRequest,
     rateLimitWindowUnits,
@@ -170,7 +171,8 @@ let serverReady = false;
  */
 v1router.get('/ready', async (req, res) => {
     try {
-        res.json(serverReady);
+        const ready = await isGatekeeperReady(serverReady, db);
+        res.status(ready ? 200 : 503).json(ready);
     } catch (error: any) {
         res.status(500).send(error.toString());
     }
diff --git a/services/gatekeeper/server/src/helpers.ts b/services/gatekeeper/server/src/helpers.ts
@@ -1,7 +1,7 @@
 import type { NextFunction, Request, Response } from 'express';
 import { BlockList, isIP } from 'net';
 
-import type { IndexExportRequest } from '@mdip/gatekeeper/types';
+import type { GatekeeperDb, IndexExportRequest } from '@mdip/gatekeeper/types';
 import { childLogger } from '@mdip/common/logger';
 
 const log = childLogger({ service: 'gatekeeper-server' });
@@ -129,6 +129,22 @@ export function shouldSkipRateLimitPath(req: Request, skipPaths: string[]): bool
         pathOnly === skipPath || pathOnly.startsWith(`${skipPath}/`));
 }
 
+export async function isGatekeeperReady(
+    serverReady: boolean,
+    db: Pick<GatekeeperDb, 'isReady'>
+): Promise<boolean> {
+    if (!serverReady) {
+        return false;
+    }
+
+    try {
+        return await db.isReady();
+    }
+    catch {
+        return false;
+    }
+}
+
 export function parseOptionalString(value: unknown, fieldName: string): string | null | undefined {
     if (value === undefined) {
         return undefined;
diff --git a/tests/gatekeeper/client.test.ts b/tests/gatekeeper/client.test.ts
@@ -84,6 +84,17 @@ describe('isReady', () => {
         expect(isReady).toBe(false);
     });
 
+    it('should return false when service is not ready', async () => {
+        nock(GatekeeperURL)
+            .get(Endpoints.ready)
+            .reply(503, 'false');
+
+        const gatekeeper = await GatekeeperClient.create({ url: GatekeeperURL });
+        const isReady = await gatekeeper.isReady();
+
+        expect(isReady).toBe(false);
+    });
+
     it('should wait until ready', async () => {
         nock(GatekeeperURL)
             .get(Endpoints.ready)
diff --git a/tests/gatekeeper/db-health.test.ts b/tests/gatekeeper/db-health.test.ts
@@ -0,0 +1,97 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { jest } from '@jest/globals';
+import DbJsonMemory from '@mdip/gatekeeper/db/json-memory.ts';
+import DbMongo from '@mdip/gatekeeper/db/mongo.ts';
+import DbPostgres from '@mdip/gatekeeper/db/postgres.ts';
+import DbRedis from '@mdip/gatekeeper/db/redis.ts';
+import DbSqlite from '@mdip/gatekeeper/db/sqlite.ts';
+import { withHealthCheckTimeout } from '@mdip/gatekeeper/db/health.ts';
+
+describe('database readiness checks', () => {
+    it('reports JSON memory as ready', async () => {
+        const db = new DbJsonMemory('health-json-memory');
+
+        await expect(db.isReady()).resolves.toBe(true);
+    });
+
+    it('reports SQLite readiness from the open handle', async () => {
+        const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gatekeeper-health-'));
+        const db = new DbSqlite('sqlite', tempDir);
+
+        try {
+            await expect(db.isReady()).resolves.toBe(false);
+            await db.start();
+            await expect(db.isReady()).resolves.toBe(true);
+            await db.stop();
+            await expect(db.isReady()).resolves.toBe(false);
+        }
+        finally {
+            fs.rmSync(tempDir, { recursive: true, force: true });
+        }
+    });
+
+    it('reports Mongo readiness from an admin ping', async () => {
+        const db = new DbMongo('health-mongo');
+        const command = jest.fn().mockResolvedValue({ ok: 1 });
+
+        await expect(db.isReady()).resolves.toBe(false);
+
+        (db as any).client = {
+            db: jest.fn(() => ({ command })),
+        };
+        (db as any).db = {};
+
+        await expect(db.isReady()).resolves.toBe(true);
+        expect(command).toHaveBeenCalledWith(
+            { ping: 1 },
+            { timeoutMS: 1_000 }
+        );
+
+        command.mockRejectedValue(new Error('mongo down'));
+        await expect(db.isReady()).resolves.toBe(false);
+    });
+
+    it('reports Redis readiness only for a ready client with a successful ping', async () => {
+        const db = new DbRedis('health-redis');
+        const ping = jest.fn().mockResolvedValue('PONG');
+
+        await expect(db.isReady()).resolves.toBe(false);
+
+        (db as any).redis = {
+            status: 'end',
+            ping,
+        };
+        await expect(db.isReady()).resolves.toBe(false);
+        expect(ping).not.toHaveBeenCalled();
+
+        (db as any).redis.status = 'ready';
+        await expect(db.isReady()).resolves.toBe(true);
+
+        ping.mockResolvedValue('NOPE');
+        await expect(db.isReady()).resolves.toBe(false);
+    });
+
+    it('reports Postgres readiness from a lightweight query', async () => {
+        const db = new DbPostgres('health-postgres');
+        const query = jest.fn().mockResolvedValue({ rows: [{ '?column?': 1 }] });
+
+        await expect(db.isReady()).resolves.toBe(false);
+
+        (db as any).pool = { query };
+        await expect(db.isReady()).resolves.toBe(true);
+        expect(query).toHaveBeenCalledWith('SELECT 1');
+
+        query.mockRejectedValue(new Error('postgres down'));
+        await expect(db.isReady()).resolves.toBe(false);
+    });
+
+    it('bounds health checks with a timeout', async () => {
+        await expect(withHealthCheckTimeout(
+            new Promise(() => undefined),
+            'health timeout',
+            1
+        )).rejects.toThrow('health timeout');
+    });
+});
diff --git a/tests/gatekeeper/server-helpers.test.ts b/tests/gatekeeper/server-helpers.test.ts
@@ -139,6 +139,24 @@ describe('gatekeeper server helpers', () => {
         } as never, ['/api/v1/status'])).toBe(false);
     });
 
+    it('checks database liveness before reporting ready', async () => {
+        const db = {
+            isReady: jest.fn<() => Promise<boolean>>().mockResolvedValue(true),
+        };
+
+        await expect(helpers.isGatekeeperReady(false, db)).resolves.toBe(false);
+        expect(db.isReady).not.toHaveBeenCalled();
+
+        await expect(helpers.isGatekeeperReady(true, db)).resolves.toBe(true);
+        expect(db.isReady).toHaveBeenCalledTimes(1);
+
+        db.isReady.mockResolvedValue(false);
+        await expect(helpers.isGatekeeperReady(true, db)).resolves.toBe(false);
+
+        db.isReady.mockRejectedValue(new Error('db down'));
+        await expect(helpers.isGatekeeperReady(true, db)).resolves.toBe(false);
+    });
+
     it('parses optional strings and positive integers', () => {
         expect(helpers.parseOptionalString(undefined, 'cursor')).toBeUndefined();
         expect(helpers.parseOptionalString(null, 'cursor')).toBeNull();
