Run release validation as a one-shot job

docker-compose.dev.yml

@@ -86,9 +86,7 @@ services:
       context: .
     env_file:
       - .env
-    environment:
-      - SIMULTANEOUS_TASKS=1
-    restart: unless-stopped
+    restart: 'no'
     entrypoint: yarn run-release-validator
     volumes:
       - ./:/usr/src/app

docker-compose.prod.yml

@@ -60,10 +60,8 @@ services:
     network_mode: host
     env_file:
       - .env
-    environment:
-      - SIMULTANEOUS_TASKS=1
-    restart: unless-stopped
-    entrypoint: /usr/local/bin/node runner.js hawk-worker-release-validator
+    restart: 'no'
+    entrypoint: /usr/local/bin/node workers/release-validator/src/index.js
 
   #
   # Notification workers: notifier, email, telegram

package.json

@@ -48,7 +48,7 @@
         "run-paymaster": "yarn worker hawk-worker-paymaster",
         "run-notifier": "yarn worker hawk-worker-notifier",
         "run-release": "yarn worker hawk-worker-release",
-        "run-release-validator": "yarn worker hawk-worker-release-validator",
+        "run-release-validator": "ts-node -T --prefer-ts-exts ./workers/release-validator/src/index.ts",
         "run-email": "yarn worker hawk-worker-email",
         "run-telegram": "yarn worker hawk-worker-telegram",
         "run-limiter": "yarn worker hawk-worker-limiter",

workers/release-validator/README.md

@@ -1,15 +1,19 @@
-# Release Validator Worker
+# Release Validator Job
 
-Checks releases after a 24-hour observation period and marks original events that no longer occur as resolved.
+One-shot MongoDB job that checks releases after a 24-hour observation period and exits. It does not use RabbitMQ.
 
-The worker processes releases from oldest to newest, stores the first release without an event in `resolvedInRelease`, and marks successfully processed releases with `fixChecked: true`.
+Events that no longer occur are marked with `resolvedInRelease`; successfully checked releases receive `fixChecked: true`. The validation window follows the archiver retention period.
 
-The worker only uses the fields required for matching releases and event groups. Records without these fields are ignored. Candidate releases must be older than 24 hours and no older than the `MAX_DAYS_NUMBER` retention period, while all available project releases are still used to compare event history. If an event's original release was archived, the original event timestamp is used as its chronological boundary.
+## Run
 
-Queue: `cron-tasks/release-validator`
-
-Run locally:
+From the workers repository root:
 
 ```sh
 yarn run-release-validator
 ```
+
+Requires the events database connection. Retention defaults to 30 days; `MAX_DAYS_NUMBER` overrides it for the environment. The `repetitions:<projectId>` collections must have the `{ groupHash: 1, release: 1 }` index named `groupHashAndRelease`.
+
+The process exits with code `0` on success, including when there is nothing to check, or `1` on failure.
+
+For Kubernetes deployment, see [KUBERNETES.md](./KUBERNETES.md).

workers/release-validator/package.json

@@ -1,10 +1,9 @@
 {
   "name": "hawk-worker-release-validator",
   "version": "0.0.1",
-  "description": "Detects events fixed by a release",
+  "description": "One-shot job that detects events fixed by a release",
   "main": "src/index.ts",
   "author": "CodeX",
   "license": "UNLICENSED",
-  "private": true,
-  "workerType": "cron-tasks/release-validator"
+  "private": true
 }

workers/release-validator/src/index.ts

@@ -1,46 +1,34 @@
+import 'dotenv/config';
 import { DatabaseController } from '../../../lib/db/controller';
-import { Worker } from '../../../lib/worker';
-import * as pkg from '../package.json';
+import createLogger from '../../../lib/logger';
 import { validateReleases } from './validate-releases';
 
+const logger = createLogger();
+
 /**
- * Worker that detects events fixed by a release.
+ * Run one release validation pass and close MongoDB before returning.
+ *
+ * @returns {Promise<void>} completion of the validation and connection cleanup
  */
-export default class ReleaseValidatorWorker extends Worker {
-  /**
-   * Worker type.
-   */
-  public readonly type: string = pkg.workerType;
+export async function runReleaseValidation(): Promise<void> {
+  logger.info('Release validation started');
 
-  /**
-   * Events database controller.
-   */
-  private eventsDb = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI);
+  const eventsDb = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI);
 
-  /**
-   * Connect to the events database and start consuming tasks.
-   */
-  public async start(): Promise<void> {
-    await this.eventsDb.connect();
-    await super.start();
-  }
+  try {
+    const db = await eventsDb.connect();
 
-  /**
-   * Stop consuming tasks and close the database connection.
-   */
-  public async finish(): Promise<void> {
-    await super.finish();
-    await this.eventsDb.close();
+    await validateReleases(db);
+  } finally {
+    await eventsDb.close();
   }
 
-  /**
-   * Handle a scheduled release validation task.
-   */
-  public async handle(): Promise<void> {
-    this.logger.info('Release validation started');
-
-    await validateReleases(this.eventsDb.getConnection());
+  logger.info('Release validation finished');
+}
 
-    this.logger.info('Release validation finished');
-  }
+if (require.main === module) {
+  runReleaseValidation().catch(error => {
+    logger.error('Release validation failed', error);
+    process.exitCode = 1;
+  });
 }

workers/release-validator/src/validate-releases.ts

@@ -24,6 +24,11 @@ const RELEASE_OBSERVATION_PERIOD_SECONDS = HOURS_IN_DAY * MINUTES_IN_HOUR * SECO
  */
 const EVENTS_BATCH_SIZE = 50;
 
+/**
+ * Default retention period when no environment override is provided.
+ */
+const DEFAULT_MAX_DAYS_NUMBER = 30;
+
 /**
  * Find unchecked releases older than the observation period.
  *
@@ -32,7 +37,7 @@ const EVENTS_BATCH_SIZE = 50;
  */
 async function findReleasesToCheck(db: Db, now: Date): Promise<ReleaseDBScheme[]> {
   const nowSeconds = Math.floor(now.getTime() / MS_IN_SEC);
-  const releaseMaxAgeDays = Number(process.env.MAX_DAYS_NUMBER);
+  const releaseMaxAgeDays = Number(process.env.MAX_DAYS_NUMBER ?? DEFAULT_MAX_DAYS_NUMBER);
 
   if (!Number.isFinite(releaseMaxAgeDays) || releaseMaxAgeDays <= 0) {
     throw new Error('MAX_DAYS_NUMBER must be a positive number');
@@ -378,6 +383,7 @@ async function validateProject(db: Db, projectId: string, releasesToCheck: Relea
  *
  * @param db - events database connection
  * @param now - current time
+ * @throws when any project fails, after attempting the remaining projects
  */
 export async function validateReleases(db: Db, now = new Date()): Promise<void> {
   /**
@@ -386,12 +392,21 @@ export async function validateReleases(db: Db, now = new Date()): Promise<void>
    */
   const releasesToCheck = await findReleasesToCheck(db, now);
   const releasesByProject = groupReleasesByProject(releasesToCheck);
+  const failedProjects: string[] = [];
+
+  logger.info(`Release validation: ${releasesToCheck.length} unchecked releases in ${releasesByProject.size} projects`);
 
   for (const [projectId, projectReleases] of releasesByProject) {
     try {
       await validateProject(db, projectId, projectReleases);
+      logger.info(`Checked ${projectReleases.length} releases for project ${projectId}`);
     } catch (error) {
+      failedProjects.push(projectId);
       logger.error(`Failed to validate releases for project ${projectId}`, error);
     }
   }
+
+  if (failedProjects.length > 0) {
+    throw new Error(`Release validation failed for ${failedProjects.length} projects: ${failedProjects.join(', ')}`);
+  }
 }

workers/release-validator/tests/index.test.ts

@@ -0,0 +1,134 @@
+import { spawnSync, SpawnSyncReturns } from 'child_process';
+import { resolve } from 'path';
+import type { Db } from 'mongodb';
+import { connect as connectRegistry } from 'amqplib';
+import { DatabaseController } from '../../../lib/db/controller';
+import { runReleaseValidation } from '../src';
+import { validateReleases } from '../src/validate-releases';
+
+jest.mock('../../../lib/db/controller');
+jest.mock('../src/validate-releases');
+
+const MockDatabaseController = DatabaseController as jest.MockedClass<typeof DatabaseController>;
+const mockValidateReleases = validateReleases as jest.MockedFunction<typeof validateReleases>;
+
+/**
+ * Execute the real CLI against the local MongoDB started by the Jest configuration.
+ *
+ * @param maxDays - configured release retention period
+ * @param uri - isolated test database, or an empty value to test configuration failure
+ * @returns {SpawnSyncReturns<string>} process output and exit status, bounded by a timeout
+ */
+function runCli(maxDays: string, uri = `mongodb://127.0.0.1:55010/release-validator-cli-${process.pid}`): SpawnSyncReturns<string> {
+  return spawnSync(process.execPath, [
+    '-r', require.resolve('ts-node/register/transpile-only'),
+    resolve(__dirname, '../src/index.ts'),
+  ], {
+    cwd: resolve(__dirname, '../../..'),
+    env: {
+      ...process.env,
+      MONGO_EVENTS_DATABASE_URI: uri,
+      MAX_DAYS_NUMBER: maxDays,
+      MONGO_RECONNECT_TRIES: '1',
+      TS_NODE_PREFER_TS_EXTS: 'true',
+      REGISTRY_URL: 'amqp://127.0.0.1:1',
+      LOG_LEVEL: 'info',
+    },
+    encoding: 'utf8',
+    timeout: 10000,
+  });
+}
+
+describe('One-shot release validation', () => {
+  const db = {} as Db;
+  const connect = jest.fn();
+  const close = jest.fn();
+  const originalUri = process.env.MONGO_EVENTS_DATABASE_URI;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    process.env.MONGO_EVENTS_DATABASE_URI = 'mongodb://test/events';
+    connect.mockResolvedValue(db);
+    close.mockResolvedValue(undefined);
+    mockValidateReleases.mockResolvedValue(undefined);
+    MockDatabaseController.mockImplementation(() => ({
+      connect,
+      close,
+    } as unknown as DatabaseController));
+  });
+
+  afterAll(() => {
+    if (originalUri === undefined) {
+      delete process.env.MONGO_EVENTS_DATABASE_URI;
+    } else {
+      process.env.MONGO_EVENTS_DATABASE_URI = originalUri;
+    }
+  });
+
+  test('should immediately validate once and close MongoDB without connecting to RabbitMQ', async () => {
+    await runReleaseValidation();
+
+    expect(MockDatabaseController).toHaveBeenCalledWith('mongodb://test/events');
+    expect(connect).toHaveBeenCalledTimes(1);
+    expect(mockValidateReleases).toHaveBeenCalledTimes(1);
+    expect(mockValidateReleases).toHaveBeenCalledWith(db);
+    expect(close).toHaveBeenCalledTimes(1);
+    expect(close.mock.invocationCallOrder[0]).toBeGreaterThan(mockValidateReleases.mock.invocationCallOrder[0]);
+    expect(connectRegistry).not.toHaveBeenCalled();
+  });
+
+  test('should close MongoDB and propagate validation failures', async () => {
+    const error = new Error('Project validation failed');
+
+    mockValidateReleases.mockRejectedValueOnce(error);
+
+    await expect(runReleaseValidation()).rejects.toBe(error);
+    expect(close).toHaveBeenCalledTimes(1);
+  });
+
+  test('should clean up after a connection failure without attempting validation', async () => {
+    const error = new Error('MongoDB unavailable');
+
+    connect.mockRejectedValueOnce(error);
+
+    await expect(runReleaseValidation()).rejects.toBe(error);
+    expect(mockValidateReleases).not.toHaveBeenCalled();
+    expect(close).toHaveBeenCalledTimes(1);
+  });
+
+  test('should report a cleanup failure rather than completing successfully', async () => {
+    close.mockRejectedValueOnce(new Error('Could not close MongoDB'));
+
+    await expect(runReleaseValidation()).rejects.toThrow('Could not close MongoDB');
+  });
+});
+
+describe('Release validator CLI', () => {
+  test('should exit successfully when there are no eligible releases, without waiting for a queue task', () => {
+    const result = runCli('30');
+
+    expect(result.error).toBeUndefined();
+    expect(result.status).toBe(0);
+    expect(result.stdout).toContain('Release validation started');
+    expect(result.stdout).toContain('0 unchecked releases in 0 projects');
+    expect(result.stdout).toContain('Release validation finished');
+  });
+
+  test('should exit with code 1 for invalid retention configuration', () => {
+    const result = runCli('0');
+    const output = result.stdout + result.stderr;
+
+    expect(result.error).toBeUndefined();
+    expect(result.status).toBe(1);
+    expect(output).toContain('MAX_DAYS_NUMBER must be a positive number');
+    expect(output).not.toContain('Release validation finished');
+  });
+
+  test('should exit with code 1 when the database URI is missing', () => {
+    const result = runCli('30', '');
+
+    expect(result.error).toBeUndefined();
+    expect(result.status).toBe(1);
+    expect(result.stdout + result.stderr).toContain('Connection URI is not specified');
+  });
+});

workers/release-validator/tests/validate-releases.test.ts

@@ -351,7 +351,7 @@ describe('validateReleases', () => {
     expect((await events.findOne({ groupHash: 'error-active-regression' })).resolvedInRelease).toBe('b');
   });
 
-  test('should continue validating other projects when one project fails', async () => {
+  test('should report failure after continuing to validate the remaining projects', async () => {
     const failedProjectId = 'failed-release-validator-project';
 
     await releases.insertMany([
@@ -391,7 +391,7 @@ describe('validateReleases', () => {
     await events.insertOne(createEvent('successful-project-event', 'a'));
 
     try {
-      await expect(validateReleases(db, NOW)).resolves.toBeUndefined();
+      await expect(validateReleases(db, NOW)).rejects.toThrow(failedProjectId);
 
       expect((await releases.findOne({
         projectId: failedProjectId,
@@ -409,6 +409,29 @@ describe('validateReleases', () => {
     }
   });
 
+  test('should default to a 30-day retention period when MAX_DAYS_NUMBER is not set', async () => {
+    const originalMaxDaysNumber = process.env.MAX_DAYS_NUMBER;
+
+    delete process.env.MAX_DAYS_NUMBER;
+
+    try {
+      await releases.insertMany([
+        createRelease('outside-default-window', 31 * 24),
+        createRelease('a', 72, true),
+        createRelease('b', 48),
+      ]);
+      await events.insertOne(createEvent('default-retention-event', 'a'));
+
+      await validateReleases(db, NOW);
+
+      expect((await releases.findOne({ release: 'outside-default-window' })).fixChecked).toBe(false);
+      expect((await releases.findOne({ release: 'b' })).fixChecked).toBe(true);
+      expect((await events.findOne({ groupHash: 'default-retention-event' })).resolvedInRelease).toBe('b');
+    } finally {
+      process.env.MAX_DAYS_NUMBER = originalMaxDaysNumber;
+    }
+  });
+
   test('should not select a release older than the configured retention period as a candidate', async () => {
     const originalMaxDaysNumber = process.env.MAX_DAYS_NUMBER;