diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index cc131d23..5826d625 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -86,9 +86,7 @@ services: context: . env_file: - .env - environment: - - SIMULTANEOUS_TASKS=1 - restart: unless-stopped + restart: 'no' entrypoint: yarn run-release-validator volumes: - ./:/usr/src/app diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a702aefc..16e81ec7 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -60,10 +60,8 @@ services: network_mode: host env_file: - .env - environment: - - SIMULTANEOUS_TASKS=1 - restart: unless-stopped - entrypoint: /usr/local/bin/node runner.js hawk-worker-release-validator + restart: 'no' + entrypoint: /usr/local/bin/node workers/release-validator/src/index.js # # Notification workers: notifier, email, telegram diff --git a/package.json b/package.json index b5bb807e..9780eab7 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "run-paymaster": "yarn worker hawk-worker-paymaster", "run-notifier": "yarn worker hawk-worker-notifier", "run-release": "yarn worker hawk-worker-release", - "run-release-validator": "yarn worker hawk-worker-release-validator", + "run-release-validator": "ts-node -T --prefer-ts-exts ./workers/release-validator/src/index.ts", "run-email": "yarn worker hawk-worker-email", "run-telegram": "yarn worker hawk-worker-telegram", "run-limiter": "yarn worker hawk-worker-limiter", diff --git a/workers/release-validator/README.md b/workers/release-validator/README.md index a64e9665..90f53625 100644 --- a/workers/release-validator/README.md +++ b/workers/release-validator/README.md @@ -1,15 +1,19 @@ -# Release Validator Worker +# Release Validator Job -Checks releases after a 24-hour observation period and marks original events that no longer occur as resolved. +One-shot MongoDB job that checks releases after a 24-hour observation period and exits. It does not use RabbitMQ. -The worker processes releases from oldest to newest, stores the first release without an event in `resolvedInRelease`, and marks successfully processed releases with `fixChecked: true`. +Events that no longer occur are marked with `resolvedInRelease`; successfully checked releases receive `fixChecked: true`. The validation window follows the archiver retention period. -The worker only uses the fields required for matching releases and event groups. Records without these fields are ignored. Candidate releases must be older than 24 hours and no older than the `MAX_DAYS_NUMBER` retention period, while all available project releases are still used to compare event history. If an event's original release was archived, the original event timestamp is used as its chronological boundary. +## Run -Queue: `cron-tasks/release-validator` - -Run locally: +From the workers repository root: ```sh yarn run-release-validator ``` + +Requires the events database connection. Retention defaults to 30 days; `MAX_DAYS_NUMBER` overrides it for the environment. The `repetitions:` collections must have the `{ groupHash: 1, release: 1 }` index named `groupHashAndRelease`. + +The process exits with code `0` on success, including when there is nothing to check, or `1` on failure. + +For Kubernetes deployment, see [KUBERNETES.md](./KUBERNETES.md). diff --git a/workers/release-validator/package.json b/workers/release-validator/package.json index 3cd8c88f..cc638361 100644 --- a/workers/release-validator/package.json +++ b/workers/release-validator/package.json @@ -1,10 +1,9 @@ { "name": "hawk-worker-release-validator", "version": "0.0.1", - "description": "Detects events fixed by a release", + "description": "One-shot job that detects events fixed by a release", "main": "src/index.ts", "author": "CodeX", "license": "UNLICENSED", - "private": true, - "workerType": "cron-tasks/release-validator" + "private": true } diff --git a/workers/release-validator/src/index.ts b/workers/release-validator/src/index.ts index ac268f6a..0e38ebd7 100644 --- a/workers/release-validator/src/index.ts +++ b/workers/release-validator/src/index.ts @@ -1,46 +1,34 @@ +import 'dotenv/config'; import { DatabaseController } from '../../../lib/db/controller'; -import { Worker } from '../../../lib/worker'; -import * as pkg from '../package.json'; +import createLogger from '../../../lib/logger'; import { validateReleases } from './validate-releases'; +const logger = createLogger(); + /** - * Worker that detects events fixed by a release. + * Run one release validation pass and close MongoDB before returning. + * + * @returns {Promise} completion of the validation and connection cleanup */ -export default class ReleaseValidatorWorker extends Worker { - /** - * Worker type. - */ - public readonly type: string = pkg.workerType; +export async function runReleaseValidation(): Promise { + logger.info('Release validation started'); - /** - * Events database controller. - */ - private eventsDb = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI); + const eventsDb = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI); - /** - * Connect to the events database and start consuming tasks. - */ - public async start(): Promise { - await this.eventsDb.connect(); - await super.start(); - } + try { + const db = await eventsDb.connect(); - /** - * Stop consuming tasks and close the database connection. - */ - public async finish(): Promise { - await super.finish(); - await this.eventsDb.close(); + await validateReleases(db); + } finally { + await eventsDb.close(); } - /** - * Handle a scheduled release validation task. - */ - public async handle(): Promise { - this.logger.info('Release validation started'); - - await validateReleases(this.eventsDb.getConnection()); + logger.info('Release validation finished'); +} - this.logger.info('Release validation finished'); - } +if (require.main === module) { + runReleaseValidation().catch(error => { + logger.error('Release validation failed', error); + process.exitCode = 1; + }); } diff --git a/workers/release-validator/src/validate-releases.ts b/workers/release-validator/src/validate-releases.ts index f8c4910a..100b908e 100644 --- a/workers/release-validator/src/validate-releases.ts +++ b/workers/release-validator/src/validate-releases.ts @@ -24,6 +24,11 @@ const RELEASE_OBSERVATION_PERIOD_SECONDS = HOURS_IN_DAY * MINUTES_IN_HOUR * SECO */ const EVENTS_BATCH_SIZE = 50; +/** + * Default retention period when no environment override is provided. + */ +const DEFAULT_MAX_DAYS_NUMBER = 30; + /** * Find unchecked releases older than the observation period. * @@ -32,7 +37,7 @@ const EVENTS_BATCH_SIZE = 50; */ async function findReleasesToCheck(db: Db, now: Date): Promise { const nowSeconds = Math.floor(now.getTime() / MS_IN_SEC); - const releaseMaxAgeDays = Number(process.env.MAX_DAYS_NUMBER); + const releaseMaxAgeDays = Number(process.env.MAX_DAYS_NUMBER ?? DEFAULT_MAX_DAYS_NUMBER); if (!Number.isFinite(releaseMaxAgeDays) || releaseMaxAgeDays <= 0) { throw new Error('MAX_DAYS_NUMBER must be a positive number'); @@ -378,6 +383,7 @@ async function validateProject(db: Db, projectId: string, releasesToCheck: Relea * * @param db - events database connection * @param now - current time + * @throws when any project fails, after attempting the remaining projects */ export async function validateReleases(db: Db, now = new Date()): Promise { /** @@ -386,12 +392,21 @@ export async function validateReleases(db: Db, now = new Date()): Promise */ const releasesToCheck = await findReleasesToCheck(db, now); const releasesByProject = groupReleasesByProject(releasesToCheck); + const failedProjects: string[] = []; + + logger.info(`Release validation: ${releasesToCheck.length} unchecked releases in ${releasesByProject.size} projects`); for (const [projectId, projectReleases] of releasesByProject) { try { await validateProject(db, projectId, projectReleases); + logger.info(`Checked ${projectReleases.length} releases for project ${projectId}`); } catch (error) { + failedProjects.push(projectId); logger.error(`Failed to validate releases for project ${projectId}`, error); } } + + if (failedProjects.length > 0) { + throw new Error(`Release validation failed for ${failedProjects.length} projects: ${failedProjects.join(', ')}`); + } } diff --git a/workers/release-validator/tests/index.test.ts b/workers/release-validator/tests/index.test.ts new file mode 100644 index 00000000..9dbb0a78 --- /dev/null +++ b/workers/release-validator/tests/index.test.ts @@ -0,0 +1,134 @@ +import { spawnSync, SpawnSyncReturns } from 'child_process'; +import { resolve } from 'path'; +import type { Db } from 'mongodb'; +import { connect as connectRegistry } from 'amqplib'; +import { DatabaseController } from '../../../lib/db/controller'; +import { runReleaseValidation } from '../src'; +import { validateReleases } from '../src/validate-releases'; + +jest.mock('../../../lib/db/controller'); +jest.mock('../src/validate-releases'); + +const MockDatabaseController = DatabaseController as jest.MockedClass; +const mockValidateReleases = validateReleases as jest.MockedFunction; + +/** + * Execute the real CLI against the local MongoDB started by the Jest configuration. + * + * @param maxDays - configured release retention period + * @param uri - isolated test database, or an empty value to test configuration failure + * @returns {SpawnSyncReturns} process output and exit status, bounded by a timeout + */ +function runCli(maxDays: string, uri = `mongodb://127.0.0.1:55010/release-validator-cli-${process.pid}`): SpawnSyncReturns { + return spawnSync(process.execPath, [ + '-r', require.resolve('ts-node/register/transpile-only'), + resolve(__dirname, '../src/index.ts'), + ], { + cwd: resolve(__dirname, '../../..'), + env: { + ...process.env, + MONGO_EVENTS_DATABASE_URI: uri, + MAX_DAYS_NUMBER: maxDays, + MONGO_RECONNECT_TRIES: '1', + TS_NODE_PREFER_TS_EXTS: 'true', + REGISTRY_URL: 'amqp://127.0.0.1:1', + LOG_LEVEL: 'info', + }, + encoding: 'utf8', + timeout: 10000, + }); +} + +describe('One-shot release validation', () => { + const db = {} as Db; + const connect = jest.fn(); + const close = jest.fn(); + const originalUri = process.env.MONGO_EVENTS_DATABASE_URI; + + beforeEach(() => { + jest.clearAllMocks(); + process.env.MONGO_EVENTS_DATABASE_URI = 'mongodb://test/events'; + connect.mockResolvedValue(db); + close.mockResolvedValue(undefined); + mockValidateReleases.mockResolvedValue(undefined); + MockDatabaseController.mockImplementation(() => ({ + connect, + close, + } as unknown as DatabaseController)); + }); + + afterAll(() => { + if (originalUri === undefined) { + delete process.env.MONGO_EVENTS_DATABASE_URI; + } else { + process.env.MONGO_EVENTS_DATABASE_URI = originalUri; + } + }); + + test('should immediately validate once and close MongoDB without connecting to RabbitMQ', async () => { + await runReleaseValidation(); + + expect(MockDatabaseController).toHaveBeenCalledWith('mongodb://test/events'); + expect(connect).toHaveBeenCalledTimes(1); + expect(mockValidateReleases).toHaveBeenCalledTimes(1); + expect(mockValidateReleases).toHaveBeenCalledWith(db); + expect(close).toHaveBeenCalledTimes(1); + expect(close.mock.invocationCallOrder[0]).toBeGreaterThan(mockValidateReleases.mock.invocationCallOrder[0]); + expect(connectRegistry).not.toHaveBeenCalled(); + }); + + test('should close MongoDB and propagate validation failures', async () => { + const error = new Error('Project validation failed'); + + mockValidateReleases.mockRejectedValueOnce(error); + + await expect(runReleaseValidation()).rejects.toBe(error); + expect(close).toHaveBeenCalledTimes(1); + }); + + test('should clean up after a connection failure without attempting validation', async () => { + const error = new Error('MongoDB unavailable'); + + connect.mockRejectedValueOnce(error); + + await expect(runReleaseValidation()).rejects.toBe(error); + expect(mockValidateReleases).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledTimes(1); + }); + + test('should report a cleanup failure rather than completing successfully', async () => { + close.mockRejectedValueOnce(new Error('Could not close MongoDB')); + + await expect(runReleaseValidation()).rejects.toThrow('Could not close MongoDB'); + }); +}); + +describe('Release validator CLI', () => { + test('should exit successfully when there are no eligible releases, without waiting for a queue task', () => { + const result = runCli('30'); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(result.stdout).toContain('Release validation started'); + expect(result.stdout).toContain('0 unchecked releases in 0 projects'); + expect(result.stdout).toContain('Release validation finished'); + }); + + test('should exit with code 1 for invalid retention configuration', () => { + const result = runCli('0'); + const output = result.stdout + result.stderr; + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(output).toContain('MAX_DAYS_NUMBER must be a positive number'); + expect(output).not.toContain('Release validation finished'); + }); + + test('should exit with code 1 when the database URI is missing', () => { + const result = runCli('30', ''); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout + result.stderr).toContain('Connection URI is not specified'); + }); +}); diff --git a/workers/release-validator/tests/validate-releases.test.ts b/workers/release-validator/tests/validate-releases.test.ts index 2d2d980e..6933b064 100644 --- a/workers/release-validator/tests/validate-releases.test.ts +++ b/workers/release-validator/tests/validate-releases.test.ts @@ -351,7 +351,7 @@ describe('validateReleases', () => { expect((await events.findOne({ groupHash: 'error-active-regression' })).resolvedInRelease).toBe('b'); }); - test('should continue validating other projects when one project fails', async () => { + test('should report failure after continuing to validate the remaining projects', async () => { const failedProjectId = 'failed-release-validator-project'; await releases.insertMany([ @@ -391,7 +391,7 @@ describe('validateReleases', () => { await events.insertOne(createEvent('successful-project-event', 'a')); try { - await expect(validateReleases(db, NOW)).resolves.toBeUndefined(); + await expect(validateReleases(db, NOW)).rejects.toThrow(failedProjectId); expect((await releases.findOne({ projectId: failedProjectId, @@ -409,6 +409,29 @@ describe('validateReleases', () => { } }); + test('should default to a 30-day retention period when MAX_DAYS_NUMBER is not set', async () => { + const originalMaxDaysNumber = process.env.MAX_DAYS_NUMBER; + + delete process.env.MAX_DAYS_NUMBER; + + try { + await releases.insertMany([ + createRelease('outside-default-window', 31 * 24), + createRelease('a', 72, true), + createRelease('b', 48), + ]); + await events.insertOne(createEvent('default-retention-event', 'a')); + + await validateReleases(db, NOW); + + expect((await releases.findOne({ release: 'outside-default-window' })).fixChecked).toBe(false); + expect((await releases.findOne({ release: 'b' })).fixChecked).toBe(true); + expect((await events.findOne({ groupHash: 'default-retention-event' })).resolvedInRelease).toBe('b'); + } finally { + process.env.MAX_DAYS_NUMBER = originalMaxDaysNumber; + } + }); + test('should not select a release older than the configured retention period as a candidate', async () => { const originalMaxDaysNumber = process.env.MAX_DAYS_NUMBER;