From 34e95d483c55d72d7d99afd577ee5bcd2eeee009 Mon Sep 17 00:00:00 2001 From: alisawavezen12 Date: Wed, 28 Oct 2026 12:18:33 +0300 Subject: [PATCH 1/3] chore --- docker-compose.dev.yml | 4 +- docker-compose.prod.yml | 6 +- package.json | 2 +- workers/release-validator/README.md | 42 +++++- workers/release-validator/package.json | 5 +- workers/release-validator/src/index.ts | 56 +++----- .../src/validate-releases.ts | 10 ++ workers/release-validator/tests/index.test.ts | 134 ++++++++++++++++++ .../tests/validate-releases.test.ts | 4 +- 9 files changed, 210 insertions(+), 53 deletions(-) create mode 100644 workers/release-validator/tests/index.test.ts diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index cc131d23..5826d625 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -86,9 +86,7 @@ services: context: . env_file: - .env - environment: - - SIMULTANEOUS_TASKS=1 - restart: unless-stopped + restart: 'no' entrypoint: yarn run-release-validator volumes: - ./:/usr/src/app diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a702aefc..16e81ec7 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -60,10 +60,8 @@ services: network_mode: host env_file: - .env - environment: - - SIMULTANEOUS_TASKS=1 - restart: unless-stopped - entrypoint: /usr/local/bin/node runner.js hawk-worker-release-validator + restart: 'no' + entrypoint: /usr/local/bin/node workers/release-validator/src/index.js # # Notification workers: notifier, email, telegram diff --git a/package.json b/package.json index b5bb807e..9780eab7 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "run-paymaster": "yarn worker hawk-worker-paymaster", "run-notifier": "yarn worker hawk-worker-notifier", "run-release": "yarn worker hawk-worker-release", - "run-release-validator": "yarn worker hawk-worker-release-validator", + "run-release-validator": "ts-node -T --prefer-ts-exts ./workers/release-validator/src/index.ts", "run-email": "yarn worker hawk-worker-email", "run-telegram": "yarn worker hawk-worker-telegram", "run-limiter": "yarn worker hawk-worker-limiter", diff --git a/workers/release-validator/README.md b/workers/release-validator/README.md index a64e9665..bb7a7a93 100644 --- a/workers/release-validator/README.md +++ b/workers/release-validator/README.md @@ -1,15 +1,45 @@ -# Release Validator Worker +# Release Validator Job -Checks releases after a 24-hour observation period and marks original events that no longer occur as resolved. +Standalone one-shot MongoDB job: connects, validates eligible releases, closes the connection and exits. It does not use RabbitMQ, `runner.ts`/`runner.js`, or a permanent metrics server. `REGISTRY_URL` and `SIMULTANEOUS_TASKS` are not needed; scheduling belongs to infrastructure, not the process. -The worker processes releases from oldest to newest, stores the first release without an event in `resolvedInRelease`, and marks successfully processed releases with `fixChecked: true`. +Releases are processed oldest first after a 24-hour observation period. The first release without an event is stored in `resolvedInRelease`; successfully processed releases receive `fixChecked: true`. Records without the fields required for matching are ignored. Candidates must be older than 24 hours and within the `MAX_DAYS_NUMBER` retention window. All available project releases are used as history; if the original release was archived, the original event timestamp is the chronological boundary. -The worker only uses the fields required for matching releases and event groups. Records without these fields are ignored. Candidate releases must be older than 24 hours and no older than the `MAX_DAYS_NUMBER` retention period, while all available project releases are still used to compare event history. If an event's original release was archived, the original event timestamp is used as its chronological boundary. +## Configuration and prerequisite -Queue: `cron-tasks/release-validator` +Required environment variables: -Run locally: +- `MONGO_EVENTS_DATABASE_URI`: events MongoDB connection URI, including the database name. +- `MAX_DAYS_NUMBER`: finite positive retention period in days; use the same value as archiver. + +Optional: `MONGO_RECONNECT_TRIES` (default `60`) and `MONGO_RECONNECT_INTERVAL` (milliseconds, default `3000`) control initial connection attempts; both accept positive integers. `MONGO_APP_NAME` labels the connection. `LOG_LEVEL` defaults to `info`. + +**Before running**, verify that every project collection `repetitions:` has the compound index `{ groupHash: 1, release: 1 }` named **`groupHashAndRelease`**. Aggregation uses this exact index name as a hint; a missing index causes validation to fail. Provision it through the existing index/migration workflow before enabling the job. + +## Run + +Locally, from the workers repository root with the required environment exported: ```sh yarn run-release-validator ``` + +The root script runs `ts-node -T --prefer-ts-exts ./workers/release-validator/src/index.ts` directly, without the worker runner. TypeScript files take precedence over JavaScript left by a previous local build. + +In the production image, from `/app` (the Dockerfile compiles JavaScript in place): + +```sh +/usr/local/bin/node workers/release-validator/src/index.js +``` + +Both Compose services use `restart: 'no'`. For an explicit one-off dev run with Compose environment files: + +```sh +docker compose -f docker-compose.dev.yml run --rm hawk-worker-release-validator +``` + +Exit codes: + +- `0`: successful run, including no eligible unchecked releases older than 24 hours. +- `1`: configuration, connection or validation errors. Project failures are collected after attempting the remaining projects, then the run fails. Successfully checked releases stay marked; failed projects' unchecked releases can be retried on the next run while within retention. + +For Kubernetes migration and cutover, see [KUBERNETES.md](./KUBERNETES.md). diff --git a/workers/release-validator/package.json b/workers/release-validator/package.json index 3cd8c88f..cc638361 100644 --- a/workers/release-validator/package.json +++ b/workers/release-validator/package.json @@ -1,10 +1,9 @@ { "name": "hawk-worker-release-validator", "version": "0.0.1", - "description": "Detects events fixed by a release", + "description": "One-shot job that detects events fixed by a release", "main": "src/index.ts", "author": "CodeX", "license": "UNLICENSED", - "private": true, - "workerType": "cron-tasks/release-validator" + "private": true } diff --git a/workers/release-validator/src/index.ts b/workers/release-validator/src/index.ts index ac268f6a..0e38ebd7 100644 --- a/workers/release-validator/src/index.ts +++ b/workers/release-validator/src/index.ts @@ -1,46 +1,34 @@ +import 'dotenv/config'; import { DatabaseController } from '../../../lib/db/controller'; -import { Worker } from '../../../lib/worker'; -import * as pkg from '../package.json'; +import createLogger from '../../../lib/logger'; import { validateReleases } from './validate-releases'; +const logger = createLogger(); + /** - * Worker that detects events fixed by a release. + * Run one release validation pass and close MongoDB before returning. + * + * @returns {Promise} completion of the validation and connection cleanup */ -export default class ReleaseValidatorWorker extends Worker { - /** - * Worker type. - */ - public readonly type: string = pkg.workerType; +export async function runReleaseValidation(): Promise { + logger.info('Release validation started'); - /** - * Events database controller. - */ - private eventsDb = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI); + const eventsDb = new DatabaseController(process.env.MONGO_EVENTS_DATABASE_URI); - /** - * Connect to the events database and start consuming tasks. - */ - public async start(): Promise { - await this.eventsDb.connect(); - await super.start(); - } + try { + const db = await eventsDb.connect(); - /** - * Stop consuming tasks and close the database connection. - */ - public async finish(): Promise { - await super.finish(); - await this.eventsDb.close(); + await validateReleases(db); + } finally { + await eventsDb.close(); } - /** - * Handle a scheduled release validation task. - */ - public async handle(): Promise { - this.logger.info('Release validation started'); - - await validateReleases(this.eventsDb.getConnection()); + logger.info('Release validation finished'); +} - this.logger.info('Release validation finished'); - } +if (require.main === module) { + runReleaseValidation().catch(error => { + logger.error('Release validation failed', error); + process.exitCode = 1; + }); } diff --git a/workers/release-validator/src/validate-releases.ts b/workers/release-validator/src/validate-releases.ts index f8c4910a..8f5ac044 100644 --- a/workers/release-validator/src/validate-releases.ts +++ b/workers/release-validator/src/validate-releases.ts @@ -378,6 +378,7 @@ async function validateProject(db: Db, projectId: string, releasesToCheck: Relea * * @param db - events database connection * @param now - current time + * @throws when any project fails, after attempting the remaining projects */ export async function validateReleases(db: Db, now = new Date()): Promise { /** @@ -386,12 +387,21 @@ export async function validateReleases(db: Db, now = new Date()): Promise */ const releasesToCheck = await findReleasesToCheck(db, now); const releasesByProject = groupReleasesByProject(releasesToCheck); + const failedProjects: string[] = []; + + logger.info(`Release validation: ${releasesToCheck.length} unchecked releases in ${releasesByProject.size} projects`); for (const [projectId, projectReleases] of releasesByProject) { try { await validateProject(db, projectId, projectReleases); + logger.info(`Checked ${projectReleases.length} releases for project ${projectId}`); } catch (error) { + failedProjects.push(projectId); logger.error(`Failed to validate releases for project ${projectId}`, error); } } + + if (failedProjects.length > 0) { + throw new Error(`Release validation failed for ${failedProjects.length} projects: ${failedProjects.join(', ')}`); + } } diff --git a/workers/release-validator/tests/index.test.ts b/workers/release-validator/tests/index.test.ts new file mode 100644 index 00000000..9dbb0a78 --- /dev/null +++ b/workers/release-validator/tests/index.test.ts @@ -0,0 +1,134 @@ +import { spawnSync, SpawnSyncReturns } from 'child_process'; +import { resolve } from 'path'; +import type { Db } from 'mongodb'; +import { connect as connectRegistry } from 'amqplib'; +import { DatabaseController } from '../../../lib/db/controller'; +import { runReleaseValidation } from '../src'; +import { validateReleases } from '../src/validate-releases'; + +jest.mock('../../../lib/db/controller'); +jest.mock('../src/validate-releases'); + +const MockDatabaseController = DatabaseController as jest.MockedClass; +const mockValidateReleases = validateReleases as jest.MockedFunction; + +/** + * Execute the real CLI against the local MongoDB started by the Jest configuration. + * + * @param maxDays - configured release retention period + * @param uri - isolated test database, or an empty value to test configuration failure + * @returns {SpawnSyncReturns} process output and exit status, bounded by a timeout + */ +function runCli(maxDays: string, uri = `mongodb://127.0.0.1:55010/release-validator-cli-${process.pid}`): SpawnSyncReturns { + return spawnSync(process.execPath, [ + '-r', require.resolve('ts-node/register/transpile-only'), + resolve(__dirname, '../src/index.ts'), + ], { + cwd: resolve(__dirname, '../../..'), + env: { + ...process.env, + MONGO_EVENTS_DATABASE_URI: uri, + MAX_DAYS_NUMBER: maxDays, + MONGO_RECONNECT_TRIES: '1', + TS_NODE_PREFER_TS_EXTS: 'true', + REGISTRY_URL: 'amqp://127.0.0.1:1', + LOG_LEVEL: 'info', + }, + encoding: 'utf8', + timeout: 10000, + }); +} + +describe('One-shot release validation', () => { + const db = {} as Db; + const connect = jest.fn(); + const close = jest.fn(); + const originalUri = process.env.MONGO_EVENTS_DATABASE_URI; + + beforeEach(() => { + jest.clearAllMocks(); + process.env.MONGO_EVENTS_DATABASE_URI = 'mongodb://test/events'; + connect.mockResolvedValue(db); + close.mockResolvedValue(undefined); + mockValidateReleases.mockResolvedValue(undefined); + MockDatabaseController.mockImplementation(() => ({ + connect, + close, + } as unknown as DatabaseController)); + }); + + afterAll(() => { + if (originalUri === undefined) { + delete process.env.MONGO_EVENTS_DATABASE_URI; + } else { + process.env.MONGO_EVENTS_DATABASE_URI = originalUri; + } + }); + + test('should immediately validate once and close MongoDB without connecting to RabbitMQ', async () => { + await runReleaseValidation(); + + expect(MockDatabaseController).toHaveBeenCalledWith('mongodb://test/events'); + expect(connect).toHaveBeenCalledTimes(1); + expect(mockValidateReleases).toHaveBeenCalledTimes(1); + expect(mockValidateReleases).toHaveBeenCalledWith(db); + expect(close).toHaveBeenCalledTimes(1); + expect(close.mock.invocationCallOrder[0]).toBeGreaterThan(mockValidateReleases.mock.invocationCallOrder[0]); + expect(connectRegistry).not.toHaveBeenCalled(); + }); + + test('should close MongoDB and propagate validation failures', async () => { + const error = new Error('Project validation failed'); + + mockValidateReleases.mockRejectedValueOnce(error); + + await expect(runReleaseValidation()).rejects.toBe(error); + expect(close).toHaveBeenCalledTimes(1); + }); + + test('should clean up after a connection failure without attempting validation', async () => { + const error = new Error('MongoDB unavailable'); + + connect.mockRejectedValueOnce(error); + + await expect(runReleaseValidation()).rejects.toBe(error); + expect(mockValidateReleases).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledTimes(1); + }); + + test('should report a cleanup failure rather than completing successfully', async () => { + close.mockRejectedValueOnce(new Error('Could not close MongoDB')); + + await expect(runReleaseValidation()).rejects.toThrow('Could not close MongoDB'); + }); +}); + +describe('Release validator CLI', () => { + test('should exit successfully when there are no eligible releases, without waiting for a queue task', () => { + const result = runCli('30'); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(result.stdout).toContain('Release validation started'); + expect(result.stdout).toContain('0 unchecked releases in 0 projects'); + expect(result.stdout).toContain('Release validation finished'); + }); + + test('should exit with code 1 for invalid retention configuration', () => { + const result = runCli('0'); + const output = result.stdout + result.stderr; + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(output).toContain('MAX_DAYS_NUMBER must be a positive number'); + expect(output).not.toContain('Release validation finished'); + }); + + test('should exit with code 1 when the database URI is missing', () => { + const result = runCli('30', ''); + + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout + result.stderr).toContain('Connection URI is not specified'); + }); +}); diff --git a/workers/release-validator/tests/validate-releases.test.ts b/workers/release-validator/tests/validate-releases.test.ts index 2d2d980e..b94c6da7 100644 --- a/workers/release-validator/tests/validate-releases.test.ts +++ b/workers/release-validator/tests/validate-releases.test.ts @@ -351,7 +351,7 @@ describe('validateReleases', () => { expect((await events.findOne({ groupHash: 'error-active-regression' })).resolvedInRelease).toBe('b'); }); - test('should continue validating other projects when one project fails', async () => { + test('should report failure after continuing to validate the remaining projects', async () => { const failedProjectId = 'failed-release-validator-project'; await releases.insertMany([ @@ -391,7 +391,7 @@ describe('validateReleases', () => { await events.insertOne(createEvent('successful-project-event', 'a')); try { - await expect(validateReleases(db, NOW)).resolves.toBeUndefined(); + await expect(validateReleases(db, NOW)).rejects.toThrow(failedProjectId); expect((await releases.findOne({ projectId: failedProjectId, From 76b508975473f1d11d41945bd5cec097da183bab Mon Sep 17 00:00:00 2001 From: alisawavezen12 Date: Wed, 28 Oct 2026 12:22:22 +0300 Subject: [PATCH 2/3] chore --- workers/release-validator/README.md | 38 +++++------------------------ 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/workers/release-validator/README.md b/workers/release-validator/README.md index bb7a7a93..0e568786 100644 --- a/workers/release-validator/README.md +++ b/workers/release-validator/README.md @@ -1,45 +1,19 @@ # Release Validator Job -Standalone one-shot MongoDB job: connects, validates eligible releases, closes the connection and exits. It does not use RabbitMQ, `runner.ts`/`runner.js`, or a permanent metrics server. `REGISTRY_URL` and `SIMULTANEOUS_TASKS` are not needed; scheduling belongs to infrastructure, not the process. +One-shot MongoDB job that checks releases after a 24-hour observation period and exits. It does not use RabbitMQ. -Releases are processed oldest first after a 24-hour observation period. The first release without an event is stored in `resolvedInRelease`; successfully processed releases receive `fixChecked: true`. Records without the fields required for matching are ignored. Candidates must be older than 24 hours and within the `MAX_DAYS_NUMBER` retention window. All available project releases are used as history; if the original release was archived, the original event timestamp is the chronological boundary. - -## Configuration and prerequisite - -Required environment variables: - -- `MONGO_EVENTS_DATABASE_URI`: events MongoDB connection URI, including the database name. -- `MAX_DAYS_NUMBER`: finite positive retention period in days; use the same value as archiver. - -Optional: `MONGO_RECONNECT_TRIES` (default `60`) and `MONGO_RECONNECT_INTERVAL` (milliseconds, default `3000`) control initial connection attempts; both accept positive integers. `MONGO_APP_NAME` labels the connection. `LOG_LEVEL` defaults to `info`. - -**Before running**, verify that every project collection `repetitions:` has the compound index `{ groupHash: 1, release: 1 }` named **`groupHashAndRelease`**. Aggregation uses this exact index name as a hint; a missing index causes validation to fail. Provision it through the existing index/migration workflow before enabling the job. +Events that no longer occur are marked with `resolvedInRelease`; successfully checked releases receive `fixChecked: true`. The validation window follows the archiver retention period. ## Run -Locally, from the workers repository root with the required environment exported: +From the workers repository root: ```sh yarn run-release-validator ``` -The root script runs `ts-node -T --prefer-ts-exts ./workers/release-validator/src/index.ts` directly, without the worker runner. TypeScript files take precedence over JavaScript left by a previous local build. - -In the production image, from `/app` (the Dockerfile compiles JavaScript in place): - -```sh -/usr/local/bin/node workers/release-validator/src/index.js -``` - -Both Compose services use `restart: 'no'`. For an explicit one-off dev run with Compose environment files: - -```sh -docker compose -f docker-compose.dev.yml run --rm hawk-worker-release-validator -``` - -Exit codes: +Requires the events database connection and retention period configured for the environment. The `repetitions:` collections must have the `{ groupHash: 1, release: 1 }` index named `groupHashAndRelease`. -- `0`: successful run, including no eligible unchecked releases older than 24 hours. -- `1`: configuration, connection or validation errors. Project failures are collected after attempting the remaining projects, then the run fails. Successfully checked releases stay marked; failed projects' unchecked releases can be retried on the next run while within retention. +The process exits with code `0` on success, including when there is nothing to check, or `1` on failure. -For Kubernetes migration and cutover, see [KUBERNETES.md](./KUBERNETES.md). +For Kubernetes deployment, see [KUBERNETES.md](./KUBERNETES.md). From 801c3e90699e7c06f4a49a4931b01bc8802d0407 Mon Sep 17 00:00:00 2001 From: alisawavezen12 Date: Wed, 28 Oct 2026 12:31:51 +0300 Subject: [PATCH 3/3] chore --- workers/release-validator/README.md | 2 +- .../src/validate-releases.ts | 7 +++++- .../tests/validate-releases.test.ts | 23 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/workers/release-validator/README.md b/workers/release-validator/README.md index 0e568786..90f53625 100644 --- a/workers/release-validator/README.md +++ b/workers/release-validator/README.md @@ -12,7 +12,7 @@ From the workers repository root: yarn run-release-validator ``` -Requires the events database connection and retention period configured for the environment. The `repetitions:` collections must have the `{ groupHash: 1, release: 1 }` index named `groupHashAndRelease`. +Requires the events database connection. Retention defaults to 30 days; `MAX_DAYS_NUMBER` overrides it for the environment. The `repetitions:` collections must have the `{ groupHash: 1, release: 1 }` index named `groupHashAndRelease`. The process exits with code `0` on success, including when there is nothing to check, or `1` on failure. diff --git a/workers/release-validator/src/validate-releases.ts b/workers/release-validator/src/validate-releases.ts index 8f5ac044..100b908e 100644 --- a/workers/release-validator/src/validate-releases.ts +++ b/workers/release-validator/src/validate-releases.ts @@ -24,6 +24,11 @@ const RELEASE_OBSERVATION_PERIOD_SECONDS = HOURS_IN_DAY * MINUTES_IN_HOUR * SECO */ const EVENTS_BATCH_SIZE = 50; +/** + * Default retention period when no environment override is provided. + */ +const DEFAULT_MAX_DAYS_NUMBER = 30; + /** * Find unchecked releases older than the observation period. * @@ -32,7 +37,7 @@ const EVENTS_BATCH_SIZE = 50; */ async function findReleasesToCheck(db: Db, now: Date): Promise { const nowSeconds = Math.floor(now.getTime() / MS_IN_SEC); - const releaseMaxAgeDays = Number(process.env.MAX_DAYS_NUMBER); + const releaseMaxAgeDays = Number(process.env.MAX_DAYS_NUMBER ?? DEFAULT_MAX_DAYS_NUMBER); if (!Number.isFinite(releaseMaxAgeDays) || releaseMaxAgeDays <= 0) { throw new Error('MAX_DAYS_NUMBER must be a positive number'); diff --git a/workers/release-validator/tests/validate-releases.test.ts b/workers/release-validator/tests/validate-releases.test.ts index b94c6da7..6933b064 100644 --- a/workers/release-validator/tests/validate-releases.test.ts +++ b/workers/release-validator/tests/validate-releases.test.ts @@ -409,6 +409,29 @@ describe('validateReleases', () => { } }); + test('should default to a 30-day retention period when MAX_DAYS_NUMBER is not set', async () => { + const originalMaxDaysNumber = process.env.MAX_DAYS_NUMBER; + + delete process.env.MAX_DAYS_NUMBER; + + try { + await releases.insertMany([ + createRelease('outside-default-window', 31 * 24), + createRelease('a', 72, true), + createRelease('b', 48), + ]); + await events.insertOne(createEvent('default-retention-event', 'a')); + + await validateReleases(db, NOW); + + expect((await releases.findOne({ release: 'outside-default-window' })).fixChecked).toBe(false); + expect((await releases.findOne({ release: 'b' })).fixChecked).toBe(true); + expect((await events.findOne({ groupHash: 'default-retention-event' })).resolvedInRelease).toBe('b'); + } finally { + process.env.MAX_DAYS_NUMBER = originalMaxDaysNumber; + } + }); + test('should not select a release older than the configured retention period as a candidate', async () => { const originalMaxDaysNumber = process.env.MAX_DAYS_NUMBER;