From 06ebebe7787890aa2556cc23eb69b4b740085204 Mon Sep 17 00:00:00 2001 From: Justin Gasper Date: Fri, 6 Nov 2026 13:34:20 +1100 Subject: [PATCH 1/3] Use deployment scripts v2.0.10 with per-image scan budgets --- .circleci/config.yml | 6 +++--- deploy/MIGRATION.md | 4 ++-- deploy/release.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 75c0c49..3ded2dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,14 +20,14 @@ commands: checkout-suite: steps: - run: - name: Verify deployment suite v2.0.5 + name: Verify deployment suite v2.0.10 command: | set -euo pipefail git init --quiet /tmp/tc-deploy-scripts git -C /tmp/tc-deploy-scripts remote add origin https://github.com/topcoder-platform/tc-deploy-scripts.git - git -C /tmp/tc-deploy-scripts fetch --quiet --depth 1 origin refs/tags/v2.0.5 + git -C /tmp/tc-deploy-scripts fetch --quiet --depth 1 origin refs/tags/v2.0.10 git -C /tmp/tc-deploy-scripts checkout --quiet --detach FETCH_HEAD - test "$(git -C /tmp/tc-deploy-scripts rev-parse HEAD)" = bff574ab5c0b46773e25d2af1eff76675fba2ba5 + test "$(git -C /tmp/tc-deploy-scripts rev-parse HEAD)" = e81a6772a33819c1b57c88a95d04494e59c559b8 jobs: build: executor: reviewed diff --git a/deploy/MIGRATION.md b/deploy/MIGRATION.md index 9918997..c2ca284 100644 --- a/deploy/MIGRATION.md +++ b/deploy/MIGRATION.md @@ -1,6 +1,6 @@ # OIDC deployment migration -This change replaces the Auth0 credential broker and SSM-to-shell deployment path with [tc-deploy-scripts v2.0.5](https://github.com/topcoder-platform/tc-deploy-scripts/releases/tag/v2.0.5), fetched by tag and verified against commit `bff574ab5c0b46773e25d2af1eff76675fba2ba5`. Build and deployment run in separate jobs. The deployer cannot read Parameter Store, Secrets Manager or decrypt KMS data, including through another assumed role. ECS continues resolving the existing runtime secret references using the execution role. +This change replaces the Auth0 credential broker and SSM-to-shell deployment path with [tc-deploy-scripts v2.0.10](https://github.com/topcoder-platform/tc-deploy-scripts/releases/tag/v2.0.10), fetched by tag and verified against commit `e81a6772a33819c1b57c88a95d04494e59c559b8`. Build and deployment run in separate jobs. The deployer cannot read Parameter Store, Secrets Manager or decrypt KMS data, including through another assumed role. ECS continues resolving the existing runtime secret references using the execution role. ## Production target @@ -39,6 +39,6 @@ IAM templates and detailed AWS baselines are maintained in the private [v6-servi ## Dev branch refresh -This PR targets `develop` and preserves its current application/dependency fixes. The shared suite is v2.0.5 (82 offline tests); service-specific adapters and their tests remain part of review. Keep this PR in draft until its documented infrastructure and integration prerequisites are met. Updating these files does not provision AWS or CircleCI resources. +This PR targets `develop` and preserves its current application/dependency fixes. The shared suite is v2.0.10 (82 offline tests); service-specific adapters and their tests remain part of review. Keep this PR in draft until its documented infrastructure and integration prerequisites are met. Updating these files does not provision AWS or CircleCI resources. Source checkout uses the public HTTPS repository and verifies the exact CircleCI commit, avoiding dependency on failing SSH checkout keys. No GitHub token is added to build or deployment contexts. diff --git a/deploy/release.py b/deploy/release.py index edd0444..10ba9d0 100644 --- a/deploy/release.py +++ b/deploy/release.py @@ -133,9 +133,9 @@ def release(aws, m, publisher, release_tag): response = aws.call('ecs', 'describe-task-definition', taskDefinition=previous['taskDefinition'], include=['TAGS']) task, tags = response['taskDefinition'], response.get('tags', []) migration_request(m, task, tags, 'preflight@sha256:' + '0' * 64) - deadline = time.monotonic() + 900 - runtime = publisher.publish('forms-api-v6', 'forms-api-v6:candidate', release_tag, deadline) - migration = publisher.publish('forms-api-v6', 'forms-api-v6:migrate-candidate', release_tag + '-migrate', deadline) + # Each image gets its own scan budget after upload and digest verification. + runtime = publisher.publish('forms-api-v6', 'forms-api-v6:candidate', release_tag) + migration = publisher.publish('forms-api-v6', 'forms-api-v6:migrate-candidate', release_tag + '-migrate') d.require(initial_stable(read_service(aws, m), previous['taskDefinition']), 'Service changed during image publication') migration_task = migrate(aws, m, previous, task, tags, migration) definition = promote(aws, m, previous, task, tags, runtime) From 6145725f2f0d64ade4a5ba2c793bf79b8efe4c6d Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Fri, 6 Nov 2026 07:34:28 +0200 Subject: [PATCH 2/3] PM-6720 - store the sendgrid message id --- docs/api.md | 7 ++ docs/data-model.md | 6 ++ .../migration.sql | 5 ++ prisma/schema.prisma | 22 +++-- src/forms/forms.service.ts | 49 ++++++++++- test/forms.integration.spec.ts | 86 +++++++++++++++++++ 6 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 prisma/migrations/20261104010000_processor_delivery_sendgrid_message/migration.sql diff --git a/docs/api.md b/docs/api.md index 6b3ec1e..ddc6699 100644 --- a/docs/api.md +++ b/docs/api.md @@ -154,6 +154,13 @@ reports `classification: "UNKNOWN"` with null details. Both accept an optional `UNKNOWN` also matches unclassified submissions. The filter is ignored for forms without classification. See forms-processor-v6 ADR 0001. +**SendGrid message IDs.** For a form with a `sendgrid-email` processor flow (Let's +talk, SpaceTech), both routes also append `sendgrid_message_ids` (label "SendGrid +logs"), last: the SendGrid message IDs email-service-v6 reported for the submission's +deliveries, oldest send first, or `[]` before any email was sent. The CSV writes the +list as a JSON array. The reports portal links each ID to +`https://app.sendgrid.com/email_logs//details`. + Dates are optional **inclusive UTC calendar dates** in `YYYY-MM-DD`; an end date includes timestamps through 23:59:59.999. Invalid calendar dates and reversed ranges return 400. Existing version-specific JSON/CSV endpoints accept these same diff --git a/docs/data-model.md b/docs/data-model.md index 083dd74..a35a096 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -141,6 +141,12 @@ the recipients and template from `SPACE_TECH_CHALLENGE_RECIPIENTS` and `SPACE_TECH_CHALLENGE_TEMPLATE_ID`. Enable it in SQL after both are set and the processor is redeployed. +Migration `20261104010000_processor_delivery_sendgrid_message` adds +`ProcessorDelivery.sendgridMessageId` and `emailSentAt`. forms-processor-v6 fills them +from email-service-v6's `external.action.email.sent` reports, matched by the email's +`:` correlation ID; the latest non-null ID wins. They are deleted +with the delivery, and the submissions report lists them as `sendgrid_message_ids`. + Processor events contain answers and must be included in personal-data retention and erasure procedures separately from `Submission`. Deleting a ProcessorEvent cascades its delivery records and removes deduplication protection, so retain it diff --git a/prisma/migrations/20261104010000_processor_delivery_sendgrid_message/migration.sql b/prisma/migrations/20261104010000_processor_delivery_sendgrid_message/migration.sql new file mode 100644 index 0000000..31c24a6 --- /dev/null +++ b/prisma/migrations/20261104010000_processor_delivery_sendgrid_message/migration.sql @@ -0,0 +1,5 @@ +-- SendGrid message IDs reported by email-service-v6 (external.action.email.sent) for +-- processor email deliveries; forms-processor-v6 matches them by :. +ALTER TABLE forms."ProcessorDelivery" + ADD COLUMN "sendgridMessageId" VARCHAR(255), + ADD COLUMN "emailSentAt" TIMESTAMPTZ(3); diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 0e3a16d..1e2ab20 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -247,15 +247,19 @@ model ProcessorEvent { /// Per-flow acceptance receipt and retry schedule; Bus API acceptance is not recipient delivery confirmation. model ProcessorDelivery { - submissionId String @db.Uuid - flowId String @db.VarChar(100) - attempts Int @default(0) - nextAttemptAt DateTime @default(now()) @db.Timestamptz(3) - lastError String? @db.VarChar(100) - deliveredAt DateTime? @db.Timestamptz(3) - createdAt DateTime @default(now()) @db.Timestamptz(3) - event ProcessorEvent @relation(fields: [submissionId], references: [submissionId], onDelete: Cascade, onUpdate: NoAction) - flow ProcessorFlow @relation(fields: [flowId], references: [id], onDelete: Restrict, onUpdate: NoAction) + submissionId String @db.Uuid + flowId String @db.VarChar(100) + attempts Int @default(0) + nextAttemptAt DateTime @default(now()) @db.Timestamptz(3) + lastError String? @db.VarChar(100) + deliveredAt DateTime? @db.Timestamptz(3) + createdAt DateTime @default(now()) @db.Timestamptz(3) + /// SendGrid X-Message-Id reported by email-service-v6 (external.action.email.sent); the latest non-null report wins. + sendgridMessageId String? @db.VarChar(255) + /// When SendGrid accepted the delivery's email, per the latest report. + emailSentAt DateTime? @db.Timestamptz(3) + event ProcessorEvent @relation(fields: [submissionId], references: [submissionId], onDelete: Cascade, onUpdate: NoAction) + flow ProcessorFlow @relation(fields: [flowId], references: [id], onDelete: Restrict, onUpdate: NoAction) @@id([submissionId, flowId]) @@index([deliveredAt, nextAttemptAt]) diff --git a/src/forms/forms.service.ts b/src/forms/forms.service.ts index de847d4..255613a 100644 --- a/src/forms/forms.service.ts +++ b/src/forms/forms.service.ts @@ -50,6 +50,11 @@ type Definition = Prisma.FormVersionGetPayload<{ const CLASSIFY_ACTION = 'ai-classify'; const UNKNOWN_LABEL = 'UNKNOWN'; +/** Processor action that emails through email-service-v6, which reports SendGrid message IDs back. */ +const EMAIL_ACTION = 'sendgrid-email'; +/** Report column listing each submission's SendGrid message IDs, after any classification columns. */ +const SENDGRID_MESSAGES_COLUMN = 'sendgrid_message_ids'; + /** Report columns appended for forms with AI classification, after the answer columns. */ const CLASSIFICATION_COLUMNS = [ 'classification', @@ -210,6 +215,11 @@ export class FormsService { (await this.db.processorFlow.count({ where: { formKey: key, action: CLASSIFY_ACTION }, })) > 0; + // Forms emailed by the processor report the SendGrid message IDs of their deliveries. + const emailed = + (await this.db.processorFlow.count({ + where: { formKey: key, action: EMAIL_ACTION }, + })) > 0; const where: Prisma.SubmissionWhereInput = { versionId: { in: form.versions.map((version) => version.id) }, createdAt: { ...dates, ...(snapshot ? { lte: snapshot } : {}) }, @@ -261,8 +271,13 @@ export class FormsService { 'form_version', ...fields.map((field) => field.key), ...(classified ? CLASSIFICATION_COLUMNS : []), + ...(emailed ? [SENDGRID_MESSAGES_COLUMN] : []), ]; - const data = submissions.slice(0, page.limit).map((submission) => { + const pageSubmissions = submissions.slice(0, page.limit); + const messageIds = emailed + ? await this.sendgridMessageIds(pageSubmissions.map(({ id }) => id)) + : new Map(); + const data = pageSubmissions.map((submission) => { const row: Record = Object.fromEntries( columns.map((column) => [column, null]), ); @@ -289,6 +304,8 @@ export class FormsService { } if (classified) Object.assign(row, classificationCells(submission.classification)); + if (emailed) + row[SENDGRID_MESSAGES_COLUMN] = messageIds.get(submission.id) ?? []; return row; }); return { @@ -297,6 +314,7 @@ export class FormsService { labels: { ...Object.fromEntries(fields.map((field) => [field.key, field.label])), ...(classified ? CLASSIFICATION_LABELS : {}), + ...(emailed ? { [SENDGRID_MESSAGES_COLUMN]: 'SendGrid logs' } : {}), }, data, total, @@ -305,6 +323,35 @@ export class FormsService { }; } + /** + * Reads the SendGrid message IDs email-service-v6 reported for these submissions' processor deliveries. + * Submission has no relation to the processor inbox, so deliveries are matched by submission ID. + * @param submissionIds One report page of submissions. + * @returns IDs per submission in send order; submissions without a reported send are absent. + * @throws Database errors on failure. + */ + private async sendgridMessageIds( + submissionIds: string[], + ): Promise> { + const deliveries = submissionIds.length + ? await this.db.processorDelivery.findMany({ + where: { + submissionId: { in: submissionIds }, + sendgridMessageId: { not: null }, + }, + orderBy: [{ emailSentAt: 'asc' }, { createdAt: 'asc' }], + select: { submissionId: true, sendgridMessageId: true }, + }) + : []; + const ids = new Map(); + for (const { submissionId, sendgridMessageId } of deliveries) + ids.set(submissionId, [ + ...(ids.get(submissionId) ?? []), + sendgridMessageId!, + ]); + return ids; + } + /** * Prepares a complete CSV export, validating the request before HTTP headers are sent. * @param key Form key. @param dates Inclusive date filters, without table pagination. diff --git a/test/forms.integration.spec.ts b/test/forms.integration.spec.ts index 59e55de..74670f9 100644 --- a/test/forms.integration.spec.ts +++ b/test/forms.integration.spec.ts @@ -436,6 +436,92 @@ describe('forms API with real PostgreSQL', () => { }); }); + it('reports SendGrid message IDs per submission only for forms with an email flow', async () => { + const key = await create({ + ...definition(), + fields: definition().fields.map((field) => ({ + ...field, + required: false, + })), + }); + const version = await db.formVersion.findFirstOrThrow({ + where: { form: { key } }, + }); + const reportPath = `/v6/forms/${key}/submissions`; + const report = async () => + ( + await request(app.getHttpServer()) + .get(reportPath) + .auth(reporter, { type: 'bearer' }) + .expect(200) + ).body as { + columns: string[]; + labels: Record; + data: Record[]; + }; + // No email flow yet: the report is unchanged. + expect((await report()).columns).not.toContain('sendgrid_message_ids'); + + await db.processorFlow.createMany({ + data: [ + { id: `${key}-internal`, formKey: key, action: 'sendgrid-email' }, + { id: `${key}-reply`, formKey: key, action: 'sendgrid-email' }, + ], + }); + const [emailed, pending] = [randomUUID(), randomUUID()]; + await db.submission.createMany({ + data: [emailed, pending].map((id, index) => ({ + id, + versionId: version.id, + idempotencyKey: randomUUID(), + requestHash: 'a'.repeat(64), + createdAt: new Date(Date.UTC(2020, 9, 1 + index)), + })), + }); + await db.processorEvent.createMany({ + data: [emailed, pending].map((submissionId) => ({ + submissionId, + formKey: key, + payload: {}, + })), + }); + await db.processorDelivery.createMany({ + data: [ + { + submissionId: emailed, + flowId: `${key}-reply`, + sendgridMessageId: 'sg-reply', + emailSentAt: new Date(Date.UTC(2020, 9, 1, 0, 2)), + }, + { + submissionId: emailed, + flowId: `${key}-internal`, + sendgridMessageId: 'sg-internal', + emailSentAt: new Date(Date.UTC(2020, 9, 1, 0, 1)), + }, + // Queued, or sent without a reported ID: no link. + { submissionId: pending, flowId: `${key}-internal` }, + ], + }); + + const body = await report(); + expect(body.columns.at(-1)).toBe('sendgrid_message_ids'); + expect(body.labels.sendgrid_message_ids).toBe('SendGrid logs'); + const byId = Object.fromEntries( + body.data.map((row) => [row.submission_id, row.sendgrid_message_ids]), + ); + expect(byId).toEqual({ + [emailed]: ['sg-internal', 'sg-reply'], + [pending]: [], + }); + const csv = await request(app.getHttpServer()) + .get(`${reportPath}/export`) + .auth(reporter, { type: 'bearer' }) + .expect(200); + expect(csv.text.split('\r\n')[0]).toContain('"sendgrid_message_ids"'); + expect(csv.text).toContain('"[""sg-internal"",""sg-reply""]"'); + }); + it('reports AI classification columns and filters by label only for forms with a classify flow', async () => { const key = await create({ ...definition(), From 70e7d15be652048bd3e8a32feb4db5194083ad10 Mon Sep 17 00:00:00 2001 From: Vasilica Olariu Date: Fri, 6 Nov 2026 08:11:15 +0200 Subject: [PATCH 3/3] Fix order for emails --- src/forms/forms.service.ts | 18 ++++++++++++------ test/forms.integration.spec.ts | 20 ++++++++++++++++++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/forms/forms.service.ts b/src/forms/forms.service.ts index 255613a..bbb2a31 100644 --- a/src/forms/forms.service.ts +++ b/src/forms/forms.service.ts @@ -338,17 +338,23 @@ export class FormsService { where: { submissionId: { in: submissionIds }, sendgridMessageId: { not: null }, + // Only email deliveries report SendGrid sends. + flow: { action: EMAIL_ACTION }, }, - orderBy: [{ emailSentAt: 'asc' }, { createdAt: 'asc' }], + // PostgreSQL sorts NULLs first ascending; sends without a reported time go last. + orderBy: [ + { emailSentAt: { sort: 'asc', nulls: 'last' } }, + { createdAt: 'asc' }, + ], select: { submissionId: true, sendgridMessageId: true }, }) : []; const ids = new Map(); - for (const { submissionId, sendgridMessageId } of deliveries) - ids.set(submissionId, [ - ...(ids.get(submissionId) ?? []), - sendgridMessageId!, - ]); + for (const { submissionId, sendgridMessageId } of deliveries) { + const list = ids.get(submissionId); + if (list) list.push(sendgridMessageId!); + else ids.set(submissionId, [sendgridMessageId!]); + } return ids; } diff --git a/test/forms.integration.spec.ts b/test/forms.integration.spec.ts index 74670f9..93f5ca5 100644 --- a/test/forms.integration.spec.ts +++ b/test/forms.integration.spec.ts @@ -466,6 +466,8 @@ describe('forms API with real PostgreSQL', () => { data: [ { id: `${key}-internal`, formKey: key, action: 'sendgrid-email' }, { id: `${key}-reply`, formKey: key, action: 'sendgrid-email' }, + { id: `${key}-copy`, formKey: key, action: 'sendgrid-email' }, + { id: `${key}-classify`, formKey: key, action: 'ai-classify' }, ], }); const [emailed, pending] = [randomUUID(), randomUUID()]; @@ -499,6 +501,18 @@ describe('forms API with real PostgreSQL', () => { sendgridMessageId: 'sg-internal', emailSentAt: new Date(Date.UTC(2020, 9, 1, 0, 1)), }, + // An ID without a reported send time sorts after the dated sends. + { + submissionId: emailed, + flowId: `${key}-copy`, + sendgridMessageId: 'sg-undated', + }, + // Only email deliveries count, whatever another flow's row holds. + { + submissionId: emailed, + flowId: `${key}-classify`, + sendgridMessageId: 'sg-not-email', + }, // Queued, or sent without a reported ID: no link. { submissionId: pending, flowId: `${key}-internal` }, ], @@ -511,7 +525,7 @@ describe('forms API with real PostgreSQL', () => { body.data.map((row) => [row.submission_id, row.sendgrid_message_ids]), ); expect(byId).toEqual({ - [emailed]: ['sg-internal', 'sg-reply'], + [emailed]: ['sg-internal', 'sg-reply', 'sg-undated'], [pending]: [], }); const csv = await request(app.getHttpServer()) @@ -519,7 +533,9 @@ describe('forms API with real PostgreSQL', () => { .auth(reporter, { type: 'bearer' }) .expect(200); expect(csv.text.split('\r\n')[0]).toContain('"sendgrid_message_ids"'); - expect(csv.text).toContain('"[""sg-internal"",""sg-reply""]"'); + expect(csv.text).toContain( + '"[""sg-internal"",""sg-reply"",""sg-undated""]"', + ); }); it('reports AI classification columns and filters by label only for forms with a classify flow', async () => {