Skip to content

Commit 5c889c3

Browse files
mchen-sentrymchen-sentry
committed
fix(pipedream): skip gate for single-region groups
When a group has one region there's nothing to filter, and services in that shape may override SENTRY_REGION per cell (uptime-checker-k8s, vector-uc-k8s) — in that case the gate compares the cell name against the group region and self-skips every cell job. Skip wrap_task entirely when std.length(regions) == 1. Multi-region groups (st) keep the gate. Updated single-region-multi-cell fixture to override SENTRY_REGION per cell, exercising the case.
1 parent 8fc25b4 commit 5c889c3

30 files changed

Lines changed: 141 additions & 119 deletions

‎libs/pipedream.libsonnet‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,11 @@ local gocd_tasks = import './gocd-tasks.libsonnet';
3131
local pipeline_name(name, region=null) =
3232
if region != null then 'deploy-' + name + '-' + region else 'deploy-' + name;
3333

34-
// POSIX-shell preamble injected at the top of every script task in a group
35-
// pipeline's region-keyed jobs. The renderer sets PIPEDREAM_GROUP_REGIONS to
36-
// the full list of regions in the group, so by default every region's job
37-
// runs. To target a subset (e.g. one region) override PIPEDREAM_GROUP_REGIONS
38-
// in GoCD's "Trigger with options" — jobs whose SENTRY_REGION isn't in the
39-
// (possibly subsetted) list exit 0 cleanly.
40-
//
41-
// Stays POSIX (single-bracket [, no [[) so it works under dash, which is the
42-
// /bin/sh on most Linux GoCD agents and would silently no-op `[[` expressions.
34+
// POSIX-shell gate injected into multi-region group jobs so an operator can
35+
// target a subset of regions via PIPEDREAM_GROUP_REGIONS in "Trigger with
36+
// options". Single-region groups skip the gate (nothing to filter, and would
37+
// self-skip if the service overrides SENTRY_REGION per cell). POSIX-only
38+
// (`[`, not `[[`) for dash compatibility on Linux GoCD agents.
4339
local deploy_target_gate = |||
4440
if [ -n "${PIPEDREAM_GROUP_REGIONS:-}" ] && [ -n "${SENTRY_REGION:-}" ]; then
4541
case ",${PIPEDREAM_GROUP_REGIONS}," in
@@ -417,7 +413,10 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde
417413
local job = stage_jobs[job_name];
418414
local job_env = if std.objectHas(job, 'environment_variables') then job.environment_variables else {};
419415
local merged_env = region_specific_env + job_env;
420-
local job_tasks = if std.objectHas(job, 'tasks') then [wrap_task(t) for t in job.tasks] else null;
416+
// Single-region groups skip the gate (see deploy_target_gate).
417+
local job_tasks = if std.objectHas(job, 'tasks') then
418+
(if std.length(regions) == 1 then job.tasks else [wrap_task(t) for t in job.tasks])
419+
else null;
421420
local gated_job = if job_tasks != null then job { tasks: job_tasks } else job;
422421
if std.length(std.objectFields(merged_env)) > 0 then
423422
gated_job { environment_variables: merged_env }

‎test/testdata/fixtures/pipedream/single-region-multi-cell.jsonnet‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,9 @@
11
local pipedream = import '../../../../libs/pipedream.libsonnet';
22

3-
// Exercises the pattern used by ops/gocd/templates/pipelines/uptime-checker-k8s
4-
// and vector-uc-k8s, where pipeline_fn is called once per single-region group
5-
// (e.g. region='de') but expands internally to multiple cells (e.g.
6-
// 'de-west-de', 'de-west-nl'). Jobs are keyed by cell name, and downstream
7-
// stages fetch artifacts using those cell-keyed names. Without the
8-
// single-region skip, transform_stage would append the group region
9-
// (e.g. 'diff-de-west-nl-de') and break the cell-keyed fetch references.
3+
// Mirrors the uptime-checker-k8s / vector-uc-k8s shape: a single-region group
4+
// whose pipeline_fn fans out to multiple cell-keyed jobs that override
5+
// SENTRY_REGION per cell. Validates that pipedream skips both the suffix and
6+
// the gate for single-region groups.
107
local pipedream_config = {
118
name: 'example',
129
auto_deploy: true,
@@ -32,6 +29,7 @@ local sample = {
3229
jobs: {
3330
['diff-' + cell]: {
3431
elastic_profile_id: 'example',
32+
environment_variables: { SENTRY_REGION: cell },
3533
tasks: [
3634
{ script: './diff.sh --cell=' + cell },
3735
],
@@ -48,6 +46,7 @@ local sample = {
4846
jobs: {
4947
['apply-' + cell]: {
5048
elastic_profile_id: 'example',
49+
environment_variables: { SENTRY_REGION: cell },
5150
tasks: [
5251
{
5352
fetch: {

‎test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
"elastic_profile_id": "example",
2828
"tasks": [
2929
{
30-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=de"
30+
"script": "./deploy.sh --region=de"
3131
}
3232
]
3333
}
@@ -78,7 +78,7 @@
7878
"elastic_profile_id": "example",
7979
"tasks": [
8080
{
81-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=s4s2"
81+
"script": "./deploy.sh --region=s4s2"
8282
}
8383
]
8484
}
@@ -212,7 +212,7 @@
212212
"elastic_profile_id": "example",
213213
"tasks": [
214214
{
215-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us"
215+
"script": "./deploy.sh --region=us"
216216
}
217217
]
218218
}
@@ -267,7 +267,7 @@
267267
"elastic_profile_id": "example",
268268
"tasks": [
269269
{
270-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us2"
270+
"script": "./deploy.sh --region=us2"
271271
}
272272
]
273273
}

‎test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
"elastic_profile_id": "example",
2727
"tasks": [
2828
{
29-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=de"
29+
"script": "./deploy.sh --region=de"
3030
}
3131
]
3232
}
@@ -72,7 +72,7 @@
7272
"elastic_profile_id": "example",
7373
"tasks": [
7474
{
75-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=s4s2"
75+
"script": "./deploy.sh --region=s4s2"
7676
}
7777
]
7878
}
@@ -196,7 +196,7 @@
196196
"elastic_profile_id": "example",
197197
"tasks": [
198198
{
199-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us"
199+
"script": "./deploy.sh --region=us"
200200
}
201201
]
202202
}
@@ -246,7 +246,7 @@
246246
"elastic_profile_id": "example",
247247
"tasks": [
248248
{
249-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us2"
249+
"script": "./deploy.sh --region=us2"
250250
}
251251
]
252252
}

‎test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
"elastic_profile_id": "example",
2828
"tasks": [
2929
{
30-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=de"
30+
"script": "./deploy.sh --region=de"
3131
}
3232
]
3333
}
@@ -82,7 +82,7 @@
8282
"elastic_profile_id": "example",
8383
"tasks": [
8484
{
85-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=s4s2"
85+
"script": "./deploy.sh --region=s4s2"
8686
}
8787
]
8888
}
@@ -216,7 +216,7 @@
216216
"elastic_profile_id": "example",
217217
"tasks": [
218218
{
219-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us"
219+
"script": "./deploy.sh --region=us"
220220
}
221221
]
222222
}
@@ -271,7 +271,7 @@
271271
"elastic_profile_id": "example",
272272
"tasks": [
273273
{
274-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us2"
274+
"script": "./deploy.sh --region=us2"
275275
}
276276
]
277277
}

‎test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
"elastic_profile_id": "example",
6060
"tasks": [
6161
{
62-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=de"
62+
"script": "./deploy.sh --region=de"
6363
}
6464
]
6565
}
@@ -109,7 +109,7 @@
109109
"elastic_profile_id": "example",
110110
"tasks": [
111111
{
112-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=s4s2"
112+
"script": "./deploy.sh --region=s4s2"
113113
}
114114
]
115115
}
@@ -233,7 +233,7 @@
233233
"elastic_profile_id": "example",
234234
"tasks": [
235235
{
236-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us"
236+
"script": "./deploy.sh --region=us"
237237
}
238238
]
239239
}
@@ -283,7 +283,7 @@
283283
"elastic_profile_id": "example",
284284
"tasks": [
285285
{
286-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us2"
286+
"script": "./deploy.sh --region=us2"
287287
}
288288
]
289289
}

‎test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
},
3232
"tasks": [
3333
{
34-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=s4s2"
34+
"script": "./deploy.sh --region=s4s2"
3535
}
3636
]
3737
}
@@ -196,7 +196,7 @@
196196
},
197197
"tasks": [
198198
{
199-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us2"
199+
"script": "./deploy.sh --region=us2"
200200
}
201201
]
202202
}

‎test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
},
3131
"tasks": [
3232
{
33-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=s4s2"
33+
"script": "./deploy.sh --region=s4s2"
3434
}
3535
]
3636
}
@@ -185,7 +185,7 @@
185185
},
186186
"tasks": [
187187
{
188-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us2"
188+
"script": "./deploy.sh --region=us2"
189189
}
190190
]
191191
}

‎test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
"elastic_profile_id": "example",
2828
"tasks": [
2929
{
30-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=de"
30+
"script": "./deploy.sh --region=de"
3131
}
3232
]
3333
}
@@ -78,7 +78,7 @@
7878
"elastic_profile_id": "example",
7979
"tasks": [
8080
{
81-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=s4s2"
81+
"script": "./deploy.sh --region=s4s2"
8282
}
8383
]
8484
}
@@ -133,7 +133,7 @@
133133
"elastic_profile_id": "example",
134134
"tasks": [
135135
{
136-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us"
136+
"script": "./deploy.sh --region=us"
137137
}
138138
]
139139
}
@@ -188,7 +188,7 @@
188188
"elastic_profile_id": "example",
189189
"tasks": [
190190
{
191-
"script": "if [ -n \"${PIPEDREAM_GROUP_REGIONS:-}\" ] && [ -n \"${SENTRY_REGION:-}\" ]; then\n case \",${PIPEDREAM_GROUP_REGIONS},\" in\n *\",${SENTRY_REGION},\"*) ;;\n *)\n echo \"Skipping $SENTRY_REGION (not in PIPEDREAM_GROUP_REGIONS=$PIPEDREAM_GROUP_REGIONS)\"\n exit 0\n ;;\n esac\nfi\n./deploy.sh --region=us2"
191+
"script": "./deploy.sh --region=us2"
192192
}
193193
]
194194
}

0 commit comments

Comments
 (0)