Skip to content

Commit bbd4d4a

Browse files
Mike Bridgeclaude
andcommitted
refactor(migration): build pre-flight SQL via SQLAlchemy core (review)
Address Beto's review comments on apache#39859: replace ``sa.text(f"...")`` SQL construction in the three pre-flight helpers (``_delete_null_fk_rows``, ``_dedupe_by_min_id``, ``_assert_no_duplicates``) with SQLAlchemy core constructs (``sa.delete``, ``sa.select``, ``sa.func``, ``.subquery()``, ``.notin_()``). A small ``_table_clause()`` helper builds a lightweight ``TableClause`` exposing the columns the queries reference; the three helpers consume it. Removes all ``# noqa: S608`` comments — they are no longer needed because there is no string-interpolated SQL. Verified the compiled SQL is identical on Postgres, MySQL, and SQLite, including the MySQL ERROR 1093 workaround (the inner aggregation is wrapped in a derived table via ``.subquery()``, producing ``... NOT IN (SELECT keep_id FROM (SELECT min(id) ...) AS keep_min)``). Also drops the redundant ``f`` prefix on the two non-interpolating lines of the ``_check_no_external_fks_to_id`` error message. 44 migration tests still pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 32ed2bc commit bbd4d4a

1 file changed

Lines changed: 46 additions & 37 deletions

File tree

superset/migrations/versions/2026-05-01_23-36_2bee73611e32_composite_pk_association_tables.py

Lines changed: 46 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -120,11 +120,19 @@ def _check_no_external_fks_to_id(conn: Connection) -> None:
120120
f"Cannot drop synthetic id from {fk['referred_table']}: "
121121
f"external FK {fk.get('name', '<unnamed>')} on {table_name} "
122122
f"references {fk['referred_table']}({fk['referred_columns']}). "
123-
f"Drop or migrate the referencing FK before applying this "
124-
f"migration."
123+
"Drop or migrate the referencing FK before applying this "
124+
"migration."
125125
)
126126

127127

128+
def _table_clause(t: AssociationTable) -> sa.sql.expression.TableClause:
129+
"""Build a lightweight SQLAlchemy ``TableClause`` for ``t`` exposing the
130+
columns the helper queries reference (``id``, ``fk1``, ``fk2``). Used so
131+
that the dedupe / cleanup / assert SQL can be expressed via SQLAlchemy
132+
core constructs rather than via string interpolation."""
133+
return sa.table(t.name, sa.column("id"), sa.column(t.fk1), sa.column(t.fk2))
134+
135+
128136
def _delete_null_fk_rows(conn: Connection, t: AssociationTable) -> int:
129137
"""Delete rows where ``t.fk1`` or ``t.fk2`` is NULL on ``t.name``.
130138
@@ -133,11 +141,9 @@ def _delete_null_fk_rows(conn: Connection, t: AssociationTable) -> int:
133141
violation if any NULL-FK rows survived. Run unconditionally on every
134142
affected table — see ``TABLES_WITH_NULLABLE_FKS`` above for the rationale.
135143
"""
136-
# Identifiers come from the AFFECTED_TABLES whitelist, not user input.
137-
sql = sa.text(
138-
f"DELETE FROM {t.name} WHERE {t.fk1} IS NULL OR {t.fk2} IS NULL" # noqa: S608
139-
)
140-
result = conn.execute(sql)
144+
tbl = _table_clause(t)
145+
stmt = sa.delete(tbl).where(sa.or_(tbl.c[t.fk1].is_(None), tbl.c[t.fk2].is_(None)))
146+
result = conn.execute(stmt)
141147
n = result.rowcount or 0
142148
if n:
143149
logger.warning(
@@ -151,35 +157,35 @@ def _delete_null_fk_rows(conn: Connection, t: AssociationTable) -> int:
151157
def _dedupe_by_min_id(conn: Connection, t: AssociationTable) -> int:
152158
"""Delete duplicate ``(t.fk1, t.fk2)`` rows from ``t.name`` keeping ``MIN(id)``.
153159
154-
Returns the deletion count. Uses the wrapped-subquery form for MySQL
155-
portability — MySQL rejects ``DELETE FROM t WHERE id NOT IN (SELECT MIN(id)
156-
FROM t GROUP BY ...)`` with ERROR 1093 unless the inner SELECT is wrapped
157-
to force materialization.
160+
Returns the deletion count. The ``NOT IN`` argument is wrapped in an
161+
extra ``SELECT keep_id FROM (...) AS s`` derived table because MySQL
162+
rejects ``DELETE FROM t WHERE id NOT IN (SELECT MIN(id) FROM t GROUP BY
163+
...)`` with ERROR 1093 unless the inner SELECT is materialized through
164+
a derived table. SQLAlchemy's ``.subquery()`` produces that wrap.
158165
159166
Logs a sample (up to 10) of the discarded ``(fk1, fk2, id)`` tuples at
160-
WARN before deletion, so operators can audit which rows are dropped — the
161-
"keep ``MIN(id)``" policy preserves the original row, which is correct
162-
in practice but discards any later, semantically-identical re-grants.
167+
WARN before deletion, so operators can audit which rows are dropped —
168+
the "keep ``MIN(id)``" policy preserves the original row, which is
169+
correct in practice but discards any later, semantically-identical
170+
re-grants.
163171
"""
164-
# Identifiers come from the AFFECTED_TABLES whitelist, not user input.
165-
sample_sql = sa.text(
166-
f"SELECT {t.fk1}, {t.fk2}, id FROM {t.name} WHERE id NOT IN (" # noqa: S608
167-
f" SELECT keep_id FROM ("
168-
f" SELECT MIN(id) AS keep_id FROM {t.name} "
169-
f"GROUP BY {t.fk1}, {t.fk2}"
170-
f" ) AS s"
171-
f") LIMIT 10"
172+
tbl = _table_clause(t)
173+
174+
keep_min = (
175+
sa.select(sa.func.min(tbl.c.id).label("keep_id"))
176+
.group_by(tbl.c[t.fk1], tbl.c[t.fk2])
177+
.subquery("keep_min")
172178
)
173-
sample = list(conn.execute(sample_sql))
174-
sql = sa.text(
175-
f"DELETE FROM {t.name} WHERE id NOT IN (" # noqa: S608
176-
f" SELECT keep_id FROM ("
177-
f" SELECT MIN(id) AS keep_id FROM {t.name} "
178-
f"GROUP BY {t.fk1}, {t.fk2}"
179-
f" ) AS s"
180-
f")"
179+
keep_ids = sa.select(keep_min.c.keep_id)
180+
discarded = tbl.c.id.notin_(keep_ids)
181+
182+
sample_stmt = (
183+
sa.select(tbl.c[t.fk1], tbl.c[t.fk2], tbl.c.id).where(discarded).limit(10)
181184
)
182-
result = conn.execute(sql)
185+
sample = list(conn.execute(sample_stmt))
186+
187+
delete_stmt = sa.delete(tbl).where(discarded)
188+
result = conn.execute(delete_stmt)
183189
n = result.rowcount or 0
184190
if n:
185191
logger.warning(
@@ -201,13 +207,16 @@ def _assert_no_duplicates(conn: Connection, t: AssociationTable) -> None:
201207
dedupe failures (e.g., a MySQL syntax issue) as an actionable error
202208
before the PK-add fires with a less-helpful constraint-violation message.
203209
"""
204-
# Identifiers come from the AFFECTED_TABLES whitelist, not user input.
205-
sql = sa.text(
206-
f"SELECT COUNT(*) FROM (" # noqa: S608
207-
f" SELECT 1 FROM {t.name} GROUP BY {t.fk1}, {t.fk2} HAVING COUNT(*) > 1"
208-
f") AS s"
210+
tbl = _table_clause(t)
211+
duplicate_groups = (
212+
sa.select(sa.literal(1))
213+
.select_from(tbl)
214+
.group_by(tbl.c[t.fk1], tbl.c[t.fk2])
215+
.having(sa.func.count() > 1)
216+
.subquery("duplicate_groups")
209217
)
210-
if remaining := conn.scalar(sql) or 0:
218+
count_stmt = sa.select(sa.func.count()).select_from(duplicate_groups)
219+
if remaining := conn.scalar(count_stmt) or 0:
211220
raise RuntimeError(
212221
f"Dedupe failed for {t.name}: {remaining} duplicate "
213222
f"({t.fk1}, {t.fk2}) groups remain after _dedupe_by_min_id. "

0 commit comments

Comments
 (0)