From 608e0e0edfcd71b5b450bf81cc4c5229a723d648 Mon Sep 17 00:00:00 2001 From: Dev-iL <6509619+Dev-iL@users.noreply.github.com> Date: Sat, 27 Jun 2026 19:50:30 +0300 Subject: [PATCH] Move asyncpg to an opt-in extra and default the postgres provider to psycopg3 The async metadata-database driver shipped by the postgres provider now defaults to psycopg3, which is safe behind transaction-mode PgBouncer with no extra configuration. asyncpg moves to the opt-in `asyncpg` extra and remains fully supported via an explicit sql_alchemy_conn_async URL. This pairs with the matching default change in Airflow 3.4.0. --- providers/postgres/README.rst | 3 ++- providers/postgres/docs/changelog.rst | 9 ++++++++ providers/postgres/docs/index.rst | 3 ++- providers/postgres/pyproject.toml | 13 +++++++---- .../providers/postgres/hooks/postgres.py | 20 ++-------------- uv.lock | 23 ++++++++++--------- 6 files changed, 35 insertions(+), 36 deletions(-) diff --git a/providers/postgres/README.rst b/providers/postgres/README.rst index 35d6ad22a3ec2..a8f06ba267234 100644 --- a/providers/postgres/README.rst +++ b/providers/postgres/README.rst @@ -58,7 +58,8 @@ PIP package Version required ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"`` ``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"`` -``asyncpg`` ``>=0.30.0`` +``psycopg[binary]`` ``>=3.2.9; python_version < "3.14"`` +``psycopg[binary]`` ``>=3.3.3; python_version >= "3.14"`` ========================================== ====================================== Cross provider package dependencies diff --git a/providers/postgres/docs/changelog.rst b/providers/postgres/docs/changelog.rst index 934ded27a23a7..a913d2541745d 100644 --- a/providers/postgres/docs/changelog.rst +++ b/providers/postgres/docs/changelog.rst @@ -27,6 +27,15 @@ Changelog --------- +.. warning:: + ``asyncpg`` is no longer installed by default — it moved to the ``asyncpg`` optional extra, and + ``psycopg`` (psycopg3) is now installed by default as the async metadata-database driver. The + default derived async connection URL changed from ``postgresql+asyncpg://`` to + ``postgresql+psycopg_async://``, which is safe behind transaction-mode PgBouncer with no extra + configuration. This aligns with the new default async Postgres driver in Airflow 3.4.0. To keep + using asyncpg, install ``apache-airflow-providers-postgres[asyncpg]`` and set + ``[database] sql_alchemy_conn_async = postgresql+asyncpg://...`` explicitly. + 6.8.0 ..... diff --git a/providers/postgres/docs/index.rst b/providers/postgres/docs/index.rst index 864aadd0f8de1..961752c0628f8 100644 --- a/providers/postgres/docs/index.rst +++ b/providers/postgres/docs/index.rst @@ -106,7 +106,8 @@ PIP package Version required ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``psycopg2-binary`` ``>=2.9.9; python_version < "3.13"`` ``psycopg2-binary`` ``>=2.9.10; python_version >= "3.13"`` -``asyncpg`` ``>=0.30.0`` +``psycopg[binary]`` ``>=3.2.9; python_version < "3.14"`` +``psycopg[binary]`` ``>=3.3.3; python_version >= "3.14"`` ========================================== ====================================== Cross provider package dependencies diff --git a/providers/postgres/pyproject.toml b/providers/postgres/pyproject.toml index 83b2b92f61053..8adda0b57802b 100644 --- a/providers/postgres/pyproject.toml +++ b/providers/postgres/pyproject.toml @@ -62,9 +62,12 @@ dependencies = [ "apache-airflow>=2.11.0", "apache-airflow-providers-common-compat>=1.12.0", "apache-airflow-providers-common-sql>=1.32.0", + # psycopg2 remains the sync driver for now; its removal and the sync migration to + # psycopg3 are tracked at https://github.com/apache/airflow/issues/68453 "psycopg2-binary>=2.9.9; python_version < '3.13'", "psycopg2-binary>=2.9.10; python_version >= '3.13'", - "asyncpg>=0.30.0", + "psycopg[binary]>=3.2.9; python_version < '3.14'", + "psycopg[binary]>=3.3.3; python_version >= '3.14'", ] # The optional dependencies should be modified in place in the generated file @@ -73,6 +76,9 @@ dependencies = [ "amazon" = [ "apache-airflow-providers-amazon>=2.6.0", ] +"asyncpg" = [ + "asyncpg>=0.30.0", +] "microsoft.azure" = [ "apache-airflow-providers-microsoft-azure>=12.8.0" ] @@ -87,10 +93,6 @@ dependencies = [ "polars" = [ "polars>=1.26.0" ] -"psycopg" = [ - "psycopg[binary]>=3.2.9; python_version < '3.14'", - "psycopg[binary]>=3.3.3; python_version >= '3.14'", -] "sqlalchemy" = [ "sqlalchemy>=1.4.54" ] @@ -108,6 +110,7 @@ dev = [ # Additional devel dependencies (do not remove this line and add extra development dependencies) "apache-airflow-providers-common-sql[pandas]", "apache-airflow-providers-common-sql[polars]", + "apache-airflow-providers-postgres[asyncpg]", "apache-airflow-providers-postgres[sqlalchemy]" ] diff --git a/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py b/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py index 77ca0537b934d..be6519ddade24 100644 --- a/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py +++ b/providers/postgres/src/airflow/providers/postgres/hooks/postgres.py @@ -70,30 +70,14 @@ class CompatConnection(Protocol): - """Protocol for type hinting psycopg2 and psycopg3 connection objects.""" + """Protocol for the common interface shared by psycopg2 and psycopg3 connection objects.""" def cursor(self, *args, **kwargs) -> Any: ... def commit(self) -> None: ... def close(self) -> None: ... - - # Context manager support - def __enter__(self) -> CompatConnection: ... + def __enter__(self) -> Any: ... def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... - # Common properties - @property - def notices(self) -> list[Any]: ... - - # psycopg3 specific (optional) - @property - def adapters(self) -> Any: ... - - @property - def row_factory(self) -> Any: ... - - # Optional method for psycopg3 - def add_notice_handler(self, handler: Any) -> None: ... - class PostgresHook(DbApiHook): """ diff --git a/uv.lock b/uv.lock index cd569574def11..727158c6401df 100644 --- a/uv.lock +++ b/uv.lock @@ -968,7 +968,7 @@ wheels = [ [[package]] name = "apache-airflow" -version = "3.3.0" +version = "3.4.0" source = { editable = "." } dependencies = [ { name = "apache-airflow-core" }, @@ -1911,7 +1911,7 @@ requires-dist = [ [[package]] name = "apache-airflow-core" -version = "3.3.0" +version = "3.4.0" source = { editable = "airflow-core" } dependencies = [ { name = "a2wsgi" }, @@ -7067,7 +7067,7 @@ dependencies = [ { name = "apache-airflow" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql" }, - { name = "asyncpg" }, + { name = "psycopg", extra = ["binary"] }, { name = "psycopg2-binary" }, ] @@ -7075,6 +7075,9 @@ dependencies = [ amazon = [ { name = "apache-airflow-providers-amazon" }, ] +asyncpg = [ + { name = "asyncpg" }, +] microsoft-azure = [ { name = "apache-airflow-providers-microsoft-azure" }, ] @@ -7087,9 +7090,6 @@ pandas = [ polars = [ { name = "polars" }, ] -psycopg = [ - { name = "psycopg", extra = ["binary"] }, -] sqlalchemy = [ { name = "sqlalchemy" }, ] @@ -7103,7 +7103,7 @@ dev = [ { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-microsoft-azure" }, { name = "apache-airflow-providers-openlineage" }, - { name = "apache-airflow-providers-postgres", extra = ["sqlalchemy"] }, + { name = "apache-airflow-providers-postgres", extra = ["asyncpg", "sqlalchemy"] }, { name = "apache-airflow-task-sdk" }, ] docs = [ @@ -7118,18 +7118,18 @@ requires-dist = [ { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-microsoft-azure", marker = "extra == 'microsoft-azure'", editable = "providers/microsoft/azure" }, { name = "apache-airflow-providers-openlineage", marker = "extra == 'openlineage'", editable = "providers/openlineage" }, - { name = "asyncpg", specifier = ">=0.30.0" }, + { name = "asyncpg", marker = "extra == 'asyncpg'", specifier = ">=0.30.0" }, { name = "pandas", marker = "python_full_version == '3.13.*' and extra == 'pandas'", specifier = ">=2.2.3" }, { name = "pandas", marker = "python_full_version < '3.13' and extra == 'pandas'", specifier = ">=2.1.2" }, { name = "pandas", marker = "python_full_version >= '3.14' and extra == 'pandas'", specifier = ">=2.3.3" }, { name = "polars", marker = "extra == 'polars'", specifier = ">=1.26.0" }, - { name = "psycopg", extras = ["binary"], marker = "python_full_version >= '3.14' and extra == 'psycopg'", specifier = ">=3.3.3" }, - { name = "psycopg", extras = ["binary"], marker = "python_full_version < '3.14' and extra == 'psycopg'", specifier = ">=3.2.9" }, + { name = "psycopg", extras = ["binary"], marker = "python_full_version < '3.14'", specifier = ">=3.2.9" }, + { name = "psycopg", extras = ["binary"], marker = "python_full_version >= '3.14'", specifier = ">=3.3.3" }, { name = "psycopg2-binary", marker = "python_full_version < '3.13'", specifier = ">=2.9.9" }, { name = "psycopg2-binary", marker = "python_full_version >= '3.13'", specifier = ">=2.9.10" }, { name = "sqlalchemy", marker = "extra == 'sqlalchemy'", specifier = ">=1.4.54" }, ] -provides-extras = ["amazon", "microsoft-azure", "openlineage", "pandas", "polars", "psycopg", "sqlalchemy"] +provides-extras = ["amazon", "asyncpg", "microsoft-azure", "openlineage", "pandas", "polars", "sqlalchemy"] [package.metadata.requires-dev] dev = [ @@ -7142,6 +7142,7 @@ dev = [ { name = "apache-airflow-providers-common-sql", extras = ["polars"], editable = "providers/common/sql" }, { name = "apache-airflow-providers-microsoft-azure", editable = "providers/microsoft/azure" }, { name = "apache-airflow-providers-openlineage", editable = "providers/openlineage" }, + { name = "apache-airflow-providers-postgres", extras = ["asyncpg"], editable = "providers/postgres" }, { name = "apache-airflow-providers-postgres", extras = ["sqlalchemy"], editable = "providers/postgres" }, { name = "apache-airflow-task-sdk", editable = "task-sdk" }, ]