Fix inconsistency between generated provider docs and pyproject.toml

dev/breeze/src/airflow_breeze/templates/PROVIDER_INDEX_TEMPLATE.rst.jinja2

@@ -73,8 +73,8 @@ The minimum Apache Airflow version supported by this provider distribution is ``
 
 {%- if CROSS_PROVIDERS_DEPENDENCIES %}
 
-Cross provider package dependencies
------------------------------------
+Optional cross provider package dependencies
+--------------------------------------------
 
 Those are dependencies that might be needed in order to use all the features of the package.
 You need to install the specified provider distributions in order to use them.
@@ -87,6 +87,7 @@ You can install such cross-provider dependencies when installing from PyPI. For
 
 
 {{ CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST | safe }}
+{%- endif %}
 
 Downloading official packages
 -----------------------------
@@ -96,7 +97,3 @@ You can download officially released packages and verify their checksums and sig
 
 * `The {{ PACKAGE_PIP_NAME }} {{ RELEASE }}{{ VERSION_SUFFIX }} sdist package <https://downloads.apache.org/airflow/providers/{{ PACKAGE_DIST_PREFIX }}-{{ RELEASE }}{{ VERSION_SUFFIX }}.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/{{ PACKAGE_DIST_PREFIX }}-{{ RELEASE }}{{ VERSION_SUFFIX }}.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/{{ PACKAGE_DIST_PREFIX }}-{{ RELEASE }}{{ VERSION_SUFFIX }}.tar.gz.sha512>`__)
 * `The {{ PACKAGE_PIP_NAME }} {{ RELEASE }}{{ VERSION_SUFFIX }} wheel package <https://downloads.apache.org/airflow/providers/{{ PACKAGE_DIST_PREFIX }}-{{ RELEASE }}{{ VERSION_SUFFIX }}-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/{{ PACKAGE_DIST_PREFIX }}-{{ RELEASE }}{{ VERSION_SUFFIX }}-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/{{ PACKAGE_DIST_PREFIX }}-{{ RELEASE }}{{ VERSION_SUFFIX }}-py3-none-any.whl.sha512>`__)
-
-
-
-{%- endif %}

dev/breeze/src/airflow_breeze/templates/PROVIDER_README_TEMPLATE.rst.jinja2

@@ -86,8 +86,8 @@ Requirements
 {%- endif %}
 {%- if CROSS_PROVIDERS_DEPENDENCIES %}
 
-Cross provider package dependencies
------------------------------------
+Optional cross provider package dependencies
+--------------------------------------------
 
 Those are dependencies that might be needed in order to use all the features of the package.
 You need to install the specified providers in order to use them.

dev/breeze/src/airflow_breeze/utils/packages.py

@@ -683,6 +683,34 @@ def get_cross_provider_dependent_packages(provider_id: str) -> list[str]:
     return get_provider_dependencies()[provider_id]["cross-providers-deps"]
 
 
+def get_cross_provider_dependencies_for_extras(provider_id: str) -> list[str]:
+    """Return cross-provider dependencies that can be rendered as installable extras.
+
+    ``cross-providers-deps`` is an import-level graph. Some entries are already
+    required dependencies, so they should not be documented as ``pkg[extra]``.
+    This mirrors the filtering used when generating provider ``pyproject.toml``.
+    """
+    cross_provider_dependencies = get_cross_provider_dependent_packages(provider_id)
+    if not cross_provider_dependencies:
+        return []
+
+    from packaging.requirements import Requirement
+    from packaging.utils import canonicalize_name
+
+    suspended_provider_ids = get_suspended_provider_ids()
+    required_package_names = {
+        canonicalize_name(Requirement(requirement).name)
+        for requirement in get_provider_requirements(provider_id)
+    }
+
+    return [
+        cross_provider_id
+        for cross_provider_id in cross_provider_dependencies
+        if cross_provider_id not in suspended_provider_ids
+        and canonicalize_name(get_pip_package_name(cross_provider_id)) not in required_package_names
+    ]
+
+
 def get_license_files(provider_id: str) -> str:
     if provider_id == "fab":
         return str(["LICENSE", "NOTICE", "3rd-party-licenses/LICENSES-*"]).replace('"', "'")
@@ -700,7 +728,7 @@ def get_provider_jinja_context(
     supported_python_versions = [
         p for p in ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS if p not in provider_details.excluded_python_versions
     ]
-    cross_providers_dependencies = get_cross_provider_dependent_packages(provider_id=provider_id)
+    cross_providers_dependencies = get_cross_provider_dependencies_for_extras(provider_id)
 
     requires_python_version: str = f">={DEFAULT_PYTHON_MAJOR_MINOR_VERSION}"
     # Most providers require the same python versions, but some may have exclusions
@@ -729,7 +757,7 @@ def get_provider_jinja_context(
         "MIN_AIRFLOW_VERSION": get_min_airflow_version(provider_id),
         "PROVIDER_REMOVED": provider_details.removed,
         "PROVIDER_INFO": get_provider_info_dict(provider_id),
-        "CROSS_PROVIDERS_DEPENDENCIES": get_cross_provider_dependent_packages(provider_id),
+        "CROSS_PROVIDERS_DEPENDENCIES": cross_providers_dependencies,
         "CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST": convert_cross_package_dependencies_to_table(
             cross_providers_dependencies, markdown=False
         ),

dev/breeze/tests/test_packages.py

@@ -31,6 +31,7 @@
     expand_all_provider_distributions,
     find_matching_long_package_names,
     get_available_distributions,
+    get_cross_provider_dependencies_for_extras,
     get_cross_provider_dependent_packages,
     get_dist_package_name_prefix,
     get_long_package_name,
@@ -263,6 +264,44 @@ def test_get_min_airflow_version(provider_id: str, min_version: str):
     assert get_min_airflow_version(provider_id) == min_version
 
 
+@pytest.mark.parametrize(
+    ("cross_provider_deps", "suspended_ids", "requirements", "expected"),
+    [
+        pytest.param(
+            ["common.compat", "common.sql", "google", "apache.beam"],
+            ["apache.beam"],
+            ["apache-airflow-providers-common-compat>=1.2.3"],
+            ["common.sql", "google"],
+            id="filters_suspended_and_required",
+        ),
+        pytest.param(
+            ["common.sql"],
+            [],
+            ["apache-airflow-providers-common-sql-extra>=1.0.0"],
+            ["common.sql"],
+            id="substring_collision_not_filtered",
+        ),
+    ],
+)
+def test_get_cross_provider_dependencies_for_extras(
+    monkeypatch, cross_provider_deps, suspended_ids, requirements, expected
+):
+    monkeypatch.setattr(
+        "airflow_breeze.utils.packages.get_cross_provider_dependent_packages",
+        lambda provider_id: cross_provider_deps,
+    )
+    monkeypatch.setattr(
+        "airflow_breeze.utils.packages.get_suspended_provider_ids",
+        lambda: suspended_ids,
+    )
+    monkeypatch.setattr(
+        "airflow_breeze.utils.packages.get_provider_requirements",
+        lambda provider_id: requirements,
+    )
+
+    assert get_cross_provider_dependencies_for_extras("test.provider") == expected
+
+
 def test_convert_cross_package_dependencies_to_table():
     EXPECTED = """
 | Dependent package                                                                       | Extra           |

providers/airbyte/README.rst

@@ -59,24 +59,5 @@ PIP package                                 Version required
 ``requests``                                ``>=2.32.0``
 ==========================================  ===================
 
-Cross provider package dependencies
------------------------------------
-
-Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified providers in order to use them.
-
-You can install such cross-provider dependencies when installing from PyPI. For example:
-
-.. code-block:: bash
-
-    pip install apache-airflow-providers-airbyte[common.compat]
-
-
-==================================================================================================================  =================
-Dependent package                                                                                                   Extra
-==================================================================================================================  =================
-`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
-==================================================================================================================  =================
-
 The changelog for the provider package can be found in the
 `changelog <https://airflow.apache.org/docs/apache-airflow-providers-airbyte/5.5.1/changelog.html>`_.

providers/airbyte/docs/index.rst

@@ -105,25 +105,6 @@ PIP package                                 Version required
 ``requests``                                ``>=2.32.0``
 ==========================================  ===================
 
-Cross provider package dependencies
------------------------------------
-
-Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified provider distributions in order to use them.
-
-You can install such cross-provider dependencies when installing from PyPI. For example:
-
-.. code-block:: bash
-
-    pip install apache-airflow-providers-airbyte[common.compat]
-
-
-==================================================================================================================  =================
-Dependent package                                                                                                   Extra
-==================================================================================================================  =================
-`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
-==================================================================================================================  =================
-
 Downloading official packages
 -----------------------------
 

providers/akeyless/README.rst

@@ -60,25 +60,6 @@ PIP package                                 Version required
 ``akeyless``                                ``>=5.0.0``
 ==========================================  ==================
 
-Cross provider package dependencies
------------------------------------
-
-Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified providers in order to use them.
-
-You can install such cross-provider dependencies when installing from PyPI. For example:
-
-.. code-block:: bash
-
-    pip install apache-airflow-providers-akeyless[common.compat]
-
-
-==================================================================================================================  =================
-Dependent package                                                                                                   Extra
-==================================================================================================================  =================
-`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
-==================================================================================================================  =================
-
 Optional dependencies
 ----------------------
 

providers/akeyless/docs/index.rst

@@ -106,25 +106,6 @@ PIP package                                 Version required
 ``akeyless``                                ``>=5.0.0``
 ==========================================  ==================
 
-Cross provider package dependencies
------------------------------------
-
-Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified provider distributions in order to use them.
-
-You can install such cross-provider dependencies when installing from PyPI. For example:
-
-.. code-block:: bash
-
-    pip install apache-airflow-providers-akeyless[common.compat]
-
-
-==================================================================================================================  =================
-Dependent package                                                                                                   Extra
-==================================================================================================================  =================
-`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
-==================================================================================================================  =================
-
 Downloading official packages
 -----------------------------
 

providers/alibaba/README.rst

@@ -62,24 +62,5 @@ PIP package                                 Version required
 ``pyodps``                                  ``>=0.12.5.1; python_version >= "3.13"``
 ==========================================  ========================================
 
-Cross provider package dependencies
------------------------------------
-
-Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified providers in order to use them.
-
-You can install such cross-provider dependencies when installing from PyPI. For example:
-
-.. code-block:: bash
-
-    pip install apache-airflow-providers-alibaba[common.compat]
-
-
-==================================================================================================================  =================
-Dependent package                                                                                                   Extra
-==================================================================================================================  =================
-`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
-==================================================================================================================  =================
-
 The changelog for the provider package can be found in the
 `changelog <https://airflow.apache.org/docs/apache-airflow-providers-alibaba/3.3.9/changelog.html>`_.

providers/alibaba/docs/index.rst

@@ -109,25 +109,6 @@ PIP package                                 Version required
 ``pyodps``                                  ``>=0.12.5.1; python_version >= "3.13"``
 ==========================================  ========================================
 
-Cross provider package dependencies
------------------------------------
-
-Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified provider distributions in order to use them.
-
-You can install such cross-provider dependencies when installing from PyPI. For example:
-
-.. code-block:: bash
-
-    pip install apache-airflow-providers-alibaba[common.compat]
-
-
-==================================================================================================================  =================
-Dependent package                                                                                                   Extra
-==================================================================================================================  =================
-`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
-==================================================================================================================  =================
-
 Downloading official packages
 -----------------------------
 

providers/amazon/README.rst

@@ -72,8 +72,8 @@ PIP package                                 Version required
 ``marshmallow``                             ``>=3``
 ==========================================  ======================================
 
-Cross provider package dependencies
------------------------------------
+Optional cross provider package dependencies
+--------------------------------------------
 
 Those are dependencies that might be needed in order to use all the features of the package.
 You need to install the specified providers in order to use them.
@@ -90,13 +90,10 @@ Dependent package
 ========================================================================================================================  ====================
 `apache-airflow-providers-apache-hive <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive>`_            ``apache.hive``
 `apache-airflow-providers-cncf-kubernetes <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes>`_    ``cncf.kubernetes``
-`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_        ``common.compat``
 `apache-airflow-providers-common-messaging <https://airflow.apache.org/docs/apache-airflow-providers-common-messaging>`_  ``common.messaging``
-`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_              ``common.sql``
 `apache-airflow-providers-exasol <https://airflow.apache.org/docs/apache-airflow-providers-exasol>`_                      ``exasol``
 `apache-airflow-providers-ftp <https://airflow.apache.org/docs/apache-airflow-providers-ftp>`_                            ``ftp``
 `apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_                      ``google``
-`apache-airflow-providers-http <https://airflow.apache.org/docs/apache-airflow-providers-http>`_                          ``http``
 `apache-airflow-providers-imap <https://airflow.apache.org/docs/apache-airflow-providers-imap>`_                          ``imap``
 `apache-airflow-providers-microsoft-azure <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure>`_    ``microsoft.azure``
 `apache-airflow-providers-mongo <https://airflow.apache.org/docs/apache-airflow-providers-mongo>`_                        ``mongo``

providers/amazon/docs/index.rst

@@ -130,8 +130,8 @@ PIP package                                 Version required
 ``marshmallow``                             ``>=3``
 ==========================================  ======================================
 
-Cross provider package dependencies
------------------------------------
+Optional cross provider package dependencies
+--------------------------------------------
 
 Those are dependencies that might be needed in order to use all the features of the package.
 You need to install the specified provider distributions in order to use them.
@@ -148,13 +148,10 @@ Dependent package
 ========================================================================================================================  ====================
 `apache-airflow-providers-apache-hive <https://airflow.apache.org/docs/apache-airflow-providers-apache-hive>`_            ``apache.hive``
 `apache-airflow-providers-cncf-kubernetes <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes>`_    ``cncf.kubernetes``
-`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_        ``common.compat``
 `apache-airflow-providers-common-messaging <https://airflow.apache.org/docs/apache-airflow-providers-common-messaging>`_  ``common.messaging``
-`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_              ``common.sql``
 `apache-airflow-providers-exasol <https://airflow.apache.org/docs/apache-airflow-providers-exasol>`_                      ``exasol``
 `apache-airflow-providers-ftp <https://airflow.apache.org/docs/apache-airflow-providers-ftp>`_                            ``ftp``
 `apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_                      ``google``
-`apache-airflow-providers-http <https://airflow.apache.org/docs/apache-airflow-providers-http>`_                          ``http``
 `apache-airflow-providers-imap <https://airflow.apache.org/docs/apache-airflow-providers-imap>`_                          ``imap``
 `apache-airflow-providers-microsoft-azure <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure>`_    ``microsoft.azure``
 `apache-airflow-providers-mongo <https://airflow.apache.org/docs/apache-airflow-providers-mongo>`_                        ``mongo``