Migrate google + pagerduty providers to httpx2

providers/.pre-commit-config.yaml

@@ -280,3 +280,12 @@ repos:
         types: [python]
         files: ^.*/src/airflow/providers/.*\.py$
         require_serial: true
+  - repo: https://github.com/boidolr/ast-grep-pre-commit
+    rev: 0.45.2
+    hooks:
+      - id: ast-grep
+        name: Check providers import httpx2 rather than httpx
+        types: [python]
+        # Pinned to this directory's config rather than relying on ast-grep walking up
+        # from the working directory, so a future root sgconfig.yml cannot shadow it.
+        args: ["--config", "./sgconfig.yml"]

providers/airbyte/src/airflow/providers/airbyte/hooks/airbyte.py

@@ -20,6 +20,9 @@
 import time
 from typing import Any, TypeVar
 
+# Stays on httpx (not httpx2): the client built below is handed to airbyte-api, which builds
+# and sends httpx.Request objects through it, and the two packages' classes are distinct.
+# Migrate once airbyte-api moves to httpx2; tracked at https://github.com/apache/airflow/issues/70522
 import httpx
 from airbyte_api import AirbyteAPI
 from airbyte_api.api import CancelJobRequest, GetJobRequest

providers/anthropic/README.rst

@@ -52,23 +52,23 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13,3.14
 Requirements
 ------------
 
-==========================================  ====================
+==========================================  ==================
 PIP package                                 Version required
-==========================================  ====================
+==========================================  ==================
 ``apache-airflow``                          ``>=3.0.0``
 ``apache-airflow-providers-common-compat``  ``>=1.12.0``
-``anthropic``                               ``>=0.121.0,<1.0.0``
-==========================================  ====================
+``anthropic``                               ``>=1.0.0``
+==========================================  ==================
 
 Optional dependencies
 ----------------------
 
 ===========  ===============================
 Extra        Dependencies
 ===========  ===============================
-``bedrock``  ``anthropic[bedrock]>=0.121.0``
-``vertex``   ``anthropic[vertex]>=0.121.0``
-``aws``      ``anthropic[aws]>=0.121.0``
+``bedrock``  ``anthropic[bedrock]>=1.0.0``
+``vertex``   ``anthropic[vertex]>=1.0.0``
+``aws``      ``anthropic[aws]>=1.0.0``
 ===========  ===============================
 
 The changelog for the provider package can be found in the

providers/anthropic/docs/changelog.rst

@@ -20,6 +20,12 @@
 Changelog
 ---------
 
+.. note::
+    The minimum required version of the ``anthropic`` SDK is now ``1.0.0``, which replaced
+    ``httpx`` with ``httpx2``. Deployments pinning an older SDK must upgrade it alongside this
+    provider, including the ``bedrock``, ``vertex`` and ``aws`` extras. The provider's own
+    hooks, operators, sensors and triggers are unchanged.
+
 0.3.0
 .....
 

providers/anthropic/docs/index.rst

@@ -133,13 +133,13 @@ Requirements
 
 The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``.
 
-==========================================  ====================
+==========================================  ==================
 PIP package                                 Version required
-==========================================  ====================
+==========================================  ==================
 ``apache-airflow``                          ``>=3.0.0``
 ``apache-airflow-providers-common-compat``  ``>=1.12.0``
-``anthropic``                               ``>=0.121.0,<1.0.0``
-==========================================  ====================
+``anthropic``                               ``>=1.0.0``
+==========================================  ==================
 
 Optional dependencies
 ---------------------
@@ -152,13 +152,13 @@ Install them when installing from PyPI. For example:
     pip install apache-airflow-providers-anthropic[bedrock]
 
 
-===========  ======================================
+===========  =============================
 Extra        Dependencies
-===========  ======================================
-``bedrock``  ``anthropic[bedrock]>=0.121.0,<1.0.0``
-``vertex``   ``anthropic[vertex]>=0.121.0,<1.0.0``
-``aws``      ``anthropic[aws]>=0.121.0,<1.0.0``
-===========  ======================================
+===========  =============================
+``bedrock``  ``anthropic[bedrock]>=1.0.0``
+``vertex``   ``anthropic[vertex]>=1.0.0``
+``aws``      ``anthropic[aws]>=1.0.0``
+===========  =============================
 
 Downloading official packages
 -----------------------------

providers/anthropic/pyproject.toml

@@ -61,24 +61,20 @@ requires-python = ">=3.10"
 dependencies = [
     "apache-airflow>=3.0.0",
     "apache-airflow-providers-common-compat>=1.12.0",
-    # 0.121.0 is the first release whose ``sessions.create`` accepts ``budget``, which is
-    # how the guide tells users to set a session ceiling. Reading a ``budget_reached`` stop
-    # reason happens to work further back -- older clients mis-build the discriminated
-    # union into the wrong variant class while preserving ``.type`` -- but only with
-    # response validation left non-strict, so it is not something to depend on.
-    # anthropic 1.0.0 replaces httpx with httpx2 and is a major release the provider
-    # has not been validated against. Remove the <1.0.0 cap (here and in the extras
-    # below) after migrating to the 1.x SDK;
-    # tracked at https://github.com/apache/airflow/issues/72071
-    "anthropic>=0.121.0,<1.0.0",
+    # 1.0.0 is the first release built on ``httpx2`` rather than ``httpx``; the two
+    # packages' ``Response``/``Request`` classes are distinct, so the SDK errors the tests
+    # construct must be built from the same package the client uses. This also subsumes the
+    # previous 0.121.0 floor, which existed because ``sessions.create`` gained ``budget``
+    # there -- the way the guide tells users to set a session ceiling.
+    "anthropic>=1.0.0",
 ]
 
 # The optional dependencies should be modified in place in the generated file
 # Any change in the dependencies is preserved when the file is regenerated
 [project.optional-dependencies]
-"bedrock" = ["anthropic[bedrock]>=0.121.0,<1.0.0"]
-"vertex" = ["anthropic[vertex]>=0.121.0,<1.0.0"]
-"aws" = ["anthropic[aws]>=0.121.0,<1.0.0"]
+"bedrock" = ["anthropic[bedrock]>=1.0.0"]
+"vertex" = ["anthropic[vertex]>=1.0.0"]
+"aws" = ["anthropic[aws]>=1.0.0"]
 
 [dependency-groups]
 dev = [

providers/anthropic/tests/unit/anthropic/hooks/test_anthropic.py

@@ -45,7 +45,7 @@
 
 pytest.importorskip("anthropic")
 
-import httpx
+import httpx2
 from anthropic import BadRequestError
 from anthropic.types import BetaMonetaryAmount
 from anthropic.types.beta import BetaManagedAgentsServerToolUsage, BetaManagedAgentsSessionUsage
@@ -283,8 +283,8 @@ def test_rejects_bad_amounts(self, amount, match):
 
 def _make_bad_request(message="cannot be archived while its status is running") -> BadRequestError:
     """A real SDK BadRequestError -- archive_session only interrupts on this, not on any error."""
-    request = httpx.Request("POST", "https://api.anthropic.com/v1/sessions/sess_1/archive")
-    return BadRequestError(message, response=httpx.Response(400, request=request), body=None)
+    request = httpx2.Request("POST", "https://api.anthropic.com/v1/sessions/sess_1/archive")
+    return BadRequestError(message, response=httpx2.Response(400, request=request), body=None)
 
 
 class TestArchiveSession:

providers/ast-grep-rules/no-httpx-import.yml

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+id: no-httpx-import
+language: python
+severity: error
+message: Import 'httpx2' instead of 'httpx' -- see https://github.com/apache/airflow/issues/70522
+note: >-
+  The 'airbyte', 'common/ai', 'weaviate' and 'microsoft/azure' providers are exempt
+  because the third-party SDKs they hand these objects to are themselves built on
+  'httpx', and 'httpx2' classes are distinct from their 'httpx' counterparts.
+files:
+  - "**/*.py"
+ignores:
+  - "airbyte/**"
+  - "common/ai/**"
+  - "weaviate/**"
+  - "microsoft/azure/**"
+rule:
+  all:
+    - kind: dotted_name
+    - regex: ^httpx($|\.)
+    - any:
+        - inside:
+            kind: import_statement
+            stopBy: end
+        - inside:
+            kind: import_from_statement
+            field: module_name

providers/google/README.rst

@@ -122,7 +122,7 @@ PIP package                                 Version required
 ``google-cloud-run``                        ``>=0.10.0``
 ``google-cloud-batch``                      ``>=0.13.0``
 ``grpcio-gcp``                              ``>=0.2.2``
-``httpx``                                   ``>=0.25.0``
+``httpx2``                                  ``>=2.0.0``
 ``looker-sdk``                              ``>=22.4.0,!=24.18.0``
 ``pandas-gbq``                              ``>=0.7.0``
 ``pandas``                                  ``>=2.1.2; python_version < "3.13"``

providers/google/docs/changelog.rst

@@ -27,6 +27,13 @@
 Changelog
 ---------
 
+.. note::
+    This provider now depends on ``httpx2`` instead of ``httpx``, and ``httpx2`` verifies TLS
+    against the operating system trust store rather than the ``certifi`` bundle. If
+    ``CloudSqlProxyRunner`` downloads the ``cloud-sql-proxy`` binary through a proxy presenting
+    a private CA, install that CA into the OS trust store or point ``SSL_CERT_FILE`` (or
+    ``SSL_CERT_DIR``) at it -- adding it to ``certifi`` alone no longer has any effect.
+
 22.4.0
 ......
 

providers/google/docs/index.rst

@@ -175,7 +175,7 @@ PIP package                                 Version required
 ``google-cloud-run``                        ``>=0.10.0``
 ``google-cloud-batch``                      ``>=0.13.0``
 ``grpcio-gcp``                              ``>=0.2.2``
-``httpx``                                   ``>=0.25.0``
+``httpx2``                                  ``>=2.0.0``
 ``looker-sdk``                              ``>=22.4.0,!=24.18.0``
 ``pandas-gbq``                              ``>=0.7.0``
 ``pandas``                                  ``>=2.1.2; python_version < "3.13"``

providers/google/pyproject.toml

@@ -132,7 +132,7 @@ dependencies = [
     "google-cloud-run>=0.10.0",
     "google-cloud-batch>=0.13.0",
     "grpcio-gcp>=0.2.2",
-    "httpx>=0.25.0",
+    "httpx2>=2.0.0",
     # looker-sdk 24.18.0 has issues in import looker_sdk.rtl,  No module named looker_sdk.rtl
     # See https://github.com/looker-open-source/sdk-codegen/issues/1518
     "looker-sdk>=22.4.0,!=24.18.0",

providers/google/src/airflow/providers/google/cloud/hooks/cloud_sql.py

@@ -34,14 +34,13 @@
 import time
 import uuid
 from collections.abc import Sequence
-from inspect import signature
 from pathlib import Path
 from subprocess import PIPE, Popen
 from tempfile import NamedTemporaryFile, _TemporaryFileWrapper, gettempdir
 from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import quote_plus
 
-import httpx
+import httpx2
 from aiohttp import ClientSession
 from gcloud.aio.auth import AioSession, Token
 from googleapiclient.discovery import Resource, build
@@ -608,12 +607,7 @@ def _download_sql_proxy_if_needed(self) -> None:
         download_url = self._get_sql_proxy_download_url()
         proxy_path_tmp = self.sql_proxy_path + ".tmp"
         self.log.info("Downloading cloud_sql_proxy from %s to %s", download_url, proxy_path_tmp)
-        # httpx has a breaking API change (follow_redirects vs allow_redirects)
-        # and this should work with both versions (cf. issue #20088)
-        if "follow_redirects" in signature(httpx.get).parameters.keys():
-            response = httpx.get(download_url, follow_redirects=True)
-        else:
-            response = httpx.get(download_url, allow_redirects=True)  # type: ignore[call-arg]
+        response = httpx2.get(download_url, follow_redirects=True)
         # Downloading to .tmp file first to avoid case where partially downloaded
         # binary is used by parallel operator which uses the same fixed binary path
         with open(proxy_path_tmp, "wb") as file:

providers/google/tests/unit/google/cloud/hooks/test_cloud_sql.py

@@ -1918,6 +1918,31 @@ def test_cloud_sql_proxy_runner_version_nok(self, version):
         with pytest.raises(ValueError, match="The sql_proxy_version should match the regular expression"):
             runner._get_sql_proxy_download_url()
 
+    @mock.patch("airflow.providers.google.cloud.hooks.cloud_sql.httpx2.get")
+    def test_download_sql_proxy_follows_redirects(self, mock_get, tmp_path):
+        """The download URL redirects, so the request must opt in to following them."""
+        mock_get.return_value = mock.Mock(status_code=200, content=b"binary")
+        runner = CloudSqlProxyRunner(
+            path_prefix=str(tmp_path / "12345678"),
+            instance_specification="project:us-east-1:instance",
+        )
+
+        runner._download_sql_proxy_if_needed()
+
+        mock_get.assert_called_once_with(runner._get_sql_proxy_download_url(), follow_redirects=True)
+        assert runner.sql_proxy_was_downloaded is True
+
+    @mock.patch("airflow.providers.google.cloud.hooks.cloud_sql.httpx2.get")
+    def test_download_sql_proxy_raises_on_error_status(self, mock_get, tmp_path):
+        mock_get.return_value = mock.Mock(status_code=404, content=b"", reason_phrase="Not Found")
+        runner = CloudSqlProxyRunner(
+            path_prefix=str(tmp_path / "12345678"),
+            instance_specification="project:us-east-1:instance",
+        )
+
+        with pytest.raises(AirflowException, match="Status code = 404. Reason = Not Found"):
+            runner._download_sql_proxy_if_needed()
+
     def test_cloud_sql_proxy_runner_adds_enable_iam_login_flag(self):
         runner = CloudSqlProxyRunner(
             path_prefix="12345678",

providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/msgraph.py

@@ -31,6 +31,9 @@
 from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import quote, urljoin, urlparse
 
+# Stays on httpx (not httpx2): the client below is handed to kiota_http, which builds and
+# consumes httpx objects, and the two packages' classes are distinct.
+# Migrate once msgraph-core/kiota_http move to httpx2; tracked at https://github.com/apache/airflow/issues/70522
 import httpx
 from azure.core.credentials_async import AsyncTokenCredential
 from azure.identity.aio import CertificateCredential, ClientSecretCredential

providers/pagerduty/README.rst

@@ -56,7 +56,7 @@ PIP package                                 Version required
 ``apache-airflow``                          ``>=2.11.0``
 ``apache-airflow-providers-common-compat``  ``>=1.10.1``
 ``apache-airflow-providers-http``
-``pagerduty``                               ``>=2.3.0``
+``pagerduty``                               ``>=7.0.0``
 ==========================================  ==================
 
 The changelog for the provider package can be found in the

providers/pagerduty/docs/changelog.rst

@@ -27,6 +27,11 @@
 Changelog
 ---------
 
+.. note::
+    The minimum required version of the ``pagerduty`` SDK is now ``7.0.0``, which replaced
+    ``httpx`` with ``httpx2``. Deployments pinning an older SDK must upgrade it alongside this
+    provider. The provider's own hooks and notifier are unchanged.
+
 5.2.6
 .....
 

providers/pagerduty/docs/index.rst

@@ -95,7 +95,7 @@ PIP package                                 Version required
 ``apache-airflow``                          ``>=2.11.0``
 ``apache-airflow-providers-common-compat``  ``>=1.10.1``
 ``apache-airflow-providers-http``
-``pagerduty``                               ``>=2.3.0``
+``pagerduty``                               ``>=7.0.0``
 ==========================================  ==================
 
 Downloading official packages

providers/pagerduty/pyproject.toml

@@ -62,7 +62,7 @@ dependencies = [
     "apache-airflow>=2.11.0",
     "apache-airflow-providers-common-compat>=1.10.1",
     "apache-airflow-providers-http",
-    "pagerduty>=2.3.0",
+    "pagerduty>=7.0.0",
 ]
 
 [dependency-groups]

providers/pagerduty/tests/unit/pagerduty/hooks/test_pagerduty_events.py

@@ -20,7 +20,7 @@
 from unittest import mock
 from unittest.mock import patch
 
-import httpx
+import httpx2
 import pagerduty
 import pytest
 from pagerduty import EventsApiV2Client
@@ -91,10 +91,10 @@ def test_create_change_event(self, mock_request, events_connections):
             "message": "Change event processed",
             "status": "success",
         }
-        mock_response = httpx.Response(
+        mock_response = httpx2.Response(
             status_code=202,
             json=mock_response_body,
-            request=httpx.Request("POST", "https://events.pagerduty.com/v2/change/enqueue"),
+            request=httpx2.Request("POST", "https://events.pagerduty.com/v2/change/enqueue"),
         )
 
         mock_response.ok = True
@@ -114,10 +114,10 @@ def test_send_event_success(self, mock_request, events_connections):
             "message": "Event processed",
             "dedup_key": dedup_key,
         }
-        mock_response = httpx.Response(
+        mock_response = httpx2.Response(
             status_code=202,
             json=mock_response_body,
-            request=httpx.Request("POST", "https://events.pagerduty.com/v2/enqueue"),
+            request=httpx2.Request("POST", "https://events.pagerduty.com/v2/enqueue"),
         )
         mock_response.ok = True
         mock_request.return_value = mock_response