Introduce tiered timeout system with per-endpoint configuration

Author: vdusek

docs/02_concepts/code/05_retries_async.py

@@ -8,7 +8,10 @@
 async def main() -> None:
     apify_client = ApifyClientAsync(
         token=TOKEN,
-        max_retries=8,
+        max_retries=4,
         min_delay_between_retries=timedelta(milliseconds=500),
-        timeout=timedelta(seconds=360),
+        timeout_short=timedelta(seconds=5),
+        timeout_medium=timedelta(seconds=30),
+        timeout_long=timedelta(seconds=360),
+        timeout_max=timedelta(seconds=360),
     )

docs/02_concepts/code/05_retries_sync.py

@@ -8,7 +8,10 @@
 def main() -> None:
     apify_client = ApifyClient(
         token=TOKEN,
-        max_retries=8,
+        max_retries=4,
         min_delay_between_retries=timedelta(milliseconds=500),
-        timeout=timedelta(seconds=360),
+        timeout_short=timedelta(seconds=5),
+        timeout_medium=timedelta(seconds=30),
+        timeout_long=timedelta(seconds=360),
+        timeout_max=timedelta(seconds=360),
     )

docs/02_concepts/code/10_default_http_client_async.py

@@ -10,5 +10,5 @@ async def main() -> None:
         token=TOKEN,
         max_retries=4,
         min_delay_between_retries=timedelta(milliseconds=500),
-        timeout=timedelta(seconds=360),
+        timeout_medium=timedelta(seconds=360),
     )

docs/02_concepts/code/10_default_http_client_sync.py

@@ -10,5 +10,5 @@ def main() -> None:
         token=TOKEN,
         max_retries=4,
         min_delay_between_retries=timedelta(milliseconds=500),
-        timeout=timedelta(seconds=360),
+        timeout_medium=timedelta(seconds=360),
     )

docs/02_concepts/code/10_plugging_in_async.py

@@ -1,7 +1,6 @@
-from datetime import timedelta
 from typing import Any
 
-from apify_client import ApifyClientAsync, HttpClientAsync, HttpResponse
+from apify_client import ApifyClientAsync, HttpClientAsync, HttpResponse, Timeout
 
 TOKEN = 'MY-APIFY-TOKEN'
 
@@ -19,7 +18,7 @@ async def call(
         data: str | bytes | bytearray | None = None,
         json: Any = None,
         stream: bool | None = None,
-        timeout: timedelta | None = None,
+        timeout: Timeout = 'medium',
     ) -> HttpResponse: ...
 
 

docs/02_concepts/code/10_plugging_in_sync.py

@@ -1,7 +1,6 @@
-from datetime import timedelta
 from typing import Any
 
-from apify_client import ApifyClient, HttpClient, HttpResponse
+from apify_client import ApifyClient, HttpClient, HttpResponse, Timeout
 
 TOKEN = 'MY-APIFY-TOKEN'
 
@@ -19,7 +18,7 @@ def call(
         data: str | bytes | bytearray | None = None,
         json: Any = None,
         stream: bool | None = None,
-        timeout: timedelta | None = None,
+        timeout: Timeout = 'medium',
     ) -> HttpResponse: ...
 
 

docs/03_guides/code/05_custom_http_client_async.py

@@ -1,14 +1,11 @@
 from __future__ import annotations
 
 import asyncio
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import httpx
 
-from apify_client import ApifyClientAsync, HttpClientAsync, HttpResponse
-
-if TYPE_CHECKING:
-    from datetime import timedelta
+from apify_client import ApifyClientAsync, HttpClientAsync, HttpResponse, Timeout
 
 TOKEN = 'MY-APIFY-TOKEN'
 
@@ -30,9 +27,9 @@ async def call(
         data: str | bytes | bytearray | None = None,
         json: Any = None,
         stream: bool | None = None,
-        timeout: timedelta | None = None,
+        timeout: Timeout = 'medium',
     ) -> HttpResponse:
-        timeout_secs = timeout.total_seconds() if timeout else 0
+        timeout_secs = self._compute_timeout(timeout, attempt=1) or 0
 
         # httpx.Response satisfies the HttpResponse protocol,
         # so it can be returned directly.

docs/03_guides/code/05_custom_http_client_sync.py

@@ -1,13 +1,10 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import httpx
 
-from apify_client import ApifyClient, HttpClient, HttpResponse
-
-if TYPE_CHECKING:
-    from datetime import timedelta
+from apify_client import ApifyClient, HttpClient, HttpResponse, Timeout
 
 TOKEN = 'MY-APIFY-TOKEN'
 
@@ -29,9 +26,9 @@ def call(
         data: str | bytes | bytearray | None = None,
         json: Any = None,
         stream: bool | None = None,
-        timeout: timedelta | None = None,
+        timeout: Timeout = 'medium',
     ) -> HttpResponse:
-        timeout_secs = timeout.total_seconds() if timeout else 0
+        timeout_secs = self._compute_timeout(timeout, attempt=1) or 0
 
         # httpx.Response satisfies the HttpResponse protocol,
         # so it can be returned directly.

src/apify_client/__init__.py

@@ -8,6 +8,7 @@
     ImpitHttpClient,
     ImpitHttpClientAsync,
 )
+from ._types import Timeout
 
 __version__ = metadata.version('apify-client')
 
@@ -19,5 +20,6 @@
     'HttpResponse',
     'ImpitHttpClient',
     'ImpitHttpClientAsync',
+    'Timeout',
     '__version__',
 ]

src/apify_client/_apify_client.py

@@ -9,7 +9,10 @@
     DEFAULT_API_URL,
     DEFAULT_MAX_RETRIES,
     DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
-    DEFAULT_TIMEOUT,
+    DEFAULT_TIMEOUT_LONG,
+    DEFAULT_TIMEOUT_MAX,
+    DEFAULT_TIMEOUT_MEDIUM,
+    DEFAULT_TIMEOUT_SHORT,
 )
 from apify_client._docs import docs_group
 from apify_client._http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync
@@ -114,7 +117,10 @@ def __init__(
         api_public_url: str | None = DEFAULT_API_URL,
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
-        timeout: timedelta = DEFAULT_TIMEOUT,
+        timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
+        timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
+        timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
+        timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
         headers: dict[str, str] | None = None,
     ) -> None:
         """Initialize the Apify API client.
@@ -132,7 +138,10 @@ def __init__(
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries: How long will the client wait between retrying requests
                 (increases exponentially from this value).
-            timeout: The socket timeout of the HTTP requests sent to the Apify API.
+            timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
+            timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
+            timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
+            timeout_max: Maximum timeout cap for exponential timeout growth across retries.
             headers: Additional HTTP headers to include in all API requests.
         """
         # We need to do this because of mocking in tests and default mutable arguments.
@@ -191,7 +200,10 @@ def __init__(
         # Configuration for the default HTTP client (used if a custom client is not provided).
         self._max_retries = max_retries
         self._min_delay_between_retries = min_delay_between_retries
-        self._timeout = timeout
+        self._timeout_short = timeout_short
+        self._timeout_medium = timeout_medium
+        self._timeout_long = timeout_long
+        self._timeout_max = timeout_max
         self._headers = headers
 
     @classmethod
@@ -249,7 +261,10 @@ def http_client(self) -> HttpClient:
         if self._http_client is None:
             self._http_client = ImpitHttpClient(
                 token=self._token,
-                timeout=self._timeout,
+                timeout_short=self._timeout_short,
+                timeout_medium=self._timeout_medium,
+                timeout_long=self._timeout_long,
+                timeout_max=self._timeout_max,
                 max_retries=self._max_retries,
                 min_delay_between_retries=self._min_delay_between_retries,
                 statistics=self._statistics,
@@ -455,7 +470,10 @@ def __init__(
         api_public_url: str | None = DEFAULT_API_URL,
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
-        timeout: timedelta = DEFAULT_TIMEOUT,
+        timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
+        timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
+        timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
+        timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
         headers: dict[str, str] | None = None,
     ) -> None:
         """Initialize the Apify API client.
@@ -473,7 +491,10 @@ def __init__(
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries: How long will the client wait between retrying requests
                 (increases exponentially from this value).
-            timeout: The socket timeout of the HTTP requests sent to the Apify API.
+            timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
+            timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
+            timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
+            timeout_max: Maximum timeout cap for exponential timeout growth across retries.
             headers: Additional HTTP headers to include in all API requests.
         """
         # We need to do this because of mocking in tests and default mutable arguments.
@@ -532,7 +553,10 @@ def __init__(
         # Configuration for the default HTTP client (used if a custom client is not provided).
         self._max_retries = max_retries
         self._min_delay_between_retries = min_delay_between_retries
-        self._timeout = timeout
+        self._timeout_short = timeout_short
+        self._timeout_medium = timeout_medium
+        self._timeout_long = timeout_long
+        self._timeout_max = timeout_max
         self._headers = headers
 
     @classmethod
@@ -590,7 +614,10 @@ def http_client(self) -> HttpClientAsync:
         if self._http_client is None:
             self._http_client = ImpitHttpClientAsync(
                 token=self._token,
-                timeout=self._timeout,
+                timeout_short=self._timeout_short,
+                timeout_medium=self._timeout_medium,
+                timeout_long=self._timeout_long,
+                timeout_max=self._timeout_max,
                 max_retries=self._max_retries,
                 min_delay_between_retries=self._min_delay_between_retries,
                 statistics=self._statistics,