diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f71369..a240d22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `AsyncInferenceExecution.status_json()` and `result()` now use the client `timeout_seconds` and raise `InferenceClientError` on an HTTP error status. - The `InferenceClient` methods `get`, `post`, `put`, `delete`, `patch`, `head`, `options` and `health` now use the client `timeout_seconds` when you do not give `timeout_seconds`. Before, these calls had no timeout. To wait longer, set `timeout_seconds` on the client or on the call. +- `InferenceClient.run_sync`, `InferenceClient.run`, `Deployment.run_sync` and `Deployment.run` now use the client `timeout_seconds` when you do not give `timeout_seconds`. Before, they always used 5 minutes, even when the client was created with a different `timeout_seconds`. - `HTTPClient` methods ignore the `allow_redirects` keyword argument. - An `InferenceClientError` for an HTTP error status now ends with the response body, cut to 500 characters. For a failed async job, `result()` returns the deployment's error status, so the error now shows the deployment's message. - **Breaking:** `InferenceClient` no longer accepts a query string or fragment in `path`. Pass query data as `params` instead. For example, replace `client.get('v1/models?limit=10')` with `client.get('v1/models', params={'limit': 10})`, and `client.run_sync(data, path='v1/completions?stream=true')` with `client.run_sync(data, path='v1/completions', params={'stream': 'true'})`. diff --git a/tests/unit_tests/inference_client/test_inference_client.py b/tests/unit_tests/inference_client/test_inference_client.py index 2fcfb42..323fae6 100644 --- a/tests/unit_tests/inference_client/test_inference_client.py +++ b/tests/unit_tests/inference_client/test_inference_client.py @@ -35,6 +35,7 @@ REDIRECT_URL = 'https://inference.example.com/v1/moved' MOVED_URL = f'{BASE_URL}/moved' EXECUTION_ID = 'exec-1' +CLIENT_TIMEOUT_SECONDS = 900 REDIRECT_STATUSES = [301, 302, 303, 307, 308] @@ -770,18 +771,6 @@ def test_request_empty_error_body_failed(self, inference_client): # assert assert str(excinfo.value).endswith(f'for url: {PREDICT_URL}') - @pytest.mark.parametrize('entry_point', ['status_json', 'result']) - def test_async_requests_client_timeout_successful(self, inference_client, entry_point): - # arrange - _, url, call = ENTRY_POINTS[entry_point] - responses.add(responses.GET, url, json={'Status': 'Completed'}) - - # act - call(inference_client) - - # assert - assert responses.calls[0].request.req_kwargs['timeout'] == inference_client.timeout_seconds - @pytest.mark.parametrize('entry_point', ['status_json', 'result']) def test_async_requests_failed(self, inference_client, entry_point): # arrange @@ -817,26 +806,41 @@ def test_http_method_helpers_successful(self, inference_client, entry_point): assert response.status_code == 200 assert len(responses.calls) == 1 - @pytest.mark.parametrize( - 'entry_point', ['get', 'post', 'put', 'delete', 'patch', 'head', 'options', 'health'] - ) - def test_http_method_helpers_client_timeout_successful(self, inference_client, entry_point): + @pytest.mark.parametrize('entry_point', list(ENTRY_POINTS)) + def test_client_timeout_successful(self, entry_point): # arrange + client = InferenceClient( + inference_key='key-123', + endpoint_base_url=BASE_URL, + timeout_seconds=CLIENT_TIMEOUT_SECONDS, + ) method, url, call = ENTRY_POINTS[entry_point] - responses.add(method, url, status=200) + # A HEAD response has no body. + body = None if method == 'HEAD' else {'Id': EXECUTION_ID, 'Status': 'Completed'} + responses.add(method, url, json=body) # act - call(inference_client) + call(client) # assert - assert responses.calls[0].request.req_kwargs['timeout'] == inference_client.timeout_seconds + assert responses.calls[0].request.req_kwargs['timeout'] == CLIENT_TIMEOUT_SECONDS - def test_http_method_helpers_explicit_timeout_successful(self, inference_client): + @pytest.mark.parametrize( + 'call', + [ + lambda c: c.get('predict', timeout_seconds=7), + lambda c: c.run_sync({'a': 1}, path='predict', timeout_seconds=7), + lambda c: c.run({'a': 1}, path='predict', timeout_seconds=7), + ], + ids=['get', 'run_sync', 'run'], + ) + def test_explicit_timeout_successful(self, inference_client, call): # arrange + responses.add(responses.POST, PREDICT_URL, json={'Id': EXECUTION_ID}) responses.add(responses.GET, PREDICT_URL, status=200) # act - inference_client.get('predict', timeout_seconds=7) + call(inference_client) # assert assert responses.calls[0].request.req_kwargs['timeout'] == 7 diff --git a/verda/containers/_containers.py b/verda/containers/_containers.py index a051331..e805400 100644 --- a/verda/containers/_containers.py +++ b/verda/containers/_containers.py @@ -487,7 +487,7 @@ def run_sync( self, data: dict[str, Any], path: str = '', - timeout_seconds: int = 60 * 5, + timeout_seconds: int | None = None, headers: dict[str, str] | None = None, http_method: str = 'POST', stream: bool = False, @@ -498,7 +498,8 @@ def run_sync( Args: data: The data to send in the request. path: The endpoint path to send the request to. - timeout_seconds: Maximum time to wait for the response. + timeout_seconds: Maximum time to wait for the response. Defaults to the + inference client ``timeout_seconds``. headers: Optional headers to include in the request. http_method: The HTTP method to use for the request. stream: Whether to stream the response. @@ -519,7 +520,7 @@ def run( self, data: dict[str, Any], path: str = '', - timeout_seconds: int = 60 * 5, + timeout_seconds: int | None = None, headers: dict[str, str] | None = None, http_method: str = 'POST', stream: bool = False, @@ -530,7 +531,8 @@ def run( Args: data: The data to send in the request. path: The endpoint path to send the request to. - timeout_seconds: Maximum time to wait for the response. + timeout_seconds: Maximum time to wait for the response. Defaults to the + inference client ``timeout_seconds``. headers: Optional headers to include in the request. http_method: The HTTP method to use for the request. stream: Whether to stream the response. diff --git a/verda/inference_client/_inference_client.py b/verda/inference_client/_inference_client.py index 872439b..e1b7ea6 100644 --- a/verda/inference_client/_inference_client.py +++ b/verda/inference_client/_inference_client.py @@ -524,7 +524,7 @@ def run_sync( self, data: dict[str, Any], path: str = '', - timeout_seconds: int = 60 * 5, + timeout_seconds: int | None = None, headers: dict[str, str] | None = None, http_method: str = 'POST', stream: bool = False, @@ -535,7 +535,8 @@ def run_sync( Args: data: The data payload to send with the request path: API endpoint path. Defaults to empty string. - timeout_seconds: Request timeout in seconds. Defaults to 5 minutes. + timeout_seconds: Request timeout in seconds. Defaults to the client + ``timeout_seconds``. headers: Optional headers to include in the request http_method: HTTP method to use. Defaults to "POST". stream: Whether to stream the response. Defaults to False. @@ -568,7 +569,7 @@ def run( self, data: dict[str, Any], path: str = '', - timeout_seconds: int = 60 * 5, + timeout_seconds: int | None = None, headers: dict[str, str] | None = None, http_method: str = 'POST', no_response: bool = False, @@ -579,7 +580,8 @@ def run( Args: data: The data payload to send with the request path: API endpoint path. Defaults to empty string. - timeout_seconds: Request timeout in seconds. Defaults to 5 minutes. + timeout_seconds: Request timeout in seconds. Defaults to the client + ``timeout_seconds``. headers: Optional headers to include in the request http_method: HTTP method to use. Defaults to "POST". no_response: If True, don't wait for response. Defaults to False.