diff --git a/airbyte_cdk/sources/declarative/decoders/json_decoder.py b/airbyte_cdk/sources/declarative/decoders/json_decoder.py index 3533fc5c8..1d13452dc 100644 --- a/airbyte_cdk/sources/declarative/decoders/json_decoder.py +++ b/airbyte_cdk/sources/declarative/decoders/json_decoder.py @@ -12,6 +12,7 @@ from airbyte_cdk.sources.declarative.decoders import CompositeRawDecoder, JsonParser from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets logger = logging.getLogger("airbyte") @@ -40,7 +41,18 @@ def decode( for element in self._decoder.decode(response): yield element has_yielded = True - except Exception: + except Exception as exc: + request_method = response.request.method if response.request else "" + body = response.content or b"" + body_preview = body[:200].decode("utf-8", errors="replace") + logger.error( + filter_secrets( + "Failed to decode JSON response: " + f"method={request_method}, url={response.url}, status_code={response.status_code}, " + f"content_type={response.headers.get('Content-Type')}, body_length={len(body)}, " + f"body_preview={body_preview!r}, error={exc}" + ) + ) yield {} if not has_yielded: diff --git a/unit_tests/sources/declarative/decoders/test_json_decoder.py b/unit_tests/sources/declarative/decoders/test_json_decoder.py index 5992bf45a..4d9528054 100644 --- a/unit_tests/sources/declarative/decoders/test_json_decoder.py +++ b/unit_tests/sources/declarative/decoders/test_json_decoder.py @@ -3,12 +3,14 @@ # import gzip import json +import logging import os import pytest import requests from airbyte_cdk.sources.declarative.decoders import CompositeRawDecoder +from airbyte_cdk.sources.declarative.decoders import json_decoder as json_decoder_module from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import JsonLineParser from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder @@ -27,6 +29,79 @@ def test_json_decoder(requests_mock, response_body, first_element): assert next(JsonDecoder(parameters={}).decode(response)) == first_element +@pytest.mark.parametrize( + ("response_body", "content_type"), + [ + ("", "application/json"), + ("error", "text/html"), + ], + ids=["empty_body", "html_body"], +) +def test_json_decoder_logs_response_details_for_invalid_json( + requests_mock, caplog, response_body, content_type +): + url = "https://airbyte.io/orders?api_key=secret" + requests_mock.register_uri( + "GET", + url, + text=response_body, + status_code=200, + headers={"Content-Type": content_type}, + ) + response = requests.get(url) + + with caplog.at_level(logging.ERROR, logger="airbyte"): + assert all(element == {} for element in JsonDecoder(parameters={}).decode(response)) + + messages = [record.message for record in caplog.records] + message = next(message for message in messages if "Failed to decode JSON response" in message) + assert "method=GET" in message + assert f"url={url}" in message + assert "status_code=200" in message + assert f"content_type={content_type}" in message + assert f"body_length={len(response_body.encode())}" in message + assert f"body_preview={response_body!r}" in message + assert "Response JSON data failed to be parsed" in message + + +def test_json_decoder_does_not_log_for_empty_json_array(requests_mock, caplog): + requests_mock.register_uri( + "GET", + "https://airbyte.io/", + text="[]", + status_code=200, + headers={"Content-Type": "application/json"}, + ) + response = requests.get("https://airbyte.io/") + + with caplog.at_level(logging.ERROR, logger="airbyte"): + assert list(JsonDecoder(parameters={}).decode(response)) == [{}] + + assert not caplog.records + + +def test_json_decoder_filters_secrets_before_logging(requests_mock, caplog, monkeypatch): + url = "https://airbyte.io/orders?api_key=secret" + requests_mock.register_uri("GET", url, text="error", status_code=200) + response = requests.get(url) + monkeypatch.setattr( + json_decoder_module, + "filter_secrets", + lambda message: message.replace("secret", "****"), + ) + + with caplog.at_level(logging.ERROR, logger="airbyte"): + list(JsonDecoder(parameters={}).decode(response)) + + message = next( + record.message + for record in caplog.records + if "Failed to decode JSON response" in record.message + ) + assert "secret" not in message + assert "****" in message + + @pytest.mark.parametrize( "response_body, expected_json", [