Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion airbyte_cdk/sources/declarative/decoders/json_decoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

from airbyte_cdk.sources.declarative.decoders import CompositeRawDecoder, JsonParser
from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets

logger = logging.getLogger("airbyte")

Expand Down Expand Up @@ -40,7 +41,18 @@
for element in self._decoder.decode(response):
yield element
has_yielded = True
except Exception:
except Exception as exc:
request_method = response.request.method if response.request else "<unknown>"
body = response.content or b""
body_preview = body[:200].decode("utf-8", errors="replace")
logger.error(
filter_secrets(
"Failed to decode JSON response: "
f"method={request_method}, url={response.url}, status_code={response.status_code}, "
f"content_type={response.headers.get('Content-Type')}, body_length={len(body)}, "
f"body_preview={body_preview!r}, error={exc}"
)
Comment on lines +49 to +54
)
yield {}

if not has_yielded:
Expand Down
75 changes: 75 additions & 0 deletions unit_tests/sources/declarative/decoders/test_json_decoder.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@
#
import gzip
import json
import logging
import os

import pytest
import requests

from airbyte_cdk.sources.declarative.decoders import CompositeRawDecoder
from airbyte_cdk.sources.declarative.decoders import json_decoder as json_decoder_module
from airbyte_cdk.sources.declarative.decoders.composite_raw_decoder import JsonLineParser
from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder

Expand All @@ -27,6 +29,79 @@ def test_json_decoder(requests_mock, response_body, first_element):
assert next(JsonDecoder(parameters={}).decode(response)) == first_element


@pytest.mark.parametrize(
("response_body", "content_type"),
[
("", "application/json"),
("<html>error</html>", "text/html"),
],
ids=["empty_body", "html_body"],
)
def test_json_decoder_logs_response_details_for_invalid_json(
requests_mock, caplog, response_body, content_type
):
url = "https://airbyte.io/orders?api_key=secret"
requests_mock.register_uri(
"GET",
url,
text=response_body,
status_code=200,
headers={"Content-Type": content_type},
)
response = requests.get(url)

with caplog.at_level(logging.ERROR, logger="airbyte"):
assert all(element == {} for element in JsonDecoder(parameters={}).decode(response))

messages = [record.message for record in caplog.records]
message = next(message for message in messages if "Failed to decode JSON response" in message)
assert "method=GET" in message
assert f"url={url}" in message
assert "status_code=200" in message
assert f"content_type={content_type}" in message
assert f"body_length={len(response_body.encode())}" in message
assert f"body_preview={response_body!r}" in message
assert "Response JSON data failed to be parsed" in message


def test_json_decoder_does_not_log_for_empty_json_array(requests_mock, caplog):
requests_mock.register_uri(
"GET",
"https://airbyte.io/",
text="[]",
status_code=200,
headers={"Content-Type": "application/json"},
)
response = requests.get("https://airbyte.io/")

with caplog.at_level(logging.ERROR, logger="airbyte"):
assert list(JsonDecoder(parameters={}).decode(response)) == [{}]

assert not caplog.records


def test_json_decoder_filters_secrets_before_logging(requests_mock, caplog, monkeypatch):
url = "https://airbyte.io/orders?api_key=secret"
requests_mock.register_uri("GET", url, text="<secret>error</secret>", status_code=200)
response = requests.get(url)
monkeypatch.setattr(
json_decoder_module,
"filter_secrets",
lambda message: message.replace("secret", "****"),
)

with caplog.at_level(logging.ERROR, logger="airbyte"):
list(JsonDecoder(parameters={}).decode(response))

message = next(
record.message
for record in caplog.records
if "Failed to decode JSON response" in record.message
)
assert "secret" not in message
assert "****" in message


@pytest.mark.parametrize(
"response_body, expected_json",
[
Expand Down
Loading