From 5b58e35b1dd32ca4c331758fdaa7501c4fdcc395 Mon Sep 17 00:00:00 2001 From: Han Ming <249250047+hanming5@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:42:32 +0000 Subject: [PATCH 1/3] fix: drop nested request_response_data copy from request logs The integrations log the request with extra={'request_response_data': dto}. BaseJSONFormatter._get_extra_fields() copies every non built-in record attribute, and the DTO is a dict, so it was added as a nested object. JSONRequestLogFormatter then merged the same DTO into the top level, so request_received_at, response_time_ms, response_sent_at and any field of a custom request_response_dto_class were written twice. Remove the nested copy in JSONRequestLogFormatter. The top-level fields stay as documented in the README, and record.request_response_data is still available to custom request formatters. Fixes #151 --- CHANGELOG.md | 10 +++++++ json_logging/formatters.py | 3 ++ tests/test_fastapi.py | 23 +++++++++++++++ tests/test_flask.py | 59 ++++++++++++++++++++++++++++++++++++++ tests/test_quart.py | 3 ++ tests/test_sanic.py | 3 ++ 6 files changed, 101 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fa4835..7d09b98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ This project adheres to [Semantic Versioning](http://semver.org/). The format is based on [Keep a Changelog](http://keepachangelog.com/). ## Unreleased +### Changed + +- Request logs no longer contain a nested `request_response_data` object. It repeated the request/response fields + (`request_received_at`, `response_time_ms`, `response_sent_at` and any field added by a custom + `request_response_dto_class`), which are still written once at the top level of the record, as the README + documents. If you read the nested object, read the top-level fields instead, for example `response_time_ms` instead + of `request_response_data.response_time_ms`. Setting `EXCLUDED_FIELDS = ("request_response_data",)` is no longer + needed to drop it, and keeping it in `EXCLUDED_FIELDS` is harmless. A custom request formatter can still read + `record.request_response_data`, and can add the nested object back if needed. (#158, fixes #151) + ## 1.7.0rc0 - 2026-11-10 ### Added diff --git a/json_logging/formatters.py b/json_logging/formatters.py index 7fcab2e..3cf608c 100644 --- a/json_logging/formatters.py +++ b/json_logging/formatters.py @@ -186,6 +186,9 @@ class JSONRequestLogFormatter(BaseJSONFormatter): def _format_log_object(self, record, request_util): json_log_object = super(JSONRequestLogFormatter, self)._format_log_object(record, request_util) + # The integrations pass the DTO as extra={'request_response_data': ...}, so the extra fields hold a nested + # copy of it. Its fields are merged into the top level below, so drop the copy to write them only once. + json_log_object.pop('request_response_data', None) request_adapter = request_util.request_adapter response_adapter = request_util.response_adapter diff --git a/tests/test_fastapi.py b/tests/test_fastapi.py index 40970a0..5ab983d 100644 --- a/tests/test_fastapi.py +++ b/tests/test_fastapi.py @@ -525,6 +525,29 @@ def test_request_instrumentation(client_and_log_handler): assert len(handler.messages) == 1 +def test_request_log_has_no_nested_request_response_data(client_and_log_handler): + """The request/response fields are written once, at the top level of the request log (#151)""" + from json_logging.formatters import JSONRequestLogFormatter + + api_client, _ = client_and_log_handler + request_logger = logging.getLogger("fastapi-request-logger") + handler = FormattedMessageCollectorHandler() + handler.setFormatter(JSONRequestLogFormatter()) + request_logger.addHandler(handler) + try: + response = api_client.get("/log/levels/debug") + finally: + request_logger.removeHandler(handler) + + assert response.status_code == 200 + assert len(handler.messages) == 1 + msg = json.loads(handler.messages[0]) + assert "request_response_data" not in msg + assert msg["type"] == "request" + for key in ("request_received_at", "response_time_ms", "response_sent_at"): + assert key in msg + + def test_excluded_from_request_instrumentation(client_and_log_handler): """Test if endpoints can be excluded from the request log""" api_client, _ = client_and_log_handler diff --git a/tests/test_flask.py b/tests/test_flask.py index 52aca51..47057a4 100644 --- a/tests/test_flask.py +++ b/tests/test_flask.py @@ -226,6 +226,65 @@ def test_request_instrumentation(client_and_log_handler): assert len(handler.messages) == 1 +def test_request_log_has_no_nested_request_response_data(client_and_log_handler): + """The request/response fields are written once, at the top level of the request log (#151)""" + from json_logging.formatters import JSONRequestLogFormatter + + api_client, _ = client_and_log_handler + request_logger = logging.getLogger("flask-request-logger") + handler = FormattedMessageCollectorHandler() + handler.setFormatter(JSONRequestLogFormatter()) + request_logger.addHandler(handler) + try: + response = api_client.get("/log/levels/debug") + finally: + request_logger.removeHandler(handler) + + assert response.status_code == 200 + assert len(handler.messages) == 1 + msg = json.loads(handler.messages[0]) + assert "request_response_data" not in msg + assert msg["type"] == "request" + for key in ("request_received_at", "response_time_ms", "response_sent_at"): + assert key in msg + + +def test_custom_request_response_dto_fields_are_not_duplicated(): + """Fields added by a custom request_response_dto_class are written once, at the top level (#151)""" + import json_logging + import json_logging.dto + from json_logging.formatters import JSONRequestLogFormatter + + class CustomDTO(json_logging.dto.DefaultRequestResponseDTO): + def on_request_complete(self, response): + super(CustomDTO, self).on_request_complete(response) + self["custom_status"] = response.status_code + + app = flask.Flask(__name__) + + @app.route("/custom-dto") + def custom_dto(): + return {} + + json_logging.init_flask(enable_json=True) + json_logging.init_request_instrument(app, request_response_dto_class=CustomDTO) + request_logger = logging.getLogger("flask-request-logger") + handler = FormattedMessageCollectorHandler() + handler.setFormatter(JSONRequestLogFormatter()) + request_logger.addHandler(handler) + try: + response = app.test_client().get("/custom-dto") + finally: + request_logger.removeHandler(handler) + undo_imports_from_package("json_logging") + + assert response.status_code == 200 + assert len(handler.messages) == 1 + msg = json.loads(handler.messages[0]) + assert "request_response_data" not in msg + assert msg["custom_status"] == 200 + + def test_excluded_from_request_instrumentation(client_and_log_handler): """Test if endpoints can be excluded from the request log""" api_client, _ = client_and_log_handler diff --git a/tests/test_quart.py b/tests/test_quart.py index 16cd959..231e7c1 100644 --- a/tests/test_quart.py +++ b/tests/test_quart.py @@ -90,6 +90,9 @@ def test_request_instrumentation_writes_request_log(app_and_handlers): assert msg["request"] == "/log" assert msg["response_status"] == 200 assert msg["type"] == "request" + # the DTO fields appear once, at the top level, without a nested copy (#151) + assert "request_response_data" not in msg + assert "request_received_at" in msg and "response_time_ms" in msg def test_excluded_url_is_not_request_logged(app_and_handlers): diff --git a/tests/test_sanic.py b/tests/test_sanic.py index 185e4e0..e92ca67 100644 --- a/tests/test_sanic.py +++ b/tests/test_sanic.py @@ -81,6 +81,9 @@ def test_request_instrumentation_writes_request_log(app_and_handlers): assert msg["request"] == "/log" assert msg["response_status"] == 200 assert msg["type"] == "request" + # the DTO fields appear once, at the top level, without a nested copy (#151) + assert "request_response_data" not in msg + assert "request_received_at" in msg and "response_time_ms" in msg def test_excluded_url_is_not_request_logged(app_and_handlers): From 5491c7650fea4bfe01fb7db167557e3240a53521 Mon Sep 17 00:00:00 2001 From: Han Ming <249250047+hanming5@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:47:11 +0000 Subject: [PATCH 2/3] docs, test: document the request log change and cover Connexion AsyncApp - README: explain that DTO fields are written once at the top level, that 1.7.0rc0 and earlier also wrote a nested request_response_data copy, and how a custom request formatter can add the nested object back. - CHANGELOG: list the change under Unreleased as a behaviour change, with the migration note. - Assert in the Connexion AsyncApp request log test that there is no nested copy. - Add the docstrings flagged by the review bot. --- README.md | 16 ++++++++++++++++ json_logging/formatters.py | 4 ++++ tests/test_connexion_async.py | 3 +++ tests/test_flask.py | 4 ++++ 4 files changed, 27 insertions(+) diff --git a/README.md b/README.md index 215ddd8..06291e0 100644 --- a/README.md +++ b/README.md @@ -360,6 +360,22 @@ json_logging.config_root_logger() A custom JSON log formatter can be passed to the init method. See the examples for more details: [non-web](example/custom_log_format.py), [web](example/custom_log_format_request.py). +### Request/response DTO fields in request logs + +The fields of the request/response DTO (`request_received_at`, `response_time_ms`, `response_sent_at`, and any field +a custom `request_response_dto_class` adds) are written once, at the top level of each request log. Version 1.7.0rc0 +and earlier also wrote them a second time in a nested `request_response_data` object; read the top-level fields +instead. A custom request formatter can still read `record.request_response_data`, and can add the nested object back +if a log consumer needs it: + +```python +class NestedDTORequestFormatter(json_logging.JSONRequestLogFormatter): + def _format_log_object(self, record, request_util): + json_log_object = super()._format_log_object(record, request_util) + json_log_object["request_response_data"] = dict(record.request_response_data) + return json_log_object +``` + ## 2.7 Exclude certain URLs from request instrumentation Certain URLs can be excluded from request instrumentation by passing a list of regular expressions to diff --git a/json_logging/formatters.py b/json_logging/formatters.py index 3cf608c..eaad256 100644 --- a/json_logging/formatters.py +++ b/json_logging/formatters.py @@ -185,6 +185,10 @@ class JSONRequestLogFormatter(BaseJSONFormatter): """ def _format_log_object(self, record, request_util): + """ + Build the request log object: the common fields, the request and response fields read through the framework + adapters, and the fields of record.request_response_data, each written once at the top level + """ json_log_object = super(JSONRequestLogFormatter, self)._format_log_object(record, request_util) # The integrations pass the DTO as extra={'request_response_data': ...}, so the extra fields hold a nested # copy of it. Its fields are merged into the top level below, so drop the copy to write them only once. diff --git a/tests/test_connexion_async.py b/tests/test_connexion_async.py index eba1b3f..dbb5e8b 100644 --- a/tests/test_connexion_async.py +++ b/tests/test_connexion_async.py @@ -86,6 +86,9 @@ def test_request_is_logged(client_and_log_handlers): assert msg["response_status"] == 200 assert msg["response_content_type"] == "application/json" assert int(msg["response_size_b"]) == len(response.content) + # the DTO fields appear once, at the top level, without a nested copy (#151) + assert "request_response_data" not in msg + assert "request_received_at" in msg and "response_time_ms" in msg @pytest.mark.parametrize("path", ["/log/async", "/log/sync"]) diff --git a/tests/test_flask.py b/tests/test_flask.py index 47057a4..057eb00 100644 --- a/tests/test_flask.py +++ b/tests/test_flask.py @@ -256,7 +256,10 @@ def test_custom_request_response_dto_fields_are_not_duplicated(): from json_logging.formatters import JSONRequestLogFormatter class CustomDTO(json_logging.dto.DefaultRequestResponseDTO): + """DTO that adds one field of its own""" + def on_request_complete(self, response): + """Add the response status code as an extra DTO field""" super(CustomDTO, self).on_request_complete(response) self["custom_status"] = response.status_code @@ -264,6 +267,7 @@ def on_request_complete(self, response): @app.route("/custom-dto") def custom_dto(): + """Endpoint whose request is logged with CustomDTO""" return {} json_logging.init_flask(enable_json=True) From b8a7d2f77d98e29214d5245ff9bf633e4817c767 Mon Sep 17 00:00:00 2001 From: Han Ming <249250047+hanming5@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:52:49 +0000 Subject: [PATCH 3/3] test: add docstrings to the request log tests touched by this change --- tests/test_connexion_async.py | 1 + tests/test_quart.py | 1 + tests/test_sanic.py | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/test_connexion_async.py b/tests/test_connexion_async.py index dbb5e8b..5a68374 100644 --- a/tests/test_connexion_async.py +++ b/tests/test_connexion_async.py @@ -71,6 +71,7 @@ async def excluded(): def test_request_is_logged(client_and_log_handlers): + """An AsyncApp request is logged with the request fields, and the DTO fields only at the top level""" api_client, _, request_handler = client_and_log_handlers response = api_client.get("/log/async", headers={"X-Correlation-Id": "abc-def"}) diff --git a/tests/test_quart.py b/tests/test_quart.py index 231e7c1..6cf744d 100644 --- a/tests/test_quart.py +++ b/tests/test_quart.py @@ -79,6 +79,7 @@ def test_get_correlation_id_returns_given_header(app_and_handlers): def test_request_instrumentation_writes_request_log(app_and_handlers): + """A request log is written with the request fields, and the DTO fields only at the top level""" app, _, request_handler = app_and_handlers response = _get(app, "/log") diff --git a/tests/test_sanic.py b/tests/test_sanic.py index e92ca67..60f7def 100644 --- a/tests/test_sanic.py +++ b/tests/test_sanic.py @@ -70,6 +70,7 @@ def test_get_correlation_id_is_generated_when_missing(app_and_handlers): def test_request_instrumentation_writes_request_log(app_and_handlers): + """A request log is written with the request fields, and the DTO fields only at the top level""" app, _, request_handler = app_and_handlers _, response = app.test_client.get("/log")