diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fa4835..7d09b98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ This project adheres to [Semantic Versioning](http://semver.org/). The format is based on [Keep a Changelog](http://keepachangelog.com/). ## Unreleased +### Changed + +- Request logs no longer contain a nested `request_response_data` object. It repeated the request/response fields + (`request_received_at`, `response_time_ms`, `response_sent_at` and any field added by a custom + `request_response_dto_class`), which are still written once at the top level of the record, as the README + documents. If you read the nested object, read the top-level fields instead, for example `response_time_ms` instead + of `request_response_data.response_time_ms`. Setting `EXCLUDED_FIELDS = ("request_response_data",)` is no longer + needed to drop it, and keeping it in `EXCLUDED_FIELDS` is harmless. A custom request formatter can still read + `record.request_response_data`, and can add the nested object back if needed. (#158, fixes #151) + ## 1.7.0rc0 - 2026-11-10 ### Added diff --git a/README.md b/README.md index 215ddd8..06291e0 100644 --- a/README.md +++ b/README.md @@ -360,6 +360,22 @@ json_logging.config_root_logger() A custom JSON log formatter can be passed to the init method. See the examples for more details: [non-web](example/custom_log_format.py), [web](example/custom_log_format_request.py). +### Request/response DTO fields in request logs + +The fields of the request/response DTO (`request_received_at`, `response_time_ms`, `response_sent_at`, and any field +a custom `request_response_dto_class` adds) are written once, at the top level of each request log. Version 1.7.0rc0 +and earlier also wrote them a second time in a nested `request_response_data` object; read the top-level fields +instead. A custom request formatter can still read `record.request_response_data`, and can add the nested object back +if a log consumer needs it: + +```python +class NestedDTORequestFormatter(json_logging.JSONRequestLogFormatter): + def _format_log_object(self, record, request_util): + json_log_object = super()._format_log_object(record, request_util) + json_log_object["request_response_data"] = dict(record.request_response_data) + return json_log_object +``` + ## 2.7 Exclude certain URLs from request instrumentation Certain URLs can be excluded from request instrumentation by passing a list of regular expressions to diff --git a/json_logging/formatters.py b/json_logging/formatters.py index 7fcab2e..eaad256 100644 --- a/json_logging/formatters.py +++ b/json_logging/formatters.py @@ -185,7 +185,14 @@ class JSONRequestLogFormatter(BaseJSONFormatter): """ def _format_log_object(self, record, request_util): + """ + Build the request log object: the common fields, the request and response fields read through the framework + adapters, and the fields of record.request_response_data, each written once at the top level + """ json_log_object = super(JSONRequestLogFormatter, self)._format_log_object(record, request_util) + # The integrations pass the DTO as extra={'request_response_data': ...}, so the extra fields hold a nested + # copy of it. Its fields are merged into the top level below, so drop the copy to write them only once. + json_log_object.pop('request_response_data', None) request_adapter = request_util.request_adapter response_adapter = request_util.response_adapter diff --git a/tests/test_connexion_async.py b/tests/test_connexion_async.py index eba1b3f..5a68374 100644 --- a/tests/test_connexion_async.py +++ b/tests/test_connexion_async.py @@ -71,6 +71,7 @@ async def excluded(): def test_request_is_logged(client_and_log_handlers): + """An AsyncApp request is logged with the request fields, and the DTO fields only at the top level""" api_client, _, request_handler = client_and_log_handlers response = api_client.get("/log/async", headers={"X-Correlation-Id": "abc-def"}) @@ -86,6 +87,9 @@ def test_request_is_logged(client_and_log_handlers): assert msg["response_status"] == 200 assert msg["response_content_type"] == "application/json" assert int(msg["response_size_b"]) == len(response.content) + # the DTO fields appear once, at the top level, without a nested copy (#151) + assert "request_response_data" not in msg + assert "request_received_at" in msg and "response_time_ms" in msg @pytest.mark.parametrize("path", ["/log/async", "/log/sync"]) diff --git a/tests/test_fastapi.py b/tests/test_fastapi.py index 40970a0..5ab983d 100644 --- a/tests/test_fastapi.py +++ b/tests/test_fastapi.py @@ -525,6 +525,29 @@ def test_request_instrumentation(client_and_log_handler): assert len(handler.messages) == 1 +def test_request_log_has_no_nested_request_response_data(client_and_log_handler): + """The request/response fields are written once, at the top level of the request log (#151)""" + from json_logging.formatters import JSONRequestLogFormatter + + api_client, _ = client_and_log_handler + request_logger = logging.getLogger("fastapi-request-logger") + handler = FormattedMessageCollectorHandler() + handler.setFormatter(JSONRequestLogFormatter()) + request_logger.addHandler(handler) + try: + response = api_client.get("/log/levels/debug") + finally: + request_logger.removeHandler(handler) + + assert response.status_code == 200 + assert len(handler.messages) == 1 + msg = json.loads(handler.messages[0]) + assert "request_response_data" not in msg + assert msg["type"] == "request" + for key in ("request_received_at", "response_time_ms", "response_sent_at"): + assert key in msg + + def test_excluded_from_request_instrumentation(client_and_log_handler): """Test if endpoints can be excluded from the request log""" api_client, _ = client_and_log_handler diff --git a/tests/test_flask.py b/tests/test_flask.py index 52aca51..057eb00 100644 --- a/tests/test_flask.py +++ b/tests/test_flask.py @@ -226,6 +226,69 @@ def test_request_instrumentation(client_and_log_handler): assert len(handler.messages) == 1 +def test_request_log_has_no_nested_request_response_data(client_and_log_handler): + """The request/response fields are written once, at the top level of the request log (#151)""" + from json_logging.formatters import JSONRequestLogFormatter + + api_client, _ = client_and_log_handler + request_logger = logging.getLogger("flask-request-logger") + handler = FormattedMessageCollectorHandler() + handler.setFormatter(JSONRequestLogFormatter()) + request_logger.addHandler(handler) + try: + response = api_client.get("/log/levels/debug") + finally: + request_logger.removeHandler(handler) + + assert response.status_code == 200 + assert len(handler.messages) == 1 + msg = json.loads(handler.messages[0]) + assert "request_response_data" not in msg + assert msg["type"] == "request" + for key in ("request_received_at", "response_time_ms", "response_sent_at"): + assert key in msg + + +def test_custom_request_response_dto_fields_are_not_duplicated(): + """Fields added by a custom request_response_dto_class are written once, at the top level (#151)""" + import json_logging + import json_logging.dto + from json_logging.formatters import JSONRequestLogFormatter + + class CustomDTO(json_logging.dto.DefaultRequestResponseDTO): + """DTO that adds one field of its own""" + + def on_request_complete(self, response): + """Add the response status code as an extra DTO field""" + super(CustomDTO, self).on_request_complete(response) + self["custom_status"] = response.status_code + + app = flask.Flask(__name__) + + @app.route("/custom-dto") + def custom_dto(): + """Endpoint whose request is logged with CustomDTO""" + return {} + + json_logging.init_flask(enable_json=True) + json_logging.init_request_instrument(app, request_response_dto_class=CustomDTO) + request_logger = logging.getLogger("flask-request-logger") + handler = FormattedMessageCollectorHandler() + handler.setFormatter(JSONRequestLogFormatter()) + request_logger.addHandler(handler) + try: + response = app.test_client().get("/custom-dto") + finally: + request_logger.removeHandler(handler) + undo_imports_from_package("json_logging") + + assert response.status_code == 200 + assert len(handler.messages) == 1 + msg = json.loads(handler.messages[0]) + assert "request_response_data" not in msg + assert msg["custom_status"] == 200 + + def test_excluded_from_request_instrumentation(client_and_log_handler): """Test if endpoints can be excluded from the request log""" api_client, _ = client_and_log_handler diff --git a/tests/test_quart.py b/tests/test_quart.py index 16cd959..6cf744d 100644 --- a/tests/test_quart.py +++ b/tests/test_quart.py @@ -79,6 +79,7 @@ def test_get_correlation_id_returns_given_header(app_and_handlers): def test_request_instrumentation_writes_request_log(app_and_handlers): + """A request log is written with the request fields, and the DTO fields only at the top level""" app, _, request_handler = app_and_handlers response = _get(app, "/log") @@ -90,6 +91,9 @@ def test_request_instrumentation_writes_request_log(app_and_handlers): assert msg["request"] == "/log" assert msg["response_status"] == 200 assert msg["type"] == "request" + # the DTO fields appear once, at the top level, without a nested copy (#151) + assert "request_response_data" not in msg + assert "request_received_at" in msg and "response_time_ms" in msg def test_excluded_url_is_not_request_logged(app_and_handlers): diff --git a/tests/test_sanic.py b/tests/test_sanic.py index 185e4e0..60f7def 100644 --- a/tests/test_sanic.py +++ b/tests/test_sanic.py @@ -70,6 +70,7 @@ def test_get_correlation_id_is_generated_when_missing(app_and_handlers): def test_request_instrumentation_writes_request_log(app_and_handlers): + """A request log is written with the request fields, and the DTO fields only at the top level""" app, _, request_handler = app_and_handlers _, response = app.test_client.get("/log") @@ -81,6 +82,9 @@ def test_request_instrumentation_writes_request_log(app_and_handlers): assert msg["request"] == "/log" assert msg["response_status"] == 200 assert msg["type"] == "request" + # the DTO fields appear once, at the top level, without a nested copy (#151) + assert "request_response_data" not in msg + assert "request_received_at" in msg and "response_time_ms" in msg def test_excluded_url_is_not_request_logged(app_and_handlers):