diff --git a/.gitignore b/.gitignore index 7daa672..5d60048 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ Thumbs.db *.log *.log.[0-9] CLAUDE.md +.vscode/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f431925..9fae9db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,35 @@ All notable changes to this project are documented in this file. -## Unreleased +## 2.0.0 - 2026-10-18 +- Fixed: `ElasticsearchTransport` now actually writes [Elastic Common + Schema](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) + fields (`@timestamp`, `log.level`, `service.name`, and, when present, + `trace.id`/`span.id`/`error.message`/`error.stack_trace`) alongside + LogQuill's own fields, via a new internal `ecs_mapper` — these are what + make Kibana's default Discover view, dashboards, and alerting rules + work on this data with zero manual field mapping, and were documented + as part of this transport from early on but never actually implemented. + The document body is now always JSON, built directly from the ECS + mapping; the `formatter` constructor parameter (inherited from + `BatchingTransport`) no longer shapes it, since Elasticsearch's bulk API + requires one JSON object per line regardless of formatter. +- Docs and 2.0.0 release prep: a starter Kibana dashboard + (`docs/kibana/logquill-dashboard.ndjson`) — an index pattern, a recent-logs + saved search, and two visualizations (records by level over time, records + by service) — importable via Kibana's Saved Objects UI, built on the ECS + fix above; `MIGRATING.md` extended to cover the license change, the + module reorganization below, and every feature added since the 1.x → 2.0 + contract work, with pointers into the README rather than duplicating it; + every README code example re-verified against the current source, which + caught and fixed two real bugs: the `span(capture_state=...)` example + didn't account for `state_diff` being a `content`-classed field (off by + default since the privacy policy work), and the `diagnose=True` example + passed a secret as a literal at the call site, which Python's traceback + source-line echo reveals regardless of `redact_local` — rewritten to bind + it to a correctly-named variable first, which is also what the feature + actually protects against. - **Breaking: license change.** Starting with this release, `logquill` is source-available under [FSL-1.1-MIT](LICENSE) (the Functional Source License, MIT Future variant) instead of MIT. Free for internal use, diff --git a/MIGRATING.md b/MIGRATING.md index 9d3457f..2bd0fa1 100644 --- a/MIGRATING.md +++ b/MIGRATING.md @@ -1,16 +1,40 @@ # Migrating from logquill 1.x to 2.0 -2.0 raises the minimum Python version and extends the record shape shared with -`logquill` on npm. Application code that only calls `logger.info(...)` and -friends needs no changes; what changes is what's on disk and on the wire, and -which Python you run. - -## 1. Python 3.10 or newer +2.0 changes the license, raises the minimum Python version, and extends the +record shape shared with `logquill` on npm. Application code that only calls +`logger.info(...)` and friends through the top-level `from logquill import +...` API needs no changes; what changes is the license, which Python you run, +what's on disk and on the wire, and — if you import a submodule directly +rather than through the top-level package — where some of those submodules +now live. + +## 1. License change: MIT → FSL-1.1-MIT + +**This only affects new releases.** 1.0.0 and every earlier version already +on PyPI stay MIT forever — relicensing later code doesn't and can't change +the terms of what's already published. If you're pinned to 1.x, nothing +here affects you. + +Starting with 2.0.0, `logquill` is source-available under +[FSL-1.1-MIT](LICENSE) (the Functional Source License, MIT Future variant) +instead of MIT. In short: free for internal use, non-commercial education +and research, and professional services you provide to a licensee; a +commercial product or service that competes with `logquill` itself needs a +separate commercial license from the author. Each version converts to plain +MIT two years after that version's own release. See the +[License](README.md#license) section of the README for the full +explanation, and talk to your own legal/compliance process before upgrading +if license terms matter for how your organization consumes dependencies — +this is a real, binding change for anyone building on 2.0+, not a formality. + +## 2. Python 3.10 or newer `logquill` 2.0 requires Python 3.10+ (it was 3.8+). It is tested on 3.10 -through 3.14. On an older interpreter, pip keeps installing the 1.x line. +through 3.14, including free-threaded (PEP 703) 3.13t/3.14t builds — see +[Thread safety](README.md#thread-safety). On an older interpreter, pip keeps +installing the 1.x line. -## 2. Every record carries `schema_version` +## 3. Every record carries `schema_version` Records now start with `"schema_version": "2.0"`: @@ -45,7 +69,7 @@ LogQuill record, or one written by a newer major version. 1.x verifies unchanged, and 2.x chains additionally cover `schema_version` and `llm`, so editing either is caught. -## 3. New optional fields +## 4. New optional fields These are defined so both languages agree on them. `logger.llm_call()` writes the `llm` block, `span(capture_state=...)` writes `state_diff`, and repeated tool @@ -70,7 +94,7 @@ has them unless you use those. dashboards need stable names for it. A record that isn't an LLM call has no `llm` key. -## 4. The published schema +## 5. The published schema `schema/record.schema.json` (JSON Schema 2020-12) defines a record precisely. The top level is closed — `schema_version`, `timestamp`, `level`, `logger`, @@ -78,7 +102,7 @@ The top level is closed — `schema_version`, `timestamp`, `level`, `logger`, free-form. `schema/golden_records.json` holds example records that this package and `logquill` on npm both test against. -## 5. Small things +## 6. Small things - `LogRecord` gained `schema_version` (required) and `llm` (optional), and `create_record()` sets them. If you build `LogRecord` values by hand in your @@ -88,7 +112,7 @@ and `logquill` on npm both test against. record instead (`{**record, "meta": new_meta}`) so `schema_version` and `llm` aren't dropped. -## 6. Content capture is off by default +## 7. Content capture is off by default The fields above that can carry a prompt, a completion, or a tool's arguments/result — `input_messages`, `output_messages`, @@ -104,3 +128,61 @@ defaults to `"off"`. by exact key name — see `logquill.privacy.FIELD_CLASSES`. Existing `RedactPlugin()` calls with no arguments are unaffected: the default key set is unchanged, just now sourced from that same classification. + +## 8. Internal modules moved into themed subpackages + +**This only affects code that imports a submodule directly** — e.g. `from +logquill.mcp import inbound` or `import logquill.logger`. Every name +documented as part of the public API and reachable via `from logquill +import ...` (`Logger`, `RunPlugin`, `MetricsPlugin`, `Neo4jTransport`, and +so on — see the README) is unaffected; this section only matters if you +bypassed that and reached into a specific file. + +Twenty-two top-level modules that had grown into more than one +implementation each were grouped into five subpackages: + +| Old path | New path | +|---|---| +| `logquill.logger`, `.levels`, `.records`, `.context`, `.exceptions`, `.handler`, `.opt` | `logquill.core.*` | +| `logquill.worker`, `.shutdown`, `.toggle`, `.runtime_level`, `.serverless` | `logquill.runtime.*` | +| `logquill.span`, `.retry`, `.mcp`, `.semconv` | `logquill.tracing.*` | +| `logquill.cli`, `.webui`, `.trace_tree` (+ the `static/` assets) | `logquill.cli.*` | +| `logquill.privacy`, `.audit` | `logquill.privacy.*` | + +`logquill.config`, `logquill.parsing`, and the `logquill.formatter` +compatibility shim stayed exactly where they were — each is a single +implementation with no sibling to group it with. The packaged `logquill` +CLI command is unaffected either way (its `console_scripts` entry point +changed internally, from `logquill.cli:main` to `logquill.cli.cli:main`, +which is invisible to anyone running the installed `logquill` command). + +## 9. What else is new + +Everything below is additive — none of it requires a change to existing +code, only an explicit opt-in if you want it. See the README section +linked for a runnable example of each: + +- **OTel GenAI alignment & `OTLPTransport`** — spans named and attributed + per the OpenTelemetry GenAI semantic conventions; `OTelLogsTransport` for + the plain logs bridge. [LLM calls & OpenTelemetry](README.md#llm-calls--opentelemetry) +- **Auto-instrumentation & MCP** — `logquill.instrument.anthropic/.openai/.litellm` + patch a provider SDK with no call-site changes; MCP trace-context + propagation helpers. [Auto-instrumentation and MCP](README.md#auto-instrumentation-and-mcp) +- **Local-first trace viewer** — `logquill trace`, `logquill serve`, and + `logquill dev`, all offline, no account, no backend. + [CLI](README.md#cli) +- **Privacy, content policy & audit trail** — field classification, + `logquill verify `, and the `AuditLogger` preset. + [`AuditLogger`: a preset for audit trails](README.md#auditlogger-a-preset-for-audit-trails) +- **Runtime intelligence** — `FlightRecorderPlugin`, `RunSummaryPlugin`, + `AdaptiveSamplingPlugin`, and changing a logger's level without a + restart. [Runtime level control](README.md#runtime-level-control) +- **Metrics bridge & graph transports** — `MetricsPlugin` (OTel/StatsD/ + Prometheus Pushgateway backends), and `Neo4jTransport`/`FalkorDBTransport`/ + `NeptuneTransport` writing a run's span tree straight into a graph + database. [Metrics bridge](README.md#metrics-bridge), + [Graph transports](README.md#graph-transports) +- **Performance & free-threaded Python** — `JSONFormatter(backend="orjson")` + (`pip install logquill[fast]`), and every batching transport/`FileTransport`/ + `HTTPTransport` now locks around its shared buffer, tested under PEP 703 + free-threaded 3.13t/3.14t in CI. [Thread safety](README.md#thread-safety) diff --git a/README.md b/README.md index 8126cfc..0ef1ae1 100644 --- a/README.md +++ b/README.md @@ -437,6 +437,43 @@ pauses sends until it elapses — dropping (not requeuing) any batch flushed during that window, since New Relic blocks the rest of that minute on a rate-limit breach anyway. +**`ElasticsearchTransport` is where Kibana support lives** — Kibana never +ingests data itself, it only visualizes what's already in Elasticsearch. +Every document carries [Elastic Common Schema](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) +fields (`@timestamp`, `log.level`, `service.name`, and, when present, +`trace.id`/`span.id`/`error.message`/`error.stack_trace`) alongside +LogQuill's own fields — ECS is permissive about extra fields, so nothing +is lost, but the *standard* fields being named correctly is what makes +Kibana's default Discover view, dashboards, and alerting rules work with +zero manual field mapping: + +```python +import json +from logquill import ElasticsearchTransport, Logger + +class FakeSender: + def __init__(self): + self.calls = [] + def __call__(self, url, batch): + self.calls.append((url, batch)) + +sender = FakeSender() +transport = ElasticsearchTransport( + url="http://localhost:9200", index="app-logs", sender=sender, max_records=1 +) +logger = Logger("app", transports=[transport]) + +logger.error("payment failed") + +_, lines = sender.calls[0] +document = json.loads(lines[1]) # lines[0] is the bulk API's index action +assert document["message"] == "payment failed" # LogQuill's own field +assert document["log"] == {"level": "error"} # added alongside it +``` + +A starter Kibana dashboard (importable via Kibana's Saved Objects UI) built +on these fields ships under [`docs/kibana/`](docs/kibana/). + `SyslogTransport` sends each record as one RFC 5424 message over UDP (default) or TCP — stdlib `socket` only, no dependency, and not a batching transport (syslog is one-message-per-call, unlike the HTTP-API transports @@ -1220,13 +1257,16 @@ to receive it. **What changed during a step.** `span(name, capture_state=...)` calls your function on entering and leaving the block and records what differs as `meta.state_diff` — only the keys that changed, deep-copied so in-place -mutation is seen, and omitted if nothing changed: +mutation is seen, and omitted if nothing changed. `state_diff` is a +`content`-classed field (see [Content capture policy](#content-capture-policy)), +so it's redacted by default — opt in with `content_policy="full"` (or +`"hash"`/`"truncate"`) on the `Logger`: ```python from logquill import CollectingTransport, Logger sink = CollectingTransport() -logger = Logger("app.agent", transports=[sink]) +logger = Logger("app.agent", transports=[sink], content_policy="full") state = {"items": 1, "user": "ada"} with logger.span("add_item", capture_state=lambda: state): @@ -1235,8 +1275,8 @@ with logger.span("add_item", capture_state=lambda: state): assert sink.records[0]["meta"]["state_diff"] == {"before": {"items": 1}, "after": {"items": 2}} ``` -`state_diff` lands in `meta`, so `PIIRedactPlugin` sees it; `RedactPlugin` only -matches top-level keys, so don't capture secrets. +Once captured, `state_diff` lands in `meta` like any other field — `PIIRedactPlugin` +scans it, and `RedactPlugin` only matches top-level keys, so don't capture secrets. **Retries.** An `.action()` that names its tool (`tool="search"`) is tracked: if the same call — same tool, same enclosing span, same `tool_call_id` if you @@ -1623,7 +1663,8 @@ def login(user, password): try: - login("ada", "hunter2") + password = "hunter2" # e.g. from a submitted form + login("ada", password) except PermissionError: record = logger.error("login failed", diagnose=True) @@ -1632,6 +1673,14 @@ assert "hunter2" not in record["meta"]["stack"] assert "password = ***" in record["meta"]["stack"] ``` +Note what this doesn't catch: Python 3.11+ tracebacks echo the *source line* +itself (`login("ada", password)`), not just captured locals — so a secret +passed as a literal at the call site (`login("ada", "hunter2")`) would still +appear verbatim in that echoed line, since there's no variable name there for +`redact_local` to match against. Keep secrets in a named variable rather than +inline at the call site, and `diagnose=True` output genuinely has nothing left +to leak. + `LogQuillHandler` bridges stdlib `logging` calls — including from third-party libraries you don't control — into a `Logger`, so they flow through the same transports and plugins instead of needing every call site diff --git a/docs/kibana/README.md b/docs/kibana/README.md new file mode 100644 index 0000000..7c87b73 --- /dev/null +++ b/docs/kibana/README.md @@ -0,0 +1,62 @@ +# Kibana starter dashboard + +A ready-to-import Kibana dashboard for logs shipped by +[`ElasticsearchTransport`](../../README.md#sql-nosql-message-queue-and-cloud-native-transports). +It only works because that transport writes [Elastic Common +Schema](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) +fields (`@timestamp`, `log.level`, `service.name`, …) alongside LogQuill's +own — Kibana never ingests data itself, it only visualizes what's already +in Elasticsearch in a shape it recognizes. + +## What's in it + +`logquill-dashboard.ndjson` is a [Kibana saved-objects +export](https://www.elastic.co/guide/en/kibana/current/managing-saved-objects.html) +containing: + +- **An index pattern** (`logquill-*`), time field `@timestamp`. +- **A saved search** — "LogQuill - recent logs", showing `log.level`, + `service.name`, and `message`, newest first. +- **Two visualizations** — a stacked count of records per `log.level` over + time (an error-rate-at-a-glance view), and a breakdown of records per + `service.name` (LogQuill's logger name). +- **A dashboard** — "LogQuill overview" — combining all three. + +## Using it + +1. Point `ElasticsearchTransport` at an index matching `logquill-*` (the + default `index="logs"` won't match the bundled index pattern — either + name your index e.g. `logquill-app`, or edit the index pattern's title + after importing to match whatever you actually use): + + ```python + from logquill import ElasticsearchTransport, Logger + + transport = ElasticsearchTransport(url="http://localhost:9200", index="logquill-app") + logger = Logger("app", transports=[transport]) + ``` + +2. In Kibana: **Stack Management → Saved Objects → Import**, and pick + `logquill-dashboard.ndjson`. +3. Open **Dashboards → LogQuill overview**. + +## Known limitations + +- **Not tested against a live Kibana instance** — this repo has no Kibana + to import into and verify against, so the NDJSON was built by hand + against the documented saved-object schema rather than exported from a + real Kibana session. The structure matches what Kibana's own exports + look like, but if your Kibana version's import rejects something, the + visualizations are simple enough (`date_histogram`/`terms` aggregations + on a `histogram`/`pie` viz) to rebuild in a couple of minutes by hand + using the field names above. +- **Aggregating on `log.level.keyword`/`service.name.keyword`** assumes + Elasticsearch's default dynamic mapping created those `.keyword` + sub-fields (true out of the box for most setups). If your index has a + custom mapping without them, point the visualizations at whatever your + equivalent keyword field is. +- This gives you an error-rate/log-browsing view, not a span-tree + waterfall — for reconstructing one agent run's exact call tree, use + `logquill trace`/`logquill serve` (see the + [CLI](../../README.md#cli) section of the README), which read JSONL or + SQLite directly and don't need Elasticsearch/Kibana at all. diff --git a/docs/kibana/logquill-dashboard.ndjson b/docs/kibana/logquill-dashboard.ndjson new file mode 100644 index 0000000..179d833 --- /dev/null +++ b/docs/kibana/logquill-dashboard.ndjson @@ -0,0 +1,5 @@ +{"id":"logquill-ecs","type":"index-pattern","attributes":{"title":"logquill-*","timeFieldName":"@timestamp"},"references":[]} +{"id":"logquill-recent-logs","type":"search","attributes":{"title":"LogQuill - recent logs","description":"Recent LogQuill records, newest first.","columns":["log.level","service.name","message"],"sort":[["@timestamp","desc"]],"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\": \"logquill-ecs\", \"query\": {\"query\": \"\", \"language\": \"kuery\"}, \"filter\": []}"}},"references":[{"id":"logquill-ecs","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}]} +{"id":"logquill-level-over-time","type":"visualization","attributes":{"title":"LogQuill - records by level over time","visState":"{\"title\": \"LogQuill - records by level over time\", \"type\": \"histogram\", \"params\": {\"grid\": {\"categoryLines\": false}, \"categoryAxes\": [{\"id\": \"CategoryAxis-1\", \"type\": \"category\", \"position\": \"bottom\", \"show\": true, \"style\": {}, \"scale\": {\"type\": \"linear\"}, \"labels\": {\"show\": true, \"truncate\": 100}, \"title\": {}}], \"valueAxes\": [{\"id\": \"ValueAxis-1\", \"name\": \"LeftAxis-1\", \"type\": \"value\", \"position\": \"left\", \"show\": true, \"style\": {}, \"scale\": {\"type\": \"linear\", \"mode\": \"normal\"}, \"labels\": {\"show\": true, \"rotate\": 0, \"filter\": false, \"truncate\": 100}, \"title\": {\"text\": \"Count\"}}], \"seriesParams\": [{\"show\": true, \"type\": \"histogram\", \"mode\": \"stacked\", \"data\": {\"label\": \"Count\", \"id\": \"1\"}, \"valueAxis\": \"ValueAxis-1\", \"drawLinesBetweenPoints\": true, \"showCirclesOnLines\": true, \"interpolate\": \"linear\", \"lineWidth\": 2}], \"addTooltip\": true, \"addLegend\": true, \"legendPosition\": \"right\", \"times\": [], \"addTimeMarker\": false, \"labels\": {\"show\": false}, \"thresholdLine\": {\"show\": false, \"value\": 10, \"width\": 1, \"style\": \"full\", \"color\": \"#E7664C\"}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"date_histogram\", \"schema\": \"segment\", \"params\": {\"field\": \"@timestamp\", \"timeRange\": {\"from\": \"now-24h\", \"to\": \"now\"}, \"useNormalizedEsInterval\": true, \"interval\": \"auto\", \"drop_partials\": false, \"min_doc_count\": 1, \"extended_bounds\": {}}}, {\"id\": \"3\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"group\", \"params\": {\"field\": \"log.level.keyword\", \"orderBy\": \"1\", \"order\": \"desc\", \"size\": 6, \"otherBucket\": false, \"otherBucketLabel\": \"Other\", \"missingBucket\": false, \"missingBucketLabel\": \"Missing\"}}]}","uiStateJSON":"{}","description":"Stacked count of records per log.level, over time - an error-rate-at-a-glance view.","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\": \"logquill-ecs\", \"query\": {\"query\": \"\", \"language\": \"kuery\"}, \"filter\": []}"}},"references":[{"id":"logquill-ecs","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}]} +{"id":"logquill-by-service","type":"visualization","attributes":{"title":"LogQuill - records by service","visState":"{\"title\": \"LogQuill - records by service\", \"type\": \"pie\", \"params\": {\"type\": \"pie\", \"addTooltip\": true, \"addLegend\": true, \"legendPosition\": \"right\", \"isDonut\": true, \"labels\": {\"show\": false, \"values\": true, \"last_level\": true, \"truncate\": 100}}, \"aggs\": [{\"id\": \"1\", \"enabled\": true, \"type\": \"count\", \"schema\": \"metric\", \"params\": {}}, {\"id\": \"2\", \"enabled\": true, \"type\": \"terms\", \"schema\": \"segment\", \"params\": {\"field\": \"service.name.keyword\", \"orderBy\": \"1\", \"order\": \"desc\", \"size\": 10, \"otherBucket\": true, \"otherBucketLabel\": \"Other\", \"missingBucket\": false, \"missingBucketLabel\": \"Missing\"}}]}","uiStateJSON":"{}","description":"Share of records per service.name (LogQuill's logger name).","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\": \"logquill-ecs\", \"query\": {\"query\": \"\", \"language\": \"kuery\"}, \"filter\": []}"}},"references":[{"id":"logquill-ecs","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}]} +{"id":"logquill-overview","type":"dashboard","attributes":{"title":"LogQuill overview","description":"Starter dashboard for LogQuill logs shipped via ElasticsearchTransport: error rate by level over time, records by service, and a recent-logs table.","panelsJSON":"[{\"version\": \"8.8.0\", \"type\": \"visualization\", \"gridData\": {\"x\": 0, \"y\": 0, \"w\": 24, \"h\": 15, \"i\": \"1\"}, \"panelIndex\": \"1\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_1\"}, {\"version\": \"8.8.0\", \"type\": \"visualization\", \"gridData\": {\"x\": 24, \"y\": 0, \"w\": 24, \"h\": 15, \"i\": \"2\"}, \"panelIndex\": \"2\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_2\"}, {\"version\": \"8.8.0\", \"type\": \"search\", \"gridData\": {\"x\": 0, \"y\": 15, \"w\": 48, \"h\": 20, \"i\": \"3\"}, \"panelIndex\": \"3\", \"embeddableConfig\": {}, \"panelRefName\": \"panel_3\"}]","optionsJSON":"{\"useMargins\": true, \"syncColors\": false, \"hidePanelTitles\": false}","timeRestore":true,"timeTo":"now","timeFrom":"now-24h","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\": {\"query\": \"\", \"language\": \"kuery\"}, \"filter\": []}"}},"references":[{"id":"logquill-level-over-time","name":"panel_1","type":"visualization"},{"id":"logquill-by-service","name":"panel_2","type":"visualization"},{"id":"logquill-recent-logs","name":"panel_3","type":"search"}]} diff --git a/logquill/__init__.py b/logquill/__init__.py index feccc53..d108da5 100644 --- a/logquill/__init__.py +++ b/logquill/__init__.py @@ -78,7 +78,7 @@ from logquill.transports.sql.sqlite_transport import SQLiteTransport from logquill.transports.transport import CollectingTransport, Transport -__version__ = "1.0.0" +__version__ = "2.0.0" __all__ = [ "AdaptiveSamplingPlugin", diff --git a/logquill/transports/cloud/ecs_mapper.py b/logquill/transports/cloud/ecs_mapper.py new file mode 100644 index 0000000..31f130b --- /dev/null +++ b/logquill/transports/cloud/ecs_mapper.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import Any + +from logquill.core.records import LogRecord + +#: The only module that knows Elastic Common Schema (ECS) field names — +#: `ElasticsearchTransport` is the only caller, same "isolate the +#: convention behind a thin mapping layer" principle as +#: `logquill.tracing.semconv` for OTel GenAI attributes. + + +def to_ecs_document(record: LogRecord) -> dict[str, Any]: + """Returns a copy of `record` with Elastic Common Schema fields added + alongside LogQuill's own — never replacing them, since ECS is + explicitly permissive about extra fields riding alongside the + standard ones. Getting the *standard* fields named correctly is what + makes Kibana's default Discover view, dashboards, and alerting rules + work on this data with zero manual field mapping: + + - `@timestamp` from `timestamp` + - `log.level` from `level`, lowercased (ECS convention) + - `service.name` from `logger` + - `trace.id` / `span.id` from `meta.trace_id` / `meta.span_id`, when present + - `error.message` / `error.stack_trace`, when `meta.stack` is present + (from `exc_info=`/`diagnose=`) — `error.message` reuses the record's + own `message`, since LogQuill doesn't keep the exception's message + separately from the formatted traceback + """ + meta = record.get("meta", {}) + document: dict[str, Any] = dict(record) + document["@timestamp"] = record["timestamp"] + document["log"] = {"level": record["level"].lower()} + document["service"] = {"name": record["logger"]} + + trace_id = meta.get("trace_id") + if trace_id is not None: + document["trace"] = {"id": trace_id} + + span_id = meta.get("span_id") + if span_id is not None: + document["span"] = {"id": span_id} + + stack = meta.get("stack") + if isinstance(stack, str): + document["error"] = {"message": record["message"], "stack_trace": stack} + + return document diff --git a/logquill/transports/cloud/elasticsearch_transport.py b/logquill/transports/cloud/elasticsearch_transport.py index 6efe673..32a0b05 100644 --- a/logquill/transports/cloud/elasticsearch_transport.py +++ b/logquill/transports/cloud/elasticsearch_transport.py @@ -8,6 +8,7 @@ from logquill.core.records import LogRecord from logquill.formatters import Formatter from logquill.transports.batching_transport import BatchingTransport +from logquill.transports.cloud.ecs_mapper import to_ecs_document ElasticsearchSender = Callable[[str, Sequence[str]], None] @@ -34,7 +35,20 @@ class ElasticsearchTransport(BatchingTransport[LogRecord]): """Batches records and POSTs them to Elasticsearch's `_bulk` API as newline-delimited action+source pairs via stdlib `urllib` — no client dependency. Pass `sender` to inject a fake for tests or an alternate - transport.""" + transport. + + This is where Kibana support actually lives — Kibana never ingests + data itself, it only visualizes what's already in Elasticsearch. Each + document is LogQuill's own record with [Elastic Common + Schema](https://www.elastic.co/guide/en/ecs/current/ecs-reference.html) + fields added alongside it (see `logquill.transports.cloud.ecs_mapper`), + so Kibana's default Discover view, dashboards, and alerting rules work + on this data with zero manual field mapping. The document body is + always JSON, built directly from the ECS mapping rather than through + `formatter` (inherited from `BatchingTransport`) — Elasticsearch's + bulk API requires a JSON object per line, so a non-JSON formatter + configured here wouldn't produce a valid request body anyway. + """ def __init__( self, @@ -57,5 +71,6 @@ def _send_batch(self, batch: Sequence[LogRecord]) -> None: lines: list[str] = [] for record in batch: lines.append(json.dumps({"index": {"_index": self.index}}, separators=(",", ":"))) - lines.append(self.format(record)) + document = to_ecs_document(record) + lines.append(json.dumps(document, separators=(",", ":"), default=str)) self._sender(self.bulk_url, lines) diff --git a/pyproject.toml b/pyproject.toml index 07984d1..83de351 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "logquill" -version = "1.0.0" +version = "2.0.0" description = "A structured, leveled logging framework with pluggable transports and a plugin pipeline." readme = "README.md" license = "FSL-1.1-MIT" diff --git a/tests/test_transports/cloud/test_elasticsearch_transport.py b/tests/test_transports/cloud/test_elasticsearch_transport.py index 327b18c..7c949c0 100644 --- a/tests/test_transports/cloud/test_elasticsearch_transport.py +++ b/tests/test_transports/cloud/test_elasticsearch_transport.py @@ -32,3 +32,74 @@ def test_sends_ndjson_action_and_source_pairs_to_bulk_endpoint() -> None: assert action == {"index": {"_index": "app-logs"}} source = json.loads(lines[1]) assert source["message"] == "hello" + + +def test_the_document_carries_ecs_fields_alongside_logquills_own() -> None: + sender = FakeSender() + transport = ElasticsearchTransport( + url="http://localhost:9200/", index="app-logs", sender=sender, max_records=1 + ) + logger = Logger("app.test", transports=[transport]) + + logger.error("payment failed") + + _url, lines = sender.calls[0] + source = json.loads(lines[1]) + assert source["message"] == "payment failed" # LogQuill's own field, untouched + assert source["@timestamp"] == source["timestamp"] + assert source["log"] == {"level": "error"} + assert source["service"] == {"name": "app.test"} + + +def test_trace_and_span_ids_map_to_ecs_trace_and_span() -> None: + sender = FakeSender() + transport = ElasticsearchTransport(url="http://localhost:9200/", sender=sender, max_records=1) + logger = Logger("app.test", transports=[transport]) + + logger.info("hello", trace_id="trace-1", span_id="span-1") + + _url, lines = sender.calls[0] + source = json.loads(lines[1]) + assert source["trace"] == {"id": "trace-1"} + assert source["span"] == {"id": "span-1"} + + +def test_no_trace_or_span_fields_when_absent() -> None: + sender = FakeSender() + transport = ElasticsearchTransport(url="http://localhost:9200/", sender=sender, max_records=1) + logger = Logger("app.test", transports=[transport]) + + logger.info("hello") + + _url, lines = sender.calls[0] + source = json.loads(lines[1]) + assert "trace" not in source + assert "span" not in source + + +def test_a_captured_exception_maps_to_ecs_error_fields() -> None: + sender = FakeSender() + transport = ElasticsearchTransport(url="http://localhost:9200/", sender=sender, max_records=1) + logger = Logger("app.test", transports=[transport]) + + try: + raise ValueError("boom") + except ValueError as exc: + logger.error("request failed", exc_info=exc) + + _url, lines = sender.calls[0] + source = json.loads(lines[1]) + assert source["error"]["message"] == "request failed" + assert "ValueError: boom" in source["error"]["stack_trace"] + + +def test_no_error_field_without_a_captured_exception() -> None: + sender = FakeSender() + transport = ElasticsearchTransport(url="http://localhost:9200/", sender=sender, max_records=1) + logger = Logger("app.test", transports=[transport]) + + logger.info("hello") + + _url, lines = sender.calls[0] + source = json.loads(lines[1]) + assert "error" not in source