diff --git a/API.md b/API.md index b32e330..f6e9b3e 100644 --- a/API.md +++ b/API.md @@ -282,6 +282,7 @@ GET /api/3/action/datastore_search "success": true, "result": { "fields": [{"id": "auction_id", "type": "integer"}, "..."], + "records_format": "objects", "records": [ {"auction_id": 144, "product_code": "DCL", "clearing_price_gbp_per_mwh": 47.82} ], @@ -295,8 +296,10 @@ GET /api/3/action/datastore_search ``` - `records_format=lists` → each record is a positional array (column order = `fields`). -- `records_format=csv` / `tsv` → `records` is a single text body (header row first), - still inside the JSON envelope. +- `records_format=csv` / `tsv` → `records` is a single text body of data rows, + still inside the JSON envelope; column names are on `fields`, not in the text. +- `result.records_format` echoes the format that was applied, so a client can tell + which `records` shape it got without re-reading its own query string. - Paginate by following `_links.next`; end-of-data is an empty `records` array. --- diff --git a/CLAUDE.md b/CLAUDE.md index 62c3fc7..74853cd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -680,6 +680,7 @@ GET /api/3/datastore_search {"id": "clearing_price_gbp_per_mwh", "type": "number"}, {"id": "volume_mwh", "type": "number"} ], + "records_format": "objects", "records": [ {"auction_id": 152, "product_code": "DCL", "delivery_start": "2025-11-05T18:30:00Z", "clearing_price_gbp_per_mwh": 39.40, "volume_mwh": 95.0}, {"auction_id": 144, "product_code": "DCL", "delivery_start": "2025-11-04T16:00:00Z", "clearing_price_gbp_per_mwh": 47.82, "volume_mwh": 120.0} @@ -699,7 +700,9 @@ non-`offset` params preserved. `start` omits `offset` (it defaults to 0); empty `records` array on the next page — there's no `prev` field today. `records_format=lists` returns each record as a positional array (column order matches `fields`). -`records_format=csv` / `tsv` return a streaming text body with the header row first. +`records_format=csv` / `tsv` return a streaming text body of data rows (no header row — column names are on `fields`). +`result.records_format` echoes back the format that was applied (always `objects` for +`datastore_search_sql`), so a client can tell which `records` shape it got. ### 6.3 `POST /api/3/datastore_upsert` @@ -793,6 +796,7 @@ GET /api/3/datastore_search_sql?sql= {"id": "avg_price", "type": "number"}, {"id": "total_volume", "type": "number"} ], + "records_format": "objects", "records": [ {"delivery_date": "2025-11-05", "product_code": "DCL", "avg_price": 41.20, "total_volume": 1840.0}, {"delivery_date": "2025-11-05", "product_code": "DCH", "avg_price": 49.75, "total_volume": 720.5}, diff --git a/datastore/schemas/responses.py b/datastore/schemas/responses.py index 8bd785d..36d4d81 100644 --- a/datastore/schemas/responses.py +++ b/datastore/schemas/responses.py @@ -160,6 +160,7 @@ class Result(BaseModel): list[dict[str, Any]], Field(deprecated="use 'schema' (Frictionless Table Schema) instead"), ] + records_format: str = "objects" records: list[dict[str, Any]] limit: int offset: int diff --git a/datastore/services/streaming.py b/datastore/services/streaming.py index 4c572af..e9ec156 100644 --- a/datastore/services/streaming.py +++ b/datastore/services/streaming.py @@ -84,6 +84,7 @@ def stream_objects( resource_id=resource_id, schema=schema, fields=fields, + records_format="objects", records_chunks=_records_object_array(columns, records), limit=limit, offset=offset, @@ -116,6 +117,7 @@ def stream_lists( resource_id=resource_id, schema=schema, fields=fields, + records_format="lists", records_chunks=_records_array_array(records), limit=limit, offset=offset, @@ -149,6 +151,7 @@ def stream_csv( resource_id=resource_id, schema=schema, fields=fields, + records_format="csv", records_chunks=_records_delimited_string(columns, records, delimiter=","), limit=limit, offset=offset, @@ -182,6 +185,7 @@ def stream_tsv( resource_id=resource_id, schema=schema, fields=fields, + records_format="tsv", records_chunks=_records_delimited_string(columns, records, delimiter="\t"), limit=limit, offset=offset, @@ -199,6 +203,7 @@ def _stream_envelope( resource_id: str, schema: dict[str, Any], fields: list[dict[str, Any]], + records_format: str, records_chunks: Iterator[bytes], limit: int, offset: int, @@ -214,6 +219,9 @@ def _stream_envelope( Column metadata is emitted in both shapes: `schema` (canonical Frictionless) and `fields` (legacy `{id, type}` list, deprecated). + `records_format` is echoed back next to `records` so a client can tell + which of the four shapes it just got without re-reading its own query + string (raw-SQL responses are always `objects`). `sql` is emitted only when supplied (i.e. for `datastore_search_sql`); `datastore_search` leaves it out. `warnings` (deprecated-input notices) is emitted at the envelope level — a sibling of `result`, matching @@ -230,6 +238,8 @@ def _stream_envelope( yield orjson.dumps(schema) yield b',"fields":' yield orjson.dumps(fields) + yield b',"records_format":' + yield orjson.dumps(records_format) yield b',"records":' yield from records_chunks yield b',"limit":' @@ -279,13 +289,15 @@ def _records_array_array(records: Iterator[tuple]) -> Iterator[bytes]: def _records_delimited_string( columns: list[str], records: Iterator[tuple], *, delimiter: str ) -> Iterator[bytes]: - """`"col1,col2\\nv1,v2\\n..."` — one JSON string containing CSV / TSV text. + """`"v1,v2\\nv3,v4\\n..."` — one JSON string containing CSV / TSV text. + + Data rows only — column names are on `result.fields`, not repeated as + a header inside the string. Yields: 1. `"` — opening quote of the JSON string value - 2. header row — `csv.writer`-encoded then JSON-escaped - 3. data rows — same per row - 4. `"` — closing quote + 2. data rows — `csv.writer`-encoded then JSON-escaped, per row + 3. `"` — closing quote """ yield b'"' for row in records: diff --git a/example_payload/datastore_search/records_format_csv.json b/example_payload/datastore_search/records_format_csv.json new file mode 100644 index 0000000..00ef2da --- /dev/null +++ b/example_payload/datastore_search/records_format_csv.json @@ -0,0 +1,6 @@ +{ + "resource_id": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "fields": "auction_id,product_code,clearing_price_gbp_per_mwh", + "records_format": "csv", + "limit": "100" +} diff --git a/example_payload/datastore_search/records_format_lists.json b/example_payload/datastore_search/records_format_lists.json new file mode 100644 index 0000000..4072524 --- /dev/null +++ b/example_payload/datastore_search/records_format_lists.json @@ -0,0 +1,6 @@ +{ + "resource_id": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09", + "fields": "auction_id,product_code,clearing_price_gbp_per_mwh", + "records_format": "lists", + "limit": "100" +} diff --git a/postman/collection.json b/postman/collection.json index 692c138..d2588a4 100644 --- a/postman/collection.json +++ b/postman/collection.json @@ -1,6 +1,6 @@ { "info": { - "_postman_id": "bba58be7-cd6a-4fc5-bbdf-11830550b9df", + "_postman_id": "2d4510dd-5f60-4f1d-860c-2da4b7b394fa", "name": "Datastore API", "description": "CKAN-compatible datastore API \u2014 auto-generated from `example_payload/`. Set `baseUrl` to your server, `apiKey` to a CKAN API key (anonymous reads are allowed; writes require a key), and `resourceId` to the table you want to hit.", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" @@ -544,6 +544,84 @@ "description": "Custom projection (CSV), multi-column sort, explicit limit / offset / include_total. Drives `_links.next`." }, "response": [] + }, + { + "name": "Search - records_format=lists", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=lists&limit=100", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "3", + "action", + "datastore_search" + ], + "query": [ + { + "key": "resource_id", + "value": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09" + }, + { + "key": "fields", + "value": "auction_id,product_code,clearing_price_gbp_per_mwh" + }, + { + "key": "records_format", + "value": "lists" + }, + { + "key": "limit", + "value": "100" + } + ] + }, + "description": "Each record is a positional array in `fields` order \u2014 smaller payload than `objects`. `result.records_format` echoes `lists`." + }, + "response": [] + }, + { + "name": "Search - records_format=csv", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=csv&limit=100", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "api", + "3", + "action", + "datastore_search" + ], + "query": [ + { + "key": "resource_id", + "value": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09" + }, + { + "key": "fields", + "value": "auction_id,product_code,clearing_price_gbp_per_mwh" + }, + { + "key": "records_format", + "value": "csv" + }, + { + "key": "limit", + "value": "100" + } + ] + }, + "description": "`records` is a single CSV string of data rows inside the usual JSON envelope (column names live on `result.fields`, not in the string); Content-Type stays `application/json`. `tsv` is the same with tabs." + }, + "response": [] } ] }, diff --git a/postman/generate_postman.py b/postman/generate_postman.py index 23d2e04..7858d00 100644 --- a/postman/generate_postman.py +++ b/postman/generate_postman.py @@ -192,6 +192,18 @@ def _get_request(action: str, body: dict[str, Any], description: str) -> dict[st "Custom projection (CSV), multi-column sort, explicit limit / " "offset / include_total. Drives `_links.next`.", ), + "datastore_search/records_format_lists": ( + "Search - records_format=lists", + "Each record is a positional array in `fields` order — smaller " + "payload than `objects`. `result.records_format` echoes `lists`.", + ), + "datastore_search/records_format_csv": ( + "Search - records_format=csv", + "`records` is a single CSV string of data rows inside the usual " + "JSON envelope (column names live on `result.fields`, not in the " + "string); Content-Type stays `application/json`. `tsv` is the " + "same with tabs.", + ), "datastore_search_sql/basic": ( "SQL - basic SELECT", "Plain SELECT with WHERE + LIMIT. Total comes from " @@ -259,6 +271,7 @@ def _get_request(action: str, body: dict[str, Any], description: str) -> dict[st ], "datastore_search": [ "basic", "with_filters", "with_full_text", "paginated_sorted", + "records_format_lists", "records_format_csv", ], "datastore_search_sql": [ "basic", "aggregate", "with_cte", diff --git a/tests/test_datastore_search.py b/tests/test_datastore_search.py index 0ede89d..b6567a4 100644 --- a/tests/test_datastore_search.py +++ b/tests/test_datastore_search.py @@ -274,6 +274,7 @@ def test_default_records_format_is_json_objects(client: TestClient) -> None: body = response.json() assert body["success"] is True assert body["result"]["records"] == [] + assert body["result"]["records_format"] == "objects" def test_records_format_lists_returns_json_envelope(client: TestClient) -> None: @@ -285,6 +286,7 @@ def test_records_format_lists_returns_json_envelope(client: TestClient) -> None: body = response.json() assert body["success"] is True assert body["result"]["records"] == [] + assert body["result"]["records_format"] == "lists" def test_records_format_csv_returns_json_envelope(client: TestClient) -> None: @@ -306,6 +308,7 @@ def test_records_format_csv_returns_json_envelope(client: TestClient) -> None: assert body["success"] is True # Placeholder engine yields no rows → empty records string. assert body["result"]["records"] == "" + assert body["result"]["records_format"] == "csv" def test_records_format_tsv_returns_json_envelope(client: TestClient) -> None: @@ -323,6 +326,7 @@ def test_records_format_tsv_returns_json_envelope(client: TestClient) -> None: body = response.json() # Placeholder engine yields no rows → empty records string. assert body["result"]["records"] == "" + assert body["result"]["records_format"] == "tsv" def test_invalid_records_format_returns_validation_error(client: TestClient) -> None: diff --git a/tests/test_datastore_search_sql.py b/tests/test_datastore_search_sql.py index d21f115..ce68626 100644 --- a/tests/test_datastore_search_sql.py +++ b/tests/test_datastore_search_sql.py @@ -38,6 +38,8 @@ def test_basic_sql_succeeds(client: TestClient) -> None: body = response.json() assert body["success"] is True assert body["result"]["records"] == [] # placeholder yields nothing + # Raw SQL always streams the objects shape. + assert body["result"]["records_format"] == "objects" def test_with_cte_succeeds(client: TestClient) -> None: