Echo records_format back on search responses

API.md

@@ -282,6 +282,7 @@ GET /api/3/action/datastore_search
   "success": true,
   "result": {
     "fields": [{"id": "auction_id", "type": "integer"}, "..."],
+    "records_format": "objects",
     "records": [
       {"auction_id": 144, "product_code": "DCL", "clearing_price_gbp_per_mwh": 47.82}
     ],
@@ -295,8 +296,10 @@ GET /api/3/action/datastore_search
 ```
 
 - `records_format=lists` → each record is a positional array (column order = `fields`).
-- `records_format=csv` / `tsv` → `records` is a single text body (header row first),
-  still inside the JSON envelope.
+- `records_format=csv` / `tsv` → `records` is a single text body of data rows,
+  still inside the JSON envelope; column names are on `fields`, not in the text.
+- `result.records_format` echoes the format that was applied, so a client can tell
+  which `records` shape it got without re-reading its own query string.
 - Paginate by following `_links.next`; end-of-data is an empty `records` array.
 
 ---

CLAUDE.md

@@ -680,6 +680,7 @@ GET /api/3/datastore_search
       {"id": "clearing_price_gbp_per_mwh", "type": "number"},
       {"id": "volume_mwh",                 "type": "number"}
     ],
+    "records_format": "objects",
     "records": [
       {"auction_id": 152, "product_code": "DCL", "delivery_start": "2025-11-05T18:30:00Z", "clearing_price_gbp_per_mwh": 39.40, "volume_mwh": 95.0},
       {"auction_id": 144, "product_code": "DCL", "delivery_start": "2025-11-04T16:00:00Z", "clearing_price_gbp_per_mwh": 47.82, "volume_mwh": 120.0}
@@ -699,7 +700,9 @@ non-`offset` params preserved. `start` omits `offset` (it defaults to 0);
 empty `records` array on the next page — there's no `prev` field today.
 
 `records_format=lists` returns each record as a positional array (column order matches `fields`).
-`records_format=csv` / `tsv` return a streaming text body with the header row first.
+`records_format=csv` / `tsv` return a streaming text body of data rows (no header row — column names are on `fields`).
+`result.records_format` echoes back the format that was applied (always `objects` for
+`datastore_search_sql`), so a client can tell which `records` shape it got.
 
 ### 6.3 `POST /api/3/datastore_upsert`
 
@@ -793,6 +796,7 @@ GET /api/3/datastore_search_sql?sql=
       {"id": "avg_price",     "type": "number"},
       {"id": "total_volume",  "type": "number"}
     ],
+    "records_format": "objects",
     "records": [
       {"delivery_date": "2025-11-05", "product_code": "DCL", "avg_price": 41.20, "total_volume": 1840.0},
       {"delivery_date": "2025-11-05", "product_code": "DCH", "avg_price": 49.75, "total_volume":  720.5},

datastore/schemas/responses.py

@@ -160,6 +160,7 @@ class Result(BaseModel):
             list[dict[str, Any]],
             Field(deprecated="use 'schema' (Frictionless Table Schema) instead"),
         ]
+        records_format: str = "objects"
         records: list[dict[str, Any]]
         limit: int
         offset: int

datastore/services/streaming.py

@@ -84,6 +84,7 @@ def stream_objects(
         resource_id=resource_id,
         schema=schema,
         fields=fields,
+        records_format="objects",
         records_chunks=_records_object_array(columns, records),
         limit=limit,
         offset=offset,
@@ -116,6 +117,7 @@ def stream_lists(
         resource_id=resource_id,
         schema=schema,
         fields=fields,
+        records_format="lists",
         records_chunks=_records_array_array(records),
         limit=limit,
         offset=offset,
@@ -149,6 +151,7 @@ def stream_csv(
         resource_id=resource_id,
         schema=schema,
         fields=fields,
+        records_format="csv",
         records_chunks=_records_delimited_string(columns, records, delimiter=","),
         limit=limit,
         offset=offset,
@@ -182,6 +185,7 @@ def stream_tsv(
         resource_id=resource_id,
         schema=schema,
         fields=fields,
+        records_format="tsv",
         records_chunks=_records_delimited_string(columns, records, delimiter="\t"),
         limit=limit,
         offset=offset,
@@ -199,6 +203,7 @@ def _stream_envelope(
     resource_id: str,
     schema: dict[str, Any],
     fields: list[dict[str, Any]],
+    records_format: str,
     records_chunks: Iterator[bytes],
     limit: int,
     offset: int,
@@ -214,6 +219,9 @@ def _stream_envelope(
 
     Column metadata is emitted in both shapes: `schema` (canonical
     Frictionless) and `fields` (legacy `{id, type}` list, deprecated).
+    `records_format` is echoed back next to `records` so a client can tell
+    which of the four shapes it just got without re-reading its own query
+    string (raw-SQL responses are always `objects`).
     `sql` is emitted only when supplied (i.e. for `datastore_search_sql`);
     `datastore_search` leaves it out. `warnings` (deprecated-input notices)
     is emitted at the envelope level — a sibling of `result`, matching
@@ -230,6 +238,8 @@ def _stream_envelope(
     yield orjson.dumps(schema)
     yield b',"fields":'
     yield orjson.dumps(fields)
+    yield b',"records_format":'
+    yield orjson.dumps(records_format)
     yield b',"records":'
     yield from records_chunks
     yield b',"limit":'
@@ -279,13 +289,15 @@ def _records_array_array(records: Iterator[tuple]) -> Iterator[bytes]:
 def _records_delimited_string(
     columns: list[str], records: Iterator[tuple], *, delimiter: str
 ) -> Iterator[bytes]:
-    """`"col1,col2\\nv1,v2\\n..."` — one JSON string containing CSV / TSV text.
+    """`"v1,v2\\nv3,v4\\n..."` — one JSON string containing CSV / TSV text.
+
+    Data rows only — column names are on `result.fields`, not repeated as
+    a header inside the string.
 
     Yields:
       1. `"`            — opening quote of the JSON string value
-      2. header row     — `csv.writer`-encoded then JSON-escaped
-      3. data rows      — same per row
-      4. `"`            — closing quote
+      2. data rows      — `csv.writer`-encoded then JSON-escaped, per row
+      3. `"`            — closing quote
     """
     yield b'"'
     for row in records:

example_payload/datastore_search/records_format_csv.json

@@ -0,0 +1,6 @@
+{
+  "resource_id": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09",
+  "fields": "auction_id,product_code,clearing_price_gbp_per_mwh",
+  "records_format": "csv",
+  "limit": "100"
+}

example_payload/datastore_search/records_format_lists.json

@@ -0,0 +1,6 @@
+{
+  "resource_id": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09",
+  "fields": "auction_id,product_code,clearing_price_gbp_per_mwh",
+  "records_format": "lists",
+  "limit": "100"
+}

postman/collection.json

@@ -1,6 +1,6 @@
 {
   "info": {
-    "_postman_id": "bba58be7-cd6a-4fc5-bbdf-11830550b9df",
+    "_postman_id": "2d4510dd-5f60-4f1d-860c-2da4b7b394fa",
     "name": "Datastore API",
     "description": "CKAN-compatible datastore API \u2014 auto-generated from `example_payload/`. Set `baseUrl` to your server, `apiKey` to a CKAN API key (anonymous reads are allowed; writes require a key), and `resourceId` to the table you want to hit.",
     "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
@@ -544,6 +544,84 @@
             "description": "Custom projection (CSV), multi-column sort, explicit limit / offset / include_total. Drives `_links.next`."
           },
           "response": []
+        },
+        {
+          "name": "Search - records_format=lists",
+          "request": {
+            "method": "GET",
+            "header": [],
+            "url": {
+              "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=lists&limit=100",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "3",
+                "action",
+                "datastore_search"
+              ],
+              "query": [
+                {
+                  "key": "resource_id",
+                  "value": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09"
+                },
+                {
+                  "key": "fields",
+                  "value": "auction_id,product_code,clearing_price_gbp_per_mwh"
+                },
+                {
+                  "key": "records_format",
+                  "value": "lists"
+                },
+                {
+                  "key": "limit",
+                  "value": "100"
+                }
+              ]
+            },
+            "description": "Each record is a positional array in `fields` order \u2014 smaller payload than `objects`. `result.records_format` echoes `lists`."
+          },
+          "response": []
+        },
+        {
+          "name": "Search - records_format=csv",
+          "request": {
+            "method": "GET",
+            "header": [],
+            "url": {
+              "raw": "{{baseUrl}}/api/3/action/datastore_search?resource_id=7a10def4-8e95-46f9-96c7-9f61bdfd1a09&fields=auction_id,product_code,clearing_price_gbp_per_mwh&records_format=csv&limit=100",
+              "host": [
+                "{{baseUrl}}"
+              ],
+              "path": [
+                "api",
+                "3",
+                "action",
+                "datastore_search"
+              ],
+              "query": [
+                {
+                  "key": "resource_id",
+                  "value": "7a10def4-8e95-46f9-96c7-9f61bdfd1a09"
+                },
+                {
+                  "key": "fields",
+                  "value": "auction_id,product_code,clearing_price_gbp_per_mwh"
+                },
+                {
+                  "key": "records_format",
+                  "value": "csv"
+                },
+                {
+                  "key": "limit",
+                  "value": "100"
+                }
+              ]
+            },
+            "description": "`records` is a single CSV string of data rows inside the usual JSON envelope (column names live on `result.fields`, not in the string); Content-Type stays `application/json`. `tsv` is the same with tabs."
+          },
+          "response": []
         }
       ]
     },

postman/generate_postman.py

@@ -192,6 +192,18 @@ def _get_request(action: str, body: dict[str, Any], description: str) -> dict[st
         "Custom projection (CSV), multi-column sort, explicit limit / "
         "offset / include_total. Drives `_links.next`.",
     ),
+    "datastore_search/records_format_lists": (
+        "Search - records_format=lists",
+        "Each record is a positional array in `fields` order — smaller "
+        "payload than `objects`. `result.records_format` echoes `lists`.",
+    ),
+    "datastore_search/records_format_csv": (
+        "Search - records_format=csv",
+        "`records` is a single CSV string of data rows inside the usual "
+        "JSON envelope (column names live on `result.fields`, not in the "
+        "string); Content-Type stays `application/json`. `tsv` is the "
+        "same with tabs.",
+    ),
     "datastore_search_sql/basic": (
         "SQL - basic SELECT",
         "Plain SELECT with WHERE + LIMIT. Total comes from "
@@ -259,6 +271,7 @@ def _get_request(action: str, body: dict[str, Any], description: str) -> dict[st
     ],
     "datastore_search": [
         "basic", "with_filters", "with_full_text", "paginated_sorted",
+        "records_format_lists", "records_format_csv",
     ],
     "datastore_search_sql": [
         "basic", "aggregate", "with_cte",

tests/test_datastore_search.py

@@ -274,6 +274,7 @@ def test_default_records_format_is_json_objects(client: TestClient) -> None:
     body = response.json()
     assert body["success"] is True
     assert body["result"]["records"] == []
+    assert body["result"]["records_format"] == "objects"
 
 
 def test_records_format_lists_returns_json_envelope(client: TestClient) -> None:
@@ -285,6 +286,7 @@ def test_records_format_lists_returns_json_envelope(client: TestClient) -> None:
     body = response.json()
     assert body["success"] is True
     assert body["result"]["records"] == []
+    assert body["result"]["records_format"] == "lists"
 
 
 def test_records_format_csv_returns_json_envelope(client: TestClient) -> None:
@@ -306,6 +308,7 @@ def test_records_format_csv_returns_json_envelope(client: TestClient) -> None:
     assert body["success"] is True
     # Placeholder engine yields no rows → empty records string.
     assert body["result"]["records"] == ""
+    assert body["result"]["records_format"] == "csv"
 
 
 def test_records_format_tsv_returns_json_envelope(client: TestClient) -> None:
@@ -323,6 +326,7 @@ def test_records_format_tsv_returns_json_envelope(client: TestClient) -> None:
     body = response.json()
     # Placeholder engine yields no rows → empty records string.
     assert body["result"]["records"] == ""
+    assert body["result"]["records_format"] == "tsv"
 
 
 def test_invalid_records_format_returns_validation_error(client: TestClient) -> None:

tests/test_datastore_search_sql.py

@@ -38,6 +38,8 @@ def test_basic_sql_succeeds(client: TestClient) -> None:
     body = response.json()
     assert body["success"] is True
     assert body["result"]["records"] == []  # placeholder yields nothing
+    # Raw SQL always streams the objects shape.
+    assert body["result"]["records_format"] == "objects"
 
 
 def test_with_cte_succeeds(client: TestClient) -> None: