diff --git a/docs/concepts/queries.md b/docs/concepts/queries.md index 9f635424..ce6c8958 100644 --- a/docs/concepts/queries.md +++ b/docs/concepts/queries.md @@ -350,6 +350,12 @@ query = HybridQuery( ) ``` +## Result ids + +For search queries, such as `VectorQuery`, `RangeQuery`, `FilterQuery` and `TextQuery`, every result's `id` is the record's full Redis key, such as `users:t1`, whatever the storage type or `return_fields`. A field the document itself names `id` never replaces it. Aggregation, hybrid and SQL results are rows of loaded fields rather than documents: they do not set `id` to the key, and a field you load under the name `id` keeps the document's value. Redis-py already drops a hash field called `id` from search results, and a JSON `FilterQuery` without `return_fields`, which returns the whole document, drops the document's own `id` in the same way. + +To read that field back, request it under another name. On JSON, either add its path to `return_fields`, as in `return_fields=["status", "$.id"]`, which returns it under the `$.id` key, or index it under an alias such as `{"name": "doc_id", "type": "tag", "path": "$.id"}` and return `doc_id`. `index.fetch()` returns the stored document unchanged, `id` field included. + ## Results and expiring documents RedisVL returns query results as a list of dictionaries, one per matched document. On Redis 8.8+ a document can occasionally come back with its field payload missing: the RediSearch worker pool now defaults to a nonzero number of background worker threads, and if a document — or one of its indexed fields — expires (via TTL or `HPEXPIRE`) at the exact moment a background search is reading it, the server returns the matched id with an empty field payload instead of dropping it from the result set. This is most likely in workloads that combine search with short TTLs, such as a semantic cache or chat history. diff --git a/redisvl/index/index.py b/redisvl/index/index.py index 6a2451fc..9222fc5b 100644 --- a/redisvl/index/index.py +++ b/redisvl/index/index.py @@ -449,8 +449,11 @@ def process_results( This function processes results from Redis, handling different storage types and query types. For JSON storage with empty return fields, it - unpacks the JSON object while retaining the document ID. The 'payload' - field is also removed from all resulting documents for consistency. + unpacks the JSON object. In every case a result's ``id`` is the record's + full Redis key; a JSON document's own ``id`` field is dropped from the + unpacked result, as redis-py already does for a hash field named ``id``. + The 'payload' field is also removed from all resulting documents for + consistency. Where it can be detected, a matched document whose field payload came back missing is silently dropped (and logged at WARNING level). On Redis 8.8+ the @@ -524,6 +527,11 @@ def _process(doc: "Document") -> Any: if isinstance(json_data, str): json_data = json.loads(json_data) if isinstance(json_data, dict): + # The Redis key always wins as "id", matching hash results + # (redis-py drops a hash field literally named "id") and every + # query that sets return_fields. A document's own "id" field + # is dropped rather than shadowing the key. + json_data.pop("id", None) return {"id": doc_dict.get("id"), **json_data} raise ValueError(f"Unable to parse json data from Redis {json_data}") diff --git a/tests/integration/test_search_results.py b/tests/integration/test_search_results.py index a9e9e9a4..2238ed45 100644 --- a/tests/integration/test_search_results.py +++ b/tests/integration/test_search_results.py @@ -2,7 +2,7 @@ import pytest -from redisvl.index import SearchIndex +from redisvl.index import AsyncSearchIndex, SearchIndex from redisvl.query import FilterQuery from redisvl.query.filter import Tag @@ -63,3 +63,64 @@ def index(sample_data, redis_url, redis_test_name): def test_process_results_unpacks_json_properly(index, filter_query): results = index.query(filter_query) assert len(results) == 4 + + +ID_CASES = [ + (storage, return_fields) + for storage in ("hash", "json") + for return_fields in (None, ["status"]) +] + + +def _id_schema(redis_test_name, storage): + return { + "index": { + "name": redis_test_name(f"id_is_key_{storage}"), + "prefix": redis_test_name(f"id_is_key_{storage}"), + "storage_type": storage, + }, + "fields": [{"name": "status", "type": "tag"}], + } + + +def test_result_id_is_redis_key(redis_url, redis_test_name): + # A document's own "id" field must never shadow the Redis key, whatever + # the storage type or return_fields. + for storage, return_fields in ID_CASES: + index = SearchIndex.from_dict( + _id_schema(redis_test_name, storage), redis_url=redis_url + ) + index.create(overwrite=True, drop=True) + try: + [key] = index.load([{"id": "t1", "status": "open"}], id_field="id") + query = FilterQuery( + filter_expression=Tag("status") == "open", + return_fields=return_fields, + ) + assert index.query(query) == [{"id": key, "status": "open"}], ( + storage, + return_fields, + ) + finally: + index.delete(drop=True) + + +async def test_async_result_id_is_redis_key(redis_url, redis_test_name): + for storage, return_fields in ID_CASES: + index = AsyncSearchIndex.from_dict( + _id_schema(redis_test_name, storage), redis_url=redis_url + ) + await index.create(overwrite=True, drop=True) + try: + [key] = await index.load([{"id": "t1", "status": "open"}], id_field="id") + query = FilterQuery( + filter_expression=Tag("status") == "open", + return_fields=return_fields, + ) + assert await index.query(query) == [{"id": key, "status": "open"}], ( + storage, + return_fields, + ) + finally: + await index.delete(drop=True) + await index.disconnect() diff --git a/tests/unit/test_query_types.py b/tests/unit/test_query_types.py index 30ce8154..539f7046 100644 --- a/tests/unit/test_query_types.py +++ b/tests/unit/test_query_types.py @@ -1332,6 +1332,12 @@ def test_filter_query_json_unpack_keeps_healthy_doc(): assert results[0] == {"id": "doc:1", "brand": "Nike"} +def test_filter_query_json_unpack_id_is_key_not_document_field(): + doc = Result([1, "doc:1", ["json", '{"id": "1", "brand": "Nike"}']], True) + results = process_results(doc, FilterQuery(Tag("brand") == "Nike"), _json_schema()) + assert results == [{"id": "doc:1", "brand": "Nike"}] + + def test_filter_query_return_fields_id_only_not_skipped(): """Over-skip guard: a legitimate return_fields=['id'] doc must be KEPT.