diff --git a/.librarian/generator-input/client-post-processing/firestore-integration.yaml b/.librarian/generator-input/client-post-processing/firestore-integration.yaml index c26c7d01e128..83f24628969e 100644 --- a/.librarian/generator-input/client-post-processing/firestore-integration.yaml +++ b/.librarian/generator-input/client-post-processing/firestore-integration.yaml @@ -92,6 +92,7 @@ replacements: ) from google.cloud.firestore_v1.pipeline_source import PipelineSource from google.cloud.firestore_v1.pipeline_types import ( + DocumentWindowFrame, FindNearestOptions, Ordering, PipelineDataType, @@ -101,6 +102,7 @@ replacements: TimePart, TimeUnit, UnnestOptions, + WindowBound, WindowSpec, ) from google.cloud.firestore_v1.query import CollectionGroup, Query @@ -199,6 +201,7 @@ replacements: "DocumentReference", "DocumentSnapshot", "DocumentTransform", + "DocumentWindowFrame", "ExistsOption", "ExplainOptions", "FieldFilter", @@ -231,6 +234,7 @@ replacements: "types", "UnnestOptions", "Watch", + "WindowBound", "WindowSpec", "WriteBatch", "WriteOption", @@ -280,6 +284,7 @@ replacements: DocumentReference, DocumentSnapshot, DocumentTransform, + DocumentWindowFrame, ExistsOption, ExplainOptions, FieldFilter, @@ -309,6 +314,7 @@ replacements: Transaction, UnnestOptions, Watch, + WindowBound, WindowSpec, WriteBatch, WriteOption, @@ -348,6 +354,7 @@ replacements: "DocumentReference", "DocumentSnapshot", "DocumentTransform", + "DocumentWindowFrame", "ExistsOption", "ExplainOptions", "FieldFilter", @@ -380,6 +387,7 @@ replacements: "types", "UnnestOptions", "Watch", + "WindowBound", "WindowSpec", "WriteBatch", "WriteOption", diff --git a/packages/google-cloud-firestore/google/cloud/firestore/__init__.py b/packages/google-cloud-firestore/google/cloud/firestore/__init__.py index 3544f97885f9..af79d1f76be2 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore/__init__.py +++ b/packages/google-cloud-firestore/google/cloud/firestore/__init__.py @@ -51,6 +51,7 @@ DocumentReference, DocumentSnapshot, DocumentTransform, + DocumentWindowFrame, ExistsOption, ExplainOptions, FieldFilter, @@ -80,6 +81,7 @@ Transaction, UnnestOptions, Watch, + WindowBound, WindowSpec, WriteBatch, WriteOption, @@ -119,6 +121,7 @@ "DocumentReference", "DocumentSnapshot", "DocumentTransform", + "DocumentWindowFrame", "ExistsOption", "ExplainOptions", "FieldFilter", @@ -151,6 +154,7 @@ "types", "UnnestOptions", "Watch", + "WindowBound", "WindowSpec", "WriteBatch", "WriteOption", diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/__init__.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/__init__.py index 9fbe1a537c95..402d0aef6bea 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/__init__.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/__init__.py @@ -69,6 +69,7 @@ ) from google.cloud.firestore_v1.pipeline_source import PipelineSource from google.cloud.firestore_v1.pipeline_types import ( + DocumentWindowFrame, FindNearestOptions, Ordering, PipelineDataType, @@ -78,6 +79,7 @@ TimePart, TimeUnit, UnnestOptions, + WindowBound, WindowSpec, ) from google.cloud.firestore_v1.query import CollectionGroup, Query @@ -176,6 +178,7 @@ "DocumentReference", "DocumentSnapshot", "DocumentTransform", + "DocumentWindowFrame", "ExistsOption", "ExplainOptions", "FieldFilter", @@ -208,6 +211,7 @@ "types", "UnnestOptions", "Watch", + "WindowBound", "WindowSpec", "WriteBatch", "WriteOption", diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py index b21d31e7db95..4e8772e8d959 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py @@ -15,6 +15,7 @@ from __future__ import annotations +from dataclasses import dataclass, fields from enum import Enum from typing import Optional, Sequence @@ -329,12 +330,69 @@ def __repr__(self): return f"{self.__class__.__name__}(index_field={self.index_field.path!r})" +class WindowBound(str, Enum): + """Enumeration of symbolic window frame boundary keywords.""" + + CURRENT = "current" + UNBOUNDED = "unbounded" + + +@dataclass +class _WindowFrame: + """Base class for window frame specifications.""" + + def _to_pb(self) -> Value: + pb_fields = { + f.name: pipeline_expressions.Expression._cast_to_expr_or_convert_to_constant( + getattr(self, f.name) + )._to_pb() + for f in fields(self) + if getattr(self, f.name) is not None + } + return Value(map_value={"fields": pb_fields}) + + +@dataclass +class DocumentWindowFrame(_WindowFrame): + """Defines a document-count based window frame relative to the current document. + + Example: + >>> from google.cloud.firestore_v1.pipeline_expressions import Field + >>> from google.cloud.firestore_v1.pipeline_types import ( + ... DocumentWindowFrame, + ... WindowBound, + ... WindowSpec, + ... ) + >>> window = WindowSpec( + ... sort=Field.of("published").ascending(), + ... frame=DocumentWindowFrame( + ... WindowBound.UNBOUNDED, WindowBound.CURRENT + ... ), + ... ) + + Args: + preceding: The lower bound of the frame (an integer offset, `WindowBound`, or `Expression`). + A positive integer includes up to that many documents before the current document. + A negative integer indicates a boundary following the current document. + following: The upper bound of the frame (an integer offset, `WindowBound`, or `Expression`). + A positive integer includes up to that many documents after the current document. + A negative integer indicates a boundary preceding the current document. + """ + + preceding: WindowBound | str | int | pipeline_expressions.Expression + following: WindowBound | str | int | pipeline_expressions.Expression + + def _to_dict(self) -> dict[str, Value]: + return {"documents": self._to_pb()} + + class WindowSpec: """Configuration for a window in an `AddWindowFields` pipeline stage. Args: partition: Field paths or expressions used to partition documents into independent windows. sort: Orderings specifying how documents are sorted within each partition. + frame: A `DocumentWindowFrame` defining a document-count window frame. """ def __init__( @@ -346,6 +404,7 @@ def __init__( | str ] = None, sort: Optional[Sequence[Ordering] | Ordering] = None, + frame: Optional[DocumentWindowFrame] = None, ): if partition is None: partition_list = [] @@ -363,6 +422,8 @@ def __init__( else: self.sort = [sort] if isinstance(sort, Ordering) else list(sort) + self.frame: Optional[DocumentWindowFrame] = frame + def __eq__(self, other: object) -> bool: if not isinstance(other, WindowSpec): return NotImplemented @@ -374,6 +435,8 @@ def __repr__(self): args.append(f"partition={self.partition!r}") if self.sort: args.append(f"sort={self.sort!r}") + if self.frame is not None: + args.append(f"frame={self.frame!r}") return f"{self.__class__.__name__}({', '.join(args)})" def _to_pb(self) -> Value: @@ -386,4 +449,6 @@ def _to_pb(self) -> Value: fields["sort"] = Value( array_value={"values": [s._to_pb() for s in self.sort]} ) + if self.frame is not None: + fields.update(self.frame._to_dict()) return Value(map_value={"fields": fields}) diff --git a/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml b/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml index 3f4d88c9967f..f8fa37b5808e 100644 --- a/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml +++ b/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml @@ -1,4 +1,129 @@ tests: + - description: "testAddWindowFields - partition, sort, documents frame, and multiple accumulators" + pipeline: + - Collection: books + - Where: + - FunctionExpression.equal_any: + - Field: genre + - [Science Fiction, Dystopian] + - AddWindowFields: + window: + WindowSpec: + partition: + - Field: genre + sort: + - Ordering: + - Field: published + - ASCENDING + frame: + DocumentWindowFrame: + preceding: unbounded + following: current + fields: + - AliasedExpression: + - FunctionExpression.sum: + - Field: rating + - "running_rating" + - AliasedExpression: + - Count + - "running_count" + - Sort: + - Ordering: + - Field: published + - ASCENDING + - Select: + - title + - genre + - running_rating + - running_count + assert_results: + - title: "1984" + genre: "Dystopian" + running_rating: 4.2 + running_count: 1 + - title: "Dune" + genre: "Science Fiction" + running_rating: 4.6 + running_count: 1 + - title: "The Hitchhiker's Guide to the Galaxy" + genre: "Science Fiction" + running_rating: 8.8 + running_count: 2 + - title: "The Handmaid's Tale" + genre: "Dystopian" + running_rating: 8.3 + running_count: 2 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - functionValue: + args: + - stringValue: Science Fiction + - stringValue: Dystopian + name: array + name: equal_any + name: where + - args: + - mapValue: + fields: + documents: + mapValue: + fields: + following: + stringValue: current + preceding: + stringValue: unbounded + partition: + arrayValue: + values: + - fieldReferenceValue: genre + sort: + arrayValue: + values: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: published + - mapValue: + fields: + running_count: + functionValue: + name: count + running_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: sum + name: add_window_fields + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: published + name: sort + - args: + - mapValue: + fields: + genre: + fieldReferenceValue: genre + running_count: + fieldReferenceValue: running_count + running_rating: + fieldReferenceValue: running_rating + title: + fieldReferenceValue: title + name: select - description: "testAddWindowFields - rank with ties" pipeline: - Collection: books diff --git a/packages/google-cloud-firestore/tests/system/test_pipeline_acceptance.py b/packages/google-cloud-firestore/tests/system/test_pipeline_acceptance.py index a25d5476195a..bbf08f90c43c 100644 --- a/packages/google-cloud-firestore/tests/system/test_pipeline_acceptance.py +++ b/packages/google-cloud-firestore/tests/system/test_pipeline_acceptance.py @@ -28,7 +28,7 @@ from test__helpers import FIRESTORE_EMULATOR, FIRESTORE_ENTERPRISE_DB, system_test_lock from google.cloud.firestore import AsyncClient, Client, GeoPoint -from google.cloud.firestore_v1 import pipeline_expressions +from google.cloud.firestore_v1 import pipeline_expressions, pipeline_types from google.cloud.firestore_v1 import pipeline_expressions as expr from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.vector import Vector @@ -324,9 +324,9 @@ def _parse_expressions(client, yaml_element: Any): yaml_args = yaml_element[cls_str] return _apply_yaml_args_to_callable(callable_obj, client, yaml_args) elif len(yaml_element) == 1 and _is_stage_string(next(iter(yaml_element))): - # build pipeline stage if possible (eg, for SampleOptions) + # build pipeline stage or pipeline_types object if possible (eg, for SampleOptions, DocumentWindowFrame) cls_str = next(iter(yaml_element)) - cls = getattr(stages, cls_str) + cls = getattr(stages, cls_str, None) or getattr(pipeline_types, cls_str) yaml_args = yaml_element[cls_str] return _apply_yaml_args_to_callable(cls, client, yaml_args) elif len(yaml_element) == 1 and list(yaml_element)[0] == "Pipeline": @@ -402,12 +402,12 @@ def _is_expr_string(yaml_str): def _is_stage_string(yaml_str): """ - Returns true if a string represents a class in pipeline_stages + Returns true if a string represents a class in pipeline_stages or pipeline_types """ return ( isinstance(yaml_str, str) and yaml_str[0].isupper() - and hasattr(stages, yaml_str) + and (hasattr(stages, yaml_str) or hasattr(pipeline_types, yaml_str)) ) diff --git a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py index e74551bc5a93..bfec49a2fdd8 100644 --- a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py +++ b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py @@ -27,7 +27,11 @@ Ordering, Rank, ) -from google.cloud.firestore_v1.pipeline_types import WindowSpec +from google.cloud.firestore_v1.pipeline_types import ( + DocumentWindowFrame, + WindowBound, + WindowSpec, +) from google.cloud.firestore_v1.types.document import Value from google.cloud.firestore_v1.vector import Vector @@ -1069,6 +1073,23 @@ def test_to_pb_with_fields(self): class TestWindowTypes: + def test_document_window_frame_to_pb(self): + frame = DocumentWindowFrame(1, WindowBound.CURRENT) + pb = frame._to_pb() + assert pb.map_value.fields == { + "preceding": Value(integer_value=1), + "following": Value(string_value="current"), + } + assert frame._to_dict() == {"documents": pb} + assert frame == DocumentWindowFrame(1, "current") + assert frame != DocumentWindowFrame(2, WindowBound.CURRENT) + + expr_pb = DocumentWindowFrame(Field.of("offset"), "unbounded")._to_pb() + assert expr_pb.map_value.fields == { + "preceding": Value(field_reference_value="offset"), + "following": Value(string_value="unbounded"), + } + def test_window_spec_coercions(self): order = Field.of("rating").descending() assert WindowSpec().partition == [] @@ -1088,16 +1109,24 @@ def test_window_spec_repr_and_equality(self): spec = WindowSpec( partition="genre", sort=Field.of("rating").descending(), + frame=DocumentWindowFrame(-1, WindowBound.CURRENT), ) assert repr(spec) == ( "WindowSpec(" "partition=[Field.of('genre')], " - "sort=[Field.of('rating').descending()]" + "sort=[Field.of('rating').descending()], " + "frame=DocumentWindowFrame(preceding=-1, following=)" ")" ) assert spec == WindowSpec( partition=[Field.of("genre")], sort=[Field.of("rating").descending()], + frame=DocumentWindowFrame(-1, "current"), + ) + assert spec != WindowSpec( + partition="genre", + sort=Field.of("rating").descending(), + frame=DocumentWindowFrame(0, WindowBound.CURRENT), ) assert spec != empty assert spec != object() @@ -1105,12 +1134,14 @@ def test_window_spec_repr_and_equality(self): def test_window_spec_to_pb(self): assert WindowSpec()._to_pb().map_value.fields == {} order = Field.of("rating").descending() - spec = WindowSpec(partition="genre", sort=order) + doc_frame = DocumentWindowFrame(-1, WindowBound.CURRENT) + spec = WindowSpec(partition="genre", sort=order, frame=doc_frame) assert spec._to_pb().map_value.fields == { "partition": Value( array_value={"values": [Value(field_reference_value="genre")]} ), "sort": Value(array_value={"values": [order._to_pb()]}), + "documents": doc_frame._to_pb(), }