From 721da84847b1054fa01bcb9b52f25c4e6568f576 Mon Sep 17 00:00:00 2001 From: ohmayr Date: Wed, 18 Nov 2026 01:01:38 +0000 Subject: [PATCH 1/2] feat(firestore): add AddWindowFields stage, WindowSpec, and Rank Introduce the foundational window functions pipeline stage for Cloud Firestore: - Add `WindowFunction` base class (making `AggregateFunction` a subclass so existing aggregates can be used in window stages) and the `Rank` expression in `pipeline_expressions.py`. - Add `WindowSpec(partition=..., sort=...)` in `pipeline_types.py` and export it from `google.cloud.firestore` and `google.cloud.firestore_v1`. - Add `AddWindowFields` stage in `pipeline_stages.py` and `pipeline.add_window_fields()` on `_BasePipeline`. - Add unit tests and `window.yaml` acceptance test (`testAddWindowFields - rank with ties`), skipping live execution under #19115 until the backend rolls out to production. --- .../firestore-integration.yaml | 4 + .../google/cloud/firestore/__init__.py | 2 + .../google/cloud/firestore_v1/__init__.py | 2 + .../cloud/firestore_v1/base_pipeline.py | 36 +++++++ .../firestore_v1/pipeline_expressions.py | 31 +++++- .../cloud/firestore_v1/pipeline_stages.py | 26 +++++ .../cloud/firestore_v1/pipeline_types.py | 60 +++++++++++ .../tests/system/pipeline_e2e/window.yaml | 102 ++++++++++++++++++ .../tests/system/test_pipeline_acceptance.py | 8 ++ .../tests/unit/v1/test_async_pipeline.py | 14 +++ .../tests/unit/v1/test_pipeline.py | 14 +++ .../unit/v1/test_pipeline_expressions.py | 15 +++ .../tests/unit/v1/test_pipeline_stages.py | 100 +++++++++++++++++ 13 files changed, 413 insertions(+), 1 deletion(-) create mode 100644 packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml diff --git a/.librarian/generator-input/client-post-processing/firestore-integration.yaml b/.librarian/generator-input/client-post-processing/firestore-integration.yaml index d31022a3b362..c26c7d01e128 100644 --- a/.librarian/generator-input/client-post-processing/firestore-integration.yaml +++ b/.librarian/generator-input/client-post-processing/firestore-integration.yaml @@ -101,6 +101,7 @@ replacements: TimePart, TimeUnit, UnnestOptions, + WindowSpec, ) from google.cloud.firestore_v1.query import CollectionGroup, Query from google.cloud.firestore_v1.query_profile import ( @@ -230,6 +231,7 @@ replacements: "types", "UnnestOptions", "Watch", + "WindowSpec", "WriteBatch", "WriteOption", ] @@ -307,6 +309,7 @@ replacements: Transaction, UnnestOptions, Watch, + WindowSpec, WriteBatch, WriteOption, async_transactional, @@ -377,6 +380,7 @@ replacements: "types", "UnnestOptions", "Watch", + "WindowSpec", "WriteBatch", "WriteOption", ] diff --git a/packages/google-cloud-firestore/google/cloud/firestore/__init__.py b/packages/google-cloud-firestore/google/cloud/firestore/__init__.py index 44a9651d6c02..3544f97885f9 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore/__init__.py +++ b/packages/google-cloud-firestore/google/cloud/firestore/__init__.py @@ -80,6 +80,7 @@ Transaction, UnnestOptions, Watch, + WindowSpec, WriteBatch, WriteOption, async_transactional, @@ -150,6 +151,7 @@ "types", "UnnestOptions", "Watch", + "WindowSpec", "WriteBatch", "WriteOption", ] diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/__init__.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/__init__.py index 4656b5b5441f..9fbe1a537c95 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/__init__.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/__init__.py @@ -78,6 +78,7 @@ TimePart, TimeUnit, UnnestOptions, + WindowSpec, ) from google.cloud.firestore_v1.query import CollectionGroup, Query from google.cloud.firestore_v1.query_profile import ( @@ -207,6 +208,7 @@ "types", "UnnestOptions", "Watch", + "WindowSpec", "WriteBatch", "WriteOption", ] diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/base_pipeline.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/base_pipeline.py index e0699b59004f..3c5ab6910dfa 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/base_pipeline.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/base_pipeline.py @@ -27,6 +27,7 @@ Field, FunctionExpression, Selectable, + WindowFunction, _PipelineValueExpression, ) from google.cloud.firestore_v1.types.pipeline import ( @@ -176,6 +177,41 @@ def add_fields(self, *fields: Selectable) -> "_BasePipeline": """ return self._append(stages.AddFields(*fields)) + def add_window_fields( + self, + *fields: AliasedExpression[WindowFunction], + window: types.WindowSpec | None = None, + ) -> "_BasePipeline": + """ + Adds new fields computed by window functions over a window of documents. + + .. note:: + This feature is currently in beta and is subject to change. + + Example: + >>> from google.cloud.firestore_v1.pipeline_expressions import Field, Rank + >>> from google.cloud.firestore_v1.pipeline_types import WindowSpec + >>> pipeline = client.pipeline().collection("books") + >>> pipeline = pipeline.add_window_fields( + ... Rank().as_("genre_rank"), + ... Field.of("price").sum().as_("genre_total_price"), + ... window=WindowSpec( + ... partition=["genre"], + ... sort=[Field.of("rating").descending()], + ... ), + ... ) + + Args: + *fields: Aliased `AggregateFunction` or `WindowFunction` expressions defining + the window calculations and their output field aliases. + window: Optional `WindowSpec` specifying the baseline window configuration + for this stage. + + Returns: + A new Pipeline object with this stage appended to the stage list. + """ + return self._append(stages.AddWindowFields(*fields, window=window)) + def remove_fields(self, *fields: Field | str) -> "_BasePipeline": """ Removes fields from outputs of previous stages. diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_expressions.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_expressions.py index 5f52d92fd3bd..0d9598fd6274 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_expressions.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_expressions.py @@ -2682,7 +2682,11 @@ def _to_pb(self): ) -class AggregateFunction(FunctionExpression): +class WindowFunction(FunctionExpression): + """A base class for window functions that operate over a window of documents.""" + + +class AggregateFunction(WindowFunction): """A base class for aggregation functions that operate across multiple inputs.""" @@ -3197,3 +3201,28 @@ def __init__(self): [], repr_function=FunctionExpression._build_standalone_repr(), ) + + +class Rank(WindowFunction): + """Creates a window function that computes the rank of each document within a window. + + Documents with equal sort values receive the same rank, with gaps in the subsequent + ranking numbers. + + Example: + >>> # Rank documents by price descending within each category + >>> db.pipeline().collection("products").add_window_fields( + ... Rank().as_("price_rank"), + ... window=WindowSpec( + ... partition=["category"], + ... sort=[Field.of("price").descending()], + ... ), + ... ) + """ + + def __init__(self): + super().__init__( + "rank", + [], + repr_function=FunctionExpression._build_standalone_repr(), + ) diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_stages.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_stages.py index e72523dc9dae..6a6fb0e13b85 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_stages.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_stages.py @@ -27,6 +27,7 @@ Expression, Field, Selectable, + WindowFunction, ) from google.cloud.firestore_v1.types.document import Pipeline as Pipeline_pb from google.cloud.firestore_v1.types.document import Value @@ -42,6 +43,7 @@ SampleOptions, SearchOptions, UnnestOptions, + WindowSpec, ) @@ -86,6 +88,30 @@ def _pb_args(self): return [Selectable._to_value(self.fields)] +class AddWindowFields(Stage): + """Adds new fields computed by window functions over a window of documents.""" + + def __init__( + self, + *args: AliasedExpression[WindowFunction], + fields: Sequence[AliasedExpression[WindowFunction]] = (), + window: WindowSpec | None = None, + ): + super().__init__("add_window_fields") + if args and fields: + raise ValueError( + "AddWindowFields stage contains both positional and keyword fields" + ) + self.fields = list(args or fields) + self.window: WindowSpec = window if window is not None else WindowSpec() + + def _pb_args(self) -> list[Value]: + return [ + self.window._to_pb(), + Selectable._to_value(self.fields), + ] + + class Aggregate(Stage): """Performs aggregation operations, optionally grouped.""" diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py index 3e9918140e1f..b21d31e7db95 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py @@ -327,3 +327,63 @@ def __init__(self, index_field: pipeline_expressions.Field | str): def __repr__(self): return f"{self.__class__.__name__}(index_field={self.index_field.path!r})" + + +class WindowSpec: + """Configuration for a window in an `AddWindowFields` pipeline stage. + + Args: + partition: Field paths or expressions used to partition documents into independent windows. + sort: Orderings specifying how documents are sorted within each partition. + """ + + def __init__( + self, + *, + partition: Optional[ + Sequence[pipeline_expressions.Expression | str] + | pipeline_expressions.Expression + | str + ] = None, + sort: Optional[Sequence[Ordering] | Ordering] = None, + ): + if partition is None: + partition_list = [] + elif isinstance(partition, (str, pipeline_expressions.Expression)): + partition_list = [partition] + else: + partition_list = list(partition) + self.partition: list[pipeline_expressions.Expression] = [ + pipeline_expressions.Field.of(p) if isinstance(p, str) else p + for p in partition_list + ] + + if sort is None: + self.sort: list[Ordering] = [] + else: + self.sort = [sort] if isinstance(sort, Ordering) else list(sort) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, WindowSpec): + return NotImplemented + return self._to_pb() == other._to_pb() + + def __repr__(self): + args = [] + if self.partition: + args.append(f"partition={self.partition!r}") + if self.sort: + args.append(f"sort={self.sort!r}") + return f"{self.__class__.__name__}({', '.join(args)})" + + def _to_pb(self) -> Value: + fields: dict[str, Value] = {} + if self.partition: + fields["partition"] = Value( + array_value={"values": [p._to_pb() for p in self.partition]} + ) + if self.sort: + fields["sort"] = Value( + array_value={"values": [s._to_pb() for s in self.sort]} + ) + return Value(map_value={"fields": fields}) diff --git a/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml b/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml new file mode 100644 index 000000000000..3f4d88c9967f --- /dev/null +++ b/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml @@ -0,0 +1,102 @@ +tests: + - description: "testAddWindowFields - rank with ties" + pipeline: + - Collection: books + - Where: + - FunctionExpression.equal_any: + - Field: genre + - [Science Fiction, Dystopian] + - AddWindowFields: + window: + WindowSpec: + sort: + - Ordering: + - Field: rating + - ASCENDING + fields: + - AliasedExpression: + - Rank + - "rating_rank" + - Sort: + - Ordering: + - Field: rating + - ASCENDING + - Ordering: + - Field: title + - ASCENDING + - Select: + - title + - rating + - rating_rank + assert_results: + - title: "The Handmaid's Tale" + rating: 4.1 + rating_rank: 1 + - title: "1984" + rating: 4.2 + rating_rank: 2 + - title: "The Hitchhiker's Guide to the Galaxy" + rating: 4.2 + rating_rank: 2 + - title: "Dune" + rating: 4.6 + rating_rank: 4 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - functionValue: + args: + - stringValue: Science Fiction + - stringValue: Dystopian + name: array + name: equal_any + name: where + - args: + - mapValue: + fields: + sort: + arrayValue: + values: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: rating + - mapValue: + fields: + rating_rank: + functionValue: + name: rank + name: add_window_fields + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: rating + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - args: + - mapValue: + fields: + rating: + fieldReferenceValue: rating + rating_rank: + fieldReferenceValue: rating_rank + title: + fieldReferenceValue: title + name: select diff --git a/packages/google-cloud-firestore/tests/system/test_pipeline_acceptance.py b/packages/google-cloud-firestore/tests/system/test_pipeline_acceptance.py index 149182b13694..a25d5476195a 100644 --- a/packages/google-cloud-firestore/tests/system/test_pipeline_acceptance.py +++ b/packages/google-cloud-firestore/tests/system/test_pipeline_acceptance.py @@ -40,6 +40,14 @@ reason="Pipeline tests are currently not supported by emulator", ) + +@pytest.fixture(autouse=True) +def _skip_window_execution_tests(request): + # TODO(#19115): Remove once add_window_fields is available in production. + if "window.yaml" in request.node.name and "parse_proto" not in request.node.name: + pytest.skip("Temporarily skipped. Not yet in production.") + + test_dir_name = os.path.dirname(__file__) id_format = lambda x: f"{x.get('file_name', '')}: {x.get('description', '')}" # noqa: E731 diff --git a/packages/google-cloud-firestore/tests/unit/v1/test_async_pipeline.py b/packages/google-cloud-firestore/tests/unit/v1/test_async_pipeline.py index 5f9e7f2749ab..304a820f0242 100644 --- a/packages/google-cloud-firestore/tests/unit/v1/test_async_pipeline.py +++ b/packages/google-cloud-firestore/tests/unit/v1/test_async_pipeline.py @@ -17,6 +17,7 @@ from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.pipeline_expressions import Field +from google.cloud.firestore_v1.pipeline_types import WindowSpec from tests.unit.v1._test_helpers import make_async_client @@ -426,6 +427,7 @@ async def test_async_pipeline_stream_stream_equivalence(): ("offset", (1,), stages.Offset), ("limit", (1,), stages.Limit), ("aggregate", (Field.of("n").as_("alias"),), stages.Aggregate), + ("add_window_fields", (Field.of("n").as_("alias"),), stages.AddWindowFields), ("distinct", ("field_name",), stages.Distinct), ("distinct", (Field.of("n"), "second"), stages.Distinct), ], @@ -450,6 +452,18 @@ def test_async_pipeline_aggregate_with_groups(): assert list(result_ppl.stages[0].accumulators) == [Field.of("title")] +def test_async_pipeline_add_window_fields_with_window(): + start_ppl = _make_async_pipeline() + window = WindowSpec(partition="genre", sort=Field.of("rating").descending()) + expr = Field.of("rating").sum().as_("sum_rating") + result_ppl = start_ppl.add_window_fields(expr, window=window) + assert len(start_ppl.stages) == 0 + assert len(result_ppl.stages) == 1 + assert isinstance(result_ppl.stages[0], stages.AddWindowFields) + assert result_ppl.stages[0].fields == [expr] + assert result_ppl.stages[0].window == window + + def test_async_pipeline_raw_stage_with_options(): from google.cloud.firestore_v1.pipeline_stages import RawStage diff --git a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline.py b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline.py index fa2f19ae109a..d67e1aa979d3 100644 --- a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline.py +++ b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline.py @@ -17,6 +17,7 @@ from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.pipeline_expressions import Field +from google.cloud.firestore_v1.pipeline_types import WindowSpec from tests.unit.v1._test_helpers import make_client @@ -417,6 +418,7 @@ def test_pipeline_execute_stream_equivalence(): ("offset", (1,), stages.Offset), ("limit", (1,), stages.Limit), ("aggregate", (Field.of("n").as_("alias"),), stages.Aggregate), + ("add_window_fields", (Field.of("n").as_("alias"),), stages.AddWindowFields), ("distinct", ("field_name",), stages.Distinct), ("distinct", (Field.of("n"), "second"), stages.Distinct), ], @@ -441,6 +443,18 @@ def test_pipeline_aggregate_with_groups(): assert list(result_ppl.stages[0].accumulators) == [Field.of("title")] +def test_pipeline_add_window_fields_with_window(): + start_ppl = _make_pipeline() + window = WindowSpec(partition="genre", sort=Field.of("rating").descending()) + expr = Field.of("rating").sum().as_("sum_rating") + result_ppl = start_ppl.add_window_fields(expr, window=window) + assert len(start_ppl.stages) == 0 + assert len(result_ppl.stages) == 1 + assert isinstance(result_ppl.stages[0], stages.AddWindowFields) + assert result_ppl.stages[0].fields == [expr] + assert result_ppl.stages[0].window == window + + def test_pipeline_raw_stage_with_options(): from google.cloud.firestore_v1.pipeline_stages import RawStage diff --git a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_expressions.py b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_expressions.py index 0c0f29df54b3..a8ca27278506 100644 --- a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_expressions.py +++ b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_expressions.py @@ -22,11 +22,14 @@ from google.cloud.firestore_v1 import _helpers from google.cloud.firestore_v1._helpers import GeoPoint from google.cloud.firestore_v1.pipeline_expressions import ( + AggregateFunction, BooleanExpression, Constant, Expression, Field, Ordering, + Rank, + WindowFunction, ) from google.cloud.firestore_v1.types import document as document_pb from google.cloud.firestore_v1.types import query as query_pb @@ -2262,3 +2265,15 @@ def test_array_index_of_all(self): assert repr(instance) == "Value.array_index_of_all(Constant.of('comedy'))" infix_instance = arg1.array_index_of_all(value) assert infix_instance == instance + + +class TestWindowExpressions: + def test_window_function_hierarchy(self): + assert issubclass(AggregateFunction, WindowFunction) + assert not issubclass(Expression, WindowFunction) + + def test_rank(self): + rank = Rank() + assert isinstance(rank, WindowFunction) + assert repr(rank) == "Rank()" + assert rank._to_pb() == Value(function_value={"name": "rank", "args": []}) diff --git a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py index c926285f462c..e74551bc5a93 100644 --- a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py +++ b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py @@ -25,7 +25,9 @@ DocumentMatches, Field, Ordering, + Rank, ) +from google.cloud.firestore_v1.pipeline_types import WindowSpec from google.cloud.firestore_v1.types.document import Value from google.cloud.firestore_v1.vector import Vector @@ -1064,3 +1066,101 @@ def test_to_pb_with_fields(self): assert "score" in result.args[0].map_value.fields assert "status" in result.args[0].map_value.fields assert len(result.options) == 0 + + +class TestWindowTypes: + def test_window_spec_coercions(self): + order = Field.of("rating").descending() + assert WindowSpec().partition == [] + assert WindowSpec().sort == [] + assert WindowSpec(partition="genre").partition == [Field.of("genre")] + assert WindowSpec(partition=Field.of("genre")).partition == [Field.of("genre")] + assert WindowSpec(partition=["genre", Field.of("author")]).partition == [ + Field.of("genre"), + Field.of("author"), + ] + assert WindowSpec(sort=order).sort == [order] + assert WindowSpec(sort=[order]).sort == [order] + + def test_window_spec_repr_and_equality(self): + empty = WindowSpec() + assert repr(empty) == "WindowSpec()" + spec = WindowSpec( + partition="genre", + sort=Field.of("rating").descending(), + ) + assert repr(spec) == ( + "WindowSpec(" + "partition=[Field.of('genre')], " + "sort=[Field.of('rating').descending()]" + ")" + ) + assert spec == WindowSpec( + partition=[Field.of("genre")], + sort=[Field.of("rating").descending()], + ) + assert spec != empty + assert spec != object() + + def test_window_spec_to_pb(self): + assert WindowSpec()._to_pb().map_value.fields == {} + order = Field.of("rating").descending() + spec = WindowSpec(partition="genre", sort=order) + assert spec._to_pb().map_value.fields == { + "partition": Value( + array_value={"values": [Value(field_reference_value="genre")]} + ), + "sort": Value(array_value={"values": [order._to_pb()]}), + } + + +class TestAddWindowFields: + def _make_one(self, *args, **kwargs): + return stages.AddWindowFields(*args, **kwargs) + + def test_ctor_positional(self): + expr1 = Field.of("rating").sum().as_("sum_rating") + expr2 = Rank().as_("rank_val") + stage = self._make_one(expr1, expr2) + assert stage.fields == [expr1, expr2] + assert stage.window == WindowSpec() + assert stage.name == "add_window_fields" + + def test_ctor_keyword(self): + expr1 = Field.of("rating").sum().as_("sum_rating") + window = WindowSpec(partition=["genre"], sort=[Field.of("rating").descending()]) + stage = self._make_one(fields=[expr1], window=window) + assert stage.fields == [expr1] + assert stage.window == window + + def test_ctor_combined_raises(self): + expr1 = Field.of("rating").sum().as_("sum_rating") + with pytest.raises( + ValueError, + match="AddWindowFields stage contains both positional and keyword fields", + ): + self._make_one(expr1, fields=[expr1]) + + def test_repr(self): + stage = self._make_one( + Rank().as_("pub_rank"), window=WindowSpec(partition="genre") + ) + assert repr(stage) == ( + "AddWindowFields(" + "fields=[Rank().as_('pub_rank')], " + "window=WindowSpec(partition=[Field.of('genre')])" + ")" + ) + + def test_to_pb(self): + expr1 = Field.of("rating").sum().as_("sum_rating") + expr2 = Rank().as_("pub_rank") + window = WindowSpec(partition="genre") + pb = self._make_one(expr1, expr2, window=window)._to_pb() + assert pb.name == "add_window_fields" + assert len(pb.options) == 0 + assert pb.args[0] == window._to_pb() + assert pb.args[1].map_value.fields == { + "sum_rating": expr1.expr._to_pb(), + "pub_rank": expr2.expr._to_pb(), + } From 82aa074bd4855d1d2ef6191dc9bd546a0546e8cf Mon Sep 17 00:00:00 2001 From: ohmayr Date: Sat, 21 Nov 2026 00:03:49 +0000 Subject: [PATCH 2/2] fix(firestore): address PR #19116 window review feedback - Make AggregateFunction and WindowFunction sibling subclasses of FunctionExpression and accept both in add_window_fields/AddWindowFields - Validate WindowSpec sort inputs and add unit test coverage - Update docstrings with preview notes, window=None, default frame semantics, Rank single-sort requirement, and fixed running-total example - Add partition aggregate, running window, and Rank error tests to window.yaml --- .../cloud/firestore_v1/base_pipeline.py | 31 ++- .../firestore_v1/pipeline_expressions.py | 15 +- .../cloud/firestore_v1/pipeline_stages.py | 13 +- .../cloud/firestore_v1/pipeline_types.py | 26 ++- .../tests/system/pipeline_e2e/window.yaml | 212 ++++++++++++++++++ .../unit/v1/test_pipeline_expressions.py | 4 +- .../tests/unit/v1/test_pipeline_stages.py | 7 + 7 files changed, 292 insertions(+), 16 deletions(-) diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/base_pipeline.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/base_pipeline.py index 3c5ab6910dfa..642611fe29d0 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/base_pipeline.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/base_pipeline.py @@ -179,22 +179,40 @@ def add_fields(self, *fields: Selectable) -> "_BasePipeline": def add_window_fields( self, - *fields: AliasedExpression[WindowFunction], + *fields: AliasedExpression[AggregateFunction] + | AliasedExpression[WindowFunction], window: types.WindowSpec | None = None, ) -> "_BasePipeline": """ - Adds new fields computed by window functions over a window of documents. + Evaluates window and aggregate functions over a window of related documents + and adds the results as new fields on each input document. + + Unlike :meth:`aggregate`, which collapses documents into summary rows, + ``add_window_fields`` preserves every input document and attaches computed + values (such as partition totals, running aggregates, or rankings) to each row. + + When ``window`` specifies ``sort`` without an explicit frame, aggregates evaluate + as a running window from the start of the partition through the current sort + value. When ``sort`` is omitted, aggregates evaluate across all documents in the + partition. .. note:: - This feature is currently in beta and is subject to change. + This feature is in preview until the backend feature launches and is + subject to change. Example: >>> from google.cloud.firestore_v1.pipeline_expressions import Field, Rank >>> from google.cloud.firestore_v1.pipeline_types import WindowSpec >>> pipeline = client.pipeline().collection("books") + >>> # Partition-wide average rating (unsorted window covers the whole partition) + >>> pipeline = pipeline.add_window_fields( + ... Field.of("rating").average().as_("genre_avg_rating"), + ... window=WindowSpec(partition=["genre"]), + ... ) + >>> # Rank and running total within each genre (sorted default frame) >>> pipeline = pipeline.add_window_fields( ... Rank().as_("genre_rank"), - ... Field.of("price").sum().as_("genre_total_price"), + ... Field.of("price").sum().as_("running_price_total"), ... window=WindowSpec( ... partition=["genre"], ... sort=[Field.of("rating").descending()], @@ -204,8 +222,9 @@ def add_window_fields( Args: *fields: Aliased `AggregateFunction` or `WindowFunction` expressions defining the window calculations and their output field aliases. - window: Optional `WindowSpec` specifying the baseline window configuration - for this stage. + window: Optional `WindowSpec` specifying how documents are partitioned and + ordered. If `None`, evaluates over a single unsorted global partition + covering all input documents (equivalent to ``WindowSpec()``). Returns: A new Pipeline object with this stage appended to the stage list. diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_expressions.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_expressions.py index 0d9598fd6274..2d492e88aa08 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_expressions.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_expressions.py @@ -2683,10 +2683,15 @@ def _to_pb(self): class WindowFunction(FunctionExpression): - """A base class for window functions that operate over a window of documents.""" + """A base class for window functions that operate over a window of documents. + + .. note:: + This feature is in preview until the backend feature launches and is + subject to change. + """ -class AggregateFunction(WindowFunction): +class AggregateFunction(FunctionExpression): """A base class for aggregation functions that operate across multiple inputs.""" @@ -3207,7 +3212,11 @@ class Rank(WindowFunction): """Creates a window function that computes the rank of each document within a window. Documents with equal sort values receive the same rank, with gaps in the subsequent - ranking numbers. + ranking numbers. The window must specify exactly one sort ordering. + + .. note:: + This feature is in preview until the backend feature launches and is + subject to change. Example: >>> # Rank documents by price descending within each category diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_stages.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_stages.py index 6a6fb0e13b85..cb8b498d3e05 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_stages.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_stages.py @@ -89,12 +89,19 @@ def _pb_args(self): class AddWindowFields(Stage): - """Adds new fields computed by window functions over a window of documents.""" + """Adds new fields computed by window functions over a window of documents. + + .. note:: + This feature is in preview until the backend feature launches and is + subject to change. + """ def __init__( self, - *args: AliasedExpression[WindowFunction], - fields: Sequence[AliasedExpression[WindowFunction]] = (), + *args: AliasedExpression[AggregateFunction] | AliasedExpression[WindowFunction], + fields: Sequence[ + AliasedExpression[AggregateFunction] | AliasedExpression[WindowFunction] + ] = (), window: WindowSpec | None = None, ): super().__init__("add_window_fields") diff --git a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py index b21d31e7db95..87e38c59bf1f 100644 --- a/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py +++ b/packages/google-cloud-firestore/google/cloud/firestore_v1/pipeline_types.py @@ -332,9 +332,19 @@ def __repr__(self): class WindowSpec: """Configuration for a window in an `AddWindowFields` pipeline stage. + .. note:: + This feature is in preview until the backend feature launches and is + subject to change. + Args: - partition: Field paths or expressions used to partition documents into independent windows. - sort: Orderings specifying how documents are sorted within each partition. + partition: Optional field paths or expressions used to partition documents into + independent windows. If `None`, all input documents are treated as a single + global partition. + sort: Optional orderings specifying how documents are sorted within each + partition. If `None`, documents are unordered and aggregates evaluate across + all documents in the partition. When specified without an explicit frame, + aggregates default to a running window from the start of the partition + through the current sort value. """ def __init__( @@ -360,8 +370,18 @@ def __init__( if sort is None: self.sort: list[Ordering] = [] + elif isinstance(sort, Ordering): + self.sort = [sort] + elif ( + isinstance(sort, (str, bytes)) + or not isinstance(sort, Sequence) + or not all(isinstance(o, Ordering) for o in sort) + ): + raise TypeError( + "sort must be an Ordering or a sequence of Ordering objects" + ) else: - self.sort = [sort] if isinstance(sort, Ordering) else list(sort) + self.sort = list(sort) def __eq__(self, other: object) -> bool: if not isinstance(other, WindowSpec): diff --git a/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml b/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml index 3f4d88c9967f..b8649f4b6abe 100644 --- a/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml +++ b/packages/google-cloud-firestore/tests/system/pipeline_e2e/window.yaml @@ -100,3 +100,215 @@ tests: title: fieldReferenceValue: title name: select + - description: "testAddWindowFields - partition with aggregate functions" + pipeline: + - Collection: books + - Where: + - FunctionExpression.equal_any: + - Field: genre + - [Science Fiction, Dystopian] + - AddWindowFields: + window: + WindowSpec: + partition: + - genre + fields: + - AliasedExpression: + - Count + - "genre_count" + - AliasedExpression: + - FunctionExpression.average: + - Field: rating + - "genre_avg_rating" + - Sort: + - Ordering: + - Field: published + - ASCENDING + - Select: + - title + - genre + - genre_count + - genre_avg_rating + assert_results: + - title: "1984" + genre: "Dystopian" + genre_count: 2 + genre_avg_rating: 4.15 + - title: "Dune" + genre: "Science Fiction" + genre_count: 2 + genre_avg_rating: 4.4 + - title: "The Hitchhiker's Guide to the Galaxy" + genre: "Science Fiction" + genre_count: 2 + genre_avg_rating: 4.4 + - title: "The Handmaid's Tale" + genre: "Dystopian" + genre_count: 2 + genre_avg_rating: 4.15 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - functionValue: + args: + - stringValue: Science Fiction + - stringValue: Dystopian + name: array + name: equal_any + name: where + - args: + - mapValue: + fields: + partition: + arrayValue: + values: + - fieldReferenceValue: genre + - mapValue: + fields: + genre_avg_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: average + genre_count: + functionValue: + name: count + name: add_window_fields + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: published + name: sort + - args: + - mapValue: + fields: + genre: + fieldReferenceValue: genre + genre_avg_rating: + fieldReferenceValue: genre_avg_rating + genre_count: + fieldReferenceValue: genre_count + title: + fieldReferenceValue: title + name: select + - description: "testAddWindowFields - partition and sort default running window" + pipeline: + - Collection: books + - Where: + - FunctionExpression.equal_any: + - Field: genre + - [Science Fiction, Dystopian] + - AddWindowFields: + window: + WindowSpec: + partition: + - genre + sort: + - Ordering: + - Field: published + - ASCENDING + fields: + - AliasedExpression: + - Count + - "running_count" + - Sort: + - Ordering: + - Field: published + - ASCENDING + - Select: + - title + - genre + - running_count + assert_results: + - title: "1984" + genre: "Dystopian" + running_count: 1 + - title: "Dune" + genre: "Science Fiction" + running_count: 1 + - title: "The Hitchhiker's Guide to the Galaxy" + genre: "Science Fiction" + running_count: 2 + - title: "The Handmaid's Tale" + genre: "Dystopian" + running_count: 2 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - functionValue: + args: + - stringValue: Science Fiction + - stringValue: Dystopian + name: array + name: equal_any + name: where + - args: + - mapValue: + fields: + partition: + arrayValue: + values: + - fieldReferenceValue: genre + sort: + arrayValue: + values: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: published + - mapValue: + fields: + running_count: + functionValue: + name: count + name: add_window_fields + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: published + name: sort + - args: + - mapValue: + fields: + genre: + fieldReferenceValue: genre + running_count: + fieldReferenceValue: running_count + title: + fieldReferenceValue: title + name: select + - description: "testAddWindowFields - error on rank() without sort" + pipeline: + - Collection: books + - AddWindowFields: + window: + WindowSpec: + partition: + - genre + fields: + - AliasedExpression: + - Rank + - "genre_rank" + assert_error: ".*sort.*" diff --git a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_expressions.py b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_expressions.py index a8ca27278506..6dd2a9ccd81b 100644 --- a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_expressions.py +++ b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_expressions.py @@ -27,6 +27,7 @@ Constant, Expression, Field, + FunctionExpression, Ordering, Rank, WindowFunction, @@ -2269,7 +2270,8 @@ def test_array_index_of_all(self): class TestWindowExpressions: def test_window_function_hierarchy(self): - assert issubclass(AggregateFunction, WindowFunction) + assert issubclass(WindowFunction, FunctionExpression) + assert not issubclass(AggregateFunction, WindowFunction) assert not issubclass(Expression, WindowFunction) def test_rank(self): diff --git a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py index e74551bc5a93..5a2713d01a02 100644 --- a/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py +++ b/packages/google-cloud-firestore/tests/unit/v1/test_pipeline_stages.py @@ -1082,6 +1082,13 @@ def test_window_spec_coercions(self): assert WindowSpec(sort=order).sort == [order] assert WindowSpec(sort=[order]).sort == [order] + @pytest.mark.parametrize( + "bad_sort", ["rating", "", b"", ["rating"], Field.of("rating")] + ) + def test_window_spec_invalid_sort(self, bad_sort): + with pytest.raises(TypeError, match="sort must be an Ordering"): + WindowSpec(sort=bad_sort) + def test_window_spec_repr_and_equality(self): empty = WindowSpec() assert repr(empty) == "WindowSpec()"