Repository navigation
feat(firestore): add DocumentWindowFrame and WindowBound for pipeline windows #19117
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
3f19d36
4f3d570
caeffe3
e9c640e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,6 +15,7 @@ | |
|
|
||
| from __future__ import annotations | ||
|
|
||
| from dataclasses import dataclass, fields | ||
| from enum import Enum | ||
| from typing import Optional, Sequence | ||
|
|
||
|
|
@@ -329,12 +330,69 @@ def __repr__(self): | |
| return f"{self.__class__.__name__}(index_field={self.index_field.path!r})" | ||
|
|
||
|
|
||
| class WindowBound(str, Enum): | ||
| """Enumeration of symbolic window frame boundary keywords.""" | ||
|
|
||
| CURRENT = "current" | ||
| UNBOUNDED = "unbounded" | ||
|
|
||
|
|
||
| @dataclass | ||
| class _WindowFrame: | ||
| """Base class for window frame specifications.""" | ||
|
|
||
| def _to_pb(self) -> Value: | ||
| pb_fields = { | ||
| f.name: pipeline_expressions.Expression._cast_to_expr_or_convert_to_constant( | ||
| getattr(self, f.name) | ||
| )._to_pb() | ||
| for f in fields(self) | ||
| if getattr(self, f.name) is not None | ||
| } | ||
| return Value(map_value={"fields": pb_fields}) | ||
|
|
||
|
|
||
| @dataclass | ||
| class DocumentWindowFrame(_WindowFrame): | ||
| """Defines a document-count based window frame relative to the current document. | ||
|
|
||
| Example: | ||
| >>> from google.cloud.firestore_v1.pipeline_expressions import Field | ||
| >>> from google.cloud.firestore_v1.pipeline_types import ( | ||
| ... DocumentWindowFrame, | ||
| ... WindowBound, | ||
| ... WindowSpec, | ||
| ... ) | ||
| >>> window = WindowSpec( | ||
| ... sort=Field.of("published").ascending(), | ||
| ... frame=DocumentWindowFrame( | ||
| ... WindowBound.UNBOUNDED, WindowBound.CURRENT | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: the example could be more clear with preceding and following kwargs present |
||
| ... ), | ||
| ... ) | ||
|
|
||
| Args: | ||
| preceding: The lower bound of the frame (an integer offset, `WindowBound`, or `Expression`). | ||
| A positive integer includes up to that many documents before the current document. | ||
| A negative integer indicates a boundary following the current document. | ||
| following: The upper bound of the frame (an integer offset, `WindowBound`, or `Expression`). | ||
| A positive integer includes up to that many documents after the current document. | ||
| A negative integer indicates a boundary preceding the current document. | ||
| """ | ||
|
|
||
| preceding: WindowBound | str | int | pipeline_expressions.Expression | ||
| following: WindowBound | str | int | pipeline_expressions.Expression | ||
|
Comment on lines
+355
to
+383
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Design Note —
|
||
|
|
||
| def _to_dict(self) -> dict[str, Value]: | ||
| return {"documents": self._to_pb()} | ||
|
|
||
|
|
||
| class WindowSpec: | ||
| """Configuration for a window in an `AddWindowFields` pipeline stage. | ||
|
|
||
| Args: | ||
| partition: Field paths or expressions used to partition documents into independent windows. | ||
| sort: Orderings specifying how documents are sorted within each partition. | ||
| frame: A `DocumentWindowFrame` defining a document-count window frame. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What happens if None? |
||
| """ | ||
|
|
||
| def __init__( | ||
|
|
@@ -346,6 +404,7 @@ def __init__( | |
| | str | ||
| ] = None, | ||
| sort: Optional[Sequence[Ordering] | Ordering] = None, | ||
| frame: Optional[DocumentWindowFrame] = None, | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Design Note —
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This seems reasonable to me, but the fact that all the other libraries use a different pattern makes me a bit nervous. But that said, I can't think of any real concerns with this current approach, and it does seem cleaner/nicer to me in isolation. So LGTM if you want to keep it as-is |
||
| ): | ||
| if partition is None: | ||
| partition_list = [] | ||
|
|
@@ -363,6 +422,8 @@ def __init__( | |
| else: | ||
| self.sort = [sort] if isinstance(sort, Ordering) else list(sort) | ||
|
|
||
| self.frame: Optional[DocumentWindowFrame] = frame | ||
|
|
||
| def __eq__(self, other: object) -> bool: | ||
| if not isinstance(other, WindowSpec): | ||
| return NotImplemented | ||
|
daniel-sanche marked this conversation as resolved.
Resolved
|
||
|
|
@@ -374,6 +435,8 @@ def __repr__(self): | |
| args.append(f"partition={self.partition!r}") | ||
| if self.sort: | ||
| args.append(f"sort={self.sort!r}") | ||
| if self.frame is not None: | ||
| args.append(f"frame={self.frame!r}") | ||
| return f"{self.__class__.__name__}({', '.join(args)})" | ||
|
|
||
| def _to_pb(self) -> Value: | ||
|
|
@@ -386,4 +449,6 @@ def _to_pb(self) -> Value: | |
| fields["sort"] = Value( | ||
| array_value={"values": [s._to_pb() for s in self.sort]} | ||
| ) | ||
| if self.frame is not None: | ||
| fields.update(self.frame._to_dict()) | ||
| return Value(map_value={"fields": fields}) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,129 @@ | ||
| tests: | ||
| - description: "testAddWindowFields - partition, sort, documents frame, and multiple accumulators" | ||
| pipeline: | ||
| - Collection: books | ||
| - Where: | ||
| - FunctionExpression.equal_any: | ||
| - Field: genre | ||
| - [Science Fiction, Dystopian] | ||
| - AddWindowFields: | ||
| window: | ||
| WindowSpec: | ||
| partition: | ||
| - Field: genre | ||
| sort: | ||
| - Ordering: | ||
| - Field: published | ||
| - ASCENDING | ||
| frame: | ||
| DocumentWindowFrame: | ||
| preceding: unbounded | ||
| following: current | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can we have a test with a numeric offset? (Maybe ask gemini for more test cases too) |
||
| fields: | ||
| - AliasedExpression: | ||
| - FunctionExpression.sum: | ||
| - Field: rating | ||
| - "running_rating" | ||
| - AliasedExpression: | ||
| - Count | ||
| - "running_count" | ||
| - Sort: | ||
| - Ordering: | ||
| - Field: published | ||
| - ASCENDING | ||
| - Select: | ||
| - title | ||
| - genre | ||
| - running_rating | ||
| - running_count | ||
| assert_results: | ||
| - title: "1984" | ||
| genre: "Dystopian" | ||
| running_rating: 4.2 | ||
| running_count: 1 | ||
| - title: "Dune" | ||
| genre: "Science Fiction" | ||
| running_rating: 4.6 | ||
| running_count: 1 | ||
| - title: "The Hitchhiker's Guide to the Galaxy" | ||
| genre: "Science Fiction" | ||
| running_rating: 8.8 | ||
| running_count: 2 | ||
| - title: "The Handmaid's Tale" | ||
| genre: "Dystopian" | ||
| running_rating: 8.3 | ||
| running_count: 2 | ||
| assert_proto: | ||
| pipeline: | ||
| stages: | ||
| - args: | ||
| - referenceValue: /books | ||
| name: collection | ||
| - args: | ||
| - functionValue: | ||
| args: | ||
| - fieldReferenceValue: genre | ||
| - functionValue: | ||
| args: | ||
| - stringValue: Science Fiction | ||
| - stringValue: Dystopian | ||
| name: array | ||
| name: equal_any | ||
| name: where | ||
| - args: | ||
| - mapValue: | ||
| fields: | ||
| documents: | ||
| mapValue: | ||
| fields: | ||
| following: | ||
| stringValue: current | ||
| preceding: | ||
| stringValue: unbounded | ||
| partition: | ||
| arrayValue: | ||
| values: | ||
| - fieldReferenceValue: genre | ||
| sort: | ||
| arrayValue: | ||
| values: | ||
| - mapValue: | ||
| fields: | ||
| direction: | ||
| stringValue: ascending | ||
| expression: | ||
| fieldReferenceValue: published | ||
| - mapValue: | ||
| fields: | ||
| running_count: | ||
| functionValue: | ||
| name: count | ||
| running_rating: | ||
| functionValue: | ||
| args: | ||
| - fieldReferenceValue: rating | ||
| name: sum | ||
| name: add_window_fields | ||
| - args: | ||
| - mapValue: | ||
| fields: | ||
| direction: | ||
| stringValue: ascending | ||
| expression: | ||
| fieldReferenceValue: published | ||
| name: sort | ||
| - args: | ||
| - mapValue: | ||
| fields: | ||
| genre: | ||
| fieldReferenceValue: genre | ||
| running_count: | ||
| fieldReferenceValue: running_count | ||
| running_rating: | ||
| fieldReferenceValue: running_rating | ||
| title: | ||
| fieldReferenceValue: title | ||
| name: select | ||
| - description: "testAddWindowFields - rank with ties" | ||
| pipeline: | ||
| - Collection: books | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
A bit more context to include: