Repository navigation
feat(firestore): add AddWindowFields stage, WindowSpec, and Rank #19116
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -27,6 +27,7 @@ | |
| Field, | ||
| FunctionExpression, | ||
| Selectable, | ||
| WindowFunction, | ||
| _PipelineValueExpression, | ||
| ) | ||
| from google.cloud.firestore_v1.types.pipeline import ( | ||
|
|
@@ -176,6 +177,60 @@ def add_fields(self, *fields: Selectable) -> "_BasePipeline": | |
| """ | ||
| return self._append(stages.AddFields(*fields)) | ||
|
|
||
| def add_window_fields( | ||
| self, | ||
| *fields: AliasedExpression[AggregateFunction] | ||
| | AliasedExpression[WindowFunction], | ||
| window: types.WindowSpec | None = None, | ||
| ) -> "_BasePipeline": | ||
| """ | ||
| Evaluates window and aggregate functions over a window of related documents | ||
| and adds the results as new fields on each input document. | ||
|
|
||
| Unlike :meth:`aggregate`, which collapses documents into summary rows, | ||
| ``add_window_fields`` preserves every input document and attaches computed | ||
| values (such as partition totals, running aggregates, or rankings) to each row. | ||
|
|
||
| When ``window`` specifies ``sort`` without an explicit frame, aggregates evaluate | ||
| as a running window from the start of the partition through the current sort | ||
| value. When ``sort`` is omitted, aggregates evaluate across all documents in the | ||
| partition. | ||
|
|
||
| .. note:: | ||
| This feature is in preview until the backend feature launches and is | ||
| subject to change. | ||
|
|
||
| Example: | ||
| >>> from google.cloud.firestore_v1.pipeline_expressions import Field, Rank | ||
| >>> from google.cloud.firestore_v1.pipeline_types import WindowSpec | ||
| >>> pipeline = client.pipeline().collection("books") | ||
| >>> # Partition-wide average rating (unsorted window covers the whole partition) | ||
| >>> pipeline = pipeline.add_window_fields( | ||
| ... Field.of("rating").average().as_("genre_avg_rating"), | ||
| ... window=WindowSpec(partition=["genre"]), | ||
| ... ) | ||
| >>> # Rank and running total within each genre (sorted default frame) | ||
| >>> pipeline = pipeline.add_window_fields( | ||
| ... Rank().as_("genre_rank"), | ||
| ... Field.of("price").sum().as_("running_price_total"), | ||
| ... window=WindowSpec( | ||
| ... partition=["genre"], | ||
| ... sort=[Field.of("rating").descending()], | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. IIUC, having this sort here means that the value under We should update the example, and make sure this sort detail is mentioned in this docstring, and in the WindowSpec
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good catch! Updated the example to show both an unsorted partition average and a sorted rank + running total. |
||
| ... ), | ||
| ... ) | ||
|
|
||
| Args: | ||
| *fields: Aliased `AggregateFunction` or `WindowFunction` expressions defining | ||
| the window calculations and their output field aliases. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It seems like other languages require at least one field. Should we do the same here?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Kept |
||
| window: Optional `WindowSpec` specifying how documents are partitioned and | ||
| ordered. If `None`, evaluates over a single unsorted global partition | ||
| covering all input documents (equivalent to ``WindowSpec()``). | ||
|
|
||
| Returns: | ||
| A new Pipeline object with this stage appended to the stage list. | ||
| """ | ||
| return self._append(stages.AddWindowFields(*fields, window=window)) | ||
|
|
||
| def remove_fields(self, *fields: Field | str) -> "_BasePipeline": | ||
| """ | ||
| Removes fields from outputs of previous stages. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -27,6 +27,7 @@ | |
| Expression, | ||
| Field, | ||
| Selectable, | ||
| WindowFunction, | ||
| ) | ||
| from google.cloud.firestore_v1.types.document import Pipeline as Pipeline_pb | ||
| from google.cloud.firestore_v1.types.document import Value | ||
|
|
@@ -42,6 +43,7 @@ | |
| SampleOptions, | ||
| SearchOptions, | ||
| UnnestOptions, | ||
| WindowSpec, | ||
| ) | ||
|
|
||
|
|
||
|
|
@@ -86,6 +88,37 @@ def _pb_args(self): | |
| return [Selectable._to_value(self.fields)] | ||
|
|
||
|
|
||
| class AddWindowFields(Stage): | ||
| """Adds new fields computed by window functions over a window of documents. | ||
|
|
||
| .. note:: | ||
| This feature is in preview until the backend feature launches and is | ||
| subject to change. | ||
| """ | ||
|
|
||
| def __init__( | ||
| self, | ||
| *args: AliasedExpression[AggregateFunction] | AliasedExpression[WindowFunction], | ||
| fields: Sequence[ | ||
| AliasedExpression[AggregateFunction] | AliasedExpression[WindowFunction] | ||
| ] = (), | ||
| window: WindowSpec | None = None, | ||
| ): | ||
| super().__init__("add_window_fields") | ||
| if args and fields: | ||
| raise ValueError( | ||
| "AddWindowFields stage contains both positional and keyword fields" | ||
| ) | ||
| self.fields = list(args or fields) | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Design Decision (
|
||
| self.window: WindowSpec = window if window is not None else WindowSpec() | ||
|
|
||
| def _pb_args(self) -> list[Value]: | ||
| return [ | ||
| self.window._to_pb(), | ||
| Selectable._to_value(self.fields), | ||
| ] | ||
|
|
||
|
|
||
| class Aggregate(Stage): | ||
| """Performs aggregation operations, optionally grouped.""" | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -327,3 +327,83 @@ def __init__(self, index_field: pipeline_expressions.Field | str): | |
|
|
||
| def __repr__(self): | ||
| return f"{self.__class__.__name__}(index_field={self.index_field.path!r})" | ||
|
|
||
|
|
||
| class WindowSpec: | ||
| """Configuration for a window in an `AddWindowFields` pipeline stage. | ||
|
|
||
| .. note:: | ||
| This feature is in preview until the backend feature launches and is | ||
| subject to change. | ||
|
|
||
| Args: | ||
| partition: Optional field paths or expressions used to partition documents into | ||
| independent windows. If `None`, all input documents are treated as a single | ||
| global partition. | ||
| sort: Optional orderings specifying how documents are sorted within each | ||
| partition. If `None`, documents are unordered and aggregates evaluate across | ||
| all documents in the partition. When specified without an explicit frame, | ||
| aggregates default to a running window from the start of the partition | ||
| through the current sort value. | ||
| """ | ||
|
|
||
| def __init__( | ||
| self, | ||
| *, | ||
| partition: Optional[ | ||
| Sequence[pipeline_expressions.Expression | str] | ||
| | pipeline_expressions.Expression | ||
| | str | ||
| ] = None, | ||
| sort: Optional[Sequence[Ordering] | Ordering] = None, | ||
| ): | ||
| if partition is None: | ||
| partition_list = [] | ||
| elif isinstance(partition, (str, pipeline_expressions.Expression)): | ||
| partition_list = [partition] | ||
| else: | ||
| partition_list = list(partition) | ||
| self.partition: list[pipeline_expressions.Expression] = [ | ||
| pipeline_expressions.Field.of(p) if isinstance(p, str) else p | ||
| for p in partition_list | ||
| ] | ||
|
|
||
| if sort is None: | ||
| self.sort: list[Ordering] = [] | ||
| elif isinstance(sort, Ordering): | ||
| self.sort = [sort] | ||
| elif ( | ||
| isinstance(sort, (str, bytes)) | ||
| or not isinstance(sort, Sequence) | ||
| or not all(isinstance(o, Ordering) for o in sort) | ||
| ): | ||
| raise TypeError( | ||
| "sort must be an Ordering or a sequence of Ordering objects" | ||
| ) | ||
| else: | ||
| self.sort = list(sort) | ||
|
|
||
| def __eq__(self, other: object) -> bool: | ||
| if not isinstance(other, WindowSpec): | ||
| return NotImplemented | ||
| return self._to_pb() == other._to_pb() | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Design Decision (
|
||
|
|
||
| def __repr__(self): | ||
| args = [] | ||
| if self.partition: | ||
| args.append(f"partition={self.partition!r}") | ||
| if self.sort: | ||
| args.append(f"sort={self.sort!r}") | ||
| return f"{self.__class__.__name__}({', '.join(args)})" | ||
|
|
||
| def _to_pb(self) -> Value: | ||
| fields: dict[str, Value] = {} | ||
| if self.partition: | ||
| fields["partition"] = Value( | ||
| array_value={"values": [p._to_pb() for p in self.partition]} | ||
| ) | ||
| if self.sort: | ||
| fields["sort"] = Value( | ||
| array_value={"values": [s._to_pb() for s in self.sort]} | ||
| ) | ||
| return Value(map_value={"fields": fields}) | ||
Uh oh!
There was an error while loading. Please reload this page.