Repository navigation
fix(metrics): preserve nested aggregate expressions #6225
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -141,25 +141,75 @@ def to_metric( | |
| class Metric(MetricMeta, frozen=True): | ||
| expanded: exp.Expr | ||
|
|
||
| def _aggregate_output_names( | ||
| self, | ||
| aggregates: t.Optional[t.List[exp.AggFunc]] = None, | ||
| used_names: t.Optional[t.Set[str]] = None, | ||
| ) -> t.List[str]: | ||
| """Returns the output column name for each aggregate in the expanded metric.""" | ||
| nested_aggregate_names: t.Dict[exp.AggFunc, str] = {} | ||
| output_names = [] | ||
| used_names = used_names if used_names is not None else set() | ||
|
|
||
| if aggregates is None: | ||
| aggregates = list(self.expanded.find_all(exp.AggFunc)) | ||
|
|
||
| for agg in aggregates: | ||
| if isinstance(agg.parent, exp.Alias): | ||
| output_names.append(agg.parent.alias) | ||
| used_names.add(agg.parent.alias) | ||
| continue | ||
|
|
||
| if agg not in nested_aggregate_names: | ||
| base_name = ( | ||
| f"__sqlmesh_metric_{self.name.replace('.', '_')}__agg_" | ||
| f"{len(nested_aggregate_names)}" | ||
| ) | ||
| output_name = base_name | ||
| suffix = 1 | ||
| while output_name in used_names: | ||
| output_name = f"{base_name}_{suffix}" | ||
| suffix += 1 | ||
| nested_aggregate_names[agg] = output_name | ||
| used_names.add(output_name) | ||
| output_names.append(nested_aggregate_names[agg]) | ||
|
|
||
| return output_names | ||
|
|
||
| @property | ||
| def aggs(self) -> t.Dict[exp.AggFunc, MeasureAndDimTables]: | ||
| """Returns a dictionary of aggregation to referenced tables. | ||
| This method removes catalog and schema information from columns. | ||
| """ | ||
| return { | ||
| t.cast( | ||
| exp.AggFunc, | ||
| t.cast(exp.Expr, agg.parent).transform( | ||
| lambda node: ( | ||
| exp.column(node.this, table=remove_namespace(node)) | ||
| if isinstance(node, exp.Column) and node.table | ||
| else node | ||
| ) | ||
| ), | ||
| ): _get_measure_and_dim_tables(agg) | ||
| for agg in self.expanded.find_all(exp.AggFunc) | ||
| } | ||
| aggregates = list(self.expanded.find_all(exp.AggFunc)) | ||
| return self._aggs_with_output_names(self._aggregate_output_names(aggregates)) | ||
|
|
||
| def _aggs_with_output_names( | ||
| self, output_names: t.List[str] | ||
| ) -> t.Dict[exp.AggFunc, MeasureAndDimTables]: | ||
| aggs: t.Dict[exp.AggFunc, MeasureAndDimTables] = {} | ||
| aggregates = list(self.expanded.find_all(exp.AggFunc)) | ||
| for agg, output_name in zip(aggregates, output_names): | ||
| projection = ( | ||
| agg.parent | ||
| if isinstance(agg.parent, exp.Alias) | ||
| else exp.alias_(agg.copy(), output_name) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Keep For
|
||
| ) | ||
| aggs[ | ||
| t.cast( | ||
| exp.AggFunc, | ||
| projection.transform( | ||
| lambda node: ( | ||
| exp.column(node.this, table=remove_namespace(node)) | ||
| if isinstance(node, exp.Column) and node.table | ||
| else node | ||
| ) | ||
| ), | ||
| ) | ||
| ] = _get_measure_and_dim_tables(agg) | ||
|
|
||
| return aggs | ||
|
|
||
| @property | ||
| def formula(self) -> exp.Expr: | ||
|
|
@@ -168,10 +218,40 @@ def formula(self) -> exp.Expr: | |
| For simple metrics it is just the metric name. For derived metrics, | ||
| it consists of the operations of the derived metrics without aggregations. | ||
| """ | ||
| return self._formula_with_output_names(self._aggregate_output_names()) | ||
|
|
||
| def _formula_with_output_names(self, output_names: t.List[str]) -> exp.Expr: | ||
| formula = ( | ||
| self.expanded.this.copy() | ||
| if isinstance(self.expanded, exp.Alias) | ||
| else self.expanded.copy() | ||
| ) | ||
| formula_aggregates = list(formula.find_all(exp.AggFunc)) | ||
| aggregate_output_names = { | ||
| id(agg): output_name for agg, output_name in zip(formula_aggregates, output_names) | ||
| } | ||
|
|
||
| def _replace_metric_alias(node: exp.Expr) -> exp.Expr: | ||
| if not isinstance(node, exp.Alias): | ||
| return node | ||
|
|
||
| if isinstance(node.this, exp.AggFunc): | ||
| return exp.Column(this=exp.to_identifier(node.alias)) | ||
| if node.this.find(exp.AggFunc): | ||
| return node.this | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Preserve operator precedence when unwrapping a derived metric. If
|
||
| return exp.Column(this=exp.to_identifier(node.alias)) | ||
|
|
||
| formula = formula.transform(_replace_metric_alias, copy=False) | ||
|
|
||
| def _replace_aggregate(node: exp.Expr) -> exp.Expr: | ||
| if not isinstance(node, exp.AggFunc): | ||
| return node | ||
| output_name = aggregate_output_names[id(node)] | ||
| return exp.Column(this=exp.to_identifier(output_name)) | ||
|
|
||
| formula = formula.transform(_replace_aggregate, copy=False) | ||
| return exp.alias_( | ||
| self.expanded.transform( | ||
| lambda node: exp.column(node.args["alias"]) if isinstance(node, exp.Alias) else node | ||
| ), | ||
| formula, | ||
| self.name, | ||
| copy=False, | ||
| ) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -40,15 +40,37 @@ def rewrite(self, expression: exp.Expr) -> exp.Expr: | |
|
|
||
| return expression | ||
|
|
||
| def _build_sources(self, projections: t.List[exp.Expr]) -> SourceAggsAndJoins: | ||
| def _build_sources( | ||
| self, projections: t.List[exp.Expr], group_by: t.List[exp.Expr] | ||
| ) -> SourceAggsAndJoins: | ||
| sources: SourceAggsAndJoins = {} | ||
| used_names = {metric.name for metric in self.metrics.values()} | ||
| used_names.update( | ||
| projection.alias if isinstance(projection, exp.Alias) else projection.output_name | ||
| for projection in projections | ||
| if projection.alias_or_name | ||
| ) | ||
| used_names.update( | ||
| expression.output_name for expression in group_by if expression.output_name | ||
| ) | ||
|
Comment on lines
+53
to
+55
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Compare reserved names using dialect identifier resolution. In Snowflake, an uppercase grain such as Reserve names according to the dialect’s identifier rules, including quoted identifiers.
|
||
| for metric in self.metrics.values(): | ||
| used_names.update( | ||
| node.alias for node in metric.expanded.find_all(exp.Alias) if node.alias | ||
| ) | ||
|
|
||
| output_names_by_metric: t.Dict[str, t.List[str]] = {} | ||
|
|
||
| for projection in projections: | ||
| for ref in find_all_in_scope(projection, d.MetricAgg): | ||
| metric = self.metrics[ref.this.name] | ||
| ref.replace(metric.formula.this) | ||
|
|
||
| for agg, (measure, dims) in metric.aggs.items(): | ||
| output_names = output_names_by_metric.get(metric.name) | ||
| if output_names is None: | ||
| aggregates = list(metric.expanded.find_all(exp.AggFunc)) | ||
| output_names = metric._aggregate_output_names(aggregates, used_names) | ||
| output_names_by_metric[metric.name] = output_names | ||
| ref.replace(metric._formula_with_output_names(output_names).this) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Parenthesize a substituted metric formula when it is an operator operand.
Keep the grouping whenever the substituted formula is an operand of an operator expression. A standalone projection such as
|
||
|
|
||
| for agg, (measure, dims) in metric._aggs_with_output_names(output_names).items(): | ||
| aggs, joins = sources.setdefault(measure, (set(), dict())) | ||
| aggs.add(agg) | ||
| for dim in dims: | ||
|
|
@@ -60,14 +82,13 @@ def _expand(self, select: exp.Select) -> None: | |
| base = select.args["from_"].this.find(exp.Table) | ||
| base_alias = base.alias_or_name | ||
| base_name = exp.table_name(base) | ||
| group = select.args.pop("group", None) | ||
| group_by = group.expressions if group else [] | ||
|
|
||
| sources: SourceAggsAndJoins = ( | ||
| {} if base_name == self.semantic_name else {base_name: (set(), {})} | ||
| ) | ||
| sources.update(self._build_sources(select.selects)) | ||
|
|
||
| group = select.args.pop("group", None) | ||
| group_by = group.expressions if group else [] | ||
| sources.update(self._build_sources(select.selects, group_by)) | ||
|
|
||
| mapping = { | ||
| remove_namespace(exp.table_name(source.assert_is(exp.Table))): name | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Keep generated aliases distinct within PostgreSQL's identifier limit.
A 50-character ASCII metric name is valid on PostgreSQL. If its formula uses
SUM(amount) + MAX(amount), both generated aliases share their first 63 bytes. PostgreSQL truncates the aliases and their references to the same name, so the rewritten query has ambiguous columns. Use bounded internal aliases and leave room for the uniqueness suffix. Keep the public metric name unchanged. (postgresql.org)