Skip to content

Commit 127a9dd

Browse files
committed
fix(duckdb): resolve 2-part catalog targets and use SELECT * REPLACE for merge coercion
- Resolve 2-part <catalog>.<table> targets via _get_target_catalog_type() before falling back to get_current_catalog() in both _supports_native_merge() and _drop_object(). - Replace manual AST column scraping in _prepare_merge_source_query() with DuckDB SELECT * REPLACE (...) so subqueries in WHEN MATCHED / merge_filter, unaliased expressions, duplicate unused join columns, struct field access, and quoted/case-mismatched identifiers are handled cleanly. Signed-off-by: Vamsi-klu <nrvamsi13@gmail.com>
1 parent 195a384 commit 127a9dd

2 files changed

Lines changed: 364 additions & 63 deletions

File tree

‎sqlmesh/core/engine_adapter/duckdb.py‎

Lines changed: 43 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -89,11 +89,8 @@ def _duckdb_supports_merge(self) -> bool:
8989
def _supports_native_merge(self, target_table: TableName) -> bool:
9090
if not self._duckdb_supports_merge:
9191
return False
92-
catalog = (
93-
exp.to_table(target_table, dialect=self.dialect).catalog or self.get_current_catalog()
94-
)
9592
# Errors from the catalog lookup propagate on purpose; only a None/unknown type falls back.
96-
catalog_type = (self._get_catalog_type(catalog) or "").lower()
93+
catalog_type = (self._get_target_catalog_type(target_table) or "").lower()
9794
return catalog_type in self.NATIVE_MERGE_CATALOG_TYPES
9895

9996
def merge(
@@ -140,65 +137,40 @@ def _prepare_merge_source_query(
140137

141138
unique_key = unique_key or []
142139

143-
if not target_columns_to_types or all(
144-
not dtype or dtype.is_type(exp.DataType.Type.UNKNOWN)
145-
for dtype in target_columns_to_types.values()
146-
):
147-
return exp.select("*").distinct(*unique_key).from_(query.subquery("_src", copy=False)) # type: ignore
140+
if target_columns_to_types:
141+
source_columns_set = (
142+
{s.lower() for s in source_columns} if source_columns is not None else None
143+
)
144+
target_cols_to_project = (
145+
[col for col in target_columns_to_types if col.lower() in source_columns_set]
146+
if source_columns_set is not None
147+
else list(target_columns_to_types)
148+
)
149+
replacements = [
150+
exp.alias_(
151+
exp.cast(exp.column(col, quoted=True), to=dtype),
152+
col,
153+
quoted=True,
154+
copy=False,
155+
)
156+
for col in target_cols_to_project
157+
if (dtype := target_columns_to_types[col])
158+
and not dtype.is_type(exp.DataType.Type.UNKNOWN)
159+
]
160+
else:
161+
replacements = []
148162

149-
source_columns_set = (
150-
{s.lower() for s in source_columns} if source_columns is not None else None
151-
)
152-
target_cols_to_project = (
153-
[col for col in target_columns_to_types if col.lower() in source_columns_set]
154-
if source_columns_set is not None
155-
else list(target_columns_to_types)
156-
)
157-
projected_target_cols = {col.lower(): col for col in target_cols_to_project}
158-
159-
projections: t.List[exp.Expr] = []
160-
for col in target_cols_to_project:
161-
dtype = target_columns_to_types[col]
162-
if dtype and not dtype.is_type(exp.DataType.Type.UNKNOWN):
163-
projections.append(exp.alias_(exp.cast(exp.column(col), to=dtype), col, copy=False))
164-
else:
165-
projections.append(exp.column(col))
166-
167-
seen_extra: t.Set[str] = set()
168-
extra_cols: t.List[str] = []
169-
target_names = {
170-
MERGE_TARGET_ALIAS,
171-
exp.to_table(target_table, dialect=self.dialect).name.upper(),
172-
}
173-
for expr in [when_matched, merge_filter, *unique_key]:
174-
if expr:
175-
for column_expr in expr.find_all(exp.Column):
176-
if column_expr.table and column_expr.table.upper() in target_names:
177-
continue
178-
col_name = column_expr.name
179-
if (
180-
col_name.lower() not in projected_target_cols
181-
and col_name.lower() not in seen_extra
182-
):
183-
seen_extra.add(col_name.lower())
184-
extra_cols.append(col_name)
185-
186-
inner_query = query.this if isinstance(query, exp.Subquery) else query
187-
if isinstance(inner_query, exp.Select):
188-
for name in inner_query.named_selects:
189-
if (
190-
name != "*"
191-
and name.lower() not in projected_target_cols
192-
and name.lower() not in seen_extra
193-
):
194-
seen_extra.add(name.lower())
195-
extra_cols.append(name)
163+
select = exp.select("*")
164+
if unique_key:
165+
select = select.distinct(*unique_key)
196166

197-
for extra in extra_cols:
198-
projections.append(exp.column(extra))
167+
if not replacements:
168+
return select.from_(query.subquery("_src"), copy=False)
199169

200-
coerced = exp.select(*projections).from_(query.subquery("_raw_src", copy=False), copy=False)
201-
return exp.select("*").distinct(*unique_key).from_(coerced.subquery("_src", copy=False)) # type: ignore
170+
coerced = exp.select(exp.Star(replace=replacements)).from_(
171+
query.subquery("_raw_src"), copy=False
172+
)
173+
return select.from_(coerced.subquery("_src", copy=False), copy=False)
202174

203175
def _merge(
204176
self,
@@ -379,12 +351,21 @@ def _drop_object(
379351
# "Cascade Drop not supported in DuckLake". Views in DuckDB are late-binding, so
380352
# dropping the underlying table without CASCADE is safe there.
381353
if cascade and kind.upper() in ("TABLE", "VIEW"):
382-
catalog = exp.to_table(name).catalog or self.get_current_catalog()
383-
if self._get_catalog_type(catalog) == "ducklake":
354+
if (self._get_target_catalog_type(name) or "").lower() == "ducklake":
384355
cascade = False
385356

386357
super()._drop_object(name=name, exists=exists, kind=kind, cascade=cascade, **drop_args)
387358

359+
def _get_target_catalog_type(self, name: TableName) -> t.Optional[str]:
360+
table = exp.to_table(name, dialect=self.dialect)
361+
if table.catalog:
362+
return self._get_catalog_type(table.catalog)
363+
if table.db:
364+
db_catalog_type = self._get_catalog_type(table.db)
365+
if db_catalog_type is not None:
366+
return db_catalog_type
367+
return self._get_catalog_type(self.get_current_catalog())
368+
388369
def _get_catalog_type(self, catalog: t.Optional[str]) -> t.Optional[str]:
389370
"""Returns the type of the given catalog (e.g. 'duckdb', 'ducklake') as reported by duckdb_databases()."""
390371
if not catalog:

0 commit comments

Comments
 (0)