Skip to content

Commit 6a563f5

Browse files
authored
fix(clickhouse): initialize virtual catalog for cleanup (#5941)
1 parent bc76e47 commit 6a563f5

2 files changed

Lines changed: 382 additions & 2 deletions

File tree

sqlmesh/core/context.py

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3070,10 +3070,17 @@ def _cleanup_environments(
30703070
expired_env = self.state_reader.get_environment(expired_env_summary.name)
30713071

30723072
if expired_env:
3073+
cleanup_default_adapter, cleanup_engine_adapters, failure = (
3074+
self._cleanup_adapters_for_environment(expired_env)
3075+
)
3076+
if failure:
3077+
logger.warning(failure)
3078+
failures.append(failure)
3079+
continue
30733080
failures.extend(
30743081
cleanup_expired_views(
3075-
default_adapter=self.engine_adapter,
3076-
engine_adapters=self.engine_adapters,
3082+
default_adapter=cleanup_default_adapter,
3083+
engine_adapters=cleanup_engine_adapters,
30773084
environments=[expired_env],
30783085
console=self.console,
30793086
)
@@ -3085,6 +3092,62 @@ def _cleanup_environments(
30853092
self.state_sync.delete_expired_environments(current_ts=current_ts, name=name)
30863093
return failures
30873094

3095+
def _cleanup_adapters_for_environment(
3096+
self, environment: Environment
3097+
) -> t.Tuple[EngineAdapter, t.Dict[str, EngineAdapter], t.Optional[str]]:
3098+
"""Create cleanup-scoped adapters for an expired environment.
3099+
3100+
Persisted catalog-qualified view names indicate that virtual catalog injection was active,
3101+
so cleanup can clone only the selected adapters with the historical catalog and leave the
3102+
context's adapters unchanged.
3103+
"""
3104+
engine_adapters = self.engine_adapters
3105+
default_adapter = self.engine_adapter
3106+
catalogs_by_gateway: t.Dict[str, t.Set[str]] = collections.defaultdict(set)
3107+
3108+
for snapshot in environment.snapshots:
3109+
if not snapshot.is_model or snapshot.is_symbolic:
3110+
continue
3111+
3112+
gateway = (
3113+
snapshot.model_gateway
3114+
if environment.gateway_managed and snapshot.model_gateway in engine_adapters
3115+
else self.selected_gateway
3116+
)
3117+
adapter = engine_adapters.get(gateway, default_adapter)
3118+
catalog = snapshot.qualified_view_name.catalog_for_environment(
3119+
environment.naming_info, dialect=adapter.dialect
3120+
)
3121+
if catalog and adapter.supports_virtual_catalog() is True:
3122+
catalogs_by_gateway[gateway].add(catalog)
3123+
3124+
for gateway, catalogs in catalogs_by_gateway.items():
3125+
if len(catalogs) > 1:
3126+
catalogs_description = ", ".join(f"'{catalog}'" for catalog in sorted(catalogs))
3127+
return (
3128+
default_adapter,
3129+
engine_adapters,
3130+
(
3131+
f"Failed to clean up expired environment '{environment.name}': gateway "
3132+
f"'{gateway}' references multiple virtual catalogs: {catalogs_description}"
3133+
),
3134+
)
3135+
3136+
cleanup_engine_adapters = engine_adapters.copy()
3137+
cleanup_default_adapter = default_adapter
3138+
for gateway, catalogs in catalogs_by_gateway.items():
3139+
cleanup_adapter = engine_adapters.get(gateway, default_adapter).with_settings()
3140+
cleanup_adapter.inject_virtual_catalog(gateway)
3141+
# inject_virtual_catalog() may initialize adapter-specific state in addition to
3142+
# _default_catalog. Override only the cleanup clone with the catalog persisted in the
3143+
# expired environment so historical names pass SINGLE_CATALOG_ONLY validation.
3144+
cleanup_adapter._default_catalog = next(iter(catalogs))
3145+
cleanup_engine_adapters[gateway] = cleanup_adapter
3146+
if gateway == self.selected_gateway:
3147+
cleanup_default_adapter = cleanup_adapter
3148+
3149+
return cleanup_default_adapter, cleanup_engine_adapters, None
3150+
30883151
def _try_connection(self, connection_name: str, validator: t.Callable[[], None]) -> None:
30893152
connection_name = connection_name.capitalize()
30903153
try:

0 commit comments

Comments
 (0)