From c0fed5278ca651af967e6c3794b59d6eaa8a4586 Mon Sep 17 00:00:00 2001 From: Ky Le Date: Thu, 19 Nov 2026 00:46:42 +0700 Subject: [PATCH] fix(state): skip query optimization during snapshot cleanup Load cleanup snapshots without preparing optimized queries while preserving normal snapshot fetch behavior. Cover cold and warm cleanup metadata and a subsequent optimized fetch. Fixes #6210 Signed-off-by: Ky Le --- sqlmesh/core/snapshot/cache.py | 32 +++++++------ sqlmesh/core/state_sync/db/snapshot.py | 11 ++++- tests/core/state_sync/test_state_sync.py | 58 +++++++++++++++++++++++- 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/sqlmesh/core/snapshot/cache.py b/sqlmesh/core/snapshot/cache.py index d46b5f0620..e582b09ee5 100644 --- a/sqlmesh/core/snapshot/cache.py +++ b/sqlmesh/core/snapshot/cache.py @@ -26,12 +26,15 @@ def get_or_load( self, snapshot_ids: t.Set[SnapshotId], loader: t.Callable[[t.Set[SnapshotId]], t.Collection[Snapshot]], + *, + load_optimized_queries: bool = True, ) -> t.Tuple[t.Dict[SnapshotId, Snapshot], t.Set[SnapshotId]]: """Fetches the target snapshots from cache or loads them using the provided loader on cache miss. Args: snapshot_ids: Target snapshot IDs to fetch. loader: The loader to load snapshot records that are missing in the cache. + load_optimized_queries: Whether to prepare optimized queries for model snapshots. Returns: A tuple where the first value represents the fetched snapshots, and the second value is a set of @@ -55,24 +58,25 @@ def get_or_load( for snapshot in loaded_snapshots: snapshots[snapshot.snapshot_id] = snapshot - with optimized_query_cache_pool(self._optimized_query_cache) as executor: - for key, entry_name in executor.map( - load_optimized_query, - ( - (snapshot.model, s_id) - for s_id, snapshot in snapshots.items() - if snapshot.is_model - ), - ): - if entry_name: - self._optimized_query_cache.with_optimized_query( - snapshots[key].model, entry_name - ) + if load_optimized_queries: + with optimized_query_cache_pool(self._optimized_query_cache) as executor: + for key, entry_name in executor.map( + load_optimized_query, + ( + (snapshot.model, s_id) + for s_id, snapshot in snapshots.items() + if snapshot.is_model + ), + ): + if entry_name: + self._optimized_query_cache.with_optimized_query( + snapshots[key].model, entry_name + ) for snapshot in snapshots.values(): self._update_node_hash_cache(snapshot) - if snapshot.is_model and c.MAX_FORK_WORKERS == 1: + if load_optimized_queries and snapshot.is_model and c.MAX_FORK_WORKERS == 1: try: self._optimized_query_cache.with_optimized_query(snapshot.model) except Exception: diff --git a/sqlmesh/core/state_sync/db/snapshot.py b/sqlmesh/core/state_sync/db/snapshot.py index 9b4337b504..458029f498 100644 --- a/sqlmesh/core/state_sync/db/snapshot.py +++ b/sqlmesh/core/state_sync/db/snapshot.py @@ -274,7 +274,10 @@ def _is_snapshot_used(snapshot: SnapshotIdAndVersion) -> bool: cleanup_targets.append((snapshot.snapshot_id, dev_table_only)) snapshot_ids_to_cleanup = [snapshot_id for snapshot_id, _ in cleanup_targets] - full_snapshots = self._get_snapshots(snapshot_ids_to_cleanup) + full_snapshots = self._get_snapshots( + snapshot_ids_to_cleanup, + load_optimized_queries=False, + ) for snapshot_id, dev_table_only in cleanup_targets: if snapshot_id in full_snapshots: cleanup_tasks.append( @@ -509,12 +512,14 @@ def _get_snapshots( self, snapshot_ids: t.Iterable[SnapshotIdLike], lock_for_update: bool = False, + load_optimized_queries: bool = True, ) -> t.Dict[SnapshotId, Snapshot]: """Fetches specified snapshots or all snapshots. Args: snapshot_ids: The collection of snapshot like objects to fetch. lock_for_update: Lock the snapshot rows for future update + load_optimized_queries: Whether to prepare optimized queries for model snapshots. Returns: A dictionary of snapshot ids to snapshots for ones that could be found. @@ -555,7 +560,9 @@ def _loader(snapshot_ids_to_load: t.Set[SnapshotId]) -> t.Collection[Snapshot]: return fetched_snapshots.values() snapshots, cached_snapshots = self._snapshot_cache.get_or_load( - {s.snapshot_id for s in snapshot_ids}, _loader + {s.snapshot_id for s in snapshot_ids}, + _loader, + load_optimized_queries=load_optimized_queries, ) if cached_snapshots: diff --git a/tests/core/state_sync/test_state_sync.py b/tests/core/state_sync/test_state_sync.py index 503f89296e..082607ec05 100644 --- a/tests/core/state_sync/test_state_sync.py +++ b/tests/core/state_sync/test_state_sync.py @@ -11,7 +11,7 @@ from pytest_mock.plugin import MockerFixture from sqlglot import exp -from sqlmesh.core import constants as c +from sqlmesh.core import constants as c, renderer from sqlmesh.core.config import EnvironmentSuffixTarget from sqlmesh.core.dialect import parse_one from sqlmesh.core.engine_adapter import create_engine_adapter @@ -1385,6 +1385,58 @@ def test_delete_expired_snapshots(state_sync: EngineAdapterStateSync, make_snaps assert not state_sync.get_snapshots(all_snapshots) +def test_get_expired_snapshots_does_not_optimize_queries( + state_sync: EngineAdapterStateSync, + make_snapshot: t.Callable, +) -> None: + snapshot = make_snapshot( + SqlModel( + name="demo.retired", + dialect="duckdb", + query=parse_one("select 1 as id"), + ), + ) + snapshot.ttl = "in 1 day" + snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + state_sync.push_snapshots([snapshot]) + + now_ts = now_timestamp() + with patch.object(renderer, "qualify", wraps=renderer.qualify) as qualify: + unexpired = state_sync.get_expired_snapshots( + batch_range=ExpiredBatchRange.all_batch_range(), + current_ts=now_ts, + ) + assert unexpired is None + assert qualify.call_count == 0 + + expired = state_sync.get_expired_snapshots( + batch_range=ExpiredBatchRange.all_batch_range(), + current_ts=now_ts + 2 * 24 * 60 * 60 * 1000, + ) + + assert expired is not None + assert expired.expired_snapshot_ids == {snapshot.snapshot_id} + assert len(expired.cleanup_tasks) == 1 + cleanup_task = expired.cleanup_tasks[0] + assert cleanup_task.dev_table_only is False + assert cleanup_task.snapshot.model_dump() == snapshot.table_info.model_dump() + assert cleanup_task.snapshot.table_name() == snapshot.table_info.table_name() + assert cleanup_task.snapshot.table_name( + is_deployable=False + ) == snapshot.table_info.table_name(is_deployable=False) + assert qualify.call_count == 0 + + # Warm snapshot metadata should not trigger optimization either. + state_sync.get_expired_snapshots( + batch_range=ExpiredBatchRange.all_batch_range(), + current_ts=now_ts + 2 * 24 * 60 * 60 * 1000, + ) + assert qualify.call_count == 0 + + state_sync.get_snapshots([snapshot.snapshot_id]) + assert qualify.call_count > 0 + + def test_get_expired_snapshot_batch(state_sync: EngineAdapterStateSync, make_snapshot: t.Callable): now_ts = now_timestamp() @@ -3692,7 +3744,9 @@ def test_snapshot_cache( state_sync.snapshot_state.push_snapshots([snapshot]) assert state_sync.get_snapshots([snapshot.snapshot_id]) == {snapshot.snapshot_id: snapshot} - cache_mock.get_or_load.assert_called_once_with({snapshot.snapshot_id}, mocker.ANY) + cache_mock.get_or_load.assert_called_once_with( + {snapshot.snapshot_id}, mocker.ANY, load_optimized_queries=True + ) # Update the snapshot in the state and make sure this update is reflected on the cached instance. assert snapshot.unpaused_ts is None