Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions sqlmesh/core/snapshot/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,15 @@ def get_or_load(
self,
snapshot_ids: t.Set[SnapshotId],
loader: t.Callable[[t.Set[SnapshotId]], t.Collection[Snapshot]],
*,
load_optimized_queries: bool = True,
) -> t.Tuple[t.Dict[SnapshotId, Snapshot], t.Set[SnapshotId]]:
"""Fetches the target snapshots from cache or loads them using the provided loader on cache miss.

Args:
snapshot_ids: Target snapshot IDs to fetch.
loader: The loader to load snapshot records that are missing in the cache.
load_optimized_queries: Whether to prepare optimized queries for model snapshots.

Returns:
A tuple where the first value represents the fetched snapshots, and the second value is a set of
Expand All @@ -55,24 +58,25 @@ def get_or_load(
for snapshot in loaded_snapshots:
snapshots[snapshot.snapshot_id] = snapshot

with optimized_query_cache_pool(self._optimized_query_cache) as executor:
for key, entry_name in executor.map(
load_optimized_query,
(
(snapshot.model, s_id)
for s_id, snapshot in snapshots.items()
if snapshot.is_model
),
):
if entry_name:
self._optimized_query_cache.with_optimized_query(
snapshots[key].model, entry_name
)
if load_optimized_queries:
with optimized_query_cache_pool(self._optimized_query_cache) as executor:
for key, entry_name in executor.map(
load_optimized_query,
(
(snapshot.model, s_id)
for s_id, snapshot in snapshots.items()
if snapshot.is_model
),
):
if entry_name:
self._optimized_query_cache.with_optimized_query(
snapshots[key].model, entry_name
)

for snapshot in snapshots.values():
self._update_node_hash_cache(snapshot)

if snapshot.is_model and c.MAX_FORK_WORKERS == 1:
if load_optimized_queries and snapshot.is_model and c.MAX_FORK_WORKERS == 1:
try:
self._optimized_query_cache.with_optimized_query(snapshot.model)
except Exception:
Expand Down
11 changes: 9 additions & 2 deletions sqlmesh/core/state_sync/db/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,10 @@ def _is_snapshot_used(snapshot: SnapshotIdAndVersion) -> bool:
cleanup_targets.append((snapshot.snapshot_id, dev_table_only))

snapshot_ids_to_cleanup = [snapshot_id for snapshot_id, _ in cleanup_targets]
full_snapshots = self._get_snapshots(snapshot_ids_to_cleanup)
full_snapshots = self._get_snapshots(
snapshot_ids_to_cleanup,
load_optimized_queries=False,
)
for snapshot_id, dev_table_only in cleanup_targets:
if snapshot_id in full_snapshots:
cleanup_tasks.append(
Expand Down Expand Up @@ -509,12 +512,14 @@ def _get_snapshots(
self,
snapshot_ids: t.Iterable[SnapshotIdLike],
lock_for_update: bool = False,
load_optimized_queries: bool = True,
) -> t.Dict[SnapshotId, Snapshot]:
"""Fetches specified snapshots or all snapshots.

Args:
snapshot_ids: The collection of snapshot like objects to fetch.
lock_for_update: Lock the snapshot rows for future update
load_optimized_queries: Whether to prepare optimized queries for model snapshots.

Returns:
A dictionary of snapshot ids to snapshots for ones that could be found.
Expand Down Expand Up @@ -555,7 +560,9 @@ def _loader(snapshot_ids_to_load: t.Set[SnapshotId]) -> t.Collection[Snapshot]:
return fetched_snapshots.values()

snapshots, cached_snapshots = self._snapshot_cache.get_or_load(
{s.snapshot_id for s in snapshot_ids}, _loader
{s.snapshot_id for s in snapshot_ids},
_loader,
load_optimized_queries=load_optimized_queries,
)

if cached_snapshots:
Expand Down
58 changes: 56 additions & 2 deletions tests/core/state_sync/test_state_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from pytest_mock.plugin import MockerFixture
from sqlglot import exp

from sqlmesh.core import constants as c
from sqlmesh.core import constants as c, renderer
from sqlmesh.core.config import EnvironmentSuffixTarget
from sqlmesh.core.dialect import parse_one
from sqlmesh.core.engine_adapter import create_engine_adapter
Expand Down Expand Up @@ -1385,6 +1385,58 @@ def test_delete_expired_snapshots(state_sync: EngineAdapterStateSync, make_snaps
assert not state_sync.get_snapshots(all_snapshots)


def test_get_expired_snapshots_does_not_optimize_queries(
state_sync: EngineAdapterStateSync,
make_snapshot: t.Callable,
) -> None:
snapshot = make_snapshot(
SqlModel(
name="demo.retired",
dialect="duckdb",
query=parse_one("select 1 as id"),
),
)
snapshot.ttl = "in 1 day"
snapshot.categorize_as(SnapshotChangeCategory.BREAKING)
state_sync.push_snapshots([snapshot])

now_ts = now_timestamp()
with patch.object(renderer, "qualify", wraps=renderer.qualify) as qualify:
unexpired = state_sync.get_expired_snapshots(
batch_range=ExpiredBatchRange.all_batch_range(),
current_ts=now_ts,
)
assert unexpired is None
assert qualify.call_count == 0

expired = state_sync.get_expired_snapshots(
batch_range=ExpiredBatchRange.all_batch_range(),
current_ts=now_ts + 2 * 24 * 60 * 60 * 1000,
)

assert expired is not None
assert expired.expired_snapshot_ids == {snapshot.snapshot_id}
assert len(expired.cleanup_tasks) == 1
cleanup_task = expired.cleanup_tasks[0]
assert cleanup_task.dev_table_only is False
assert cleanup_task.snapshot.model_dump() == snapshot.table_info.model_dump()
assert cleanup_task.snapshot.table_name() == snapshot.table_info.table_name()
assert cleanup_task.snapshot.table_name(
is_deployable=False
) == snapshot.table_info.table_name(is_deployable=False)
assert qualify.call_count == 0

# Warm snapshot metadata should not trigger optimization either.
state_sync.get_expired_snapshots(
batch_range=ExpiredBatchRange.all_batch_range(),
current_ts=now_ts + 2 * 24 * 60 * 60 * 1000,
)
assert qualify.call_count == 0

state_sync.get_snapshots([snapshot.snapshot_id])
assert qualify.call_count > 0


def test_get_expired_snapshot_batch(state_sync: EngineAdapterStateSync, make_snapshot: t.Callable):
now_ts = now_timestamp()

Expand Down Expand Up @@ -3692,7 +3744,9 @@ def test_snapshot_cache(
state_sync.snapshot_state.push_snapshots([snapshot])

assert state_sync.get_snapshots([snapshot.snapshot_id]) == {snapshot.snapshot_id: snapshot}
cache_mock.get_or_load.assert_called_once_with({snapshot.snapshot_id}, mocker.ANY)
cache_mock.get_or_load.assert_called_once_with(
{snapshot.snapshot_id}, mocker.ANY, load_optimized_queries=True
)

# Update the snapshot in the state and make sure this update is reflected on the cached instance.
assert snapshot.unpaused_ts is None
Expand Down