Skip to content

Commit 0b4d551

Browse files
committed
fix(plan): resolve unfinalized restatement baseline and unbounded model propagation
- Capture exact plan-creation baseline snapshots on EvaluatablePlan.previous_environment_snapshots ({} for absent/expired target environments, None when recreating an existing environment from another create_from, and context_diff.environment_snapshots otherwise) without altering Environment.previous_finalized_snapshots - Compare RestatementStage.expected_previous_snapshots symmetrically against live environment state across finalized, unfinalized recovery, same-plan retry, and concurrent promotion scenarios - Widen cross-environment interval clear requests when multiple upstream prod restatements reach the same downstream dev snapshot - Propagate start-extension restatement ranges through non-materialized VIEW and EMBEDDED models regardless of model.start, and rebuild populated FULL and materialized-view models while passing only the extended range downstream Signed-off-by: Vamsi-klu <nrvamsi13@gmail.com>
1 parent af5fbcc commit 0b4d551

8 files changed

Lines changed: 1353 additions & 58 deletions

File tree

‎sqlmesh/core/plan/builder.py‎

Lines changed: 97 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
)
3333
from sqlmesh.core.snapshot import (
3434
DeployabilityIndex,
35+
Intervals,
3536
Snapshot,
3637
SnapshotChangeCategory,
3738
)
@@ -507,35 +508,96 @@ def _build_restatements(
507508
)
508509
)
509510
)
510-
if extension_ranges and (is_explicit or floored_snapshot_start < aligned_extension[1]):
511-
# The extended range flows to the downstream models even if this snapshot isn't restated
512-
# itself, as long as it holds (or will hold) data for that range.
513-
start_extension_flow[s_id] = aligned_extension
511+
is_passthrough = snapshot.is_embedded or (
512+
snapshot.is_view and not snapshot.is_materialized_view
513+
)
514+
is_full_refresh = snapshot.is_full or snapshot.is_materialized_view
515+
existing_intervals = _intervals(snapshot, deployability_index)
514516

515-
if not is_explicit:
516-
if floored_snapshot_start >= aligned_extension[1]:
517-
if _has_intervals_in_range(
518-
snapshot, aligned_extension[0], aligned_extension[1], deployability_index
519-
):
517+
if is_passthrough:
518+
if extension_ranges:
519+
start_extension_flow[s_id] = aligned_extension
520+
if not is_explicit:
521+
continue
522+
elif is_full_refresh:
523+
if is_explicit:
524+
if extension_ranges:
525+
start_extension_flow[s_id] = aligned_extension
526+
else:
527+
execution_ts = to_timestamp(self.execution_time)
528+
upper_bound_ts = (
529+
execution_ts
530+
if self._ignore_cron
531+
else to_timestamp(snapshot.node.cron_floor(execution_ts))
532+
)
533+
if not snapshot.allow_partials:
534+
upper_bound_ts = to_timestamp(
535+
snapshot.node.interval_unit.cron_floor(upper_bound_ts)
536+
)
537+
if self.end:
538+
plan_end_ts = snapshot.inclusive_exclusive(
539+
self.end, self.end, strict=False
540+
)[1]
541+
upper_bound_ts = min(upper_bound_ts, plan_end_ts)
542+
543+
can_backfill_in_plan = floored_snapshot_start < upper_bound_ts
544+
if not existing_intervals:
545+
if extension_ranges and can_backfill_in_plan:
546+
start_extension_flow[s_id] = aligned_extension
547+
continue
548+
if not can_backfill_in_plan:
520549
outside_backfill_scope.append(snapshot)
521550
stale_ranges[s_id] = aligned_extension
522-
continue
523-
524-
if self._is_dev and (
525-
not snapshot.is_paused or deployability_index.is_deployable(snapshot)
526-
):
527-
# The physical table of this version is the one used in production.
528-
if _has_intervals_in_range(
529-
snapshot, aligned_extension[0], aligned_extension[1], deployability_index
551+
continue
552+
if extension_ranges:
553+
start_extension_flow[s_id] = aligned_extension
554+
if self._is_dev and (
555+
not snapshot.is_paused or deployability_index.is_deployable(snapshot)
530556
):
531557
shared_with_prod.append(snapshot)
532558
start_extension_flow.pop(s_id, None)
533-
continue
534-
# Only existing intervals can be stale. Anything else is backfilled as usual.
535-
if not _has_intervals_in_range(
536-
snapshot, aligned_extension[0], aligned_extension[1], deployability_index
559+
continue
560+
else:
561+
if extension_ranges and (
562+
is_explicit or floored_snapshot_start < aligned_extension[1]
537563
):
538-
continue
564+
# The extended range flows to the downstream models even if this snapshot isn't restated
565+
# itself, as long as it holds (or will hold) data for that range.
566+
start_extension_flow[s_id] = aligned_extension
567+
568+
if not is_explicit:
569+
if floored_snapshot_start >= aligned_extension[1]:
570+
if _has_intervals_in_range(
571+
snapshot,
572+
aligned_extension[0],
573+
aligned_extension[1],
574+
deployability_index,
575+
):
576+
outside_backfill_scope.append(snapshot)
577+
stale_ranges[s_id] = aligned_extension
578+
continue
579+
580+
if self._is_dev and (
581+
not snapshot.is_paused or deployability_index.is_deployable(snapshot)
582+
):
583+
# The physical table of this version is the one used in production.
584+
if _has_intervals_in_range(
585+
snapshot,
586+
aligned_extension[0],
587+
aligned_extension[1],
588+
deployability_index,
589+
):
590+
shared_with_prod.append(snapshot)
591+
start_extension_flow.pop(s_id, None)
592+
continue
593+
# Only existing intervals can be stale. Anything else is backfilled as usual.
594+
if not _has_intervals_in_range(
595+
snapshot,
596+
aligned_extension[0],
597+
aligned_extension[1],
598+
deployability_index,
599+
):
600+
continue
539601

540602
if not (forward_only_preview_needed and is_explicit):
541603
if self._is_dev and not snapshot.is_paused:
@@ -598,6 +660,11 @@ def _build_restatements(
598660
ext_removal = snapshot.get_removal_interval(
599661
extension_start, extension_end, self._execution_time, strict=False
600662
)
663+
if is_full_refresh and existing_intervals and not (is_explicit and is_preview):
664+
ext_removal = (
665+
min(ext_removal[0], existing_intervals[0][0]),
666+
max(ext_removal[1], existing_intervals[-1][1]),
667+
)
601668
possible_intervals.add(ext_removal)
602669
if snapshot.is_incremental:
603670
start_extension_flow[s_id] = _align_to_interval_unit(snapshot, *ext_removal)
@@ -1222,14 +1289,18 @@ def _range_args(start: int, end: int) -> str:
12221289
return f"--start '{to_ts(start)}' --end '{to_ts(end)}'"
12231290

12241291

1225-
def _has_intervals_in_range(
1226-
snapshot: Snapshot, start: int, end: int, deployability_index: DeployabilityIndex
1227-
) -> bool:
1228-
intervals = (
1292+
def _intervals(snapshot: Snapshot, deployability_index: DeployabilityIndex) -> Intervals:
1293+
return (
12291294
snapshot.intervals
12301295
if deployability_index.is_deployable(snapshot)
12311296
else snapshot.dev_intervals
12321297
)
1298+
1299+
1300+
def _has_intervals_in_range(
1301+
snapshot: Snapshot, start: int, end: int, deployability_index: DeployabilityIndex
1302+
) -> bool:
1303+
intervals = _intervals(snapshot, deployability_index)
12331304
return any(i_start < end and start < i_end for i_start, i_end in intervals)
12341305

12351306

‎sqlmesh/core/plan/common.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,11 @@ def identify_restatement_intervals_across_snapshot_versions(
137137
snapshot=affected_snapshot.id_and_version, interval=interval
138138
)
139139
snapshot_intervals_to_clear[affected_snapshot.snapshot_id] = clear_request
140+
else:
141+
clear_request.interval = (
142+
min(clear_request.interval[0], interval[0]),
143+
max(clear_request.interval[1], interval[1]),
144+
)
140145

141146
clear_request.environment_names |= set([env.name])
142147

‎sqlmesh/core/plan/definition.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,11 +268,20 @@ def is_selected_for_backfill(self, model_fqn: str) -> bool:
268268
return self.models_to_backfill is None or model_fqn in self.models_to_backfill
269269

270270
def to_evaluatable(self) -> EvaluatablePlan:
271+
if not self.context_diff.is_new_environment:
272+
previous_environment_snapshots: t.Optional[t.Dict[str, SnapshotTableInfo]] = {
273+
s.name: s.table_info for s in self.context_diff.environment_snapshots
274+
}
275+
elif self.previous_plan_id is None:
276+
previous_environment_snapshots = {}
277+
else:
278+
previous_environment_snapshots = None
271279
return EvaluatablePlan(
272280
start=self.start,
273281
end=self.end,
274282
new_snapshots=self.new_snapshots,
275283
environment=self.environment,
284+
previous_environment_snapshots=previous_environment_snapshots,
276285
no_gaps=self.no_gaps,
277286
skip_backfill=self.skip_backfill,
278287
empty_backfill=self.empty_backfill,
@@ -319,6 +328,7 @@ class EvaluatablePlan(PydanticModel):
319328
end: TimeLike
320329
new_snapshots: t.List[Snapshot]
321330
environment: Environment
331+
previous_environment_snapshots: t.Optional[t.Dict[str, SnapshotTableInfo]] = None
322332
no_gaps: bool
323333
skip_backfill: bool
324334
empty_backfill: bool

‎sqlmesh/core/plan/evaluator.py‎

Lines changed: 45 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -322,41 +322,65 @@ def visit_restatement_stage(
322322
str, t.Tuple[SnapshotTableInfo, SnapshotTableInfo]
323323
] = {} # tuple of (restated_snapshot, current_prod_snapshot)
324324

325-
if deployed_env := self.state_sync.get_environment(plan.environment.name):
326-
promoted_snapshots_by_name = {s.name: s for s in deployed_env.snapshots}
325+
current_plan_id = getattr(plan, "plan_id", None)
326+
if not isinstance(current_plan_id, str):
327+
current_plan_id = plan.environment.plan_id
328+
329+
if (
330+
(deployed_env := self.state_sync.get_environment(plan.environment.name))
331+
and not deployed_env.expired
332+
and deployed_env.plan_id != current_plan_id
333+
):
334+
ensure_finalized = getattr(plan, "ensure_finalized_snapshots", False) is True
335+
if ensure_finalized:
336+
if deployed_env.plan_id == plan.environment.previous_plan_id:
337+
snapshot_maps = [
338+
{s.name: s for s in deployed_env.finalized_or_current_snapshots}
339+
]
340+
else:
341+
snapshot_maps = [
342+
{s.name: s for s in deployed_env.snapshots},
343+
{s.name: s for s in deployed_env.finalized_or_current_snapshots},
344+
]
345+
else:
346+
snapshot_maps = [{s.name: s for s in deployed_env.snapshots}]
327347

328348
if stage.expected_previous_snapshots is not None:
329349
for name in plan.restatements:
330350
snapshot = stage.all_snapshots[name]
331351
expected_prev = stage.expected_previous_snapshots.get(name)
332-
prod_snapshot = promoted_snapshots_by_name.get(name)
333352

334-
if expected_prev is not None:
335-
if prod_snapshot is None or prod_snapshot.version != expected_prev.version:
336-
deployed_during_restatement[name] = (
337-
snapshot.table_info,
338-
prod_snapshot.table_info if prod_snapshot else expected_prev,
339-
)
340-
else:
341-
if (
342-
prod_snapshot is not None
343-
and prod_snapshot.version != snapshot.table_info.version
344-
):
353+
for promoted_snapshots_by_name in snapshot_maps:
354+
prod_snapshot = promoted_snapshots_by_name.get(name)
355+
if expected_prev is not None:
356+
if (
357+
prod_snapshot is None
358+
or prod_snapshot.version != expected_prev.version
359+
):
360+
deployed_during_restatement[name] = (
361+
snapshot.table_info,
362+
prod_snapshot.table_info if prod_snapshot else expected_prev,
363+
)
364+
break
365+
elif prod_snapshot is not None:
345366
deployed_during_restatement[name] = (
346367
snapshot.table_info,
347368
prod_snapshot.table_info,
348369
)
370+
break
349371
else:
350372
for name in plan.restatements:
351373
snapshot = stage.all_snapshots[name]
352374
version = snapshot.table_info.version
353-
if (
354-
prod_snapshot := promoted_snapshots_by_name.get(name)
355-
) and prod_snapshot.version != version:
356-
deployed_during_restatement[name] = (
357-
snapshot.table_info,
358-
prod_snapshot.table_info,
359-
)
375+
for promoted_snapshots_by_name in snapshot_maps:
376+
if (
377+
prod_snapshot := promoted_snapshots_by_name.get(name)
378+
) and prod_snapshot.version != version:
379+
deployed_during_restatement[name] = (
380+
snapshot.table_info,
381+
prod_snapshot.table_info,
382+
)
383+
break
360384

361385
# we need to *not* clear the intervals on the snapshots where new versions were deployed while the restatement was running in order to prevent
362386
# subsequent plans from having unexpected intervals to backfill.

‎sqlmesh/core/plan/stages.py‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -477,16 +477,35 @@ def _get_restatement_stage(
477477

478478
if plan.restatements:
479479
expected_previous_snapshots: t.Optional[t.Dict[str, SnapshotTableInfo]] = None
480-
baseline_snapshots = plan.environment.previous_finalized_snapshots
481-
if baseline_snapshots is None:
480+
previous_env_snapshots = getattr(plan, "previous_environment_snapshots", None)
481+
if isinstance(previous_env_snapshots, dict):
482+
expected_previous_snapshots = previous_env_snapshots
483+
else:
484+
ensure_finalized = getattr(plan, "ensure_finalized_snapshots", False) is True
482485
if existing_environment is None:
483486
existing_environment = self.state_reader.get_environment(
484487
plan.environment.name
485488
)
486-
if existing_environment is not None:
487-
baseline_snapshots = existing_environment.snapshots
488-
if baseline_snapshots is not None:
489-
expected_previous_snapshots = {s.name: s.table_info for s in baseline_snapshots}
489+
if isinstance(existing_environment, Environment) and (
490+
plan.environment.previous_finalized_snapshots is None
491+
or existing_environment.plan_id == plan.environment.previous_plan_id
492+
):
493+
if existing_environment.expired:
494+
expected_previous_snapshots = {}
495+
else:
496+
baseline = (
497+
existing_environment.finalized_or_current_snapshots
498+
if ensure_finalized
499+
else existing_environment.snapshots
500+
)
501+
expected_previous_snapshots = {s.name: s.table_info for s in baseline}
502+
elif plan.environment.previous_finalized_snapshots is not None:
503+
expected_previous_snapshots = {
504+
s.name: s.table_info
505+
for s in plan.environment.previous_finalized_snapshots
506+
}
507+
elif existing_environment is None:
508+
expected_previous_snapshots = {}
490509

491510
return RestatementStage(
492511
all_snapshots=snapshots_by_name,

0 commit comments

Comments
 (0)