|
32 | 32 | ) |
33 | 33 | from sqlmesh.core.snapshot import ( |
34 | 34 | DeployabilityIndex, |
| 35 | + Intervals, |
35 | 36 | Snapshot, |
36 | 37 | SnapshotChangeCategory, |
37 | 38 | ) |
@@ -507,35 +508,96 @@ def _build_restatements( |
507 | 508 | ) |
508 | 509 | ) |
509 | 510 | ) |
510 | | - if extension_ranges and (is_explicit or floored_snapshot_start < aligned_extension[1]): |
511 | | - # The extended range flows to the downstream models even if this snapshot isn't restated |
512 | | - # itself, as long as it holds (or will hold) data for that range. |
513 | | - start_extension_flow[s_id] = aligned_extension |
| 511 | + is_passthrough = snapshot.is_embedded or ( |
| 512 | + snapshot.is_view and not snapshot.is_materialized_view |
| 513 | + ) |
| 514 | + is_full_refresh = snapshot.is_full or snapshot.is_materialized_view |
| 515 | + existing_intervals = _intervals(snapshot, deployability_index) |
514 | 516 |
|
515 | | - if not is_explicit: |
516 | | - if floored_snapshot_start >= aligned_extension[1]: |
517 | | - if _has_intervals_in_range( |
518 | | - snapshot, aligned_extension[0], aligned_extension[1], deployability_index |
519 | | - ): |
| 517 | + if is_passthrough: |
| 518 | + if extension_ranges: |
| 519 | + start_extension_flow[s_id] = aligned_extension |
| 520 | + if not is_explicit: |
| 521 | + continue |
| 522 | + elif is_full_refresh: |
| 523 | + if is_explicit: |
| 524 | + if extension_ranges: |
| 525 | + start_extension_flow[s_id] = aligned_extension |
| 526 | + else: |
| 527 | + execution_ts = to_timestamp(self.execution_time) |
| 528 | + upper_bound_ts = ( |
| 529 | + execution_ts |
| 530 | + if self._ignore_cron |
| 531 | + else to_timestamp(snapshot.node.cron_floor(execution_ts)) |
| 532 | + ) |
| 533 | + if not snapshot.allow_partials: |
| 534 | + upper_bound_ts = to_timestamp( |
| 535 | + snapshot.node.interval_unit.cron_floor(upper_bound_ts) |
| 536 | + ) |
| 537 | + if self.end: |
| 538 | + plan_end_ts = snapshot.inclusive_exclusive( |
| 539 | + self.end, self.end, strict=False |
| 540 | + )[1] |
| 541 | + upper_bound_ts = min(upper_bound_ts, plan_end_ts) |
| 542 | + |
| 543 | + can_backfill_in_plan = floored_snapshot_start < upper_bound_ts |
| 544 | + if not existing_intervals: |
| 545 | + if extension_ranges and can_backfill_in_plan: |
| 546 | + start_extension_flow[s_id] = aligned_extension |
| 547 | + continue |
| 548 | + if not can_backfill_in_plan: |
520 | 549 | outside_backfill_scope.append(snapshot) |
521 | 550 | stale_ranges[s_id] = aligned_extension |
522 | | - continue |
523 | | - |
524 | | - if self._is_dev and ( |
525 | | - not snapshot.is_paused or deployability_index.is_deployable(snapshot) |
526 | | - ): |
527 | | - # The physical table of this version is the one used in production. |
528 | | - if _has_intervals_in_range( |
529 | | - snapshot, aligned_extension[0], aligned_extension[1], deployability_index |
| 551 | + continue |
| 552 | + if extension_ranges: |
| 553 | + start_extension_flow[s_id] = aligned_extension |
| 554 | + if self._is_dev and ( |
| 555 | + not snapshot.is_paused or deployability_index.is_deployable(snapshot) |
530 | 556 | ): |
531 | 557 | shared_with_prod.append(snapshot) |
532 | 558 | start_extension_flow.pop(s_id, None) |
533 | | - continue |
534 | | - # Only existing intervals can be stale. Anything else is backfilled as usual. |
535 | | - if not _has_intervals_in_range( |
536 | | - snapshot, aligned_extension[0], aligned_extension[1], deployability_index |
| 559 | + continue |
| 560 | + else: |
| 561 | + if extension_ranges and ( |
| 562 | + is_explicit or floored_snapshot_start < aligned_extension[1] |
537 | 563 | ): |
538 | | - continue |
| 564 | + # The extended range flows to the downstream models even if this snapshot isn't restated |
| 565 | + # itself, as long as it holds (or will hold) data for that range. |
| 566 | + start_extension_flow[s_id] = aligned_extension |
| 567 | + |
| 568 | + if not is_explicit: |
| 569 | + if floored_snapshot_start >= aligned_extension[1]: |
| 570 | + if _has_intervals_in_range( |
| 571 | + snapshot, |
| 572 | + aligned_extension[0], |
| 573 | + aligned_extension[1], |
| 574 | + deployability_index, |
| 575 | + ): |
| 576 | + outside_backfill_scope.append(snapshot) |
| 577 | + stale_ranges[s_id] = aligned_extension |
| 578 | + continue |
| 579 | + |
| 580 | + if self._is_dev and ( |
| 581 | + not snapshot.is_paused or deployability_index.is_deployable(snapshot) |
| 582 | + ): |
| 583 | + # The physical table of this version is the one used in production. |
| 584 | + if _has_intervals_in_range( |
| 585 | + snapshot, |
| 586 | + aligned_extension[0], |
| 587 | + aligned_extension[1], |
| 588 | + deployability_index, |
| 589 | + ): |
| 590 | + shared_with_prod.append(snapshot) |
| 591 | + start_extension_flow.pop(s_id, None) |
| 592 | + continue |
| 593 | + # Only existing intervals can be stale. Anything else is backfilled as usual. |
| 594 | + if not _has_intervals_in_range( |
| 595 | + snapshot, |
| 596 | + aligned_extension[0], |
| 597 | + aligned_extension[1], |
| 598 | + deployability_index, |
| 599 | + ): |
| 600 | + continue |
539 | 601 |
|
540 | 602 | if not (forward_only_preview_needed and is_explicit): |
541 | 603 | if self._is_dev and not snapshot.is_paused: |
@@ -598,6 +660,11 @@ def _build_restatements( |
598 | 660 | ext_removal = snapshot.get_removal_interval( |
599 | 661 | extension_start, extension_end, self._execution_time, strict=False |
600 | 662 | ) |
| 663 | + if is_full_refresh and existing_intervals and not (is_explicit and is_preview): |
| 664 | + ext_removal = ( |
| 665 | + min(ext_removal[0], existing_intervals[0][0]), |
| 666 | + max(ext_removal[1], existing_intervals[-1][1]), |
| 667 | + ) |
601 | 668 | possible_intervals.add(ext_removal) |
602 | 669 | if snapshot.is_incremental: |
603 | 670 | start_extension_flow[s_id] = _align_to_interval_unit(snapshot, *ext_removal) |
@@ -1222,14 +1289,18 @@ def _range_args(start: int, end: int) -> str: |
1222 | 1289 | return f"--start '{to_ts(start)}' --end '{to_ts(end)}'" |
1223 | 1290 |
|
1224 | 1291 |
|
1225 | | -def _has_intervals_in_range( |
1226 | | - snapshot: Snapshot, start: int, end: int, deployability_index: DeployabilityIndex |
1227 | | -) -> bool: |
1228 | | - intervals = ( |
| 1292 | +def _intervals(snapshot: Snapshot, deployability_index: DeployabilityIndex) -> Intervals: |
| 1293 | + return ( |
1229 | 1294 | snapshot.intervals |
1230 | 1295 | if deployability_index.is_deployable(snapshot) |
1231 | 1296 | else snapshot.dev_intervals |
1232 | 1297 | ) |
| 1298 | + |
| 1299 | + |
| 1300 | +def _has_intervals_in_range( |
| 1301 | + snapshot: Snapshot, start: int, end: int, deployability_index: DeployabilityIndex |
| 1302 | +) -> bool: |
| 1303 | + intervals = _intervals(snapshot, deployability_index) |
1233 | 1304 | return any(i_start < end and start < i_end for i_start, i_end in intervals) |
1234 | 1305 |
|
1235 | 1306 |
|
|
0 commit comments