From 78a50b4f6124b884817dbcc357f630accfc40b25 Mon Sep 17 00:00:00 2001 From: NIne-WIngEd Date: Thu, 22 Oct 2026 20:50:14 -0500 Subject: [PATCH 1/2] prune: Use calendar-aware retention intervals Fixes borgbackup/borg#9929. --- docs/misc/prune-example-interval.txt | 24 ++-- docs/usage/general/date-time.rst.inc | 16 ++- src/borg/archiver/prune_cmd.py | 24 +++- src/borg/helpers/__init__.py | 2 +- src/borg/helpers/parseformat.py | 51 +------ src/borg/testsuite/archiver/prune_cmd_test.py | 127 +++++++++++++++--- .../testsuite/helpers/parseformat_test.py | 68 ++-------- 7 files changed, 167 insertions(+), 145 deletions(-) diff --git a/docs/misc/prune-example-interval.txt b/docs/misc/prune-example-interval.txt index e96e0a0baa..b8ba9a6938 100644 --- a/docs/misc/prune-example-interval.txt +++ b/docs/misc/prune-example-interval.txt @@ -59,7 +59,7 @@ Calendar view 8 9 10 11 12 13 14 15y16 15 16 17 18 19 20 21 17 18 19 20 21 22 23 22 23 24 25 26 27 28 - 24 25 26 27 28 29 30 29 30 31m + 24 25 26 27 28 29 30 29 30 31y 2026 January February March @@ -83,12 +83,13 @@ List view --keep-daily 1w --keep-weekly 4 --keep-monthly 5m --keep-yearly 2 -------------------------------------------------------------------------------- - 1. 2025-11-15 (oldest) + 1. 2025-12-31 + 2. 2025-11-15 (oldest) 1. 2026-06-02 1. 2026-05-23 1. 2026-04-30 2. 2026-06-01 2. 2026-05-17 2. 2026-03-30 3. 2026-05-31 3. 2026-05-10 3. 2026-02-28 4. 2026-05-30 4. 2026-05-03 4. 2026-01-31 -5. 2026-05-29 5. 2025-12-31 +5. 2026-05-29 6. 2026-05-28 2026-06-04 is additionally kept due to `--from`. @@ -132,14 +133,12 @@ Since 2026-05-24 and 2026-03-31 were skipped, Borg substitutes the next best candidate for each: 2026-05-23 fills the week 21 weekly slot, and 2026-03-30 fills the March monthly slot. -The implementation of `--keep-monthly 5m` is somewhat special: Borg defines a -month as a fixed 31-day period, independent of the actual calendar dates -involved. As a result, `5m` corresponds to 5 × 31 = 155 days. The archive from -2025-12-31 16:00 is exactly 155 days older than the reference time and is -therefore retained by `--keep-monthly`. +The `--keep-monthly 5m` window starts five calendar months before the +reference time, on 2026-01-04 at 16:00. The archive from 2025-12-31 is +outside this window, so it is not retained by `--keep-monthly`. -As a result, there are no true yearly candidates. In the absence of a better -candidate, `--keep-yearly 2` only matches the oldest archive, 2025-11-15. +Instead, `--keep-yearly 2` keeps 2025-12-31 as its yearly candidate. It +also keeps the oldest archive, 2025-11-15, to fill the remaining slot. Since interval rules define time windows rather than competing for a fixed number of slots, their interplay is simpler than count-based rules. An archive @@ -150,6 +149,5 @@ Intervals and counts can be mixed freely. Yearly retention in this example is done by retention count instead of intervals. A count rule paired with interval rules behaves just as it would if all preceding rules were also counts: Archives already kept by earlier rules are excluded from consideration. In this -example, every yearly slot is consumed by earlier rules; no true yearly -candidate remains, so the oldest archive at 2025-11-15 is kept to preserve -rolling backup semantics. +example, the 2026 yearly slot is consumed by earlier rules. December 31 +fills the 2025 slot, and November 15 is retained as the oldest archive. diff --git a/docs/usage/general/date-time.rst.inc b/docs/usage/general/date-time.rst.inc index db1fcca608..9e2420f845 100644 --- a/docs/usage/general/date-time.rst.inc +++ b/docs/usage/general/date-time.rst.inc @@ -23,7 +23,15 @@ The ``borg prune`` ``--keep-*`` retention options accept either a plain count When using interval-based retention, ``--from`` may be specified to set the reference timestamp for the interval (defaults to the current time). -Please note that Borg treats months (e.g. ``12m``) as fixed 31-day periods -rather than calendar months. As a result, ``12m`` corresponds to -12 × 31 = 372 days. Similarly, years (e.g. ``2y``) are treated as fixed -365-day periods and do not take leap years into account. +Months and years are calendar-aware. For example, ``12m`` before +2026-07-22 is 2025-07-22, and ``3y`` before 2026-07-22 is 2023-07-22. +The time of day is preserved. If the target month has fewer days, the date +is clamped to its last day: one month before March 31 is February 28 +(or February 29 in a leap year). + +Weeks, days, hours, minutes, and seconds keep their fixed-length meanings. +For example, ``1w`` is seven days. Plain counts and ``all`` are unchanged. + +When combining interval-based pruning rules, their lengths are compared at +the same reference timestamp. Thus, ``1m`` can be shorter than, equal to, +or longer than ``30d`` depending on the month. diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 0e5efcfe93..757589b195 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -12,6 +12,7 @@ from ..helpers import GroupBySpec from ..helpers import json_print, basic_json_data from ..helpers.argparsing import ArgumentParser +from ..helpers.time import calculate_relative_offset from ..manifest import AI_GROUP_BY_KEYS, ArchiveInfo, format_group_key, group_archives from ..logger import create_logger @@ -198,6 +199,18 @@ class PruneMixIn: @with_repository() def do_prune(self, args, repository, manifest): """Prune archives according to specified rules.""" + from_timestamp = getattr(args, PRUNE_FROM.key) + base_timestamp = from_timestamp if from_timestamp is not None else datetime.now().astimezone() + + for rule in PRUNING_RULES: + value = getattr(args, rule.key) + if not isinstance(value, str): + continue + try: + cutoff = calculate_relative_offset(value, base_timestamp, earlier=True) + except (ValueError, OverflowError) as error: + raise CommandError(f"Invalid retention interval {value!r} for {rule.key}: {error}") from None + setattr(args, rule.key, base_timestamp - cutoff) self._validate_prune_args(args) match = archive_match_patterns(args) @@ -210,10 +223,6 @@ def do_prune(self, args, repository, manifest): group_by = tuple(args.group_by.split(",")) if args.group_by else () groups = group_archives(archives, group_by) - # All groups use the same reference timestamp, so interval based rules do not drift apart. - from_timestamp = getattr(args, PRUNE_FROM.key) - base_timestamp = from_timestamp if from_timestamp is not None else datetime.now().astimezone() - # Archives to keep along with the rule that ensured them being kept keep = {} group_of = {} @@ -465,7 +474,8 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): that time window. When ``--from`` is given together with an interval retention, the interval is measured backwards from that timestamp instead of from the current time. See ``Date and Time`` docs for exact - INTERVAL format. + INTERVAL format. Months and years are calendar-aware: a month before + March 31 is February 28 (or 29 in a leap year), at the same time of day. The ``--keep-secondly``, ``--keep-minutely``, ``--keep-hourly``, ``--keep-daily``, ``--keep-weekly``, ``--keep-monthly``, @@ -505,7 +515,9 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): weekly interval is already covered by the daily one and so the weekly interval is effectively useless. An error is emitted upon running ``borg prune`` if such a combination of flags is given. The order of - flags on the command line is not significant. + flags on the command line is not significant. Calendar intervals are + compared at the reference timestamp: ``1m`` can be shorter than, equal + to, or longer than ``30d`` depending on the month. A practical approach for recurring backups is to use rules diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 9587a715e1..3ba11a3826 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -34,7 +34,7 @@ from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper from .parseformat import octal_int, bin_to_hex, hex_to_bin, safe_encode, safe_decode from .parseformat import text_to_json, binary_to_json, remove_surrogates, join_cmd -from .parseformat import eval_escapes, decode_dict, interval, int_or_interval, nonnegative_seconds +from .parseformat import eval_escapes, decode_dict, int_or_interval, nonnegative_seconds from .parseformat import ( PathSpec, FilesystemPathSpec, diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 789232a5d7..73146bf336 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -13,7 +13,7 @@ import uuid from pathlib import Path from typing import ClassVar, Any, TYPE_CHECKING, Literal -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from functools import lru_cache, partial from hashlib import sha256 from string import Formatter @@ -141,58 +141,17 @@ def nonnegative_seconds(s): return seconds -def interval(s): - """Parse an interval string (e.g. ``7d``, ``2w``, ``30M``) into a timedelta. - - Supported units: y (years, 365d), m (months, 31d), w (weeks), d (days), - H (hours), M (minutes), S (seconds). The value must be nonnegative. - """ - if isinstance(s, timedelta): - return s - - seconds_in_a_minute = 60 - seconds_in_an_hour = 60 * seconds_in_a_minute - seconds_in_a_day = 24 * seconds_in_an_hour - seconds_in_a_week = 7 * seconds_in_a_day - seconds_in_a_month = 31 * seconds_in_a_day - seconds_in_a_year = 365 * seconds_in_a_day - multiplier = dict( - y=seconds_in_a_year, - m=seconds_in_a_month, - w=seconds_in_a_week, - d=seconds_in_a_day, - H=seconds_in_an_hour, - M=seconds_in_a_minute, - S=1, - ) - - if s.endswith(tuple(multiplier.keys())): - number = s[:-1] - suffix = s[-1] - else: - raise ArgumentTypeError(f'Unexpected time unit "{s[-1]}": choose from {", ".join(multiplier)}') - - try: - seconds = int(number) * multiplier[suffix] - except ValueError: - seconds = -1 - - if seconds < 0: - raise ArgumentTypeError(f'Invalid number "{number}": expected nonnegative integer') - - return timedelta(seconds=seconds) - - def int_or_interval(s): """Parse *s* as an :class:`int` or, failing that, as an interval string. Returns :class:`int` if *s* can be parsed as an integer (e.g. ``"7"``), - or :class:`datetime.timedelta` if *s* is a valid interval (e.g. ``"7d"``). + or the validated interval string (e.g. ``"7d"``). Months and years need a + reference timestamp before they can be converted to a duration. Raises :class:`ArgumentTypeError` if *s* is neither an integer nor a valid interval. """ - if isinstance(s, (int, timedelta)): + if isinstance(s, int): return s # Explicitly check 'all' as a shortcut to 'infinite' sentinel value `-1`. @@ -205,7 +164,7 @@ def int_or_interval(s): pass try: - return interval(s) + return relative_time_marker_validator(s) except ArgumentTypeError as e: raise ArgumentTypeError(f"Value is neither an integer nor an interval: {e}") diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index e101d4cd1a..aaa15bd8b3 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -22,6 +22,7 @@ from argparse import ArgumentTypeError from ...helpers import CommandError +from ...helpers.time import calculate_relative_offset from ...manifest import ArchiveInfo from . import cmd, RK_ENCRYPTION, generate_archiver_tests @@ -202,18 +203,12 @@ def test_prune_repository_example_interval(archivers, request, backup_files): for i, name in enumerate(weekly_kept, 1): assert re.search(rf"Keeping archive \(rule: weekly #{i}\):\s+{name}", output) - monthly_kept = [ - "backup_2026-04-30", - "backup_2026-03-30", - "backup_2026-02-28", - "backup_2026-01-31", - "backup_2025-12-31", - ] + monthly_kept = ["backup_2026-04-30", "backup_2026-03-30", "backup_2026-02-28", "backup_2026-01-31"] for i, name in enumerate(monthly_kept, 1): assert re.search(rf"Keeping archive \(rule: monthly #{i}\):\s+{name}", output) - # No true yearly candidates remain; only the oldest archive fills the slot. - assert re.search(r"Keeping archive \(rule: yearly\[oldest\] #1\):\s+backup_2025-11-15", output) + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+backup_2025-12-31", output) + assert re.search(r"Keeping archive \(rule: yearly\[oldest\] #2\):\s+backup_2025-11-15", output) # 05-27 was kept by daily yesterday (window started 05-26) but falls out # today (window shifted to 05-28). W22 slot is consumed by daily-held 05-31, @@ -653,11 +648,13 @@ def test_prune_keep_monthly_int_or_interval(archivers, request, backup_files, ke archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc) - _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=31)) - _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=62)).replace(microsecond=1)) - _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=62)) - _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(days=93)) - _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=93, microseconds=1)) + _create_archive_dt(archiver, backup_files, "test-1", datetime(2023, 11, 30, tzinfo=timezone.utc)) + _create_archive_dt(archiver, backup_files, "test-2", datetime(2023, 10, 31, microsecond=1, tzinfo=timezone.utc)) + _create_archive_dt(archiver, backup_files, "test-3", datetime(2023, 10, 31, tzinfo=timezone.utc)) + _create_archive_dt(archiver, backup_files, "test-4", datetime(2023, 9, 30, tzinfo=timezone.utc)) + _create_archive_dt( + archiver, backup_files, "test-5", datetime(2023, 9, 30, tzinfo=timezone.utc) - timedelta(microseconds=1) + ) output = prune_ungrouped(archiver, "--list", "--dry-run", "--from", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output) @@ -774,7 +771,7 @@ def test_prune_no_args(archivers, request): assert f"keep-{flag}" in output -@pytest.mark.parametrize("keep_arg,value", list(product([rule.key for rule in PRUNING_RULES], ["0", "0S"]))) +@pytest.mark.parametrize("keep_arg,value", list(product([rule.key for rule in PRUNING_RULES], ["0", "0S", "0m", "0y"]))) def test_prune_all_zero_args_one(archivers, request, keep_arg, value): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) @@ -861,17 +858,17 @@ def test_prune_from_prefiltered_archives_ignored_in_pruning(archivers, request, def test_prune_interval_rolling_schedule_oldest_retention(): daily_interval = timedelta(days=6) - monthly_interval = timedelta(days=31 * 3) # Matching --keep-monthly=3m after argument parsing start_date = datetime(2024, 1, 1, tzinfo=timezone.utc) previous_archives = [] archives = [] - for day_offset in range(95): + for day_offset in range(93): backup_ts = start_date + timedelta(days=day_offset) previous_archives = archives archives = [mock_archive(backup_ts, day_offset), *archives] + monthly_interval = backup_ts - calculate_relative_offset("3m", backup_ts, earlier=True) keep = {} keep |= prune(archives, PRUNE_DAILY, daily_interval, backup_ts, False, keep) keep |= prune(archives, PRUNE_MONTHLY, monthly_interval, backup_ts, True, keep) @@ -885,10 +882,6 @@ def test_prune_interval_rolling_schedule_oldest_retention(): archives = sorted(keep.keys(), key=lambda a: a.ts, reverse=True) - # It is now 2024-04-04. On this final run jan-01 falls outside the - # 93d monthly interval and is pruned. It was kept until now by the - # oldest-retention rule — on 2024-04-03 it was exactly at the - # inclusive boundary and retained as oldest. assert previous_archives[-1].ts.strftime("%m-%d") == "01-01" assert archives[-1].ts.strftime("%m-%d") == "01-31" @@ -1074,3 +1067,95 @@ def test_prune_group_by_invalid_key(archivers, request): cmd(archiver, "repo-create", RK_ENCRYPTION) output = cmd(archiver, "prune", "--group-by", "bogus", "--keep-daily=1", exit_code=2) assert "Invalid group-by key: bogus" in output + + +@pytest.mark.parametrize( + "reference,interval,cutoff", + [ + ("2026-07-22", "12m", "2025-07-22"), + ("2026-07-22", "3y", "2023-07-22"), + ("2026-03-31", "1m", "2026-02-28"), + ("2024-03-31", "1m", "2024-02-29"), + ("2024-02-29", "1y", "2023-02-28"), + ("2026-01-31", "2m", "2025-11-30"), + ("2026-03-31T12:34:56.123456+05:30", "1m", "2026-02-28T12:34:56.123456+05:30"), + ], +) +def test_prune_calendar_interval(archivers, request, backup_files, reference, interval, cutoff): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + reference = datetime.fromisoformat(reference) + cutoff = datetime.fromisoformat(cutoff) + if reference.tzinfo is None: + reference = reference.replace(tzinfo=timezone.utc) + cutoff = cutoff.replace(tzinfo=timezone.utc) + archive_times = { + "too-old": cutoff - timedelta(microseconds=1), + "boundary": cutoff, + "inside": cutoff + timedelta(microseconds=1), + "from": reference, + "newer": reference + timedelta(microseconds=1), + } + for name, archive_time in archive_times.items(): + _create_archive_dt(archiver, backup_files, name, archive_time) + + prune_args = ("--from", reference.isoformat(), "--keep", interval) + output = json.loads(prune_ungrouped(archiver, "--json", "--dry-run", *prune_args)) + assert {a["name"]: a["kept"] for a in output["archives"]} == {name: name != "too-old" for name in archive_times} + assert {a["name"] for a in json.loads(cmd(archiver, "repo-list", "--json"))["archives"]} == set(archive_times) + + prune_ungrouped(archiver, *prune_args) + remaining = json.loads(cmd(archiver, "repo-list", "--json"))["archives"] + assert {a["name"] for a in remaining} == set(archive_times) - {"too-old"} + + +@pytest.mark.parametrize( + "reference,shorter,longer,valid", + [ + ("2026-03-01", "1m", "30d", True), + ("2026-04-01", "1m", "30d", False), + ("2026-05-01", "1m", "30d", False), + ("2024-07-22", "365d", "1y", True), + ("2025-07-22", "365d", "1y", False), + ("2026-07-22", "12m", "1y", False), + ("2026-03-01", "1m", "all", True), + ("2026-03-01", "all", "1m", False), + ], +) +def test_prune_calendar_interval_order(archivers, request, reference, shorter, longer, valid): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + prune_args = ("--from", reference + "T00:00:00+00:00", "--keep-daily", shorter, "--keep-monthly", longer) + if valid: + cmd(archiver, "prune", "--dry-run", *prune_args) + else: + assert "effectively useless" in _cmd_prune_error(archiver, "--dry-run", *prune_args) + + +def test_prune_calendar_interval_without_from(archiver, backup_files, monkeypatch): + from ...archiver import prune_cmd + + reference = datetime(2026, 3, 1, tzinfo=timezone.utc) + + class FixedDatetime(datetime): + @classmethod + def now(cls, tz=None): + return reference.astimezone(tz) + + cmd(archiver, "repo-create", RK_ENCRYPTION) + _create_archive_ts(archiver, backup_files, "too-old", 2026, 1, 31) + _create_archive_ts(archiver, backup_files, "boundary", 2026, 2, 1) + monkeypatch.setattr(prune_cmd, "datetime", FixedDatetime) + output = json.loads(prune_ungrouped(archiver, "--json", "--dry-run", "--keep", "1m")) + assert {a["name"]: a["kept"] for a in output["archives"]} == {"too-old": False, "boundary": True} + + +@pytest.mark.parametrize("interval", ["99999y", "99999999999999999999d"]) +def test_prune_calendar_interval_out_of_range(archivers, request, backup_files, interval): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "untouched", backup_files) + output = _cmd_prune_error(archiver, "--keep", interval, "--from", "2026-07-22T00:00:00+00:00") + assert "Invalid retention interval" in output + archives = json.loads(cmd(archiver, "repo-list", "--json"))["archives"] + assert [a["name"] for a in archives] == ["untouched"] diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index c1abc6f728..fbf739f7aa 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -5,7 +5,7 @@ import re import unicodedata -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone import pytest @@ -21,7 +21,6 @@ text_validator, format_file_size, parse_file_size, - interval, int_or_interval, nonnegative_seconds, partial_format, @@ -444,53 +443,6 @@ def test_format_timedelta(): assert format_timedelta(t1 - t0) == "2 hours 1.100 seconds" -@pytest.mark.parametrize( - "timeframe, num_secs", - [ - ("0S", timedelta(seconds=0)), - ("5S", timedelta(seconds=5)), - ("2M", timedelta(minutes=2)), - ("1H", timedelta(hours=1)), - ("1d", timedelta(days=1)), - ("1w", timedelta(days=7)), - ("1m", timedelta(days=31)), - ("1y", timedelta(days=365)), - ], -) -def test_interval(timeframe, num_secs): - assert interval(timeframe) == num_secs - - -@pytest.mark.parametrize( - "invalid_interval, error_tuple", - [ - ("H", ('Invalid number "": expected nonnegative integer',)), - ("-1d", ('Invalid number "-1": expected nonnegative integer',)), - ("food", ('Invalid number "foo": expected nonnegative integer',)), - ], -) -def test_interval_time_unit(invalid_interval, error_tuple): - with pytest.raises(ArgumentTypeError) as exc: - interval(invalid_interval) - assert exc.value.args == error_tuple - - -@pytest.mark.parametrize( - "invalid_input, error_regex", - [ - ("x", r'^Unexpected time unit "x": choose from'), - ("-1t", r'^Unexpected time unit "t": choose from'), - ("fool", r'^Unexpected time unit "l": choose from'), - ("abc", r'^Unexpected time unit "c": choose from'), - (" abc ", r'^Unexpected time unit " ": choose from'), - ], -) -def test_interval_invalid_time_format(invalid_input, error_regex): - with pytest.raises(ArgumentTypeError) as exc: - interval(invalid_input) - assert re.search(error_regex, exc.value.args[0]) - - @pytest.mark.parametrize("input, result", [("0", 0.0), ("30", 30.0), ("2.5", 2.5), ("1800", 1800.0)]) def test_nonnegative_seconds(input, result): assert nonnegative_seconds(input) == result @@ -510,14 +462,12 @@ def test_nonnegative_seconds_invalid(invalid_input): (" 999 ", 999), ("-1", -1), ("all", -1), - ("0S", timedelta(seconds=0)), - ("5S", timedelta(seconds=5)), - ("1m", timedelta(days=31)), + ("0S", "0S"), + ("5S", "5S"), + ("1m", "1m"), # already-converted values (jsonargparse idempotency) (0, 0), (5, 5), - (timedelta(seconds=5), timedelta(seconds=5)), - (timedelta(days=31), timedelta(days=31)), ], ) def test_int_or_interval(input, result): @@ -530,6 +480,10 @@ def test_int_or_interval(input, result): ("H", r"Value is neither an integer nor an interval:"), ("-1d", r"Value is neither an integer nor an interval:"), ("food", r"Value is neither an integer nor an interval:"), + ("", r"Value is neither an integer nor an interval:"), + ("1t", r"Value is neither an integer nor an interval:"), + ("1m2d", r"Value is neither an integer nor an interval:"), + ("1.5y", r"Value is neither an integer nor an interval:"), ], ) def test_int_or_interval_time_unit(invalid_input, error_regex): @@ -1021,3 +975,9 @@ def test_diff_formatter_time_precision(ctime1_ns, ctime2_ns): assert re.search(r"\.\d{9} ", m.group(1)) # 9-digit fraction is shown assert re.search(r"\.\d{9} ", m.group(2)) assert m.group(1) != m.group(2) # timestamps are distinguishable + + +@pytest.mark.parametrize("marker", ["0S", "0m", "0y", "5S", "2M", "1H", "7d", "2w", "1m", "12m", "3y"]) +def test_int_or_interval_preserves_calendar_marker(marker): + assert int_or_interval(marker) == marker + assert int_or_interval(int_or_interval(marker)) == marker From 6348c4e447ac97ddaa3f4136e930eacb71d5bca7 Mon Sep 17 00:00:00 2001 From: NIne-WIngEd Date: Sat, 7 Nov 2026 17:30:07 -0600 Subject: [PATCH 2/2] prune: Allow redundant retention rules Address review feedback for #10541. --- docs/misc/prune-example-interval.txt | 42 +++---- docs/usage/general/date-time.rst.inc | 25 ++-- src/borg/archiver/prune_cmd.py | 67 +++-------- src/borg/testsuite/archiver/prune_cmd_test.py | 108 +++++++++++++----- 4 files changed, 130 insertions(+), 112 deletions(-) diff --git a/docs/misc/prune-example-interval.txt b/docs/misc/prune-example-interval.txt index b8ba9a6938..04257932b7 100644 --- a/docs/misc/prune-example-interval.txt +++ b/docs/misc/prune-example-interval.txt @@ -12,14 +12,14 @@ days going back to late 2025. Today, `borg create` took a little longer than usual. It's 16:12 now and you run `borg prune`. You want Borg to keep one archive per day for one week, four weekly archives, -one archive per month for five months, and two yearly backups. For that, you +one archive per month for six months, and two yearly backups. For that, you use the following command: ``` borg prune \ --keep-daily 1w \ --keep-weekly 4 \ - --keep-monthly 5m \ + --keep-monthly 6m \ --keep-yearly 2 \ --from '2026-06-04 16:00' ``` @@ -49,6 +49,7 @@ archives kept by the `--keep-weekly` rule are marked by a "w" to the right, archives kept by the `--keep-monthly` rule are marked by a "m" to the right, archives kept by the `--keep-yearly` rule are marked by a "y" to the right, and archives kept by the `--from` rule are marked by a "x" to the right. +An "n" marks a day when no backup was taken. Calendar view @@ -59,7 +60,7 @@ Calendar view 8 9 10 11 12 13 14 15y16 15 16 17 18 19 20 21 17 18 19 20 21 22 23 22 23 24 25 26 27 28 - 24 25 26 27 28 29 30 29 30 31y + 24 25 26 27 28 29 30 29 30 31m 2026 January February March @@ -68,28 +69,27 @@ Calendar view 12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15 19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22 26 27 28 29 30 31m 23 24 25 26 27 28m 23 24 25 26 27 28 29 - 30m31 + 30m31n April May June - 1 2 3 4 5 1 2 3w 1d 2d 3 4x + 1 2 3 4 5 1 2 3w 1d 2d 3n 4x 6 7 8 9 10 11 12 4 5 6 7 8 9 10w 13 14 15 16 17 18 19 11 12 13 14 15 16 17w -20 21 22 23 24 25 26 18 19 20 21 22 23w24 +20 21 22 23 24 25 26 18 19 20 21 22 23w24n 27 28 29 30m 25 26 27 28d29d30d31d List view --------- ---keep-daily 1w --keep-weekly 4 --keep-monthly 5m --keep-yearly 2 +--keep-daily 1w --keep-weekly 4 --keep-monthly 6m --keep-yearly 2 -------------------------------------------------------------------------------- - 1. 2025-12-31 - 2. 2025-11-15 (oldest) + 1. 2025-11-15 (oldest) 1. 2026-06-02 1. 2026-05-23 1. 2026-04-30 2. 2026-06-01 2. 2026-05-17 2. 2026-03-30 3. 2026-05-31 3. 2026-05-10 3. 2026-02-28 4. 2026-05-30 4. 2026-05-03 4. 2026-01-31 -5. 2026-05-29 +5. 2026-05-29 5. 2025-12-31 6. 2026-05-28 2026-06-04 is additionally kept due to `--from`. @@ -115,7 +115,7 @@ later reference time, e.g. `--from '2026-06-04 23:59:59'`. 2026-05-31 is considered not only by `--keep-daily`, but by `--keep-weekly` and `--keep-monthly`, too. The archive is effectively kept by `--keep-daily`, but how this affects other rules differs between *count*- and *interval*-based -policies. For *interval*-based rules like `--keep-monthly 5m` it has no effect: +policies. For *interval*-based rules like `--keep-monthly 6m` it has no effect: The rule simply keeps one archive fewer in that case. For *count*-based rules like `--keep-weekly 4` it has an effect: The policy @@ -133,21 +133,17 @@ Since 2026-05-24 and 2026-03-31 were skipped, Borg substitutes the next best candidate for each: 2026-05-23 fills the week 21 weekly slot, and 2026-03-30 fills the March monthly slot. -The `--keep-monthly 5m` window starts five calendar months before the -reference time, on 2026-01-04 at 16:00. The archive from 2025-12-31 is -outside this window, so it is not retained by `--keep-monthly`. - -Instead, `--keep-yearly 2` keeps 2025-12-31 as its yearly candidate. It -also keeps the oldest archive, 2025-11-15, to fill the remaining slot. +The `--keep-monthly 6m` window starts on 2025-12-04 at 16:00. +The archive from 2025-12-31 falls within this window and is kept by +`--keep-monthly`. Since interval rules define time windows rather than competing for a fixed number of slots, their interplay is simpler than count-based rules. An archive is kept by an interval rule as long as it falls within the specified window; the next rule simply considers whatever remains. -Intervals and counts can be mixed freely. Yearly retention in this example is -done by retention count instead of intervals. A count rule paired with interval -rules behaves just as it would if all preceding rules were also counts: -Archives already kept by earlier rules are excluded from consideration. In this -example, the 2026 yearly slot is consumed by earlier rules. December 31 -fills the 2025 slot, and November 15 is retained as the oldest archive. +Intervals and counts can be mixed freely. Archives already kept by earlier +rules do not count again toward a count-based rule. Here, `--keep-monthly` +keeps 2025-12-31, so it does not fill a `--keep-yearly` slot. The yearly +rule keeps 2025-11-15 as the oldest remaining archive to preserve rolling +backup semantics. diff --git a/docs/usage/general/date-time.rst.inc b/docs/usage/general/date-time.rst.inc index 9e2420f845..b5200ab7d4 100644 --- a/docs/usage/general/date-time.rst.inc +++ b/docs/usage/general/date-time.rst.inc @@ -17,21 +17,20 @@ a number of years (e.g. ``2y``), months (e.g. ``12m``), weeks (e.g. ``2w``), days (e.g. ``7d``), hours (e.g. ``8H``), minutes (e.g. ``30M``), or seconds (e.g. ``150S``). +Weeks, days, hours, minutes, and seconds have fixed lengths. For example, +``1w`` is 7 days and ``1d`` is ``86400S``. Months and years are interpreted +relative to a reference timestamp. ``1m`` can be shorter than, equal to, +or longer than ``30d``. ``1y`` spans 365 or 366 days and always matches +12 months. + +Months and years follow the calendar. For example, ``12m`` before +2026-07-22 is 2025-07-22. ``3y`` before 2026-07-22 is 2023-07-22. +The time of day is preserved. If the target month has fewer days, the date +is clamped to its last day. One month before 2024-03-31 is 2024-02-29. +One year before 2024-02-29 is 2023-02-28. + The ``borg prune`` ``--keep-*`` retention options accept either a plain count (e.g. ``--keep-daily 7``, keeping up to 7 daily archives) or a time interval (e.g. ``--keep-daily 7d``, keeping one daily archive per day within a 7-day window). When using interval-based retention, ``--from`` may be specified to set the reference timestamp for the interval (defaults to the current time). - -Months and years are calendar-aware. For example, ``12m`` before -2026-07-22 is 2025-07-22, and ``3y`` before 2026-07-22 is 2023-07-22. -The time of day is preserved. If the target month has fewer days, the date -is clamped to its last day: one month before March 31 is February 28 -(or February 29 in a leap year). - -Weeks, days, hours, minutes, and seconds keep their fixed-length meanings. -For example, ``1w`` is seven days. Plain counts and ``all`` are unchanged. - -When combining interval-based pruning rules, their lengths are compared at -the same reference timestamp. Thus, ``1m`` can be shorter than, equal to, -or longer than ``30d`` depending on the month. diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 757589b195..2ed1e73d0e 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -4,7 +4,7 @@ import math from functools import partial, wraps import os -from itertools import count, combinations +from itertools import count from ._common import with_repository, Highlander, archive_match_patterns from ..constants import * # NOQA from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error @@ -202,16 +202,16 @@ def do_prune(self, args, repository, manifest): from_timestamp = getattr(args, PRUNE_FROM.key) base_timestamp = from_timestamp if from_timestamp is not None else datetime.now().astimezone() - for rule in PRUNING_RULES: - value = getattr(args, rule.key) + keep_args = {rule.key: getattr(args, rule.key) for rule in PRUNING_RULES if getattr(args, rule.key) is not None} + for name, value in keep_args.items(): if not isinstance(value, str): continue try: cutoff = calculate_relative_offset(value, base_timestamp, earlier=True) except (ValueError, OverflowError) as error: - raise CommandError(f"Invalid retention interval {value!r} for {rule.key}: {error}") from None - setattr(args, rule.key, base_timestamp - cutoff) - self._validate_prune_args(args) + raise CommandError(f"Invalid retention interval {value!r} for {name}: {error}") from None + keep_args[name] = base_timestamp - cutoff + self._validate_prune_args(keep_args) match = archive_match_patterns(args) archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True) @@ -228,7 +228,7 @@ def do_prune(self, args, repository, manifest): group_of = {} group_summaries = [] for key, group in groups.items(): - group_keep = self._compute_keep(group, args, base_timestamp) + group_keep = self._compute_keep(group, args, base_timestamp, keep_args) keep |= group_keep group_of.update(dict.fromkeys(group, key)) group_summaries.append((key, len(group), len(group_keep))) @@ -318,7 +318,7 @@ def do_prune(self, args, repository, manifest): if sig_int: raise Error("Got Ctrl-C / SIGINT.") - def _compute_keep(self, archives, args, base_timestamp): + def _compute_keep(self, archives, args, base_timestamp, keep_args): """ Apply the retention rules to *archives* (sorted by timestamp, newest first). @@ -341,26 +341,20 @@ def _compute_keep(self, archives, args, base_timestamp): # Apply each retention rule to all candidate archives. The # `previously_kept` parameter prevents later (coarser-grained) rules # from double-counting archives already retained by earlier rules. - active_rules = [ - (rule, getattr(args, rule.key)) for rule in PRUNING_RULES if getattr(args, rule.key) is not None - ] + active_rules = [(rule, keep_args[rule.key]) for rule in PRUNING_RULES if rule.key in keep_args] for rule, n_or_interval in active_rules: keep |= prune( archives=candidate_archives, rule=rule, n_or_interval=n_or_interval, base_timestamp=base_timestamp, - keep_oldest=( - rule == active_rules[-1][0] - ), # Activate keep_oldest rule only for the largest active interval + keep_oldest=rule == active_rules[-1][0], previously_kept=frozenset(keep), ) return keep - def _validate_prune_args(self, args): - keep_args = {rule.key: getattr(args, rule.key) for rule in PRUNING_RULES if getattr(args, rule.key) is not None} - + def _validate_prune_args(self, keep_args): if len(keep_args) == 0: raise CommandError( 'At least one of the "keep", "keep-secondly", "keep-minutely", "keep-hourly", "keep-daily", ' @@ -375,26 +369,6 @@ def _validate_prune_args(self, args): "At least one must be non-zero." ) - def lo_hi_mismatch_errmsg(lo_arg, lo_val, hi_arg, hi_val): - return ( - f"The combination of \"{lo_arg}='{lo_val}'\" and \"{hi_arg}='{hi_val}'\" is invalid. It is effectively " - f"useless since every archive matched by {hi_arg} would have already been matched by {lo_arg}." - ) - - interval_args = [(arg, val) for arg, val in keep_args.items() if isinstance(val, timedelta) or val == -1] - for (lo_arg, lo_val), (hi_arg, hi_val) in combinations(interval_args, 2): - if hi_val == -1: - # 'Infinity' is always bigger - continue - - if lo_val == -1 or lo_val >= hi_val: - raise CommandError(lo_hi_mismatch_errmsg(lo_arg, lo_val, hi_arg, hi_val)) - - int_args = [(arg, val) for arg, val in keep_args.items() if isinstance(val, int)] - for (lo_arg, lo_val), (hi_arg, hi_val) in combinations(int_args, 2): - if lo_val == -1: - raise CommandError(lo_hi_mismatch_errmsg(lo_arg, lo_val, hi_arg, hi_val)) - def build_parser_prune(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog from ._common import define_archive_filters_group @@ -474,8 +448,8 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): that time window. When ``--from`` is given together with an interval retention, the interval is measured backwards from that timestamp instead of from the current time. See ``Date and Time`` docs for exact - INTERVAL format. Months and years are calendar-aware: a month before - March 31 is February 28 (or 29 in a leap year), at the same time of day. + INTERVAL format. Months and years follow the calendar. One month before + 2024-03-31 is 2024-02-29, at the same time of day. The ``--keep-secondly``, ``--keep-minutely``, ``--keep-hourly``, ``--keep-daily``, ``--keep-weekly``, ``--keep-monthly``, @@ -508,16 +482,11 @@ def build_parser_prune(self, subparsers, common_parser, mid_common_parser): schemes, where the earliest backup in a retention window should survive until the next tier's interval naturally replaces it. - When using interval-based pruning with multiple ``--keep-*`` options, - the intervals must be specified in increasing length matching the - periods chosen. For example, ``--keep-daily 7d --keep-weekly 4w`` is - valid, but ``--keep-daily 30d --keep-weekly 7d`` is not, because the - weekly interval is already covered by the daily one and so the weekly - interval is effectively useless. An error is emitted upon running - ``borg prune`` if such a combination of flags is given. The order of - flags on the command line is not significant. Calendar intervals are - compared at the reference timestamp: ``1m`` can be shorter than, equal - to, or longer than ``30d`` depending on the month. + Retention rules may overlap. For example, + ``--keep-daily 30d --keep-weekly 7d`` is allowed. The weekly rule does + not keep additional archives because the daily rule already covers + its time window. The order of flags on the command line does not + change the order in which the rules are applied. A practical approach for recurring backups is to use rules diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index aaa15bd8b3..ba5d8242b0 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -174,7 +174,7 @@ def test_prune_repository_example_interval(archivers, request, backup_files): "--from=2026-06-04T16:00:00+00:00", "--keep-daily=1w", "--keep-weekly=4", - "--keep-monthly=5m", + "--keep-monthly=6m", "--keep-yearly=2", ) @@ -203,12 +203,17 @@ def test_prune_repository_example_interval(archivers, request, backup_files): for i, name in enumerate(weekly_kept, 1): assert re.search(rf"Keeping archive \(rule: weekly #{i}\):\s+{name}", output) - monthly_kept = ["backup_2026-04-30", "backup_2026-03-30", "backup_2026-02-28", "backup_2026-01-31"] + monthly_kept = [ + "backup_2026-04-30", + "backup_2026-03-30", + "backup_2026-02-28", + "backup_2026-01-31", + "backup_2025-12-31", + ] for i, name in enumerate(monthly_kept, 1): assert re.search(rf"Keeping archive \(rule: monthly #{i}\):\s+{name}", output) - assert re.search(r"Keeping archive \(rule: yearly #1\):\s+backup_2025-12-31", output) - assert re.search(r"Keeping archive \(rule: yearly\[oldest\] #2\):\s+backup_2025-11-15", output) + assert re.search(r"Keeping archive \(rule: yearly\[oldest\] #1\):\s+backup_2025-11-15", output) # 05-27 was kept by daily yesterday (window started 05-26) but falls out # today (window shifted to 05-28). W22 slot is consumed by daily-held 05-31, @@ -790,16 +795,32 @@ def test_prune_all_zero_multiple_multiple(archivers, request): @pytest.mark.parametrize( - "lo_val,hi_val", - [("14d", "7d"), ("-1", "7d"), ("-1", "1"), ("-1", "-1"), ("all", "7d"), ("all", "1"), ("all", "-1")], + "hourly,daily,keep_old", + [ + ("14d", "7d", False), + ("-1", "7d", True), + ("-1", "1", True), + ("-1", "-1", True), + ("all", "7d", True), + ("all", "1", True), + ("all", "-1", True), + ("30d", "7d", False), + ("12m", "1y", True), + ("1m", "30d", False), + ("365d", "1y", True), + ], ) -def test_prune_warns_on_redundant_interval_flags(archivers, request, lo_val, hi_val): +def test_prune_allows_redundant_rules(archivers, request, backup_files, hourly, daily, keep_old): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - output = _cmd_prune_error(archiver, "--dry-run", f"--keep-hourly={lo_val}", f"--keep-daily={hi_val}") - assert "hourly=" in output - assert "daily=" in output - assert "effectively useless" in output + _create_archive_ts(archiver, backup_files, "old", 2025, 12, 1) + _create_archive_ts(archiver, backup_files, "recent", 2026, 1, 31) + prune_args = ("--from=2026-02-01T00:00:00+00:00", f"--keep-hourly={hourly}", f"--keep-daily={daily}") + output = json.loads(prune_ungrouped(archiver, "--json", "--dry-run", *prune_args)) + assert {a["name"]: a["kept"] for a in output["archives"]} == {"old": keep_old, "recent": True} + prune_ungrouped(archiver, *prune_args) + archives = json.loads(cmd(archiver, "repo-list", "--json"))["archives"] + assert {a["name"] for a in archives} == ({"old", "recent"} if keep_old else {"recent"}) @pytest.mark.parametrize("lo_val,hi_val", [("7d", "14d"), ("7d", "-1"), ("1", "-1"), ("7d", "all"), ("1", "all")]) @@ -882,7 +903,8 @@ def test_prune_interval_rolling_schedule_oldest_retention(): archives = sorted(keep.keys(), key=lambda a: a.ts, reverse=True) - + # 2024-01-01 is on the three-month boundary on 2024-04-01. + # It falls outside the window on 2024-04-02. assert previous_archives[-1].ts.strftime("%m-%d") == "01-01" assert archives[-1].ts.strftime("%m-%d") == "01-31" @@ -1110,26 +1132,58 @@ def test_prune_calendar_interval(archivers, request, backup_files, reference, in @pytest.mark.parametrize( - "reference,shorter,longer,valid", + "reference,weekly,monthly,cutoff", [ - ("2026-03-01", "1m", "30d", True), - ("2026-04-01", "1m", "30d", False), - ("2026-05-01", "1m", "30d", False), - ("2024-07-22", "365d", "1y", True), - ("2025-07-22", "365d", "1y", False), - ("2026-07-22", "12m", "1y", False), - ("2026-03-01", "1m", "all", True), - ("2026-03-01", "all", "1m", False), + ("2026-03-10", "4w", "1m", "2026-02-10"), + ("2026-04-10", "4w", "1m", "2026-03-10"), + ("2026-05-01", "13w", "3m", "2026-01-30"), + ("2026-05-31", "13w", "3m", "2026-02-28"), ], ) -def test_prune_calendar_interval_order(archivers, request, reference, shorter, longer, valid): +def test_prune_calendar_rule_combinations(archivers, request, backup_files, reference, weekly, monthly, cutoff): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - prune_args = ("--from", reference + "T00:00:00+00:00", "--keep-daily", shorter, "--keep-monthly", longer) - if valid: - cmd(archiver, "prune", "--dry-run", *prune_args) - else: - assert "effectively useless" in _cmd_prune_error(archiver, "--dry-run", *prune_args) + reference = datetime.fromisoformat(reference).replace(tzinfo=timezone.utc) + cutoff = datetime.fromisoformat(cutoff).replace(tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "old", cutoff - timedelta(microseconds=1)) + _create_archive_dt(archiver, backup_files, "boundary", cutoff) + _create_archive_dt(archiver, backup_files, "from", reference) + prune_args = ("--from", reference.isoformat(), "--keep-weekly", weekly, "--keep-monthly", monthly) + output = json.loads(prune_ungrouped(archiver, "--json", "--dry-run", *prune_args)) + assert {a["name"]: a["kept"] for a in output["archives"]} == {"old": False, "boundary": True, "from": True} + archives = json.loads(cmd(archiver, "repo-list", "--json"))["archives"] + assert {a["name"] for a in archives} == {"old", "boundary", "from"} + prune_ungrouped(archiver, *prune_args) + archives = json.loads(cmd(archiver, "repo-list", "--json"))["archives"] + assert {a["name"] for a in archives} == {"boundary", "from"} + + +def test_prune_preserves_retention_args(archiver, backup_files, monkeypatch): + from ...archiver.prune_cmd import PruneMixIn + + seen = [] + original_compute_keep = PruneMixIn._compute_keep + + def compute_keep(self, archives, args, *rest): + seen.append((args.keep, args.daily, args.monthly)) + return original_compute_keep(self, archives, args, *rest) + + monkeypatch.setattr(PruneMixIn, "_compute_keep", compute_keep) + cmd(archiver, "repo-create", RK_ENCRYPTION) + for host in ("host1", "host2"): + monkeypatch.setenv("BORG_HOSTNAME", host) + _create_archive_ts(archiver, backup_files, "home", 2026, 3, 30) + cmd( + archiver, + "prune", + "--dry-run", + "--group-by=host", + "--from=2026-03-31T00:00:00+00:00", + "--keep=1m", + "--keep-daily=1", + "--keep-monthly=2m", + ) + assert seen == [("1m", 1, "2m"), ("1m", 1, "2m")] def test_prune_calendar_interval_without_from(archiver, backup_files, monkeypatch):