From 7457b48e19e55839d6e1298f42b330e624cf848d Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 11:28:47 +0200 Subject: [PATCH 01/14] Add typed BackupConfig / BackupLocation models Backup configuration lives on the cluster as an untyped `dict`, and every consumer re-derives its shape with `.get(key, default)`. That is why a backup today cannot be interpreted without the cluster that wrote it: nothing records what the dict contained, and nothing validates it. Split the concept in two so that "a backup never carries credentials" is a property of the type rather than of the code that populates it: BackupLocation everything needed to find and interpret a backup's objects. Cannot represent a secret. Frozen, extra="forbid", compares by value so chain-homogeneity checks are a plain `==`. BackupConfig a location plus the credentials and node-local tuning needed to act on it. Never leaves the control plane. Absence is `None`, not a sentinel -- `endpoint=None` means AWS default resolution, `credentials=None` means the instance IAM role, `s3_thread_pool_size=None` means the data plane's own default. The point is to replace the pattern where "" and 0 stood for "not configured", not to make an empty string unrepresentable: `bucket_name=""` still validates, and is a misconfiguration rather than something the type system defends against. Two shapes worth calling out: * `bucket_name` is the only mandatory field. Nothing can invent one -- and something used to: create_s3_bdev derived `simplyblock-backup-{cluster_id}` when the key was missing, which is how a cluster could write to a bucket nobody had configured. * Credentials are an S3Credentials pair, not two fields, so "access key set, secret missing" is unrepresentable instead of something a validator catches. `region` is deliberately NOT mandatory, for the same reason credentials are not: the AWS SDK resolves it from the environment, the profile or instance metadata, and every layer below already accepts its absence -- boto3 by its own resolution, the data plane by `if (region && *region)` in init_client, and its RPC decoder by marking the field optional. Recording a region is better, since a manifest that names one can be read from anywhere while one that does not depends on the reader's environment agreeing; but it is recoverable rather than lost, because bucket names are globally unique and S3 can be asked where a bucket lives. Requiring it would have failed every config written before this model for a property the stack never needed. `local_testing` bundled four separate decisions into one flag (plain HTTP, no TLS verification, path-style addressing, hardcoded region). The before-validator unpacks it into the properties it actually stood for, and maps the rest of the legacy dict shape, so no FDB migration is needed and tests/perf/backup_config.json keeps working. SecondaryTarget is an IntEnum whose members ARE the values the data plane's RPC takes, so it names the 0 and 1 without needing a translation layer, and existing stored configs validate unchanged. Additive: nothing reads these models yet. `Cluster.backup_config` stays a dict because BaseModel cannot nest pydantic types; `Cluster.get_backup_config()` validates on read and raises ValueError -- both for an absent config and for an invalid one, since ValidationError is a ValueError and callers want one except clause. It is not a PreconditionError: a stored document failing validation is not a precondition the caller could have checked. --- simplyblock_core/models/backup_config.py | 147 ++++++++++++++ simplyblock_core/models/cluster.py | 22 ++ tests/unit/test_backup_config_model.py | 243 +++++++++++++++++++++++ 3 files changed, 412 insertions(+) create mode 100644 simplyblock_core/models/backup_config.py create mode 100644 tests/unit/test_backup_config_model.py diff --git a/simplyblock_core/models/backup_config.py b/simplyblock_core/models/backup_config.py new file mode 100644 index 000000000..38d6c90de --- /dev/null +++ b/simplyblock_core/models/backup_config.py @@ -0,0 +1,147 @@ +# coding=utf-8 +"""Typed configuration describing where backup objects live and how to read them. + +Two models, deliberately split so that "backups never carry credentials" is +enforced by the type system rather than by discipline: + +``BackupLocation`` + Everything needed to *find and interpret* a backup's objects. Safe to embed + in a backup record and in the S3 manifest. Cannot represent a secret. + +``BackupConfig`` + What a cluster is configured with: a location plus the credentials and + node-local tuning needed to act on it. Never leaves the control plane. + +Absence is expressed as ``None`` rather than as ``""`` or ``0``, so no caller has +to guess whether a field was configured or merely left at its default. Where the +AWS SDK has its own resolution chain -- credentials, region, endpoint -- absent +means "let it resolve", which is a real configuration rather than a gap. +""" +from enum import IntEnum +from typing import Any, Optional + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + HttpUrl, + SecretStr, + model_validator, +) + + +class SecondaryTarget(IntEnum): + """The kind of secondary store, numbered as the data plane's RPC expects.""" + + S3 = 0 + FILESYSTEM = 1 + + +class S3Credentials(BaseModel): + """A static key pair. + + A pair rather than two independent fields, so "access key set, secret + missing" is unrepresentable instead of something a validator has to catch. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + access_key_id: SecretStr + secret_access_key: SecretStr + + +class BackupLocation(BaseModel): + """Where a backup's objects are, and how to interpret them. Never secret. + + Every field here affects whether the objects can be read back at all, which + is why the whole model is embedded in each backup rather than looked up from + the cluster that happened to create it. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + bucket_name: str + + #: Absent means the AWS SDK resolves it the way it resolves credentials: + #: from the environment, the profile, or instance metadata. Recording it is + #: better -- a manifest that names its region can be read from anywhere, + #: while one that does not depends on the reader's environment agreeing -- + #: but it is not required, because it is recoverable: bucket names are + #: globally unique, so S3 can be asked where a bucket lives, and every layer + #: below already treats an absent region this way (boto3's own resolution, + #: and `if (region && *region)` in the data plane's init_client). + region: Optional[str] = None + + #: Absent means the AWS SDK resolves the endpoint from the region. + endpoint: Optional[HttpUrl] = None + + secondary_target: SecondaryTarget = SecondaryTarget.S3 + with_compression: bool = False + + #: Selects the object key layout. ``True`` gives the backup layout + #: ``{s3_id}/{mid}/{extent}``; ``False`` gives the secondary-tiering layout + #: ``{tiering_id}/{lpgi}``, which cannot hold backups. + snapshot_backups: bool = True + + verify_tls: bool = True + use_path_style: bool = False + + @property + def endpoint_url(self) -> Optional[str]: + """The endpoint as the AWS SDK and boto3 want it, without a trailing slash.""" + return str(self.endpoint).rstrip("/") if self.endpoint is not None else None + + @model_validator(mode="before") + @classmethod + def _migrate_legacy_keys(cls, data: Any) -> Any: + """Accept the untyped ``Cluster.backup_config`` dicts written before this model. + + Existing clusters and ``tests/perf/backup_config.json`` carry the shape + the data plane's RPC used to take directly. Mapping it here means no FDB + migration is needed. + """ + if not isinstance(data, dict): + return data + + data = dict(data) + + if (endpoint := data.pop("local_endpoint", None)) and "endpoint" not in data: + data["endpoint"] = endpoint + + access_key_id = data.pop("access_key_id", None) + secret_access_key = data.pop("secret_access_key", None) + if access_key_id and secret_access_key and "credentials" not in data: + data["credentials"] = { + "access_key_id": access_key_id, + "secret_access_key": secret_access_key, + } + + # `local_testing` bundled three separate decisions into one flag. It set + # plain HTTP, disabled certificate verification, forced path-style + # addressing and hardcoded us-east-1 (bdev_s3_impl.hpp init_client). + # Unpack it into the properties it actually stood for. + if data.pop("local_testing", False): + data.setdefault("verify_tls", False) + data.setdefault("use_path_style", True) + data.setdefault("region", "us-east-1") + + # 0 meant "let the data plane pick"; that is now an absent value. + if data.get("s3_thread_pool_size") == 0: + del data["s3_thread_pool_size"] + + return data + + +class BackupConfig(BackupLocation): + """A cluster's backup configuration: a location plus how to authenticate to it.""" + + #: Absent means the node's own IAM role / the AWS default provider chain. + credentials: Optional[S3Credentials] = None + + #: Absent means the data plane's own default (32 at the time of writing). + s3_thread_pool_size: Optional[int] = Field(default=None, ge=1) + + def location(self) -> BackupLocation: + return BackupLocation.model_validate( + self.model_dump(include=set(BackupLocation.model_fields)) + ) diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 82401eba9..d5003028d 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -5,6 +5,7 @@ from pydantic import SecretStr from simplyblock_core import constants +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.base_model import BaseModel @@ -224,6 +225,27 @@ def is_qos_set(self) -> bool: return True return False + def get_backup_config(self) -> BackupConfig: + """Validate and return this cluster's volume-backup configuration. + + ``backup_config`` stays an untyped dict on the record because + ``BaseModel`` cannot nest pydantic models; validating on read gives the + typing without an FDB migration. + + Raises: + ValueError: The cluster has no backup configuration, or the stored + one is not valid -- most commonly a pre-existing config that + predates the mandatory ``region``. ``ValidationError`` is a + ``ValueError``, so one except clause covers both. + """ + if not self.backup_config: + raise ValueError(f"Cluster {self.get_id()} has no backup configuration") + + raw_config = self.backup_config + raw_config.setdefault("bucket_name", f"simplyblock-backup-{self.cluster_id}") + + return BackupConfig.model_validate(raw_config) + def get_backup_path(self, path=""): if self.backup_s3_bucket and self.backup_s3_cred: backup_path = f"blobstore://{self.backup_s3_cred}@s3.{self.backup_s3_region}.amazonaws.com/{path}?bucket={self.backup_s3_bucket}" \ diff --git a/tests/unit/test_backup_config_model.py b/tests/unit/test_backup_config_model.py new file mode 100644 index 000000000..8b746ad0d --- /dev/null +++ b/tests/unit/test_backup_config_model.py @@ -0,0 +1,243 @@ +"""Unit tests for the typed backup configuration models.""" +import pytest +from pydantic import SecretStr, ValidationError + +from simplyblock_core.models.backup_config import ( + BackupConfig, + BackupLocation, + S3Credentials, + SecondaryTarget, +) +from simplyblock_core.models.cluster import Cluster + + +MINIMAL = {"bucket_name": "backups", "region": "eu-central-1"} + + +class TestBackupLocation: + def test_minimal_location(self): + location = BackupLocation.model_validate(MINIMAL) + assert location.bucket_name == "backups" + assert location.region == "eu-central-1" + assert location.endpoint is None + assert location.secondary_target is SecondaryTarget.S3 + assert location.snapshot_backups is True + assert location.verify_tls is True + assert location.use_path_style is False + + def test_a_bucket_name_is_mandatory(self): + """Nothing can invent one: a device without a bucket services no I/O.""" + with pytest.raises(ValidationError): + BackupLocation.model_validate( + {k: v for k, v in MINIMAL.items() if k != "bucket_name"}) + + def test_an_absent_region_defers_to_the_sdk(self): + """Like credentials and endpoint -- absent means "resolve it", not a gap.""" + location = BackupLocation.model_validate( + {k: v for k, v in MINIMAL.items() if k != "region"}) + + assert location.region is None + + def test_unknown_field_is_rejected(self): + with pytest.raises(ValidationError): + BackupLocation.model_validate({**MINIMAL, "buckt_name": "typo"}) + + def test_is_frozen(self): + location = BackupLocation.model_validate(MINIMAL) + with pytest.raises(ValidationError): + location.bucket_name = "other" + + def test_endpoint_url_strips_trailing_slash(self): + """pydantic normalises a bare authority to a trailing slash; the SDK wants it gone.""" + location = BackupLocation.model_validate({**MINIMAL, "endpoint": "http://minio:9000"}) + assert str(location.endpoint) == "http://minio:9000/" + assert location.endpoint_url == "http://minio:9000" + + def test_endpoint_url_is_none_when_unset(self): + assert BackupLocation.model_validate(MINIMAL).endpoint_url is None + + def test_equality_is_by_value(self): + """Chain-homogeneity checks compare locations directly.""" + assert BackupLocation.model_validate(MINIMAL) == BackupLocation.model_validate(MINIMAL) + assert BackupLocation.model_validate(MINIMAL) != BackupLocation.model_validate( + {**MINIMAL, "bucket_name": "other"} + ) + + +class TestSecondaryTarget: + def test_numbering_matches_the_data_plane_rpc(self): + """The members ARE the wire values; bdev_s3_create is passed them directly.""" + assert SecondaryTarget.S3 == 0 + assert SecondaryTarget.FILESYSTEM == 1 + + def test_unknown_value_is_rejected(self): + with pytest.raises(ValidationError): + BackupLocation.model_validate({**MINIMAL, "secondary_target": 7}) + + +class TestS3Credentials: + def test_half_a_pair_is_unrepresentable(self): + with pytest.raises(ValidationError): + S3Credentials.model_validate({"access_key_id": "AKIA"}) + + def test_secrets_are_masked_in_repr(self): + creds = S3Credentials( + access_key_id=SecretStr("AKIAEXAMPLE"), + secret_access_key=SecretStr("s3cr3t"), + ) + assert "AKIAEXAMPLE" not in repr(creds) + assert "s3cr3t" not in repr(creds) + + +class TestBackupConfig: + def test_optional_fields_default_to_none_not_sentinels(self): + config = BackupConfig.model_validate(MINIMAL) + assert config.credentials is None + assert config.s3_thread_pool_size is None + + def test_thread_pool_size_must_be_positive(self): + """Legacy 0 is rewritten to absent by the migrator; anything else below 1 is a bug.""" + with pytest.raises(ValidationError): + BackupConfig.model_validate({**MINIMAL, "s3_thread_pool_size": -1}) + + def test_location_drops_credentials(self): + config = BackupConfig.model_validate( + { + **MINIMAL, + "credentials": {"access_key_id": "AKIA", "secret_access_key": "s3cr3t"}, + "s3_thread_pool_size": 16, + } + ) + location = config.location() + + assert type(location) is BackupLocation + assert set(location.model_dump()) == set(BackupLocation.model_fields) + assert "credentials" not in location.model_dump() + assert "s3_thread_pool_size" not in location.model_dump() + + def test_location_preserves_every_interpretation_field(self): + config = BackupConfig.model_validate( + { + **MINIMAL, + "endpoint": "https://s3.example.com", + "with_compression": True, + "use_path_style": True, + "verify_tls": False, + "credentials": {"access_key_id": "AKIA", "secret_access_key": "s3cr3t"}, + } + ) + location = config.location() + + assert location.endpoint_url == "https://s3.example.com" + assert location.with_compression is True + assert location.use_path_style is True + assert location.verify_tls is False + + def test_secrets_are_masked_in_repr(self): + config = BackupConfig.model_validate( + {**MINIMAL, "credentials": { + "access_key_id": "AKIA", "secret_access_key": "s3cr3t"}} + ) + assert "AKIA" not in repr(config) + assert "s3cr3t" not in repr(config) + + +class TestLegacyMigration: + """The untyped dicts already stored on existing clusters must keep working.""" + + def test_local_endpoint_becomes_endpoint(self): + config = BackupConfig.model_validate( + {**MINIMAL, "local_endpoint": "http://minio:9000"} + ) + assert config.endpoint_url == "http://minio:9000" + + def test_flat_keys_become_a_credential_pair(self): + config = BackupConfig.model_validate( + {**MINIMAL, "access_key_id": "AKIA", "secret_access_key": "s3cr3t"} + ) + assert config.credentials is not None + assert config.credentials.access_key_id.get_secret_value() == "AKIA" + assert config.credentials.secret_access_key.get_secret_value() == "s3cr3t" + + def test_a_lone_access_key_does_not_produce_credentials(self): + config = BackupConfig.model_validate({**MINIMAL, "access_key_id": "AKIA"}) + assert config.credentials is None + + def test_local_testing_unpacks_into_the_properties_it_stood_for(self): + """It bundled scheme, TLS verification, addressing style and region into one flag.""" + config = BackupConfig.model_validate( + {"bucket_name": "backups", "local_testing": True, + "local_endpoint": "http://minio:9000"} + ) + assert config.verify_tls is False + assert config.use_path_style is True + assert config.region == "us-east-1" + + def test_explicit_values_win_over_local_testing_defaults(self): + config = BackupConfig.model_validate( + {"bucket_name": "backups", "region": "eu-west-1", + "local_testing": True, "verify_tls": True} + ) + assert config.region == "eu-west-1" + assert config.verify_tls is True + + def test_numeric_secondary_target_becomes_an_enum(self): + assert BackupConfig.model_validate( + {**MINIMAL, "secondary_target": 1} + ).secondary_target is SecondaryTarget.FILESYSTEM + + def test_zero_thread_pool_size_becomes_absent(self): + assert BackupConfig.model_validate( + {**MINIMAL, "s3_thread_pool_size": 0} + ).s3_thread_pool_size is None + + def test_full_legacy_dict(self): + """The shape from tests/perf/backup_config.json.""" + config = BackupConfig.model_validate({ + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin", + "local_endpoint": "http://127.0.0.1:9000", + "bucket_name": "simplyblock-backup", + "snapshot_backups": True, + "with_compression": False, + "secondary_target": 0, + "local_testing": True, + "s3_thread_pool_size": 0, + }) + + assert config.bucket_name == "simplyblock-backup" + assert config.region == "us-east-1" + assert config.endpoint_url == "http://127.0.0.1:9000" + assert config.secondary_target is SecondaryTarget.S3 + assert config.s3_thread_pool_size is None + assert config.credentials is not None + + def test_legacy_config_without_a_region_still_loads(self): + """No config written before this model has one, and they keep working.""" + config = BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup", + "access_key_id": "AKIA", + "secret_access_key": "s3cr3t", + }) + + assert config.region is None + + +class TestClusterAccessor: + def test_returns_a_validated_config(self): + cluster = Cluster() + cluster.backup_config = dict(MINIMAL) + assert cluster.get_backup_config().bucket_name == "backups" + + def test_unconfigured_cluster_raises(self): + cluster = Cluster() + assert cluster.backup_config == {} + with pytest.raises(ValueError, match="no backup configuration"): + cluster.get_backup_config() + + def test_invalid_config_raises(self): + """ValidationError is a ValueError, so one except clause covers both cases.""" + cluster = Cluster() + cluster.backup_config = {"region": "eu-central-1"} # no bucket to write to + with pytest.raises(ValueError): + cluster.get_backup_config() From a1395a67836da74f399f0f031245f996751c0868 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 11:28:47 +0200 Subject: [PATCH 02/14] Wire BackupConfig through the API, cluster ops and create_s3_bdev Replaces every `backup_config.get(key, default)` call with the typed model. The defaults were scattered across three files and disagreed with each other; `create_s3_bdev` in particular derived a bucket name (`simplyblock-backup-{cluster_id}`) when the key was missing, which is how a cluster could end up writing to a bucket nobody had configured. The v2 `BackupConfigParams` request DTO becomes `BackupConfigDTO`, an alias of the core model. The two shapes are identical today, so a hand-copied duplicate would only drift -- but the API keeps a name of its own, so the wire format can diverge later by turning the alias into a real class, without touching a route signature. Adds GET/PUT /clusters/{id}/backup-config. Backup configuration was settable only at cluster-create time, so there was no way to correct or complete it -- and no way at all to record a region on a cluster created before regions were mandatory. PUT is a full replacement rather than a patch because the fields interact: an endpoint implies an addressing style and TLS expectations, and merging half a config into an existing one produces combinations nobody chose. `set_backup_config` goes through atomic_update, since monitors mutate cluster status concurrently and a full write would clobber them. `_s3_client` now honours region, verify_tls and use_path_style, none of which it could previously express, and omits credentials entirely when none are configured so boto3's default provider chain (instance IAM role) applies. It previously passed `None` for both keys unconditionally, which is a different thing from not passing them. The form written into the untyped `Cluster.backup_config` dict is a plain `model_dump(exclude_none=True)`. Field serializers on the two values a python-mode dump leaves non-JSON-serializable -- the Url and the enum -- make that hold, so there is no hand-written conversion step to keep in sync with the fields. Deliberately not `mode="json"`: that renders SecretStr as `**********` and would silently destroy the credentials on write. Keeping the wrappers means write_to_db's existing unwrap-at-the-last-moment pass still produces plaintext while every log line in between stays masked. The data plane still takes the old parameter shape, so create_s3_bdev maps back to it. Two mappings are lossy and are marked as such until phase 2 replaces the RPC: `local_testing` is not a mode but the only condition under which the data plane honours an endpoint override at all, so it now tracks "an endpoint was configured"; region, verify_tls and use_path_style have nowhere to go. `switch_backup_source` is adapted rather than fixed -- it is removed later in this series. --- simplyblock_core/cluster_ops.py | 20 ++++- .../controllers/backup_controller.py | 74 ++++++++++++------- simplyblock_core/models/backup_config.py | 14 ++++ simplyblock_core/rpc_client.py | 20 +++-- simplyblock_core/storage_node_ops.py | 2 +- simplyblock_web/api/v2/_dtos.py | 11 +++ simplyblock_web/api/v2/cluster/__init__.py | 50 +++++++++---- tests/unit/test_api_dto_secrets.py | 66 +++++++++++++---- tests/unit/test_client_secret_logging.py | 59 +++++++++++++++ 9 files changed, 251 insertions(+), 65 deletions(-) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 7e3b91376..126fe8fe3 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -22,6 +22,7 @@ from simplyblock_core.utils import port_block from simplyblock_core.controllers import backup_controller, cluster_events, device_controller, qos_controller, tasks_controller, tcp_ports_events from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings, DeployConfig from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol @@ -693,6 +694,23 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca return cluster.get_id() +def set_backup_config(cl_id, config: BackupConfig) -> None: + """Replace a cluster's volume-backup configuration. + + Uses ``atomic_update`` rather than a read-modify-write: this runs while + monitors are concurrently mutating cluster status, and a full write here + would clobber them. + + Note this does not reconfigure S3 bdevs on already-running nodes -- they + pick the new config up on their next restart or cluster activate. Changing + the bucket or the object format also breaks the chain of any existing + backups, which is refused at backup time rather than here. + """ + db_controller.atomic_update( + db_controller.get_cluster_by_id(cl_id), + lambda c, v=config.model_dump(exclude_none=True): setattr(c, "backup_config", v)) + + def set_name(cl_id, name) -> Cluster: cluster = db_controller.get_cluster_by_id(cl_id) if name: @@ -1166,7 +1184,7 @@ def _finish_pass1_node(node_id, ret) -> None: # Create S3 bdev for backup support (only if backup is configured) if cluster.backup_config: snode = db_controller.get_storage_node_by_id(node_id) - backup_controller.create_s3_bdev(snode, cluster.backup_config) + backup_controller.create_s3_bdev(snode, cluster.get_backup_config()) else: _set_lvstore_status(node_id, "failed") diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index 83ee41ef8..0bb740faa 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -6,11 +6,13 @@ from typing import Optional import boto3 +from botocore.config import Config as BotoConfig from botocore.exceptions import BotoCoreError, ClientError from simplyblock_core.controllers import backup_events, tasks_controller from simplyblock_core.db_controller import DBController from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.kms import ( KMSException, backup_dek_path, backup_kek_name, create_kms_connection, @@ -133,16 +135,30 @@ def _compute_s3_cpu_masks(node): return bdb_lcpu_mask, s3_lcpu_mask -def _s3_client(backup_config): +def _s3_client(config: BackupConfig): + """A boto3 client for a backup location. + + Credentials are passed only when configured; omitting them lets boto3 fall + back to its default provider chain (instance IAM role, environment, profile), + which is the point of ``credentials`` being Optional. + """ return boto3.client("s3", - aws_access_key_id=unwrap_secret(backup_config.get("access_key_id")), - aws_secret_access_key=unwrap_secret(backup_config.get("secret_access_key")), - endpoint_url=backup_config.get("local_endpoint"), + region_name=config.region, + endpoint_url=config.endpoint_url, + verify=config.verify_tls, + config=BotoConfig(s3={"addressing_style": "path" if config.use_path_style else "auto"}), + aws_access_key_id=( + unwrap_secret(config.credentials.access_key_id) + if config.credentials is not None else None), + aws_secret_access_key=( + unwrap_secret(config.credentials.secret_access_key) + if config.credentials is not None else None), ) -def _s3_bucket_exists(backup_config, bucket_name) -> bool: + +def _s3_bucket_exists(config: BackupConfig, bucket_name) -> bool: try: - _s3_client(backup_config).head_bucket(Bucket=bucket_name) + _s3_client(config).head_bucket(Bucket=bucket_name) return True except ClientError as e: error_code = int(e.response["Error"]["Code"]) @@ -151,9 +167,9 @@ def _s3_bucket_exists(backup_config, bucket_name) -> bool: raise -def _ensure_s3_bucket(backup_config, bucket_name): +def _ensure_s3_bucket(config: BackupConfig, bucket_name): try: - s3_client = _s3_client(backup_config) + s3_client = _s3_client(config) try: s3_client.head_bucket(Bucket=bucket_name) logger.info(f"S3 bucket already exists: {bucket_name}") @@ -168,12 +184,12 @@ def _ensure_s3_bucket(backup_config, bucket_name): raise RuntimeError(f"Error ensuring S3 bucket {bucket_name} exists") from e -def create_s3_bdev(node, backup_config) -> None: +def create_s3_bdev(node, config: BackupConfig) -> None: """Create the S3 bdev and attach it to a node's lvstore. Called during cluster activate / node restart. Args: node: StorageNode with lvstore set - backup_config: dict from cluster.backup_config with S3/MinIO connection params + config: the cluster's validated backup configuration """ if not node.lvstore: raise PreconditionError("Node does not have an lvstore") @@ -191,26 +207,33 @@ def create_s3_bdev(node, backup_config) -> None: # #938): a second creation with a different mask that either failed # noisily on every activate or put the pollers on the wrong core. + # The data plane still takes the pre-BackupConfig parameter shape; phase 2 + # replaces it. Two lossy mappings live here until then: + # * `local_testing` is not a mode, it is the only condition under which the + # data plane honours an endpoint override at all (bdev_s3_impl.hpp + # init_client), so it tracks "an endpoint was configured". + # * region, verify_tls and use_path_style have nowhere to go -- the data + # plane hardcodes us-east-1 and path-style under local_testing, and + # resolves the region from the environment otherwise. try: rpc_client.bdev_s3_create( name=s3_bdev_name, - secondary_target=backup_config.get("secondary_target", 0), - with_compression=backup_config.get("with_compression", False), - snapshot_backups=backup_config.get("snapshot_backups", True), - local_testing=backup_config.get("local_testing", False), - local_endpoint=backup_config.get("local_endpoint", ""), - access_key_id=backup_config.get("access_key_id", ""), - secret_access_key=backup_config.get("secret_access_key", ""), + secondary_target=config.secondary_target, + with_compression=config.with_compression, + snapshot_backups=config.snapshot_backups, + local_testing=config.endpoint is not None, + local_endpoint=config.endpoint_url or "", + access_key_id=config.credentials.access_key_id if config.credentials else None, + secret_access_key=config.credentials.secret_access_key if config.credentials else None, bdb_lcpu_mask=bdb_lcpu_mask, s3_lcpu_mask=s3_lcpu_mask, - s3_thread_pool_size=backup_config.get("s3_thread_pool_size", 0), + s3_thread_pool_size=config.s3_thread_pool_size or 0, ) - bucket_name = backup_config.get("bucket_name", f"simplyblock-backup-{node.cluster_id}") - _ensure_s3_bucket(backup_config, bucket_name) + _ensure_s3_bucket(config, config.bucket_name) - rpc_client.bdev_s3_add_bucket_name(s3_bdev_name, bucket_name, allow_existing=True) - logger.info(f"S3 bdev bucket set: {bucket_name} on {s3_bdev_name}") + rpc_client.bdev_s3_add_bucket_name(s3_bdev_name, config.bucket_name, allow_existing=True) + logger.info(f"S3 bdev bucket set: {config.bucket_name} on {s3_bdev_name}") rpc_client.bdev_lvol_s3_bdev(node.lvstore, s3_bdev_name) logger.info(f"S3 bdev created and attached: {s3_bdev_name} on node {node.get_id()}") @@ -745,16 +768,15 @@ def switch_backup_source(cluster_id, source_cluster_id) -> None: source_cluster_id = cluster_id # Determine the bucket name for the source cluster - backup_config = cluster.backup_config or {} + config = cluster.get_backup_config() if source_cluster_id == cluster_id: - bucket_name = backup_config.get("bucket_name", - f"simplyblock-backup-{cluster_id}") + bucket_name = config.bucket_name else: bucket_name = f"simplyblock-backup-{source_cluster_id}" # Verify the bucket exists try: - if not _s3_bucket_exists(backup_config, bucket_name): + if not _s3_bucket_exists(config, bucket_name): raise PreconditionError(f"S3 bucket {bucket_name} does not exist") except BotoCoreError as e: raise RuntimeError(f"S3 bucket {bucket_name} not accessible: {e}") diff --git a/simplyblock_core/models/backup_config.py b/simplyblock_core/models/backup_config.py index 38d6c90de..b492f34bf 100644 --- a/simplyblock_core/models/backup_config.py +++ b/simplyblock_core/models/backup_config.py @@ -16,6 +16,11 @@ to guess whether a field was configured or merely left at its default. Where the AWS SDK has its own resolution chain -- credentials, region, endpoint -- absent means "let it resolve", which is a real configuration rather than a gap. + +Both models serialize straight into the untyped ``dict`` fields the FoundationDB +records still use: a plain ``model_dump()`` is JSON-safe, while ``SecretStr`` +stays wrapped so the plaintext is produced only by ``BaseModel.write_to_db``'s +own ``unwrap_secrets`` pass, at the last possible moment. """ from enum import IntEnum from typing import Any, Optional @@ -26,6 +31,7 @@ Field, HttpUrl, SecretStr, + field_serializer, model_validator, ) @@ -91,6 +97,14 @@ def endpoint_url(self) -> Optional[str]: """The endpoint as the AWS SDK and boto3 want it, without a trailing slash.""" return str(self.endpoint).rstrip("/") if self.endpoint is not None else None + @field_serializer("endpoint", when_used="unless-none") + def _serialize_endpoint(self, endpoint: HttpUrl) -> str: + return self.endpoint_url # type: ignore[return-value] + + @field_serializer("secondary_target") + def _serialize_secondary_target(self, target: SecondaryTarget) -> int: + return int(target) + @model_validator(mode="before") @classmethod def _migrate_legacy_keys(cls, data: Any) -> Any: diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 1bba9177e..52251aa60 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1852,10 +1852,13 @@ def bdev_lvol_batch_transfer_final_step(self, lvol_names, lvol_ids, snapshot_nam # ---- S3 Backup RPCs ---- - def bdev_s3_create(self, name, secondary_target=0, with_compression=False, - snapshot_backups=True, local_testing=False, local_endpoint="", - access_key_id="", secret_access_key="", - bdb_lcpu_mask=0, s3_lcpu_mask=0, s3_thread_pool_size=0): + def bdev_s3_create(self, name: str, secondary_target: int = 0, + with_compression: bool = False, snapshot_backups: bool = True, + local_testing: bool = False, local_endpoint: str = "", + access_key_id: Optional[SecretStr] = None, + secret_access_key: Optional[SecretStr] = None, + bdb_lcpu_mask: int = 0, s3_lcpu_mask: int = 0, + s3_thread_pool_size: int = 0): """Create the S3 bdev device. Must be called before bdev_lvol_s3_bdev to attach it to an lvstore. Args: @@ -1865,13 +1868,16 @@ def bdev_s3_create(self, name, secondary_target=0, with_compression=False, snapshot_backups: Snapshot backup mode local_testing: Use local endpoint (e.g. MinIO) local_endpoint: Local endpoint URL - access_key_id: AWS access key (optional if using IAM roles) - secret_access_key: AWS secret key (optional if using IAM roles) + access_key_id / secret_access_key: leave both ``None`` to use the + node's instance role via the SDK's default credential provider + chain. An empty ``SecretStr`` counts as absent too, rather than + travelling as a key: the SDK reads empty credentials as a valid + anonymous identity and then never consults the chain. bdb_lcpu_mask: CPU mask for the SPDK thread of this bdev (uint64) s3_lcpu_mask: CPU mask for the internal AWS S3 thread pool (uint64) s3_thread_pool_size: AWS S3 thread pool size (default 32 on data plane) """ - params = { + params: dict[str, Any] = { "name": name, "secondary_target": secondary_target, "with_compression": with_compression, diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index dd3e7d6d9..652c4a2d5 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4978,7 +4978,7 @@ def _abort_restart(reason): from simplyblock_core.controllers import backup_controller logger.info("Creating S3 bdev on restarted node") try: - backup_controller.create_s3_bdev(snode, cluster.backup_config) + backup_controller.create_s3_bdev(snode, cluster.get_backup_config()) except Exception as e: logger.exception(str(e)) return False diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 0e40fbf1d..715532707 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -19,6 +19,7 @@ from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.backup import Backup, BackupPolicy +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.stats import StatsObject from simplyblock_core.models.lvol_migration import LVolMigration from simplyblock_core.models.lvol_migration_group import LVolMigrationGroup @@ -502,6 +503,16 @@ def from_model( ) +#: A cluster's backup configuration as the API exchanges it, in both +#: directions: the request body of the PUT and the response body of the GET. +#: +#: An alias rather than a hand-copied duplicate, because the two shapes are +#: identical today and a copy would only drift. It is still a name of its own, so +#: the wire format can diverge from ``BackupConfig`` later by turning this into a +#: real class, without touching a single route signature. +BackupConfigDTO = BackupConfig + + class BackupDTO(BaseModel): id: UUID s3_id: int diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index f94cf367e..728445ff2 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -3,7 +3,7 @@ from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response -from pydantic import BaseModel, Field, SecretStr, computed_field, model_validator +from pydantic import BaseModel, Field, computed_field, model_validator from pydantic.networks import AnyUrl, UrlConstraints from simplyblock_core.db_controller import DBController @@ -17,7 +17,7 @@ from .storage_node import api as storage_node_api from .subsystem import api as subsystem_api from .task import api as task_api -from .._dtos import ClusterDTO +from .._dtos import BackupConfigDTO, ClusterDTO from .. import util as util @@ -36,18 +36,6 @@ class _UpdateParams(BaseModel): restart: bool = Field(False) -class BackupConfigParams(BaseModel): - access_key_id: Optional[SecretStr] = None - secret_access_key: Optional[SecretStr] = None - local_endpoint: Optional[str] = None - bucket_name: Optional[str] = None - snapshot_backups: Optional[bool] = None - with_compression: Optional[bool] = None - secondary_target: Optional[int] = Field(default=None, ge=0) - local_testing: Optional[bool] = None - s3_thread_pool_size: Optional[int] = Field(default=None, ge=0) - - class HashicorpVaultSettings(BaseModel): base_url: Optional[Annotated[AnyUrl, UrlConstraints(allowed_schemes=["https"])]] = None transit_mount: str = "simplyblock/transit" @@ -82,7 +70,7 @@ class ClusterParams(BaseModel): nvmf_base_port: int = 4420 rpc_base_port: int = 8080 snode_api_port: int = 50001 - backup_config: Optional[BackupConfigParams] = None + backup_config: Optional[BackupConfigDTO] = None hashicorp_vault_settings: Optional[HashicorpVaultSettings] = None enable_failure_domain: bool = False @@ -115,6 +103,8 @@ def add(request: Request, parameters: ClusterParams, response_format: util.Creat params = parameters.model_dump(exclude_none=True) if "hashicorp_vault_settings" in params: params["hashicorp_vault_settings"] = ModelVaultSettings(params["hashicorp_vault_settings"]) + if parameters.backup_config is not None: + params["backup_config"] = parameters.backup_config.model_dump(exclude_none=True) cluster_id_or_false = cluster_ops.add_cluster(**params) except ValueError as e: raise HTTPException(status_code=409, detail=str(e)) @@ -156,6 +146,36 @@ def update(cluster: Cluster, parameters: UpdatableClusterParameters): return Response(status_code=204) +@instance_api.get('/backup-config', name='clusters:backup-config:get') +def get_backup_config(cluster: Cluster) -> BackupConfigDTO: + """The cluster's backup configuration, with credentials masked. + + The credentials are ``SecretStr``, which FastAPI's JSON serialization + renders as ``**********``. + """ + try: + return cluster.get_backup_config() + except ValueError as e: + raise HTTPException(404, str(e)) from e + + +@instance_api.put('/backup-config', name='clusters:backup-config:set', + status_code=204, responses={204: {"content": None}}) +def set_backup_config(cluster: Cluster, parameters: BackupConfigDTO) -> Response: + """Replace the cluster's backup configuration. + + Backup configuration used to be settable only at cluster-create time, which + left no way to correct or complete it -- notably no way to record a region + on a cluster created before it was mandatory. + + A full replacement rather than a patch: the fields interact (an endpoint + implies addressing style and TLS expectations), so merging half a config + into an existing one produces combinations nobody chose. + """ + cluster_ops.set_backup_config(cluster.get_id(), parameters) + return Response(status_code=204) + + @instance_api.delete('/', name='clusters:delete', status_code=204, responses={204: {"content": None}}) def delete(cluster: Cluster) -> Response: try: diff --git a/tests/unit/test_api_dto_secrets.py b/tests/unit/test_api_dto_secrets.py index 95e7685f1..47df88432 100644 --- a/tests/unit/test_api_dto_secrets.py +++ b/tests/unit/test_api_dto_secrets.py @@ -3,8 +3,8 @@ from pydantic import SecretStr -from simplyblock_web.api.v2.cluster import BackupConfigParams -from simplyblock_web.api.v2._dtos import ClusterDTO, CapacityStatDTO +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_web.api.v2._dtos import BackupConfigDTO, ClusterDTO, CapacityStatDTO from uuid import uuid4 @@ -13,27 +13,63 @@ def _build_capacity(): return CapacityStatDTO.from_model(StatsObject()) +_BACKUP_CONFIG = { + "bucket_name": "backups", + "region": "eu-central-1", + "access_key_id": "AKID", + "secret_access_key": "SK", +} + + def test_backup_config_params_carry_secretstr(): - params = BackupConfigParams.model_validate({ - "access_key_id": "AKID", - "secret_access_key": "SK", - }) - assert isinstance(params.access_key_id, SecretStr) - assert isinstance(params.secret_access_key, SecretStr) - assert params.access_key_id.get_secret_value() == "AKID" - assert params.secret_access_key.get_secret_value() == "SK" + params = BackupConfigDTO.model_validate(_BACKUP_CONFIG) + assert params.credentials is not None + assert isinstance(params.credentials.access_key_id, SecretStr) + assert isinstance(params.credentials.secret_access_key, SecretStr) + assert params.credentials.access_key_id.get_secret_value() == "AKID" + assert params.credentials.secret_access_key.get_secret_value() == "SK" def test_backup_config_repr_masks_secret_values(): - params = BackupConfigParams.model_validate({ - "access_key_id": "AKID", - "secret_access_key": "SK", - }) - text = repr(params) + text = repr(BackupConfigDTO.model_validate(_BACKUP_CONFIG)) assert "AKID" not in text assert "SK" not in text +def test_backup_config_dump_keeps_secrets_wrapped(): + """write_to_db unwraps at the last moment; anything earlier leaks into logs.""" + stored = BackupConfig.model_validate(_BACKUP_CONFIG).model_dump(exclude_none=True) + assert isinstance(stored["credentials"]["access_key_id"], SecretStr) + assert "AKID" not in repr(stored) + + +def test_backup_config_dump_is_json_serializable(): + """Cluster.backup_config is a plain dict written through BaseModel to FDB. + + A python-mode dump is normally not JSON-safe. Field serializers on the two + offenders -- the URL and the enum -- are what make this hold without a + hand-written conversion step. + """ + from simplyblock_core.models.base_model import BaseModel as CoreBaseModel + + stored = BackupConfig.model_validate({ + **_BACKUP_CONFIG, "local_endpoint": "http://minio:9000", + }).model_dump(exclude_none=True) + + class _Holder(CoreBaseModel): + backup_config: dict = {} + + holder = _Holder() + holder.backup_config = stored + persisted = json.loads(json.dumps(holder.to_dict(unwrap_secrets=True))) + + assert stored["endpoint"] == "http://minio:9000" + assert stored["secondary_target"] == 0 + assert persisted["backup_config"]["credentials"]["access_key_id"] == "AKID" + assert BackupConfig.model_validate(persisted["backup_config"]).endpoint_url == \ + "http://minio:9000" + + def _build_cluster_dto(): return ClusterDTO( id=uuid4(), diff --git a/tests/unit/test_client_secret_logging.py b/tests/unit/test_client_secret_logging.py index 77c5f727f..5b6f077cb 100644 --- a/tests/unit/test_client_secret_logging.py +++ b/tests/unit/test_client_secret_logging.py @@ -80,6 +80,65 @@ def test_rpc_client_response_body_logged_when_flag_on(rpc_client, caplog, monkey assert "RESPVALUE" in _captured_logs_text(caplog) +def _sent_params(client): + return json.loads(client._fake_session.post.call_args.kwargs["data"])["params"] + + +def test_bdev_s3_create_keys_reach_the_wire_but_not_the_log(rpc_client, caplog): + # bdev_s3_create is the only RPC carrying S3 keys, and it goes through + # _request3, which logs its parameter dict directly -- only a SecretStr + # masks there. + rpc_client._fake_session.post.return_value = _make_json_response({ + "jsonrpc": "2.0", "id": 1, "result": True, + }) + + with caplog.at_level(logging.DEBUG): + rpc_client.bdev_s3_create( + name="s3_lvs_test", + access_key_id=SecretStr("AKIAEXAMPLE"), + secret_access_key=SecretStr("s3cr3t"), + ) + + params = _sent_params(rpc_client) + assert params["access_key_id"] == "AKIAEXAMPLE" + assert params["secret_access_key"] == "s3cr3t" + + logged = _captured_logs_text(caplog) + assert "AKIAEXAMPLE" not in logged + assert "s3cr3t" not in logged + assert "**********" in logged + + +def test_bdev_s3_create_omits_absent_credentials(rpc_client): + rpc_client._fake_session.post.return_value = _make_json_response({ + "jsonrpc": "2.0", "id": 1, "result": True, + }) + + rpc_client.bdev_s3_create(name="s3_lvs_test") + + params = _sent_params(rpc_client) + assert "access_key_id" not in params + assert "secret_access_key" not in params + + +def test_bdev_s3_create_does_not_send_empty_credentials_as_keys(rpc_client): + # An empty key pair is not an absent one to the AWS SDK: it reads as a valid + # anonymous identity, and the default provider chain (the node's instance + # role) is then never consulted. + rpc_client._fake_session.post.return_value = _make_json_response({ + "jsonrpc": "2.0", "id": 1, "result": True, + }) + + rpc_client.bdev_s3_create( + name="s3_lvs_test", + access_key_id=SecretStr(""), secret_access_key=SecretStr(""), + ) + + params = _sent_params(rpc_client) + assert "access_key_id" not in params + assert "secret_access_key" not in params + + @pytest.fixture def snode_client(): with patch("simplyblock_core.snode_client.requests.session") as session_factory: From 6da95a746ef91f98acdaaf78d2a7cbc1ed788a48 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 11:28:47 +0200 Subject: [PATCH 03/14] Record each backup's location; allocate s3_id from a sequence Three changes that together make a Backup record say where its own data is. Backup.location A BackupLocation, resolved once per chain by the caller and passed into _create_single_backup rather than read from the cluster per backup, so every backup in one chain is guaranteed to share it. get_location() validates on read and raises ValueError, matching Cluster.get_backup_config -- a stored document failing validation is not a precondition the caller could have checked. Backup.s3_metadata is deleted It was written in two places and read in none: export_backups rebuilt its own dict from the model fields instead. It was a partial duplicate of fields already on the same record, in the same database, so it survived exactly as well as the cluster did -- while the docstring and the (also deleted, never referenced) BACKUP_S3_METADATA_BUCKET constant claimed it went to S3. The real S3 manifest lands in the next commit; leaving this behind would only keep a second, staler copy of the same facts. s3_id allocation _next_s3_id was max-plus-one over the local cluster's Backup records. It raced, so two concurrent backups could get the same id; it recycled the id of a deleted backup whose objects may still exist, since nothing reclaims them (bdev_lvol_s3_delete does not exist on the data plane); and after an import it counted foreign backups it had no business counting. Replaced with a monotonic FDB sequence, reusing the _VUID_SEQ_KEY pattern already in db_controller, which was introduced for this exact class of problem. Unlike vuid the space is bounded: the data plane packs s3_id into 30 bits and masks rather than validates, so BACKUP_MAX_S3_ID is now explicit and exhaustion raises instead of silently aliasing onto another backup's keys. export/import now carry location and encrypted. Import previously dropped `encrypted` entirely, so an imported encrypted backup restored with use_crypto=False -- a plaintext volume over ciphertext, silently. Entries missing either field are rejected in the pre-check loop, so a stale export file fails whole rather than half-importing. That rejection is a ValueError: the shape of a supplied entry is a bad request, where "this backup id already exists" is a precondition -- the v2 import endpoint maps the former to 400 explicitly, since only PreconditionError has a global handler. _auto_backup_lvol resolves node, cluster and location before taking the snapshot. It used to snapshot first and discover afterwards that the cluster was unusable, leaving an orphaned auto_* snapshot behind on every scheduler tick. backup_snapshot's `if not snapshot.lvol` guard is removed: it cannot fire for any snapshot read from the database, because SnapShot.write_to_db builds a SnapShotMini whose from_snapshot calls LVolMini().from_lvol unconditionally (snapshot.py:87), so a snapshot without an lvol cannot be persisted in the first place. Dropping it also collapses a duplicated storage-node lookup, where the first fetch swallowed the KeyError that the second then reported. Tests: TestImportBackups and TestBackupSnapshot are converted off the stubbed-DB pattern onto real FoundationDB, per tests/AGENTS.md. Converting them is what surfaced the dead guard above -- the old test only reached it by mocking the database away. Also, TestCreateS3Bdev asserted only pytest.raises(Exception), which passes on any error including an AttributeError from a wrong argument type; tightened to the specific exceptions, and test_exception_handled now raises RPCException rather than a bare Exception, which the code under test never caught, so that test had been passing for the wrong reason. --- simplyblock_core/constants.py | 7 +- .../controllers/backup_controller.py | 129 +++--- simplyblock_core/db_controller.py | 55 +++ simplyblock_core/models/backup.py | 23 +- simplyblock_web/api/v2/cluster/backup.py | 8 +- tests/integration/test_backup.py | 398 ++++++++++-------- .../test_backup_s3_id_allocation.py | 73 ++++ 7 files changed, 449 insertions(+), 244 deletions(-) create mode 100644 tests/integration/test_backup_s3_id_allocation.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index bda9fba3e..f5c404e09 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -605,6 +605,11 @@ def get_config_var(name, default=None): BACKUP_POLL_INTERVAL_SEC = 5 BACKUP_MAX_RETRIES = 10 BACKUP_MERGE_SERVICE_INTERVAL_SEC = 60 -BACKUP_S3_METADATA_BUCKET = "simplyblock-backup-metadata" + +#: Upper bound on a backup's s3_id. The data plane packs it into bits 33..62 of +#: the synthetic bdev offset (S3_ID_BITS in spdk_internal/lvolstore.h) and masks +#: rather than validates, so a larger value silently aliases onto another +#: backup's object keys. +BACKUP_MAX_S3_ID = (1 << 30) - 1 TASKS_RETENTION_PERIOD_SEC = 60*60*24*30 # 30 days \ No newline at end of file diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index 0bb740faa..1ac7686d3 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -12,7 +12,7 @@ from simplyblock_core.controllers import backup_events, tasks_controller from simplyblock_core.db_controller import DBController from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment -from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.backup_config import BackupConfig, BackupLocation from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.kms import ( KMSException, backup_dek_path, backup_kek_name, create_kms_connection, @@ -31,15 +31,6 @@ def _generate_backup_id(): return str(uuid.uuid4()) -def _next_s3_id(cluster_id): - """Return the next cluster-wide unique s3_id (uint32) for data-plane RPCs.""" - max_id = 0 - for b in db_controller.get_backups(cluster_id): - if b.s3_id > max_id: - max_id = b.s3_id - return max_id + 1 - - def _parse_age_string(age_str): """Parse age strings like '2d', '12h', '1w', '30m' into seconds.""" match = re.match(r'^(\d+)([mhdw])$', age_str.strip()) @@ -77,29 +68,6 @@ def _parse_schedule(schedule_str): return tiers -def _write_s3_metadata(rpc_client, backup): - """Write backup metadata to the S3 metadata bucket. - This metadata is needed for cross-cluster recovery.""" - metadata = { - "backup_id": backup.uuid, - "lvol_id": backup.lvol_id, - "lvol_name": backup.lvol_name, - "snapshot_id": backup.snapshot_id, - "snapshot_name": backup.snapshot_name, - "node_id": backup.node_id, - "cluster_id": backup.cluster_id, - "prev_backup_id": backup.prev_backup_id, - "created_at": backup.created_at, - "size": backup.size, - "allowed_hosts": backup.allowed_hosts, - } - backup.s3_metadata = metadata - # The actual S3 metadata write is done via the data plane's S3 bdev. - # For now we store it in the backup object itself. - # In production, this would write to the metadata bucket via S3 API. - return metadata - - def _get_latest_backup_for_lvol(lvol_id): """Get the most recent non-failed backup for a given lvol. @@ -281,16 +249,25 @@ def _snapshot_has_backup(snapshot_id): Backup.STATUS_COMPLETED, Backup.STATUS_MERGED) for b in backups) -def _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup): +def _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location: BackupLocation): """Create a single backup record and task for one snapshot. - Returns the created Backup object.""" + + Args: + location: where this backup's objects will be written. Passed in rather + than read from the cluster here so that every backup in one chain is + guaranteed to share it -- the caller resolves it once and validates + the chain against it. + + Returns the created Backup object. + """ backup_id = _generate_backup_id() backup = Backup() backup.uuid = backup_id - backup.s3_id = _next_s3_id(cluster_id) + backup.s3_id = db_controller.next_s3_id() backup.cluster_id = cluster_id - backup.source_cluster_id = cluster_id # local backup + backup.source_cluster_id = cluster_id # provenance only + backup.location = location.model_dump(mode="json") backup.lvol_id = lvol.get_id() backup.lvol_name = lvol.lvol_name backup.snapshot_id = snapshot.get_id() @@ -317,9 +294,6 @@ def _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup): backup.write_to_db() - _write_s3_metadata(None, backup) - backup.write_to_db() - backup_events.backup_created(cluster_id, node_id, backup) tasks_controller.add_backup_task(backup) @@ -341,21 +315,6 @@ def backup_snapshot(snapshot_id, cluster_id=None): except KeyError as e: return None, str(e) - # Block new backups when S3 source is switched to an external cluster - node_id = snapshot.lvol.node_id if snapshot.lvol else None - if node_id: - try: - snode = db_controller.get_storage_node_by_id(node_id) - if not is_local_backup_source(snode.cluster_id): - return None, ("Cannot create backups while backup source is " - "switched to an external cluster. Switch back " - "to local first.") - except KeyError: - pass - - if not snapshot.lvol: - return None, "Snapshot has no associated lvol" - lvol = snapshot.lvol node_id = lvol.node_id try: @@ -363,12 +322,23 @@ def backup_snapshot(snapshot_id, cluster_id=None): except KeyError as e: return None, str(e) + # Block new backups when S3 source is switched to an external cluster + if not is_local_backup_source(snode.cluster_id): + return None, ("Cannot create backups while backup source is " + "switched to an external cluster. Switch back " + "to local first.") + if snode.status != StorageNode.STATUS_ONLINE: return None, f"Node {node_id} is not online (status: {snode.status})" if not cluster_id: cluster_id = snode.cluster_id + try: + location = db_controller.get_cluster_by_id(cluster_id).get_backup_config().location() + except (KeyError, ValueError) as e: + return None, str(e) + snap_chain = _get_snapshot_chain(snapshot) chain_snapshot_ids = [snap.get_id() for snap in snap_chain] acquired, existing_lock = db_controller.acquire_backup_chain_locks( @@ -397,7 +367,7 @@ def backup_snapshot(snapshot_id, cluster_id=None): prev_backup = existing continue - backup = _create_single_backup(snap, lvol, node_id, cluster_id, prev_backup) + backup = _create_single_backup(snap, lvol, node_id, cluster_id, prev_backup, location) time.sleep(1) prev_backup = backup if snap.get_id() == snapshot_id: @@ -644,6 +614,8 @@ def export_backups(cluster_id=None, lvol_name=None): "prev_backup_id": b.prev_backup_id, "size": b.size, "allowed_hosts": b.allowed_hosts, + "location": b.location, + "encrypted": b.encrypted, "created_at": b.created_at, }) return result @@ -662,6 +634,7 @@ def import_backups(s3_metadata_list, cluster_id=None): restore so the backups are visible in the local cluster's DB. Raises: + ValueError: One of the given entries is not a usable backup description. PreconditionError: One of the given backup IDs is already known. Backup lookups are not scoped by cluster, so a UUID reused across clusters would make either record unaddressable. All IDs are checked before @@ -674,7 +647,20 @@ def import_backups(s3_metadata_list, cluster_id=None): continue if backup_id in pending: - raise PreconditionError(f"Backup {backup_id} is listed more than once") + raise ValueError(f"Backup {backup_id} is listed more than once") + + # An entry that cannot say where its objects are, or whether they are + # encrypted, is not importable at any price: the first produces a + # restore against whatever bucket happens to be configured, the second a + # plaintext volume over ciphertext. Checked here so a stale export file + # is rejected whole rather than half-imported. + for required in ("location", "encrypted"): + if required not in meta: + raise ValueError( + f"Backup {backup_id} is missing '{required}'; it predates " + "self-describing backups and cannot be imported") + + BackupLocation.model_validate(meta["location"]) try: existing = db_controller.get_backup_by_id(backup_id) @@ -702,7 +688,10 @@ def import_backups(s3_metadata_list, cluster_id=None): backup.allowed_hosts = meta.get("allowed_hosts", []) backup.created_at = meta.get("created_at", 0) backup.status = Backup.STATUS_COMPLETED - backup.s3_metadata = meta + backup.location = meta["location"] + # Import used to drop this, so an imported encrypted backup restored as + # use_crypto=False -- a plaintext volume over ciphertext, silently. + backup.encrypted = meta["encrypted"] backup.write_to_db() return len(pending) @@ -1046,6 +1035,20 @@ def _auto_backup_lvol(lvol): incremental chain is maintained without re-backing all ancestors. """ from simplyblock_core.controllers import snapshot_controller + + # Resolve everything the backup needs BEFORE taking the snapshot. This used + # to create the snapshot first and discover afterwards that the node or + # cluster was unusable, leaving an orphaned auto_* snapshot behind on every + # scheduler tick. + node_id = lvol.node_id + try: + snode = db_controller.get_storage_node_by_id(node_id) + cluster_id = snode.cluster_id + location = db_controller.get_cluster_by_id(cluster_id).get_backup_config().location() + except (KeyError, ValueError) as e: + logger.warning(f"Auto-backup skipped for lvol {lvol.get_id()}: {e}") + return + snap_name = f"auto_{lvol.lvol_name}_{int(time.time())}" snap_id, error = snapshot_controller.add(lvol.get_id(), snap_name) if error: @@ -1058,16 +1061,8 @@ def _auto_backup_lvol(lvol): logger.warning(f"Auto-backup: snapshot {snap_id} not found after creation") return - node_id = lvol.node_id - try: - snode = db_controller.get_storage_node_by_id(node_id) - except KeyError: - logger.warning(f"Auto-backup: node {node_id} not found") - return - - cluster_id = snode.cluster_id prev_backup = _get_latest_backup_for_lvol(lvol.get_id()) - _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup) + _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location) def _trigger_merge(keep_backup, old_backup): diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index ea475de42..bddce38c1 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -1045,6 +1045,61 @@ def next_vuid(self) -> int: fdb.transactional(DBController._seed_vuid_tx)(self, self.kv_store, seed) return fdb.transactional(DBController._incr_vuid_tx)(self, self.kv_store) + # ---- s3_id allocation (monotonic sequence) ---- + # + # An s3_id names a backup's object keys in S3 ({s3_id}/{mid}/{extent}). The + # old allocator was max-plus-one over the local cluster's Backup records, + # which had three problems: it raced (two concurrent backups got the same + # id), it recycled the id of a deleted backup whose objects may still exist + # (nothing reclaims them -- bdev_lvol_s3_delete does not exist on the data + # plane), and after an import it counted foreign backups it should not have. + # A monotonic sequence makes reuse impossible by construction. + # + # Unlike vuid this space is NOT unbounded: the data plane packs s3_id into + # 30 bits and masks rather than validates, so callers must check + # BACKUP_MAX_S3_ID. 2^30 is ~1.07e9 backups per control plane. + _S3_ID_SEQ_KEY = b"sequence/s3_id" + + def _incr_s3_id_tx(self, tr): + raw = tr.get(DBController._S3_ID_SEQ_KEY).wait() + if not raw.present(): + return None + nxt = int(json.loads(raw)) + 1 + tr[DBController._S3_ID_SEQ_KEY] = json.dumps(nxt).encode() + return nxt + + def _seed_s3_id_tx(self, tr, seed): + # Only-if-absent CAS, as for vuid: the first allocator across all API + # workers seeds it; concurrent racers see it present and skip. + raw = tr.get(DBController._S3_ID_SEQ_KEY).wait() + if raw.present(): + return + tr[DBController._S3_ID_SEQ_KEY] = json.dumps(int(seed)).encode() + + def _max_existing_s3_id(self) -> int: + """Highest s3_id in use across every backup this control plane knows of. + + Read once to seed the counter on an upgraded cluster so the sequence + never reuses an id the old max-plus-one allocator handed out; never read + again. Deliberately unscoped by cluster -- imported backups keep their + originating cluster's ids, and seeding above those too costs nothing. + """ + return max((b.s3_id or 0 for b in self.get_backups()), default=0) + + def next_s3_id(self) -> int: + """Allocate the next globally-unique s3_id (monotonic, O(1)).""" + val = fdb.transactional(DBController._incr_s3_id_tx)(self, self.kv_store) + if val is None: + seed = self._max_existing_s3_id() + fdb.transactional(DBController._seed_s3_id_tx)(self, self.kv_store, seed) + val = fdb.transactional(DBController._incr_s3_id_tx)(self, self.kv_store) + + if val > constants.BACKUP_MAX_S3_ID: + raise ValueError( + f"s3_id space exhausted: {val} exceeds the data plane's " + f"{constants.BACKUP_MAX_S3_ID} limit") + return val + # ---- snapshot indexes (replace per-create cluster-wide scans) ---- # # Snapshot create used to read EVERY snapshot in the cluster on each request diff --git a/simplyblock_core/models/backup.py b/simplyblock_core/models/backup.py index fccee7da8..6cbdc49e4 100644 --- a/simplyblock_core/models/backup.py +++ b/simplyblock_core/models/backup.py @@ -2,6 +2,7 @@ import datetime from typing import List +from simplyblock_core.models.backup_config import BackupLocation from simplyblock_core.models.base_model import BaseModel @@ -41,13 +42,31 @@ class Backup(BaseModel): error_message: str = "" # Security params from the source lvol (for cross-cluster restore) allowed_hosts: List[dict] = [] - # S3 metadata written to metadata bucket - s3_metadata: dict = {} + #: Where this backup's objects live and how to interpret them, as a + #: ``BackupLocation``. Stored as a dict because ``BaseModel`` cannot nest + #: pydantic models; read it through :meth:`get_location`. + location: dict = {} encrypted: bool = False def get_id(self): return "%s/%s" % (self.cluster_id, self.uuid) + def get_location(self) -> BackupLocation: + """Validate and return where this backup's objects live. + + Raises: + ValueError: The backup predates self-describing locations, or its + recorded location is not valid. Either way it cannot be read + without knowing what wrote it. ``ValidationError`` is a + ``ValueError``, so one except clause covers both. + """ + if not self.location: + raise ValueError( + f"Backup {self.uuid} has no recorded location " + "(created before backups became self-describing)") + + return BackupLocation.model_validate(self.location) + def write_to_db(self, kv_store=None): self.updated_at = str(datetime.datetime.now(datetime.timezone.utc)) super().write_to_db(kv_store) diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index be8147246..56e3b1168 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -65,7 +65,13 @@ class _ImportParams(BaseModel): @api.post('/import', name='clusters:backups:import') def import_backups(cluster: Cluster, parameters: _ImportParams): - count = backup_controller.import_backups(parameters.metadata, cluster_id=cluster.get_id()) + try: + count = backup_controller.import_backups(parameters.metadata, cluster_id=cluster.get_id()) + except ValueError as e: + # The request body could not be read as backup descriptions, which is a + # bad request rather than an unmet precondition (those reach 400 through + # app.py's PreconditionError handler). + raise HTTPException(400, str(e)) from e return {"imported": count} diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index cb733e230..7f3be350a 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -24,6 +24,7 @@ import pytest +from simplyblock_core.controllers.backup_controller import backup_snapshot, import_backups from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment @@ -54,6 +55,22 @@ def _node(uuid="node-1", status=StorageNode.STATUS_ONLINE, lvstore="lvs_test", return n +def _backup_config(**overrides): + from simplyblock_core.models.backup_config import BackupConfig + return BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", + "region": "eu-central-1", + **overrides, + }) + + +def _cluster(uuid="cluster-1", **config_overrides): + c = Cluster() + c.uuid = uuid + c.backup_config = _backup_config(**config_overrides).model_dump(exclude_none=True) + return c + + def _backup(uuid="backup-1", lvol_id="lvol-1", status=Backup.STATUS_COMPLETED, node_id="node-1", cluster_id="cluster-1", prev_backup_id="", created_at=None, snapshot_id="snap-1", s3_id=1): @@ -74,6 +91,18 @@ def _backup(uuid="backup-1", lvol_id="lvol-1", status=Backup.STATUS_COMPLETED, return b +def _meta(backup_id, cluster_id="cluster-1", **overrides): + """One entry of an export/import payload.""" + return { + "backup_id": backup_id, + "lvol_id": "l-1", + "cluster_id": cluster_id, + "location": _backup_config().location().model_dump(mode="json"), + "encrypted": False, + **overrides, + } + + def _snapshot(uuid="snap-1", lvol_uuid="lvol-1", node_id="node-1"): s = SnapShot() s.uuid = uuid @@ -119,7 +148,7 @@ def test_default_fields(self): self.assertEqual(b.size, 0) self.assertEqual(b.created_at, 0) self.assertEqual(b.completed_at, 0) - self.assertEqual(b.s3_metadata, {}) + self.assertEqual(b.location, {}) self.assertEqual(b.error_message, "") def test_status_constants(self): @@ -291,9 +320,7 @@ def test_success(self, MockRPC, mock_boto3_client): from simplyblock_core.controllers.backup_controller import create_s3_bdev node = _node() - config = {"secondary_target": 0, "with_compression": False, - "snapshot_backups": True} - create_s3_bdev(node, config) + create_s3_bdev(node, _backup_config()) mock_rpc.bdev_s3_create.assert_called_once() # Verify CPU masks: bdb_lcpu_mask=app_thread(0x8=8), s3_lcpu_mask=all 8 vCPUs(0xFF=255) @@ -309,8 +336,8 @@ def test_success(self, MockRPC, mock_boto3_client): def test_no_lvstore(self, MockRPC, _mock_boto3_client): from simplyblock_core.controllers.backup_controller import create_s3_bdev node = _node(lvstore="") - with pytest.raises(Exception): - create_s3_bdev(node, {}) + with pytest.raises(PreconditionError): + create_s3_bdev(node, _backup_config()) MockRPC.assert_not_called() @@ -321,8 +348,8 @@ def test_bdev_s3_create_fails(self, MockRPC): from simplyblock_core.controllers.backup_controller import create_s3_bdev node = _node() - with pytest.raises(Exception): - create_s3_bdev(node, {}) + with pytest.raises(RuntimeError): + create_s3_bdev(node, _backup_config()) mock_rpc.bdev_s3_add_bucket_name.assert_not_called() mock_rpc.bdev_lvol_s3_bdev.assert_not_called() @@ -338,8 +365,8 @@ def test_bucket_name_fails(self, MockRPC, mock_boto3_client): from simplyblock_core.controllers.backup_controller import create_s3_bdev node = _node() - with pytest.raises(Exception): - create_s3_bdev(node, {}) + with pytest.raises(RuntimeError): + create_s3_bdev(node, _backup_config()) mock_rpc.bdev_lvol_s3_bdev.assert_not_called() @patch("simplyblock_core.controllers.backup_controller.boto3.client") @@ -355,8 +382,8 @@ def test_attach_fails(self, MockRPC, mock_boto3_client): from simplyblock_core.controllers.backup_controller import create_s3_bdev node = _node() - with pytest.raises(Exception): - create_s3_bdev(node, {}) + with pytest.raises(RuntimeError): + create_s3_bdev(node, _backup_config()) @patch("simplyblock_core.controllers.backup_controller.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") @@ -368,38 +395,63 @@ def test_local_testing_params(self, MockRPC, mock_boto3_client): mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} + from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.controllers.backup_controller import create_s3_bdev node = _node() - config = { - "secondary_target": 0, + # A genuine pre-BackupConfig dict: no region, local_testing standing in + # for four separate decisions. + create_s3_bdev(node, BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", "local_testing": True, "local_endpoint": "http://minio:9000", "access_key_id": "minioadmin", "secret_access_key": "minioadmin", - } - create_s3_bdev(node, config) + })) + # The data plane still takes the legacy shape; `local_testing` there is + # not a mode but the only condition under which it honours an endpoint + # override at all, so it tracks "an endpoint was configured". _, kwargs = mock_rpc.bdev_s3_create.call_args - self.assertTrue(kwargs.get("local_testing", False)) - self.assertEqual(kwargs.get("local_endpoint", ""), "http://minio:9000") - self.assertEqual(kwargs.get("access_key_id", ""), "minioadmin") - self.assertEqual(kwargs.get("secret_access_key", ""), "minioadmin") - mock_boto3_client.assert_called_once_with( - "s3", - aws_access_key_id="minioadmin", - aws_secret_access_key="minioadmin", - endpoint_url="http://minio:9000", - ) + self.assertTrue(kwargs["local_testing"]) + self.assertEqual(kwargs["local_endpoint"], "http://minio:9000") + self.assertEqual(kwargs["access_key_id"].get_secret_value(), "minioadmin") + self.assertEqual(kwargs["secret_access_key"].get_secret_value(), "minioadmin") + + _, boto_kwargs = mock_boto3_client.call_args + self.assertEqual(boto_kwargs["aws_access_key_id"], "minioadmin") + self.assertEqual(boto_kwargs["aws_secret_access_key"], "minioadmin") + self.assertEqual(boto_kwargs["endpoint_url"], "http://minio:9000") + # local_testing unpacked into the properties it actually stood for. + self.assertEqual(boto_kwargs["region_name"], "us-east-1") + self.assertFalse(boto_kwargs["verify"]) + + @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.models.storage_node.RPCClient") + def test_no_credentials_defers_to_the_provider_chain(self, MockRPC, mock_boto3_client): + """An absent key pair must mean "use the node's IAM role", not "send empty keys".""" + mock_rpc = MockRPC.return_value + mock_rpc.bdev_s3_create.return_value = True + mock_rpc.bdev_s3_add_bucket_name.return_value = (True, None) + mock_rpc.bdev_lvol_s3_bdev.return_value = True + mock_boto3_client.return_value.head_bucket.return_value = {} + + from simplyblock_core.controllers.backup_controller import create_s3_bdev + create_s3_bdev(_node(), _backup_config()) + + _, boto_kwargs = mock_boto3_client.call_args + self.assertIsNone(boto_kwargs["aws_access_key_id"]) + self.assertIsNone(boto_kwargs["aws_secret_access_key"]) @patch("simplyblock_core.models.storage_node.RPCClient") def test_exception_handled(self, MockRPC): + from simplyblock_core.rpc_client import RPCException mock_rpc = MockRPC.return_value - mock_rpc.bdev_s3_create.side_effect = Exception("connection refused") + mock_rpc.bdev_s3_create.side_effect = RPCException("connection refused") from simplyblock_core.controllers.backup_controller import create_s3_bdev node = _node() - with pytest.raises(Exception): - create_s3_bdev(node, {}) + with pytest.raises(RuntimeError): + create_s3_bdev(node, _backup_config()) # =========================================================================== @@ -407,143 +459,121 @@ def test_exception_handled(self, MockRPC): # =========================================================================== class TestBackupSnapshot(unittest.TestCase): + """Real FDB: backup_snapshot reads cluster/node/snapshot state and writes Backups. + + Only what sits above the database is mocked -- the storage node's RPC client, + the task runner and event emission. + """ + + def setUp(self): + self.db = DBController() + _cluster().write_to_db(self.db.kv_store) + _node().write_to_db(self.db.kv_store) + + def _persist(self, snapshot): + snapshot.lvol.write_to_db(self.db.kv_store) + snapshot.write_to_db(self.db.kv_store) + return snapshot - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup_controller.tasks_controller") @patch("simplyblock_core.controllers.backup_controller.backup_events") - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_success(self, mock_db, _mock_local_source, mock_events, mock_tasks, _mock_write): - snap = _snapshot() - mock_db.get_snapshot_by_id.return_value = snap - mock_db.get_storage_node_by_id.return_value = _node() - mock_db.get_backups_by_lvol_id.return_value = [] - mock_db.get_backups.return_value = [] - mock_db.get_backups_by_snapshot_id.return_value = [] - mock_db.acquire_backup_chain_locks.return_value = (True, None) - mock_tasks.add_backup_task.return_value = "task-1" + def test_success(self, mock_events, mock_tasks): + snap = self._persist(_snapshot()) - from simplyblock_core.controllers.backup_controller import backup_snapshot - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", return_value=[snap]): + with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", + return_value=[snap]): backup_id, error = backup_snapshot("snap-1") - self.assertIsNotNone(backup_id) self.assertIsNone(error) + self.assertIsNotNone(backup_id) mock_tasks.add_backup_task.assert_called_once() - mock_events.backup_created.assert_called_once() - # Verify s3_id is assigned - created_backup = mock_events.backup_created.call_args[0][2] - self.assertEqual(created_backup.s3_id, 1) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_snapshot_not_found(self, mock_db): - mock_db.get_snapshot_by_id.side_effect = KeyError("not found") + stored = self.db.get_backup_by_id(backup_id) + self.assertEqual(stored.status, Backup.STATUS_PENDING) + self.assertGreater(stored.s3_id, 0) + # Self-describing from the moment it is created. + self.assertEqual(stored.get_location().bucket_name, + "simplyblock-backup-cluster-1") + + @patch("simplyblock_core.controllers.backup_controller.tasks_controller") + @patch("simplyblock_core.controllers.backup_controller.backup_events") + def test_incremental_backup(self, mock_events, mock_tasks): + snap = self._persist(_snapshot()) + prev = _backup(uuid="prev-backup", s3_id=3, snapshot_id="snap-0", + status=Backup.STATUS_COMPLETED) + prev.write_to_db(self.db.kv_store) + + with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", + return_value=[snap]): + backup_id, error = backup_snapshot("snap-1") + + self.assertIsNone(error) + stored = self.db.get_backup_by_id(backup_id) + self.assertEqual(stored.prev_backup_id, "prev-backup") + # Monotonic allocation, so strictly above the existing backup's id. + self.assertGreater(stored.s3_id, 3) - from simplyblock_core.controllers.backup_controller import backup_snapshot + def test_snapshot_not_found(self): backup_id, error = backup_snapshot("missing") self.assertIsNone(backup_id) self.assertIn("not found", error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_no_lvol(self, mock_db): - snap = _snapshot() - snap.lvol = None - mock_db.get_snapshot_by_id.return_value = snap + def test_node_not_online(self): + _node(status=StorageNode.STATUS_OFFLINE).write_to_db(self.db.kv_store) + self._persist(_snapshot()) - from simplyblock_core.controllers.backup_controller import backup_snapshot backup_id, error = backup_snapshot("snap-1") self.assertIsNone(backup_id) - self.assertIn("no associated lvol", error.lower()) + self.assertIn("not online", error) - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_node_not_online(self, mock_db, _mock_local_source): - snap = _snapshot() - node = _node(status=StorageNode.STATUS_OFFLINE) - mock_db.get_snapshot_by_id.return_value = snap - mock_db.get_storage_node_by_id.return_value = node + def test_cluster_without_backup_config_is_refused(self): + """Refused before the chain lock, the KMS work or any task is created.""" + cluster = _cluster() + cluster.backup_config = {} + cluster.write_to_db(self.db.kv_store) + self._persist(_snapshot()) - from simplyblock_core.controllers.backup_controller import backup_snapshot backup_id, error = backup_snapshot("snap-1") self.assertIsNone(backup_id) - self.assertIn("not online", error) + self.assertIn("backup configuration", error) + self.assertEqual(self.db.get_backups(), []) - @patch.object(Backup, 'write_to_db') @patch("simplyblock_core.controllers.backup_controller.tasks_controller") @patch("simplyblock_core.controllers.backup_controller.backup_events") - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_incremental_backup(self, mock_db, _mock_local_source, mock_events, mock_tasks, _mock_write): - snap = _snapshot() - prev = _backup(uuid="prev-backup", s3_id=3) - mock_db.get_snapshot_by_id.return_value = snap - mock_db.get_storage_node_by_id.return_value = _node() - mock_db.get_backups_by_lvol_id.return_value = [prev] - mock_db.get_backups.return_value = [prev] - mock_db.acquire_backup_chain_locks.return_value = (True, None) - mock_tasks.add_backup_task.return_value = "task-1" - - from simplyblock_core.controllers.backup_controller import backup_snapshot - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", return_value=[snap]): - backup_id, error = backup_snapshot("snap-1") - - self.assertIsNotNone(backup_id) - self.assertIsNone(error) - # Verify the backup object passed to events has prev_backup_id set - created_backup = mock_events.backup_created.call_args[0][2] - self.assertEqual(created_backup.prev_backup_id, "prev-backup") - self.assertEqual(created_backup.s3_id, 4) - - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.backup_events") - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_chain_backup_acquires_and_releases_lock(self, mock_db, _mock_local_source, mock_events, mock_tasks, _mock_write): - snap1 = _snapshot(uuid="snap-1") + def test_chain_backup_acquires_and_releases_lock(self, mock_events, mock_tasks): + snap1 = self._persist(_snapshot(uuid="snap-1")) snap1.created_at = 1 - snap2 = _snapshot(uuid="snap-2") + snap2 = self._persist(_snapshot(uuid="snap-2")) snap2.created_at = 2 - snap2.lvol.uuid = "lvol-1" - mock_db.get_snapshot_by_id.return_value = snap2 - mock_db.get_storage_node_by_id.return_value = _node() - mock_db.get_backups_by_lvol_id.return_value = [] - mock_db.get_backups.return_value = [] - mock_db.get_backups_by_snapshot_id.return_value = [] - mock_db.acquire_backup_chain_locks.return_value = (True, None) - mock_tasks.add_backup_task.return_value = "task-1" - from simplyblock_core.controllers.backup_controller import backup_snapshot - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", return_value=[snap1, snap2]): + with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", + return_value=[snap1, snap2]): backup_id, error = backup_snapshot("snap-2") - self.assertIsNotNone(backup_id) self.assertIsNone(error) - mock_db.acquire_backup_chain_locks.assert_called_once_with(["snap-1", "snap-2"], "snap-2", "lvol-1") - mock_db.release_backup_chain_locks.assert_called_once_with(["snap-1", "snap-2"]) + self.assertIsNotNone(backup_id) self.assertEqual(mock_tasks.add_backup_task.call_count, 2) - self.assertEqual(mock_events.backup_created.call_count, 2) + # Locks released, so a second request for the same chain can proceed. + self.assertIsNone(self.db.get_backup_chain_lock("snap-1")) + self.assertIsNone(self.db.get_backup_chain_lock("snap-2")) - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_chain_backup_lock_conflict(self, mock_db, _mock_local_source): - snap = _snapshot(uuid="snap-4") - snap.created_at = 4 - existing_lock = MagicMock(requested_snapshot_id="snap-2", snapshot_id="snap-2") - mock_db.get_snapshot_by_id.return_value = snap - mock_db.get_storage_node_by_id.return_value = _node() - mock_db.acquire_backup_chain_locks.return_value = (False, existing_lock) + def test_chain_backup_lock_conflict(self): + snap = self._persist(_snapshot(uuid="snap-4")) + acquired, _ = self.db.acquire_backup_chain_locks(["snap-4"], "snap-2", "lvol-1") + self.assertTrue(acquired) - from simplyblock_core.controllers.backup_controller import backup_snapshot - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", return_value=[snap]): + with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", + return_value=[snap]): backup_id, error = backup_snapshot("snap-4") self.assertIsNone(backup_id) self.assertIn("already preparing this snapshot chain", error) - mock_db.release_backup_chain_locks.assert_not_called() + # The conflicting holder's lock must survive. + self.assertIsNotNone(self.db.get_backup_chain_lock("snap-4")) + # =========================================================================== @@ -987,57 +1017,71 @@ def test_fewer_than_two_backups(self, mock_db, mock_tasks): # =========================================================================== class TestImportBackups(unittest.TestCase): + """Real FDB: import writes Backup records that later reads must see.""" - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_import_new(self, mock_db, _mock_write): - mock_db.get_backup_by_id.side_effect = KeyError("not found") - mock_db.get_backups.return_value = [] + def setUp(self): + self.db = DBController() - from simplyblock_core.controllers.backup_controller import import_backups + def test_import_new(self): count = import_backups([ - {"backup_id": "b-1", "lvol_id": "l-1", "cluster_id": "c-1"}, - {"backup_id": "b-2", "lvol_id": "l-1", "cluster_id": "c-1"}, - ]) + _meta("b-1", cluster_id="c-1"), + _meta("b-2", cluster_id="c-1"), + ], cluster_id="cluster-1") self.assertEqual(count, 2) + self.assertEqual( + {b.uuid for b in self.db.get_backups("cluster-1")}, {"b-1", "b-2"}) - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_existing_fails(self, mock_db, mock_write): - existing = _backup(uuid="b-1") - mock_db.get_backup_by_id.side_effect = lambda bid: existing if bid == "b-1" else (_ for _ in ()).throw(KeyError()) - mock_db.get_backups.return_value = [existing] + def test_existing_fails(self): + _backup(uuid="b-1").write_to_db(self.db.kv_store) - from simplyblock_core.controllers.backup_controller import import_backups with self.assertRaises(PreconditionError): - import_backups([ - {"backup_id": "b-2", "lvol_id": "l-1", "cluster_id": "cluster-1"}, - {"backup_id": "b-1", "lvol_id": "l-1", "cluster_id": "cluster-1"}, - ]) + import_backups([_meta("b-2"), _meta("b-1")], cluster_id="cluster-1") - mock_write.assert_not_called() + # Nothing imported: the pre-check runs before the first write. + self.assertEqual({b.uuid for b in self.db.get_backups()}, {"b-1"}) - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_duplicate_in_metadata_fails(self, mock_db, mock_write): - mock_db.get_backup_by_id.side_effect = KeyError("not found") - mock_db.get_backups.return_value = [] + def test_duplicate_in_metadata_fails(self): + with self.assertRaises(ValueError): + import_backups([_meta("b-1"), _meta("b-1")], cluster_id="cluster-1") - from simplyblock_core.controllers.backup_controller import import_backups - with self.assertRaises(PreconditionError): - import_backups([ - {"backup_id": "b-1", "lvol_id": "l-1", "cluster_id": "cluster-1"}, - {"backup_id": "b-1", "lvol_id": "l-1", "cluster_id": "cluster-1"}, - ]) + self.assertEqual(self.db.get_backups(), []) - mock_write.assert_not_called() + def test_skip_no_backup_id(self): + self.assertEqual(import_backups([{"lvol_id": "l-1"}]), 0) + + def test_encrypted_flag_survives_import(self): + """Import used to drop this, restoring a plaintext volume over ciphertext.""" + import_backups([_meta("b-1", encrypted=True)], cluster_id="cluster-1") + + self.assertTrue(self.db.get_backup_by_id("b-1").encrypted) + + def test_location_survives_import(self): + import_backups([_meta("b-1")], cluster_id="cluster-1") + + self.assertEqual( + self.db.get_backup_by_id("b-1").get_location().bucket_name, + "simplyblock-backup-cluster-1") + + def test_entry_without_a_location_is_rejected(self): + """A pre-self-describing export would otherwise restore against whatever + bucket the importing cluster happens to have configured.""" + stale = _meta("b-1") + del stale["location"] + + with self.assertRaises(ValueError): + import_backups([stale], cluster_id="cluster-1") + + self.assertEqual(self.db.get_backups(), []) + + def test_invalid_location_rejects_the_whole_batch(self): + with self.assertRaises(ValueError): + import_backups( + [_meta("b-1"), _meta("b-2", location={"bucket_name": "b"})], + cluster_id="cluster-1") + + self.assertEqual(self.db.get_backups(), []) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_skip_no_backup_id(self, mock_db): - from simplyblock_core.controllers.backup_controller import import_backups - count = import_backups([{"lvol_id": "l-1"}]) - self.assertEqual(count, 0) # =========================================================================== @@ -1198,20 +1242,28 @@ def test_backup_handlers_exist(self): # =========================================================================== -# 20. _write_s3_metadata +# 20. Backup.get_location # =========================================================================== -class TestWriteS3Metadata(unittest.TestCase): +class TestBackupLocationAccessor(unittest.TestCase): - def test_metadata_stored_on_backup(self): - from simplyblock_core.controllers.backup_controller import _write_s3_metadata + def test_recorded_location_round_trips(self): b = _backup() - meta = _write_s3_metadata(None, b) + b.location = {"bucket_name": "backups", "region": "eu-central-1"} - self.assertEqual(meta["backup_id"], "backup-1") - self.assertEqual(meta["lvol_id"], "lvol-1") - self.assertEqual(meta["snapshot_id"], "snap-1") - self.assertEqual(b.s3_metadata, meta) + location = b.get_location() + self.assertEqual(location.bucket_name, "backups") + self.assertEqual(location.region, "eu-central-1") + + def test_backup_without_a_location_raises(self): + with self.assertRaises(ValueError): + _backup().get_location() + + def test_invalid_location_raises_value_error(self): + b = _backup() + b.location = {"bucket_name": "backups"} # no region + with self.assertRaises(ValueError): + b.get_location() # =========================================================================== diff --git a/tests/integration/test_backup_s3_id_allocation.py b/tests/integration/test_backup_s3_id_allocation.py new file mode 100644 index 000000000..9cfe5fabd --- /dev/null +++ b/tests/integration/test_backup_s3_id_allocation.py @@ -0,0 +1,73 @@ +"""s3_id allocation against real FoundationDB. + +An s3_id names a backup's object keys in S3 (``{s3_id}/{mid}/{extent}``), and +nothing on the data plane reclaims those objects. Reusing an id therefore aims a +new backup's writes at another backup's keys, so the properties worth pinning are +monotonicity and non-reuse -- not just "returns a number". +""" +import json + +import pytest + +from simplyblock_core import constants +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup + + +@pytest.fixture +def db(): + return DBController() + + +def _backup(uuid, cluster_id, s3_id): + backup = Backup() + backup.uuid = uuid + backup.cluster_id = cluster_id + backup.s3_id = s3_id + backup.status = Backup.STATUS_COMPLETED + return backup + + +class TestS3IdAllocation: + + def test_allocations_are_strictly_increasing(self, db): + allocated = [db.next_s3_id() for _ in range(5)] + assert allocated == sorted(allocated) + assert len(set(allocated)) == len(allocated) + + def test_first_allocation_is_usable(self, db): + """0 is rejected by the data plane (s3_id == 0 -> -EINVAL).""" + assert db.next_s3_id() > 0 + + def test_seeds_above_pre_existing_backups(self, db): + """Upgrade path: ids handed out by the old max-plus-one allocator must not repeat.""" + _backup("b-legacy", "cl-1", 42).write_to_db(db.kv_store) + + assert db.next_s3_id() > 42 + + def test_seeds_above_imported_foreign_backups(self, db): + """Imported backups keep their originating cluster's ids; seeding ignores cluster scope.""" + _backup("b-foreign", "cl-other", 900).write_to_db(db.kv_store) + + assert db.next_s3_id() > 900 + + def test_deleting_a_backup_does_not_recycle_its_id(self, db): + """The old allocator recycled the top id, aiming new writes at orphaned objects.""" + first = db.next_s3_id() + backup = _backup("b-1", "cl-1", first) + backup.write_to_db(db.kv_store) + backup.remove(db.kv_store) + + assert db.next_s3_id() > first + + def test_exhaustion_is_reported_not_wrapped(self, db): + """The data plane masks s3_id to 30 bits, so an overflow would silently alias.""" + db.kv_store[DBController._S3_ID_SEQ_KEY] = json.dumps( + constants.BACKUP_MAX_S3_ID).encode() + + with pytest.raises(ValueError, match="exhausted"): + db.next_s3_id() + + def test_max_s3_id_matches_the_data_plane_field_width(self): + """S3_ID_BITS is 30 in spdk_internal/lvolstore.h; a wider value aliases.""" + assert constants.BACKUP_MAX_S3_ID == (1 << 30) - 1 From e77acce0f799e8998c035c0c4e3a09e0e98abc80 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 11:28:47 +0200 Subject: [PATCH 04/14] Write a manifest into the bucket; discover backups from it alone The data plane writes only opaque objects keyed {s3_id}/{mid}/{extent}. Nothing in them records which volume they came from, how they are encoded, or which other backups they depend on -- all of which lived exclusively in the originating cluster's FoundationDB. That is precisely what a disaster recovery does not have. A manifest now goes into the same bucket, at manifests/{backup_id}.json. The prefix is a leading non-numeric segment, so it cannot collide with the data plane's decimal keyspace. It carries the location, the id of the backup this one is a delta against, the volume's shape, source provenance, and the object layout. It carries no credentials: it says where the objects are and how to read them, never how to authenticate. The reader supplies that. Chains are derived, not stored. backup_manifest.chain_of walks prev_backup_id across the manifests in the bucket. Storing the chain in each manifest would mean every merge invalidates the manifest of every descendant of the backup it folded away: the write cost of a merge becomes the length of the chain, and a partial failure leaves the bucket advertising object keys the data plane has already unmapped. With only the immediate link stored, a merge is exactly two objects -- republish the survivor, whose prev_backup_id just moved, and delete the merged-away one -- and every other manifest stays true without being touched. Publication order is deliberate. The manifest is written BEFORE the backup is marked COMPLETED, so that status implies "identifiable from the bucket alone", and a manifest failure fails the backup. Data in a bucket with no manifest is data nobody can attribute to a volume later. Absence is spelled as absence. A chain root has prev_backup_id=None rather than ""; a volume's settings and the object size are Optional and absent together with the volume or cluster they came from, because 0 is a real answer for a QoS cap (it means unlimited) and for a priority class. What is always knowable off the backup record -- ids, timestamps, size, whether it is encrypted -- is mandatory, with no default to fall back on. The volume settings are recorded even though restore still uses hardcoded defaults, because a manifest is read years after it is written and a backup cannot be given a shape retroactively once its volume is gone. export_backups and discover_backups return the manifests themselves rather than dicts; whoever writes them out decides how they are rendered. import_backups takes manifests too, so "is this a manifest at all" is answered by whoever read the bytes -- the API by its request body's type, the CLI when it parses the file -- and reported against the thing the operator actually supplied. export_backups also stops emitting a third, narrower format of its own. That format is how `encrypted` came to be omitted: two shapes for the same concept, and only one of them was maintained. New: discover_backups / import_from_bucket, and POST /backups/discover. Given a bucket and credentials for it, these answer "what is in here" and "register it" with no reference to any cluster, live or dead. That is the disaster-recovery entry point the feature was missing. POST /backups/import takes a union of two bodies rather than one model with two optional fields and a validator forbidding both-or-neither: pydantic then rejects a malformed request itself, the manifests are typed on the way in, and the OpenAPI schema says "one of these two" instead of "everything optional, good luck". Status codes: an unreadable bucket is 400, not 502 -- nothing here proxies for S3, and the bucket named in the request is the only thing that can be wrong from this side. PreconditionError is no longer caught locally and turned into 409; app.py maps it to 400 for the whole API, and a second, disagreeing mapping in one router only made the API inconsistent with itself. Two deliberate non-deletions: * delete_backups does not remove manifests. bdev_lvol_s3_delete does not exist on the data plane, so the objects outlive the call; the manifest is the only thing that can still identify them, and dropping it would turn a reclaimable orphan set into anonymous bucket weight. * list_all and _parse refuse an unreadable or unknown-version manifest rather than skipping it. Silently omitting a backup from a recovery listing is how an operator concludes their data is gone. The S3 client moves to the new module, so there is one place that knows how to talk to a bucket rather than a copy in the controller. build_manifest's docstring now records where Backup and BackupManifest overlap without justification, and where the overlap is already wrong: dataplane.cluster_size is recomputed from the current cluster, so re-exporting an imported backup restamps it with the importing cluster's page size. Nothing reads it yet. Collapsing the two documents is left as its own change. Tests: manifest schema handling and chain derivation are unit-tested; assembly and the export -> wipe the database -> import round-trip run against real FoundationDB, with boto3 mocked at the client boundary as an external service. The import endpoint's union, its 400s and the absence of a local 409 are covered in the v2 endpoint tests. TestImportBackups is deleted from test_backup.py rather than ported -- the new file covers the same ground against the manifest format with real fixtures, and maintaining two copies is what let the old format drift. --- simplyblock_cli/clibase.py | 26 +- simplyblock_core/backup_manifest.py | 310 ++++++++++++++++++ .../controllers/backup_controller.py | 310 ++++++++++++------ .../services/tasks_runner_backup.py | 35 +- simplyblock_web/api/v2/_dtos.py | 10 + simplyblock_web/api/v2/cluster/backup.py | 67 +++- tests/integration/test_backup.py | 98 +----- .../integration/test_backup_manifest_flow.py | 302 +++++++++++++++++ tests/unit/test_backup_manifest.py | 159 +++++++++ .../unit/web/api/v2/test_backup_endpoints.py | 121 +++++++ 10 files changed, 1229 insertions(+), 209 deletions(-) create mode 100644 simplyblock_core/backup_manifest.py create mode 100644 tests/integration/test_backup_manifest_flow.py create mode 100644 tests/unit/test_backup_manifest.py diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index eb0ec2a00..1caf5c754 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -9,6 +9,7 @@ import argcomplete from simplyblock_core import cluster_ops, utils, db_controller, constants +from simplyblock_core.backup_manifest import BackupManifest from simplyblock_core.exceptions import MigrationConflictError, PreconditionError from simplyblock_core import storage_node_ops as storage_ops from simplyblock_core import mgmt_node_ops as mgmt_ops @@ -1052,18 +1053,18 @@ def backup__restore(self, sub_command, args): return True def backup__export(self, sub_command, args): - data = backup_controller.export_backups( + manifests = backup_controller.export_backups( cluster_id=getattr(args, 'cluster_id', None), lvol_name=getattr(args, 'lvol_name', None)) - if not data: + if not manifests: print("No completed backups found") return False - output = _format_json(data) + output = _format_json([m.model_dump(mode="json") for m in manifests]) output_file = getattr(args, 'output', None) if output_file: with open(output_file, 'w') as f: f.write(output) - print(f"Exported {len(data)} backup(s) to {output_file}") + print(f"Exported {len(manifests)} backup(s) to {output_file}") else: print(output) return True @@ -1071,14 +1072,23 @@ def backup__export(self, sub_command, args): def backup__import(self, sub_command, args): try: with open(args.metadata_file, 'r') as f: - metadata_list = json.load(f) + entries = json.load(f) except Exception as e: print(f"Error reading metadata file: {e}") return False - if not isinstance(metadata_list, list): - metadata_list = [metadata_list] + if not isinstance(entries, list): + entries = [entries] + + # Parsed here rather than in the controller so a malformed file is + # reported as a problem with the file, naming it. + try: + manifests = [BackupManifest.model_validate(entry) for entry in entries] + except ValueError as e: + print(f"{args.metadata_file} is not a backup export: {e}") + return False + count = backup_controller.import_backups( - metadata_list, cluster_id=getattr(args, 'cluster_id', None)) + manifests, cluster_id=getattr(args, 'cluster_id', None)) print(f"Imported {count} backup(s)") return True diff --git a/simplyblock_core/backup_manifest.py b/simplyblock_core/backup_manifest.py new file mode 100644 index 000000000..1346d12e2 --- /dev/null +++ b/simplyblock_core/backup_manifest.py @@ -0,0 +1,310 @@ +# coding=utf-8 +"""The self-describing part of a backup: a JSON manifest stored alongside its data. + +The data plane writes only opaque objects keyed ``{s3_id}/{mid}/{extent}``, and +nothing in them records which volume they came from, how they are encoded, or +which other backups they depend on. All of that used to live exclusively in the +originating cluster's FoundationDB, which is precisely what a disaster recovery +no longer has. + +A manifest closes that gap. It is written to the same bucket as the data it +describes, under a ``manifests/`` prefix -- a leading non-numeric segment, so it +cannot collide with the data plane's decimal keyspace. Given a bucket and +credentials for it, every backup in it can be enumerated, understood and +restored with no other input. + +Credentials are deliberately absent: a manifest says *where* the objects are and +*how* to read them, never how to authenticate. The reader supplies that. + +Each manifest describes exactly one backup and names only its immediate +predecessor. Chains are walked at read time by :func:`chain_of`, not stored: a +stored chain would have to be rewritten in every descendant's manifest each time +a merge folded a backup away, so a single merge would cost a write per descendant +and could half-fail, leaving the bucket advertising keys the data plane had +already unmapped. + +This document overlaps the ``Backup`` record in FoundationDB by design, and +substantially -- see the note at the top of ``controllers/backup_controller.py`` +for where the two genuinely differ and where they should be collapsed. +""" +import json +import logging +from typing import Iterable, List, Optional + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError +from pydantic import BaseModel, ConfigDict + +from simplyblock_core.models.backup_config import BackupConfig, BackupLocation +from simplyblock_core.utils.secrets import unwrap_secret + + +logger = logging.getLogger() + +MANIFEST_PREFIX = "manifests/" + +#: Bumped when the manifest's meaning changes in a way an older reader would +#: misinterpret. A reader must refuse a version it does not know rather than +#: guess -- restoring from a misread manifest corrupts the volume silently. +MANIFEST_SCHEMA_VERSION = 1 + + +def manifest_key(backup_id: str) -> str: + return f"{MANIFEST_PREFIX}{backup_id}.json" + + +class Source(BaseModel): + """Where this backup came from. Provenance for an operator reading a bucket. + + Nothing may resolve configuration or keys through these -- that dependency + on the originating cluster is the whole problem being removed. + """ + model_config = ConfigDict(extra="forbid") + + cluster_id: str + node_id: str + + #: Absent when the cluster's own record of its name was no longer readable + #: at the time the manifest was written. + cluster_name: Optional[str] = None + + +class Volume(BaseModel): + """The shape of the volume this backup was taken from. + + Split in two by what is knowable. The identity and size come off the backup + record and are always present. The settings below them come off the live + volume, so they are absent together once that volume is deleted -- and + absent is not the same answer as ``0``, which for a QoS cap means + "unlimited" and for a priority class is a real class. + + Nothing reads the settings yet; restore still creates its volume with + hardcoded defaults. They are recorded anyway because a manifest is read + years after it is written, and a backup taken today cannot be given a shape + retroactively once its volume is gone. + """ + model_config = ConfigDict(extra="forbid") + + lvol_id: str + lvol_name: str + snapshot_id: str + snapshot_name: str + size: int + allowed_hosts: List[dict] = [] + + pool_name: Optional[str] = None + ha_type: Optional[str] = None + fabric: Optional[str] = None + lvol_priority_class: Optional[int] = None + max_size: Optional[int] = None + rw_ios_per_sec: Optional[int] = None + rw_mbytes_per_sec: Optional[int] = None + r_mbytes_per_sec: Optional[int] = None + w_mbytes_per_sec: Optional[int] = None + + +class DataPlane(BaseModel): + """How the objects are laid out, so a later format change is detectable.""" + model_config = ConfigDict(extra="forbid") + + #: Object key template. ``mid=1`` is metadata (``/1/0`` is the data plane's + #: own root record, ``/1/n`` the extent map); ``mid=0`` is a data cluster. + key_format: str = "{s3_id}/{mid}/{extent}" + + #: Object body size for both data and metadata objects. Absent when the + #: writing cluster's record was unreadable -- a reader then has to fall back + #: on the data plane's own default, which is why it is not silently 0. + cluster_size: Optional[int] = None + + +class BackupManifest(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: int = MANIFEST_SCHEMA_VERSION + backup_id: str + s3_id: int + created_at: int + completed_at: int + size: int + encrypted: bool + + #: The backup this one is a delta against, or absent when it is a full + #: backup and therefore the root of its chain. + #: + #: The chain itself is deliberately NOT stored. It is derivable by following + #: these links across the manifests in the bucket, and storing it would make + #: every merge invalidate the manifest of every descendant of the merged-away + #: backup -- so the write amplification of a merge would be the length of the + #: chain, and a partial failure would leave the bucket advertising object keys + #: the data plane had already unmapped. + prev_backup_id: Optional[str] = None + + location: BackupLocation + source: Source + volume: Volume + dataplane: DataPlane + + +class ManifestError(Exception): + """A manifest could not be read, written, or understood.""" + + +def s3_client(config: BackupConfig): + """A boto3 client for a backup location. + + Credentials are passed only when configured; omitting them lets boto3 fall + back to its default provider chain (instance IAM role, environment, + profile), which is what an absent ``credentials`` means. + """ + return boto3.client("s3", + region_name=config.region, + endpoint_url=config.endpoint_url, + verify=config.verify_tls, + config=BotoConfig(s3={"addressing_style": "path" if config.use_path_style else "auto"}), + aws_access_key_id=( + unwrap_secret(config.credentials.access_key_id) + if config.credentials is not None else None), + aws_secret_access_key=( + unwrap_secret(config.credentials.secret_access_key) + if config.credentials is not None else None), + ) + + +def write(config: BackupConfig, manifest: BackupManifest) -> None: + """Store a manifest next to the data it describes. + + Raises: + ManifestError: The manifest could not be stored. Callers must treat this + as a failed backup: data in the bucket with no manifest is data + nobody can identify later. + """ + try: + s3_client(config).put_object( + Bucket=config.bucket_name, + Key=manifest_key(manifest.backup_id), + Body=manifest.model_dump_json().encode(), + ContentType="application/json", + ) + except (BotoCoreError, ClientError) as e: + raise ManifestError( + f"Failed to write manifest for backup {manifest.backup_id}") from e + + logger.info("Wrote manifest for backup %s to %s", + manifest.backup_id, config.bucket_name) + + +def read(config: BackupConfig, backup_id: str) -> BackupManifest: + """Load one manifest by backup id. + + Raises: + ManifestError: It is absent, unreadable, or a schema version this build + does not understand. + """ + try: + body = s3_client(config).get_object( + Bucket=config.bucket_name, Key=manifest_key(backup_id))["Body"].read() + except (BotoCoreError, ClientError) as e: + raise ManifestError( + f"Failed to read manifest for backup {backup_id} " + f"from {config.bucket_name}") from e + + return _parse(body, manifest_key(backup_id)) + + +def list_all(config: BackupConfig) -> List[BackupManifest]: + """Every manifest in the bucket, newest first. + + This is the disaster-recovery entry point: with a bucket and credentials it + answers "what is in here" without reference to any cluster. + + Raises: + ManifestError: The bucket could not be listed, or one of its manifests + could not be parsed. Deliberately not best-effort -- silently + omitting an unreadable backup from a recovery listing is how an + operator concludes their data is gone. + """ + client = s3_client(config) + manifests = [] + + try: + pages = client.get_paginator("list_objects_v2").paginate( + Bucket=config.bucket_name, Prefix=MANIFEST_PREFIX) + for page in pages: + for entry in page.get("Contents", []): + key = entry["Key"] + if not key.endswith(".json"): + continue + body = client.get_object(Bucket=config.bucket_name, Key=key)["Body"].read() + manifests.append(_parse(body, key)) + except (BotoCoreError, ClientError) as e: + raise ManifestError(f"Failed to list manifests in {config.bucket_name}") from e + + manifests.sort(key=lambda m: (m.created_at, m.backup_id), reverse=True) + return manifests + + +def delete(config: BackupConfig, backup_id: str) -> None: + """Remove a manifest. + + Only for a backup whose data is genuinely gone -- a merge folding it into + its successor. Deleting the manifest of a backup whose objects still exist + turns them into weight nobody can identify or reclaim. + """ + try: + s3_client(config).delete_object( + Bucket=config.bucket_name, Key=manifest_key(backup_id)) + except (BotoCoreError, ClientError) as e: + raise ManifestError(f"Failed to delete manifest for backup {backup_id}") from e + + +def _parse(body: bytes, key: str) -> BackupManifest: + try: + data = json.loads(body) + except ValueError as e: + raise ManifestError(f"Manifest {key} is not valid JSON") from e + + version = data.get("schema_version") + if version != MANIFEST_SCHEMA_VERSION: + # Refuse rather than guess: a manifest written by a newer control plane + # may mean something different by the same field names, and restoring + # from a misread manifest corrupts the volume without an error. + raise ManifestError( + f"Manifest {key} has schema version {version}, " + f"this build understands {MANIFEST_SCHEMA_VERSION}") + + try: + return BackupManifest.model_validate(data) + except ValueError as e: + raise ManifestError(f"Manifest {key} is malformed: {e}") from e + + +def chain_of(manifest: BackupManifest, + manifests: Iterable[BackupManifest]) -> List[BackupManifest]: + """The chain ending at ``manifest``, oldest first and including itself. + + Derived by following ``prev_backup_id`` through the manifests supplied, + rather than read from a list stored in each one. That keeps a merge a + two-object write -- republish the survivor, delete the merged-away one -- + instead of a rewrite of every descendant's manifest. + + Raises: + ManifestError: A link points at a backup that is not among the manifests + supplied, so the chain cannot be completed from them. Reported rather + than truncated: a short chain restores a volume with holes in it. + """ + by_id = {m.backup_id: m for m in manifests} + + chain = [manifest] + while (previous := chain[-1].prev_backup_id) is not None: + if previous not in by_id: + raise ManifestError( + f"Backup {chain[-1].backup_id} is a delta against {previous}, " + "which is not among the manifests given") + if previous in {m.backup_id for m in chain}: + raise ManifestError( + f"Backup {manifest.backup_id} has a cyclic chain at {previous}") + chain.append(by_id[previous]) + + chain.reverse() + return chain diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index 1ac7686d3..5299e7b39 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -3,12 +3,11 @@ import re import time import uuid -from typing import Optional +from typing import Iterable, List, Optional -import boto3 -from botocore.config import Config as BotoConfig from botocore.exceptions import BotoCoreError, ClientError +from simplyblock_core import backup_manifest, constants from simplyblock_core.controllers import backup_events, tasks_controller from simplyblock_core.db_controller import DBController from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment @@ -18,7 +17,6 @@ KMSException, backup_dek_path, backup_kek_name, create_kms_connection, lvol_dek_path, pool_kek_name, ) -from simplyblock_core.utils.secrets import unwrap_secret from simplyblock_core.exceptions import PreconditionError from simplyblock_core.rpc_client import RPCException @@ -103,30 +101,143 @@ def _compute_s3_cpu_masks(node): return bdb_lcpu_mask, s3_lcpu_mask -def _s3_client(config: BackupConfig): - """A boto3 client for a backup location. - - Credentials are passed only when configured; omitting them lets boto3 fall - back to its default provider chain (instance IAM role, environment, profile), - which is the point of ``credentials`` being Optional. +def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: + """Assemble the self-describing record for a completed backup. + + Everything a restore needs is collected here, from the backup itself and + from the volume/pool/cluster it came from, so that after this point no part + of the restore path has to consult the originating cluster. + + This function exists because `Backup` and `BackupManifest` describe the same + thing in two shapes. Most of that overlap is not justified, and this is the + seam where it shows: + + * Justified: `status` and `error_message` are on the record and not in the + manifest, because they are mutable control-plane state with no meaning in a + bucket. `schema_version` and `dataplane` are in the manifest and not on the + record, because they are claims about the byte format, which the cluster + that wrote them does not need told back to it. + * Not justified: `pool_uuid` on the record against `volume.pool_name` in the + manifest -- the same fact, keyed differently, so neither can be derived from + the other. `encrypted` living both as its own field and inside `encryption`, + which is why this function has to overlay one onto the other so a manifest + cannot contradict itself. And the volume's settings, which the manifest + records and the record does not, so an imported backup knows less about its + volume than the manifest it was imported from did. + * Actively wrong: `dataplane.cluster_size` is recomputed here from the + *current* cluster. Re-exporting an imported backup therefore restamps it + with the importing cluster's page size, silently, even though it describes + objects a different cluster wrote. Nothing reads it yet, so nothing is + broken today. + + The fix for all three is the same and is not attempted here: make the + manifest the canonical document, store it on the record, and reduce `Backup` + to control-plane state plus the fields FoundationDB is queried by. That is + cheaper than it looks -- every backup query in `db_controller` already filters + in Python over a full scan, so nesting costs nothing there -- but it touches + `BackupDTO`, the `backup list` table and existing records, so it wants its own + change. """ - return boto3.client("s3", - region_name=config.region, - endpoint_url=config.endpoint_url, - verify=config.verify_tls, - config=BotoConfig(s3={"addressing_style": "path" if config.use_path_style else "auto"}), - aws_access_key_id=( - unwrap_secret(config.credentials.access_key_id) - if config.credentials is not None else None), - aws_secret_access_key=( - unwrap_secret(config.credentials.secret_access_key) - if config.credentials is not None else None), + volume = backup_manifest.Volume( + lvol_id=backup.lvol_id, + lvol_name=backup.lvol_name, + snapshot_id=backup.snapshot_id, + snapshot_name=backup.snapshot_name, + size=backup.size, + allowed_hosts=backup.allowed_hosts or [], ) + # The volume's own settings, where it still exists. Absent together once it + # is gone, which is a different answer from 0 -- for a QoS cap that means + # unlimited. + try: + lvol = db_controller.get_lvol_by_id(backup.lvol_id) + except KeyError: + logger.warning("Volume %s is gone; manifest for backup %s records only " + "the shape carried on the backup itself", + backup.lvol_id, backup.uuid) + else: + volume = volume.model_copy(update={ + "pool_name": lvol.pool_name, + "ha_type": lvol.ha_type or "default", + "fabric": lvol.fabric or "tcp", + "lvol_priority_class": lvol.lvol_priority_class, + "max_size": lvol.max_size, + "rw_ios_per_sec": lvol.rw_ios_per_sec, + "rw_mbytes_per_sec": lvol.rw_mbytes_per_sec, + "r_mbytes_per_sec": lvol.r_mbytes_per_sec, + "w_mbytes_per_sec": lvol.w_mbytes_per_sec, + }) + + cluster_name = None + cluster_size = None + try: + cluster = db_controller.get_cluster_by_id(backup.cluster_id) + except KeyError: + logger.warning("Cluster %s is gone; manifest for backup %s records no " + "object size", backup.cluster_id, backup.uuid) + else: + cluster_name = cluster.cluster_name + cluster_size = cluster.page_size_in_blocks * constants.LVOL_CLUSTER_RATIO + + return backup_manifest.BackupManifest( + backup_id=backup.uuid, + s3_id=backup.s3_id, + created_at=backup.created_at, + completed_at=backup.completed_at, + size=backup.size, + encrypted=backup.encrypted, + prev_backup_id=backup.prev_backup_id or None, + location=backup.get_location(), + source=backup_manifest.Source( + cluster_id=backup.source_cluster_id or backup.cluster_id, + cluster_name=cluster_name, + node_id=backup.node_id, + ), + volume=volume, + dataplane=backup_manifest.DataPlane(cluster_size=cluster_size), + ) + + +def _config_for(backup: Backup) -> BackupConfig: + """Credentials for a backup's own bucket. + + The location comes from the backup; only the credentials come from the + cluster, and only because a manifest must never carry them. + + Raises: + PreconditionError: The cluster's configured bucket is not the one this + backup lives in, so its credentials cannot be assumed to reach it. + """ + config = db_controller.get_cluster_by_id(backup.cluster_id).get_backup_config() + location = backup.get_location() + + if config.location() != location: + raise PreconditionError( + f"Backup {backup.uuid} lives in bucket {location.bucket_name}, but " + f"cluster {backup.cluster_id} is configured for " + f"{config.bucket_name}; supply credentials for the backup's bucket") + + return config + + +def write_manifest(backup: Backup) -> None: + """Publish a backup's manifest. + + Raises: + ManifestError: the manifest could not be stored. + PreconditionError: the backup's bucket is not the cluster's own. + """ + backup_manifest.write(_config_for(backup), build_manifest(backup)) + + +def delete_manifest(backup: Backup) -> None: + backup_manifest.delete(_config_for(backup), backup.uuid) + def _s3_bucket_exists(config: BackupConfig, bucket_name) -> bool: try: - _s3_client(config).head_bucket(Bucket=bucket_name) + backup_manifest.s3_client(config).head_bucket(Bucket=bucket_name) return True except ClientError as e: error_code = int(e.response["Error"]["Code"]) @@ -137,7 +248,7 @@ def _s3_bucket_exists(config: BackupConfig, bucket_name) -> bool: def _ensure_s3_bucket(config: BackupConfig, bucket_name): try: - s3_client = _s3_client(config) + s3_client = backup_manifest.s3_client(config) try: s3_client.head_bucket(Bucket=bucket_name) logger.info(f"S3 bucket already exists: {bucket_name}") @@ -524,7 +635,16 @@ def _cleanup_backup_kms_keys(backups): def delete_backups(lvol_id): """Delete all backups for a given lvol. - Returns (success, error_message).""" + + Removes the database records, not the objects: bdev_lvol_s3_delete does not + exist on the data plane, so the S3 data outlives this call. The manifests + are deliberately left in place too -- they are the only thing that can still + identify those objects, and deleting them would turn a reclaimable orphan + set into anonymous bucket weight. `backup discover` therefore keeps showing + them, which is the honest answer about what the bucket contains. + + Returns (success, error_message). + """ backups = db_controller.get_backups_by_lvol_id(lvol_id) if not backups: return False, f"No backups found for lvol {lvol_id}" @@ -589,114 +709,112 @@ def list_backups(cluster_id=None): return data -def export_backups(cluster_id=None, lvol_name=None): - """Export completed backup metadata as a list of dicts suitable for import - into another cluster via import_backups(). +def export_backups(cluster_id=None, lvol_name=None) -> List[backup_manifest.BackupManifest]: + """Export completed backups as manifests, for import into another cluster. + + Emits the same shape that lives in the bucket, so a hand-carried file and a + bucket read are interchangeable. Previously this produced a third, narrower + format of its own -- which is how it came to omit `encrypted`. - Returns a list of metadata dicts including s3_id, chain links, and size. + Returns the manifests themselves; whoever is writing them out decides how + they are rendered. """ backups = db_controller.get_backups(cluster_id) completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] if lvol_name: completed = [b for b in completed if b.lvol_name == lvol_name] - result = [] - for b in completed: - result.append({ - "backup_id": b.uuid, - "s3_id": b.s3_id, - "cluster_id": b.cluster_id, - "lvol_id": b.lvol_id, - "lvol_name": b.lvol_name, - "snapshot_id": b.snapshot_id, - "snapshot_name": b.snapshot_name, - "node_id": b.node_id, - "prev_backup_id": b.prev_backup_id, - "size": b.size, - "allowed_hosts": b.allowed_hosts, - "location": b.location, - "encrypted": b.encrypted, - "created_at": b.created_at, - }) - return result + return [build_manifest(b) for b in completed] + + +def discover_backups(config: BackupConfig) -> List[backup_manifest.BackupManifest]: + """Every backup in a bucket, read from its manifests alone. + The disaster-recovery entry point: given a bucket and credentials for it, + this answers "what is in here" with no reference to any cluster, live or + dead. -def import_backups(s3_metadata_list, cluster_id=None): - """Import backup metadata from another cluster's S3 metadata. + Raises: + ManifestError: The bucket could not be listed, or one of its manifests + could not be parsed. + """ + return backup_manifest.list_all(config) + + +def import_backups(manifests: Iterable[backup_manifest.BackupManifest], + cluster_id=None) -> int: + """Register backups described by manifests into this cluster's database. - Backups are stored in the local cluster's DB namespace but keep their - original s3_ids (scoped to source_cluster_id). The source_cluster_id - field tracks which cluster originally created the backup. + The backups keep their original ids -- both their uuid and their s3_id, + which names their objects in the bucket and therefore cannot be reassigned. Args: - s3_metadata_list: list of dicts with backup metadata. - cluster_id: Target cluster to import into. Required for cross-cluster - restore so the backups are visible in the local cluster's DB. + manifests: validated manifests, from `discover_backups`, + `export_backups`, or a file parsed into them. Taking the models + rather than dicts means "is this a manifest at all" is answered by + whoever read the bytes -- the API by its request body's type, the CLI + when it parses the file -- and reported where the input came from. + cluster_id: Target cluster to import into, so the backups are visible in + its namespace. Raises: - ValueError: One of the given entries is not a usable backup description. - PreconditionError: One of the given backup IDs is already known. Backup + PreconditionError: One of the backup IDs is already known -- backup lookups are not scoped by cluster, so a UUID reused across clusters - would make either record unaddressable. All IDs are checked before - the first record is written, so nothing is imported in that case. + would make either record unaddressable. Everything is checked before + the first record is written, so a bad batch imports nothing rather + than half of itself. + ValueError: The same backup is listed twice. """ - pending = {} - for meta in s3_metadata_list: - backup_id = meta.get("backup_id") - if not backup_id: - continue + pending: dict = {} + for manifest in manifests: + backup_id = manifest.backup_id if backup_id in pending: raise ValueError(f"Backup {backup_id} is listed more than once") - # An entry that cannot say where its objects are, or whether they are - # encrypted, is not importable at any price: the first produces a - # restore against whatever bucket happens to be configured, the second a - # plaintext volume over ciphertext. Checked here so a stale export file - # is rejected whole rather than half-imported. - for required in ("location", "encrypted"): - if required not in meta: - raise ValueError( - f"Backup {backup_id} is missing '{required}'; it predates " - "self-describing backups and cannot be imported") - - BackupLocation.model_validate(meta["location"]) - try: existing = db_controller.get_backup_by_id(backup_id) except KeyError: - pending[backup_id] = meta + pending[backup_id] = manifest else: raise PreconditionError(f"Backup {backup_id} already exists in cluster {existing.cluster_id}") - for backup_id, meta in pending.items(): - source_cluster = meta.get("cluster_id", "") - target_cluster = cluster_id or source_cluster - + for backup_id, manifest in pending.items(): backup = Backup() backup.uuid = backup_id - backup.s3_id = meta.get("s3_id", 0) - backup.cluster_id = target_cluster - backup.source_cluster_id = source_cluster - backup.lvol_id = meta.get("lvol_id", "") - backup.lvol_name = meta.get("lvol_name", "") - backup.snapshot_id = meta.get("snapshot_id", "") - backup.snapshot_name = meta.get("snapshot_name", "") - backup.node_id = meta.get("node_id", "") - backup.prev_backup_id = meta.get("prev_backup_id", "") - backup.size = meta.get("size", 0) - backup.allowed_hosts = meta.get("allowed_hosts", []) - backup.created_at = meta.get("created_at", 0) + backup.s3_id = manifest.s3_id + backup.cluster_id = cluster_id or manifest.source.cluster_id + backup.source_cluster_id = manifest.source.cluster_id + backup.lvol_id = manifest.volume.lvol_id + backup.lvol_name = manifest.volume.lvol_name + backup.snapshot_id = manifest.volume.snapshot_id + backup.snapshot_name = manifest.volume.snapshot_name + backup.node_id = manifest.source.node_id + backup.prev_backup_id = manifest.prev_backup_id or "" + backup.size = manifest.size + backup.allowed_hosts = manifest.volume.allowed_hosts + backup.created_at = manifest.created_at + backup.completed_at = manifest.completed_at backup.status = Backup.STATUS_COMPLETED - backup.location = meta["location"] + backup.location = manifest.location.model_dump(mode="json") # Import used to drop this, so an imported encrypted backup restored as # use_crypto=False -- a plaintext volume over ciphertext, silently. - backup.encrypted = meta["encrypted"] + backup.encrypted = manifest.encrypted backup.write_to_db() return len(pending) +def import_from_bucket(config: BackupConfig, cluster_id=None) -> int: + """Import every backup found in a bucket. + + Raises: + ManifestError: the bucket could not be read. + PreconditionError: the manifests it holds cannot be imported as a batch. + """ + return import_backups(discover_backups(config), cluster_id=cluster_id) + + def get_backup_sources(cluster_id): """List all distinct backup sources (local + imported clusters). diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index 8f0ed8af0..a08f3e2fe 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -10,7 +10,9 @@ import time from simplyblock_core import constants, db_controller, utils -from simplyblock_core.controllers import backup_events +from simplyblock_core.backup_manifest import ManifestError +from simplyblock_core.controllers import backup_controller, backup_events +from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule @@ -109,8 +111,19 @@ def _run_backup(task): if stat and isinstance(stat, dict): state = stat.get("transfer_state", "") if state == "Done": - backup.status = Backup.STATUS_COMPLETED backup.completed_at = int(time.time()) + + # Publish the manifest BEFORE marking the backup completed, so that + # COMPLETED implies "identifiable from the bucket alone". Data with + # no manifest is data nobody can attribute to a volume later, so a + # manifest failure fails the backup rather than leaving that behind. + try: + backup_controller.write_manifest(backup) + except (ManifestError, PreconditionError) as e: + _fail_backup(backup, task, f"Failed to publish backup manifest: {e}") + return + + backup.status = Backup.STATUS_COMPLETED backup.write_to_db() backup_events.backup_completed(backup.cluster_id, backup.node_id, backup) task.function_result = "Backup completed" @@ -360,6 +373,24 @@ def _run_merge(task): old_backup.status = Backup.STATUS_MERGED old_backup.write_to_db() + # Two objects, and only two: the survivor's manifest, whose prev_backup_id + # just changed, and the merged-away one, which describes keys the data plane + # has unmapped. Every descendant's manifest stays valid because none of them + # names anything but its own immediate predecessor -- the chain is walked at + # read time rather than stored, precisely so a merge does not have to rewrite + # the whole line of descent and cannot half-succeed at it. + try: + backup_controller.write_manifest(keep_backup) + backup_controller.delete_manifest(old_backup) + except (ManifestError, PreconditionError) as e: + # The S3 merge already happened and is not reversible, so the task + # cannot be failed here -- retry the manifest work instead. + task.function_result = f"Merge done, manifest update failed: {e}" + task.retry += 1 + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return + task.function_result = "Merge completed" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 715532707..f0ba841e2 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -19,6 +19,7 @@ from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.backup import Backup, BackupPolicy +from simplyblock_core.backup_manifest import BackupManifest from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.stats import StatsObject from simplyblock_core.models.lvol_migration import LVolMigration @@ -512,6 +513,15 @@ def from_model( #: real class, without touching a single route signature. BackupConfigDTO = BackupConfig +#: A backup's manifest as the API exchanges it: the response body of +#: export/discover and the entries of an inline import. +#: +#: An alias for the same reason ``BackupConfigDTO`` is one -- except that here the +#: shapes have a reason to stay locked together, since the wire form of a manifest +#: is also its form in the bucket. Naming it separately still lets the API grow a +#: field the stored document does not have. +BackupManifestDTO = BackupManifest + class BackupDTO(BaseModel): id: UUID diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index 56e3b1168..42aaaa894 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -1,16 +1,17 @@ -from typing import List, Optional +from typing import List, Optional, Union from uuid import UUID from fastapi import APIRouter, HTTPException, Query, Request, Response -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict +from simplyblock_core.backup_manifest import ManifestError from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import backup_controller from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.lvol_model import LVol from .._dependencies import BackupResource, Cluster, Policy -from .._dtos import BackupDTO, BackupPolicyDTO +from .._dtos import BackupConfigDTO, BackupDTO, BackupManifestDTO, BackupPolicyDTO from ..util import CreationResponseFormatParameter, creation_response @@ -59,28 +60,71 @@ def restore_backup(cluster: Cluster, parameters: _RestoreParams): parameters.backup_id, parameters.lvol_name, parameters.pool, target_node_id=parameters.target_node_id)} -class _ImportParams(BaseModel): - metadata: list[dict] +class _ImportManifests(BaseModel): + """Manifests carried in the request itself, e.g. from an export file.""" + model_config = ConfigDict(extra="forbid") + + metadata: List[BackupManifestDTO] + + +class _ImportFromBucket(BaseModel): + """Import whatever a bucket turns out to contain. + + The disaster-recovery path: it needs a bucket and credentials for it, and + nothing from the cluster that wrote the backups. + """ + model_config = ConfigDict(extra="forbid") + + bucket: BackupConfigDTO + + +#: The two ways to name what to import. A union rather than one model with two +#: optional fields and a validator forbidding both/neither: pydantic then rejects +#: a malformed body itself, and the OpenAPI schema says "one of these two" rather +#: than "everything optional, good luck". +_ImportParams = Union[_ImportManifests, _ImportFromBucket] @api.post('/import', name='clusters:backups:import') def import_backups(cluster: Cluster, parameters: _ImportParams): try: - count = backup_controller.import_backups(parameters.metadata, cluster_id=cluster.get_id()) + count = ( + backup_controller.import_from_bucket( + parameters.bucket, cluster_id=cluster.get_id()) + if isinstance(parameters, _ImportFromBucket) else + backup_controller.import_backups( + parameters.metadata, cluster_id=cluster.get_id()) + ) + except ManifestError as e: + # The bucket named in the request could not be read. 400 rather than + # 502: nothing here proxies for an upstream service, and what the caller + # supplied is the only thing that can be wrong from here. + raise HTTPException(400, str(e)) from e except ValueError as e: - # The request body could not be read as backup descriptions, which is a - # bad request rather than an unmet precondition (those reach 400 through - # app.py's PreconditionError handler). raise HTTPException(400, str(e)) from e return {"imported": count} +@api.post('/discover', name='clusters:backups:discover') +def discover_backups(parameters: BackupConfigDTO) -> List[BackupManifestDTO]: + """List the backups a bucket contains, without importing anything. + + A POST because it carries credentials, which have no business in a query + string. Takes no cluster state at all: this is what an operator runs when + the cluster that wrote the backups no longer exists. + """ + try: + return backup_controller.discover_backups(parameters) + except ManifestError as e: + raise HTTPException(400, str(e)) from e + + @api.get('/export', name='clusters:backups:export') def export_backups( cluster: Cluster, backup_id: Optional[str] = Query(None, description="Export only the chain containing this backup UUID"), lvol_name: Optional[str] = Query(None, description="Export all completed backups for this lvol name"), -): +) -> List[BackupManifestDTO]: lvol_name_filter = lvol_name if backup_id and not lvol_name_filter: try: @@ -88,9 +132,8 @@ def export_backups( lvol_name_filter = backup.lvol_name except KeyError: raise HTTPException(404, f"Backup {backup_id} not found") - data = backup_controller.export_backups( + return backup_controller.export_backups( cluster_id=cluster.get_id(), lvol_name=lvol_name_filter) - return data class _BackupSourceSwitchParams(BaseModel): diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index 7f3be350a..bafe44403 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -24,7 +24,7 @@ import pytest -from simplyblock_core.controllers.backup_controller import backup_snapshot, import_backups +from simplyblock_core.controllers.backup_controller import backup_snapshot from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment @@ -91,18 +91,6 @@ def _backup(uuid="backup-1", lvol_id="lvol-1", status=Backup.STATUS_COMPLETED, return b -def _meta(backup_id, cluster_id="cluster-1", **overrides): - """One entry of an export/import payload.""" - return { - "backup_id": backup_id, - "lvol_id": "l-1", - "cluster_id": cluster_id, - "location": _backup_config().location().model_dump(mode="json"), - "encrypted": False, - **overrides, - } - - def _snapshot(uuid="snap-1", lvol_uuid="lvol-1", node_id="node-1"): s = SnapShot() s.uuid = uuid @@ -308,7 +296,7 @@ def test_large_cpu_count(self): class TestCreateS3Bdev(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_success(self, MockRPC, mock_boto3_client): mock_rpc = MockRPC.return_value @@ -331,7 +319,7 @@ def test_success(self, MockRPC, mock_boto3_client): "s3_lvs_test", "simplyblock-backup-cluster-1", allow_existing=True) mock_rpc.bdev_lvol_s3_bdev.assert_called_once_with("lvs_test", "s3_lvs_test") - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_no_lvstore(self, MockRPC, _mock_boto3_client): from simplyblock_core.controllers.backup_controller import create_s3_bdev @@ -353,7 +341,7 @@ def test_bdev_s3_create_fails(self, MockRPC): mock_rpc.bdev_s3_add_bucket_name.assert_not_called() mock_rpc.bdev_lvol_s3_bdev.assert_not_called() - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_bucket_name_fails(self, MockRPC, mock_boto3_client): from simplyblock_core.rpc_client import RPCRemoteError @@ -369,7 +357,7 @@ def test_bucket_name_fails(self, MockRPC, mock_boto3_client): create_s3_bdev(node, _backup_config()) mock_rpc.bdev_lvol_s3_bdev.assert_not_called() - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_attach_fails(self, MockRPC, mock_boto3_client): from simplyblock_core.rpc_client import RPCRemoteError @@ -385,7 +373,7 @@ def test_attach_fails(self, MockRPC, mock_boto3_client): with pytest.raises(RuntimeError): create_s3_bdev(node, _backup_config()) - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_local_testing_params(self, MockRPC, mock_boto3_client): mock_rpc = MockRPC.return_value @@ -425,7 +413,7 @@ def test_local_testing_params(self, MockRPC, mock_boto3_client): self.assertEqual(boto_kwargs["region_name"], "us-east-1") self.assertFalse(boto_kwargs["verify"]) - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_no_credentials_defers_to_the_provider_chain(self, MockRPC, mock_boto3_client): """An absent key pair must mean "use the node's IAM role", not "send empty keys".""" @@ -1012,78 +1000,6 @@ def test_fewer_than_two_backups(self, mock_db, mock_tasks): mock_tasks.add_backup_merge_task.assert_not_called() -# =========================================================================== -# 13. Import backups -# =========================================================================== - -class TestImportBackups(unittest.TestCase): - """Real FDB: import writes Backup records that later reads must see.""" - - def setUp(self): - self.db = DBController() - - def test_import_new(self): - count = import_backups([ - _meta("b-1", cluster_id="c-1"), - _meta("b-2", cluster_id="c-1"), - ], cluster_id="cluster-1") - - self.assertEqual(count, 2) - self.assertEqual( - {b.uuid for b in self.db.get_backups("cluster-1")}, {"b-1", "b-2"}) - - def test_existing_fails(self): - _backup(uuid="b-1").write_to_db(self.db.kv_store) - - with self.assertRaises(PreconditionError): - import_backups([_meta("b-2"), _meta("b-1")], cluster_id="cluster-1") - - # Nothing imported: the pre-check runs before the first write. - self.assertEqual({b.uuid for b in self.db.get_backups()}, {"b-1"}) - - def test_duplicate_in_metadata_fails(self): - with self.assertRaises(ValueError): - import_backups([_meta("b-1"), _meta("b-1")], cluster_id="cluster-1") - - self.assertEqual(self.db.get_backups(), []) - - def test_skip_no_backup_id(self): - self.assertEqual(import_backups([{"lvol_id": "l-1"}]), 0) - - def test_encrypted_flag_survives_import(self): - """Import used to drop this, restoring a plaintext volume over ciphertext.""" - import_backups([_meta("b-1", encrypted=True)], cluster_id="cluster-1") - - self.assertTrue(self.db.get_backup_by_id("b-1").encrypted) - - def test_location_survives_import(self): - import_backups([_meta("b-1")], cluster_id="cluster-1") - - self.assertEqual( - self.db.get_backup_by_id("b-1").get_location().bucket_name, - "simplyblock-backup-cluster-1") - - def test_entry_without_a_location_is_rejected(self): - """A pre-self-describing export would otherwise restore against whatever - bucket the importing cluster happens to have configured.""" - stale = _meta("b-1") - del stale["location"] - - with self.assertRaises(ValueError): - import_backups([stale], cluster_id="cluster-1") - - self.assertEqual(self.db.get_backups(), []) - - def test_invalid_location_rejects_the_whole_batch(self): - with self.assertRaises(ValueError): - import_backups( - [_meta("b-1"), _meta("b-2", location={"bucket_name": "b"})], - cluster_id="cluster-1") - - self.assertEqual(self.db.get_backups(), []) - - - # =========================================================================== # 14. List policies # =========================================================================== diff --git a/tests/integration/test_backup_manifest_flow.py b/tests/integration/test_backup_manifest_flow.py new file mode 100644 index 000000000..62322030b --- /dev/null +++ b/tests/integration/test_backup_manifest_flow.py @@ -0,0 +1,302 @@ +"""Manifest assembly and the export -> import round-trip, against real FoundationDB. + +The point of a manifest is that a backup can be understood without the cluster +that wrote it, so the test that matters is: build manifests, throw the database +away, and rebuild usable Backup records from the manifests alone. + +Only boto3 is mocked -- it is an external service client. The database is real. +""" +from unittest.mock import patch + +import pytest + +from simplyblock_core import backup_manifest +from simplyblock_core.controllers import backup_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.pool import Pool + + +CLUSTER_ID = "cluster-1" + + +def _config(**overrides): + return BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", + "region": "eu-central-1", + **overrides, + }) + + +@pytest.fixture +def db(): + return DBController() + + +@pytest.fixture +def cluster(db): + c = Cluster() + c.uuid = CLUSTER_ID + c.cluster_name = "primary" + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +@pytest.fixture +def lvol(db): + pool = Pool() + pool.uuid = "pool-1" + pool.pool_name = "testpool" + pool.cluster_id = CLUSTER_ID + pool.write_to_db(db.kv_store) + + volume = LVol() + volume.uuid = "lvol-1" + volume.lvol_name = "vol" + volume.pool_uuid = "pool-1" + volume.pool_name = "testpool" + volume.node_id = "node-1" + volume.size = 4096 + volume.ha_type = "ha" + volume.fabric = "tcp" + volume.rw_ios_per_sec = 5000 + volume.max_size = 8192 + volume.write_to_db(db.kv_store) + return volume + + +def _backup(db, uuid, s3_id, prev="", **overrides): + b = Backup() + b.uuid = uuid + b.s3_id = s3_id + b.cluster_id = CLUSTER_ID + b.source_cluster_id = CLUSTER_ID + b.lvol_id = "lvol-1" + b.lvol_name = "vol" + b.snapshot_id = f"snap-{uuid}" + b.snapshot_name = f"snap_{uuid}" + b.node_id = "node-1" + b.pool_uuid = "pool-1" + b.prev_backup_id = prev + b.size = 4096 + b.created_at = 1000 + s3_id + b.completed_at = 2000 + s3_id + b.allowed_hosts = [{"nqn": "nqn.2024-01.io.test:host"}] + b.status = Backup.STATUS_COMPLETED + b.location = _config().location().model_dump(mode="json") + for key, value in overrides.items(): + setattr(b, key, value) + b.write_to_db(db.kv_store) + return b + + +class TestBuildManifest: + + def test_records_the_backup_location(self, db, cluster, lvol): + manifest = backup_controller.build_manifest(_backup(db, "b-1", 1)) + + assert manifest.location.bucket_name == "simplyblock-backup-cluster-1" + assert manifest.location.region == "eu-central-1" + + def test_records_only_the_immediate_predecessor(self, db, cluster, lvol): + """Not the whole chain: storing that would make a merge rewrite the + manifest of every descendant of the backup it folded away.""" + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, prev="b-1") + third = _backup(db, "b-3", 3, prev="b-2") + + manifest = backup_controller.build_manifest(third) + + assert manifest.prev_backup_id == "b-2" + + def test_a_full_backup_has_no_predecessor(self, db, cluster, lvol): + """Absent rather than "", so a chain root is a state and not a blank.""" + assert backup_controller.build_manifest(_backup(db, "b-1", 1)).prev_backup_id is None + + def test_the_chain_reconstructs_from_the_links(self, db, cluster, lvol): + """What the stored chain was for, done from the bucket's own contents.""" + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, prev="b-1") + _backup(db, "b-3", 3, prev="b-2") + manifests = [backup_controller.build_manifest(b) for b in db.get_backups()] + last = next(m for m in manifests if m.backup_id == "b-3") + + chain = backup_manifest.chain_of(last, manifests) + + assert [(m.backup_id, m.s3_id) for m in chain] == [ + ("b-1", 1), ("b-2", 2), ("b-3", 3)] + + def test_records_the_volume_shape(self, db, cluster, lvol): + """Restore currently hardcodes these; recording them is what lets it stop.""" + manifest = backup_controller.build_manifest(_backup(db, "b-1", 1)) + + assert manifest.volume.pool_name == "testpool" + assert manifest.volume.ha_type == "ha" + assert manifest.volume.rw_ios_per_sec == 5000 + assert manifest.volume.max_size == 8192 + assert manifest.volume.allowed_hosts == [{"nqn": "nqn.2024-01.io.test:host"}] + + def test_survives_a_deleted_volume(self, db, cluster, lvol): + """A backup outlives its volume; that must not stop the manifest.""" + backup = _backup(db, "b-1", 1) + lvol.remove(db.kv_store) + + manifest = backup_controller.build_manifest(backup) + + assert manifest.volume.lvol_name == "vol" # carried on the backup itself + assert manifest.volume.pool_name is None # only on the volume + # Absent, not 0 -- which for a QoS cap would read as "unlimited". + assert manifest.volume.rw_ios_per_sec is None + + def test_records_source_as_provenance(self, db, cluster, lvol): + manifest = backup_controller.build_manifest(_backup(db, "b-1", 1)) + + assert manifest.source.cluster_id == CLUSTER_ID + assert manifest.source.cluster_name == "primary" + assert manifest.source.node_id == "node-1" + + def test_records_the_object_layout(self, db, cluster, lvol): + manifest = backup_controller.build_manifest(_backup(db, "b-1", 1)) + + assert manifest.dataplane.key_format == "{s3_id}/{mid}/{extent}" + assert manifest.dataplane.cluster_size == cluster.page_size_in_blocks + + def test_backup_without_a_location_is_refused(self, db, cluster, lvol): + backup = _backup(db, "b-1", 1) + backup.location = {} + + with pytest.raises(ValueError): + backup_controller.build_manifest(backup) + + +class TestExportImportRoundTrip: + """Export, wipe the database, import -- the disaster-recovery shape.""" + + def test_backups_survive_losing_the_database(self, db, cluster, lvol): + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, prev="b-1", encrypted=True) + + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + for backup in db.get_backups(): + backup.remove(db.kv_store) + assert db.get_backups() == [] + + count = backup_controller.import_backups(exported, cluster_id="cluster-2") + + assert count == 2 + restored = db.get_backup_by_id("b-2") + assert restored.s3_id == 2 + assert restored.prev_backup_id == "b-1" + assert restored.cluster_id == "cluster-2" + assert restored.source_cluster_id == CLUSTER_ID + assert restored.get_location().bucket_name == "simplyblock-backup-cluster-1" + assert restored.encrypted is True + + def test_encrypted_flag_survives(self, db, cluster, lvol): + """It used to be dropped, restoring a plaintext volume over ciphertext.""" + _backup(db, "b-1", 1, encrypted=True) + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + db.get_backup_by_id("b-1").remove(db.kv_store) + + backup_controller.import_backups(exported, cluster_id="cluster-2") + + assert db.get_backup_by_id("b-1").encrypted is True + + def test_export_emits_the_same_shape_as_the_bucket(self, db, cluster, lvol): + """One format, so a file and a bucket read are interchangeable.""" + _backup(db, "b-1", 1) + + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + + assert backup_manifest._parse( + exported[0].model_dump_json().encode(), "k") == exported[0] + + def test_only_completed_backups_are_exported(self, db, cluster, lvol): + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, status=Backup.STATUS_FAILED) + + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + + assert [m.backup_id for m in exported] == ["b-1"] + + def test_a_malformed_entry_never_reaches_the_controller(self, db, cluster, lvol): + """import_backups takes manifests, not dicts, so an unusable entry is + rejected by whoever read the bytes -- naming the file or the request.""" + with pytest.raises(ValueError): + backup_manifest.BackupManifest.model_validate( + {"backup_id": "b-9", "s3_id": "not-an-int"}) + + def test_duplicate_id_rejects_the_whole_batch(self, db, cluster, lvol): + _backup(db, "b-1", 1) + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + + with pytest.raises(PreconditionError, match="already exists"): + backup_controller.import_backups(exported, cluster_id="cluster-2") + + def test_same_id_listed_twice_rejects_the_whole_batch(self, db, cluster, lvol): + """Backup lookups are not cluster-scoped, so a reused uuid unaddresses both.""" + _backup(db, "b-1", 1) + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + db.get_backup_by_id("b-1").remove(db.kv_store) + + with pytest.raises(ValueError, match="listed more than once"): + backup_controller.import_backups(exported + exported, cluster_id="cluster-2") + + assert db.get_backups() == [] + + def test_nothing_to_import_is_not_an_error(self, db, cluster, lvol): + assert backup_controller.import_backups([], cluster_id="cluster-2") == 0 + + +class TestBucketDiscovery: + """The path that needs nothing but a bucket and credentials.""" + + def test_discover_reads_every_manifest(self, db, cluster, lvol): + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, prev="b-1") + manifests = [backup_controller.build_manifest(b) for b in db.get_backups()] + + with patch.object(backup_manifest, "list_all", return_value=manifests): + found = backup_controller.discover_backups(_config()) + + assert {m.backup_id for m in found} == {"b-1", "b-2"} + # Models, not dicts: rendering them is the caller's decision. + assert all(isinstance(m, backup_manifest.BackupManifest) for m in found) + + def test_import_from_bucket_needs_no_prior_records(self, db, cluster, lvol): + _backup(db, "b-1", 1) + manifests = [backup_controller.build_manifest(db.get_backup_by_id("b-1"))] + db.get_backup_by_id("b-1").remove(db.kv_store) + + with patch.object(backup_manifest, "list_all", return_value=manifests): + count = backup_controller.import_from_bucket(_config(), cluster_id="cluster-2") + + assert count == 1 + assert db.get_backup_by_id("b-1").cluster_id == "cluster-2" + + +class TestManifestPublication: + + def test_write_manifest_puts_the_object_in_the_backup_bucket(self, db, cluster, lvol): + backup = _backup(db, "b-1", 1) + + with patch.object(backup_manifest, "s3_client") as mock_client: + backup_controller.write_manifest(backup) + + _, kwargs = mock_client.return_value.put_object.call_args + assert kwargs["Bucket"] == "simplyblock-backup-cluster-1" + assert kwargs["Key"] == "manifests/b-1.json" + + def test_refuses_when_the_cluster_points_at_another_bucket(self, db, cluster, lvol): + """The cluster's credentials cannot be assumed to reach a foreign bucket.""" + backup = _backup(db, "b-1", 1) + backup.location = _config(bucket_name="somewhere-else").location().model_dump(mode="json") + backup.write_to_db(db.kv_store) + + with pytest.raises(PreconditionError, match="somewhere-else"): + backup_controller.write_manifest(backup) diff --git a/tests/unit/test_backup_manifest.py b/tests/unit/test_backup_manifest.py new file mode 100644 index 000000000..428a0438f --- /dev/null +++ b/tests/unit/test_backup_manifest.py @@ -0,0 +1,159 @@ +"""Manifest schema: serialization, version handling, and malformed input. + +Pure logic -- the S3 plumbing around it is exercised in the integration tier. +""" +import json + +import pytest + +from simplyblock_core import backup_manifest +from simplyblock_core.backup_manifest import ( + BackupManifest, + DataPlane, + ManifestError, + MANIFEST_SCHEMA_VERSION, + Source, + Volume, +) +from simplyblock_core.models.backup_config import BackupLocation + + +LOCATION = {"bucket_name": "backups", "region": "eu-central-1"} + + +def _manifest(**overrides): + fields = { + "backup_id": "b-1", + "s3_id": 7, + "created_at": 100, + "completed_at": 200, + "size": 4096, + "encrypted": False, + "location": BackupLocation.model_validate(LOCATION), + "source": Source(cluster_id="c-1", node_id="n-1"), + "volume": Volume(lvol_id="l-1", lvol_name="vol", snapshot_id="s-1", + snapshot_name="snap", size=4096), + "dataplane": DataPlane(), + } + fields.update(overrides) + return BackupManifest(**fields) + + +class TestManifestKey: + def test_key_cannot_collide_with_the_data_plane_keyspace(self): + """Data objects are {s3_id}/{mid}/{extent}, all decimal segments.""" + key = backup_manifest.manifest_key("b-1") + assert key.startswith("manifests/") + assert not key.split("/")[0].isdigit() + + +class TestSchema: + def test_round_trip(self): + original = _manifest(encrypted=True, prev_backup_id="b-0") + + restored = backup_manifest._parse(original.model_dump_json().encode(), "k") + + assert restored == original + + def test_serializes_to_plain_json(self): + data = json.loads(_manifest().model_dump_json()) + assert data["schema_version"] == MANIFEST_SCHEMA_VERSION + assert data["location"]["bucket_name"] == "backups" + + def test_carries_no_credential_field(self): + """A manifest sits next to the ciphertext; it must not carry keys.""" + data = json.loads(_manifest().model_dump_json()) + assert "credentials" not in data["location"] + assert "access_key_id" not in json.dumps(data) + + def test_unknown_field_is_rejected(self): + with pytest.raises(ValueError): + BackupManifest.model_validate({ + **json.loads(_manifest().model_dump_json()), "extra": 1}) + + def test_a_root_backup_has_no_predecessor(self): + """Absent, not "" -- a chain root is a state, not a missing value.""" + assert _manifest().prev_backup_id is None + + def test_the_chain_is_not_stored(self): + """It is derived from prev_backup_id, so a merge rewrites two objects + rather than every descendant's manifest.""" + assert "chain" not in json.loads(_manifest().model_dump_json()) + + def test_volume_settings_are_absent_rather_than_zero(self): + """0 is a real answer for a QoS cap -- it means unlimited.""" + volume = _manifest().volume + assert volume.rw_ios_per_sec is None + assert volume.pool_name is None + + +class TestChainOf: + def _line(self): + return [ + _manifest(backup_id="b-0", s3_id=1), + _manifest(backup_id="b-1", s3_id=2, prev_backup_id="b-0"), + _manifest(backup_id="b-2", s3_id=3, prev_backup_id="b-1"), + ] + + def test_walks_to_the_root_oldest_first(self): + line = self._line() + chain = backup_manifest.chain_of(line[-1], line) + assert [m.backup_id for m in chain] == ["b-0", "b-1", "b-2"] + + def test_a_full_backup_is_its_own_chain(self): + line = self._line() + assert backup_manifest.chain_of(line[0], line) == [line[0]] + + def test_order_does_not_matter(self): + line = self._line() + chain = backup_manifest.chain_of(line[-1], list(reversed(line))) + assert [m.backup_id for m in chain] == ["b-0", "b-1", "b-2"] + + def test_ignores_manifests_outside_the_chain(self): + line = self._line() + unrelated = _manifest(backup_id="other", s3_id=9) + chain = backup_manifest.chain_of(line[-1], line + [unrelated]) + assert [m.backup_id for m in chain] == ["b-0", "b-1", "b-2"] + + def test_a_missing_ancestor_is_reported_not_truncated(self): + """Truncating would restore a volume with holes in it.""" + line = self._line() + with pytest.raises(ManifestError, match="b-0"): + backup_manifest.chain_of(line[-1], line[1:]) + + def test_a_cycle_is_reported_rather_than_looping(self): + a = _manifest(backup_id="b-a", prev_backup_id="b-b") + b = _manifest(backup_id="b-b", prev_backup_id="b-a") + with pytest.raises(ManifestError, match="cyclic"): + backup_manifest.chain_of(a, [a, b]) + + +class TestParse: + def test_rejects_a_newer_schema_version(self): + """Guessing at an unknown schema restores a corrupt volume with no error.""" + data = json.loads(_manifest().model_dump_json()) + data["schema_version"] = MANIFEST_SCHEMA_VERSION + 1 + + with pytest.raises(ManifestError, match="schema version"): + backup_manifest._parse(json.dumps(data).encode(), "k") + + def test_rejects_a_missing_schema_version(self): + data = json.loads(_manifest().model_dump_json()) + del data["schema_version"] + + with pytest.raises(ManifestError, match="schema version"): + backup_manifest._parse(json.dumps(data).encode(), "k") + + def test_rejects_invalid_json(self): + with pytest.raises(ManifestError, match="not valid JSON"): + backup_manifest._parse(b"{not json", "k") + + def test_rejects_a_malformed_manifest(self): + with pytest.raises(ManifestError, match="malformed"): + backup_manifest._parse( + json.dumps({"schema_version": MANIFEST_SCHEMA_VERSION}).encode(), "k") + + def test_names_the_key_it_could_not_read(self): + """An operator sweeping a recovery bucket needs to know which object failed.""" + with pytest.raises(ManifestError, match="manifests/b-9.json"): + backup_manifest._parse(b"{not json", "manifests/b-9.json") diff --git a/tests/unit/web/api/v2/test_backup_endpoints.py b/tests/unit/web/api/v2/test_backup_endpoints.py index 59ac27868..d6902ee68 100644 --- a/tests/unit/web/api/v2/test_backup_endpoints.py +++ b/tests/unit/web/api/v2/test_backup_endpoints.py @@ -84,6 +84,127 @@ def test_restores_backup(self, client, db, cluster, backup_controller): BACKUP_ID, 'restored-volume', 'pool-1', target_node_id=None) +class TestImportBackups: + """The body is a union of two shapes, not one model with everything optional.""" + + _MANIFEST = { + 'schema_version': 1, + 'backup_id': BACKUP_ID, + 's3_id': 7, + 'created_at': 100, + 'completed_at': 200, + 'size': 4096, + 'encrypted': False, + 'location': {'bucket_name': 'backups', 'region': 'eu-central-1'}, + 'source': {'cluster_id': CLUSTER_ID, 'node_id': 'node-1'}, + 'volume': {'lvol_id': VOLUME_ID, 'lvol_name': 'vol', + 'snapshot_id': SNAPSHOT_ID, 'snapshot_name': 'snap', + 'size': 4096}, + 'dataplane': {}, + } + + _BUCKET = {'bucket_name': 'backups', 'region': 'eu-central-1'} + + def test_inline_manifests_are_validated_by_the_body_type( + self, client, db, cluster, backup_controller): + backup_controller.import_backups.return_value = 1 + + response = client.post(f'{BASE}/import', json={'metadata': [self._MANIFEST]}) + + assert response.status_code == 200 + assert response.json() == {'imported': 1} + (manifests,), kwargs = backup_controller.import_backups.call_args + assert [m.backup_id for m in manifests] == [BACKUP_ID] + + def test_a_malformed_manifest_is_rejected_before_the_controller( + self, client, db, cluster, backup_controller): + response = client.post( + f'{BASE}/import', + json={'metadata': [{**self._MANIFEST, 's3_id': 'not-an-int'}]}) + + assert response.status_code == 422 + backup_controller.import_backups.assert_not_called() + + def test_a_bucket_reads_the_manifests_itself( + self, client, db, cluster, backup_controller): + backup_controller.import_from_bucket.return_value = 3 + + response = client.post(f'{BASE}/import', json={'bucket': self._BUCKET}) + + assert response.status_code == 200 + assert response.json() == {'imported': 3} + backup_controller.import_backups.assert_not_called() + + def test_naming_both_sources_is_rejected( + self, client, db, cluster, backup_controller): + """extra="forbid" on both arms is what makes the union decide.""" + response = client.post(f'{BASE}/import', json={ + 'metadata': [self._MANIFEST], 'bucket': self._BUCKET}) + + assert response.status_code == 422 + + def test_naming_neither_source_is_rejected( + self, client, db, cluster, backup_controller): + response = client.post(f'{BASE}/import', json={}) + + assert response.status_code == 422 + + def test_an_unreadable_bucket_is_a_bad_request_not_a_bad_gateway( + self, client, db, cluster, backup_controller): + """Nothing here proxies for S3, and the bucket came from the request.""" + from simplyblock_core.backup_manifest import ManifestError + backup_controller.import_from_bucket.side_effect = ManifestError('no such bucket') + + response = client.post(f'{BASE}/import', json={'bucket': self._BUCKET}) + + assert response.status_code == 400 + assert 'no such bucket' in response.json()['detail'] + + def test_a_precondition_error_is_not_mapped_here( + self, client, db, cluster, backup_controller): + """It used to become a 409, contradicting app.py, which maps every + PreconditionError to 400 for the whole API. The endpoint lets it through. + + (This test app deliberately mounts only the routers, so an unhandled + exception surfaces here instead of reaching that handler.) + """ + import pytest + from simplyblock_core.exceptions import PreconditionError + backup_controller.import_backups.side_effect = PreconditionError('already exists') + + with pytest.raises(PreconditionError): + client.post(f'{BASE}/import', json={'metadata': [self._MANIFEST]}) + + +class TestDiscoverBackups: + + def test_returns_the_manifests_the_bucket_holds( + self, client, db, backup_controller): + from simplyblock_core.backup_manifest import BackupManifest + backup_controller.discover_backups.return_value = [ + BackupManifest.model_validate(TestImportBackups._MANIFEST)] + + response = client.post( + f'{BASE}/discover', + json={'bucket_name': 'backups', 'region': 'eu-central-1'}) + + assert response.status_code == 200 + assert [entry['backup_id'] for entry in response.json()] == [BACKUP_ID] + + def test_credentials_are_masked_in_the_response_of_a_failure( + self, client, db, backup_controller): + from simplyblock_core.backup_manifest import ManifestError + backup_controller.discover_backups.side_effect = ManifestError('unreachable') + + response = client.post(f'{BASE}/discover', json={ + 'bucket_name': 'backups', 'region': 'eu-central-1', + 'credentials': {'access_key_id': 'AKIA', 'secret_access_key': 's3cr3t'}, + }) + + assert response.status_code == 400 + assert 's3cr3t' not in response.text + + class TestBackupPolicies: def test_list_policies(self, client, db, backup_policy): From 56ad9eee4a2b6da1b3466436ccd1cac992d817ce Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 11:28:47 +0200 Subject: [PATCH 05/14] Record how an encrypted backup's key can be reached An encrypted volume's backup is ciphertext in a bucket whose key lives in a KMS. Nothing recorded which KMS, at what path, or under which key -- the dependency was implicit, and only discovered during a recovery. Worse, for an imported backup `encrypted` was always False, so restoring one produced a plaintext volume over ciphertext, silently. A backup now carries an `encryption` document holding a key descriptor: which backend held the key, which Vault and mounts where that applies, and at what path. Never key material. Restore resolves the key from it before creating the volume, so a restore that cannot decrypt fails leaving nothing behind, and the error names the path, the cluster and the KMS -- an operator mid-recovery needs to know what is missing. The working assumption is that a KMS is recoverable independently of the cluster that used it: a Vault deployment outlives one cluster, and the FoundationDB behind LocalKMS is itself backed up. So recording the dependency is enough, and no key material has to travel with the ciphertext. This is why the descriptor is a document rather than loose fields: a scheme that wraps the keys under an operator-held secret adds a sibling field to Encryption and a branch in _resolve_crypto_key, and touches nothing else -- including no change to what is already written in any bucket, since a reader treats an absent field as absent. Invalid combinations are unrepresentable rather than checked at each use. `kms` is a Literal, so a manifest naming a backend this build does not implement is refused instead of read with the wrong fields meaning something; the Vault mounts are Optional and absent for the local backend rather than ""; and an Encryption validator requires a descriptor exactly when `encrypted` is set, so "encrypted, key location unknown" cannot be constructed at all. Backup.encrypted stays authoritative over the copy inside the encryption document, and build_manifest overlays it so the two cannot disagree in a manifest. Two places recording one fact can drift, and drift here decides whether a restore decrypts. An unreachable KMS is a RuntimeError, not a PreconditionError: there is no condition the caller could have checked to avoid it. A backup that records nothing at all about its key stays a PreconditionError, since that is a property of the request's target rather than a failure. --- simplyblock_core/backup_manifest.py | 57 ++++- .../controllers/backup_controller.py | 88 ++++++-- simplyblock_core/models/backup.py | 4 + simplyblock_web/api/v2/_dtos.py | 2 + simplyblock_web/api/v2/cluster/backup.py | 3 +- tests/integration/test_backup_encryption.py | 207 ++++++++++++++++++ .../integration/test_backup_manifest_flow.py | 6 +- tests/unit/test_backup_manifest.py | 8 +- .../unit/web/api/v2/test_backup_endpoints.py | 21 +- 9 files changed, 374 insertions(+), 22 deletions(-) create mode 100644 tests/integration/test_backup_encryption.py diff --git a/simplyblock_core/backup_manifest.py b/simplyblock_core/backup_manifest.py index 1346d12e2..764a1a4f5 100644 --- a/simplyblock_core/backup_manifest.py +++ b/simplyblock_core/backup_manifest.py @@ -29,12 +29,12 @@ """ import json import logging -from typing import Iterable, List, Optional +from typing import Iterable, List, Literal, Optional import boto3 from botocore.config import Config as BotoConfig from botocore.exceptions import BotoCoreError, ClientError -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, model_validator from simplyblock_core.models.backup_config import BackupConfig, BackupLocation from simplyblock_core.utils.secrets import unwrap_secret @@ -104,6 +104,57 @@ class Volume(BaseModel): w_mbytes_per_sec: Optional[int] = None +class KeyDescriptor(BaseModel): + """Where this backup's data encryption key lives. Never the key itself. + + Restoring an encrypted backup means reaching the KMS named here. The working + assumption is that a KMS is recoverable independently of the cluster that + used it -- a Vault deployment outlives one cluster, and the FoundationDB + behind LocalKMS is itself backed up -- so recording the dependency is enough, + and no key material has to travel with the ciphertext. + + Recording it as a document rather than as loose fields is also what leaves + room to change that assumption: a scheme that wraps the keys under an + operator-held secret adds a sibling field here and a branch in + _resolve_crypto_key, and touches nothing else. + """ + model_config = ConfigDict(extra="forbid") + + #: Which backend holds the key. Named rather than free text, because a reader + #: years later has to know which of the fields below mean anything. + kms: Literal["hashicorp_vault", "local"] + + dek_path: str + kek_name: str + + #: Vault only; absent for the local backend. + vault_base_url: Optional[str] = None + transit_mount: Optional[str] = None + kv_mount: Optional[str] = None + + +class Encryption(BaseModel): + """Whether this backup is ciphertext, and if so how to reach its key.""" + model_config = ConfigDict(extra="forbid") + + encrypted: bool + + #: Where the key lives. Required for an encrypted backup and meaningless + #: otherwise, so the two cannot disagree. + descriptor: Optional[KeyDescriptor] = None + + @model_validator(mode="after") + def _descriptor_matches_encrypted(self) -> "Encryption": + if self.encrypted and self.descriptor is None: + raise ValueError( + "An encrypted backup must record where its key lives; without " + "that, nothing can decrypt it and nothing can say why") + if not self.encrypted and self.descriptor is not None: + raise ValueError( + "An unencrypted backup has no key to describe") + return self + + class DataPlane(BaseModel): """How the objects are laid out, so a later format change is detectable.""" model_config = ConfigDict(extra="forbid") @@ -127,7 +178,6 @@ class BackupManifest(BaseModel): created_at: int completed_at: int size: int - encrypted: bool #: The backup this one is a delta against, or absent when it is a full #: backup and therefore the root of its chain. @@ -141,6 +191,7 @@ class BackupManifest(BaseModel): prev_backup_id: Optional[str] = None location: BackupLocation + encryption: Encryption source: Source volume: Volume dataplane: DataPlane diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index 5299e7b39..af8053b95 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -186,8 +186,11 @@ def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: created_at=backup.created_at, completed_at=backup.completed_at, size=backup.size, - encrypted=backup.encrypted, prev_backup_id=backup.prev_backup_id or None, + # backup.encrypted is authoritative -- overlaying it here means the two + # cannot disagree in a manifest, whatever is stored in the dict. + encryption=backup_manifest.Encryption.model_validate( + {**(backup.encryption or {}), "encrypted": backup.encrypted}), location=backup.get_location(), source=backup_manifest.Source( cluster_id=backup.source_cluster_id or backup.cluster_id, @@ -199,6 +202,48 @@ def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: ) +def _resolve_crypto_key(backup: Backup, cluster): + """Recover the key needed to read an encrypted backup. + + An encrypted backup is ciphertext whose key lives in a KMS, and the backup + records which one. Restoring it therefore needs that KMS reachable -- the + assumption being that a KMS is recoverable independently of any one cluster. + Nothing about the key travels with the backup. + + Returns None for an unencrypted backup. + + Raises: + PreconditionError: The backup records nothing about its key, so no amount + of reachable infrastructure can decrypt it. + RuntimeError: The recorded KMS could not be reached. Raised before the + volume is created, so a restore that cannot decrypt fails without + leaving a half-built volume behind -- and, more importantly, without + silently producing a plaintext volume over ciphertext. + """ + if not backup.encrypted: + return None + + encryption = backup_manifest.Encryption.model_validate( + {**(backup.encryption or {}), "encrypted": backup.encrypted}) + + descriptor = encryption.descriptor + if descriptor is None: + raise PreconditionError( + f"Backup {backup.uuid} is encrypted but records nothing about its " + "key; it predates self-describing backups and cannot be restored") + + try: + with create_kms_connection(cluster) as kms: + return kms.get_data_encryption_keys(descriptor.dek_path, descriptor.kek_name) + except KMSException as e: + raise RuntimeError( + f"Cannot reach the key for backup {backup.uuid} at " + f"{descriptor.dek_path}. It was written by cluster " + f"{backup.source_cluster_id or backup.cluster_id} using " + f"{descriptor.kms}, which has to be reachable to restore it: " + f"{e}") from e + + def _config_for(backup: Backup) -> BackupConfig: """Credentials for a backup's own bucket. @@ -360,6 +405,31 @@ def _snapshot_has_backup(snapshot_id): Backup.STATUS_COMPLETED, Backup.STATUS_MERGED) for b in backups) +def _build_encryption(cluster, backup: Backup) -> backup_manifest.Encryption: + """Describe where an encrypted backup's key lives. + + The dependency on a KMS is not removed -- it is written down. Nothing before + this recorded it at all, so an encrypted backup's key was reachable only by + someone who already knew which cluster had made it and how that cluster was + configured. + """ + descriptor = backup_manifest.KeyDescriptor( + kms="local", + dek_path=backup_dek_path(cluster.get_id(), backup.uuid), + kek_name=backup_kek_name(backup.uuid), + ) + if cluster.hashicorp_vault_settings is not None: + vault = cluster.hashicorp_vault_settings + descriptor = descriptor.model_copy(update={ + "kms": "hashicorp_vault", + "vault_base_url": vault.base_url, + "transit_mount": vault.transit_mount, + "kv_mount": vault.kv_mount, + }) + + return backup_manifest.Encryption(encrypted=True, descriptor=descriptor) + + def _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location: BackupLocation): """Create a single backup record and task for one snapshot. @@ -402,6 +472,7 @@ def _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, loca backup_dek_path(cluster_id, backup.uuid), backup_kek_name(backup.uuid), ) + backup.encryption = _build_encryption(cluster, backup).model_dump(mode="json") backup.write_to_db() @@ -557,17 +628,7 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, raise PreconditionError( f"Target node {target_node_id} has no lvstore (S3 bdev requires lvstore)") - if backup.encrypted: - with create_kms_connection(cluster) as kms: - try: - crypto_key = kms.get_data_encryption_keys( - backup_dek_path(pool.cluster_id, backup.uuid), - backup_kek_name(backup.uuid), - ) - except KMSException as e: - raise RuntimeError("Failed to retrieve backup crypto keys") from e - else: - crypto_key = None + crypto_key = _resolve_crypto_key(backup, cluster) logger.info(f"Backup allowed hosts: {backup.allowed_hosts}") lvol_id, error = lvol_controller.add_lvol_ha( @@ -799,7 +860,8 @@ def import_backups(manifests: Iterable[backup_manifest.BackupManifest], backup.location = manifest.location.model_dump(mode="json") # Import used to drop this, so an imported encrypted backup restored as # use_crypto=False -- a plaintext volume over ciphertext, silently. - backup.encrypted = manifest.encrypted + backup.encrypted = manifest.encryption.encrypted + backup.encryption = manifest.encryption.model_dump(mode="json") backup.write_to_db() return len(pending) diff --git a/simplyblock_core/models/backup.py b/simplyblock_core/models/backup.py index 6cbdc49e4..d78b21275 100644 --- a/simplyblock_core/models/backup.py +++ b/simplyblock_core/models/backup.py @@ -47,6 +47,10 @@ class Backup(BaseModel): #: pydantic models; read it through :meth:`get_location`. location: dict = {} encrypted: bool = False + #: Which KMS holds this backup's key, and under what path. A + #: ``backup_manifest.Encryption``; stored as a dict for the same reason + #: ``location`` is. Empty for an unencrypted backup. + encryption: dict = {} def get_id(self): return "%s/%s" % (self.cluster_id, self.uuid) diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index f0ba841e2..658317d0b 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -538,6 +538,7 @@ class BackupDTO(BaseModel): created_at: int completed_at: int source_cluster_id: str + encrypted: bool @staticmethod def from_model(model: Backup): @@ -556,6 +557,7 @@ def from_model(model: Backup): created_at=model.created_at, completed_at=model.completed_at, source_cluster_id=model.source_cluster_id or "", + encrypted=model.encrypted, ) diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index 42aaaa894..0e74cbbdb 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -57,7 +57,8 @@ class _RestoreParams(BaseModel): @api.post('/restore', name='clusters:backups:restore', status_code=202) def restore_backup(cluster: Cluster, parameters: _RestoreParams): return {"lvol_id": backup_controller.restore_backup( - parameters.backup_id, parameters.lvol_name, parameters.pool, target_node_id=parameters.target_node_id)} + parameters.backup_id, parameters.lvol_name, parameters.pool, + target_node_id=parameters.target_node_id)} class _ImportManifests(BaseModel): diff --git a/tests/integration/test_backup_encryption.py b/tests/integration/test_backup_encryption.py new file mode 100644 index 000000000..9b69d9835 --- /dev/null +++ b/tests/integration/test_backup_encryption.py @@ -0,0 +1,207 @@ +"""Encrypted backups: what is recorded about their key, and what a restore reaches. + +An encrypted backup is ciphertext in a bucket; the key is in a KMS. Nothing used +to record which KMS, so the dependency was implicit and only discovered during a +recovery. These tests pin down that it is written down now, that it never carries +key material, and that a restore which cannot reach the key fails instead of +producing a plaintext volume over ciphertext. +""" +import pytest + +from simplyblock_core import backup_manifest +from simplyblock_core.controllers import backup_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.kms import LocalKMS, backup_dek_path, backup_kek_name +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings + + +CLUSTER_ID = "cluster-1" +KEYS = ("a" * 64, "b" * 64) + + +def _config(**overrides): + return BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", + "region": "eu-central-1", + **overrides, + }) + + +@pytest.fixture +def db(): + return DBController() + + +def _cluster(db, **config_overrides): + c = Cluster() + c.uuid = CLUSTER_ID + c.backup_config = _config(**config_overrides).model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +def _descriptor(**overrides): + return { + "kms": "local", + "dek_path": backup_dek_path(CLUSTER_ID, "b-1"), + "kek_name": backup_kek_name("b-1"), + **overrides, + } + + +def _backup(db, uuid="b-1", encrypted=True, encryption=None): + b = Backup() + b.uuid = uuid + b.s3_id = 1 + b.cluster_id = CLUSTER_ID + b.source_cluster_id = CLUSTER_ID + b.lvol_id = "lvol-1" + b.lvol_name = "vol" + b.size = 4096 + b.status = Backup.STATUS_COMPLETED + b.location = _config().location().model_dump(exclude_none=True) + b.encrypted = encrypted + b.encryption = encryption or {} + b.write_to_db(db.kv_store) + return b + + +class TestKeyDescriptor: + + def test_local_kms_is_recorded_as_such(self, db): + cluster = _cluster(db) + backup = _backup(db) + + encryption = backup_controller._build_encryption(cluster, backup) + + assert encryption.descriptor.kms == "local" + + def test_vault_settings_are_recorded(self, db): + cluster = _cluster(db) + cluster.hashicorp_vault_settings = HashicorpVaultSettings() + cluster.hashicorp_vault_settings.base_url = "https://vault.example.com" + cluster.hashicorp_vault_settings.transit_mount = "sb/transit" + cluster.hashicorp_vault_settings.kv_mount = "sb/kv" + backup = _backup(db) + + encryption = backup_controller._build_encryption(cluster, backup) + + assert encryption.descriptor.kms == "hashicorp_vault" + assert encryption.descriptor.vault_base_url == "https://vault.example.com" + assert encryption.descriptor.transit_mount == "sb/transit" + assert encryption.descriptor.kv_mount == "sb/kv" + + def test_local_kms_records_no_vault_mounts(self, db): + """Absent rather than "" -- they mean nothing for this backend.""" + encryption = backup_controller._build_encryption(_cluster(db), _backup(db)) + + assert encryption.descriptor.vault_base_url is None + assert encryption.descriptor.transit_mount is None + assert encryption.descriptor.kv_mount is None + + def test_descriptor_points_at_the_key_path(self, db): + cluster = _cluster(db) + backup = _backup(db) + + encryption = backup_controller._build_encryption(cluster, backup) + + assert encryption.descriptor.dek_path == backup_dek_path(CLUSTER_ID, "b-1") + assert encryption.descriptor.kek_name == backup_kek_name("b-1") + + def test_descriptor_carries_no_key_material(self, db): + cluster = _cluster(db) + backup = _backup(db) + with LocalKMS(cluster) as kms: + kms.import_data_encryption_keys( + backup_dek_path(CLUSTER_ID, "b-1"), backup_kek_name("b-1"), KEYS) + + encryption = backup_controller._build_encryption(cluster, backup) + + assert KEYS[0] not in encryption.model_dump_json() + + def test_an_unknown_backend_is_refused_rather_than_guessed(self, db): + """Which fields of a descriptor mean anything depends on the backend.""" + with pytest.raises(ValueError): + backup_manifest.KeyDescriptor.model_validate(_descriptor(kms="something-new")) + + +class TestEncryptionDocument: + + def test_an_encrypted_backup_must_say_where_its_key_is(self): + """Without that, nothing can decrypt it and nothing can say why.""" + with pytest.raises(ValueError): + backup_manifest.Encryption(encrypted=True) + + def test_an_unencrypted_backup_has_no_key_to_describe(self): + with pytest.raises(ValueError): + backup_manifest.Encryption.model_validate( + {"encrypted": False, "descriptor": _descriptor()}) + + def test_manifest_cannot_disagree_with_the_backup(self, db): + """Backup.encrypted is authoritative; the stored sub-document is a copy. + + Two places recording the same fact can drift, and drift here means a + restore reads the wrong one. build_manifest overlays the authoritative + value so a manifest can never carry the stale copy. + """ + _cluster(db) + backup = _backup(db, encrypted=True, + encryption={"encrypted": False, "descriptor": _descriptor()}) + + manifest = backup_controller.build_manifest(backup) + + assert manifest.encryption.encrypted is True + + def test_survives_a_manifest_round_trip(self, db): + cluster = _cluster(db) + backup = _backup(db) + backup.encryption = backup_controller._build_encryption( + cluster, backup).model_dump(exclude_none=True) + backup.write_to_db(db.kv_store) + + parsed = backup_manifest.BackupManifest.model_validate( + backup_controller.build_manifest(backup).model_dump(mode="json")) + + assert parsed.encryption.descriptor.dek_path == backup_dek_path(CLUSTER_ID, "b-1") + + +class TestKeyResolutionOnRestore: + + def test_unencrypted_backup_needs_no_key(self, db): + cluster = _cluster(db) + assert backup_controller._resolve_crypto_key( + _backup(db, encrypted=False), cluster) is None + + def test_the_key_comes_from_the_kms_the_descriptor_names(self, db): + cluster = _cluster(db) + backup = _backup(db, encryption={"encrypted": True, "descriptor": _descriptor()}) + with LocalKMS(cluster) as kms: + kms.import_data_encryption_keys( + backup_dek_path(CLUSTER_ID, "b-1"), backup_kek_name("b-1"), KEYS) + + assert backup_controller._resolve_crypto_key(backup, cluster) == KEYS + + def test_unreachable_key_names_what_is_missing(self, db): + """The operator needs to know which cluster and KMS held the key.""" + cluster = _cluster(db) + backup = _backup(db, encryption={ + "encrypted": True, + "descriptor": _descriptor(dek_path="cluster/gone/backup/b-1")}) + + with pytest.raises(RuntimeError) as excinfo: + backup_controller._resolve_crypto_key(backup, cluster) + + message = str(excinfo.value) + assert "cluster/gone/backup/b-1" in message + assert "local" in message + + def test_encrypted_backup_with_no_encryption_record_is_refused(self, db): + """Rather than silently restoring a plaintext volume over ciphertext.""" + cluster = _cluster(db) + backup = _backup(db, encrypted=True, encryption={}) + + with pytest.raises(PreconditionError, match="records nothing about its"): + backup_controller._resolve_crypto_key(backup, cluster) diff --git a/tests/integration/test_backup_manifest_flow.py b/tests/integration/test_backup_manifest_flow.py index 62322030b..53b2981b1 100644 --- a/tests/integration/test_backup_manifest_flow.py +++ b/tests/integration/test_backup_manifest_flow.py @@ -179,7 +179,8 @@ class TestExportImportRoundTrip: def test_backups_survive_losing_the_database(self, db, cluster, lvol): _backup(db, "b-1", 1) - _backup(db, "b-2", 2, prev="b-1", encrypted=True) + _backup(db, "b-2", 2, prev="b-1", encrypted=True, + encryption={"encrypted": True, "descriptor": {"kms": "local"}}) exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) for backup in db.get_backups(): @@ -199,7 +200,8 @@ def test_backups_survive_losing_the_database(self, db, cluster, lvol): def test_encrypted_flag_survives(self, db, cluster, lvol): """It used to be dropped, restoring a plaintext volume over ciphertext.""" - _backup(db, "b-1", 1, encrypted=True) + _backup(db, "b-1", 1, encrypted=True, + encryption={"encrypted": True, "descriptor": {"kms": "local"}}) exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) db.get_backup_by_id("b-1").remove(db.kv_store) diff --git a/tests/unit/test_backup_manifest.py b/tests/unit/test_backup_manifest.py index 428a0438f..529876692 100644 --- a/tests/unit/test_backup_manifest.py +++ b/tests/unit/test_backup_manifest.py @@ -10,6 +10,8 @@ from simplyblock_core.backup_manifest import ( BackupManifest, DataPlane, + Encryption, + KeyDescriptor, ManifestError, MANIFEST_SCHEMA_VERSION, Source, @@ -28,7 +30,7 @@ def _manifest(**overrides): "created_at": 100, "completed_at": 200, "size": 4096, - "encrypted": False, + "encryption": Encryption(encrypted=False), "location": BackupLocation.model_validate(LOCATION), "source": Source(cluster_id="c-1", node_id="n-1"), "volume": Volume(lvol_id="l-1", lvol_name="vol", snapshot_id="s-1", @@ -49,7 +51,9 @@ def test_key_cannot_collide_with_the_data_plane_keyspace(self): class TestSchema: def test_round_trip(self): - original = _manifest(encrypted=True, prev_backup_id="b-0") + original = _manifest(prev_backup_id="b-0", encryption=Encryption( + encrypted=True, + descriptor=KeyDescriptor(kms="local", dek_path="p", kek_name="k"))) restored = backup_manifest._parse(original.model_dump_json().encode(), "k") diff --git a/tests/unit/web/api/v2/test_backup_endpoints.py b/tests/unit/web/api/v2/test_backup_endpoints.py index d6902ee68..2c790fb11 100644 --- a/tests/unit/web/api/v2/test_backup_endpoints.py +++ b/tests/unit/web/api/v2/test_backup_endpoints.py @@ -83,6 +83,25 @@ def test_restores_backup(self, client, db, cluster, backup_controller): backup_controller.restore_backup.assert_called_once_with( BACKUP_ID, 'restored-volume', 'pool-1', target_node_id=None) + def test_a_precondition_error_is_not_mapped_here(self, client, db, cluster, + backup_controller): + """app.py maps PreconditionError to 400 for the whole API; a second, + disagreeing mapping in this one router made it inconsistent with itself. + + (This test app deliberately mounts only the routers, so an unhandled + exception surfaces here instead of reaching that handler.) + """ + import pytest + from simplyblock_core.exceptions import PreconditionError + backup_controller.restore_backup.side_effect = PreconditionError('node offline') + + with pytest.raises(PreconditionError): + client.post(f'{BASE}/restore', json={ + 'backup_id': BACKUP_ID, + 'lvol_name': 'restored-volume', + 'pool': 'pool-1', + }) + class TestImportBackups: """The body is a union of two shapes, not one model with everything optional.""" @@ -94,7 +113,7 @@ class TestImportBackups: 'created_at': 100, 'completed_at': 200, 'size': 4096, - 'encrypted': False, + 'encryption': {'encrypted': False}, 'location': {'bucket_name': 'backups', 'region': 'eu-central-1'}, 'source': {'cluster_id': CLUSTER_ID, 'node_id': 'node-1'}, 'volume': {'lvol_id': VOLUME_ID, 'lvol_name': 'vol', From 9ba836b71c555c0277eadb83bbb2183fe4b16861 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 11:27:44 +0200 Subject: [PATCH 06/14] Refuse unrestorable backups up front rather than at recovery time Every rule here already existed implicitly, enforced by whatever failed first -- usually the data plane, usually mid-operation, sometimes not until someone tried the restore the backup was taken for. They are now checked at the earliest layer that can check them, before any side effect. The rules are predicates: chain_fits(length) the data plane copies the decoded array into a fixed 40-element stack buffer (vbdev_lvol_rpc.c), so beyond BACKUP_MAX_CHAIN_LENGTH it smashes the storage node's stack. Refusing here is the only guard until those buffers are sized properly. location_holds_backups(loc) snapshot_backups=False selects the secondary-tiering key layout {tiering_id}/{lpgi}; a backup written there is unreadable by a restore, which addresses {s3_id}/{mid}/{extent}. chain_is_coherent(...) a restore reads clusters from the whole chain in one operation, against one bucket, with one key. Nothing in the stack could express a chain split across buckets or half encrypted. This is what silently broke when a cluster's bucket was reconfigured mid-chain. They return booleans, so a rule can answer a question as well as block an operation -- "can this bucket hold backups" is a thing a caller may want to know without being refused. require_restorable is the one place that turns a false answer into a PreconditionError, so the wording an operator sees is written once rather than at each of the three entry points that enforce the same three rules. Applied at backup creation before the chain lock, before any KMS key and before any task; at restore before the volume is created, so a doomed restore leaves no half-built volume behind; and at import before the first record is written. Import needs one rule of its own, so it has its own gate. Whether every ancestor is either in this batch or already in the database is a question only the importer can ask, and the answer is what stops an import from landing a delta whose ancestors are missing -- something that looks restorable in `backup list` and fails only when tried, typically during the recovery it was meant to serve. It walks prev_backup_id through the batch, continues into the database when a link lands on a record already there (so existing ancestry counts towards the length the data plane has to accept), and refuses a cycle rather than looping on it. Tests assert the absence of side effects, not just the error: no Backup record, no task, no chain lock, no volume. The predicates are also tested directly, at their boundaries. TestRestoreBackup is converted off the stubbed-DB pattern onto real FoundationDB, per tests/AGENTS.md; add_lvol_ha and the task runner stay mocked, since they sit above the database and drive RPC. --- simplyblock_core/constants.py | 6 + .../controllers/backup_controller.py | 206 ++++++++++- tests/integration/test_backup.py | 98 ++--- tests/integration/test_backup_validation.py | 349 ++++++++++++++++++ .../test_backup_restore_node_selection.py | 4 + 5 files changed, 619 insertions(+), 44 deletions(-) create mode 100644 tests/integration/test_backup_validation.py diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index f5c404e09..ef81e7a72 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -606,6 +606,12 @@ def get_config_var(name, default=None): BACKUP_MAX_RETRIES = 10 BACKUP_MERGE_SERVICE_INTERVAL_SEC = 60 +#: Longest backup chain the data plane will accept. bdev_lvol_s3_backup and +#: bdev_lvol_s3_recovery copy the decoded arrays into fixed 40-element stack +#: buffers (vbdev_lvol_rpc.c), so a longer chain corrupts the node's stack. The +#: control plane refuses first; raise this only together with those buffers. +BACKUP_MAX_CHAIN_LENGTH = 40 + #: Upper bound on a backup's s3_id. The data plane packs it into bits 33..62 of #: the synthetic bdev offset (S3_ID_BITS in spdk_internal/lvolstore.h) and masks #: rather than validates, so a larger value silently aliases onto another diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index af8053b95..c8d55a60f 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -101,6 +101,122 @@ def _compute_s3_cpu_masks(node): return bdb_lcpu_mask, s3_lcpu_mask +# --- Restorability rules --------------------------------------------------- +# +# Predicates, so the same rule can answer a question ("can this bucket hold +# backups?") as well as block an operation. `require_restorable` is the one place +# that turns a false answer into a refusal, so the wording an operator sees is +# written once rather than at each of the entry points that enforce the rules. + + +def chain_fits(length: int) -> bool: + """Whether the data plane can accept a chain this long. + + Beyond this it copies the decoded array into a fixed stack buffer, smashing + the storage node's stack rather than returning an error. + """ + return length <= constants.BACKUP_MAX_CHAIN_LENGTH + + +def location_holds_backups(location: BackupLocation) -> bool: + """Whether backups written to this location could be read back. + + ``snapshot_backups=False`` selects the secondary-tiering object layout, whose + keys are ``{tiering_id}/{lpgi}``. The restore path addresses + ``{s3_id}/{mid}/{extent}``, so it can never find them. + """ + return location.snapshot_backups + + +def chain_is_coherent(backups, location: BackupLocation, + encrypted: Optional[bool] = None) -> bool: + """Whether these backups can be restored together. + + A restore reads clusters from every backup in the chain in one operation, + against one bucket, decrypting all of it with one key. So the chain has to + agree on where it lives, how it is encoded, and whether it is encrypted -- + nothing anywhere in the stack could express a chain split across two buckets + or half encrypted. + + ``encrypted`` folds in a backup that does not exist yet, which is the case at + creation time. + """ + if any(backup.get_location() != location for backup in backups): + return False + + variants = {backup.encrypted for backup in backups} + if encrypted is not None: + variants.add(encrypted) + return len(variants) <= 1 + + +def _describe_incoherence(backups, location: BackupLocation, + encrypted: Optional[bool]) -> str: + for backup in backups: + if backup.get_location() != location: + return ( + f"backup {backup.uuid} lives in bucket " + f"{backup.get_location().bucket_name}, but the rest of its chain " + f"is in {location.bucket_name}. A chain cannot span buckets or " + "encodings; start a new chain with a full backup") + + return ( + "a chain cannot mix encrypted and unencrypted backups: " + + ", ".join(f"{b.uuid}={'encrypted' if b.encrypted else 'plain'}" + for b in backups) + + (f", new backup={'encrypted' if encrypted else 'plain'}" + if encrypted is not None else "")) + + +def require_restorable(location: BackupLocation, backups=(), + chain_length: Optional[int] = None, + encrypted: Optional[bool] = None, + what: str = "This chain") -> None: + """Refuse a chain that could not be restored, naming the rule it breaks. + + Applied at creation, at import and at restore, because each is a point where + a chain could otherwise become unrestorable without anyone noticing -- and + each used to find out from whatever failed first, usually the data plane + mid-operation. + + Args: + chain_length: The eventual length, where it differs from ``len(backups)`` + -- at creation the ancestors are snapshots that have no backup yet. + encrypted: Whether the backup about to be created will be encrypted. + + Raises: + PreconditionError: One of the rules above does not hold. + """ + if not location_holds_backups(location): + raise PreconditionError( + f"Bucket {location.bucket_name} is configured with snapshot_backups " + "disabled, which selects the secondary-tiering object layout. " + "Backups cannot be written there.") + + length = len(backups) if chain_length is None else chain_length + if not chain_fits(length): + raise PreconditionError( + f"{what} is {length} backups long; the data plane accepts at most " + f"{constants.BACKUP_MAX_CHAIN_LENGTH}. Merge older backups to " + "shorten the chain, or start a new chain with a full backup.") + + if not chain_is_coherent(backups, location, encrypted): + raise PreconditionError( + f"{what} cannot be restored as a unit: " + + _describe_incoherence(backups, location, encrypted)) + + +def _existing_chain_backups(snap_chain) -> list: + """The backups that already exist for a snapshot chain, oldest first.""" + existing = [] + for snap in snap_chain: + for backup in db_controller.get_backups_by_snapshot_id(snap.get_id()): + if backup.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS, + Backup.STATUS_COMPLETED): + existing.append(backup) + return existing + + def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: """Assemble the self-describing record for a completed backup. @@ -516,12 +632,24 @@ def backup_snapshot(snapshot_id, cluster_id=None): if not cluster_id: cluster_id = snode.cluster_id + snap_chain = _get_snapshot_chain(snapshot) + + # Everything that could make this backup unrestorable is checked here, + # before the chain lock is taken, before any KMS key is created and before + # any task is enqueued. A backup either is restorable or was never created. try: location = db_controller.get_cluster_by_id(cluster_id).get_backup_config().location() - except (KeyError, ValueError) as e: + require_restorable( + location, + backups=_existing_chain_backups(snap_chain), + # Every snapshot in the chain gets a backup below, including the ones + # that have none yet, so the eventual length is the chain's. + chain_length=len(snap_chain), + encrypted=bool(lvol.crypto_bdev), + what="This snapshot chain") + except (KeyError, ValueError, PreconditionError) as e: return None, str(e) - snap_chain = _get_snapshot_chain(snapshot) chain_snapshot_ids = [snap.get_id() for snap in snap_chain] acquired, existing_lock = db_controller.acquire_backup_chain_locks( chain_snapshot_ids, snapshot_id, lvol.get_id()) @@ -610,6 +738,12 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, f"is {active_src[:8]}. Use 'sbctl backup source-switch " f"{backup_src}' first.") + # The chain has to be restorable as a unit, and short enough for the data + # plane. Checked before the volume is created so a doomed restore leaves + # nothing behind. + require_restorable(backup.get_location(), backups=chain, + what=f"The chain ending at backup {backup_id}") + size = backup.size if size <= 0: raise PreconditionError("Backup has no size information") @@ -802,6 +936,72 @@ def discover_backups(config: BackupConfig) -> List[backup_manifest.BackupManifes return backup_manifest.list_all(config) +def _require_importable(pending: dict) -> None: + """Refuse a batch of manifests that would not form something restorable. + + An import that lands a backup whose ancestors are missing produces a record + that looks restorable in ``backup list`` and fails only when someone tries it + -- typically during the recovery it was meant to serve. Each manifest names + its predecessor, so this is answerable up front by walking the batch. + + Separate from ``require_restorable`` because the rule is about the batch, not + about a chain: whether every ancestor is either in this import or already in + the database is a question only the importer can ask. The rules that ARE about + the chain are deferred to it. + + Raises: + PreconditionError: A chain is incomplete, cyclic, too long for the data + plane, or spans buckets. + """ + for backup_id, manifest in pending.items(): + chain, seen = [manifest], {backup_id} + + while (previous := chain[-1].prev_backup_id) is not None: + if previous in seen: + raise PreconditionError( + f"Backup {backup_id} has a cyclic chain at {previous}") + seen.add(previous) + + if previous in pending: + chain.append(pending[previous]) + continue + + if not _backup_exists(previous): + raise PreconditionError( + f"Backup {backup_id} is a delta against {previous}, which is " + "neither in this import nor already known. A backup cannot " + "be restored without its chain.") + + # Already imported, and checked then. Its own ancestry still counts + # towards the length the data plane has to accept. + chain.extend(db_controller.get_backup_chain(previous)) + break + + # Manifests carry their location as a value, so coherence over the batch + # is a plain comparison; require_restorable wants Backup records, and + # these are not in the database yet. + divergent = next( + (m for m in chain if m.location != manifest.location), None) + if divergent is not None: + raise PreconditionError( + f"Backup {backup_id} shares a chain with {divergent.backup_id}, " + "which is in a different bucket or encoding") + + if not chain_fits(len(chain)): + raise PreconditionError( + f"The chain of backup {backup_id} is {len(chain)} backups long; " + f"the data plane accepts at most " + f"{constants.BACKUP_MAX_CHAIN_LENGTH}.") + + +def _backup_exists(backup_id: str) -> bool: + try: + db_controller.get_backup_by_id(backup_id) + except KeyError: + return False + return True + + def import_backups(manifests: Iterable[backup_manifest.BackupManifest], cluster_id=None) -> int: """Register backups described by manifests into this cluster's database. @@ -840,6 +1040,8 @@ def import_backups(manifests: Iterable[backup_manifest.BackupManifest], else: raise PreconditionError(f"Backup {backup_id} already exists in cluster {existing.cluster_id}") + _require_importable(pending) + for backup_id, manifest in pending.items(): backup = Backup() backup.uuid = backup_id diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index bafe44403..71d96a4cb 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -603,65 +603,79 @@ def test_release_backup_chain_locks_uses_unbound_method_with_db_handle(self, moc # =========================================================================== class TestRestoreBackup(unittest.TestCase): + """Real FDB: restore reads backup/pool/cluster state and creates a volume. - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_success(self, mock_db, mock_tasks): - cluster_uuid = "00000000-0000-0000-0000-000000000001" - backup = _backup(s3_id=5, cluster_id=cluster_uuid) - mock_db.get_backup_by_id.return_value = backup - mock_db.get_backup_chain.return_value = [backup] - mock_tasks.add_backup_restore_task.return_value = True + add_lvol_ha and the task runner are mocked -- they sit above the database + and drive RPC to storage nodes. + """ + + CLUSTER_ID = "00000000-0000-0000-0000-000000000001" + + def setUp(self): + from simplyblock_core.models.pool import Pool + self.db = DBController() + + cluster = _cluster(uuid=self.CLUSTER_ID) + cluster.write_to_db(self.db.kv_store) + + pool = Pool() + pool.uuid = "pool-1" + pool.pool_name = "pool-1" # resolved by name: "pool-1" is not a UUID + pool.cluster_id = self.CLUSTER_ID + pool.write_to_db(self.db.kv_store) - mock_cluster = MagicMock() - mock_cluster.uuid = cluster_uuid - mock_cluster.backup_source = "" - mock_db.get_cluster_by_id.return_value = mock_cluster + def _backup(self, **overrides): + backup = _backup(cluster_id=self.CLUSTER_ID, **overrides) + backup.location = _backup_config().location().model_dump(mode="json") + backup.write_to_db(self.db.kv_store) + return backup - # Mock the lvol created by add_lvol_ha - mock_lvol = MagicMock() - mock_lvol.node_id = "node-1" - mock_lvol.lvs_name = "lvs_test" - mock_lvol.lvol_bdev = "LVOL_123" - mock_lvol.write_to_db = MagicMock() - mock_db.get_lvol_by_id.return_value = mock_lvol + @patch("simplyblock_core.controllers.backup_controller.tasks_controller") + def test_success(self, mock_tasks): + self._backup(s3_id=5) + mock_tasks.add_backup_restore_task.return_value = True - with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha") as mock_add_ha: - mock_add_ha.return_value = ("lvol-new", None) + lvol = LVol() + lvol.uuid = "lvol-new" + lvol.node_id = "node-1" + lvol.lvs_name = "lvs_test" + lvol.lvol_bdev = "LVOL_123" + lvol.pool_uuid = "pool-1" + lvol.write_to_db(self.db.kv_store) + with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha", + return_value=("lvol-new", None)): from simplyblock_core.controllers.backup_controller import restore_backup result = restore_backup("backup-1", "restored_lvol", "pool-1") self.assertEqual(result, "lvol-new") - # Verify s3_id integers are passed, not UUIDs - call_args = mock_tasks.add_backup_restore_task.call_args - self.assertEqual(call_args[0][4], [5]) - - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_backup_not_found(self, mock_db): - mock_db.get_backup_by_id.side_effect = KeyError("not found") + # s3_id integers reach the data plane, not backup UUIDs. + self.assertEqual(mock_tasks.add_backup_restore_task.call_args[0][4], [5]) + self.assertEqual(self.db.get_lvol_by_id("lvol-new").status, LVol.STATUS_RESTORING) + def test_backup_not_found(self): from simplyblock_core.controllers.backup_controller import restore_backup with self.assertRaises(PreconditionError): restore_backup("missing", "lvol", "pool-1") - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_add_lvol_ha_fails(self, mock_db): - cluster_uuid = "00000000-0000-0000-0000-000000000001" - mock_db.get_backup_by_id.return_value = _backup(cluster_id=cluster_uuid) - mock_db.get_backup_chain.return_value = [_backup(cluster_id=cluster_uuid)] - - mock_cluster = MagicMock() - mock_cluster.uuid = cluster_uuid - mock_cluster.backup_source = "" - mock_db.get_cluster_by_id.return_value = mock_cluster - - with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha") as mock_add_ha: - mock_add_ha.return_value = (None, "Pool not found") + def test_add_lvol_ha_fails(self): + self._backup() + with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha", + return_value=(None, "Pool not found")): from simplyblock_core.controllers.backup_controller import restore_backup with self.assertRaisesRegex(RuntimeError, "Failed to create restore volume"): - restore_backup("backup-1", "lvol", "bad-pool") + restore_backup("backup-1", "lvol", "pool-1") + + def test_incomplete_chain_is_refused(self): + self._backup(uuid="b-old", s3_id=1, status=Backup.STATUS_IN_PROGRESS) + self._backup(uuid="backup-1", s3_id=2, prev_backup_id="b-old") + + from simplyblock_core.controllers.backup_controller import restore_backup + with self.assertRaisesRegex(PreconditionError, "Incomplete backups in chain"): + restore_backup("backup-1", "lvol", "pool-1") + + self.assertEqual(self.db.get_lvols(), []) # =========================================================================== diff --git a/tests/integration/test_backup_validation.py b/tests/integration/test_backup_validation.py new file mode 100644 index 000000000..8d3c886cb --- /dev/null +++ b/tests/integration/test_backup_validation.py @@ -0,0 +1,349 @@ +"""Preconditions on backup creation, restore and import, against real FoundationDB. + +Every rule here has the same shape: it must fire *before* any side effect. A +refused backup must leave no Backup record, no KMS key and no task; a refused +restore must leave no volume. The point is that a backup either is restorable or +was never created -- discovering the problem during a recovery is too late. +""" +from unittest.mock import patch + +import pytest + +from simplyblock_core import constants +from simplyblock_core.backup_manifest import BackupManifest +from simplyblock_core.controllers import backup_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.pool import Pool +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode + + +CLUSTER_ID = "cluster-1" + + +def _config(**overrides): + return BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", + "region": "eu-central-1", + **overrides, + }) + + +@pytest.fixture +def db(): + return DBController() + + +@pytest.fixture +def cluster(db): + c = Cluster() + c.uuid = CLUSTER_ID + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +@pytest.fixture +def pool(db): + p = Pool() + p.uuid = "pool-1" + p.pool_name = "pool-1" # resolved by name: "pool-1" is not a UUID + p.cluster_id = CLUSTER_ID + p.write_to_db(db.kv_store) + return p + + +@pytest.fixture +def node(db): + n = StorageNode() + n.uuid = "node-1" + n.cluster_id = CLUSTER_ID + n.status = StorageNode.STATUS_ONLINE + n.lvstore = "lvs_test" + n.mgmt_ip = "10.0.0.1" + n.rpc_port = 5260 + n.write_to_db(db.kv_store) + return n + + +def _snapshot(db, uuid="snap-1", crypto=False): + volume = LVol() + volume.uuid = "lvol-1" + volume.lvol_name = "vol" + volume.node_id = "node-1" + volume.lvs_name = "lvs_test" + volume.pool_uuid = "pool-1" + volume.size = 4096 + if crypto: + volume.crypto_bdev = "crypto_lvol-1" + volume.write_to_db(db.kv_store) + + s = SnapShot() + s.uuid = uuid + s.snap_uuid = uuid + s.snap_name = uuid + s.snap_bdev = f"lvs_test/{uuid}" + s.size = 4096 + s.status = SnapShot.STATUS_ONLINE + s.lvol = volume + s.write_to_db(db.kv_store) + return s + + +def _backup(db, uuid, s3_id, snapshot_id, prev="", location=None, encrypted=False): + b = Backup() + b.uuid = uuid + b.s3_id = s3_id + b.cluster_id = CLUSTER_ID + b.source_cluster_id = CLUSTER_ID + b.lvol_id = "lvol-1" + b.lvol_name = "vol" + b.snapshot_id = snapshot_id + b.prev_backup_id = prev + b.size = 4096 + b.status = Backup.STATUS_COMPLETED + b.location = (location or _config().location()).model_dump(exclude_none=True) + b.encrypted = encrypted + b.write_to_db(db.kv_store) + return b + + +def _assert_no_side_effects(db): + assert db.get_backups() == [], "a refused backup must leave no record" + assert db.get_job_tasks(CLUSTER_ID) == [], "a refused backup must enqueue no task" + + +class TestBackupCreationPreconditions: + + def test_missing_backup_config_is_refused(self, db, cluster, node): + cluster.backup_config = {} + cluster.write_to_db(db.kv_store) + _snapshot(db) + + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "backup configuration" in error + _assert_no_side_effects(db) + + def test_tiering_layout_bucket_is_refused(self, db, cluster, node): + """snapshot_backups=False selects {tiering_id}/{lpgi}, which restore cannot read.""" + cluster.backup_config = _config(snapshot_backups=False).model_dump(exclude_none=True) + cluster.write_to_db(db.kv_store) + _snapshot(db) + + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "snapshot_backups disabled" in error + _assert_no_side_effects(db) + + def test_overlong_chain_is_refused(self, db, cluster, node): + """Longer than the data plane's fixed arrays, where it smashes the stack.""" + snap = _snapshot(db) + too_long = [snap] * (constants.BACKUP_MAX_CHAIN_LENGTH + 1) + + with patch.object(backup_controller, "_get_snapshot_chain", return_value=too_long): + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "data plane accepts at most" in error + _assert_no_side_effects(db) + + def test_chain_in_another_bucket_is_refused(self, db, cluster, node): + """The cluster's bucket changed since the ancestors were written.""" + snap = _snapshot(db) + _backup(db, "b-old", 1, snapshot_id="snap-0", + location=_config(bucket_name="the-old-bucket").location()) + + with patch.object(backup_controller, "_get_snapshot_chain", + return_value=[_named(snap, "snap-0"), snap]): + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "cannot span buckets" in error + assert db.get_backups() == [db.get_backup_by_id("b-old")] + + def test_encrypted_volume_over_a_plain_chain_is_refused(self, db, cluster, node): + snap = _snapshot(db, crypto=True) + _backup(db, "b-plain", 1, snapshot_id="snap-0", encrypted=False) + + with patch.object(backup_controller, "_get_snapshot_chain", + return_value=[_named(snap, "snap-0"), snap]): + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "cannot mix encrypted and unencrypted" in error + + def test_refusal_happens_before_the_chain_lock(self, db, cluster, node): + """Otherwise a refused request would block the next one.""" + cluster.backup_config = {} + cluster.write_to_db(db.kv_store) + _snapshot(db) + + backup_controller.backup_snapshot("snap-1") + + assert db.get_backup_chain_lock("snap-1") is None + + +def _named(snapshot, uuid): + """A shallow copy of a snapshot under a different id, for chain fixtures.""" + clone = SnapShot() + clone.from_dict(snapshot.to_dict()) + clone.uuid = uuid + clone.snap_uuid = uuid + return clone + + +class TestRestorePreconditions: + + def test_chain_spanning_buckets_is_refused(self, db, cluster, node, pool): + _backup(db, "b-1", 1, snapshot_id="snap-1", + location=_config(bucket_name="elsewhere").location()) + _backup(db, "b-2", 2, snapshot_id="snap-2", prev="b-1") + + with pytest.raises(PreconditionError, match="cannot span buckets"): + backup_controller.restore_backup("b-2", "restored", "pool-1") + + def test_chain_mixing_encryption_is_refused(self, db, cluster, node, pool): + _backup(db, "b-1", 1, snapshot_id="snap-1", encrypted=True) + _backup(db, "b-2", 2, snapshot_id="snap-2", prev="b-1", encrypted=False) + + with pytest.raises(PreconditionError, match="mix encrypted and unencrypted"): + backup_controller.restore_backup("b-2", "restored", "pool-1") + + def test_overlong_chain_is_refused(self, db, cluster, node, pool): + previous = "" + for index in range(constants.BACKUP_MAX_CHAIN_LENGTH + 1): + previous = _backup(db, f"b-{index}", index + 1, + snapshot_id=f"snap-{index}", prev=previous).uuid + + with pytest.raises(PreconditionError, match="data plane accepts at most"): + backup_controller.restore_backup(previous, "restored", "pool-1") + + def test_no_volume_is_created_when_a_precondition_fails(self, db, cluster, node, pool): + _backup(db, "b-1", 1, snapshot_id="snap-1", encrypted=True) + _backup(db, "b-2", 2, snapshot_id="snap-2", prev="b-1", encrypted=False) + + with pytest.raises(PreconditionError): + backup_controller.restore_backup("b-2", "restored", "pool-1") + + assert db.get_lvols() == [] + + +class TestImportPreconditions: + + def _manifest(self, backup_id, prev=None, s3_id=1, + bucket="simplyblock-backup-cluster-1"): + return BackupManifest.model_validate({ + "schema_version": 1, + "backup_id": backup_id, + "s3_id": s3_id, + "created_at": 100, + "completed_at": 200, + "size": 4096, + "encrypted": False, + "prev_backup_id": prev, + "location": _config(bucket_name=bucket).location().model_dump(mode="json"), + "source": {"cluster_id": CLUSTER_ID, "node_id": "node-1"}, + "volume": {"lvol_id": "lvol-1", "lvol_name": "vol", + "snapshot_id": f"snap-{backup_id}", "snapshot_name": "s", + "size": 4096}, + "dataplane": {}, + }) + + def _line(self, length, **overrides): + """A chain of `length` manifests, oldest first.""" + line, prev = [], None + for index in range(length): + line.append(self._manifest(f"b-{index}", prev=prev, s3_id=index + 1, + **overrides)) + prev = line[-1].backup_id + return line + + def test_incomplete_chain_is_refused(self, db, cluster): + """A delta whose ancestors are missing looks restorable until it is tried.""" + with pytest.raises(PreconditionError, match="neither in this import nor already known"): + backup_controller.import_backups( + [self._manifest("b-2", prev="b-1")], cluster_id=CLUSTER_ID) + + assert db.get_backups() == [] + + def test_chain_satisfied_within_the_batch_is_accepted(self, db, cluster): + count = backup_controller.import_backups(self._line(2), cluster_id=CLUSTER_ID) + + assert count == 2 + + def test_chain_satisfied_by_existing_records_is_accepted(self, db, cluster): + _backup(db, "b-1", 1, snapshot_id="snap-1") + + count = backup_controller.import_backups( + [self._manifest("b-2", prev="b-1")], cluster_id=CLUSTER_ID) + + assert count == 1 + + def test_chain_spanning_buckets_is_refused(self, db, cluster): + with pytest.raises(PreconditionError, match="different bucket or encoding"): + backup_controller.import_backups( + [self._manifest("b-1", bucket="elsewhere"), + self._manifest("b-2", prev="b-1")], + cluster_id=CLUSTER_ID) + + assert db.get_backups() == [] + + def test_overlong_chain_is_refused(self, db, cluster): + with pytest.raises(PreconditionError, match="data plane accepts at most"): + backup_controller.import_backups( + self._line(constants.BACKUP_MAX_CHAIN_LENGTH + 1), + cluster_id=CLUSTER_ID) + + assert db.get_backups() == [] + + def test_a_chain_lengthened_past_the_limit_by_existing_records_is_refused( + self, db, cluster): + """The ancestry already in the database counts towards the limit.""" + previous = "" + for index in range(constants.BACKUP_MAX_CHAIN_LENGTH): + previous = _backup(db, f"old-{index}", index + 1, + snapshot_id=f"snap-old-{index}", prev=previous).uuid + + with pytest.raises(PreconditionError, match="data plane accepts at most"): + backup_controller.import_backups( + [self._manifest("b-new", prev=previous, s3_id=999)], + cluster_id=CLUSTER_ID) + + def test_a_cyclic_chain_is_refused_rather_than_looping(self, db, cluster): + with pytest.raises(PreconditionError, match="cyclic"): + backup_controller.import_backups( + [self._manifest("b-1", prev="b-2"), self._manifest("b-2", prev="b-1")], + cluster_id=CLUSTER_ID) + + +class TestPredicates: + """The rules answer a yes/no question as well as blocking an operation.""" + + def test_chain_fits_at_the_limit(self): + assert backup_controller.chain_fits(constants.BACKUP_MAX_CHAIN_LENGTH) + assert not backup_controller.chain_fits(constants.BACKUP_MAX_CHAIN_LENGTH + 1) + + def test_a_tiering_bucket_holds_no_backups(self): + assert backup_controller.location_holds_backups(_config().location()) + assert not backup_controller.location_holds_backups( + _config(snapshot_backups=False).location()) + + def test_an_empty_chain_is_coherent(self): + assert backup_controller.chain_is_coherent([], _config().location()) + + def test_coherence_covers_a_backup_that_does_not_exist_yet(self, db, cluster): + chain = [_backup(db, "b-1", 1, snapshot_id="snap-1", encrypted=False)] + + assert backup_controller.chain_is_coherent(chain, _config().location()) + assert backup_controller.chain_is_coherent( + chain, _config().location(), encrypted=False) + assert not backup_controller.chain_is_coherent( + chain, _config().location(), encrypted=True) diff --git a/tests/unit/test_backup_restore_node_selection.py b/tests/unit/test_backup_restore_node_selection.py index fa089b8b7..cd5d4a7bf 100644 --- a/tests/unit/test_backup_restore_node_selection.py +++ b/tests/unit/test_backup_restore_node_selection.py @@ -20,6 +20,9 @@ SOURCE_CLUSTER = "00000000-0000-0000-0000-00000000000f" +LOCATION = {"bucket_name": "backups", "region": "eu-central-1"} + + def _backup(node_id, cluster_id=TARGET_CLUSTER, source_cluster_id=""): backup = Backup() backup.uuid = "backup-1" @@ -29,6 +32,7 @@ def _backup(node_id, cluster_id=TARGET_CLUSTER, source_cluster_id=""): backup.source_cluster_id = source_cluster_id backup.size = 1024 backup.status = Backup.STATUS_COMPLETED + backup.location = dict(LOCATION) return backup From 0624ac0b5787fdd80df01698115dbd8a6939743a Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 12:40:06 +0200 Subject: [PATCH 07/14] Restore from a bucket that is not the cluster's own An S3 device holds exactly one bucket with one set of credentials, so reading another cluster's bucket means attaching another device. The lvstore already supports that -- its transfer devices are a list -- so a restore now attaches a device for the backup's own recorded location, reads through it, and drops it again. This is what the recorded location was for. The device's whole lifecycle belongs to the task runner, not to the restore request. Two reasons, either sufficient: * The node is not known when the restore is requested. target_node_id may be None, in which case add_lvol_ha chooses the node; there is nothing to attach a device to until it has. * A node restart mid-restore takes the device with it. _run_restore already re-issues after STATUS_SUSPENDED, so the runner is the only component that can put the device back -- and it already owned teardown, so creation belongs with it. The device name is derived from the backup id, so a retry re-derives the same name rather than leaking a device per attempt, and creation is re-run on every attempt instead of once. Cleanup happens on every terminal path, including the two that abandon the restore because the volume was deleted underneath it, and the timeout / retry-ceiling path in _terminate_task. It is best-effort and logged rather than raised: a cleanup failure must not turn a completed restore into a failed one. A leaked device is worth noticing though, since a non-empty transfer_devs list blocks the lvstore from being destroyed (vbdev_lvol.c:502). Credentials for a foreign bucket travel in the task's parameters, because the runner needs them on every attempt. They are scrubbed when the restore reaches a terminal state -- a task record is retained for weeks afterwards, and another cluster's S3 keys have no business outliving the restore that needed them. foreign_bucket_config returns one value and raises on error, rather than the tuple-plus-flag it started as. A None result means what it says: there is no foreign bucket, so there is nothing to describe. The device name is no longer threaded through as an empty-string sentinel; the runner derives it, which it can do because it knows the node. Refusing a foreign bucket with no credentials is deliberate: the cluster's own static keys say nothing about someone else's bucket, and falling back to them fails deep in the data plane with nothing pointing at the cause. A cluster with no static credentials at all is a different matter -- there the nodes' instance role is the only answer, and it is allowed through. bdev_lvol_s3_recovery gains an optional s3_bdev, and bdev_s3_delete is added to the RPC client. The data-plane side of that parameter lands in the commit that makes it required; until then the data plane ignores it and picks the first attached S3 device, which is exactly the ambiguity being removed. On the exceptions here: refusing a foreign bucket with no credentials is a PreconditionError in its intended sense -- the caller could have supplied them, and the message says so. The broad `except Exception` around the device teardown is deliberate and logged rather than re-raised, per the reason above; it replaces a `(RPCException, Exception)` tuple that needed a noqa to silence the redundancy it introduced. --- .../controllers/backup_controller.py | 137 +++++++++- .../controllers/tasks_controller.py | 16 +- simplyblock_core/rpc_client.py | 24 +- .../services/tasks_runner_backup.py | 72 +++++- simplyblock_web/api/v2/cluster/backup.py | 8 +- .../integration/test_backup_restore_source.py | 240 ++++++++++++++++++ .../test_backup_restore_node_selection.py | 11 +- .../unit/web/api/v2/test_backup_endpoints.py | 19 +- 8 files changed, 513 insertions(+), 14 deletions(-) create mode 100644 tests/integration/test_backup_restore_source.py diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index c8d55a60f..c862e43b6 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -11,7 +11,8 @@ from simplyblock_core.controllers import backup_events, tasks_controller from simplyblock_core.db_controller import DBController from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment -from simplyblock_core.models.backup_config import BackupConfig, BackupLocation +from simplyblock_core.models.backup_config import ( + BackupConfig, BackupLocation, S3Credentials) from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.kms import ( KMSException, backup_dek_path, backup_kek_name, create_kms_connection, @@ -318,6 +319,125 @@ def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: ) +def primary_s3_bdev_name(node) -> str: + """The S3 device holding the cluster's own backup bucket.""" + return f"s3_{node.lvstore}" + + +def create_restore_s3_bdev(node, config: BackupConfig, name: str) -> None: + """Attach a second S3 device to a node, for a bucket that is not its own. + + A restore from a foreign bucket needs different credentials, a different + endpoint and a different region than the node's own backup device carries. + Since a device holds exactly one bucket, the way to read another one is to + create another device -- which the lvstore supports, its transfer devices + being a list. + + The caller owns the result and must delete it when the restore ends. + """ + rpc_client = node.rpc_client() + bdb_lcpu_mask, s3_lcpu_mask = _compute_s3_cpu_masks(node) + + try: + rpc_client.bdev_s3_create( + name=name, + secondary_target=config.secondary_target, + with_compression=config.with_compression, + snapshot_backups=config.snapshot_backups, + local_testing=config.endpoint is not None, + local_endpoint=config.endpoint_url or "", + access_key_id=config.credentials.access_key_id if config.credentials else None, + secret_access_key=config.credentials.secret_access_key if config.credentials else None, + bdb_lcpu_mask=bdb_lcpu_mask, + s3_lcpu_mask=s3_lcpu_mask, + s3_thread_pool_size=config.s3_thread_pool_size or 0, + ) + rpc_client.bdev_s3_add_bucket_name(name, config.bucket_name, allow_existing=True) + rpc_client.bdev_lvol_s3_bdev(node.lvstore, name) + except RPCException as e: + raise RuntimeError( + f"Failed to attach S3 device {name} for bucket {config.bucket_name} " + f"on node {node.get_id()}") from e + + logger.info("Attached restore S3 device %s for bucket %s on node %s", + name, config.bucket_name, node.get_id()) + + +def delete_restore_s3_bdev(node, name: str) -> None: + """Detach a device created by :func:`create_restore_s3_bdev`. + + Best-effort by design: this runs on the restore's terminal paths, and a + failure to clean up must not turn a completed restore into a failed one. It + is logged rather than raised, because the consequence is a leaked device -- + which does block the lvstore from being destroyed, so it is worth noticing. + """ + try: + node.rpc_client().bdev_s3_delete(name) + except Exception as e: + # Deliberately broad and deliberately not re-raised: this runs on a + # restore's terminal paths, where the alternative to a leaked device is + # reporting a completed restore as failed. + logger.warning("Could not delete restore S3 device %s on node %s: %s", + name, node.get_id(), e) + else: + logger.info("Deleted restore S3 device %s on node %s", name, node.get_id()) + + +def foreign_bucket_config(backup: Backup, cluster, + credentials: Optional[S3Credentials]) -> Optional[BackupConfig]: + """How to reach this backup's bucket, when it is not the cluster's own. + + Returns ``None`` when the node's existing backup device already points at + the right bucket -- there is no foreign bucket, so there is nothing to + describe. Otherwise returns the configuration for a device that reads the + backup's own recorded location, which is what makes a restore from another + cluster's bucket possible at all. + + Decides only. The device itself is created by the task runner, which is the + component that knows which node the volume landed on and the only one that + can put the device back after a node restart mid-restore. + + Raises: + PreconditionError: The bucket is foreign and unreachable -- no + credentials were supplied for it, while the cluster uses static + credentials that say nothing about it. + """ + location = backup.get_location() + + try: + own = cluster.get_backup_config() + except ValueError: + # No usable configuration of its own, so every bucket is foreign to it. + own = None + + if own is not None and own.location() == location and credentials is None: + return None + + config = BackupConfig.model_validate({ + **location.model_dump(exclude_none=True), + **({"credentials": credentials.model_dump()} if credentials is not None else {}), + }) + + if config.credentials is None and own is not None and own.credentials is not None: + # Falling back to the cluster's own static credentials would fail deep + # in the data plane with nothing to point at the cause. + raise PreconditionError( + f"Backup {backup.uuid} lives in bucket {location.bucket_name}, which " + f"is not this cluster's own. Supply credentials for that bucket, or " + "configure the nodes with an instance role that can read it.") + + return config + + +def restore_s3_bdev_name(backup_id: str) -> str: + """Name of the device created to read a foreign bucket for one restore. + + Derived from the backup id so a retry re-derives the same name rather than + leaking a device per attempt. + """ + return f"s3_restore_{backup_id[:8]}" + + def _resolve_crypto_key(backup: Backup, cluster): """Recover the key needed to read an encrypted backup. @@ -369,6 +489,9 @@ def _config_for(backup: Backup) -> BackupConfig: Raises: PreconditionError: The cluster's configured bucket is not the one this backup lives in, so its credentials cannot be assumed to reach it. + A precondition rather than a failure: it means the cluster's backup + configuration was repointed while this backup was in flight, which is + visible through GET /clusters/{id}/backup-config. """ config = db_controller.get_cluster_by_id(backup.cluster_id).get_backup_config() location = backup.get_location() @@ -693,7 +816,8 @@ def backup_snapshot(snapshot_id, cluster_id=None): def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, - target_node_id: Optional[str] = None): + target_node_id: Optional[str] = None, + s3_credentials: Optional[S3Credentials] = None): """Restore a backup chain into a new fully-accessible lvol. Creates the volume (with subsystem, listeners, namespace) via @@ -702,6 +826,8 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, until the data transfer completes. Args: + s3_credentials: Credentials for the backup's bucket, when that is not + the cluster's own. Omit to use the nodes' instance role. target_node_id: Optional node to restore onto. If not provided, a node of the target cluster is auto-selected. Any node in the cluster can restore any backup because S3 keys are node-agnostic @@ -763,6 +889,10 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, f"Target node {target_node_id} has no lvstore (S3 bdev requires lvstore)") crypto_key = _resolve_crypto_key(backup, cluster) + # Resolved before the volume exists, so an unreachable bucket is refused + # here rather than after a volume has been created for a restore that + # cannot run. + s3_config = foreign_bucket_config(backup, cluster, s3_credentials) logger.info(f"Backup allowed hosts: {backup.allowed_hosts}") lvol_id, error = lvol_controller.add_lvol_ha( @@ -805,7 +935,8 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, # incremental data wins, with older backups filling any remaining gaps. if not tasks_controller.add_backup_restore_task( pool.cluster_id, lvol.node_id, backup_id, bdev_name, - [b.s3_id for b in reversed(chain)], lvol_id=lvol_id): + [b.s3_id for b in reversed(chain)], lvol_id=lvol_id, + s3_config=s3_config.model_dump(exclude_none=True) if s3_config is not None else None): raise RuntimeError("Failed to create restore task") return lvol_id diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e92d3bdb5..5a7e233a1 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -1102,8 +1102,18 @@ def add_backup_task(backup): ) -def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids, lvol_id=""): - """Create the task that restores an S3 backup chain into a new lvol.""" +def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids, + lvol_id="", s3_bdev="", s3_config=None): + """Create the task that restores an S3 backup chain into a new lvol. + + Args: + s3_bdev: which S3 device on the node to read from. + s3_config: when set, the device named above does not exist yet and the + runner creates it from this configuration -- the case where the + backup lives in a bucket that is not the cluster's own. The runner + owns that device and deletes it, and scrubs this field, when the + restore reaches a terminal state. + """ return _add_task( JobSchedule.FN_BACKUP_RESTORE, cluster_id, @@ -1115,6 +1125,8 @@ def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids "lvol_name": lvol_name, "lvol_id": lvol_id, "chain_ids": chain_ids, + "s3_bdev": s3_bdev, + "s3_config": s3_config, }, ) diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 52251aa60..e75ff93c3 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1968,18 +1968,34 @@ def bdev_lvol_s3_merge(self, s3_id, old_s3_id, cluster_batch, lvs_name=None): params["lvs_name"] = lvs_name return self._request("bdev_lvol_s3_merge", params) - def bdev_lvol_s3_recovery(self, lvol_name, s3_ids, cluster_batch): + def bdev_lvol_s3_recovery(self, lvol_name, s3_ids, cluster_batch, s3_bdev=None): """Restore a chain of S3 backups into a new lvol. Args: lvol_name: target lvol name to restore into - s3_ids: list of S3 backup IDs (uint32) forming the chain (oldest first) + s3_ids: list of S3 backup IDs (uint32) forming the chain, NEWEST + first: the data plane claims each cluster for the first id that + offers it (prepare_s3_clusters is first-writer-wins), so the + newest backup's data must win. cluster_batch: batch size in clusters + s3_bdev: which S3 device to read from. Omitted, the data plane picks + the first S3 device attached to the lvstore, which is ambiguous + once a restore has attached a second one for a foreign bucket. """ - return self._request("bdev_lvol_s3_recovery", { + params = { "lvol_name": lvol_name, "cluster_batch": cluster_batch, "s3_ids": s3_ids, - }) + } + if s3_bdev: + params["s3_bdev"] = s3_bdev + return self._request("bdev_lvol_s3_recovery", params) + + def bdev_s3_delete(self, name): + """Delete an S3 bdev. + + Used to release the device a restore attached for a foreign bucket. + """ + return self._request3("bdev_s3_delete", name=name) def bdev_lvol_s3_delete(self, s3_ids): """Delete all S3 backups for the given IDs (list of uint32).""" diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index a08f3e2fe..ae27add90 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -14,6 +14,7 @@ from simplyblock_core.controllers import backup_controller, backup_events from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.storage_node import StorageNode @@ -192,6 +193,51 @@ def _set_lvol_restore_failed(task, reason): logger.warning(f"Restored lvol {lvol_id} not found in DB") +def _restore_s3_bdev(task, snode) -> str: + """The S3 device this restore reads from. + + A foreign bucket gets its own device, named from the backup id so a retry + re-derives the same name instead of leaking one per attempt. Otherwise the + node's own backup device already points at the right bucket. + """ + if task.function_params.get("s3_config"): + return backup_controller.restore_s3_bdev_name( + task.function_params["backup_id"]) + return backup_controller.primary_s3_bdev_name(snode) + + +def _ensure_restore_s3_bdev(task, snode) -> None: + """Create the foreign-bucket device if this restore needs one. + + Idempotent, and re-run on every attempt rather than once: a node restart + mid-restore takes the device with it, and the runner is the only component + positioned to put it back. + """ + config = task.function_params.get("s3_config") + if not config: + return + + backup_controller.create_restore_s3_bdev( + snode, BackupConfig.model_validate(config), _restore_s3_bdev(task, snode)) + + +def _release_restore_s3_bdev(task, snode) -> None: + """Delete the foreign-bucket device and forget its credentials. + + Called on every terminal path. The credentials are scrubbed from the task + record because a task is retained for weeks after it finishes, and there is + no reason for another cluster's S3 keys to outlive the restore that needed + them. + """ + if not task.function_params.get("s3_config"): + return + + if snode is not None: + backup_controller.delete_restore_s3_bdev(snode, _restore_s3_bdev(task, snode)) + + task.function_params["s3_config"] = None + + def _run_restore(task): backup_id = task.function_params.get("backup_id") lvol_name = task.function_params.get("lvol_name") @@ -222,19 +268,35 @@ def _run_restore(task): from simplyblock_core.models.lvol_model import LVol lvol = db.get_lvol_by_id(lvol_id) if lvol.status == LVol.STATUS_IN_DELETION: + _release_restore_s3_bdev(task, snode) task.function_result = f"Restore target {lvol_id} has been deleted" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) return except KeyError: + _release_restore_s3_bdev(task, snode) task.function_result = f"Restore target {lvol_id} no longer exists" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) return if not recovery_started: + # The device is established here, not when the restore was requested: + # only now is the node known (add_lvol_ha chooses it), and only the + # runner can put it back after a node restart wipes it mid-restore. + try: + _ensure_restore_s3_bdev(task, snode) + except (RuntimeError, ValueError) as e: + task.function_result = f"Could not attach the backup's bucket: {e}" + task.retry += 1 + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return + try: - ret = rpc_client.bdev_lvol_s3_recovery(lvol_name, chain_ids, cluster_batch=16) + ret = rpc_client.bdev_lvol_s3_recovery( + lvol_name, chain_ids, cluster_batch=16, + s3_bdev=_restore_s3_bdev(task, snode)) if not ret: task.function_result = "bdev_lvol_s3_recovery RPC failed" task.retry += 1 @@ -274,6 +336,7 @@ def _run_restore(task): task.cluster_id, node_id, backup, lvol_name) except KeyError: pass + _release_restore_s3_bdev(task, snode) task.function_result = f"Restore completed: {lvol_name}" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) @@ -292,6 +355,7 @@ def _run_restore(task): logger.warning( "Backup %s not found in DB; restore-failed event skipped for lvol %s", backup_id, lvol_name) + _release_restore_s3_bdev(task, snode) task.status = JobSchedule.STATUS_DONE else: task.retry += 1 @@ -415,6 +479,12 @@ def _terminate_task(task, reason): pass elif task.function_name == JobSchedule.FN_BACKUP_RESTORE: _set_lvol_restore_failed(task, reason) + # This is the timeout / retry-ceiling path, so it is terminal too and + # owes the same cleanup as a normal failure. + try: + _release_restore_s3_bdev(task, db.get_storage_node_by_id(task.node_id)) + except KeyError: + _release_restore_s3_bdev(task, None) elif task.function_name == JobSchedule.FN_BACKUP_MERGE: old_bid = task.function_params.get("old_backup_id") if old_bid: diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index 0e74cbbdb..7cbeda3ed 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -7,6 +7,7 @@ from simplyblock_core.backup_manifest import ManifestError from simplyblock_core.db_controller import DBController from simplyblock_core.controllers import backup_controller +from simplyblock_core.models.backup_config import S3Credentials from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.lvol_model import LVol @@ -54,11 +55,16 @@ class _RestoreParams(BaseModel): target_node_id: Optional[str] = None + #: Credentials for the backup's bucket, when that is not this cluster's own + #: -- the disaster-recovery case. Omit to use the nodes' instance role. + s3_credentials: Optional[S3Credentials] = None + @api.post('/restore', name='clusters:backups:restore', status_code=202) def restore_backup(cluster: Cluster, parameters: _RestoreParams): return {"lvol_id": backup_controller.restore_backup( parameters.backup_id, parameters.lvol_name, parameters.pool, - target_node_id=parameters.target_node_id)} + target_node_id=parameters.target_node_id, + s3_credentials=parameters.s3_credentials)} class _ImportManifests(BaseModel): diff --git a/tests/integration/test_backup_restore_source.py b/tests/integration/test_backup_restore_source.py new file mode 100644 index 000000000..c47abc8c4 --- /dev/null +++ b/tests/integration/test_backup_restore_source.py @@ -0,0 +1,240 @@ +"""Restoring from a bucket that is not the cluster's own, against real FoundationDB. + +This is the disaster-recovery path: the backup's own recorded location says +where its objects are, and the node gets a second S3 device pointed at that +bucket for the duration of the restore. + +The device's whole lifecycle belongs to the task runner, not to the restore +request -- the runner is the only component that knows which node the volume +landed on, and the only one that can put the device back after a node restart +mid-restore. +""" +from unittest.mock import MagicMock, patch + +import pytest + +from simplyblock_core.controllers import backup_controller +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig, S3Credentials +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import tasks_runner_backup + + +CLUSTER_ID = "cluster-1" +OWN_BUCKET = "simplyblock-backup-cluster-1" +FOREIGN_BUCKET = "someone-elses-bucket" + + +def _config(bucket=OWN_BUCKET, **overrides): + return BackupConfig.model_validate({ + "bucket_name": bucket, "region": "eu-central-1", **overrides}) + + +@pytest.fixture +def db(): + return DBController() + + +@pytest.fixture +def cluster(db): + c = Cluster() + c.uuid = CLUSTER_ID + c.backup_config = _config( + credentials={"access_key_id": "own", "secret_access_key": "own"} + ).model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +@pytest.fixture +def node(db): + n = StorageNode() + n.uuid = "node-1" + n.cluster_id = CLUSTER_ID + n.status = StorageNode.STATUS_ONLINE + n.lvstore = "lvs_test" + n.mgmt_ip = "10.0.0.1" + n.rpc_port = 5260 + n.app_thread_mask = "0x8" + n.cpu = 8 + n.write_to_db(db.kv_store) + return n + + +def _backup(db, bucket=OWN_BUCKET, uuid="b-1"): + b = Backup() + b.uuid = uuid + b.s3_id = 1 + b.cluster_id = CLUSTER_ID + b.lvol_id = "lvol-1" + b.size = 4096 + b.status = Backup.STATUS_COMPLETED + b.location = _config(bucket).location().model_dump(mode="json") + b.write_to_db(db.kv_store) + return b + + +class TestSourceSelection: + + def test_own_bucket_needs_no_new_device(self, db, cluster, node): + assert backup_controller.foreign_bucket_config( + _backup(db), cluster, None) is None + + def test_foreign_bucket_yields_its_own_config(self, db, cluster, node): + config = backup_controller.foreign_bucket_config( + _backup(db, FOREIGN_BUCKET), cluster, + S3Credentials(access_key_id="theirs", secret_access_key="theirs")) + + assert config is not None + assert config.bucket_name == FOREIGN_BUCKET + assert config.credentials.access_key_id.get_secret_value() == "theirs" + + def test_foreign_bucket_without_credentials_is_refused(self, db, cluster, node): + """The cluster's own static keys say nothing about someone else's bucket.""" + with pytest.raises(PreconditionError, match="not this cluster's own"): + backup_controller.foreign_bucket_config( + _backup(db, FOREIGN_BUCKET), cluster, None) + + def test_instance_role_cluster_may_reach_a_foreign_bucket(self, db, node): + """With no static credentials anywhere, the node's role is the only answer.""" + c = Cluster() + c.uuid = CLUSTER_ID + c.backup_config = _config().model_dump(exclude_none=True) # no credentials + c.write_to_db(db.kv_store) + + config = backup_controller.foreign_bucket_config( + _backup(db, FOREIGN_BUCKET), c, None) + + assert config.bucket_name == FOREIGN_BUCKET + assert config.credentials is None + + def test_explicit_credentials_override_the_own_bucket_shortcut(self, db, cluster, node): + """Restoring the cluster's own bucket with other credentials is legitimate.""" + config = backup_controller.foreign_bucket_config( + _backup(db), cluster, + S3Credentials(access_key_id="other", secret_access_key="other")) + + assert config is not None + assert config.bucket_name == OWN_BUCKET + + def test_device_name_is_derived_from_the_backup(self, db): + """Stable across retries, so an attempt cannot leak a device per try.""" + assert (backup_controller.restore_s3_bdev_name("b-1234567890") + == backup_controller.restore_s3_bdev_name("b-1234567890")) + assert backup_controller.restore_s3_bdev_name( + "b-1234567890") != backup_controller.restore_s3_bdev_name("c-1234567890") + + +def _restore_task(db, s3_config=None, **params): + task = JobSchedule() + task.uuid = "task-1" + task.cluster_id = CLUSTER_ID + task.node_id = "node-1" + task.function_name = JobSchedule.FN_BACKUP_RESTORE + task.status = JobSchedule.STATUS_RUNNING + task.function_params = { + "backup_id": "b-1", + "lvol_name": "lvs_test/LVOL_1", + "lvol_id": "", + "chain_ids": [1], + "s3_config": s3_config, + **params, + } + task.write_to_db(db.kv_store) + return task + + +class TestRunnerOwnsTheDevice: + + def test_own_bucket_creates_nothing(self, db, cluster, node): + task = _restore_task(db) + + with patch.object(backup_controller, "create_restore_s3_bdev") as create: + tasks_runner_backup._ensure_restore_s3_bdev(task, node) + + create.assert_not_called() + + def test_foreign_bucket_device_is_created_by_the_runner(self, db, cluster, node): + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + with patch.object(backup_controller, "create_restore_s3_bdev") as create: + tasks_runner_backup._ensure_restore_s3_bdev(task, node) + + _, kwargs = create.call_args + args = create.call_args[0] + assert args[0] is node + assert args[1].bucket_name == FOREIGN_BUCKET + assert args[2] == backup_controller.restore_s3_bdev_name("b-1") + + def test_creation_is_idempotent_across_retries(self, db, cluster, node): + """A node restart mid-restore takes the device with it; the runner rebuilds it.""" + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + with patch.object(backup_controller, "create_restore_s3_bdev") as create: + tasks_runner_backup._ensure_restore_s3_bdev(task, node) + tasks_runner_backup._ensure_restore_s3_bdev(task, node) + + assert create.call_count == 2 + assert {c[0][2] for c in create.call_args_list} == { + backup_controller.restore_s3_bdev_name("b-1")} + + def test_release_deletes_the_device(self, db, cluster, node): + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + with patch.object(backup_controller, "delete_restore_s3_bdev") as delete: + tasks_runner_backup._release_restore_s3_bdev(task, node) + + delete.assert_called_once_with( + node, backup_controller.restore_s3_bdev_name("b-1")) + + def test_release_scrubs_the_credentials(self, db, cluster, node): + """A task record outlives the restore by weeks; foreign keys must not.""" + task = _restore_task(db, s3_config=_config( + FOREIGN_BUCKET, + credentials={"access_key_id": "theirs", "secret_access_key": "theirs"}, + ).model_dump(exclude_none=True)) + + with patch.object(backup_controller, "delete_restore_s3_bdev"): + tasks_runner_backup._release_restore_s3_bdev(task, node) + + assert task.function_params["s3_config"] is None + + def test_release_is_a_noop_for_the_own_bucket(self, db, cluster, node): + """The node's own device is shared; a restore must never delete it.""" + task = _restore_task(db) + + with patch.object(backup_controller, "delete_restore_s3_bdev") as delete: + tasks_runner_backup._release_restore_s3_bdev(task, node) + + delete.assert_not_called() + + def test_release_survives_a_missing_node(self, db, cluster): + """Cleanup runs on terminal paths, including ones reached because the node is gone.""" + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + tasks_runner_backup._release_restore_s3_bdev(task, None) + + assert task.function_params["s3_config"] is None + + def test_delete_failure_does_not_raise(self, db, cluster, node): + """A cleanup failure must not turn a completed restore into a failed one.""" + node.rpc_client = MagicMock() + node.rpc_client.return_value.bdev_s3_delete.side_effect = RuntimeError("gone") + + backup_controller.delete_restore_s3_bdev(node, "s3_restore_b-1") + + def test_recovery_names_the_device_it_reads_from(self, db, cluster, node): + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + assert tasks_runner_backup._restore_s3_bdev(task, node) == \ + backup_controller.restore_s3_bdev_name("b-1") + + def test_recovery_falls_back_to_the_nodes_own_device(self, db, cluster, node): + task = _restore_task(db) + + assert tasks_runner_backup._restore_s3_bdev(task, node) == \ + backup_controller.primary_s3_bdev_name(node) diff --git a/tests/unit/test_backup_restore_node_selection.py b/tests/unit/test_backup_restore_node_selection.py index cd5d4a7bf..584d89b69 100644 --- a/tests/unit/test_backup_restore_node_selection.py +++ b/tests/unit/test_backup_restore_node_selection.py @@ -13,6 +13,7 @@ from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup +from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.storage_node import StorageNode @@ -52,11 +53,17 @@ def db(): pool.cluster_id = TARGET_CLUSTER db.get_pool_by_id_or_name.return_value = pool - cluster = MagicMock() + # A real Cluster, not a mock: restore compares the backup's recorded + # location against the cluster's own configuration, and a mock compares + # unequal to everything. + cluster = Cluster() cluster.uuid = TARGET_CLUSTER - cluster.backup_source = "" + cluster.backup_config = dict(LOCATION) db.get_cluster_by_id.return_value = cluster + node = _node("target-node", TARGET_CLUSTER) + db.get_storage_node_by_id.return_value = node + lvol = MagicMock() lvol.node_id = "target-node" lvol.lvs_name = "lvs_test" diff --git a/tests/unit/web/api/v2/test_backup_endpoints.py b/tests/unit/web/api/v2/test_backup_endpoints.py index 2c790fb11..626034dc1 100644 --- a/tests/unit/web/api/v2/test_backup_endpoints.py +++ b/tests/unit/web/api/v2/test_backup_endpoints.py @@ -81,7 +81,24 @@ def test_restores_backup(self, client, db, cluster, backup_controller): assert response.status_code == 202 assert response.json() == {'lvol_id': VOLUME_ID} backup_controller.restore_backup.assert_called_once_with( - BACKUP_ID, 'restored-volume', 'pool-1', target_node_id=None) + BACKUP_ID, 'restored-volume', 'pool-1', target_node_id=None, + s3_credentials=None) + + def test_passes_bucket_credentials_through(self, client, db, cluster, + backup_controller): + """Restoring another cluster's bucket needs credentials for it.""" + backup_controller.restore_backup.return_value = VOLUME_ID + + response = client.post(f'{BASE}/restore', json={ + 'backup_id': BACKUP_ID, + 'lvol_name': 'restored-volume', + 'pool': 'pool-1', + 's3_credentials': {'access_key_id': 'AKIA', 'secret_access_key': 'shh'}, + }) + + assert response.status_code == 202 + credentials = backup_controller.restore_backup.call_args.kwargs['s3_credentials'] + assert credentials.access_key_id.get_secret_value() == 'AKIA' def test_a_precondition_error_is_not_mapped_here(self, client, db, cluster, backup_controller): From 8c16518522523b8aed85b335e217bcdef8a239d8 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 13:11:04 +0200 Subject: [PATCH 08/14] Remove the backup-source switch, and source_cluster_id with it The mechanism this served never worked. The S3 bdev picks its bucket as bucket_names[idx] where idx comes from bit 63 of the packed offset (bdev_s3_impl.cpp:611), and every s3_pack_offset call site in lib/lvol passes msb_flag=false. So idx is permanently 0, only the first registered bucket is ever addressed, and bdev_s3_add_bucket_name -- which appends rather than replaces -- was adding state nothing reads. Switching the source did nothing at all. Two guards enforced that fiction and are removed with it: * Backup creation was blocked cluster-wide while the source pointed elsewhere. There is no cluster-wide source any more. * Restore refused a backup from another cluster unless the whole cluster had first been re-pointed at that cluster's bucket. Restore now attaches a device for the backup's own recorded location, so there is nothing to re-point and nothing to refuse. Also gone: Cluster.backup_source, get_backup_sources, switch_backup_source, is_local_backup_source, `sbctl backup source-list` / `source-switch`, the v2 /source-switch and /sources endpoints, and _s3_bucket_exists, which existed only to pre-flight the switch. cli.py is regenerated from the reference rather than edited. Backup.source_cluster_id goes too, along with BackupDTO.source_cluster_id and the "Source" column in `backup list`. Every remaining reader was either the switch itself or a listing built to explain it: the field was written at creation as a copy of cluster_id, and on import as the manifest's source. Which means it recorded something real in exactly one case -- an imported backup -- and what it recorded is already in the manifest that import read. The cost of removing it is one narrow regression, and it is the same one already noted on build_manifest: re-exporting an imported backup now stamps `source` with the importing cluster rather than the originating one, because the record keeps nothing from the manifest it was imported from. Nothing reads that field, and the manifests in a bucket -- which is what a recovery actually reads -- are written once, by the cluster that made the backup, and stay correct. Fixing it properly means storing the manifest on the record, which is the same change that fixes dataplane.cluster_size. The guard test that asserted nothing resolves a cluster through the field goes with the field. It was defending against reintroducing get_cluster_by_id(backup.source_cluster_id); with no such attribute, that is now a NameError rather than a subtle dependency on a dead cluster. --- simplyblock_cli/cli-reference.yaml | 18 -- simplyblock_cli/cli.py | 15 -- simplyblock_cli/clibase.py | 25 --- .../controllers/backup_controller.py | 154 ++---------------- simplyblock_core/models/backup.py | 1 - simplyblock_core/models/cluster.py | 1 - simplyblock_web/api/v2/_dtos.py | 2 - simplyblock_web/api/v2/cluster/backup.py | 16 -- tests/integration/test_backup_encryption.py | 1 - .../integration/test_backup_manifest_flow.py | 2 - tests/integration/test_backup_validation.py | 1 - .../test_backup_restore_node_selection.py | 25 ++- 12 files changed, 21 insertions(+), 240 deletions(-) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 86b6366a7..8cff69738 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -2612,24 +2612,6 @@ commands: help: "The target id (storage pool or logical volume id)." dest: target_id type: str - - name: source-list - help: "List backup sources (local and imported clusters)." - arguments: - - name: "--cluster-id" - help: "The cluster id." - dest: cluster_id - type: str - - name: source-switch - help: "Switch the active S3 backup source to a different cluster. Use 'local' or the local cluster id to switch back." - arguments: - - name: "source_cluster_id" - help: "The source cluster id or 'local'." - dest: source_cluster_id - type: str - - name: "--cluster-id" - help: "The cluster id." - dest: cluster_id - type: str - name: "qos" help: "QoS Commands" weight: 700 diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 16ab15886..ba1c86afb 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -1018,8 +1018,6 @@ def init_backup(self): self.init_backup__policy_list(subparser) self.init_backup__policy_attach(subparser) self.init_backup__policy_detach(subparser) - self.init_backup__source_list(subparser) - self.init_backup__source_switch(subparser) def init_backup__list(self, subparser): @@ -1076,15 +1074,6 @@ def init_backup__policy_detach(self, subparser): subcommand.add_argument('target_type', help='The target type.', type=str, choices=['pool','lvol',]) subcommand.add_argument('target_id', help='The target id (storage pool or logical volume id).', type=str) - def init_backup__source_list(self, subparser): - subcommand = self.add_sub_command(subparser, 'source-list', 'List backup sources (local and imported clusters).') - subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id') - - def init_backup__source_switch(self, subparser): - subcommand = self.add_sub_command(subparser, 'source-switch', 'Switch the active S3 backup source to a different cluster. Use \'local\' or the local cluster id to switch back.') - subcommand.add_argument('source_cluster_id', help='The source cluster id or \'local\'.', type=str) - subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id') - def init_qos(self): subparser = self.add_command('qos', 'QoS Commands') @@ -1569,10 +1558,6 @@ def run(self): ret = self.backup__policy_attach(sub_command, args) elif sub_command in ['policy-detach']: ret = self.backup__policy_detach(sub_command, args) - elif sub_command in ['source-list']: - ret = self.backup__source_list(sub_command, args) - elif sub_command in ['source-switch']: - ret = self.backup__source_switch(sub_command, args) else: self.parser.print_help() diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 1caf5c754..5a881353e 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -1137,31 +1137,6 @@ def backup__policy_detach(self, sub_command, args): print("Policy detached") return True - def backup__source_list(self, sub_command, args): - cluster_id = args.cluster_id - if not cluster_id: - db = db_controller.DBController() - clusters = db.get_clusters() - if clusters: - cluster_id = clusters[0].get_id() - sources = backup_controller.get_backup_sources(cluster_id) - return sources - - def backup__source_switch(self, sub_command, args): - cluster_id = args.cluster_id - if not cluster_id: - db = db_controller.DBController() - clusters = db.get_clusters() - if clusters: - cluster_id = clusters[0].get_id() - backup_controller.switch_backup_source(cluster_id, args.source_cluster_id) - target = args.source_cluster_id - if target == cluster_id or target == "local": - print("Switched to local backup source") - else: - print(f"Switched to external backup source: {target}") - return True - def db_backup__create(self, sub_command, args): return fdb_backup_controller.add_backup_task(args.cluster_id) diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index c862e43b6..2a8539f51 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -241,11 +241,13 @@ def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: cannot contradict itself. And the volume's settings, which the manifest records and the record does not, so an imported backup knows less about its volume than the manifest it was imported from did. - * Actively wrong: `dataplane.cluster_size` is recomputed here from the - *current* cluster. Re-exporting an imported backup therefore restamps it - with the importing cluster's page size, silently, even though it describes - objects a different cluster wrote. Nothing reads it yet, so nothing is - broken today. + * Actively wrong: `dataplane.cluster_size` and `source` are recomputed here + from the *current* cluster, because the record does not keep what an import + read. Re-exporting an imported backup therefore restamps both with the + importing cluster's identity and page size, silently, though they describe + objects a different cluster wrote. Nothing reads either yet, so nothing is + broken today -- and a bucket's own manifests, which is what a recovery + reads, are written once by the cluster that made the backup and are correct. The fix for all three is the same and is not attempted here: make the manifest the canonical document, store it on the record, and reduce `Backup` @@ -310,7 +312,7 @@ def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: {**(backup.encryption or {}), "encrypted": backup.encrypted}), location=backup.get_location(), source=backup_manifest.Source( - cluster_id=backup.source_cluster_id or backup.cluster_id, + cluster_id=backup.cluster_id, cluster_name=cluster_name, node_id=backup.node_id, ), @@ -474,10 +476,8 @@ def _resolve_crypto_key(backup: Backup, cluster): except KMSException as e: raise RuntimeError( f"Cannot reach the key for backup {backup.uuid} at " - f"{descriptor.dek_path}. It was written by cluster " - f"{backup.source_cluster_id or backup.cluster_id} using " - f"{descriptor.kms}, which has to be reachable to restore it: " - f"{e}") from e + f"{descriptor.dek_path} using {descriptor.kms}, which has to be " + f"reachable to restore it: {e}") from e def _config_for(backup: Backup) -> BackupConfig: @@ -519,17 +519,6 @@ def delete_manifest(backup: Backup) -> None: backup_manifest.delete(_config_for(backup), backup.uuid) -def _s3_bucket_exists(config: BackupConfig, bucket_name) -> bool: - try: - backup_manifest.s3_client(config).head_bucket(Bucket=bucket_name) - return True - except ClientError as e: - error_code = int(e.response["Error"]["Code"]) - if error_code == 404: - return False - raise - - def _ensure_s3_bucket(config: BackupConfig, bucket_name): try: s3_client = backup_manifest.s3_client(config) @@ -686,7 +675,6 @@ def _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, loca backup.uuid = backup_id backup.s3_id = db_controller.next_s3_id() backup.cluster_id = cluster_id - backup.source_cluster_id = cluster_id # provenance only backup.location = location.model_dump(mode="json") backup.lvol_id = lvol.get_id() backup.lvol_name = lvol.lvol_name @@ -743,12 +731,6 @@ def backup_snapshot(snapshot_id, cluster_id=None): except KeyError as e: return None, str(e) - # Block new backups when S3 source is switched to an external cluster - if not is_local_backup_source(snode.cluster_id): - return None, ("Cannot create backups while backup source is " - "switched to an external cluster. Switch back " - "to local first.") - if snode.status != StorageNode.STATUS_ONLINE: return None, f"Node {node_id} is not online (status: {snode.status})" @@ -853,17 +835,6 @@ def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, except KeyError as e: raise PreconditionError(str(e)) from e - # Verify the backup's source matches the active S3 source. - # If the backup came from an external cluster, the S3 bdev must be - # switched to that cluster's bucket before restoring. - backup_src = backup.source_cluster_id or backup.cluster_id - active_src = cluster.backup_source or cluster.uuid - if backup_src != active_src: - raise PreconditionError( - f"Backup source is {backup_src[:8]} but active S3 source " - f"is {active_src[:8]}. Use 'sbctl backup source-switch " - f"{backup_src}' first.") - # The chain has to be restorable as a unit, and short enough for the data # plane. Checked before the volume is created so a doomed restore leaves # nothing behind. @@ -1018,8 +989,6 @@ def list_backups(cluster_id=None): data = [] for b in backups: logger.debug(b) - source = b.source_cluster_id or b.cluster_id - is_external = source != b.cluster_id entry = { "ID": b.uuid, "S3 ID": b.s3_id, @@ -1029,7 +998,6 @@ def list_backups(cluster_id=None): "Status": b.status, "Prev": b.prev_backup_id[:8] if b.prev_backup_id else "-", "Created": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(b.created_at)) if b.created_at else "", - "Source": source[:8] if is_external else "local", } data.append(entry) return data @@ -1178,7 +1146,6 @@ def import_backups(manifests: Iterable[backup_manifest.BackupManifest], backup.uuid = backup_id backup.s3_id = manifest.s3_id backup.cluster_id = cluster_id or manifest.source.cluster_id - backup.source_cluster_id = manifest.source.cluster_id backup.lvol_id = manifest.volume.lvol_id backup.lvol_name = manifest.volume.lvol_name backup.snapshot_id = manifest.volume.snapshot_id @@ -1210,107 +1177,6 @@ def import_from_bucket(config: BackupConfig, cluster_id=None) -> int: return import_backups(discover_backups(config), cluster_id=cluster_id) -def get_backup_sources(cluster_id): - """List all distinct backup sources (local + imported clusters). - - Returns a list of dicts with source_cluster_id, count, and whether - it is the currently active source. - """ - try: - cluster = db_controller.get_cluster_by_id(cluster_id) - except KeyError: - return [] - - backups = db_controller.get_backups(cluster_id) - sources = {} - for b in backups: - src = b.source_cluster_id or cluster_id - if src not in sources: - sources[src] = {"source_cluster_id": src, "count": 0, "is_local": src == cluster_id} - sources[src]["count"] += 1 - - active_source = cluster.backup_source or cluster_id - result = [] - for src_id, info in sources.items(): - info["active"] = (src_id == active_source) - result.append(info) - - # Always include local even if no backups - if cluster_id not in sources: - result.append({ - "source_cluster_id": cluster_id, - "count": 0, - "is_local": True, - "active": active_source == cluster_id, - }) - - return result - - -def switch_backup_source(cluster_id, source_cluster_id) -> None: - """Switch the active backup source for all nodes in the cluster. - - Reconfigures the S3 bdev on every node to read from the bucket - belonging to source_cluster_id. While switched to an external - source, new backups cannot be created. - - Args: - cluster_id: The local cluster ID. - source_cluster_id: The cluster ID whose S3 bucket to activate. - Use the local cluster_id (or "local") to switch back. - - Returns (success, error_message). - """ - try: - cluster = db_controller.get_cluster_by_id(cluster_id) - except KeyError as e: - raise PreconditionError("Precondition not met") from e - - if source_cluster_id == "local": - source_cluster_id = cluster_id - - # Determine the bucket name for the source cluster - config = cluster.get_backup_config() - if source_cluster_id == cluster_id: - bucket_name = config.bucket_name - else: - bucket_name = f"simplyblock-backup-{source_cluster_id}" - - # Verify the bucket exists - try: - if not _s3_bucket_exists(config, bucket_name): - raise PreconditionError(f"S3 bucket {bucket_name} does not exist") - except BotoCoreError as e: - raise RuntimeError(f"S3 bucket {bucket_name} not accessible: {e}") - - # Reconfigure S3 bdev bucket on all online nodes - nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id) - for node in nodes: - if node.status != StorageNode.STATUS_ONLINE or not node.lvstore: - continue - - rpc_client = node.rpc_client() - s3_bdev_name = f"s3_{node.lvstore}" - rpc_client.bdev_s3_add_bucket_name(s3_bdev_name, bucket_name, allow_existing=True) - logger.info(f"Switched S3 bucket to {bucket_name} on node {node.get_id()}") - - # Persist the active source in the cluster record. Atomic: the long - # per-node RPC loop above means a concurrent cluster.status change could be - # clobbered by a full write here (lost-update class — incident 2026-06-18). - db_controller.atomic_update( - db_controller.get_cluster_by_id(cluster_id), - lambda c, v=source_cluster_id: setattr(c, "backup_source", v)) - - -def is_local_backup_source(cluster_id): - """Check if the cluster is currently using its own local backup source.""" - try: - cluster = db_controller.get_cluster_by_id(cluster_id) - except KeyError: - return True - return not cluster.backup_source or cluster.backup_source == cluster_id - - # ---- Backup Policy Management ---- def add_policy(cluster_id, name, max_versions=0, max_age="", schedule=""): diff --git a/simplyblock_core/models/backup.py b/simplyblock_core/models/backup.py index d78b21275..532bfeccd 100644 --- a/simplyblock_core/models/backup.py +++ b/simplyblock_core/models/backup.py @@ -36,7 +36,6 @@ class Backup(BaseModel): prev_backup_id: str = "" pool_uuid: str = "" size: int = 0 - source_cluster_id: str = "" # original cluster that created this backup created_at: int = 0 completed_at: int = 0 error_message: str = "" diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index d5003028d..1e35c793e 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -191,7 +191,6 @@ def is_topology_owned(self) -> bool: client_data_nic: str = "" max_fault_tolerance: int = 1 backup_config: dict = {} - backup_source: str = "" # active backup source cluster_id ("" = local) backup_timeout_seconds: int = 14400 # 4 hours default nvmf_base_port: int = 4420 rpc_base_port: int = 8080 diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 658317d0b..4fe57ed6a 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -537,7 +537,6 @@ class BackupDTO(BaseModel): allowed_hosts: List[dict] created_at: int completed_at: int - source_cluster_id: str encrypted: bool @staticmethod @@ -556,7 +555,6 @@ def from_model(model: Backup): allowed_hosts=model.allowed_hosts or [], created_at=model.created_at, completed_at=model.completed_at, - source_cluster_id=model.source_cluster_id or "", encrypted=model.encrypted, ) diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index 7cbeda3ed..296d58b4a 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -143,22 +143,6 @@ def export_backups( cluster_id=cluster.get_id(), lvol_name=lvol_name_filter) -class _BackupSourceSwitchParams(BaseModel): - source_cluster_id: str - - -@api.post('/source-switch', name='clusters:backups:source-switch') -def source_switch(cluster: Cluster, parameters: _BackupSourceSwitchParams): - backup_controller.switch_backup_source(cluster.get_id(), parameters.source_cluster_id) - return {"source_cluster_id": parameters.source_cluster_id} - - -@api.get('/sources', name='clusters:backups:sources') -def list_sources(cluster: Cluster): - sources = backup_controller.get_backup_sources(cluster.get_id()) - return sources - - def _lookup_lvol_in_cluster(volume_id: str, cluster: ClusterModel) -> LVol: try: volume = db.get_lvol_by_id(volume_id) diff --git a/tests/integration/test_backup_encryption.py b/tests/integration/test_backup_encryption.py index 9b69d9835..fcce7d73d 100644 --- a/tests/integration/test_backup_encryption.py +++ b/tests/integration/test_backup_encryption.py @@ -57,7 +57,6 @@ def _backup(db, uuid="b-1", encrypted=True, encryption=None): b.uuid = uuid b.s3_id = 1 b.cluster_id = CLUSTER_ID - b.source_cluster_id = CLUSTER_ID b.lvol_id = "lvol-1" b.lvol_name = "vol" b.size = 4096 diff --git a/tests/integration/test_backup_manifest_flow.py b/tests/integration/test_backup_manifest_flow.py index 53b2981b1..f7dd2e743 100644 --- a/tests/integration/test_backup_manifest_flow.py +++ b/tests/integration/test_backup_manifest_flow.py @@ -75,7 +75,6 @@ def _backup(db, uuid, s3_id, prev="", **overrides): b.uuid = uuid b.s3_id = s3_id b.cluster_id = CLUSTER_ID - b.source_cluster_id = CLUSTER_ID b.lvol_id = "lvol-1" b.lvol_name = "vol" b.snapshot_id = f"snap-{uuid}" @@ -194,7 +193,6 @@ def test_backups_survive_losing_the_database(self, db, cluster, lvol): assert restored.s3_id == 2 assert restored.prev_backup_id == "b-1" assert restored.cluster_id == "cluster-2" - assert restored.source_cluster_id == CLUSTER_ID assert restored.get_location().bucket_name == "simplyblock-backup-cluster-1" assert restored.encrypted is True diff --git a/tests/integration/test_backup_validation.py b/tests/integration/test_backup_validation.py index 8d3c886cb..7979cf400 100644 --- a/tests/integration/test_backup_validation.py +++ b/tests/integration/test_backup_validation.py @@ -100,7 +100,6 @@ def _backup(db, uuid, s3_id, snapshot_id, prev="", location=None, encrypted=Fals b.uuid = uuid b.s3_id = s3_id b.cluster_id = CLUSTER_ID - b.source_cluster_id = CLUSTER_ID b.lvol_id = "lvol-1" b.lvol_name = "vol" b.snapshot_id = snapshot_id diff --git a/tests/unit/test_backup_restore_node_selection.py b/tests/unit/test_backup_restore_node_selection.py index 584d89b69..c640d8c15 100644 --- a/tests/unit/test_backup_restore_node_selection.py +++ b/tests/unit/test_backup_restore_node_selection.py @@ -24,13 +24,12 @@ LOCATION = {"bucket_name": "backups", "region": "eu-central-1"} -def _backup(node_id, cluster_id=TARGET_CLUSTER, source_cluster_id=""): +def _backup(node_id, cluster_id=TARGET_CLUSTER): backup = Backup() backup.uuid = "backup-1" backup.s3_id = 5 backup.node_id = node_id backup.cluster_id = cluster_id - backup.source_cluster_id = source_cluster_id backup.size = 1024 backup.status = Backup.STATUS_COMPLETED backup.location = dict(LOCATION) @@ -95,29 +94,26 @@ class TestImplicitNode: def test_backup_node_is_not_used_for_placement(self, db, add_lvol_ha, tasks): """An imported backup's node_id points into the source cluster.""" - backup = _backup(node_id="source-cluster-node", source_cluster_id=SOURCE_CLUSTER) + backup = _backup(node_id="source-cluster-node") db.get_backup_by_id.return_value = backup db.get_backup_chain.return_value = [backup] - db.get_cluster_by_id.return_value.backup_source = SOURCE_CLUSTER assert _restore() == "lvol-new" assert not add_lvol_ha.call_args.kwargs["host_id_or_name"] def test_no_node_lookup_without_explicit_target(self, db, add_lvol_ha, tasks): - backup = _backup(node_id="source-cluster-node", source_cluster_id=SOURCE_CLUSTER) + backup = _backup(node_id="source-cluster-node") db.get_backup_by_id.return_value = backup db.get_backup_chain.return_value = [backup] - db.get_cluster_by_id.return_value.backup_source = SOURCE_CLUSTER _restore() db.get_storage_node_by_id.assert_not_called() def test_restore_task_targets_the_node_the_volume_landed_on(self, db, add_lvol_ha, tasks): - backup = _backup(node_id="source-cluster-node", source_cluster_id=SOURCE_CLUSTER) + backup = _backup(node_id="source-cluster-node") db.get_backup_by_id.return_value = backup db.get_backup_chain.return_value = [backup] - db.get_cluster_by_id.return_value.backup_source = SOURCE_CLUSTER _restore() @@ -182,13 +178,14 @@ def _local_backup(self, db): db.get_backup_by_id.return_value = backup db.get_backup_chain.return_value = [backup] - def test_source_mismatch_is_a_precondition(self, db, add_lvol_ha): - db.get_cluster_by_id.return_value.backup_source = SOURCE_CLUSTER - - with pytest.raises(PreconditionError, match="source-switch"): - _restore() + def test_a_backup_from_another_cluster_needs_no_switch(self, db, add_lvol_ha, tasks): + """It used to be refused unless the whole cluster had been re-pointed at + that cluster's bucket. Same bucket, so nothing to re-point.""" + backup = _backup(node_id="source-cluster-node") + db.get_backup_by_id.return_value = backup + db.get_backup_chain.return_value = [backup] - add_lvol_ha.assert_not_called() + assert _restore() == "lvol-new" def test_incomplete_chain_is_rejected_before_creating_a_volume(self, db, add_lvol_ha): db.get_backup_chain.return_value = [_backup(node_id="target-node")] From ac84594df66da600a44e68f617ef7514cc37541c Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 13:56:52 +0200 Subject: [PATCH 09/14] dataplane: one bucket per S3 device A device now takes its bucket at creation and keeps it. That is what a bucket actually is here -- it comes with its own credentials, endpoint and region, so a device serving several would need several clients. Reading a second bucket is done by creating a second device, which the lvstore already supports: its transfer devices are a list. What this replaces was a mutable vector indexed by bit 63 of the I/O offset. Nothing ever set that bit -- every s3_pack_offset call site in lib/lvol passes msb_flag=false -- so only entry 0 was ever addressed, and the index read past the end whenever the vector held just the one bucket the control plane registered. bdev_s3_add_bucket_name is deleted along with it; a device can no longer exist in a bucket-less state, so there is no window in which one is attached to an lvstore with nothing to read. Aws::InitAPI / ShutdownAPI are process-global and were called per device, so deleting a second S3 device shut the SDK down underneath the first -- which a restore attaching a device for a foreign bucket does routinely. A device now holds a share of a t_sdk_session, whose constructor calls InitAPI and whose destructor calls ShutdownAPI, handed out through a weak_ptr so there is at most one alive at a time. A shared_ptr rather than a counter because the pairing then holds by construction rather than by remembering to write the other half. It survives the paths a counter would leak on: try_shutdown_client refuses while requests are still pending, ~t_disk runs whether or not that was ever called, and init_client can throw partway through building its two clients. Keeping the SDKOptions inside the session also means InitAPI and ShutdownAPI are handed the same object, whose lifetime brackets both calls exactly -- as two statics they merely happened to. The member is declared ahead of the clients so it is destroyed after them. Endpoint, region, TLS verification and addressing style are separate parameters. They were bundled into a `local_testing` flag that forced HTTP, disabled certificate verification and hardcoded us-east-1, and an endpoint was honoured only when it was set. That made every non-AWS S3-compatible store a testing-only configuration. verify_tls defaults to true, set explicitly because the surrounding memset would otherwise default it to "do not verify". With no key configured the SDK's default provider chain is used, so the instance role works as the header has always claimed. Passing an empty AWSCredentials, as this did, is not the same thing: the SDK takes it as a valid anonymous identity and never consults the chain. Two fixes in add_directory_name, which shares this code: * The bdev lookup was dereferenced before its NULL check, so an unknown name segfaulted the storage node. The same path leaked its context. * It read and wrote bucket_names rather than directory_names, so every call after the first corrupted the bucket list instead of registering a directory. The filesystem target's directory selection has the same unbounded index as the bucket selection did; bounded rather than reworked, since giving it one directory per device is a separate change. NOT BUILT, but syntax and type checked. A full build needs SPDK configured and simplyblock's aws-sdk-cpp fork, neither of which exists in this monorepo (see CLAUDE.md). `-fsyntax-only` against the *upstream* SDK does work, and reports no errors for this translation unit, with the same 102 -Wall -Wextra warnings as before the change -- all of them pre-existing. The recipe, its two necessary shims and what it does not prove are recorded in COMPILE_CHECK.md later in this series. It does not link and does not run, so it says nothing about whether a transfer completes or whether the SDK ends up initialised the right number of times at runtime; that still needs a real build in the ultra workspace against the forked SDK. --- .../controllers/backup_controller.py | 30 ++++----- simplyblock_core/rpc_client.py | 63 +++++++++---------- tests/integration/test_backup.py | 42 +++++++------ tests/unit/test_client_secret_logging.py | 6 +- 4 files changed, 68 insertions(+), 73 deletions(-) diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index 2a8539f51..c1924963a 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -343,18 +343,20 @@ def create_restore_s3_bdev(node, config: BackupConfig, name: str) -> None: try: rpc_client.bdev_s3_create( name=name, + bucket_name=config.bucket_name, secondary_target=config.secondary_target, with_compression=config.with_compression, snapshot_backups=config.snapshot_backups, - local_testing=config.endpoint is not None, - local_endpoint=config.endpoint_url or "", + endpoint=config.endpoint_url or "", + region=config.region, + verify_tls=config.verify_tls, + use_path_style=config.use_path_style, access_key_id=config.credentials.access_key_id if config.credentials else None, secret_access_key=config.credentials.secret_access_key if config.credentials else None, bdb_lcpu_mask=bdb_lcpu_mask, s3_lcpu_mask=s3_lcpu_mask, s3_thread_pool_size=config.s3_thread_pool_size or 0, ) - rpc_client.bdev_s3_add_bucket_name(name, config.bucket_name, allow_existing=True) rpc_client.bdev_lvol_s3_bdev(node.lvstore, name) except RPCException as e: raise RuntimeError( @@ -559,22 +561,19 @@ def create_s3_bdev(node, config: BackupConfig) -> None: # #938): a second creation with a different mask that either failed # noisily on every activate or put the pollers on the wrong core. - # The data plane still takes the pre-BackupConfig parameter shape; phase 2 - # replaces it. Two lossy mappings live here until then: - # * `local_testing` is not a mode, it is the only condition under which the - # data plane honours an endpoint override at all (bdev_s3_impl.hpp - # init_client), so it tracks "an endpoint was configured". - # * region, verify_tls and use_path_style have nowhere to go -- the data - # plane hardcodes us-east-1 and path-style under local_testing, and - # resolves the region from the environment otherwise. try: + _ensure_s3_bucket(config, config.bucket_name) + rpc_client.bdev_s3_create( name=s3_bdev_name, + bucket_name=config.bucket_name, secondary_target=config.secondary_target, with_compression=config.with_compression, snapshot_backups=config.snapshot_backups, - local_testing=config.endpoint is not None, - local_endpoint=config.endpoint_url or "", + endpoint=config.endpoint_url or "", + region=config.region, + verify_tls=config.verify_tls, + use_path_style=config.use_path_style, access_key_id=config.credentials.access_key_id if config.credentials else None, secret_access_key=config.credentials.secret_access_key if config.credentials else None, bdb_lcpu_mask=bdb_lcpu_mask, @@ -582,11 +581,6 @@ def create_s3_bdev(node, config: BackupConfig) -> None: s3_thread_pool_size=config.s3_thread_pool_size or 0, ) - _ensure_s3_bucket(config, config.bucket_name) - - rpc_client.bdev_s3_add_bucket_name(s3_bdev_name, config.bucket_name, allow_existing=True) - logger.info(f"S3 bdev bucket set: {config.bucket_name} on {s3_bdev_name}") - rpc_client.bdev_lvol_s3_bdev(node.lvstore, s3_bdev_name) logger.info(f"S3 bdev created and attached: {s3_bdev_name} on node {node.get_id()}") except (RPCException, RuntimeError) as e: diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index e75ff93c3..fb2a13cc3 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1852,41 +1852,56 @@ def bdev_lvol_batch_transfer_final_step(self, lvol_names, lvol_ids, snapshot_nam # ---- S3 Backup RPCs ---- - def bdev_s3_create(self, name: str, secondary_target: int = 0, + def bdev_s3_create(self, name: str, bucket_name: str, secondary_target: int = 0, with_compression: bool = False, snapshot_backups: bool = True, - local_testing: bool = False, local_endpoint: str = "", + endpoint: str = "", region: str = "", verify_tls: bool = True, + use_path_style: bool = False, access_key_id: Optional[SecretStr] = None, secret_access_key: Optional[SecretStr] = None, bdb_lcpu_mask: int = 0, s3_lcpu_mask: int = 0, s3_thread_pool_size: int = 0): - """Create the S3 bdev device. - Must be called before bdev_lvol_s3_bdev to attach it to an lvstore. + """Create an S3 bdev for one bucket. + + A device serves exactly one bucket with one set of credentials, so + reading a second bucket means creating a second device. Attach it to an + lvstore with bdev_lvol_s3_bdev. + Args: - name: Bdev name - secondary_target: 0=S3, 1=FileSystem - with_compression: Enable ISA-L compression - snapshot_backups: Snapshot backup mode - local_testing: Use local endpoint (e.g. MinIO) - local_endpoint: Local endpoint URL + bucket_name: the bucket this device reads and writes. Required -- a + device without one cannot service any I/O. + endpoint: an S3-compatible endpoint, e.g. "http://minio:9000". + Empty means the SDK resolves AWS's endpoint from the region. + region: AWS region. Empty means the SDK resolves it from the + environment or instance metadata. + verify_tls: verify the endpoint's certificate. + use_path_style: path-style addressing, needed by MinIO and most + S3-compatible stores. access_key_id / secret_access_key: leave both ``None`` to use the node's instance role via the SDK's default credential provider chain. An empty ``SecretStr`` counts as absent too, rather than travelling as a key: the SDK reads empty credentials as a valid anonymous identity and then never consults the chain. + secondary_target: 0=S3, 1=FileSystem + with_compression: Enable ISA-L compression + snapshot_backups: selects the backup object layout + ({s3_id}/{mid}/{extent}) rather than the tiering one bdb_lcpu_mask: CPU mask for the SPDK thread of this bdev (uint64) s3_lcpu_mask: CPU mask for the internal AWS S3 thread pool (uint64) s3_thread_pool_size: AWS S3 thread pool size (default 32 on data plane) """ params: dict[str, Any] = { "name": name, + "bucket_name": bucket_name, "secondary_target": secondary_target, "with_compression": with_compression, "snapshot_backups": snapshot_backups, + "verify_tls": verify_tls, + "use_path_style": use_path_style, } - if local_testing: - params["local_testing"] = True - if local_endpoint: - params["local_endpoint"] = local_endpoint + if endpoint: + params["endpoint"] = endpoint + if region: + params["region"] = region if access_key_id: params["access_key_id"] = access_key_id if secret_access_key: @@ -1916,26 +1931,6 @@ def bdev_lvol_s3_bdev(self, lvs_name, bdev_name): "s3_bdev": bdev_name, }) - def bdev_s3_add_bucket_name(self, name, bucket_name, allow_existing: bool = False): - """Register a bucket name with the S3 bdev. - Must be called after bdev_s3_create and before any backup/recovery operations. - Args: - name: S3 bdev name (e.g. 's3_LVS_1234') - bucket_name: S3/MinIO bucket name to use for data storage - Returns (result, error) tuple. - """ - try: - return self._request3( - "bdev_s3_add_bucket_name", - name=name, - bucket_name=bucket_name, - ) - except RPCRemoteError as e: - if allow_existing and e.code == -17: - logger.debug("Bucket %s already registered with %s", name, bucket_name) - return None - raise - def bdev_lvol_s3_backup(self, s3_id, snapshot_names, cluster_batch=1): """Start an async backup of snapshots to S3. Args: diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index 71d96a4cb..8d094117f 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -315,8 +315,7 @@ def test_success(self, MockRPC, mock_boto3_client): _, kwargs = mock_rpc.bdev_s3_create.call_args self.assertEqual(kwargs["bdb_lcpu_mask"], 0x8) self.assertEqual(kwargs["s3_lcpu_mask"], 0xFF) - mock_rpc.bdev_s3_add_bucket_name.assert_called_once_with( - "s3_lvs_test", "simplyblock-backup-cluster-1", allow_existing=True) + self.assertEqual(kwargs["bucket_name"], "simplyblock-backup-cluster-1") mock_rpc.bdev_lvol_s3_bdev.assert_called_once_with("lvs_test", "s3_lvs_test") @patch("simplyblock_core.backup_manifest.boto3.client") @@ -343,19 +342,21 @@ def test_bdev_s3_create_fails(self, MockRPC): @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") - def test_bucket_name_fails(self, MockRPC, mock_boto3_client): - from simplyblock_core.rpc_client import RPCRemoteError + def test_bucket_is_a_create_parameter(self, MockRPC, mock_boto3_client): + """A device cannot exist without its bucket, so there is no window in + which one is attached to an lvstore with no bucket registered.""" mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.side_effect = RPCRemoteError("error", code=-1) - mock_s3 = mock_boto3_client.return_value - mock_s3.head_bucket.return_value = {} + mock_rpc.bdev_lvol_s3_bdev.return_value = True + mock_boto3_client.return_value.head_bucket.return_value = {} from simplyblock_core.controllers.backup_controller import create_s3_bdev - node = _node() - with pytest.raises(RuntimeError): - create_s3_bdev(node, _backup_config()) - mock_rpc.bdev_lvol_s3_bdev.assert_not_called() + create_s3_bdev(_node(), _backup_config()) + + _, kwargs = mock_rpc.bdev_s3_create.call_args + assert kwargs["bucket_name"] == "simplyblock-backup-cluster-1" + assert not hasattr(mock_rpc, "_mock_children") or \ + "bdev_s3_add_bucket_name" not in mock_rpc.method_calls @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") @@ -396,12 +397,11 @@ def test_local_testing_params(self, MockRPC, mock_boto3_client): "secret_access_key": "minioadmin", })) - # The data plane still takes the legacy shape; `local_testing` there is - # not a mode but the only condition under which it honours an endpoint - # override at all, so it tracks "an endpoint was configured". _, kwargs = mock_rpc.bdev_s3_create.call_args - self.assertTrue(kwargs["local_testing"]) - self.assertEqual(kwargs["local_endpoint"], "http://minio:9000") + self.assertEqual(kwargs["endpoint"], "http://minio:9000") + self.assertEqual(kwargs["region"], "us-east-1") + self.assertFalse(kwargs["verify_tls"]) + self.assertTrue(kwargs["use_path_style"]) self.assertEqual(kwargs["access_key_id"].get_secret_value(), "minioadmin") self.assertEqual(kwargs["secret_access_key"].get_secret_value(), "minioadmin") @@ -1078,9 +1078,15 @@ def test_bdev_s3_create_exists(self): from simplyblock_core.rpc_client import RPCClient self.assertTrue(hasattr(RPCClient, 'bdev_s3_create')) - def test_bdev_s3_add_bucket_name_exists(self): + def test_bdev_s3_add_bucket_name_is_gone(self): + """The bucket is a create parameter now; the separate call was the + mechanism behind the non-functional source switch.""" + from simplyblock_core.rpc_client import RPCClient + self.assertFalse(hasattr(RPCClient, 'bdev_s3_add_bucket_name')) + + def test_bdev_s3_delete_exists(self): from simplyblock_core.rpc_client import RPCClient - self.assertTrue(hasattr(RPCClient, 'bdev_s3_add_bucket_name')) + self.assertTrue(hasattr(RPCClient, 'bdev_s3_delete')) def test_bdev_lvol_s3_bdev_exists(self): from simplyblock_core.rpc_client import RPCClient diff --git a/tests/unit/test_client_secret_logging.py b/tests/unit/test_client_secret_logging.py index 5b6f077cb..471fc99ae 100644 --- a/tests/unit/test_client_secret_logging.py +++ b/tests/unit/test_client_secret_logging.py @@ -94,7 +94,7 @@ def test_bdev_s3_create_keys_reach_the_wire_but_not_the_log(rpc_client, caplog): with caplog.at_level(logging.DEBUG): rpc_client.bdev_s3_create( - name="s3_lvs_test", + name="s3_lvs_test", bucket_name="bucket", access_key_id=SecretStr("AKIAEXAMPLE"), secret_access_key=SecretStr("s3cr3t"), ) @@ -114,7 +114,7 @@ def test_bdev_s3_create_omits_absent_credentials(rpc_client): "jsonrpc": "2.0", "id": 1, "result": True, }) - rpc_client.bdev_s3_create(name="s3_lvs_test") + rpc_client.bdev_s3_create(name="s3_lvs_test", bucket_name="bucket") params = _sent_params(rpc_client) assert "access_key_id" not in params @@ -130,7 +130,7 @@ def test_bdev_s3_create_does_not_send_empty_credentials_as_keys(rpc_client): }) rpc_client.bdev_s3_create( - name="s3_lvs_test", + name="s3_lvs_test", bucket_name="bucket", access_key_id=SecretStr(""), secret_access_key=SecretStr(""), ) From e5ce63fe10e267414a7e50a370bdab8295615eb9 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 15:06:17 +0200 Subject: [PATCH 10/14] dataplane: name the S3 device per transfer, and fix the array bounds The transfer RPCs now take a required s3_bdev, and spdk_find_s3_bdev resolves it by name instead of returning the first entry with is_s3 set. An lvolstore can carry several S3 devices -- its own backup bucket, plus one a restore attached for another cluster's bucket -- so "the first" is ambiguous exactly during the operation that needs it to be right. Required rather than optional, deliberately. The only available default is the old first-match behaviour, which is the ambiguity being removed, and the caller always knows the name. spdk_json_decode_object is the non-relaxed form, so an omitted parameter is rejected rather than guessed, and a caller from before this change fails loudly instead of writing to an arbitrary bucket. A named device that is absent now returns -ENODEV and says which name it looked for, rather than -EINVAL with nothing to go on. Two memory-safety fixes in the same handlers: * snapshot_chain and s3_ids_chain were fixed 40-element stack arrays fed by a decoder bounded at RPC_MAX_LVOL_VBDEV (255). A 41-link chain overran the stack of the process serving live volumes, reachable from an ordinary tiered retention schedule. Both are now sized to the decoder's own bound. * s3_id is validated against S3_ID_BITS in all three handlers. The offset packing masks it to 30 bits without checking, so a larger value silently aliased onto another backup's object keys -- one backup overwriting another's data. The ordering contract is now stated where callers read it. snapshot_names and s3_ids must be NEWEST first, because prepare_s3_clusters is first-writer-wins (blobstore.c:15515). rpc_client.py documented the opposite and passed reversed(chain), so it was correct by accident; the SPDK binding said only "Ordered list", which is not wrong but not usable either. Both now say which end comes first and why, and scripts/rpc.py says it in its --help. Also in scripts/rpc.py: bdev_lvol_s3_recovery passed offset= to a binding that has no such parameter, so invoking it from the SPDK CLI raised TypeError before reaching the target. The argument is dropped along with the call. Verified by compiling. include/spdk/config.h is normally generated by ./configure, which needs the DPDK submodule this subtree does not wire up; generating a stub (the file is gitignored) makes gcc -fsyntax-only work, and lvol.c, vbdev_lvol_rpc.c and vbdev_lvol.c all compile clean with -Wall. That covers syntax and types, not linking or behaviour. S3_ID_BITS was checked to expand to 30 through vbdev_lvol.h rather than silently vanishing. Sizing those buffers to 255 also retires the reason the control plane's BACKUP_MAX_CHAIN_LENGTH was 40, so its comment and chain_fits are corrected here rather than left justifying a bound that no longer exists. The limit stays at 40, now as a policy: a restore reads every backup in the chain in one operation, so chain length multiplies restore time and the objects a recovery has to fetch. Raising it is safe up to 255 and needs no data-plane change. --- simplyblock_core/constants.py | 17 ++++++--- .../controllers/backup_controller.py | 7 ++-- simplyblock_core/rpc_client.py | 35 +++++++++++-------- .../services/tasks_runner_backup.py | 9 +++-- tests/integration/test_backup.py | 7 ---- 5 files changed, 45 insertions(+), 30 deletions(-) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index ef81e7a72..0c1126681 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -606,10 +606,19 @@ def get_config_var(name, default=None): BACKUP_MAX_RETRIES = 10 BACKUP_MERGE_SERVICE_INTERVAL_SEC = 60 -#: Longest backup chain the data plane will accept. bdev_lvol_s3_backup and -#: bdev_lvol_s3_recovery copy the decoded arrays into fixed 40-element stack -#: buffers (vbdev_lvol_rpc.c), so a longer chain corrupts the node's stack. The -#: control plane refuses first; raise this only together with those buffers. +#: Longest backup chain the control plane will accept. +#: +#: This used to be a memory-safety bound: bdev_lvol_s3_backup and +#: bdev_lvol_s3_recovery copied the decoded arrays into fixed 40-element stack +#: buffers, so a longer chain corrupted the storage node's stack. Those buffers +#: are now sized to the decoder's own bound (RPC_MAX_LVOL_VBDEV, 255), so the +#: data plane rejects rather than overruns and this number is the control plane's +#: own policy. +#: +#: It stays at 40 because a restore reads every backup in the chain in one +#: operation: the chain length is a multiplier on restore time and on the objects +#: a recovery has to fetch, and 40 tiered backups is already a long retention +#: history. Raising it is safe up to 255 and needs no data-plane change. BACKUP_MAX_CHAIN_LENGTH = 40 #: Upper bound on a backup's s3_id. The data plane packs it into bits 33..62 of diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index c1924963a..6796b252d 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -111,10 +111,11 @@ def _compute_s3_cpu_masks(node): def chain_fits(length: int) -> bool: - """Whether the data plane can accept a chain this long. + """Whether a chain this long is accepted. - Beyond this it copies the decoded array into a fixed stack buffer, smashing - the storage node's stack rather than returning an error. + The bound is the control plane's own; see BACKUP_MAX_CHAIN_LENGTH for what it + is a bound on. The data plane's own limit is higher and refuses rather than + overruns, so this is where a too-long chain is reported usefully. """ return length <= constants.BACKUP_MAX_CHAIN_LENGTH diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index fb2a13cc3..084c00590 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -1931,17 +1931,22 @@ def bdev_lvol_s3_bdev(self, lvs_name, bdev_name): "s3_bdev": bdev_name, }) - def bdev_lvol_s3_backup(self, s3_id, snapshot_names, cluster_batch=1): + def bdev_lvol_s3_backup(self, s3_id, snapshot_names, s3_bdev, cluster_batch=1): """Start an async backup of snapshots to S3. Args: - s3_id: unique backup identifier (uint32) - snapshot_names: list of snapshot composite bdev names + s3_id: unique backup identifier (uint32, < 2**30) + snapshot_names: snapshot composite bdev names, NEWEST first -- the + data plane unions their cluster maps first-writer-wins, so the + newest snapshot's allocation must be seen first. + s3_bdev: which S3 device to write through. Required: an lvstore may + carry several, and "the first" is ambiguous. cluster_batch: batch size in clusters (default 1) Returns RPC result (truthy on success). Poll with bdev_lvol_transfer_stat. """ params = { "s3_id": s3_id, "snapshot_names": snapshot_names, + "s3_bdev": s3_bdev, "cluster_batch": cluster_batch, } return self._request("bdev_lvol_s3_backup", params) @@ -1951,19 +1956,23 @@ def bdev_lvol_s3_backup(self, s3_id, snapshot_names, cluster_batch=1): # (pass snapshot bdev name) and recovery (pass target lvol name). # Merge has lvol=NULL on data plane so transfer_stat cannot poll it. - def bdev_lvol_s3_merge(self, s3_id, old_s3_id, cluster_batch, lvs_name=None): + def bdev_lvol_s3_merge(self, s3_id, old_s3_id, cluster_batch, s3_bdev, lvs_name=None): """Merge two backups: keep s3_id and merge old_s3_id into it. - This shortens the backup chain.""" + + This shortens the backup chain. Both backups must live in the bucket + served by s3_bdev -- a merge reads one and writes the other. + """ params = { "s3_id": s3_id, "old_s3_id": old_s3_id, "cluster_batch": cluster_batch, + "s3_bdev": s3_bdev, } if lvs_name: params["lvs_name"] = lvs_name return self._request("bdev_lvol_s3_merge", params) - def bdev_lvol_s3_recovery(self, lvol_name, s3_ids, cluster_batch, s3_bdev=None): + def bdev_lvol_s3_recovery(self, lvol_name, s3_ids, cluster_batch, s3_bdev): """Restore a chain of S3 backups into a new lvol. Args: lvol_name: target lvol name to restore into @@ -1972,18 +1981,16 @@ def bdev_lvol_s3_recovery(self, lvol_name, s3_ids, cluster_batch, s3_bdev=None): offers it (prepare_s3_clusters is first-writer-wins), so the newest backup's data must win. cluster_batch: batch size in clusters - s3_bdev: which S3 device to read from. Omitted, the data plane picks - the first S3 device attached to the lvstore, which is ambiguous - once a restore has attached a second one for a foreign bucket. + s3_bdev: which S3 device to read from. Required: a restore from a + foreign bucket attaches a second device, so "the first" is + ambiguous exactly when it matters. """ - params = { + return self._request("bdev_lvol_s3_recovery", { "lvol_name": lvol_name, "cluster_batch": cluster_batch, "s3_ids": s3_ids, - } - if s3_bdev: - params["s3_bdev"] = s3_bdev - return self._request("bdev_lvol_s3_recovery", params) + "s3_bdev": s3_bdev, + }) def bdev_s3_delete(self, name): """Delete an S3 bdev. diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index ae27add90..d4312c908 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -84,7 +84,9 @@ def _run_backup(task): if backup.status == Backup.STATUS_PENDING: try: - ret = rpc_client.bdev_lvol_s3_backup(backup.s3_id, [snap_bdev_name], cluster_batch=16) + ret = rpc_client.bdev_lvol_s3_backup( + backup.s3_id, [snap_bdev_name], + backup_controller.primary_s3_bdev_name(snode), cluster_batch=16) if not ret: _fail_backup(backup, task, "bdev_lvol_s3_backup RPC failed") return @@ -408,7 +410,10 @@ def _run_merge(task): if not merge_started: try: - ret = rpc_client.bdev_lvol_s3_merge(keep_backup.s3_id, old_backup.s3_id, cluster_batch=16, lvs_name=snode.lvstore) + ret = rpc_client.bdev_lvol_s3_merge( + keep_backup.s3_id, old_backup.s3_id, cluster_batch=16, + s3_bdev=backup_controller.primary_s3_bdev_name(snode), + lvs_name=snode.lvstore) if not ret: task.function_result = "bdev_lvol_s3_merge RPC failed" task.retry += 1 diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index 8d094117f..a0e39d307 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -301,7 +301,6 @@ class TestCreateS3Bdev(unittest.TestCase): def test_success(self, MockRPC, mock_boto3_client): mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.return_value = (True, None) mock_rpc.bdev_lvol_s3_bdev.return_value = True mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} @@ -337,7 +336,6 @@ def test_bdev_s3_create_fails(self, MockRPC): node = _node() with pytest.raises(RuntimeError): create_s3_bdev(node, _backup_config()) - mock_rpc.bdev_s3_add_bucket_name.assert_not_called() mock_rpc.bdev_lvol_s3_bdev.assert_not_called() @patch("simplyblock_core.backup_manifest.boto3.client") @@ -355,8 +353,6 @@ def test_bucket_is_a_create_parameter(self, MockRPC, mock_boto3_client): _, kwargs = mock_rpc.bdev_s3_create.call_args assert kwargs["bucket_name"] == "simplyblock-backup-cluster-1" - assert not hasattr(mock_rpc, "_mock_children") or \ - "bdev_s3_add_bucket_name" not in mock_rpc.method_calls @patch("simplyblock_core.backup_manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") @@ -364,7 +360,6 @@ def test_attach_fails(self, MockRPC, mock_boto3_client): from simplyblock_core.rpc_client import RPCRemoteError mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.return_value = (True, None) mock_rpc.bdev_lvol_s3_bdev.side_effect = RPCRemoteError("attach failed", code=-1) mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} @@ -379,7 +374,6 @@ def test_attach_fails(self, MockRPC, mock_boto3_client): def test_local_testing_params(self, MockRPC, mock_boto3_client): mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.return_value = (True, None) mock_rpc.bdev_lvol_s3_bdev.return_value = True mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} @@ -419,7 +413,6 @@ def test_no_credentials_defers_to_the_provider_chain(self, MockRPC, mock_boto3_c """An absent key pair must mean "use the node's IAM role", not "send empty keys".""" mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.return_value = (True, None) mock_rpc.bdev_lvol_s3_bdev.return_value = True mock_boto3_client.return_value.head_bucket.return_value = {} From 056bf9205809925319656d7e5f3df220ad137275 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Mon, 17 Aug 2026 17:16:20 +0200 Subject: [PATCH 11/14] Add `backup discover`, and bucket credentials to import and restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI could not reach a bucket except through a cluster's own configuration, which is exactly what a disaster recovery does not have. Three additions: sbctl backup discover --bucket … --region … lists what a bucket contains, reading its manifests. Takes no cluster at all. sbctl backup import --bucket … registers those backups, as an alternative to --from-file. sbctl backup restore … --access-key-id … restores from a bucket that is --secret-access-key … not this cluster's own. `backup import` grows --from-file and requires exactly one of that or --bucket; the old positional metadata_file is gone, because "a file" is no longer the only place manifests come from. Its file is parsed into manifests here rather than in the controller, so a malformed export is reported against the file, by name. The discover listing works from the manifest models the controller hands back. It reports chain length by walking prev_backup_id over the set rather than reading a stored chain, and prints "incomplete" for a backup whose ancestor is missing from the bucket rather than refusing the whole listing -- an incomplete chain is the finding an operator is looking for, not an error. It names the KMS an encrypted backup depends on, which is what decides whether a recovery can proceed. cli.py is regenerated with `tox run -e generate`, never edited. Also cleans up bdev_s3_create's signature, which had carried sentinel defaults over from the shape it replaced. `0` for the two CPU masks and the thread-pool size, and `""` for endpoint and region, all meant "not specified" -- and the data plane already reads zero that way (bdev_s3_impl.cpp:1204, 1222, 1239), so an explicit 0 behaved as absent while reading as a deliberate choice. Those are now Optional and omitted from the payload when None. Nothing else is special-cased: a caller that passes an empty string meant to, and the data plane treats it as absent. secondary_target, with_compression and snapshot_backups lose their defaults entirely: they are decisions the caller has already made, and defaulting them invites the wrong one silently. `region` goes the other way and joins the optional group, because the data plane leaves its SDK config untouched when it is absent (`if (region && *region)`) and its RPC decoder marks the field optional -- so it behaves exactly like the credentials, and requiring it here would have been this layer inventing a constraint the layer below does not have. The S3 RPC surface is annotated, and the node parameters of the functions that drive it with it. Between them they close a hole: `node` was implicitly Any, so `node.rpc_client()` was Any too and every argument to every S3 RPC went unchecked, however carefully bdev_s3_create itself was typed. With both ends annotated, mypy now rejects a str where a list is wanted, or an Optional where a str is -- which is the class of mistake that made region look mandatory in the first place. _compute_s3_cpu_masks returns None rather than 0 where the node does not say, since a zero mask selects no CPUs and only ever meant "unset". --- simplyblock_cli/cli-reference.yaml | 82 ++++++++++++- simplyblock_cli/cli.py | 26 +++- simplyblock_cli/clibase.py | 110 ++++++++++++++--- .../controllers/backup_controller.py | 31 ++--- simplyblock_core/rpc_client.py | 106 +++++++++------- tests/integration/test_backup.py | 6 +- tests/unit/rpc/test_client.py | 1 + tests/unit/test_backup_cli.py | 113 ++++++++++++++++++ tests/unit/test_client_secret_logging.py | 30 ++--- 9 files changed, 402 insertions(+), 103 deletions(-) create mode 100644 tests/unit/test_backup_cli.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 8cff69738..ca556f4bd 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -2511,6 +2511,14 @@ commands: help: "The target storage node id." dest: node type: str + - name: "--access-key-id" + help: "Access key for the backup's bucket, when it is not this cluster's own." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the backup's bucket, when it is not this cluster's own." + dest: secret_access_key + type: secret - name: export help: "Export backup metadata to a JSON file for cross-cluster restore." arguments: @@ -2528,17 +2536,81 @@ commands: help: "The output file path." dest: output type: str - - name: import - help: "Import backup metadata from a JSON file." + - name: discover + help: "List the backups a bucket contains, reading its manifests. Needs no cluster." arguments: - - name: "metadata_file" - help: "The path to JSON metadata file." - dest: metadata_file + - name: "--bucket" + help: "The bucket holding the backups." + dest: bucket + type: str + required: true + - name: "--region" + help: "The bucket's region. Omit to let the AWS SDK resolve it." + dest: region + type: str + - name: "--endpoint" + help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS." + dest: endpoint type: str + - name: "--access-key-id" + help: "Access key for the bucket. Omit to use the node's instance role." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the bucket. Omit to use the node's instance role." + dest: secret_access_key + type: secret + - name: "--no-verify-tls" + help: "Skip certificate verification for the endpoint." + dest: no_verify_tls + type: bool + action: store_true + - name: "--path-style" + help: "Use path-style addressing, as MinIO and most S3-compatible stores need." + dest: path_style + type: bool + action: store_true + - name: import + help: "Register backups into this cluster, from a bucket or from an exported file." + arguments: - name: "--cluster-id" help: "The target cluster to import into (required for cross-cluster restore)." dest: cluster_id type: str + - name: "--from-file" + help: "Path to a JSON file produced by 'backup export'. Mutually exclusive with --bucket." + dest: from_file + type: str + - name: "--bucket" + help: "Read manifests straight from this bucket. Mutually exclusive with --from-file." + dest: bucket + type: str + - name: "--region" + help: "The bucket's region. Omit to let the AWS SDK resolve it." + dest: region + type: str + - name: "--endpoint" + help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS." + dest: endpoint + type: str + - name: "--access-key-id" + help: "Access key for the bucket. Omit to use the node's instance role." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the bucket. Omit to use the node's instance role." + dest: secret_access_key + type: secret + - name: "--no-verify-tls" + help: "Skip certificate verification for the endpoint." + dest: no_verify_tls + type: bool + action: store_true + - name: "--path-style" + help: "Use path-style addressing, as MinIO and most S3-compatible stores need." + dest: path_style + type: bool + action: store_true - name: policy-add help: "Create a new backup policy." arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index ba1c86afb..b8cb315a9 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -1012,6 +1012,7 @@ def init_backup(self): self.init_backup__delete(subparser) self.init_backup__restore(subparser) self.init_backup__export(subparser) + self.init_backup__discover(subparser) self.init_backup__import(subparser) self.init_backup__policy_add(subparser) self.init_backup__policy_remove(subparser) @@ -1034,6 +1035,8 @@ def init_backup__restore(self, subparser): subcommand.add_argument('--lvol', help='The new logical volume name.', type=str, dest='lvol_name', required=True) subcommand.add_argument('--pool', help='The target pool name or id.', type=str, dest='pool', required=True) subcommand.add_argument('--node', help='The target storage node id.', type=str, dest='node') + subcommand.add_argument('--access-key-id', help='Access key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='secret_access_key') def init_backup__export(self, subparser): subcommand = self.add_sub_command(subparser, 'export', 'Export backup metadata to a JSON file for cross-cluster restore.') @@ -1041,10 +1044,27 @@ def init_backup__export(self, subparser): subcommand.add_argument('--lvol', help='Filter exports to a specific logical volume name.', type=str, dest='lvol_name') subcommand.add_argument('-o', '--output', help='The output file path.', type=str, dest='output') + def init_backup__discover(self, subparser): + subcommand = self.add_sub_command(subparser, 'discover', 'List the backups a bucket contains, reading its manifests. Needs no cluster.') + subcommand.add_argument('--bucket', help='The bucket holding the backups.', type=str, dest='bucket', required=True) + subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region') + subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint') + subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key') + subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true') + subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true') + def init_backup__import(self, subparser): - subcommand = self.add_sub_command(subparser, 'import', 'Import backup metadata from a JSON file.') - subcommand.add_argument('metadata_file', help='The path to JSON metadata file.', type=str) + subcommand = self.add_sub_command(subparser, 'import', 'Register backups into this cluster, from a bucket or from an exported file.') subcommand.add_argument('--cluster-id', help='The target cluster to import into (required for cross-cluster restore).', type=str, dest='cluster_id') + subcommand.add_argument('--from-file', help='Path to a JSON file produced by \'backup export\'. Mutually exclusive with --bucket.', type=str, dest='from_file') + subcommand.add_argument('--bucket', help='Read manifests straight from this bucket. Mutually exclusive with --from-file.', type=str, dest='bucket') + subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region') + subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint') + subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key') + subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true') + subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true') def init_backup__policy_add(self, subparser): subcommand = self.add_sub_command(subparser, 'policy-add', 'Create a new backup policy.') @@ -1546,6 +1566,8 @@ def run(self): ret = self.backup__restore(sub_command, args) elif sub_command in ['export']: ret = self.backup__export(sub_command, args) + elif sub_command in ['discover']: + ret = self.backup__discover(sub_command, args) elif sub_command in ['import']: ret = self.backup__import(sub_command, args) elif sub_command in ['policy-add']: diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 5a881353e..20d6161b9 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -9,6 +9,7 @@ import argcomplete from simplyblock_core import cluster_ops, utils, db_controller, constants +from simplyblock_core import backup_manifest from simplyblock_core.backup_manifest import BackupManifest from simplyblock_core.exceptions import MigrationConflictError, PreconditionError from simplyblock_core import storage_node_ops as storage_ops @@ -17,6 +18,8 @@ tasks_controller, qos_controller, migration_controller, backup_controller, fdb_backup_controller from simplyblock_core.controllers import health_controller from simplyblock_core.models.pool import Pool +from simplyblock_core.backup_manifest import ManifestError +from simplyblock_core.models.backup_config import BackupConfig, S3Credentials from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings @@ -73,6 +76,44 @@ def _format_result(data, *, json: bool) -> str: return _format_json(data) if json else utils.print_table(data, unwrap_secrets=True) +def _format_timestamp(seconds) -> str: + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(seconds)) if seconds else "" + + +def _s3_credentials(args): + """The bucket credentials given on the command line, if any. + + Returns None when neither key is supplied, which means "use the node's + instance role" rather than "use empty credentials". + """ + access_key_id = getattr(args, 'access_key_id', None) + secret_access_key = getattr(args, 'secret_access_key', None) + if not access_key_id and not secret_access_key: + return None + if not (access_key_id and secret_access_key): + raise ValueError( + "give both --access-key-id and --secret-access-key, or neither") + return S3Credentials(access_key_id=access_key_id, + secret_access_key=secret_access_key) + + +def _bucket_config(args) -> BackupConfig: + """A backup configuration assembled from bucket arguments. + + Used by the commands that read a bucket directly rather than through a + cluster -- which is the point of them, since after a disaster there may be no + cluster left to ask. + """ + return BackupConfig( + bucket_name=args.bucket, + region=getattr(args, 'region', None) or None, + endpoint=getattr(args, 'endpoint', None) or None, + verify_tls=not getattr(args, 'no_verify_tls', False), + use_path_style=getattr(args, 'path_style', False), + credentials=_s3_credentials(args), + ) + + class CLIWrapperBase: def __init__(self): @@ -1045,13 +1086,44 @@ def backup__restore(self, sub_command, args): try: lvol_id = backup_controller.restore_backup( args.backup_id, args.lvol_name, args.pool, - target_node_id=getattr(args, 'node', None)) - except (PreconditionError, RuntimeError) as e: + target_node_id=getattr(args, 'node', None), + s3_credentials=_s3_credentials(args)) + except (PreconditionError, RuntimeError, ValueError) as e: print(f"Error: {e}") return False print(f"Restoring backup {args.backup_id} into new volume {lvol_id}") return True + def backup__discover(self, sub_command, args): + try: + manifests = backup_controller.discover_backups(_bucket_config(args)) + except (ManifestError, ValueError) as e: + print(f"Error: {e}") + return False + if not manifests: + print(f"No backups found in {args.bucket}") + return False + # Each manifest names only its predecessor, so the chain is walked over + # the set. A bucket holding a manifest whose ancestor is missing is worth + # showing rather than refusing: that IS the finding. + def chain_length(manifest): + try: + return str(len(backup_manifest.chain_of(manifest, manifests))) + except ManifestError: + return "incomplete" + + return [{ + "ID": m.backup_id, + "Volume": m.volume.lvol_name, + "Snapshot": m.volume.snapshot_name, + "Size": m.size, + "Chain": chain_length(m), + "Encrypted": "yes" if m.encryption.encrypted else "no", + "Needs KMS": ( + m.encryption.descriptor.kms if m.encryption.descriptor else "-"), + "Created": _format_timestamp(m.created_at), + } for m in manifests] + def backup__export(self, sub_command, args): manifests = backup_controller.export_backups( cluster_id=getattr(args, 'cluster_id', None), @@ -1070,25 +1142,31 @@ def backup__export(self, sub_command, args): return True def backup__import(self, sub_command, args): - try: - with open(args.metadata_file, 'r') as f: - entries = json.load(f) - except Exception as e: - print(f"Error reading metadata file: {e}") + from_file = getattr(args, 'from_file', None) + bucket = getattr(args, 'bucket', None) + + if bool(from_file) == bool(bucket): + print("Error: give exactly one of --from-file or --bucket") return False - if not isinstance(entries, list): - entries = [entries] - # Parsed here rather than in the controller so a malformed file is - # reported as a problem with the file, naming it. + cluster_id = getattr(args, 'cluster_id', None) try: - manifests = [BackupManifest.model_validate(entry) for entry in entries] - except ValueError as e: - print(f"{args.metadata_file} is not a backup export: {e}") + if bucket: + count = backup_controller.import_from_bucket( + _bucket_config(args), cluster_id=cluster_id) + else: + with open(str(from_file), 'r') as f: + entries = json.load(f) + if not isinstance(entries, list): + entries = [entries] + # Parsed here rather than in the controller so a malformed file + # is reported as a problem with the file, naming it. + manifests = [BackupManifest.model_validate(e) for e in entries] + count = backup_controller.import_backups(manifests, cluster_id=cluster_id) + except (ManifestError, PreconditionError, ValueError, OSError) as e: + print(f"Error: {e}") return False - count = backup_controller.import_backups( - manifests, cluster_id=getattr(args, 'cluster_id', None)) print(f"Imported {count} backup(s)") return True diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py index 6796b252d..34c168ba7 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup_controller.py @@ -83,21 +83,24 @@ def _get_latest_backup_for_lvol(lvol_id): return valid[0] -def _compute_s3_cpu_masks(node): - """Compute CPU masks for the S3 bdev. +def _compute_s3_cpu_masks(node: StorageNode): + """CPU masks for the S3 bdev, or None where the node does not say. + Returns (bdb_lcpu_mask, s3_lcpu_mask): bdb_lcpu_mask: app_thread core (SPDK lightweight thread, low overhead) s3_lcpu_mask: all system vCPUs (no pinning — let Linux scheduler handle the AWS SDK thread pool; the data plane default would wrongly pin onto SPDK reactor cores) + + None rather than 0 for "the node does not tell us": a zero mask selects no + CPUs at all, and the data plane reads it as "unset" anyway, so returning it + would be a sentinel dressed as a value. """ # SPDK thread for the bdev poller — reuse the app thread core - bdb_lcpu_mask = 0 - if node.app_thread_mask: - bdb_lcpu_mask = int(node.app_thread_mask, 16) + bdb_lcpu_mask = int(node.app_thread_mask, 16) if node.app_thread_mask else None # AWS SDK thread pool — set all system vCPU bits so threads are unconstrained - s3_lcpu_mask = (1 << node.cpu) - 1 if node.cpu > 0 else 0 + s3_lcpu_mask = (1 << node.cpu) - 1 if node.cpu > 0 else None return bdb_lcpu_mask, s3_lcpu_mask @@ -322,12 +325,12 @@ def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: ) -def primary_s3_bdev_name(node) -> str: +def primary_s3_bdev_name(node: StorageNode) -> str: """The S3 device holding the cluster's own backup bucket.""" return f"s3_{node.lvstore}" -def create_restore_s3_bdev(node, config: BackupConfig, name: str) -> None: +def create_restore_s3_bdev(node: StorageNode, config: BackupConfig, name: str) -> None: """Attach a second S3 device to a node, for a bucket that is not its own. A restore from a foreign bucket needs different credentials, a different @@ -348,7 +351,7 @@ def create_restore_s3_bdev(node, config: BackupConfig, name: str) -> None: secondary_target=config.secondary_target, with_compression=config.with_compression, snapshot_backups=config.snapshot_backups, - endpoint=config.endpoint_url or "", + endpoint=config.endpoint_url, region=config.region, verify_tls=config.verify_tls, use_path_style=config.use_path_style, @@ -356,7 +359,7 @@ def create_restore_s3_bdev(node, config: BackupConfig, name: str) -> None: secret_access_key=config.credentials.secret_access_key if config.credentials else None, bdb_lcpu_mask=bdb_lcpu_mask, s3_lcpu_mask=s3_lcpu_mask, - s3_thread_pool_size=config.s3_thread_pool_size or 0, + s3_thread_pool_size=config.s3_thread_pool_size, ) rpc_client.bdev_lvol_s3_bdev(node.lvstore, name) except RPCException as e: @@ -368,7 +371,7 @@ def create_restore_s3_bdev(node, config: BackupConfig, name: str) -> None: name, config.bucket_name, node.get_id()) -def delete_restore_s3_bdev(node, name: str) -> None: +def delete_restore_s3_bdev(node: StorageNode, name: str) -> None: """Detach a device created by :func:`create_restore_s3_bdev`. Best-effort by design: this runs on the restore's terminal paths, and a @@ -539,7 +542,7 @@ def _ensure_s3_bucket(config: BackupConfig, bucket_name): raise RuntimeError(f"Error ensuring S3 bucket {bucket_name} exists") from e -def create_s3_bdev(node, config: BackupConfig) -> None: +def create_s3_bdev(node: StorageNode, config: BackupConfig) -> None: """Create the S3 bdev and attach it to a node's lvstore. Called during cluster activate / node restart. Args: @@ -571,7 +574,7 @@ def create_s3_bdev(node, config: BackupConfig) -> None: secondary_target=config.secondary_target, with_compression=config.with_compression, snapshot_backups=config.snapshot_backups, - endpoint=config.endpoint_url or "", + endpoint=config.endpoint_url, region=config.region, verify_tls=config.verify_tls, use_path_style=config.use_path_style, @@ -579,7 +582,7 @@ def create_s3_bdev(node, config: BackupConfig) -> None: secret_access_key=config.credentials.secret_access_key if config.credentials else None, bdb_lcpu_mask=bdb_lcpu_mask, s3_lcpu_mask=s3_lcpu_mask, - s3_thread_pool_size=config.s3_thread_pool_size or 0, + s3_thread_pool_size=config.s3_thread_pool_size, ) rpc_client.bdev_lvol_s3_bdev(node.lvstore, s3_bdev_name) diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 084c00590..a3b23c48b 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -2,7 +2,7 @@ import json from enum import IntEnum from json import JSONDecodeError -from typing import Any, Optional +from typing import Any, List, Optional import jsonschema import requests @@ -1752,7 +1752,7 @@ def bdev_lvol_transfer(self, name, offset, batch_size, bdev_name, operation="mig "operation": operation, }) - def bdev_lvol_transfer_stat(self, name): + def bdev_lvol_transfer_stat(self, name: str): """ Return transfer status for *name* (source composite bdev). @@ -1852,42 +1852,57 @@ def bdev_lvol_batch_transfer_final_step(self, lvol_names, lvol_ids, snapshot_nam # ---- S3 Backup RPCs ---- - def bdev_s3_create(self, name: str, bucket_name: str, secondary_target: int = 0, - with_compression: bool = False, snapshot_backups: bool = True, - endpoint: str = "", region: str = "", verify_tls: bool = True, + def bdev_s3_create(self, name: str, bucket_name: str, + secondary_target: int, with_compression: bool, + snapshot_backups: bool, verify_tls: bool = True, use_path_style: bool = False, + region: Optional[str] = None, + endpoint: Optional[str] = None, access_key_id: Optional[SecretStr] = None, secret_access_key: Optional[SecretStr] = None, - bdb_lcpu_mask: int = 0, s3_lcpu_mask: int = 0, - s3_thread_pool_size: int = 0): + bdb_lcpu_mask: Optional[int] = None, + s3_lcpu_mask: Optional[int] = None, + s3_thread_pool_size: Optional[int] = None): """Create an S3 bdev for one bucket. A device serves exactly one bucket with one set of credentials, so reading a second bucket means creating a second device. Attach it to an lvstore with bdev_lvol_s3_bdev. + What the device cannot function without has no default. What the data + plane has its own default for is Optional and omitted from the payload + when absent. Nothing here uses ``0`` or ``""`` to mean "unset": the data + plane already reads zero that way for the masks and the pool size + (bdev_s3_impl.cpp:1204, 1222, 1239), so an explicit 0 would behave as + absent while reading as a deliberate choice. + Args: - bucket_name: the bucket this device reads and writes. Required -- a - device without one cannot service any I/O. - endpoint: an S3-compatible endpoint, e.g. "http://minio:9000". - Empty means the SDK resolves AWS's endpoint from the region. - region: AWS region. Empty means the SDK resolves it from the - environment or instance metadata. + bucket_name: the bucket this device reads and writes. A device + without one cannot service any I/O. + region: the bucket's region. Absent means the SDK resolves it, the + same way it resolves credentials -- the data plane leaves its + config untouched when this is not given. + secondary_target: 0=S3, 1=FileSystem. The caller holds a + SecondaryTarget enum and converts at this boundary. + with_compression: enable ISA-L compression. + snapshot_backups: selects the backup object layout + ({s3_id}/{mid}/{extent}) rather than the tiering one. verify_tls: verify the endpoint's certificate. use_path_style: path-style addressing, needed by MinIO and most S3-compatible stores. - access_key_id / secret_access_key: leave both ``None`` to use the - node's instance role via the SDK's default credential provider - chain. An empty ``SecretStr`` counts as absent too, rather than - travelling as a key: the SDK reads empty credentials as a valid - anonymous identity and then never consults the chain. - secondary_target: 0=S3, 1=FileSystem - with_compression: Enable ISA-L compression - snapshot_backups: selects the backup object layout - ({s3_id}/{mid}/{extent}) rather than the tiering one - bdb_lcpu_mask: CPU mask for the SPDK thread of this bdev (uint64) - s3_lcpu_mask: CPU mask for the internal AWS S3 thread pool (uint64) - s3_thread_pool_size: AWS S3 thread pool size (default 32 on data plane) + endpoint: an S3-compatible endpoint, e.g. "http://minio:9000". + Absent means the SDK resolves AWS's endpoint from the region. + access_key_id / secret_access_key: absent means the node's instance + role, via the SDK's default credential provider chain. Callers + derive them from an S3Credentials pair, so they arrive together + or not at all. + bdb_lcpu_mask: CPU mask for this bdev's SPDK thread. Absent lets the + data plane derive one from the app core mask. + s3_lcpu_mask: CPU mask for the AWS SDK thread pool. Absent leaves the + data plane's own choice, which pins onto SPDK reactor cores -- + so callers that care should compute one. + s3_thread_pool_size: AWS SDK thread pool size. Absent means the data + plane's default of 32. """ params: dict[str, Any] = { "name": name, @@ -1898,20 +1913,16 @@ def bdev_s3_create(self, name: str, bucket_name: str, secondary_target: int = 0, "verify_tls": verify_tls, "use_path_style": use_path_style, } - if endpoint: - params["endpoint"] = endpoint - if region: - params["region"] = region - if access_key_id: - params["access_key_id"] = access_key_id - if secret_access_key: - params["secret_access_key"] = secret_access_key - if bdb_lcpu_mask: - params["bdb_lcpu_mask"] = bdb_lcpu_mask - if s3_lcpu_mask: - params["s3_lcpu_mask"] = s3_lcpu_mask - if s3_thread_pool_size: - params["s3_thread_pool_size"] = s3_thread_pool_size + optional: dict[str, Any] = { + "region": region, + "endpoint": endpoint, + "access_key_id": access_key_id, + "secret_access_key": secret_access_key, + "bdb_lcpu_mask": bdb_lcpu_mask, + "s3_lcpu_mask": s3_lcpu_mask, + "s3_thread_pool_size": s3_thread_pool_size, + } + params.update({k: v for k, v in optional.items() if v is not None}) return self._request3("bdev_s3_create", **params) def bdev_lvol_create_poller_group(self, cpu_mask): @@ -1922,7 +1933,7 @@ def bdev_lvol_create_poller_group(self, cpu_mask): """ return self._request3("bdev_lvol_create_poller_group", cpu_mask=cpu_mask) - def bdev_lvol_s3_bdev(self, lvs_name, bdev_name): + def bdev_lvol_s3_bdev(self, lvs_name: str, bdev_name: str): """Attach an S3 bdev to the given lvstore. The S3 bdev must already exist (created via bdev_s3_create). Called once per lvstore at setup time (cluster activate, node restart).""" @@ -1931,7 +1942,8 @@ def bdev_lvol_s3_bdev(self, lvs_name, bdev_name): "s3_bdev": bdev_name, }) - def bdev_lvol_s3_backup(self, s3_id, snapshot_names, s3_bdev, cluster_batch=1): + def bdev_lvol_s3_backup(self, s3_id: int, snapshot_names: List[str], + s3_bdev: str, cluster_batch: int = 1): """Start an async backup of snapshots to S3. Args: s3_id: unique backup identifier (uint32, < 2**30) @@ -1956,13 +1968,14 @@ def bdev_lvol_s3_backup(self, s3_id, snapshot_names, s3_bdev, cluster_batch=1): # (pass snapshot bdev name) and recovery (pass target lvol name). # Merge has lvol=NULL on data plane so transfer_stat cannot poll it. - def bdev_lvol_s3_merge(self, s3_id, old_s3_id, cluster_batch, s3_bdev, lvs_name=None): + def bdev_lvol_s3_merge(self, s3_id: int, old_s3_id: int, cluster_batch: int, + s3_bdev: str, lvs_name: Optional[str] = None): """Merge two backups: keep s3_id and merge old_s3_id into it. This shortens the backup chain. Both backups must live in the bucket served by s3_bdev -- a merge reads one and writes the other. """ - params = { + params: dict[str, Any] = { "s3_id": s3_id, "old_s3_id": old_s3_id, "cluster_batch": cluster_batch, @@ -1972,7 +1985,8 @@ def bdev_lvol_s3_merge(self, s3_id, old_s3_id, cluster_batch, s3_bdev, lvs_name= params["lvs_name"] = lvs_name return self._request("bdev_lvol_s3_merge", params) - def bdev_lvol_s3_recovery(self, lvol_name, s3_ids, cluster_batch, s3_bdev): + def bdev_lvol_s3_recovery(self, lvol_name: str, s3_ids: List[int], + cluster_batch: int, s3_bdev: str): """Restore a chain of S3 backups into a new lvol. Args: lvol_name: target lvol name to restore into @@ -1992,14 +2006,14 @@ def bdev_lvol_s3_recovery(self, lvol_name, s3_ids, cluster_batch, s3_bdev): "s3_bdev": s3_bdev, }) - def bdev_s3_delete(self, name): + def bdev_s3_delete(self, name: str): """Delete an S3 bdev. Used to release the device a restore attached for a foreign bucket. """ return self._request3("bdev_s3_delete", name=name) - def bdev_lvol_s3_delete(self, s3_ids): + def bdev_lvol_s3_delete(self, s3_ids: List[int]): """Delete all S3 backups for the given IDs (list of uint32).""" # RPC still missing on data plane — use dummy return self._request("bdev_lvol_s3_delete", { diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index a0e39d307..8b8dc4776 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -271,7 +271,9 @@ def test_no_app_thread_mask(self): node = _node() node.app_thread_mask = "" bdb, s3 = _compute_s3_cpu_masks(node) - self.assertEqual(bdb, 0) # falls back to data plane default + # None, not 0: a zero mask selects no CPUs, and the RPC omits the + # parameter so the data plane derives one from the app core mask. + self.assertIsNone(bdb) self.assertEqual(s3, 0xFF) def test_no_cpu_count(self): @@ -280,7 +282,7 @@ def test_no_cpu_count(self): node.cpu = 0 bdb, s3 = _compute_s3_cpu_masks(node) self.assertEqual(bdb, 0x8) - self.assertEqual(s3, 0) # falls back to data plane default + self.assertIsNone(s3) # omitted; data plane picks def test_large_cpu_count(self): from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks diff --git a/tests/unit/rpc/test_client.py b/tests/unit/rpc/test_client.py index 1a146cfac..246bb96e5 100644 --- a/tests/unit/rpc/test_client.py +++ b/tests/unit/rpc/test_client.py @@ -91,3 +91,4 @@ def test_subsystem_get_other_rpc_error_propagates(self, mock_req): if __name__ == "__main__": unittest.main() + diff --git a/tests/unit/test_backup_cli.py b/tests/unit/test_backup_cli.py new file mode 100644 index 000000000..86441e32d --- /dev/null +++ b/tests/unit/test_backup_cli.py @@ -0,0 +1,113 @@ +"""The backup CLI's argument handling. + +Focused on the two helpers that turn command-line arguments into a +``BackupConfig``, because that is where a disaster-recovery operator's typing +becomes the thing that decides whether a bucket can be read at all. +""" +import pytest + +from simplyblock_cli import clibase +from simplyblock_core.models.backup_config import BackupConfig + + +class _Args: + """Stand-in for the argparse namespace, with only the attributes set.""" + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + +class TestCredentials: + + def test_absent_keys_mean_the_instance_role(self): + """Not "empty credentials" -- that is what broke IAM roles in the first place.""" + assert clibase._s3_credentials(_Args()) is None + + def test_both_keys_are_carried(self): + credentials = clibase._s3_credentials( + _Args(access_key_id="AKIA", secret_access_key="shh")) + + assert credentials.access_key_id.get_secret_value() == "AKIA" + assert credentials.secret_access_key.get_secret_value() == "shh" + + @pytest.mark.parametrize("given", [ + {"access_key_id": "AKIA"}, + {"secret_access_key": "shh"}, + ]) + def test_half_a_pair_is_refused(self, given): + """Half a key pair reaches S3 as an authentication failure with no clue why.""" + with pytest.raises(ValueError, match="both"): + clibase._s3_credentials(_Args(**given)) + + +class TestBucketConfig: + + def _args(self, **overrides): + return _Args(**{"bucket": "backups", "region": "eu-central-1", **overrides}) + + def test_minimal_bucket(self): + config = clibase._bucket_config(self._args()) + + assert config.bucket_name == "backups" + assert config.region == "eu-central-1" + assert config.endpoint is None + assert config.credentials is None + assert config.verify_tls is True + assert config.use_path_style is False + + def test_endpoint_and_addressing(self): + config = clibase._bucket_config(self._args( + endpoint="http://minio:9000", path_style=True, no_verify_tls=True)) + + assert config.endpoint_url == "http://minio:9000" + assert config.use_path_style is True + assert config.verify_tls is False + + def test_empty_endpoint_is_absent_not_blank(self): + """argparse gives "" for an unset str; the model must not see that.""" + assert clibase._bucket_config(self._args(endpoint="")).endpoint is None + + def test_the_flag_is_negated_the_way_it_reads(self): + """--no-verify-tls sets verify_tls=False, not the other way round.""" + assert clibase._bucket_config(self._args()).verify_tls is True + assert clibase._bucket_config( + self._args(no_verify_tls=True)).verify_tls is False + + def test_an_omitted_region_defers_to_the_sdk(self): + """--region is optional, like the credentials: absent means "resolve it". + + Both spellings argparse can produce for an unsupplied option reach the + model as absence, rather than one of them becoming a region named "". + """ + assert clibase._bucket_config(_Args(bucket="backups", region=None)).region is None + assert clibase._bucket_config(_Args(bucket="backups", region="")).region is None + + def test_produces_a_usable_config(self): + config = clibase._bucket_config(self._args( + access_key_id="AKIA", secret_access_key="shh")) + + assert isinstance(config, BackupConfig) + assert config.location().bucket_name == "backups" + + +class TestRegisteredCommands: + + def _cli(self): + import sys + sys.argv = ['sbcli'] + from simplyblock_cli.cli import CLIWrapper + return CLIWrapper() + + def test_discover_is_registered(self): + """The disaster-recovery entry point: needs a bucket, not a cluster.""" + assert hasattr(self._cli(), 'init_backup__discover') + assert hasattr(clibase.CLIWrapperBase, 'backup__discover') + + def test_source_switch_is_gone(self): + """It never switched anything; see the data-plane bucket selection.""" + cli = self._cli() + assert not hasattr(cli, 'init_backup__source_switch') + assert not hasattr(cli, 'init_backup__source_list') + assert not hasattr(clibase.CLIWrapperBase, 'backup__source_switch') + assert not hasattr(clibase.CLIWrapperBase, 'backup__source_list') diff --git a/tests/unit/test_client_secret_logging.py b/tests/unit/test_client_secret_logging.py index 471fc99ae..87662dea1 100644 --- a/tests/unit/test_client_secret_logging.py +++ b/tests/unit/test_client_secret_logging.py @@ -94,7 +94,8 @@ def test_bdev_s3_create_keys_reach_the_wire_but_not_the_log(rpc_client, caplog): with caplog.at_level(logging.DEBUG): rpc_client.bdev_s3_create( - name="s3_lvs_test", bucket_name="bucket", + name="s3_lvs_test", bucket_name="bucket", region="eu-central-1", + secondary_target=0, with_compression=False, snapshot_backups=True, access_key_id=SecretStr("AKIAEXAMPLE"), secret_access_key=SecretStr("s3cr3t"), ) @@ -114,29 +115,22 @@ def test_bdev_s3_create_omits_absent_credentials(rpc_client): "jsonrpc": "2.0", "id": 1, "result": True, }) - rpc_client.bdev_s3_create(name="s3_lvs_test", bucket_name="bucket") + rpc_client.bdev_s3_create( + name="s3_lvs_test", bucket_name="bucket", region="eu-central-1", + secondary_target=0, with_compression=False, snapshot_backups=True) params = _sent_params(rpc_client) assert "access_key_id" not in params assert "secret_access_key" not in params + # Same for every other optional: the data plane reads 0 as "unset" for the + # masks and the pool size, so sending one would be indistinguishable from + # omitting it while reading as a deliberate choice. + for absent in ("endpoint", "bdb_lcpu_mask", "s3_lcpu_mask", "s3_thread_pool_size"): + assert absent not in params -def test_bdev_s3_create_does_not_send_empty_credentials_as_keys(rpc_client): - # An empty key pair is not an absent one to the AWS SDK: it reads as a valid - # anonymous identity, and the default provider chain (the node's instance - # role) is then never consulted. - rpc_client._fake_session.post.return_value = _make_json_response({ - "jsonrpc": "2.0", "id": 1, "result": True, - }) - - rpc_client.bdev_s3_create( - name="s3_lvs_test", bucket_name="bucket", - access_key_id=SecretStr(""), secret_access_key=SecretStr(""), - ) - - params = _sent_params(rpc_client) - assert "access_key_id" not in params - assert "secret_access_key" not in params + # ... and what is not optional is always present. + assert params["region"] == "eu-central-1" @pytest.fixture From 18005ff54cdbaf310b41c5d6a314ea8cb2034176 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Tue, 18 Aug 2026 14:06:50 +0200 Subject: [PATCH 12/14] Split the backup code into a controllers/backup package backup_controller.py had grown to ~1400 lines holding five separable concerns, and backup_manifest.py sat at simplyblock_core top level despite being backup's alone. Both are now one package, in dependency order: manifest the self-describing record written into the bucket, and the only thing that can interpret a backup's objects once the cluster that wrote them is gone. Owns the control plane's S3 access. validation whether a chain can be restored: the predicates, plus the one function that refuses. device the S3 devices a node reads and writes through, one bucket each, and the names they get. controller creating, restoring, importing, exporting, discovering. policy retention limits, tiered schedules, and the merges they cause. Each module may import the ones above it and none below, so the graph is a DAG: device and controller reach manifest, controller reaches validation, policy reaches controller. Nothing reaches back. Pure movement -- no function body is changed. The split was done by slicing the file at top-level definition boundaries and reassembling, so nothing was retyped and nothing can have drifted from what was reviewed in the preceding commits. Two names lose their leading underscore, create_single_backup and get_latest_backup_for_lvol, because policy legitimately calls them and an underscore means private to a module rather than to a package. __init__.py deliberately re-exports nothing. A caller writes from simplyblock_core.controllers.backup import controller as backup_controller so the import says which part of the subsystem it depends on -- and the callers show the split was worth making: cluster_ops and storage_node_ops need only device, tasks_runner_backup_merge only policy, and the v2 router and the CLI turn out to use controller and policy as two separate things. Flattening that back into one namespace through the package would undo the reason for splitting it. Every reference is updated, including the patch targets in the tests, which have to follow each function to the module it now lives in. tests/unit/test_imports.py imports all five modules, so a circular import between them fails the fast tier. --- simplyblock_cli/clibase.py | 20 +- simplyblock_core/cluster_ops.py | 5 +- .../controllers/backup/__init__.py | 26 + .../controller.py} | 610 +----------------- simplyblock_core/controllers/backup/device.py | 183 ++++++ .../backup/manifest.py} | 2 +- simplyblock_core/controllers/backup/policy.py | 340 ++++++++++ .../controllers/backup/validation.py | 112 ++++ .../controllers/snapshot_controller.py | 2 +- .../services/tasks_runner_backup.py | 18 +- .../services/tasks_runner_backup_merge.py | 6 +- simplyblock_core/storage_node_ops.py | 4 +- simplyblock_web/api/v2/_dtos.py | 2 +- simplyblock_web/api/v2/cluster/backup.py | 13 +- .../cluster/storage_pool/volume/__init__.py | 3 +- tests/integration/expansion_sim/conftest.py | 2 +- tests/integration/test_backup.py | 198 +++--- tests/integration/test_backup_encryption.py | 4 +- .../integration/test_backup_manifest_flow.py | 4 +- .../integration/test_backup_restore_source.py | 33 +- tests/integration/test_backup_validation.py | 21 +- tests/unit/test_backup_manifest.py | 4 +- .../test_backup_restore_node_selection.py | 6 +- tests/unit/test_imports.py | 6 +- tests/unit/web/api/v2/conftest.py | 6 + .../unit/web/api/v2/test_backup_endpoints.py | 6 +- 26 files changed, 863 insertions(+), 773 deletions(-) create mode 100644 simplyblock_core/controllers/backup/__init__.py rename simplyblock_core/controllers/{backup_controller.py => backup/controller.py} (60%) create mode 100644 simplyblock_core/controllers/backup/device.py rename simplyblock_core/{backup_manifest.py => controllers/backup/manifest.py} (99%) create mode 100644 simplyblock_core/controllers/backup/policy.py create mode 100644 simplyblock_core/controllers/backup/validation.py diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 20d6161b9..7058745bc 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -9,16 +9,18 @@ import argcomplete from simplyblock_core import cluster_ops, utils, db_controller, constants -from simplyblock_core import backup_manifest -from simplyblock_core.backup_manifest import BackupManifest +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup import policy as backup_policy +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, ManifestError) from simplyblock_core.exceptions import MigrationConflictError, PreconditionError from simplyblock_core import storage_node_ops as storage_ops from simplyblock_core import mgmt_node_ops as mgmt_ops from simplyblock_core.controllers import pool_controller, lvol_controller, snapshot_controller, device_controller, \ - tasks_controller, qos_controller, migration_controller, backup_controller, fdb_backup_controller + tasks_controller, qos_controller, migration_controller, fdb_backup_controller from simplyblock_core.controllers import health_controller from simplyblock_core.models.pool import Pool -from simplyblock_core.backup_manifest import ManifestError from simplyblock_core.models.backup_config import BackupConfig, S3Credentials from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings @@ -1171,7 +1173,7 @@ def backup__import(self, sub_command, args): return True def backup__policy_add(self, sub_command, args): - policy_id, error = backup_controller.add_policy( + policy_id, error = backup_policy.add_policy( args.cluster_id, args.name, max_versions=args.versions or 0, max_age=args.age or "", @@ -1183,7 +1185,7 @@ def backup__policy_add(self, sub_command, args): return True def backup__policy_remove(self, sub_command, args): - success, error = backup_controller.remove_policy(args.policy_id) + success, error = backup_policy.remove_policy(args.policy_id) if error: print(f"Error: {error}") return False @@ -1192,13 +1194,13 @@ def backup__policy_remove(self, sub_command, args): def backup__policy_list(self, sub_command, args): cluster_id = getattr(args, 'cluster_id', None) - data = backup_controller.list_policies(cluster_id) + data = backup_policy.list_policies(cluster_id) if data: return utils.print_table(data) return "No policies found" def backup__policy_attach(self, sub_command, args): - att_id, error = backup_controller.attach_policy( + att_id, error = backup_policy.attach_policy( args.policy_id, args.target_type, args.target_id) if error: print(f"Error: {error}") @@ -1207,7 +1209,7 @@ def backup__policy_attach(self, sub_command, args): return True def backup__policy_detach(self, sub_command, args): - success, error = backup_controller.detach_policy( + success, error = backup_policy.detach_policy( args.policy_id, args.target_type, args.target_id) if error: print(f"Error: {error}") diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 126fe8fe3..d89f4903e 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -20,7 +20,8 @@ from simplyblock_core import utils, scripts, constants, mgmt_node_ops, storage_node_ops from simplyblock_core.utils import port_block -from simplyblock_core.controllers import backup_controller, cluster_events, device_controller, qos_controller, tasks_controller, tcp_ports_events +from simplyblock_core.controllers import cluster_events, device_controller, qos_controller, tasks_controller, tcp_ports_events +from simplyblock_core.controllers.backup import device as backup_device from simplyblock_core.db_controller import DBController from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings, DeployConfig @@ -1184,7 +1185,7 @@ def _finish_pass1_node(node_id, ret) -> None: # Create S3 bdev for backup support (only if backup is configured) if cluster.backup_config: snode = db_controller.get_storage_node_by_id(node_id) - backup_controller.create_s3_bdev(snode, cluster.get_backup_config()) + backup_device.create_s3_bdev(snode, cluster.get_backup_config()) else: _set_lvstore_status(node_id, "failed") diff --git a/simplyblock_core/controllers/backup/__init__.py b/simplyblock_core/controllers/backup/__init__.py new file mode 100644 index 000000000..34a72bc40 --- /dev/null +++ b/simplyblock_core/controllers/backup/__init__.py @@ -0,0 +1,26 @@ +# coding=utf-8 +"""Volume backup to a secondary store. + +Five modules, in dependency order -- each one may use the ones above it: + +``manifest`` + The self-describing record written into the bucket next to a backup's data, + and the only thing that can interpret those objects once the cluster that + wrote them is gone. Owns the control plane's S3 access. +``validation`` + Whether a chain can be restored. Predicates, plus the one function that + refuses. +``device`` + The S3 devices a node reads and writes through. One bucket each. +``controller`` + Creating, restoring, importing, exporting and discovering backups. +``policy`` + Retention limits and tiered schedules, and the merges they cause. + +Deliberately not a facade: nothing is re-exported here. A caller writes + + from simplyblock_core.controllers.backup import controller as backup_controller + +so the import says which part of the subsystem it depends on. Flattening that +back into one namespace would undo the reason for splitting it. +""" diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup/controller.py similarity index 60% rename from simplyblock_core/controllers/backup_controller.py rename to simplyblock_core/controllers/backup/controller.py index 34c168ba7..c8b8d4124 100644 --- a/simplyblock_core/controllers/backup_controller.py +++ b/simplyblock_core/controllers/backup/controller.py @@ -1,16 +1,17 @@ # coding=utf-8 +"""Creating, restoring, importing, exporting and discovering backups.""" import logging -import re import time import uuid from typing import Iterable, List, Optional -from botocore.exceptions import BotoCoreError, ClientError - -from simplyblock_core import backup_manifest, constants +from simplyblock_core import constants from simplyblock_core.controllers import backup_events, tasks_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.validation import ( + chain_fits, require_restorable) from simplyblock_core.db_controller import DBController -from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment +from simplyblock_core.models.backup import Backup from simplyblock_core.models.backup_config import ( BackupConfig, BackupLocation, S3Credentials) from simplyblock_core.models.storage_node import StorageNode @@ -19,7 +20,6 @@ lvol_dek_path, pool_kek_name, ) from simplyblock_core.exceptions import PreconditionError -from simplyblock_core.rpc_client import RPCException logger = logging.getLogger() @@ -30,44 +30,7 @@ def _generate_backup_id(): return str(uuid.uuid4()) -def _parse_age_string(age_str): - """Parse age strings like '2d', '12h', '1w', '30m' into seconds.""" - match = re.match(r'^(\d+)([mhdw])$', age_str.strip()) - if not match: - raise ValueError(f"Invalid age format: {age_str}. Use e.g. 2d, 12h, 1w") - value = int(match.group(1)) - unit = match.group(2) - multipliers = {'m': 60, 'h': 3600, 'd': 86400, 'w': 604800} - return value * multipliers[unit] - - -def _parse_schedule(schedule_str): - """Parse schedule string like '15m,4 60m,11 24h,7' into list of (interval_seconds, keep_count) tuples. - Returns sorted list by interval ascending. Raises ValueError on invalid input.""" - if not schedule_str or not schedule_str.strip(): - return [] - tiers = [] - for part in schedule_str.strip().split(): - parts = part.split(',') - if len(parts) != 2: - raise ValueError(f"Invalid schedule tier: {part}. Expected format: , e.g. 15m,4") - interval_seconds = _parse_age_string(parts[0]) - try: - keep_count = int(parts[1]) - except ValueError: - raise ValueError(f"Invalid keep count in tier: {part}. Must be an integer.") - if keep_count < 1: - raise ValueError(f"Keep count must be >= 1 in tier: {part}") - tiers.append((interval_seconds, keep_count)) - tiers.sort(key=lambda t: t[0]) - # Validate intervals are strictly increasing - for i in range(1, len(tiers)): - if tiers[i][0] <= tiers[i - 1][0]: - raise ValueError("Schedule tier intervals must be strictly increasing") - return tiers - - -def _get_latest_backup_for_lvol(lvol_id): +def get_latest_backup_for_lvol(lvol_id): """Get the most recent non-failed backup for a given lvol. Includes pending/in-progress backups so that chain links are set @@ -83,134 +46,6 @@ def _get_latest_backup_for_lvol(lvol_id): return valid[0] -def _compute_s3_cpu_masks(node: StorageNode): - """CPU masks for the S3 bdev, or None where the node does not say. - - Returns (bdb_lcpu_mask, s3_lcpu_mask): - bdb_lcpu_mask: app_thread core (SPDK lightweight thread, low overhead) - s3_lcpu_mask: all system vCPUs (no pinning — let Linux scheduler handle - the AWS SDK thread pool; the data plane default would - wrongly pin onto SPDK reactor cores) - - None rather than 0 for "the node does not tell us": a zero mask selects no - CPUs at all, and the data plane reads it as "unset" anyway, so returning it - would be a sentinel dressed as a value. - """ - # SPDK thread for the bdev poller — reuse the app thread core - bdb_lcpu_mask = int(node.app_thread_mask, 16) if node.app_thread_mask else None - - # AWS SDK thread pool — set all system vCPU bits so threads are unconstrained - s3_lcpu_mask = (1 << node.cpu) - 1 if node.cpu > 0 else None - - return bdb_lcpu_mask, s3_lcpu_mask - - -# --- Restorability rules --------------------------------------------------- -# -# Predicates, so the same rule can answer a question ("can this bucket hold -# backups?") as well as block an operation. `require_restorable` is the one place -# that turns a false answer into a refusal, so the wording an operator sees is -# written once rather than at each of the entry points that enforce the rules. - - -def chain_fits(length: int) -> bool: - """Whether a chain this long is accepted. - - The bound is the control plane's own; see BACKUP_MAX_CHAIN_LENGTH for what it - is a bound on. The data plane's own limit is higher and refuses rather than - overruns, so this is where a too-long chain is reported usefully. - """ - return length <= constants.BACKUP_MAX_CHAIN_LENGTH - - -def location_holds_backups(location: BackupLocation) -> bool: - """Whether backups written to this location could be read back. - - ``snapshot_backups=False`` selects the secondary-tiering object layout, whose - keys are ``{tiering_id}/{lpgi}``. The restore path addresses - ``{s3_id}/{mid}/{extent}``, so it can never find them. - """ - return location.snapshot_backups - - -def chain_is_coherent(backups, location: BackupLocation, - encrypted: Optional[bool] = None) -> bool: - """Whether these backups can be restored together. - - A restore reads clusters from every backup in the chain in one operation, - against one bucket, decrypting all of it with one key. So the chain has to - agree on where it lives, how it is encoded, and whether it is encrypted -- - nothing anywhere in the stack could express a chain split across two buckets - or half encrypted. - - ``encrypted`` folds in a backup that does not exist yet, which is the case at - creation time. - """ - if any(backup.get_location() != location for backup in backups): - return False - - variants = {backup.encrypted for backup in backups} - if encrypted is not None: - variants.add(encrypted) - return len(variants) <= 1 - - -def _describe_incoherence(backups, location: BackupLocation, - encrypted: Optional[bool]) -> str: - for backup in backups: - if backup.get_location() != location: - return ( - f"backup {backup.uuid} lives in bucket " - f"{backup.get_location().bucket_name}, but the rest of its chain " - f"is in {location.bucket_name}. A chain cannot span buckets or " - "encodings; start a new chain with a full backup") - - return ( - "a chain cannot mix encrypted and unencrypted backups: " - + ", ".join(f"{b.uuid}={'encrypted' if b.encrypted else 'plain'}" - for b in backups) - + (f", new backup={'encrypted' if encrypted else 'plain'}" - if encrypted is not None else "")) - - -def require_restorable(location: BackupLocation, backups=(), - chain_length: Optional[int] = None, - encrypted: Optional[bool] = None, - what: str = "This chain") -> None: - """Refuse a chain that could not be restored, naming the rule it breaks. - - Applied at creation, at import and at restore, because each is a point where - a chain could otherwise become unrestorable without anyone noticing -- and - each used to find out from whatever failed first, usually the data plane - mid-operation. - - Args: - chain_length: The eventual length, where it differs from ``len(backups)`` - -- at creation the ancestors are snapshots that have no backup yet. - encrypted: Whether the backup about to be created will be encrypted. - - Raises: - PreconditionError: One of the rules above does not hold. - """ - if not location_holds_backups(location): - raise PreconditionError( - f"Bucket {location.bucket_name} is configured with snapshot_backups " - "disabled, which selects the secondary-tiering object layout. " - "Backups cannot be written there.") - - length = len(backups) if chain_length is None else chain_length - if not chain_fits(length): - raise PreconditionError( - f"{what} is {length} backups long; the data plane accepts at most " - f"{constants.BACKUP_MAX_CHAIN_LENGTH}. Merge older backups to " - "shorten the chain, or start a new chain with a full backup.") - - if not chain_is_coherent(backups, location, encrypted): - raise PreconditionError( - f"{what} cannot be restored as a unit: " - + _describe_incoherence(backups, location, encrypted)) - - def _existing_chain_backups(snap_chain) -> list: """The backups that already exist for a snapshot chain, oldest first.""" existing = [] @@ -325,72 +160,6 @@ def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: ) -def primary_s3_bdev_name(node: StorageNode) -> str: - """The S3 device holding the cluster's own backup bucket.""" - return f"s3_{node.lvstore}" - - -def create_restore_s3_bdev(node: StorageNode, config: BackupConfig, name: str) -> None: - """Attach a second S3 device to a node, for a bucket that is not its own. - - A restore from a foreign bucket needs different credentials, a different - endpoint and a different region than the node's own backup device carries. - Since a device holds exactly one bucket, the way to read another one is to - create another device -- which the lvstore supports, its transfer devices - being a list. - - The caller owns the result and must delete it when the restore ends. - """ - rpc_client = node.rpc_client() - bdb_lcpu_mask, s3_lcpu_mask = _compute_s3_cpu_masks(node) - - try: - rpc_client.bdev_s3_create( - name=name, - bucket_name=config.bucket_name, - secondary_target=config.secondary_target, - with_compression=config.with_compression, - snapshot_backups=config.snapshot_backups, - endpoint=config.endpoint_url, - region=config.region, - verify_tls=config.verify_tls, - use_path_style=config.use_path_style, - access_key_id=config.credentials.access_key_id if config.credentials else None, - secret_access_key=config.credentials.secret_access_key if config.credentials else None, - bdb_lcpu_mask=bdb_lcpu_mask, - s3_lcpu_mask=s3_lcpu_mask, - s3_thread_pool_size=config.s3_thread_pool_size, - ) - rpc_client.bdev_lvol_s3_bdev(node.lvstore, name) - except RPCException as e: - raise RuntimeError( - f"Failed to attach S3 device {name} for bucket {config.bucket_name} " - f"on node {node.get_id()}") from e - - logger.info("Attached restore S3 device %s for bucket %s on node %s", - name, config.bucket_name, node.get_id()) - - -def delete_restore_s3_bdev(node: StorageNode, name: str) -> None: - """Detach a device created by :func:`create_restore_s3_bdev`. - - Best-effort by design: this runs on the restore's terminal paths, and a - failure to clean up must not turn a completed restore into a failed one. It - is logged rather than raised, because the consequence is a leaked device -- - which does block the lvstore from being destroyed, so it is worth noticing. - """ - try: - node.rpc_client().bdev_s3_delete(name) - except Exception as e: - # Deliberately broad and deliberately not re-raised: this runs on a - # restore's terminal paths, where the alternative to a leaked device is - # reporting a completed restore as failed. - logger.warning("Could not delete restore S3 device %s on node %s: %s", - name, node.get_id(), e) - else: - logger.info("Deleted restore S3 device %s on node %s", name, node.get_id()) - - def foreign_bucket_config(backup: Backup, cluster, credentials: Optional[S3Credentials]) -> Optional[BackupConfig]: """How to reach this backup's bucket, when it is not the cluster's own. @@ -437,15 +206,6 @@ def foreign_bucket_config(backup: Backup, cluster, return config -def restore_s3_bdev_name(backup_id: str) -> str: - """Name of the device created to read a foreign bucket for one restore. - - Derived from the backup id so a retry re-derives the same name rather than - leaking a device per attempt. - """ - return f"s3_restore_{backup_id[:8]}" - - def _resolve_crypto_key(backup: Backup, cluster): """Recover the key needed to read an encrypted backup. @@ -525,72 +285,6 @@ def delete_manifest(backup: Backup) -> None: backup_manifest.delete(_config_for(backup), backup.uuid) -def _ensure_s3_bucket(config: BackupConfig, bucket_name): - try: - s3_client = backup_manifest.s3_client(config) - try: - s3_client.head_bucket(Bucket=bucket_name) - logger.info(f"S3 bucket already exists: {bucket_name}") - except ClientError as e: - error_code = int(e.response["Error"]["Code"]) - if error_code == 404: - s3_client.create_bucket(Bucket=bucket_name) - logger.info(f"S3 bucket created: {bucket_name}") - else: - raise - except BotoCoreError as e: - raise RuntimeError(f"Error ensuring S3 bucket {bucket_name} exists") from e - - -def create_s3_bdev(node: StorageNode, config: BackupConfig) -> None: - """Create the S3 bdev and attach it to a node's lvstore. - Called during cluster activate / node restart. - Args: - node: StorageNode with lvstore set - config: the cluster's validated backup configuration - """ - if not node.lvstore: - raise PreconditionError("Node does not have an lvstore") - - rpc_client = node.rpc_client() - s3_bdev_name = f"s3_{node.lvstore}" - - bdb_lcpu_mask, s3_lcpu_mask = _compute_s3_cpu_masks(node) - - # NO bdev_lvol_create_poller_group here: the lvstore-create poller group - # is created exactly ONCE per SPDK process lifetime — right after - # framework init in the add-node / restart-node flows, on the JC - # singleton's thread/core. This function used to re-call it with - # app_thread_mask (fix f0fed785, which predates the bring-up call from - # #938): a second creation with a different mask that either failed - # noisily on every activate or put the pollers on the wrong core. - - try: - _ensure_s3_bucket(config, config.bucket_name) - - rpc_client.bdev_s3_create( - name=s3_bdev_name, - bucket_name=config.bucket_name, - secondary_target=config.secondary_target, - with_compression=config.with_compression, - snapshot_backups=config.snapshot_backups, - endpoint=config.endpoint_url, - region=config.region, - verify_tls=config.verify_tls, - use_path_style=config.use_path_style, - access_key_id=config.credentials.access_key_id if config.credentials else None, - secret_access_key=config.credentials.secret_access_key if config.credentials else None, - bdb_lcpu_mask=bdb_lcpu_mask, - s3_lcpu_mask=s3_lcpu_mask, - s3_thread_pool_size=config.s3_thread_pool_size, - ) - - rpc_client.bdev_lvol_s3_bdev(node.lvstore, s3_bdev_name) - logger.info(f"S3 bdev created and attached: {s3_bdev_name} on node {node.get_id()}") - except (RPCException, RuntimeError) as e: - raise RuntimeError(f"Error S3 bdev on node {node.get_id()}") from e - - def _get_snapshot_chain(snapshot): """Build the snapshot chain ending at this snapshot, oldest first. @@ -656,7 +350,7 @@ def _build_encryption(cluster, backup: Backup) -> backup_manifest.Encryption: return backup_manifest.Encryption(encrypted=True, descriptor=descriptor) -def _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location: BackupLocation): +def create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location: BackupLocation): """Create a single backup record and task for one snapshot. Args: @@ -763,7 +457,7 @@ def backup_snapshot(snapshot_id, cluster_id=None): + (f" (requested snapshot {lock_snapshot})" if lock_snapshot else "") ) - prev_backup = _get_latest_backup_for_lvol(lvol.get_id()) + prev_backup = get_latest_backup_for_lvol(lvol.get_id()) final_backup_id = None try: # Walk the snapshot chain and back up all unbacked ancestors first @@ -780,7 +474,7 @@ def backup_snapshot(snapshot_id, cluster_id=None): prev_backup = existing continue - backup = _create_single_backup(snap, lvol, node_id, cluster_id, prev_backup, location) + backup = create_single_backup(snap, lvol, node_id, cluster_id, prev_backup, location) time.sleep(1) prev_backup = backup if snap.get_id() == snapshot_id: @@ -1173,287 +867,3 @@ def import_from_bucket(config: BackupConfig, cluster_id=None) -> int: PreconditionError: the manifests it holds cannot be imported as a batch. """ return import_backups(discover_backups(config), cluster_id=cluster_id) - - -# ---- Backup Policy Management ---- - -def add_policy(cluster_id, name, max_versions=0, max_age="", schedule=""): - """Create a new backup policy. - Returns (policy_id, error_message).""" - max_age_seconds = 0 - if max_age: - try: - max_age_seconds = _parse_age_string(max_age) - except ValueError as e: - return None, str(e) - - if schedule: - try: - _parse_schedule(schedule) - except ValueError as e: - return None, str(e) - - if max_versions <= 0 and max_age_seconds <= 0 and not schedule: - return None, "At least one of --versions, --age, or --schedule must be specified" - - # Check name uniqueness - for p in db_controller.get_backup_policies(cluster_id): - if p.policy_name == name: - return None, f"Policy name already exists: {name}" - - policy = BackupPolicy() - policy.uuid = str(uuid.uuid4()) - policy.cluster_id = cluster_id - policy.policy_name = name - policy.max_versions = max_versions - policy.max_age_seconds = max_age_seconds - policy.max_age_display = max_age - policy.backup_schedule = schedule - policy.status = BackupPolicy.STATUS_ACTIVE - policy.write_to_db() - - return policy.uuid, None - - -def remove_policy(policy_id): - """Remove a backup policy and all its attachments. - Returns (success, error_message).""" - try: - policy = db_controller.get_backup_policy_by_id(policy_id) - except KeyError as e: - return False, str(e) - - # Remove attachments - for att in db_controller.get_backup_policy_attachments(policy.cluster_id): - if att.policy_id == policy_id: - att.remove(db_controller.kv_store) - - policy.remove(db_controller.kv_store) - return True, None - - -def attach_policy(policy_id, target_type, target_id): - """Attach a backup policy to a pool or lvol. - Returns (attachment_id, error_message).""" - try: - policy = db_controller.get_backup_policy_by_id(policy_id) - except KeyError as e: - return None, str(e) - - if target_type not in ("pool", "lvol"): - return None, f"Invalid target_type: {target_type}. Use 'pool' or 'lvol'" - - # Validate target exists - try: - if target_type == "pool": - db_controller.get_pool_by_id(target_id) - else: - db_controller.get_lvol_by_id(target_id) - except KeyError as e: - return None, str(e) - - # Check if already attached - for att in db_controller.get_backup_policy_attachments(policy.cluster_id): - if att.policy_id == policy_id and att.target_type == target_type and att.target_id == target_id: - return att.uuid, None # already attached - - att = BackupPolicyAttachment() - att.uuid = str(uuid.uuid4()) - att.cluster_id = policy.cluster_id - att.policy_id = policy_id - att.target_type = target_type - att.target_id = target_id - att.write_to_db() - - return att.uuid, None - - -def detach_policy(policy_id, target_type, target_id): - """Detach a backup policy from a pool or lvol. - Returns (success, error_message).""" - try: - policy = db_controller.get_backup_policy_by_id(policy_id) - except KeyError as e: - return False, str(e) - - for att in db_controller.get_backup_policy_attachments(policy.cluster_id): - if att.policy_id == policy_id and att.target_type == target_type and att.target_id == target_id: - att.remove(db_controller.kv_store) - return True, None - - return False, "Attachment not found" - - -def list_policies(cluster_id=None): - """List all backup policies.""" - policies = db_controller.get_backup_policies(cluster_id) - data = [] - for p in policies: - data.append({ - "ID": p.uuid, - "Name": p.policy_name, - "Versions": p.max_versions if p.max_versions > 0 else "-", - "Max Age": p.max_age_display if p.max_age_display else "-", - "Schedule": p.backup_schedule if p.backup_schedule else "-", - "Status": p.status, - }) - return data - - -def evaluate_policy(lvol): - """Evaluate backup policy for an lvol and trigger merges if needed. - Called by the backup merge service.""" - policy = db_controller.get_policy_for_lvol(lvol) - if not policy: - return - - backups = db_controller.get_backups_by_lvol_id(lvol.get_id()) - completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] - if len(completed) < 2: - return - - completed.sort(key=lambda b: b.created_at) - now = int(time.time()) - - versions_exceeded = policy.max_versions > 0 and len(completed) > policy.max_versions - age_exceeded = False - if policy.max_age_seconds > 0 and completed: - oldest_age = now - completed[0].created_at - age_exceeded = oldest_age > policy.max_age_seconds - - # Either condition triggers a merge - if versions_exceeded or age_exceeded: - oldest = completed[0] - second = completed[1] - _trigger_merge(second, oldest) - - -def evaluate_schedule(lvol): - """Evaluate the backup schedule for an lvol and trigger auto-backups + tiered merges. - Called by the backup merge service.""" - policy = db_controller.get_policy_for_lvol(lvol) - if not policy or not policy.backup_schedule: - return - - try: - tiers = _parse_schedule(policy.backup_schedule) - except ValueError: - return - - if not tiers: - return - - now = int(time.time()) - - # Check if we need to create a new auto-backup based on the smallest tier interval - smallest_interval = tiers[0][0] - backups = db_controller.get_backups_by_lvol_id(lvol.get_id()) - completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] - pending_or_running = [b for b in backups if b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS)] - - # Don't create a new backup if one is already in progress - if not pending_or_running: - needs_backup = True - if completed: - completed.sort(key=lambda b: b.created_at, reverse=True) - latest = completed[0] - elapsed = now - latest.created_at - if elapsed < smallest_interval: - needs_backup = False - - if needs_backup: - _auto_backup_lvol(lvol) - return # Skip merge evaluation this cycle — let the backup complete first - - # Tiered merge: enforce keep_count per tier. - # Each tier covers an age range. Backups age from tier 0 (newest) - # into higher tiers. When a tier exceeds its keep_count, the oldest - # backup in that tier is merged into its successor. - # All tiers are evaluated each cycle so limits are maintained in parallel. - if len(completed) < 2: - return - - completed.sort(key=lambda b: b.created_at) - - # Don't merge while another merge is already in progress - merging = [b for b in backups if b.status == Backup.STATUS_MERGING] - if merging: - return - - for tier_idx, (interval, keep_count) in enumerate(tiers): - # Age boundaries for this tier - if tier_idx == 0: - lower_age = 0 - else: - lower_age = tiers[tier_idx - 1][0] - - if tier_idx + 1 < len(tiers): - upper_age = tiers[tier_idx + 1][0] - else: - upper_age = float('inf') - - tier_backups = [b for b in completed - if lower_age <= (now - b.created_at) < upper_age] - - if len(tier_backups) > keep_count: - tier_backups.sort(key=lambda b: b.created_at) - oldest = tier_backups[0] - second = tier_backups[1] - _trigger_merge(second, oldest) - return # One merge per cycle to avoid conflicts - - -def _auto_backup_lvol(lvol): - """Create an automatic snapshot + backup for scheduled backups. - - Unlike manual backup_snapshot() which walks the full ancestor chain, - auto-backups create a single snapshot and a single backup for it. - The prev_backup_id is set to the latest existing backup so the - incremental chain is maintained without re-backing all ancestors. - """ - from simplyblock_core.controllers import snapshot_controller - - # Resolve everything the backup needs BEFORE taking the snapshot. This used - # to create the snapshot first and discover afterwards that the node or - # cluster was unusable, leaving an orphaned auto_* snapshot behind on every - # scheduler tick. - node_id = lvol.node_id - try: - snode = db_controller.get_storage_node_by_id(node_id) - cluster_id = snode.cluster_id - location = db_controller.get_cluster_by_id(cluster_id).get_backup_config().location() - except (KeyError, ValueError) as e: - logger.warning(f"Auto-backup skipped for lvol {lvol.get_id()}: {e}") - return - - snap_name = f"auto_{lvol.lvol_name}_{int(time.time())}" - snap_id, error = snapshot_controller.add(lvol.get_id(), snap_name) - if error: - logger.warning(f"Auto-backup snapshot failed for lvol {lvol.get_id()}: {error}") - return - - try: - snapshot = db_controller.get_snapshot_by_id(snap_id) - except KeyError: - logger.warning(f"Auto-backup: snapshot {snap_id} not found after creation") - return - - prev_backup = _get_latest_backup_for_lvol(lvol.get_id()) - _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location) - - -def _trigger_merge(keep_backup, old_backup): - """Trigger a merge of old_backup into keep_backup.""" - if old_backup.status != Backup.STATUS_COMPLETED: - return - if keep_backup.status != Backup.STATUS_COMPLETED: - return - - old_backup.status = Backup.STATUS_MERGING - old_backup.write_to_db() - - tasks_controller.add_backup_merge_task( - keep_backup.cluster_id, - keep_backup.node_id, - keep_backup.uuid, - old_backup.uuid) diff --git a/simplyblock_core/controllers/backup/device.py b/simplyblock_core/controllers/backup/device.py new file mode 100644 index 000000000..f7f4b05e2 --- /dev/null +++ b/simplyblock_core/controllers/backup/device.py @@ -0,0 +1,183 @@ +# coding=utf-8 +"""The S3 devices a node reads and writes backups through. + +A device holds exactly one bucket with one set of credentials, so a node runs one +for the cluster's own backup bucket and, during a restore from somebody else's +bucket, a second one attached for the duration. Naming them is part of this +module's job: a restore device's name derives from the backup id, so a retry +re-derives it instead of leaking a device per attempt. +""" +import logging + +from botocore.exceptions import BotoCoreError, ClientError + +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCException + +logger = logging.getLogger() + + +def _compute_s3_cpu_masks(node: StorageNode): + """CPU masks for the S3 bdev, or None where the node does not say. + + Returns (bdb_lcpu_mask, s3_lcpu_mask): + bdb_lcpu_mask: app_thread core (SPDK lightweight thread, low overhead) + s3_lcpu_mask: all system vCPUs (no pinning — let Linux scheduler handle + the AWS SDK thread pool; the data plane default would + wrongly pin onto SPDK reactor cores) + + None rather than 0 for "the node does not tell us": a zero mask selects no + CPUs at all, and the data plane reads it as "unset" anyway, so returning it + would be a sentinel dressed as a value. + """ + # SPDK thread for the bdev poller — reuse the app thread core + bdb_lcpu_mask = int(node.app_thread_mask, 16) if node.app_thread_mask else None + + # AWS SDK thread pool — set all system vCPU bits so threads are unconstrained + s3_lcpu_mask = (1 << node.cpu) - 1 if node.cpu > 0 else None + + return bdb_lcpu_mask, s3_lcpu_mask + + +def primary_s3_bdev_name(node: StorageNode) -> str: + """The S3 device holding the cluster's own backup bucket.""" + return f"s3_{node.lvstore}" + + +def restore_s3_bdev_name(backup_id: str) -> str: + """Name of the device created to read a foreign bucket for one restore. + + Derived from the backup id so a retry re-derives the same name rather than + leaking a device per attempt. + """ + return f"s3_restore_{backup_id[:8]}" + + +def create_restore_s3_bdev(node: StorageNode, config: BackupConfig, name: str) -> None: + """Attach a second S3 device to a node, for a bucket that is not its own. + + A restore from a foreign bucket needs different credentials, a different + endpoint and a different region than the node's own backup device carries. + Since a device holds exactly one bucket, the way to read another one is to + create another device -- which the lvstore supports, its transfer devices + being a list. + + The caller owns the result and must delete it when the restore ends. + """ + rpc_client = node.rpc_client() + bdb_lcpu_mask, s3_lcpu_mask = _compute_s3_cpu_masks(node) + + try: + rpc_client.bdev_s3_create( + name=name, + bucket_name=config.bucket_name, + secondary_target=config.secondary_target, + with_compression=config.with_compression, + snapshot_backups=config.snapshot_backups, + endpoint=config.endpoint_url, + region=config.region, + verify_tls=config.verify_tls, + use_path_style=config.use_path_style, + access_key_id=config.credentials.access_key_id if config.credentials else None, + secret_access_key=config.credentials.secret_access_key if config.credentials else None, + bdb_lcpu_mask=bdb_lcpu_mask, + s3_lcpu_mask=s3_lcpu_mask, + s3_thread_pool_size=config.s3_thread_pool_size, + ) + rpc_client.bdev_lvol_s3_bdev(node.lvstore, name) + except RPCException as e: + raise RuntimeError( + f"Failed to attach S3 device {name} for bucket {config.bucket_name} " + f"on node {node.get_id()}") from e + + logger.info("Attached restore S3 device %s for bucket %s on node %s", + name, config.bucket_name, node.get_id()) + + +def delete_restore_s3_bdev(node: StorageNode, name: str) -> None: + """Detach a device created by :func:`create_restore_s3_bdev`. + + Best-effort by design: this runs on the restore's terminal paths, and a + failure to clean up must not turn a completed restore into a failed one. It + is logged rather than raised, because the consequence is a leaked device -- + which does block the lvstore from being destroyed, so it is worth noticing. + """ + try: + node.rpc_client().bdev_s3_delete(name) + except Exception as e: + # Deliberately broad and deliberately not re-raised: this runs on a + # restore's terminal paths, where the alternative to a leaked device is + # reporting a completed restore as failed. + logger.warning("Could not delete restore S3 device %s on node %s: %s", + name, node.get_id(), e) + else: + logger.info("Deleted restore S3 device %s on node %s", name, node.get_id()) + + +def _ensure_s3_bucket(config: BackupConfig, bucket_name): + try: + s3_client = backup_manifest.s3_client(config) + try: + s3_client.head_bucket(Bucket=bucket_name) + logger.info(f"S3 bucket already exists: {bucket_name}") + except ClientError as e: + error_code = int(e.response["Error"]["Code"]) + if error_code == 404: + s3_client.create_bucket(Bucket=bucket_name) + logger.info(f"S3 bucket created: {bucket_name}") + else: + raise + except BotoCoreError as e: + raise RuntimeError(f"Error ensuring S3 bucket {bucket_name} exists") from e + + +def create_s3_bdev(node: StorageNode, config: BackupConfig) -> None: + """Create the S3 bdev and attach it to a node's lvstore. + Called during cluster activate / node restart. + Args: + node: StorageNode with lvstore set + config: the cluster's validated backup configuration + """ + if not node.lvstore: + raise PreconditionError("Node does not have an lvstore") + + rpc_client = node.rpc_client() + s3_bdev_name = f"s3_{node.lvstore}" + + bdb_lcpu_mask, s3_lcpu_mask = _compute_s3_cpu_masks(node) + + # NO bdev_lvol_create_poller_group here: the lvstore-create poller group + # is created exactly ONCE per SPDK process lifetime — right after + # framework init in the add-node / restart-node flows, on the JC + # singleton's thread/core. This function used to re-call it with + # app_thread_mask (fix f0fed785, which predates the bring-up call from + # #938): a second creation with a different mask that either failed + # noisily on every activate or put the pollers on the wrong core. + + try: + _ensure_s3_bucket(config, config.bucket_name) + + rpc_client.bdev_s3_create( + name=s3_bdev_name, + bucket_name=config.bucket_name, + secondary_target=config.secondary_target, + with_compression=config.with_compression, + snapshot_backups=config.snapshot_backups, + endpoint=config.endpoint_url, + region=config.region, + verify_tls=config.verify_tls, + use_path_style=config.use_path_style, + access_key_id=config.credentials.access_key_id if config.credentials else None, + secret_access_key=config.credentials.secret_access_key if config.credentials else None, + bdb_lcpu_mask=bdb_lcpu_mask, + s3_lcpu_mask=s3_lcpu_mask, + s3_thread_pool_size=config.s3_thread_pool_size, + ) + + rpc_client.bdev_lvol_s3_bdev(node.lvstore, s3_bdev_name) + logger.info(f"S3 bdev created and attached: {s3_bdev_name} on node {node.get_id()}") + except (RPCException, RuntimeError) as e: + raise RuntimeError(f"Error S3 bdev on node {node.get_id()}") from e diff --git a/simplyblock_core/backup_manifest.py b/simplyblock_core/controllers/backup/manifest.py similarity index 99% rename from simplyblock_core/backup_manifest.py rename to simplyblock_core/controllers/backup/manifest.py index 764a1a4f5..690b33583 100644 --- a/simplyblock_core/backup_manifest.py +++ b/simplyblock_core/controllers/backup/manifest.py @@ -24,7 +24,7 @@ already unmapped. This document overlaps the ``Backup`` record in FoundationDB by design, and -substantially -- see the note at the top of ``controllers/backup_controller.py`` +substantially -- see the note on ``controller.build_manifest`` for where the two genuinely differ and where they should be collapsed. """ import json diff --git a/simplyblock_core/controllers/backup/policy.py b/simplyblock_core/controllers/backup/policy.py new file mode 100644 index 000000000..9112e1a15 --- /dev/null +++ b/simplyblock_core/controllers/backup/policy.py @@ -0,0 +1,340 @@ +# coding=utf-8 +"""Backup policies: retention limits, tiered schedules, and the merges they cause. + +A policy decides *when* a backup is taken and when two are folded together. What +a backup is and how it is written is :mod:`controller`'s business, which this +module calls into rather than reimplements. +""" +import logging +import re +import time +import uuid + +from simplyblock_core.controllers import tasks_controller +from simplyblock_core.controllers.backup.controller import ( + create_single_backup, get_latest_backup_for_lvol) +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment + +logger = logging.getLogger() + +db_controller = DBController() + + +def _parse_age_string(age_str): + """Parse age strings like '2d', '12h', '1w', '30m' into seconds.""" + match = re.match(r'^(\d+)([mhdw])$', age_str.strip()) + if not match: + raise ValueError(f"Invalid age format: {age_str}. Use e.g. 2d, 12h, 1w") + value = int(match.group(1)) + unit = match.group(2) + multipliers = {'m': 60, 'h': 3600, 'd': 86400, 'w': 604800} + return value * multipliers[unit] + + +def _parse_schedule(schedule_str): + """Parse schedule string like '15m,4 60m,11 24h,7' into list of (interval_seconds, keep_count) tuples. + Returns sorted list by interval ascending. Raises ValueError on invalid input.""" + if not schedule_str or not schedule_str.strip(): + return [] + tiers = [] + for part in schedule_str.strip().split(): + parts = part.split(',') + if len(parts) != 2: + raise ValueError(f"Invalid schedule tier: {part}. Expected format: , e.g. 15m,4") + interval_seconds = _parse_age_string(parts[0]) + try: + keep_count = int(parts[1]) + except ValueError: + raise ValueError(f"Invalid keep count in tier: {part}. Must be an integer.") + if keep_count < 1: + raise ValueError(f"Keep count must be >= 1 in tier: {part}") + tiers.append((interval_seconds, keep_count)) + tiers.sort(key=lambda t: t[0]) + # Validate intervals are strictly increasing + for i in range(1, len(tiers)): + if tiers[i][0] <= tiers[i - 1][0]: + raise ValueError("Schedule tier intervals must be strictly increasing") + return tiers + + +def add_policy(cluster_id, name, max_versions=0, max_age="", schedule=""): + """Create a new backup policy. + Returns (policy_id, error_message).""" + max_age_seconds = 0 + if max_age: + try: + max_age_seconds = _parse_age_string(max_age) + except ValueError as e: + return None, str(e) + + if schedule: + try: + _parse_schedule(schedule) + except ValueError as e: + return None, str(e) + + if max_versions <= 0 and max_age_seconds <= 0 and not schedule: + return None, "At least one of --versions, --age, or --schedule must be specified" + + # Check name uniqueness + for p in db_controller.get_backup_policies(cluster_id): + if p.policy_name == name: + return None, f"Policy name already exists: {name}" + + policy = BackupPolicy() + policy.uuid = str(uuid.uuid4()) + policy.cluster_id = cluster_id + policy.policy_name = name + policy.max_versions = max_versions + policy.max_age_seconds = max_age_seconds + policy.max_age_display = max_age + policy.backup_schedule = schedule + policy.status = BackupPolicy.STATUS_ACTIVE + policy.write_to_db() + + return policy.uuid, None + + +def remove_policy(policy_id): + """Remove a backup policy and all its attachments. + Returns (success, error_message).""" + try: + policy = db_controller.get_backup_policy_by_id(policy_id) + except KeyError as e: + return False, str(e) + + # Remove attachments + for att in db_controller.get_backup_policy_attachments(policy.cluster_id): + if att.policy_id == policy_id: + att.remove(db_controller.kv_store) + + policy.remove(db_controller.kv_store) + return True, None + + +def attach_policy(policy_id, target_type, target_id): + """Attach a backup policy to a pool or lvol. + Returns (attachment_id, error_message).""" + try: + policy = db_controller.get_backup_policy_by_id(policy_id) + except KeyError as e: + return None, str(e) + + if target_type not in ("pool", "lvol"): + return None, f"Invalid target_type: {target_type}. Use 'pool' or 'lvol'" + + # Validate target exists + try: + if target_type == "pool": + db_controller.get_pool_by_id(target_id) + else: + db_controller.get_lvol_by_id(target_id) + except KeyError as e: + return None, str(e) + + # Check if already attached + for att in db_controller.get_backup_policy_attachments(policy.cluster_id): + if att.policy_id == policy_id and att.target_type == target_type and att.target_id == target_id: + return att.uuid, None # already attached + + att = BackupPolicyAttachment() + att.uuid = str(uuid.uuid4()) + att.cluster_id = policy.cluster_id + att.policy_id = policy_id + att.target_type = target_type + att.target_id = target_id + att.write_to_db() + + return att.uuid, None + + +def detach_policy(policy_id, target_type, target_id): + """Detach a backup policy from a pool or lvol. + Returns (success, error_message).""" + try: + policy = db_controller.get_backup_policy_by_id(policy_id) + except KeyError as e: + return False, str(e) + + for att in db_controller.get_backup_policy_attachments(policy.cluster_id): + if att.policy_id == policy_id and att.target_type == target_type and att.target_id == target_id: + att.remove(db_controller.kv_store) + return True, None + + return False, "Attachment not found" + + +def list_policies(cluster_id=None): + """List all backup policies.""" + policies = db_controller.get_backup_policies(cluster_id) + data = [] + for p in policies: + data.append({ + "ID": p.uuid, + "Name": p.policy_name, + "Versions": p.max_versions if p.max_versions > 0 else "-", + "Max Age": p.max_age_display if p.max_age_display else "-", + "Schedule": p.backup_schedule if p.backup_schedule else "-", + "Status": p.status, + }) + return data + + +def evaluate_policy(lvol): + """Evaluate backup policy for an lvol and trigger merges if needed. + Called by the backup merge service.""" + policy = db_controller.get_policy_for_lvol(lvol) + if not policy: + return + + backups = db_controller.get_backups_by_lvol_id(lvol.get_id()) + completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] + if len(completed) < 2: + return + + completed.sort(key=lambda b: b.created_at) + now = int(time.time()) + + versions_exceeded = policy.max_versions > 0 and len(completed) > policy.max_versions + age_exceeded = False + if policy.max_age_seconds > 0 and completed: + oldest_age = now - completed[0].created_at + age_exceeded = oldest_age > policy.max_age_seconds + + # Either condition triggers a merge + if versions_exceeded or age_exceeded: + oldest = completed[0] + second = completed[1] + _trigger_merge(second, oldest) + + +def evaluate_schedule(lvol): + """Evaluate the backup schedule for an lvol and trigger auto-backups + tiered merges. + Called by the backup merge service.""" + policy = db_controller.get_policy_for_lvol(lvol) + if not policy or not policy.backup_schedule: + return + + try: + tiers = _parse_schedule(policy.backup_schedule) + except ValueError: + return + + if not tiers: + return + + now = int(time.time()) + + # Check if we need to create a new auto-backup based on the smallest tier interval + smallest_interval = tiers[0][0] + backups = db_controller.get_backups_by_lvol_id(lvol.get_id()) + completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] + pending_or_running = [b for b in backups if b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS)] + + # Don't create a new backup if one is already in progress + if not pending_or_running: + needs_backup = True + if completed: + completed.sort(key=lambda b: b.created_at, reverse=True) + latest = completed[0] + elapsed = now - latest.created_at + if elapsed < smallest_interval: + needs_backup = False + + if needs_backup: + _auto_backup_lvol(lvol) + return # Skip merge evaluation this cycle — let the backup complete first + + # Tiered merge: enforce keep_count per tier. + # Each tier covers an age range. Backups age from tier 0 (newest) + # into higher tiers. When a tier exceeds its keep_count, the oldest + # backup in that tier is merged into its successor. + # All tiers are evaluated each cycle so limits are maintained in parallel. + if len(completed) < 2: + return + + completed.sort(key=lambda b: b.created_at) + + # Don't merge while another merge is already in progress + merging = [b for b in backups if b.status == Backup.STATUS_MERGING] + if merging: + return + + for tier_idx, (interval, keep_count) in enumerate(tiers): + # Age boundaries for this tier + if tier_idx == 0: + lower_age = 0 + else: + lower_age = tiers[tier_idx - 1][0] + + if tier_idx + 1 < len(tiers): + upper_age = tiers[tier_idx + 1][0] + else: + upper_age = float('inf') + + tier_backups = [b for b in completed + if lower_age <= (now - b.created_at) < upper_age] + + if len(tier_backups) > keep_count: + tier_backups.sort(key=lambda b: b.created_at) + oldest = tier_backups[0] + second = tier_backups[1] + _trigger_merge(second, oldest) + return # One merge per cycle to avoid conflicts + + +def _auto_backup_lvol(lvol): + """Create an automatic snapshot + backup for scheduled backups. + + Unlike manual backup_snapshot() which walks the full ancestor chain, + auto-backups create a single snapshot and a single backup for it. + The prev_backup_id is set to the latest existing backup so the + incremental chain is maintained without re-backing all ancestors. + """ + from simplyblock_core.controllers import snapshot_controller + + # Resolve everything the backup needs BEFORE taking the snapshot. This used + # to create the snapshot first and discover afterwards that the node or + # cluster was unusable, leaving an orphaned auto_* snapshot behind on every + # scheduler tick. + node_id = lvol.node_id + try: + snode = db_controller.get_storage_node_by_id(node_id) + cluster_id = snode.cluster_id + location = db_controller.get_cluster_by_id(cluster_id).get_backup_config().location() + except (KeyError, ValueError) as e: + logger.warning(f"Auto-backup skipped for lvol {lvol.get_id()}: {e}") + return + + snap_name = f"auto_{lvol.lvol_name}_{int(time.time())}" + snap_id, error = snapshot_controller.add(lvol.get_id(), snap_name) + if error: + logger.warning(f"Auto-backup snapshot failed for lvol {lvol.get_id()}: {error}") + return + + try: + snapshot = db_controller.get_snapshot_by_id(snap_id) + except KeyError: + logger.warning(f"Auto-backup: snapshot {snap_id} not found after creation") + return + + prev_backup = get_latest_backup_for_lvol(lvol.get_id()) + create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location) + + +def _trigger_merge(keep_backup, old_backup): + """Trigger a merge of old_backup into keep_backup.""" + if old_backup.status != Backup.STATUS_COMPLETED: + return + if keep_backup.status != Backup.STATUS_COMPLETED: + return + + old_backup.status = Backup.STATUS_MERGING + old_backup.write_to_db() + + tasks_controller.add_backup_merge_task( + keep_backup.cluster_id, + keep_backup.node_id, + keep_backup.uuid, + old_backup.uuid) diff --git a/simplyblock_core/controllers/backup/validation.py b/simplyblock_core/controllers/backup/validation.py new file mode 100644 index 000000000..d11cfc6f7 --- /dev/null +++ b/simplyblock_core/controllers/backup/validation.py @@ -0,0 +1,112 @@ +# coding=utf-8 +"""When a backup chain can be restored, and when it cannot. + +Predicates rather than assertions, so a rule can answer a question as well as +block an operation -- ``location_holds_backups`` is a thing a caller may want to +know without being refused. :func:`require_restorable` is the one place that turns +a false answer into a refusal, so the wording an operator sees is written once +rather than at each entry point that enforces the same rules. +""" +from typing import Optional + +from simplyblock_core import constants +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup_config import BackupLocation + + +def chain_fits(length: int) -> bool: + """Whether a chain this long is accepted. + + The bound is the control plane's own; see BACKUP_MAX_CHAIN_LENGTH for what it + is a bound on. The data plane's own limit is higher and refuses rather than + overruns, so this is where a too-long chain is reported usefully. + """ + return length <= constants.BACKUP_MAX_CHAIN_LENGTH + + +def location_holds_backups(location: BackupLocation) -> bool: + """Whether backups written to this location could be read back. + + ``snapshot_backups=False`` selects the secondary-tiering object layout, whose + keys are ``{tiering_id}/{lpgi}``. The restore path addresses + ``{s3_id}/{mid}/{extent}``, so it can never find them. + """ + return location.snapshot_backups + + +def chain_is_coherent(backups, location: BackupLocation, + encrypted: Optional[bool] = None) -> bool: + """Whether these backups can be restored together. + + A restore reads clusters from every backup in the chain in one operation, + against one bucket, decrypting all of it with one key. So the chain has to + agree on where it lives, how it is encoded, and whether it is encrypted -- + nothing anywhere in the stack could express a chain split across two buckets + or half encrypted. + + ``encrypted`` folds in a backup that does not exist yet, which is the case at + creation time. + """ + if any(backup.get_location() != location for backup in backups): + return False + + variants = {backup.encrypted for backup in backups} + if encrypted is not None: + variants.add(encrypted) + return len(variants) <= 1 + + +def _describe_incoherence(backups, location: BackupLocation, + encrypted: Optional[bool]) -> str: + for backup in backups: + if backup.get_location() != location: + return ( + f"backup {backup.uuid} lives in bucket " + f"{backup.get_location().bucket_name}, but the rest of its chain " + f"is in {location.bucket_name}. A chain cannot span buckets or " + "encodings; start a new chain with a full backup") + + return ( + "a chain cannot mix encrypted and unencrypted backups: " + + ", ".join(f"{b.uuid}={'encrypted' if b.encrypted else 'plain'}" + for b in backups) + + (f", new backup={'encrypted' if encrypted else 'plain'}" + if encrypted is not None else "")) + + +def require_restorable(location: BackupLocation, backups=(), + chain_length: Optional[int] = None, + encrypted: Optional[bool] = None, + what: str = "This chain") -> None: + """Refuse a chain that could not be restored, naming the rule it breaks. + + Applied at creation, at import and at restore, because each is a point where + a chain could otherwise become unrestorable without anyone noticing -- and + each used to find out from whatever failed first, usually the data plane + mid-operation. + + Args: + chain_length: The eventual length, where it differs from ``len(backups)`` + -- at creation the ancestors are snapshots that have no backup yet. + encrypted: Whether the backup about to be created will be encrypted. + + Raises: + PreconditionError: One of the rules above does not hold. + """ + if not location_holds_backups(location): + raise PreconditionError( + f"Bucket {location.bucket_name} is configured with snapshot_backups " + "disabled, which selects the secondary-tiering object layout. " + "Backups cannot be written there.") + + length = len(backups) if chain_length is None else chain_length + if not chain_fits(length): + raise PreconditionError( + f"{what} is {length} backups long; the data plane accepts at most " + f"{constants.BACKUP_MAX_CHAIN_LENGTH}. Merge older backups to " + "shorten the chain, or start a new chain with a full backup.") + + if not chain_is_coherent(backups, location, encrypted): + raise PreconditionError( + f"{what} cannot be restored as a unit: " + + _describe_incoherence(backups, location, encrypted)) diff --git a/simplyblock_core/controllers/snapshot_controller.py b/simplyblock_core/controllers/snapshot_controller.py index acb23babe..e9b86bef6 100644 --- a/simplyblock_core/controllers/snapshot_controller.py +++ b/simplyblock_core/controllers/snapshot_controller.py @@ -800,7 +800,7 @@ def add(lvol_id, snapshot_name, backup=False, lock=True, all_snaps=None, all_lvo pass if backup: - from simplyblock_core.controllers import backup_controller + from simplyblock_core.controllers.backup import controller as backup_controller backup_id, backup_err = backup_controller.backup_snapshot(snap.uuid) if backup_err: logger.warning(f"Snapshot created but backup failed: {backup_err}") diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index d4312c908..718f3a1f0 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -10,8 +10,10 @@ import time from simplyblock_core import constants, db_controller, utils -from simplyblock_core.backup_manifest import ManifestError -from simplyblock_core.controllers import backup_controller, backup_events +from simplyblock_core.controllers import backup_events +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import device as backup_device +from simplyblock_core.controllers.backup.manifest import ManifestError from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup from simplyblock_core.models.backup_config import BackupConfig @@ -86,7 +88,7 @@ def _run_backup(task): try: ret = rpc_client.bdev_lvol_s3_backup( backup.s3_id, [snap_bdev_name], - backup_controller.primary_s3_bdev_name(snode), cluster_batch=16) + backup_device.primary_s3_bdev_name(snode), cluster_batch=16) if not ret: _fail_backup(backup, task, "bdev_lvol_s3_backup RPC failed") return @@ -203,9 +205,9 @@ def _restore_s3_bdev(task, snode) -> str: node's own backup device already points at the right bucket. """ if task.function_params.get("s3_config"): - return backup_controller.restore_s3_bdev_name( + return backup_device.restore_s3_bdev_name( task.function_params["backup_id"]) - return backup_controller.primary_s3_bdev_name(snode) + return backup_device.primary_s3_bdev_name(snode) def _ensure_restore_s3_bdev(task, snode) -> None: @@ -219,7 +221,7 @@ def _ensure_restore_s3_bdev(task, snode) -> None: if not config: return - backup_controller.create_restore_s3_bdev( + backup_device.create_restore_s3_bdev( snode, BackupConfig.model_validate(config), _restore_s3_bdev(task, snode)) @@ -235,7 +237,7 @@ def _release_restore_s3_bdev(task, snode) -> None: return if snode is not None: - backup_controller.delete_restore_s3_bdev(snode, _restore_s3_bdev(task, snode)) + backup_device.delete_restore_s3_bdev(snode, _restore_s3_bdev(task, snode)) task.function_params["s3_config"] = None @@ -412,7 +414,7 @@ def _run_merge(task): try: ret = rpc_client.bdev_lvol_s3_merge( keep_backup.s3_id, old_backup.s3_id, cluster_batch=16, - s3_bdev=backup_controller.primary_s3_bdev_name(snode), + s3_bdev=backup_device.primary_s3_bdev_name(snode), lvs_name=snode.lvstore) if not ret: task.function_result = "bdev_lvol_s3_merge RPC failed" diff --git a/simplyblock_core/services/tasks_runner_backup_merge.py b/simplyblock_core/services/tasks_runner_backup_merge.py index d28cfdc9a..45d09ea88 100644 --- a/simplyblock_core/services/tasks_runner_backup_merge.py +++ b/simplyblock_core/services/tasks_runner_backup_merge.py @@ -6,7 +6,7 @@ import time from simplyblock_core import constants, db_controller, utils -from simplyblock_core.controllers import backup_controller +from simplyblock_core.controllers.backup import policy as backup_policy from simplyblock_core.models.cluster import Cluster logger = utils.get_logger(__name__) @@ -31,11 +31,11 @@ def main(): lvols = db.get_lvols(cl.get_id()) for lvol in lvols: try: - backup_controller.evaluate_policy(lvol) + backup_policy.evaluate_policy(lvol) except Exception as e: logger.error(f"Error evaluating policy for lvol {lvol.get_id()}: {e}") try: - backup_controller.evaluate_schedule(lvol) + backup_policy.evaluate_schedule(lvol) except Exception as e: logger.error(f"Error evaluating schedule for lvol {lvol.get_id()}: {e}") except Exception as e: diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index 652c4a2d5..3dc073864 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4975,10 +4975,10 @@ def _abort_restart(reason): # Create S3 bdev for backup support (only if backup is configured) if cluster.backup_config: - from simplyblock_core.controllers import backup_controller + from simplyblock_core.controllers.backup import device as backup_device logger.info("Creating S3 bdev on restarted node") try: - backup_controller.create_s3_bdev(snode, cluster.get_backup_config()) + backup_device.create_s3_bdev(snode, cluster.get_backup_config()) except Exception as e: logger.exception(str(e)) return False diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 4fe57ed6a..11af748e1 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -19,7 +19,7 @@ from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.backup import Backup, BackupPolicy -from simplyblock_core.backup_manifest import BackupManifest +from simplyblock_core.controllers.backup.manifest import BackupManifest from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.stats import StatsObject from simplyblock_core.models.lvol_migration import LVolMigration diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index 296d58b4a..fcc812fb3 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -4,9 +4,10 @@ from fastapi import APIRouter, HTTPException, Query, Request, Response from pydantic import BaseModel, ConfigDict -from simplyblock_core.backup_manifest import ManifestError from simplyblock_core.db_controller import DBController -from simplyblock_core.controllers import backup_controller +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import policy as backup_policy +from simplyblock_core.controllers.backup.manifest import ManifestError from simplyblock_core.models.backup_config import S3Credentials from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.lvol_model import LVol @@ -195,7 +196,7 @@ class _PolicyCreateParams(BaseModel): @policy_api.post('/', name='clusters:backup-policies:create', status_code=201, responses={201: {"content": None}}) def create_policy(cluster: Cluster, parameters: _PolicyCreateParams) -> Response: - policy_id, error = backup_controller.add_policy( + policy_id, error = backup_policy.add_policy( cluster.get_id(), parameters.name, max_versions=parameters.versions or 0, max_age=parameters.age or "", @@ -219,7 +220,7 @@ def _validate_attachment_target(target_type: str, target_id: str, cluster: Clust @policy_api.delete('/{policy_id}', name='clusters:backup-policies:delete', status_code=204, responses={204: {"content": None}}) def delete_policy(cluster: Cluster, policy: Policy) -> Response: - success, error = backup_controller.remove_policy(policy.uuid) + success, error = backup_policy.remove_policy(policy.uuid) if error: raise HTTPException(400, error) return Response(status_code=204) @@ -233,7 +234,7 @@ class _AttachParams(BaseModel): @policy_api.post('/{policy_id}/attach', name='clusters:backup-policies:attach', status_code=201) def attach_policy(cluster: Cluster, policy: Policy, parameters: _AttachParams): _validate_attachment_target(parameters.target_type, parameters.target_id, cluster) - att_id, error = backup_controller.attach_policy( + att_id, error = backup_policy.attach_policy( policy.uuid, parameters.target_type, parameters.target_id) if error: raise HTTPException(400, error) @@ -243,7 +244,7 @@ def attach_policy(cluster: Cluster, policy: Policy, parameters: _AttachParams): @policy_api.post('/{policy_id}/detach', name='clusters:backup-policies:detach', status_code=204, responses={204: {"content": None}}) def detach_policy(cluster: Cluster, policy: Policy, parameters: _AttachParams) -> Response: _validate_attachment_target(parameters.target_type, parameters.target_id, cluster) - success, error = backup_controller.detach_policy( + success, error = backup_policy.detach_policy( policy.uuid, parameters.target_type, parameters.target_id) if error: raise HTTPException(400, error) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py index d430a6c63..03b7fe469 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py @@ -6,7 +6,8 @@ from simplyblock_core.db_controller import DBController from simplyblock_core import utils as core_utils -from simplyblock_core.controllers import backup_controller, lvol_controller, snapshot_controller +from simplyblock_core.controllers import lvol_controller, snapshot_controller +from simplyblock_core.controllers.backup import controller as backup_controller from simplyblock_core.models.lvol_model import LVol from ...._dependencies import Cluster, StoragePool, Volume diff --git a/tests/integration/expansion_sim/conftest.py b/tests/integration/expansion_sim/conftest.py index 85c604df1..b1f2d0b83 100644 --- a/tests/integration/expansion_sim/conftest.py +++ b/tests/integration/expansion_sim/conftest.py @@ -161,7 +161,7 @@ def patched_rpc_router(): "simplyblock_core.controllers.device_controller", "simplyblock_core.controllers.snapshot_controller", "simplyblock_core.controllers.pool_controller", - "simplyblock_core.controllers.backup_controller", + "simplyblock_core.controllers.backup.device", ] saved_rpc = install_rpc_router(target_modules) saved_fw = install_firewall_stub([ diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index 8b8dc4776..6fd80800b 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -24,7 +24,7 @@ import pytest -from simplyblock_core.controllers.backup_controller import backup_snapshot +from simplyblock_core.controllers.backup.controller import backup_snapshot from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment @@ -223,33 +223,33 @@ def test_backup_config_stored(self): class TestParseAgeString(unittest.TestCase): def test_minutes(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string("30m"), 1800) def test_hours(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string("12h"), 43200) def test_days(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string("2d"), 172800) def test_weeks(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string("1w"), 604800) def test_invalid_format(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string with self.assertRaises(ValueError): _parse_age_string("abc") def test_invalid_unit(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string with self.assertRaises(ValueError): _parse_age_string("5x") def test_whitespace(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string(" 3d "), 259200) @@ -260,14 +260,14 @@ def test_whitespace(self): class TestComputeS3CpuMasks(unittest.TestCase): def test_masks_from_node(self): - from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks + from simplyblock_core.controllers.backup.device import _compute_s3_cpu_masks node = _node() # app_thread_mask="0x8", cpu=8 bdb, s3 = _compute_s3_cpu_masks(node) self.assertEqual(bdb, 0x8) # app thread core 3 self.assertEqual(s3, 0xFF) # all 8 vCPUs — no pinning def test_no_app_thread_mask(self): - from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks + from simplyblock_core.controllers.backup.device import _compute_s3_cpu_masks node = _node() node.app_thread_mask = "" bdb, s3 = _compute_s3_cpu_masks(node) @@ -277,7 +277,7 @@ def test_no_app_thread_mask(self): self.assertEqual(s3, 0xFF) def test_no_cpu_count(self): - from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks + from simplyblock_core.controllers.backup.device import _compute_s3_cpu_masks node = _node() node.cpu = 0 bdb, s3 = _compute_s3_cpu_masks(node) @@ -285,7 +285,7 @@ def test_no_cpu_count(self): self.assertIsNone(s3) # omitted; data plane picks def test_large_cpu_count(self): - from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks + from simplyblock_core.controllers.backup.device import _compute_s3_cpu_masks node = _node() node.cpu = 32 bdb, s3 = _compute_s3_cpu_masks(node) @@ -298,7 +298,7 @@ def test_large_cpu_count(self): class TestCreateS3Bdev(unittest.TestCase): - @patch("simplyblock_core.backup_manifest.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_success(self, MockRPC, mock_boto3_client): mock_rpc = MockRPC.return_value @@ -307,7 +307,7 @@ def test_success(self, MockRPC, mock_boto3_client): mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() create_s3_bdev(node, _backup_config()) @@ -319,10 +319,10 @@ def test_success(self, MockRPC, mock_boto3_client): self.assertEqual(kwargs["bucket_name"], "simplyblock-backup-cluster-1") mock_rpc.bdev_lvol_s3_bdev.assert_called_once_with("lvs_test", "s3_lvs_test") - @patch("simplyblock_core.backup_manifest.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_no_lvstore(self, MockRPC, _mock_boto3_client): - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node(lvstore="") with pytest.raises(PreconditionError): create_s3_bdev(node, _backup_config()) @@ -334,13 +334,13 @@ def test_bdev_s3_create_fails(self, MockRPC): mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = None - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() with pytest.raises(RuntimeError): create_s3_bdev(node, _backup_config()) mock_rpc.bdev_lvol_s3_bdev.assert_not_called() - @patch("simplyblock_core.backup_manifest.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_bucket_is_a_create_parameter(self, MockRPC, mock_boto3_client): """A device cannot exist without its bucket, so there is no window in @@ -350,13 +350,13 @@ def test_bucket_is_a_create_parameter(self, MockRPC, mock_boto3_client): mock_rpc.bdev_lvol_s3_bdev.return_value = True mock_boto3_client.return_value.head_bucket.return_value = {} - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev create_s3_bdev(_node(), _backup_config()) _, kwargs = mock_rpc.bdev_s3_create.call_args assert kwargs["bucket_name"] == "simplyblock-backup-cluster-1" - @patch("simplyblock_core.backup_manifest.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_attach_fails(self, MockRPC, mock_boto3_client): from simplyblock_core.rpc_client import RPCRemoteError @@ -366,12 +366,12 @@ def test_attach_fails(self, MockRPC, mock_boto3_client): mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() with pytest.raises(RuntimeError): create_s3_bdev(node, _backup_config()) - @patch("simplyblock_core.backup_manifest.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_local_testing_params(self, MockRPC, mock_boto3_client): mock_rpc = MockRPC.return_value @@ -381,7 +381,7 @@ def test_local_testing_params(self, MockRPC, mock_boto3_client): mock_s3.head_bucket.return_value = {} from simplyblock_core.models.backup_config import BackupConfig - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() # A genuine pre-BackupConfig dict: no region, local_testing standing in # for four separate decisions. @@ -409,7 +409,7 @@ def test_local_testing_params(self, MockRPC, mock_boto3_client): self.assertEqual(boto_kwargs["region_name"], "us-east-1") self.assertFalse(boto_kwargs["verify"]) - @patch("simplyblock_core.backup_manifest.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_no_credentials_defers_to_the_provider_chain(self, MockRPC, mock_boto3_client): """An absent key pair must mean "use the node's IAM role", not "send empty keys".""" @@ -418,7 +418,7 @@ def test_no_credentials_defers_to_the_provider_chain(self, MockRPC, mock_boto3_c mock_rpc.bdev_lvol_s3_bdev.return_value = True mock_boto3_client.return_value.head_bucket.return_value = {} - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev create_s3_bdev(_node(), _backup_config()) _, boto_kwargs = mock_boto3_client.call_args @@ -431,7 +431,7 @@ def test_exception_handled(self, MockRPC): mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.side_effect = RPCException("connection refused") - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() with pytest.raises(RuntimeError): create_s3_bdev(node, _backup_config()) @@ -458,12 +458,12 @@ def _persist(self, snapshot): snapshot.write_to_db(self.db.kv_store) return snapshot - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.backup_events") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.backup_events") def test_success(self, mock_events, mock_tasks): snap = self._persist(_snapshot()) - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", + with patch("simplyblock_core.controllers.backup.controller._get_snapshot_chain", return_value=[snap]): backup_id, error = backup_snapshot("snap-1") @@ -478,15 +478,15 @@ def test_success(self, mock_events, mock_tasks): self.assertEqual(stored.get_location().bucket_name, "simplyblock-backup-cluster-1") - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.backup_events") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.backup_events") def test_incremental_backup(self, mock_events, mock_tasks): snap = self._persist(_snapshot()) prev = _backup(uuid="prev-backup", s3_id=3, snapshot_id="snap-0", status=Backup.STATUS_COMPLETED) prev.write_to_db(self.db.kv_store) - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", + with patch("simplyblock_core.controllers.backup.controller._get_snapshot_chain", return_value=[snap]): backup_id, error = backup_snapshot("snap-1") @@ -524,15 +524,15 @@ def test_cluster_without_backup_config_is_refused(self): self.assertIn("backup configuration", error) self.assertEqual(self.db.get_backups(), []) - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.backup_events") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.backup_events") def test_chain_backup_acquires_and_releases_lock(self, mock_events, mock_tasks): snap1 = self._persist(_snapshot(uuid="snap-1")) snap1.created_at = 1 snap2 = self._persist(_snapshot(uuid="snap-2")) snap2.created_at = 2 - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", + with patch("simplyblock_core.controllers.backup.controller._get_snapshot_chain", return_value=[snap1, snap2]): backup_id, error = backup_snapshot("snap-2") @@ -548,7 +548,7 @@ def test_chain_backup_lock_conflict(self): acquired, _ = self.db.acquire_backup_chain_locks(["snap-4"], "snap-2", "lvol-1") self.assertTrue(acquired) - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", + with patch("simplyblock_core.controllers.backup.controller._get_snapshot_chain", return_value=[snap]): backup_id, error = backup_snapshot("snap-4") @@ -625,7 +625,7 @@ def _backup(self, **overrides): backup.write_to_db(self.db.kv_store) return backup - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") def test_success(self, mock_tasks): self._backup(s3_id=5) mock_tasks.add_backup_restore_task.return_value = True @@ -640,7 +640,7 @@ def test_success(self, mock_tasks): with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha", return_value=("lvol-new", None)): - from simplyblock_core.controllers.backup_controller import restore_backup + from simplyblock_core.controllers.backup.controller import restore_backup result = restore_backup("backup-1", "restored_lvol", "pool-1") self.assertEqual(result, "lvol-new") @@ -649,7 +649,7 @@ def test_success(self, mock_tasks): self.assertEqual(self.db.get_lvol_by_id("lvol-new").status, LVol.STATUS_RESTORING) def test_backup_not_found(self): - from simplyblock_core.controllers.backup_controller import restore_backup + from simplyblock_core.controllers.backup.controller import restore_backup with self.assertRaises(PreconditionError): restore_backup("missing", "lvol", "pool-1") @@ -658,7 +658,7 @@ def test_add_lvol_ha_fails(self): with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha", return_value=(None, "Pool not found")): - from simplyblock_core.controllers.backup_controller import restore_backup + from simplyblock_core.controllers.backup.controller import restore_backup with self.assertRaisesRegex(RuntimeError, "Failed to create restore volume"): restore_backup("backup-1", "lvol", "pool-1") @@ -666,7 +666,7 @@ def test_incomplete_chain_is_refused(self): self._backup(uuid="b-old", s3_id=1, status=Backup.STATUS_IN_PROGRESS) self._backup(uuid="backup-1", s3_id=2, prev_backup_id="b-old") - from simplyblock_core.controllers.backup_controller import restore_backup + from simplyblock_core.controllers.backup.controller import restore_backup with self.assertRaisesRegex(PreconditionError, "Incomplete backups in chain"): restore_backup("backup-1", "lvol", "pool-1") @@ -679,9 +679,9 @@ def test_incomplete_chain_is_refused(self): class TestDeleteBackups(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.backup_events") + @patch("simplyblock_core.controllers.backup.controller.backup_events") @patch("simplyblock_core.models.storage_node.RPCClient") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_success(self, mock_db, MockRPC, mock_events): b1 = _backup(uuid="b-1") mock_db.get_backups_by_lvol_id.return_value = [b1] @@ -689,18 +689,18 @@ def test_success(self, mock_db, MockRPC, mock_events): b1.remove = MagicMock() MockRPC.return_value.bdev_lvol_s3_delete.return_value = True - from simplyblock_core.controllers.backup_controller import delete_backups + from simplyblock_core.controllers.backup.controller import delete_backups success, error = delete_backups("lvol-1") self.assertTrue(success) self.assertIsNone(error) b1.remove.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_no_backups(self, mock_db): mock_db.get_backups_by_lvol_id.return_value = [] - from simplyblock_core.controllers.backup_controller import delete_backups + from simplyblock_core.controllers.backup.controller import delete_backups success, error = delete_backups("lvol-1") self.assertFalse(success) @@ -713,34 +713,34 @@ def test_no_backups(self, mock_db): class TestListBackups(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_list_empty(self, mock_db): mock_db.get_backups.return_value = [] - from simplyblock_core.controllers.backup_controller import list_backups + from simplyblock_core.controllers.backup.controller import list_backups data = list_backups() self.assertEqual(data, []) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_list_with_backups(self, mock_db): b = _backup() mock_db.get_backups.return_value = [b] - from simplyblock_core.controllers.backup_controller import list_backups + from simplyblock_core.controllers.backup.controller import list_backups data = list_backups() self.assertEqual(len(data), 1) self.assertEqual(data[0]["ID"], "backup-1") self.assertEqual(data[0]["Status"], Backup.STATUS_COMPLETED) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_list_sorted_newest_first_with_seconds(self, mock_db): older = _backup(uuid="older", created_at=1710000000) newer = _backup(uuid="newer", created_at=1710000005) mock_db.get_backups.return_value = [older, newer] - from simplyblock_core.controllers.backup_controller import list_backups + from simplyblock_core.controllers.backup.controller import list_backups data = list_backups() self.assertEqual([row["ID"] for row in data], ["newer", "older"]) @@ -754,38 +754,38 @@ def test_list_sorted_newest_first_with_seconds(self, mock_db): class TestPolicyAdd(unittest.TestCase): @patch.object(BackupPolicy, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_success(self, mock_db, _mock_write): mock_db.get_backup_policies.return_value = [] - from simplyblock_core.controllers.backup_controller import add_policy + from simplyblock_core.controllers.backup.policy import add_policy policy_id, error = add_policy("cluster-1", "daily", max_versions=5, max_age="2d") self.assertIsNotNone(policy_id) self.assertIsNone(error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_no_limits(self, mock_db): - from simplyblock_core.controllers.backup_controller import add_policy + from simplyblock_core.controllers.backup.policy import add_policy policy_id, error = add_policy("cluster-1", "empty", max_versions=0, max_age="") self.assertIsNone(policy_id) self.assertIn("must be specified", error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_duplicate_name(self, mock_db): existing = _policy(name="daily") mock_db.get_backup_policies.return_value = [existing] - from simplyblock_core.controllers.backup_controller import add_policy + from simplyblock_core.controllers.backup.policy import add_policy policy_id, error = add_policy("cluster-1", "daily", max_versions=5) self.assertIsNone(policy_id) self.assertIn("already exists", error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_invalid_age(self, mock_db): - from simplyblock_core.controllers.backup_controller import add_policy + from simplyblock_core.controllers.backup.policy import add_policy policy_id, error = add_policy("cluster-1", "test", max_age="invalid") self.assertIsNone(policy_id) @@ -794,25 +794,25 @@ def test_invalid_age(self, mock_db): class TestPolicyRemove(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_success(self, mock_db): p = _policy() p.remove = MagicMock() mock_db.get_backup_policy_by_id.return_value = p mock_db.get_backup_policy_attachments.return_value = [] - from simplyblock_core.controllers.backup_controller import remove_policy + from simplyblock_core.controllers.backup.policy import remove_policy success, error = remove_policy("policy-1") self.assertTrue(success) self.assertIsNone(error) p.remove.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_not_found(self, mock_db): mock_db.get_backup_policy_by_id.side_effect = KeyError("not found") - from simplyblock_core.controllers.backup_controller import remove_policy + from simplyblock_core.controllers.backup.policy import remove_policy success, error = remove_policy("missing") self.assertFalse(success) @@ -822,25 +822,25 @@ def test_not_found(self, mock_db): class TestPolicyAttach(unittest.TestCase): @patch.object(BackupPolicyAttachment, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_success(self, mock_db, _mock_write): p = _policy() mock_db.get_backup_policy_by_id.return_value = p mock_db.get_lvol_by_id.return_value = MagicMock() mock_db.get_backup_policy_attachments.return_value = [] - from simplyblock_core.controllers.backup_controller import attach_policy + from simplyblock_core.controllers.backup.policy import attach_policy att_id, error = attach_policy("policy-1", "lvol", "lvol-1") self.assertIsNotNone(att_id) self.assertIsNone(error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_invalid_target_type(self, mock_db): p = _policy() mock_db.get_backup_policy_by_id.return_value = p - from simplyblock_core.controllers.backup_controller import attach_policy + from simplyblock_core.controllers.backup.policy import attach_policy att_id, error = attach_policy("policy-1", "invalid", "target-1") self.assertIsNone(att_id) @@ -849,7 +849,7 @@ def test_invalid_target_type(self, mock_db): class TestPolicyDetach(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_success(self, mock_db): p = _policy() att = BackupPolicyAttachment() @@ -861,20 +861,20 @@ def test_success(self, mock_db): mock_db.get_backup_policy_by_id.return_value = p mock_db.get_backup_policy_attachments.return_value = [att] - from simplyblock_core.controllers.backup_controller import detach_policy + from simplyblock_core.controllers.backup.policy import detach_policy success, error = detach_policy("policy-1", "lvol", "lvol-1") self.assertTrue(success) self.assertIsNone(error) att.remove.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_not_found(self, mock_db): p = _policy() mock_db.get_backup_policy_by_id.return_value = p mock_db.get_backup_policy_attachments.return_value = [] - from simplyblock_core.controllers.backup_controller import detach_policy + from simplyblock_core.controllers.backup.policy import detach_policy success, error = detach_policy("policy-1", "lvol", "lvol-1") self.assertFalse(success) @@ -887,20 +887,20 @@ def test_not_found(self, mock_db): class TestEvaluatePolicy(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_no_policy(self, mock_db, mock_tasks): mock_db.get_policy_for_lvol.return_value = None - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_not_called() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_version_limit_exceeded(self, mock_db, mock_tasks, _mock_write): policy = _policy(max_versions=2, max_age_seconds=0) mock_db.get_policy_for_lvol.return_value = policy @@ -911,14 +911,14 @@ def test_version_limit_exceeded(self, mock_db, mock_tasks, _mock_write): b3 = _backup(uuid="b3", created_at=now - 100) mock_db.get_backups_by_lvol_id.return_value = [b1, b2, b3] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_under_version_limit(self, mock_db, mock_tasks): policy = _policy(max_versions=5, max_age_seconds=0) mock_db.get_policy_for_lvol.return_value = policy @@ -928,15 +928,15 @@ def test_under_version_limit(self, mock_db, mock_tasks): b2 = _backup(uuid="b2", created_at=now - 200) mock_db.get_backups_by_lvol_id.return_value = [b1, b2] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_not_called() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): policy = _policy(max_versions=0) policy.max_age_seconds = 3600 # 1 hour @@ -947,15 +947,15 @@ def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): b2 = _backup(uuid="b2", created_at=now - 100) mock_db.get_backups_by_lvol_id.return_value = [b1, b2] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_called_once() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): """When both versions and age are set, either limit can trigger a merge.""" policy = _policy(max_versions=3) @@ -967,15 +967,15 @@ def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): backups = [_backup(uuid=f"b{i}", created_at=now - (i * 60)) for i in range(4)] mock_db.get_backups_by_lvol_id.return_value = backups - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_called_once() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_both_conditions_met(self, mock_db, mock_tasks, _mock_write): """When both limits set and both exceeded, merge triggers.""" policy = _policy(max_versions=2) @@ -988,21 +988,21 @@ def test_both_conditions_met(self, mock_db, mock_tasks, _mock_write): b3 = _backup(uuid="b3", created_at=now - 100) mock_db.get_backups_by_lvol_id.return_value = [b1, b2, b3] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_fewer_than_two_backups(self, mock_db, mock_tasks): """Never merge with fewer than 2 completed backups.""" policy = _policy(max_versions=1) mock_db.get_policy_for_lvol.return_value = policy mock_db.get_backups_by_lvol_id.return_value = [_backup()] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) @@ -1015,13 +1015,13 @@ def test_fewer_than_two_backups(self, mock_db, mock_tasks): class TestListPolicies(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_list_with_policies(self, mock_db): p = _policy(max_versions=5) p.max_age_display = "2d" mock_db.get_backup_policies.return_value = [p] - from simplyblock_core.controllers.backup_controller import list_policies + from simplyblock_core.controllers.backup.policy import list_policies data = list_policies() self.assertEqual(len(data), 1) @@ -1203,9 +1203,9 @@ def test_invalid_location_raises_value_error(self): class TestTriggerMerge(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") def test_trigger_merge_marks_old_as_merging(self, mock_tasks): - from simplyblock_core.controllers.backup_controller import _trigger_merge + from simplyblock_core.controllers.backup.policy import _trigger_merge keep = _backup(uuid="keep") old = _backup(uuid="old") old.write_to_db = MagicMock() @@ -1216,9 +1216,9 @@ def test_trigger_merge_marks_old_as_merging(self, mock_tasks): old.write_to_db.assert_called_once() mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") def test_skip_if_not_completed(self, mock_tasks): - from simplyblock_core.controllers.backup_controller import _trigger_merge + from simplyblock_core.controllers.backup.policy import _trigger_merge keep = _backup(uuid="keep") old = _backup(uuid="old", status=Backup.STATUS_PENDING) diff --git a/tests/integration/test_backup_encryption.py b/tests/integration/test_backup_encryption.py index fcce7d73d..c6a9f0167 100644 --- a/tests/integration/test_backup_encryption.py +++ b/tests/integration/test_backup_encryption.py @@ -8,8 +8,8 @@ """ import pytest -from simplyblock_core import backup_manifest -from simplyblock_core.controllers import backup_controller +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.kms import LocalKMS, backup_dek_path, backup_kek_name diff --git a/tests/integration/test_backup_manifest_flow.py b/tests/integration/test_backup_manifest_flow.py index f7dd2e743..30f72afdf 100644 --- a/tests/integration/test_backup_manifest_flow.py +++ b/tests/integration/test_backup_manifest_flow.py @@ -10,8 +10,8 @@ import pytest -from simplyblock_core import backup_manifest -from simplyblock_core.controllers import backup_controller +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup diff --git a/tests/integration/test_backup_restore_source.py b/tests/integration/test_backup_restore_source.py index c47abc8c4..1f71e2333 100644 --- a/tests/integration/test_backup_restore_source.py +++ b/tests/integration/test_backup_restore_source.py @@ -13,7 +13,8 @@ import pytest -from simplyblock_core.controllers import backup_controller +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import device as backup_device from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup @@ -123,10 +124,10 @@ def test_explicit_credentials_override_the_own_bucket_shortcut(self, db, cluster def test_device_name_is_derived_from_the_backup(self, db): """Stable across retries, so an attempt cannot leak a device per try.""" - assert (backup_controller.restore_s3_bdev_name("b-1234567890") - == backup_controller.restore_s3_bdev_name("b-1234567890")) - assert backup_controller.restore_s3_bdev_name( - "b-1234567890") != backup_controller.restore_s3_bdev_name("c-1234567890") + assert (backup_device.restore_s3_bdev_name("b-1234567890") + == backup_device.restore_s3_bdev_name("b-1234567890")) + assert backup_device.restore_s3_bdev_name( + "b-1234567890") != backup_device.restore_s3_bdev_name("c-1234567890") def _restore_task(db, s3_config=None, **params): @@ -153,7 +154,7 @@ class TestRunnerOwnsTheDevice: def test_own_bucket_creates_nothing(self, db, cluster, node): task = _restore_task(db) - with patch.object(backup_controller, "create_restore_s3_bdev") as create: + with patch.object(backup_device, "create_restore_s3_bdev") as create: tasks_runner_backup._ensure_restore_s3_bdev(task, node) create.assert_not_called() @@ -161,35 +162,35 @@ def test_own_bucket_creates_nothing(self, db, cluster, node): def test_foreign_bucket_device_is_created_by_the_runner(self, db, cluster, node): task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) - with patch.object(backup_controller, "create_restore_s3_bdev") as create: + with patch.object(backup_device, "create_restore_s3_bdev") as create: tasks_runner_backup._ensure_restore_s3_bdev(task, node) _, kwargs = create.call_args args = create.call_args[0] assert args[0] is node assert args[1].bucket_name == FOREIGN_BUCKET - assert args[2] == backup_controller.restore_s3_bdev_name("b-1") + assert args[2] == backup_device.restore_s3_bdev_name("b-1") def test_creation_is_idempotent_across_retries(self, db, cluster, node): """A node restart mid-restore takes the device with it; the runner rebuilds it.""" task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) - with patch.object(backup_controller, "create_restore_s3_bdev") as create: + with patch.object(backup_device, "create_restore_s3_bdev") as create: tasks_runner_backup._ensure_restore_s3_bdev(task, node) tasks_runner_backup._ensure_restore_s3_bdev(task, node) assert create.call_count == 2 assert {c[0][2] for c in create.call_args_list} == { - backup_controller.restore_s3_bdev_name("b-1")} + backup_device.restore_s3_bdev_name("b-1")} def test_release_deletes_the_device(self, db, cluster, node): task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) - with patch.object(backup_controller, "delete_restore_s3_bdev") as delete: + with patch.object(backup_device, "delete_restore_s3_bdev") as delete: tasks_runner_backup._release_restore_s3_bdev(task, node) delete.assert_called_once_with( - node, backup_controller.restore_s3_bdev_name("b-1")) + node, backup_device.restore_s3_bdev_name("b-1")) def test_release_scrubs_the_credentials(self, db, cluster, node): """A task record outlives the restore by weeks; foreign keys must not.""" @@ -198,7 +199,7 @@ def test_release_scrubs_the_credentials(self, db, cluster, node): credentials={"access_key_id": "theirs", "secret_access_key": "theirs"}, ).model_dump(exclude_none=True)) - with patch.object(backup_controller, "delete_restore_s3_bdev"): + with patch.object(backup_device, "delete_restore_s3_bdev"): tasks_runner_backup._release_restore_s3_bdev(task, node) assert task.function_params["s3_config"] is None @@ -207,7 +208,7 @@ def test_release_is_a_noop_for_the_own_bucket(self, db, cluster, node): """The node's own device is shared; a restore must never delete it.""" task = _restore_task(db) - with patch.object(backup_controller, "delete_restore_s3_bdev") as delete: + with patch.object(backup_device, "delete_restore_s3_bdev") as delete: tasks_runner_backup._release_restore_s3_bdev(task, node) delete.assert_not_called() @@ -231,10 +232,10 @@ def test_recovery_names_the_device_it_reads_from(self, db, cluster, node): task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) assert tasks_runner_backup._restore_s3_bdev(task, node) == \ - backup_controller.restore_s3_bdev_name("b-1") + backup_device.restore_s3_bdev_name("b-1") def test_recovery_falls_back_to_the_nodes_own_device(self, db, cluster, node): task = _restore_task(db) assert tasks_runner_backup._restore_s3_bdev(task, node) == \ - backup_controller.primary_s3_bdev_name(node) + backup_device.primary_s3_bdev_name(node) diff --git a/tests/integration/test_backup_validation.py b/tests/integration/test_backup_validation.py index 7979cf400..82434766c 100644 --- a/tests/integration/test_backup_validation.py +++ b/tests/integration/test_backup_validation.py @@ -10,8 +10,9 @@ import pytest from simplyblock_core import constants -from simplyblock_core.backup_manifest import BackupManifest -from simplyblock_core.controllers import backup_controller +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import validation +from simplyblock_core.controllers.backup.manifest import BackupManifest from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup @@ -327,22 +328,22 @@ class TestPredicates: """The rules answer a yes/no question as well as blocking an operation.""" def test_chain_fits_at_the_limit(self): - assert backup_controller.chain_fits(constants.BACKUP_MAX_CHAIN_LENGTH) - assert not backup_controller.chain_fits(constants.BACKUP_MAX_CHAIN_LENGTH + 1) + assert validation.chain_fits(constants.BACKUP_MAX_CHAIN_LENGTH) + assert not validation.chain_fits(constants.BACKUP_MAX_CHAIN_LENGTH + 1) def test_a_tiering_bucket_holds_no_backups(self): - assert backup_controller.location_holds_backups(_config().location()) - assert not backup_controller.location_holds_backups( + assert validation.location_holds_backups(_config().location()) + assert not validation.location_holds_backups( _config(snapshot_backups=False).location()) def test_an_empty_chain_is_coherent(self): - assert backup_controller.chain_is_coherent([], _config().location()) + assert validation.chain_is_coherent([], _config().location()) def test_coherence_covers_a_backup_that_does_not_exist_yet(self, db, cluster): chain = [_backup(db, "b-1", 1, snapshot_id="snap-1", encrypted=False)] - assert backup_controller.chain_is_coherent(chain, _config().location()) - assert backup_controller.chain_is_coherent( + assert validation.chain_is_coherent(chain, _config().location()) + assert validation.chain_is_coherent( chain, _config().location(), encrypted=False) - assert not backup_controller.chain_is_coherent( + assert not validation.chain_is_coherent( chain, _config().location(), encrypted=True) diff --git a/tests/unit/test_backup_manifest.py b/tests/unit/test_backup_manifest.py index 529876692..7f797d07a 100644 --- a/tests/unit/test_backup_manifest.py +++ b/tests/unit/test_backup_manifest.py @@ -6,8 +6,8 @@ import pytest -from simplyblock_core import backup_manifest -from simplyblock_core.backup_manifest import ( +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( BackupManifest, DataPlane, Encryption, diff --git a/tests/unit/test_backup_restore_node_selection.py b/tests/unit/test_backup_restore_node_selection.py index c640d8c15..3fcf51880 100644 --- a/tests/unit/test_backup_restore_node_selection.py +++ b/tests/unit/test_backup_restore_node_selection.py @@ -47,7 +47,7 @@ def _node(uuid, cluster_id, status=StorageNode.STATUS_ONLINE, lvstore="lvs_test" @pytest.fixture def db(): - with patch("simplyblock_core.controllers.backup_controller.db_controller") as db: + with patch("simplyblock_core.controllers.backup.controller.db_controller") as db: pool = MagicMock() pool.cluster_id = TARGET_CLUSTER db.get_pool_by_id_or_name.return_value = pool @@ -80,13 +80,13 @@ def add_lvol_ha(): @pytest.fixture def tasks(): - with patch("simplyblock_core.controllers.backup_controller.tasks_controller") as tasks: + with patch("simplyblock_core.controllers.backup.controller.tasks_controller") as tasks: tasks.add_backup_restore_task.return_value = True yield tasks def _restore(**kwargs): - from simplyblock_core.controllers.backup_controller import restore_backup + from simplyblock_core.controllers.backup.controller import restore_backup return restore_backup("backup-1", "restored_lvol", "pool-1", **kwargs) diff --git a/tests/unit/test_imports.py b/tests/unit/test_imports.py index 1187d0570..039d7a0bd 100644 --- a/tests/unit/test_imports.py +++ b/tests/unit/test_imports.py @@ -35,7 +35,11 @@ "simplyblock_core.controllers.lvol_controller", "simplyblock_core.controllers.snapshot_controller", "simplyblock_core.controllers.migration_controller", - "simplyblock_core.controllers.backup_controller", + "simplyblock_core.controllers.backup.controller", + "simplyblock_core.controllers.backup.device", + "simplyblock_core.controllers.backup.manifest", + "simplyblock_core.controllers.backup.policy", + "simplyblock_core.controllers.backup.validation", "simplyblock_core.controllers.tasks_controller", # --- Models --- "simplyblock_core.models.lvol_migration", diff --git a/tests/unit/web/api/v2/conftest.py b/tests/unit/web/api/v2/conftest.py index 64ba57786..0a1534109 100644 --- a/tests/unit/web/api/v2/conftest.py +++ b/tests/unit/web/api/v2/conftest.py @@ -147,9 +147,15 @@ def snapshot_controller(monkeypatch): @pytest.fixture() def backup_controller(monkeypatch): + """One mock standing in for both halves of the backup package. + + The router reaches `controller` for backups and `policy` for policies; the + tests assert against a single object, so the same mock is installed as both. + """ mock = MagicMock() monkeypatch.setattr(volume_module, 'backup_controller', mock) monkeypatch.setattr(backup_module, 'backup_controller', mock) + monkeypatch.setattr(backup_module, 'backup_policy', mock) return mock diff --git a/tests/unit/web/api/v2/test_backup_endpoints.py b/tests/unit/web/api/v2/test_backup_endpoints.py index 626034dc1..17b0fff84 100644 --- a/tests/unit/web/api/v2/test_backup_endpoints.py +++ b/tests/unit/web/api/v2/test_backup_endpoints.py @@ -188,7 +188,7 @@ def test_naming_neither_source_is_rejected( def test_an_unreadable_bucket_is_a_bad_request_not_a_bad_gateway( self, client, db, cluster, backup_controller): """Nothing here proxies for S3, and the bucket came from the request.""" - from simplyblock_core.backup_manifest import ManifestError + from simplyblock_core.controllers.backup.manifest import ManifestError backup_controller.import_from_bucket.side_effect = ManifestError('no such bucket') response = client.post(f'{BASE}/import', json={'bucket': self._BUCKET}) @@ -216,7 +216,7 @@ class TestDiscoverBackups: def test_returns_the_manifests_the_bucket_holds( self, client, db, backup_controller): - from simplyblock_core.backup_manifest import BackupManifest + from simplyblock_core.controllers.backup.manifest import BackupManifest backup_controller.discover_backups.return_value = [ BackupManifest.model_validate(TestImportBackups._MANIFEST)] @@ -229,7 +229,7 @@ def test_returns_the_manifests_the_bucket_holds( def test_credentials_are_masked_in_the_response_of_a_failure( self, client, db, backup_controller): - from simplyblock_core.backup_manifest import ManifestError + from simplyblock_core.controllers.backup.manifest import ManifestError backup_controller.discover_backups.side_effect = ManifestError('unreachable') response = client.post(f'{BASE}/discover', json={ From 1a8a9ee4bfb3133a49c448ba5dc5c111d7c1090b Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Tue, 18 Aug 2026 17:54:06 +0200 Subject: [PATCH 13/14] fixup! Add typed BackupConfig / BackupLocation models --- simplyblock_core/models/cluster.py | 51 +++++++++++--- tests/unit/test_backup_config_model.py | 92 +++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 9 deletions(-) diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 1e35c793e..7b2e9d626 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -1,6 +1,6 @@ # coding=utf-8 import os.path -from typing import List, Optional +from typing import Any, List, Mapping, Optional from pydantic import SecretStr @@ -224,26 +224,61 @@ def is_qos_set(self) -> bool: return True return False + def default_backup_bucket_name(self) -> str: + """The bucket this cluster backs up to when its configuration names none. + + A config that names no bucket gets the one the bucket name used to be + derived from at device-creation time, so a cluster configured before + ``bucket_name`` existed keeps addressing the bucket it has been writing + to. Callers that configure a bucket per cluster (``StorageCluster``'s + ``spec.backup`` has no bucket field at all) never name one, so this is + the normal case rather than a fallback. + """ + return f"simplyblock-backup-{self.uuid}" + + def _resolve_backup_config(self, config: Mapping[str, Any]) -> BackupConfig: + return BackupConfig.model_validate({ + "bucket_name": self.default_backup_bucket_name(), + **config, + }) + def get_backup_config(self) -> BackupConfig: """Validate and return this cluster's volume-backup configuration. ``backup_config`` stays an untyped dict on the record because ``BaseModel`` cannot nest pydantic models; validating on read gives the - typing without an FDB migration. + typing without an FDB migration. Stored configs are never rewritten by + reading them, so :meth:`default_backup_bucket_name` stays derived rather + than frozen into the record. Raises: ValueError: The cluster has no backup configuration, or the stored - one is not valid -- most commonly a pre-existing config that - predates the mandatory ``region``. ``ValidationError`` is a - ``ValueError``, so one except clause covers both. + one is not valid. ``ValidationError`` is a ``ValueError``, so one + except clause covers both. """ if not self.backup_config: raise ValueError(f"Cluster {self.get_id()} has no backup configuration") - raw_config = self.backup_config - raw_config.setdefault("bucket_name", f"simplyblock-backup-{self.cluster_id}") + return self._resolve_backup_config(self.backup_config) + + def set_backup_config(self, config: Mapping[str, Any]) -> None: + """Validate a raw backup configuration and store it on this record. - return BackupConfig.model_validate(raw_config) + The only way to put a configuration on a cluster. Validating on write as + well as on read is what stops a configuration no cluster could act on + from being accepted at cluster-create and then failing activation on + every node, arbitrarily long after the mistake was made and with nothing + in the failure pointing back at it. + + What gets stored is what the caller passed, so an absent ``bucket_name`` + stays absent and keeps resolving through + :meth:`default_backup_bucket_name`. + + Raises: + ValueError: The configuration is not one this cluster could act on. + """ + self._resolve_backup_config(config) + self.backup_config = dict(config) def get_backup_path(self, path=""): if self.backup_s3_bucket and self.backup_s3_cred: diff --git a/tests/unit/test_backup_config_model.py b/tests/unit/test_backup_config_model.py index 8b746ad0d..748492d0e 100644 --- a/tests/unit/test_backup_config_model.py +++ b/tests/unit/test_backup_config_model.py @@ -238,6 +238,96 @@ def test_unconfigured_cluster_raises(self): def test_invalid_config_raises(self): """ValidationError is a ValueError, so one except clause covers both cases.""" cluster = Cluster() - cluster.backup_config = {"region": "eu-central-1"} # no bucket to write to + cluster.backup_config = {"region": "eu-central-1", "endpoint": "minio:9000"} with pytest.raises(ValueError): cluster.get_backup_config() + + def test_a_config_without_a_bucket_gets_the_derived_one(self): + """The shape every operator-created cluster stores. + + ``StorageCluster.spec.backup`` has no bucket field (the operator's + ``utils.BackupConfig`` cannot express one), so the config that reaches + ``Cluster.backup_config`` names credentials and an endpoint and nothing + else. Rejecting it here fails cluster activation, which validates the + config for every node it brings up + (``cluster_ops._finish_pass1_node`` -> ``create_s3_bdev``). + """ + cluster = Cluster() + cluster.uuid = "7f4c1b2e-0000-4000-8000-000000000001" + cluster.backup_config = { + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin", + "local_endpoint": "http://minio:9000", + } + + config = cluster.get_backup_config() + + assert config.bucket_name == "simplyblock-backup-7f4c1b2e-0000-4000-8000-000000000001" + + def test_an_explicitly_configured_bucket_is_not_overridden(self): + cluster = Cluster() + cluster.uuid = "7f4c1b2e-0000-4000-8000-000000000001" + cluster.backup_config = {"bucket_name": "chosen-by-the-operator"} + + assert cluster.get_backup_config().bucket_name == "chosen-by-the-operator" + + def test_deriving_a_bucket_leaves_the_record_alone(self): + """Otherwise the next write of this cluster persists the derived name as + if someone had configured it, and renaming the derivation would orphan + every backup written before the rename.""" + cluster = Cluster() + cluster.uuid = "7f4c1b2e-0000-4000-8000-000000000001" + cluster.backup_config = {"region": "eu-central-1"} + + cluster.get_backup_config() + + assert cluster.backup_config == {"region": "eu-central-1"} + + +class TestClusterMutator: + """``Cluster.set_backup_config`` is the only way a config reaches a record. + + Before it existed, ``add_cluster`` stored whatever dict the caller handed it + -- a JSON file for ``sbctl cluster create --use-backup``, a request body for + the API -- and nothing looked at it until activation validated it once per + node. So a typo made at cluster-create surfaced as a failed activation. + """ + + def test_a_config_that_cannot_be_validated_is_refused(self): + cluster = Cluster() + + with pytest.raises(ValueError): + cluster.set_backup_config({**MINIMAL, "endpoint": "minio:9000"}) + + assert cluster.backup_config == {} + + def test_a_misspelled_field_is_refused(self): + """The failure this catches earliest: ``extra="forbid"`` turns a typo into + a rejection, but only where something validates. Stored unvalidated, a + misspelled key is a setting that silently does nothing.""" + cluster = Cluster() + + with pytest.raises(ValueError): + cluster.set_backup_config({"buckt_name": "backups"}) + + def test_a_config_without_a_bucket_is_accepted_and_stored_as_given(self): + """The bucket is the one field a valid config may omit, and omitting it + has to survive the round trip -- storing the derived name would freeze + this cluster's bucket into a record that never chose one.""" + cluster = Cluster() + cluster.uuid = "7f4c1b2e-0000-4000-8000-000000000001" + + cluster.set_backup_config({"region": "eu-central-1"}) + + assert cluster.backup_config == {"region": "eu-central-1"} + assert cluster.get_backup_config().bucket_name == cluster.default_backup_bucket_name() + + def test_a_stored_config_is_not_the_caller_dict(self): + """A caller that keeps mutating its dict must not be editing the record.""" + cluster = Cluster() + config = dict(MINIMAL) + + cluster.set_backup_config(config) + config["bucket_name"] = "somewhere-else" + + assert cluster.get_backup_config().bucket_name == "backups" From 5c59fbc041101ea1b895a0f33d7fec337928f7a2 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Tue, 18 Aug 2026 17:54:13 +0200 Subject: [PATCH 14/14] fixup! Wire BackupConfig through the API, cluster ops and create_s3_bdev --- simplyblock_core/cluster_ops.py | 11 ++++++++--- tests/integration/test_backup.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index d89f4903e..e34cc450a 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -296,6 +296,11 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, if not dns_name: raise ValueError("--dns-name is required when --ingress-host-source is dns or loadbalancer") + if backup_config: + # Validate the backup config before doing real work, standing in for the + # bucket name Cluster.set_backup_config defaults below. + BackupConfig.model_validate({"bucket_name": "dummy", **backup_config}) + if name and db_controller.kv_store is not None: existing_clusters = db_controller.get_clusters() for existing in existing_clusters: @@ -429,7 +434,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.tls_config = nvmeof_tls_config if backup_config: - cluster.backup_config = backup_config + cluster.set_backup_config(backup_config) if not disable_monitoring: utils.render_and_deploy_alerting_configs(contact_point, cluster.grafana_endpoint, cluster.uuid, cluster.secret.get_secret_value()) @@ -684,7 +689,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca cluster.snode_api_port = snode_api_port cluster.hashicorp_vault_settings = hashicorp_vault_settings if backup_config: - cluster.backup_config = backup_config + cluster.set_backup_config(backup_config) cluster.backup_local_path = os.path.join(constants.KVD_DB_BACKUP_PATH, cluster.uuid) cluster.status = Cluster.STATUS_UNREADY @@ -709,7 +714,7 @@ def set_backup_config(cl_id, config: BackupConfig) -> None: """ db_controller.atomic_update( db_controller.get_cluster_by_id(cl_id), - lambda c, v=config.model_dump(exclude_none=True): setattr(c, "backup_config", v)) + lambda c, v=config.model_dump(exclude_none=True): c.set_backup_config(v)) def set_name(cl_id, name) -> Cluster: diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index 6fd80800b..0378d524a 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -356,6 +356,30 @@ def test_bucket_is_a_create_parameter(self, MockRPC, mock_boto3_client): _, kwargs = mock_rpc.bdev_s3_create.call_args assert kwargs["bucket_name"] == "simplyblock-backup-cluster-1" + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") + @patch("simplyblock_core.models.storage_node.RPCClient") + def test_a_cluster_that_configured_no_bucket_still_gets_a_device( + self, MockRPC, mock_boto3_client): + """Activation hands ``cluster.get_backup_config()`` straight to this + function for every node it brings up (``cluster_ops._finish_pass1_node``), + so a stored config that names no bucket -- what a cluster created through + the operator has, its CR having no bucket field -- failed the activation + rather than the backup that would have used the bucket.""" + mock_rpc = MockRPC.return_value + mock_rpc.bdev_s3_create.return_value = True + mock_rpc.bdev_lvol_s3_bdev.return_value = True + mock_boto3_client.return_value.head_bucket.return_value = {} + + cluster = Cluster() + cluster.uuid = "cluster-1" + cluster.backup_config = {"region": "eu-central-1"} + + from simplyblock_core.controllers.backup.device import create_s3_bdev + create_s3_bdev(_node(), cluster.get_backup_config()) + + _, kwargs = mock_rpc.bdev_s3_create.call_args + assert kwargs["bucket_name"] == "simplyblock-backup-cluster-1" + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_attach_fails(self, MockRPC, mock_boto3_client):