diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 86b6366a7..ca556f4bd 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -2511,6 +2511,14 @@ commands: help: "The target storage node id." dest: node type: str + - name: "--access-key-id" + help: "Access key for the backup's bucket, when it is not this cluster's own." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the backup's bucket, when it is not this cluster's own." + dest: secret_access_key + type: secret - name: export help: "Export backup metadata to a JSON file for cross-cluster restore." arguments: @@ -2528,17 +2536,81 @@ commands: help: "The output file path." dest: output type: str - - name: import - help: "Import backup metadata from a JSON file." + - name: discover + help: "List the backups a bucket contains, reading its manifests. Needs no cluster." arguments: - - name: "metadata_file" - help: "The path to JSON metadata file." - dest: metadata_file + - name: "--bucket" + help: "The bucket holding the backups." + dest: bucket + type: str + required: true + - name: "--region" + help: "The bucket's region. Omit to let the AWS SDK resolve it." + dest: region type: str + - name: "--endpoint" + help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS." + dest: endpoint + type: str + - name: "--access-key-id" + help: "Access key for the bucket. Omit to use the node's instance role." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the bucket. Omit to use the node's instance role." + dest: secret_access_key + type: secret + - name: "--no-verify-tls" + help: "Skip certificate verification for the endpoint." + dest: no_verify_tls + type: bool + action: store_true + - name: "--path-style" + help: "Use path-style addressing, as MinIO and most S3-compatible stores need." + dest: path_style + type: bool + action: store_true + - name: import + help: "Register backups into this cluster, from a bucket or from an exported file." + arguments: - name: "--cluster-id" help: "The target cluster to import into (required for cross-cluster restore)." dest: cluster_id type: str + - name: "--from-file" + help: "Path to a JSON file produced by 'backup export'. Mutually exclusive with --bucket." + dest: from_file + type: str + - name: "--bucket" + help: "Read manifests straight from this bucket. Mutually exclusive with --from-file." + dest: bucket + type: str + - name: "--region" + help: "The bucket's region. Omit to let the AWS SDK resolve it." + dest: region + type: str + - name: "--endpoint" + help: "Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS." + dest: endpoint + type: str + - name: "--access-key-id" + help: "Access key for the bucket. Omit to use the node's instance role." + dest: access_key_id + type: secret + - name: "--secret-access-key" + help: "Secret key for the bucket. Omit to use the node's instance role." + dest: secret_access_key + type: secret + - name: "--no-verify-tls" + help: "Skip certificate verification for the endpoint." + dest: no_verify_tls + type: bool + action: store_true + - name: "--path-style" + help: "Use path-style addressing, as MinIO and most S3-compatible stores need." + dest: path_style + type: bool + action: store_true - name: policy-add help: "Create a new backup policy." arguments: @@ -2612,24 +2684,6 @@ commands: help: "The target id (storage pool or logical volume id)." dest: target_id type: str - - name: source-list - help: "List backup sources (local and imported clusters)." - arguments: - - name: "--cluster-id" - help: "The cluster id." - dest: cluster_id - type: str - - name: source-switch - help: "Switch the active S3 backup source to a different cluster. Use 'local' or the local cluster id to switch back." - arguments: - - name: "source_cluster_id" - help: "The source cluster id or 'local'." - dest: source_cluster_id - type: str - - name: "--cluster-id" - help: "The cluster id." - dest: cluster_id - type: str - name: "qos" help: "QoS Commands" weight: 700 diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 16ab15886..b8cb315a9 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -1012,14 +1012,13 @@ def init_backup(self): self.init_backup__delete(subparser) self.init_backup__restore(subparser) self.init_backup__export(subparser) + self.init_backup__discover(subparser) self.init_backup__import(subparser) self.init_backup__policy_add(subparser) self.init_backup__policy_remove(subparser) self.init_backup__policy_list(subparser) self.init_backup__policy_attach(subparser) self.init_backup__policy_detach(subparser) - self.init_backup__source_list(subparser) - self.init_backup__source_switch(subparser) def init_backup__list(self, subparser): @@ -1036,6 +1035,8 @@ def init_backup__restore(self, subparser): subcommand.add_argument('--lvol', help='The new logical volume name.', type=str, dest='lvol_name', required=True) subcommand.add_argument('--pool', help='The target pool name or id.', type=str, dest='pool', required=True) subcommand.add_argument('--node', help='The target storage node id.', type=str, dest='node') + subcommand.add_argument('--access-key-id', help='Access key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the backup\'s bucket, when it is not this cluster\'s own.', type=SecretStr, dest='secret_access_key') def init_backup__export(self, subparser): subcommand = self.add_sub_command(subparser, 'export', 'Export backup metadata to a JSON file for cross-cluster restore.') @@ -1043,10 +1044,27 @@ def init_backup__export(self, subparser): subcommand.add_argument('--lvol', help='Filter exports to a specific logical volume name.', type=str, dest='lvol_name') subcommand.add_argument('-o', '--output', help='The output file path.', type=str, dest='output') + def init_backup__discover(self, subparser): + subcommand = self.add_sub_command(subparser, 'discover', 'List the backups a bucket contains, reading its manifests. Needs no cluster.') + subcommand.add_argument('--bucket', help='The bucket holding the backups.', type=str, dest='bucket', required=True) + subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region') + subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint') + subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key') + subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true') + subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true') + def init_backup__import(self, subparser): - subcommand = self.add_sub_command(subparser, 'import', 'Import backup metadata from a JSON file.') - subcommand.add_argument('metadata_file', help='The path to JSON metadata file.', type=str) + subcommand = self.add_sub_command(subparser, 'import', 'Register backups into this cluster, from a bucket or from an exported file.') subcommand.add_argument('--cluster-id', help='The target cluster to import into (required for cross-cluster restore).', type=str, dest='cluster_id') + subcommand.add_argument('--from-file', help='Path to a JSON file produced by \'backup export\'. Mutually exclusive with --bucket.', type=str, dest='from_file') + subcommand.add_argument('--bucket', help='Read manifests straight from this bucket. Mutually exclusive with --from-file.', type=str, dest='bucket') + subcommand.add_argument('--region', help='The bucket\'s region. Omit to let the AWS SDK resolve it.', type=str, dest='region') + subcommand.add_argument('--endpoint', help='Endpoint of an S3-compatible store, e.g. http://minio:9000. Omit for AWS.', type=str, dest='endpoint') + subcommand.add_argument('--access-key-id', help='Access key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='access_key_id') + subcommand.add_argument('--secret-access-key', help='Secret key for the bucket. Omit to use the node\'s instance role.', type=SecretStr, dest='secret_access_key') + subcommand.add_argument('--no-verify-tls', help='Skip certificate verification for the endpoint.', dest='no_verify_tls', action='store_true') + subcommand.add_argument('--path-style', help='Use path-style addressing, as MinIO and most S3-compatible stores need.', dest='path_style', action='store_true') def init_backup__policy_add(self, subparser): subcommand = self.add_sub_command(subparser, 'policy-add', 'Create a new backup policy.') @@ -1076,15 +1094,6 @@ def init_backup__policy_detach(self, subparser): subcommand.add_argument('target_type', help='The target type.', type=str, choices=['pool','lvol',]) subcommand.add_argument('target_id', help='The target id (storage pool or logical volume id).', type=str) - def init_backup__source_list(self, subparser): - subcommand = self.add_sub_command(subparser, 'source-list', 'List backup sources (local and imported clusters).') - subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id') - - def init_backup__source_switch(self, subparser): - subcommand = self.add_sub_command(subparser, 'source-switch', 'Switch the active S3 backup source to a different cluster. Use \'local\' or the local cluster id to switch back.') - subcommand.add_argument('source_cluster_id', help='The source cluster id or \'local\'.', type=str) - subcommand.add_argument('--cluster-id', help='The cluster id.', type=str, dest='cluster_id') - def init_qos(self): subparser = self.add_command('qos', 'QoS Commands') @@ -1557,6 +1566,8 @@ def run(self): ret = self.backup__restore(sub_command, args) elif sub_command in ['export']: ret = self.backup__export(sub_command, args) + elif sub_command in ['discover']: + ret = self.backup__discover(sub_command, args) elif sub_command in ['import']: ret = self.backup__import(sub_command, args) elif sub_command in ['policy-add']: @@ -1569,10 +1580,6 @@ def run(self): ret = self.backup__policy_attach(sub_command, args) elif sub_command in ['policy-detach']: ret = self.backup__policy_detach(sub_command, args) - elif sub_command in ['source-list']: - ret = self.backup__source_list(sub_command, args) - elif sub_command in ['source-switch']: - ret = self.backup__source_switch(sub_command, args) else: self.parser.print_help() diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index eb0ec2a00..7058745bc 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -9,13 +9,19 @@ import argcomplete from simplyblock_core import cluster_ops, utils, db_controller, constants +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup import policy as backup_policy +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, ManifestError) from simplyblock_core.exceptions import MigrationConflictError, PreconditionError from simplyblock_core import storage_node_ops as storage_ops from simplyblock_core import mgmt_node_ops as mgmt_ops from simplyblock_core.controllers import pool_controller, lvol_controller, snapshot_controller, device_controller, \ - tasks_controller, qos_controller, migration_controller, backup_controller, fdb_backup_controller + tasks_controller, qos_controller, migration_controller, fdb_backup_controller from simplyblock_core.controllers import health_controller from simplyblock_core.models.pool import Pool +from simplyblock_core.models.backup_config import BackupConfig, S3Credentials from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings @@ -72,6 +78,44 @@ def _format_result(data, *, json: bool) -> str: return _format_json(data) if json else utils.print_table(data, unwrap_secrets=True) +def _format_timestamp(seconds) -> str: + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(seconds)) if seconds else "" + + +def _s3_credentials(args): + """The bucket credentials given on the command line, if any. + + Returns None when neither key is supplied, which means "use the node's + instance role" rather than "use empty credentials". + """ + access_key_id = getattr(args, 'access_key_id', None) + secret_access_key = getattr(args, 'secret_access_key', None) + if not access_key_id and not secret_access_key: + return None + if not (access_key_id and secret_access_key): + raise ValueError( + "give both --access-key-id and --secret-access-key, or neither") + return S3Credentials(access_key_id=access_key_id, + secret_access_key=secret_access_key) + + +def _bucket_config(args) -> BackupConfig: + """A backup configuration assembled from bucket arguments. + + Used by the commands that read a bucket directly rather than through a + cluster -- which is the point of them, since after a disaster there may be no + cluster left to ask. + """ + return BackupConfig( + bucket_name=args.bucket, + region=getattr(args, 'region', None) or None, + endpoint=getattr(args, 'endpoint', None) or None, + verify_tls=not getattr(args, 'no_verify_tls', False), + use_path_style=getattr(args, 'path_style', False), + credentials=_s3_credentials(args), + ) + + class CLIWrapperBase: def __init__(self): @@ -1044,46 +1088,92 @@ def backup__restore(self, sub_command, args): try: lvol_id = backup_controller.restore_backup( args.backup_id, args.lvol_name, args.pool, - target_node_id=getattr(args, 'node', None)) - except (PreconditionError, RuntimeError) as e: + target_node_id=getattr(args, 'node', None), + s3_credentials=_s3_credentials(args)) + except (PreconditionError, RuntimeError, ValueError) as e: print(f"Error: {e}") return False print(f"Restoring backup {args.backup_id} into new volume {lvol_id}") return True + def backup__discover(self, sub_command, args): + try: + manifests = backup_controller.discover_backups(_bucket_config(args)) + except (ManifestError, ValueError) as e: + print(f"Error: {e}") + return False + if not manifests: + print(f"No backups found in {args.bucket}") + return False + # Each manifest names only its predecessor, so the chain is walked over + # the set. A bucket holding a manifest whose ancestor is missing is worth + # showing rather than refusing: that IS the finding. + def chain_length(manifest): + try: + return str(len(backup_manifest.chain_of(manifest, manifests))) + except ManifestError: + return "incomplete" + + return [{ + "ID": m.backup_id, + "Volume": m.volume.lvol_name, + "Snapshot": m.volume.snapshot_name, + "Size": m.size, + "Chain": chain_length(m), + "Encrypted": "yes" if m.encryption.encrypted else "no", + "Needs KMS": ( + m.encryption.descriptor.kms if m.encryption.descriptor else "-"), + "Created": _format_timestamp(m.created_at), + } for m in manifests] + def backup__export(self, sub_command, args): - data = backup_controller.export_backups( + manifests = backup_controller.export_backups( cluster_id=getattr(args, 'cluster_id', None), lvol_name=getattr(args, 'lvol_name', None)) - if not data: + if not manifests: print("No completed backups found") return False - output = _format_json(data) + output = _format_json([m.model_dump(mode="json") for m in manifests]) output_file = getattr(args, 'output', None) if output_file: with open(output_file, 'w') as f: f.write(output) - print(f"Exported {len(data)} backup(s) to {output_file}") + print(f"Exported {len(manifests)} backup(s) to {output_file}") else: print(output) return True def backup__import(self, sub_command, args): + from_file = getattr(args, 'from_file', None) + bucket = getattr(args, 'bucket', None) + + if bool(from_file) == bool(bucket): + print("Error: give exactly one of --from-file or --bucket") + return False + + cluster_id = getattr(args, 'cluster_id', None) try: - with open(args.metadata_file, 'r') as f: - metadata_list = json.load(f) - except Exception as e: - print(f"Error reading metadata file: {e}") + if bucket: + count = backup_controller.import_from_bucket( + _bucket_config(args), cluster_id=cluster_id) + else: + with open(str(from_file), 'r') as f: + entries = json.load(f) + if not isinstance(entries, list): + entries = [entries] + # Parsed here rather than in the controller so a malformed file + # is reported as a problem with the file, naming it. + manifests = [BackupManifest.model_validate(e) for e in entries] + count = backup_controller.import_backups(manifests, cluster_id=cluster_id) + except (ManifestError, PreconditionError, ValueError, OSError) as e: + print(f"Error: {e}") return False - if not isinstance(metadata_list, list): - metadata_list = [metadata_list] - count = backup_controller.import_backups( - metadata_list, cluster_id=getattr(args, 'cluster_id', None)) + print(f"Imported {count} backup(s)") return True def backup__policy_add(self, sub_command, args): - policy_id, error = backup_controller.add_policy( + policy_id, error = backup_policy.add_policy( args.cluster_id, args.name, max_versions=args.versions or 0, max_age=args.age or "", @@ -1095,7 +1185,7 @@ def backup__policy_add(self, sub_command, args): return True def backup__policy_remove(self, sub_command, args): - success, error = backup_controller.remove_policy(args.policy_id) + success, error = backup_policy.remove_policy(args.policy_id) if error: print(f"Error: {error}") return False @@ -1104,13 +1194,13 @@ def backup__policy_remove(self, sub_command, args): def backup__policy_list(self, sub_command, args): cluster_id = getattr(args, 'cluster_id', None) - data = backup_controller.list_policies(cluster_id) + data = backup_policy.list_policies(cluster_id) if data: return utils.print_table(data) return "No policies found" def backup__policy_attach(self, sub_command, args): - att_id, error = backup_controller.attach_policy( + att_id, error = backup_policy.attach_policy( args.policy_id, args.target_type, args.target_id) if error: print(f"Error: {error}") @@ -1119,7 +1209,7 @@ def backup__policy_attach(self, sub_command, args): return True def backup__policy_detach(self, sub_command, args): - success, error = backup_controller.detach_policy( + success, error = backup_policy.detach_policy( args.policy_id, args.target_type, args.target_id) if error: print(f"Error: {error}") @@ -1127,31 +1217,6 @@ def backup__policy_detach(self, sub_command, args): print("Policy detached") return True - def backup__source_list(self, sub_command, args): - cluster_id = args.cluster_id - if not cluster_id: - db = db_controller.DBController() - clusters = db.get_clusters() - if clusters: - cluster_id = clusters[0].get_id() - sources = backup_controller.get_backup_sources(cluster_id) - return sources - - def backup__source_switch(self, sub_command, args): - cluster_id = args.cluster_id - if not cluster_id: - db = db_controller.DBController() - clusters = db.get_clusters() - if clusters: - cluster_id = clusters[0].get_id() - backup_controller.switch_backup_source(cluster_id, args.source_cluster_id) - target = args.source_cluster_id - if target == cluster_id or target == "local": - print("Switched to local backup source") - else: - print(f"Switched to external backup source: {target}") - return True - def db_backup__create(self, sub_command, args): return fdb_backup_controller.add_backup_task(args.cluster_id) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index 7e3b91376..e34cc450a 100644 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -20,8 +20,10 @@ from simplyblock_core import utils, scripts, constants, mgmt_node_ops, storage_node_ops from simplyblock_core.utils import port_block -from simplyblock_core.controllers import backup_controller, cluster_events, device_controller, qos_controller, tasks_controller, tcp_ports_events +from simplyblock_core.controllers import cluster_events, device_controller, qos_controller, tasks_controller, tcp_ports_events +from simplyblock_core.controllers.backup import device as backup_device from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings, DeployConfig from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.lvol_model import LVol @@ -294,6 +296,11 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, if not dns_name: raise ValueError("--dns-name is required when --ingress-host-source is dns or loadbalancer") + if backup_config: + # Validate the backup config before doing real work, standing in for the + # bucket name Cluster.set_backup_config defaults below. + BackupConfig.model_validate({"bucket_name": "dummy", **backup_config}) + if name and db_controller.kv_store is not None: existing_clusters = db_controller.get_clusters() for existing in existing_clusters: @@ -427,7 +434,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.tls_config = nvmeof_tls_config if backup_config: - cluster.backup_config = backup_config + cluster.set_backup_config(backup_config) if not disable_monitoring: utils.render_and_deploy_alerting_configs(contact_point, cluster.grafana_endpoint, cluster.uuid, cluster.secret.get_secret_value()) @@ -682,7 +689,7 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca cluster.snode_api_port = snode_api_port cluster.hashicorp_vault_settings = hashicorp_vault_settings if backup_config: - cluster.backup_config = backup_config + cluster.set_backup_config(backup_config) cluster.backup_local_path = os.path.join(constants.KVD_DB_BACKUP_PATH, cluster.uuid) cluster.status = Cluster.STATUS_UNREADY @@ -693,6 +700,23 @@ def _add_cluster_impl(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_ca return cluster.get_id() +def set_backup_config(cl_id, config: BackupConfig) -> None: + """Replace a cluster's volume-backup configuration. + + Uses ``atomic_update`` rather than a read-modify-write: this runs while + monitors are concurrently mutating cluster status, and a full write here + would clobber them. + + Note this does not reconfigure S3 bdevs on already-running nodes -- they + pick the new config up on their next restart or cluster activate. Changing + the bucket or the object format also breaks the chain of any existing + backups, which is refused at backup time rather than here. + """ + db_controller.atomic_update( + db_controller.get_cluster_by_id(cl_id), + lambda c, v=config.model_dump(exclude_none=True): c.set_backup_config(v)) + + def set_name(cl_id, name) -> Cluster: cluster = db_controller.get_cluster_by_id(cl_id) if name: @@ -1166,7 +1190,7 @@ def _finish_pass1_node(node_id, ret) -> None: # Create S3 bdev for backup support (only if backup is configured) if cluster.backup_config: snode = db_controller.get_storage_node_by_id(node_id) - backup_controller.create_s3_bdev(snode, cluster.backup_config) + backup_device.create_s3_bdev(snode, cluster.get_backup_config()) else: _set_lvstore_status(node_id, "failed") diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index bda9fba3e..0c1126681 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -605,6 +605,26 @@ def get_config_var(name, default=None): BACKUP_POLL_INTERVAL_SEC = 5 BACKUP_MAX_RETRIES = 10 BACKUP_MERGE_SERVICE_INTERVAL_SEC = 60 -BACKUP_S3_METADATA_BUCKET = "simplyblock-backup-metadata" + +#: Longest backup chain the control plane will accept. +#: +#: This used to be a memory-safety bound: bdev_lvol_s3_backup and +#: bdev_lvol_s3_recovery copied the decoded arrays into fixed 40-element stack +#: buffers, so a longer chain corrupted the storage node's stack. Those buffers +#: are now sized to the decoder's own bound (RPC_MAX_LVOL_VBDEV, 255), so the +#: data plane rejects rather than overruns and this number is the control plane's +#: own policy. +#: +#: It stays at 40 because a restore reads every backup in the chain in one +#: operation: the chain length is a multiplier on restore time and on the objects +#: a recovery has to fetch, and 40 tiered backups is already a long retention +#: history. Raising it is safe up to 255 and needs no data-plane change. +BACKUP_MAX_CHAIN_LENGTH = 40 + +#: Upper bound on a backup's s3_id. The data plane packs it into bits 33..62 of +#: the synthetic bdev offset (S3_ID_BITS in spdk_internal/lvolstore.h) and masks +#: rather than validates, so a larger value silently aliases onto another +#: backup's object keys. +BACKUP_MAX_S3_ID = (1 << 30) - 1 TASKS_RETENTION_PERIOD_SEC = 60*60*24*30 # 30 days \ No newline at end of file diff --git a/simplyblock_core/controllers/backup/__init__.py b/simplyblock_core/controllers/backup/__init__.py new file mode 100644 index 000000000..34a72bc40 --- /dev/null +++ b/simplyblock_core/controllers/backup/__init__.py @@ -0,0 +1,26 @@ +# coding=utf-8 +"""Volume backup to a secondary store. + +Five modules, in dependency order -- each one may use the ones above it: + +``manifest`` + The self-describing record written into the bucket next to a backup's data, + and the only thing that can interpret those objects once the cluster that + wrote them is gone. Owns the control plane's S3 access. +``validation`` + Whether a chain can be restored. Predicates, plus the one function that + refuses. +``device`` + The S3 devices a node reads and writes through. One bucket each. +``controller`` + Creating, restoring, importing, exporting and discovering backups. +``policy`` + Retention limits and tiered schedules, and the merges they cause. + +Deliberately not a facade: nothing is re-exported here. A caller writes + + from simplyblock_core.controllers.backup import controller as backup_controller + +so the import says which part of the subsystem it depends on. Flattening that +back into one namespace would undo the reason for splitting it. +""" diff --git a/simplyblock_core/controllers/backup/controller.py b/simplyblock_core/controllers/backup/controller.py new file mode 100644 index 000000000..c8b8d4124 --- /dev/null +++ b/simplyblock_core/controllers/backup/controller.py @@ -0,0 +1,869 @@ +# coding=utf-8 +"""Creating, restoring, importing, exporting and discovering backups.""" +import logging +import time +import uuid +from typing import Iterable, List, Optional + +from simplyblock_core import constants +from simplyblock_core.controllers import backup_events, tasks_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.validation import ( + chain_fits, require_restorable) +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import ( + BackupConfig, BackupLocation, S3Credentials) +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.kms import ( + KMSException, backup_dek_path, backup_kek_name, create_kms_connection, + lvol_dek_path, pool_kek_name, +) +from simplyblock_core.exceptions import PreconditionError + +logger = logging.getLogger() + +db_controller = DBController() + + +def _generate_backup_id(): + return str(uuid.uuid4()) + + +def get_latest_backup_for_lvol(lvol_id): + """Get the most recent non-failed backup for a given lvol. + + Includes pending/in-progress backups so that chain links are set + even when multiple backups are created in quick succession before + the earlier ones complete. + """ + backups = db_controller.get_backups_by_lvol_id(lvol_id) + valid = [b for b in backups if b.status in ( + Backup.STATUS_COMPLETED, Backup.STATUS_IN_PROGRESS, Backup.STATUS_PENDING)] + if not valid: + return None + valid.sort(key=lambda b: b.created_at, reverse=True) + return valid[0] + + +def _existing_chain_backups(snap_chain) -> list: + """The backups that already exist for a snapshot chain, oldest first.""" + existing = [] + for snap in snap_chain: + for backup in db_controller.get_backups_by_snapshot_id(snap.get_id()): + if backup.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS, + Backup.STATUS_COMPLETED): + existing.append(backup) + return existing + + +def build_manifest(backup: Backup) -> backup_manifest.BackupManifest: + """Assemble the self-describing record for a completed backup. + + Everything a restore needs is collected here, from the backup itself and + from the volume/pool/cluster it came from, so that after this point no part + of the restore path has to consult the originating cluster. + + This function exists because `Backup` and `BackupManifest` describe the same + thing in two shapes. Most of that overlap is not justified, and this is the + seam where it shows: + + * Justified: `status` and `error_message` are on the record and not in the + manifest, because they are mutable control-plane state with no meaning in a + bucket. `schema_version` and `dataplane` are in the manifest and not on the + record, because they are claims about the byte format, which the cluster + that wrote them does not need told back to it. + * Not justified: `pool_uuid` on the record against `volume.pool_name` in the + manifest -- the same fact, keyed differently, so neither can be derived from + the other. `encrypted` living both as its own field and inside `encryption`, + which is why this function has to overlay one onto the other so a manifest + cannot contradict itself. And the volume's settings, which the manifest + records and the record does not, so an imported backup knows less about its + volume than the manifest it was imported from did. + * Actively wrong: `dataplane.cluster_size` and `source` are recomputed here + from the *current* cluster, because the record does not keep what an import + read. Re-exporting an imported backup therefore restamps both with the + importing cluster's identity and page size, silently, though they describe + objects a different cluster wrote. Nothing reads either yet, so nothing is + broken today -- and a bucket's own manifests, which is what a recovery + reads, are written once by the cluster that made the backup and are correct. + + The fix for all three is the same and is not attempted here: make the + manifest the canonical document, store it on the record, and reduce `Backup` + to control-plane state plus the fields FoundationDB is queried by. That is + cheaper than it looks -- every backup query in `db_controller` already filters + in Python over a full scan, so nesting costs nothing there -- but it touches + `BackupDTO`, the `backup list` table and existing records, so it wants its own + change. + """ + volume = backup_manifest.Volume( + lvol_id=backup.lvol_id, + lvol_name=backup.lvol_name, + snapshot_id=backup.snapshot_id, + snapshot_name=backup.snapshot_name, + size=backup.size, + allowed_hosts=backup.allowed_hosts or [], + ) + + # The volume's own settings, where it still exists. Absent together once it + # is gone, which is a different answer from 0 -- for a QoS cap that means + # unlimited. + try: + lvol = db_controller.get_lvol_by_id(backup.lvol_id) + except KeyError: + logger.warning("Volume %s is gone; manifest for backup %s records only " + "the shape carried on the backup itself", + backup.lvol_id, backup.uuid) + else: + volume = volume.model_copy(update={ + "pool_name": lvol.pool_name, + "ha_type": lvol.ha_type or "default", + "fabric": lvol.fabric or "tcp", + "lvol_priority_class": lvol.lvol_priority_class, + "max_size": lvol.max_size, + "rw_ios_per_sec": lvol.rw_ios_per_sec, + "rw_mbytes_per_sec": lvol.rw_mbytes_per_sec, + "r_mbytes_per_sec": lvol.r_mbytes_per_sec, + "w_mbytes_per_sec": lvol.w_mbytes_per_sec, + }) + + cluster_name = None + cluster_size = None + try: + cluster = db_controller.get_cluster_by_id(backup.cluster_id) + except KeyError: + logger.warning("Cluster %s is gone; manifest for backup %s records no " + "object size", backup.cluster_id, backup.uuid) + else: + cluster_name = cluster.cluster_name + cluster_size = cluster.page_size_in_blocks * constants.LVOL_CLUSTER_RATIO + + return backup_manifest.BackupManifest( + backup_id=backup.uuid, + s3_id=backup.s3_id, + created_at=backup.created_at, + completed_at=backup.completed_at, + size=backup.size, + prev_backup_id=backup.prev_backup_id or None, + # backup.encrypted is authoritative -- overlaying it here means the two + # cannot disagree in a manifest, whatever is stored in the dict. + encryption=backup_manifest.Encryption.model_validate( + {**(backup.encryption or {}), "encrypted": backup.encrypted}), + location=backup.get_location(), + source=backup_manifest.Source( + cluster_id=backup.cluster_id, + cluster_name=cluster_name, + node_id=backup.node_id, + ), + volume=volume, + dataplane=backup_manifest.DataPlane(cluster_size=cluster_size), + ) + + +def foreign_bucket_config(backup: Backup, cluster, + credentials: Optional[S3Credentials]) -> Optional[BackupConfig]: + """How to reach this backup's bucket, when it is not the cluster's own. + + Returns ``None`` when the node's existing backup device already points at + the right bucket -- there is no foreign bucket, so there is nothing to + describe. Otherwise returns the configuration for a device that reads the + backup's own recorded location, which is what makes a restore from another + cluster's bucket possible at all. + + Decides only. The device itself is created by the task runner, which is the + component that knows which node the volume landed on and the only one that + can put the device back after a node restart mid-restore. + + Raises: + PreconditionError: The bucket is foreign and unreachable -- no + credentials were supplied for it, while the cluster uses static + credentials that say nothing about it. + """ + location = backup.get_location() + + try: + own = cluster.get_backup_config() + except ValueError: + # No usable configuration of its own, so every bucket is foreign to it. + own = None + + if own is not None and own.location() == location and credentials is None: + return None + + config = BackupConfig.model_validate({ + **location.model_dump(exclude_none=True), + **({"credentials": credentials.model_dump()} if credentials is not None else {}), + }) + + if config.credentials is None and own is not None and own.credentials is not None: + # Falling back to the cluster's own static credentials would fail deep + # in the data plane with nothing to point at the cause. + raise PreconditionError( + f"Backup {backup.uuid} lives in bucket {location.bucket_name}, which " + f"is not this cluster's own. Supply credentials for that bucket, or " + "configure the nodes with an instance role that can read it.") + + return config + + +def _resolve_crypto_key(backup: Backup, cluster): + """Recover the key needed to read an encrypted backup. + + An encrypted backup is ciphertext whose key lives in a KMS, and the backup + records which one. Restoring it therefore needs that KMS reachable -- the + assumption being that a KMS is recoverable independently of any one cluster. + Nothing about the key travels with the backup. + + Returns None for an unencrypted backup. + + Raises: + PreconditionError: The backup records nothing about its key, so no amount + of reachable infrastructure can decrypt it. + RuntimeError: The recorded KMS could not be reached. Raised before the + volume is created, so a restore that cannot decrypt fails without + leaving a half-built volume behind -- and, more importantly, without + silently producing a plaintext volume over ciphertext. + """ + if not backup.encrypted: + return None + + encryption = backup_manifest.Encryption.model_validate( + {**(backup.encryption or {}), "encrypted": backup.encrypted}) + + descriptor = encryption.descriptor + if descriptor is None: + raise PreconditionError( + f"Backup {backup.uuid} is encrypted but records nothing about its " + "key; it predates self-describing backups and cannot be restored") + + try: + with create_kms_connection(cluster) as kms: + return kms.get_data_encryption_keys(descriptor.dek_path, descriptor.kek_name) + except KMSException as e: + raise RuntimeError( + f"Cannot reach the key for backup {backup.uuid} at " + f"{descriptor.dek_path} using {descriptor.kms}, which has to be " + f"reachable to restore it: {e}") from e + + +def _config_for(backup: Backup) -> BackupConfig: + """Credentials for a backup's own bucket. + + The location comes from the backup; only the credentials come from the + cluster, and only because a manifest must never carry them. + + Raises: + PreconditionError: The cluster's configured bucket is not the one this + backup lives in, so its credentials cannot be assumed to reach it. + A precondition rather than a failure: it means the cluster's backup + configuration was repointed while this backup was in flight, which is + visible through GET /clusters/{id}/backup-config. + """ + config = db_controller.get_cluster_by_id(backup.cluster_id).get_backup_config() + location = backup.get_location() + + if config.location() != location: + raise PreconditionError( + f"Backup {backup.uuid} lives in bucket {location.bucket_name}, but " + f"cluster {backup.cluster_id} is configured for " + f"{config.bucket_name}; supply credentials for the backup's bucket") + + return config + + +def write_manifest(backup: Backup) -> None: + """Publish a backup's manifest. + + Raises: + ManifestError: the manifest could not be stored. + PreconditionError: the backup's bucket is not the cluster's own. + """ + backup_manifest.write(_config_for(backup), build_manifest(backup)) + + +def delete_manifest(backup: Backup) -> None: + backup_manifest.delete(_config_for(backup), backup.uuid) + + +def _get_snapshot_chain(snapshot): + """Build the snapshot chain ending at this snapshot, oldest first. + + For cloned volumes, walks snap_ref_id upward. For regular volumes + (no snap_ref_id), collects all snapshots of the same lvol that were + created at or before this snapshot, ordered by created_at. + """ + if snapshot.snap_ref_id: + # Clone-based chain: walk snap_ref_id + chain = [snapshot] + current = snapshot + while current.snap_ref_id: + try: + parent = db_controller.get_snapshot_by_id(current.snap_ref_id) + chain.append(parent) + current = parent + except KeyError: + break + chain.reverse() # oldest first + return chain + + # Regular volume: all snapshots of the same lvol up to this one + lvol_id = snapshot.lvol.get_id() if snapshot.lvol else None + if not lvol_id: + return [snapshot] + + all_snaps = db_controller.get_snapshots_by_lvol_id(lvol_id) + # Filter to snapshots created at or before this one, sort oldest first + chain = [s for s in all_snaps if s.created_at <= snapshot.created_at] + chain.sort(key=lambda s: s.created_at) + return chain + + +def _snapshot_has_backup(snapshot_id): + """Check if a snapshot already has a non-failed backup.""" + backups = db_controller.get_backups_by_snapshot_id(snapshot_id) + return any(b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS, + Backup.STATUS_COMPLETED, Backup.STATUS_MERGED) for b in backups) + + +def _build_encryption(cluster, backup: Backup) -> backup_manifest.Encryption: + """Describe where an encrypted backup's key lives. + + The dependency on a KMS is not removed -- it is written down. Nothing before + this recorded it at all, so an encrypted backup's key was reachable only by + someone who already knew which cluster had made it and how that cluster was + configured. + """ + descriptor = backup_manifest.KeyDescriptor( + kms="local", + dek_path=backup_dek_path(cluster.get_id(), backup.uuid), + kek_name=backup_kek_name(backup.uuid), + ) + if cluster.hashicorp_vault_settings is not None: + vault = cluster.hashicorp_vault_settings + descriptor = descriptor.model_copy(update={ + "kms": "hashicorp_vault", + "vault_base_url": vault.base_url, + "transit_mount": vault.transit_mount, + "kv_mount": vault.kv_mount, + }) + + return backup_manifest.Encryption(encrypted=True, descriptor=descriptor) + + +def create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location: BackupLocation): + """Create a single backup record and task for one snapshot. + + Args: + location: where this backup's objects will be written. Passed in rather + than read from the cluster here so that every backup in one chain is + guaranteed to share it -- the caller resolves it once and validates + the chain against it. + + Returns the created Backup object. + """ + backup_id = _generate_backup_id() + + backup = Backup() + backup.uuid = backup_id + backup.s3_id = db_controller.next_s3_id() + backup.cluster_id = cluster_id + backup.location = location.model_dump(mode="json") + backup.lvol_id = lvol.get_id() + backup.lvol_name = lvol.lvol_name + backup.snapshot_id = snapshot.get_id() + backup.snapshot_name = snapshot.snap_name + backup.node_id = node_id + backup.pool_uuid = lvol.pool_uuid + backup.prev_backup_id = prev_backup.uuid if prev_backup else "" + backup.size = snapshot.size + backup.allowed_hosts = lvol.allowed_hosts + backup.created_at = int(time.time()) + backup.status = Backup.STATUS_PENDING + backup.encrypted = bool(lvol.crypto_bdev) + + if backup.encrypted: + cluster = db_controller.get_cluster_by_id(cluster_id) + with create_kms_connection(cluster) as kms: + kms.create_key_encryption_key(backup_kek_name(backup.uuid)) + kms.rekey_data_encryption_keys( + lvol_dek_path(cluster_id, lvol.get_id()), + pool_kek_name(lvol.pool_uuid), + backup_dek_path(cluster_id, backup.uuid), + backup_kek_name(backup.uuid), + ) + backup.encryption = _build_encryption(cluster, backup).model_dump(mode="json") + + backup.write_to_db() + + backup_events.backup_created(cluster_id, node_id, backup) + tasks_controller.add_backup_task(backup) + + return backup + + +def backup_snapshot(snapshot_id, cluster_id=None): + """Create a backup from an existing snapshot. + + Walks the snapshot chain to ensure all ancestor snapshots are also + backed up, since a single snapshot backup is only a delta and cannot + be restored without its ancestors. + + Returns (backup_id, error_message) where backup_id is the ID of the + backup for the requested snapshot. + """ + try: + snapshot = db_controller.get_snapshot_by_id(snapshot_id) + except KeyError as e: + return None, str(e) + + lvol = snapshot.lvol + node_id = lvol.node_id + try: + snode = db_controller.get_storage_node_by_id(node_id) + except KeyError as e: + return None, str(e) + + if snode.status != StorageNode.STATUS_ONLINE: + return None, f"Node {node_id} is not online (status: {snode.status})" + + if not cluster_id: + cluster_id = snode.cluster_id + + snap_chain = _get_snapshot_chain(snapshot) + + # Everything that could make this backup unrestorable is checked here, + # before the chain lock is taken, before any KMS key is created and before + # any task is enqueued. A backup either is restorable or was never created. + try: + location = db_controller.get_cluster_by_id(cluster_id).get_backup_config().location() + require_restorable( + location, + backups=_existing_chain_backups(snap_chain), + # Every snapshot in the chain gets a backup below, including the ones + # that have none yet, so the eventual length is the chain's. + chain_length=len(snap_chain), + encrypted=bool(lvol.crypto_bdev), + what="This snapshot chain") + except (KeyError, ValueError, PreconditionError) as e: + return None, str(e) + + chain_snapshot_ids = [snap.get_id() for snap in snap_chain] + acquired, existing_lock = db_controller.acquire_backup_chain_locks( + chain_snapshot_ids, snapshot_id, lvol.get_id()) + if not acquired: + lock_snapshot = getattr(existing_lock, "requested_snapshot_id", "") or getattr(existing_lock, "snapshot_id", "") + return None, ( + "A backup request is already preparing this snapshot chain" + + (f" (requested snapshot {lock_snapshot})" if lock_snapshot else "") + ) + + prev_backup = get_latest_backup_for_lvol(lvol.get_id()) + final_backup_id = None + try: + # Walk the snapshot chain and back up all unbacked ancestors first + for snap in snap_chain: + if _snapshot_has_backup(snap.get_id()): + # Already backed up — update prev_backup pointer for chain linking + backups = db_controller.get_backups_by_snapshot_id(snap.get_id()) + existing = next( + (b for b in backups if b.status in ( + Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS, + Backup.STATUS_COMPLETED)), + None) + if existing: + prev_backup = existing + continue + + backup = create_single_backup(snap, lvol, node_id, cluster_id, prev_backup, location) + time.sleep(1) + prev_backup = backup + if snap.get_id() == snapshot_id: + final_backup_id = backup.uuid + finally: + db_controller.release_backup_chain_locks(chain_snapshot_ids) + + if not final_backup_id: + # The target snapshot was already backed up + return None, f"Snapshot {snapshot_id} already has a backup" + + return final_backup_id, None + + +def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, + target_node_id: Optional[str] = None, + s3_credentials: Optional[S3Credentials] = None): + """Restore a backup chain into a new fully-accessible lvol. + + Creates the volume (with subsystem, listeners, namespace) via + lvol_controller.add_lvol_ha, then schedules an async task to + fill in the data from S3. The volume is in STATUS_RESTORING + until the data transfer completes. + + Args: + s3_credentials: Credentials for the backup's bucket, when that is not + the cluster's own. Omit to use the nodes' instance role. + target_node_id: Optional node to restore onto. If not provided, a node + of the target cluster is auto-selected. Any node in the cluster + can restore any backup because S3 keys are node-agnostic + ({s3_id}/{mid_flag}/{extent}) and all nodes share the same + S3 bucket and credentials. + + Returns the uuid of the created volume. + """ + from simplyblock_core.controllers import lvol_controller + from simplyblock_core.models.lvol_model import LVol + + try: + backup = db_controller.get_backup_by_id(backup_id) + pool = db_controller.get_pool_by_id_or_name(pool_id_or_name) + cluster = db_controller.get_cluster_by_id(pool.cluster_id) + target_node = db_controller.get_storage_node_by_id(target_node_id) if target_node_id is not None else None + chain = db_controller.get_backup_chain(backup_id) + if (incomplete := [ + backup for backup in chain + if backup.status != Backup.STATUS_COMPLETED + ]): + raise PreconditionError("Incomplete backups in chain: " + ", ".join(backup.uuid for backup in incomplete)) + except KeyError as e: + raise PreconditionError(str(e)) from e + + # The chain has to be restorable as a unit, and short enough for the data + # plane. Checked before the volume is created so a doomed restore leaves + # nothing behind. + require_restorable(backup.get_location(), backups=chain, + what=f"The chain ending at backup {backup_id}") + + size = backup.size + if size <= 0: + raise PreconditionError("Backup has no size information") + + if target_node is not None: + if target_node.cluster_id != cluster.uuid: + raise PreconditionError( + f"Target node {target_node_id} belongs to cluster " + f"{target_node.cluster_id[:8]}, not {cluster.uuid[:8]}") + + if target_node.status != StorageNode.STATUS_ONLINE: + raise PreconditionError(f"Target node {target_node_id} is not online " + f"(status: {target_node.status})") + + if not target_node.lvstore: + raise PreconditionError( + f"Target node {target_node_id} has no lvstore (S3 bdev requires lvstore)") + + crypto_key = _resolve_crypto_key(backup, cluster) + # Resolved before the volume exists, so an unreachable bucket is refused + # here rather than after a volume has been created for a restore that + # cannot run. + s3_config = foreign_bucket_config(backup, cluster, s3_credentials) + + logger.info(f"Backup allowed hosts: {backup.allowed_hosts}") + lvol_id, error = lvol_controller.add_lvol_ha( + name=lvol_name, + size=size, + pool_id_or_name=pool_id_or_name, + use_crypto=backup.encrypted, + max_size=0, + max_rw_iops=0, + max_rw_mbytes=0, + max_r_mbytes=0, + max_w_mbytes=0, + host_id_or_name=target_node_id, + ha_type="default", + crypto_key=crypto_key, + use_comp=False, + distr_vuid=0, + lvol_priority_class=0, + allowed_hosts=[h["nqn"] if isinstance(h, dict) else h + for h in (backup.allowed_hosts or [])] or None, + fabric="tcp", + ) + if error or not lvol_id: + raise RuntimeError(f"Failed to create restore volume: {error}") + + # Mark volume as restoring + try: + lvol = db_controller.get_lvol_by_id(lvol_id) + except KeyError as e: + raise RuntimeError(f"Volume created but not found in DB: {lvol_id}") from e + + lvol.status = LVol.STATUS_RESTORING + lvol.write_to_db() + + # The bdev name the data plane expects (e.g. LVS_7744/LVOL_12345) + bdev_name = f"{lvol.lvs_name}/{lvol.lvol_bdev}" + + # Data plane processes s3_ids in array order: the first entry's clusters + # take priority (skip-if-populated). Newest-first means the latest + # incremental data wins, with older backups filling any remaining gaps. + if not tasks_controller.add_backup_restore_task( + pool.cluster_id, lvol.node_id, backup_id, bdev_name, + [b.s3_id for b in reversed(chain)], lvol_id=lvol_id, + s3_config=s3_config.model_dump(exclude_none=True) if s3_config is not None else None): + raise RuntimeError("Failed to create restore task") + + return lvol_id + + +def _cleanup_backup_kms_keys(backups): + encrypted = [b for b in backups if b.encrypted] + if not encrypted: + return + try: + cluster = db_controller.get_cluster_by_id(encrypted[0].cluster_id) + with create_kms_connection(cluster) as kms: + for b in encrypted: + try: + kms.delete_data_encryption_keys(backup_dek_path(b.cluster_id, b.uuid)) + kms.delete_key_encryption_key(backup_kek_name(b.uuid)) + except KMSException: + logger.exception(f"Failed to delete keys for backup {b.uuid}") + except (KMSException, KeyError): + logger.exception("Failed to clean up backup KMS keys") + + +def delete_backups(lvol_id): + """Delete all backups for a given lvol. + + Removes the database records, not the objects: bdev_lvol_s3_delete does not + exist on the data plane, so the S3 data outlives this call. The manifests + are deliberately left in place too -- they are the only thing that can still + identify those objects, and deleting them would turn a reclaimable orphan + set into anonymous bucket weight. `backup discover` therefore keeps showing + them, which is the honest answer about what the bucket contains. + + Returns (success, error_message). + """ + backups = db_controller.get_backups_by_lvol_id(lvol_id) + if not backups: + return False, f"No backups found for lvol {lvol_id}" + + _cleanup_backup_kms_keys(backups) + + # Find node to run delete RPC on + completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] + if not completed: + # Just remove from DB + for b in backups: + b.remove(db_controller.kv_store) + return True, None + + node_id = completed[0].node_id + try: + snode = db_controller.get_storage_node_by_id(node_id) + except KeyError: + # Node gone, just clean up DB + for b in backups: + b.remove(db_controller.kv_store) + return True, None + + # Call S3 delete RPC (dummy for now) + if snode.status == StorageNode.STATUS_ONLINE: + rpc_client = snode.rpc_client() + s3_ids = [b.s3_id for b in completed] + try: + rpc_client.bdev_lvol_s3_delete(s3_ids) + except Exception as e: + logger.error(f"Error deleting S3 backups: {e}") + + cluster_id = completed[0].cluster_id + for b in backups: + backup_events.backup_deleted(cluster_id, node_id, b) + b.remove(db_controller.kv_store) + + return True, None + + +def list_backups(cluster_id=None): + """List all backups, optionally filtered by cluster.""" + backups = db_controller.get_backups(cluster_id) + backups = sorted(backups, key=lambda b: (b.created_at, b.uuid), reverse=True) + data = [] + for b in backups: + logger.debug(b) + entry = { + "ID": b.uuid, + "S3 ID": b.s3_id, + "LVol": b.lvol_name, + "Snapshot": b.snapshot_name, + "Node": b.node_id[:8] if b.node_id else "", + "Status": b.status, + "Prev": b.prev_backup_id[:8] if b.prev_backup_id else "-", + "Created": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(b.created_at)) if b.created_at else "", + } + data.append(entry) + return data + + +def export_backups(cluster_id=None, lvol_name=None) -> List[backup_manifest.BackupManifest]: + """Export completed backups as manifests, for import into another cluster. + + Emits the same shape that lives in the bucket, so a hand-carried file and a + bucket read are interchangeable. Previously this produced a third, narrower + format of its own -- which is how it came to omit `encrypted`. + + Returns the manifests themselves; whoever is writing them out decides how + they are rendered. + """ + backups = db_controller.get_backups(cluster_id) + completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] + if lvol_name: + completed = [b for b in completed if b.lvol_name == lvol_name] + + return [build_manifest(b) for b in completed] + + +def discover_backups(config: BackupConfig) -> List[backup_manifest.BackupManifest]: + """Every backup in a bucket, read from its manifests alone. + + The disaster-recovery entry point: given a bucket and credentials for it, + this answers "what is in here" with no reference to any cluster, live or + dead. + + Raises: + ManifestError: The bucket could not be listed, or one of its manifests + could not be parsed. + """ + return backup_manifest.list_all(config) + + +def _require_importable(pending: dict) -> None: + """Refuse a batch of manifests that would not form something restorable. + + An import that lands a backup whose ancestors are missing produces a record + that looks restorable in ``backup list`` and fails only when someone tries it + -- typically during the recovery it was meant to serve. Each manifest names + its predecessor, so this is answerable up front by walking the batch. + + Separate from ``require_restorable`` because the rule is about the batch, not + about a chain: whether every ancestor is either in this import or already in + the database is a question only the importer can ask. The rules that ARE about + the chain are deferred to it. + + Raises: + PreconditionError: A chain is incomplete, cyclic, too long for the data + plane, or spans buckets. + """ + for backup_id, manifest in pending.items(): + chain, seen = [manifest], {backup_id} + + while (previous := chain[-1].prev_backup_id) is not None: + if previous in seen: + raise PreconditionError( + f"Backup {backup_id} has a cyclic chain at {previous}") + seen.add(previous) + + if previous in pending: + chain.append(pending[previous]) + continue + + if not _backup_exists(previous): + raise PreconditionError( + f"Backup {backup_id} is a delta against {previous}, which is " + "neither in this import nor already known. A backup cannot " + "be restored without its chain.") + + # Already imported, and checked then. Its own ancestry still counts + # towards the length the data plane has to accept. + chain.extend(db_controller.get_backup_chain(previous)) + break + + # Manifests carry their location as a value, so coherence over the batch + # is a plain comparison; require_restorable wants Backup records, and + # these are not in the database yet. + divergent = next( + (m for m in chain if m.location != manifest.location), None) + if divergent is not None: + raise PreconditionError( + f"Backup {backup_id} shares a chain with {divergent.backup_id}, " + "which is in a different bucket or encoding") + + if not chain_fits(len(chain)): + raise PreconditionError( + f"The chain of backup {backup_id} is {len(chain)} backups long; " + f"the data plane accepts at most " + f"{constants.BACKUP_MAX_CHAIN_LENGTH}.") + + +def _backup_exists(backup_id: str) -> bool: + try: + db_controller.get_backup_by_id(backup_id) + except KeyError: + return False + return True + + +def import_backups(manifests: Iterable[backup_manifest.BackupManifest], + cluster_id=None) -> int: + """Register backups described by manifests into this cluster's database. + + The backups keep their original ids -- both their uuid and their s3_id, + which names their objects in the bucket and therefore cannot be reassigned. + + Args: + manifests: validated manifests, from `discover_backups`, + `export_backups`, or a file parsed into them. Taking the models + rather than dicts means "is this a manifest at all" is answered by + whoever read the bytes -- the API by its request body's type, the CLI + when it parses the file -- and reported where the input came from. + cluster_id: Target cluster to import into, so the backups are visible in + its namespace. + + Raises: + PreconditionError: One of the backup IDs is already known -- backup + lookups are not scoped by cluster, so a UUID reused across clusters + would make either record unaddressable. Everything is checked before + the first record is written, so a bad batch imports nothing rather + than half of itself. + ValueError: The same backup is listed twice. + """ + pending: dict = {} + for manifest in manifests: + backup_id = manifest.backup_id + + if backup_id in pending: + raise ValueError(f"Backup {backup_id} is listed more than once") + + try: + existing = db_controller.get_backup_by_id(backup_id) + except KeyError: + pending[backup_id] = manifest + else: + raise PreconditionError(f"Backup {backup_id} already exists in cluster {existing.cluster_id}") + + _require_importable(pending) + + for backup_id, manifest in pending.items(): + backup = Backup() + backup.uuid = backup_id + backup.s3_id = manifest.s3_id + backup.cluster_id = cluster_id or manifest.source.cluster_id + backup.lvol_id = manifest.volume.lvol_id + backup.lvol_name = manifest.volume.lvol_name + backup.snapshot_id = manifest.volume.snapshot_id + backup.snapshot_name = manifest.volume.snapshot_name + backup.node_id = manifest.source.node_id + backup.prev_backup_id = manifest.prev_backup_id or "" + backup.size = manifest.size + backup.allowed_hosts = manifest.volume.allowed_hosts + backup.created_at = manifest.created_at + backup.completed_at = manifest.completed_at + backup.status = Backup.STATUS_COMPLETED + backup.location = manifest.location.model_dump(mode="json") + # Import used to drop this, so an imported encrypted backup restored as + # use_crypto=False -- a plaintext volume over ciphertext, silently. + backup.encrypted = manifest.encryption.encrypted + backup.encryption = manifest.encryption.model_dump(mode="json") + backup.write_to_db() + + return len(pending) + + +def import_from_bucket(config: BackupConfig, cluster_id=None) -> int: + """Import every backup found in a bucket. + + Raises: + ManifestError: the bucket could not be read. + PreconditionError: the manifests it holds cannot be imported as a batch. + """ + return import_backups(discover_backups(config), cluster_id=cluster_id) diff --git a/simplyblock_core/controllers/backup/device.py b/simplyblock_core/controllers/backup/device.py new file mode 100644 index 000000000..f7f4b05e2 --- /dev/null +++ b/simplyblock_core/controllers/backup/device.py @@ -0,0 +1,183 @@ +# coding=utf-8 +"""The S3 devices a node reads and writes backups through. + +A device holds exactly one bucket with one set of credentials, so a node runs one +for the cluster's own backup bucket and, during a restore from somebody else's +bucket, a second one attached for the duration. Naming them is part of this +module's job: a restore device's name derives from the backup id, so a retry +re-derives it instead of leaking a device per attempt. +""" +import logging + +from botocore.exceptions import BotoCoreError, ClientError + +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.rpc_client import RPCException + +logger = logging.getLogger() + + +def _compute_s3_cpu_masks(node: StorageNode): + """CPU masks for the S3 bdev, or None where the node does not say. + + Returns (bdb_lcpu_mask, s3_lcpu_mask): + bdb_lcpu_mask: app_thread core (SPDK lightweight thread, low overhead) + s3_lcpu_mask: all system vCPUs (no pinning — let Linux scheduler handle + the AWS SDK thread pool; the data plane default would + wrongly pin onto SPDK reactor cores) + + None rather than 0 for "the node does not tell us": a zero mask selects no + CPUs at all, and the data plane reads it as "unset" anyway, so returning it + would be a sentinel dressed as a value. + """ + # SPDK thread for the bdev poller — reuse the app thread core + bdb_lcpu_mask = int(node.app_thread_mask, 16) if node.app_thread_mask else None + + # AWS SDK thread pool — set all system vCPU bits so threads are unconstrained + s3_lcpu_mask = (1 << node.cpu) - 1 if node.cpu > 0 else None + + return bdb_lcpu_mask, s3_lcpu_mask + + +def primary_s3_bdev_name(node: StorageNode) -> str: + """The S3 device holding the cluster's own backup bucket.""" + return f"s3_{node.lvstore}" + + +def restore_s3_bdev_name(backup_id: str) -> str: + """Name of the device created to read a foreign bucket for one restore. + + Derived from the backup id so a retry re-derives the same name rather than + leaking a device per attempt. + """ + return f"s3_restore_{backup_id[:8]}" + + +def create_restore_s3_bdev(node: StorageNode, config: BackupConfig, name: str) -> None: + """Attach a second S3 device to a node, for a bucket that is not its own. + + A restore from a foreign bucket needs different credentials, a different + endpoint and a different region than the node's own backup device carries. + Since a device holds exactly one bucket, the way to read another one is to + create another device -- which the lvstore supports, its transfer devices + being a list. + + The caller owns the result and must delete it when the restore ends. + """ + rpc_client = node.rpc_client() + bdb_lcpu_mask, s3_lcpu_mask = _compute_s3_cpu_masks(node) + + try: + rpc_client.bdev_s3_create( + name=name, + bucket_name=config.bucket_name, + secondary_target=config.secondary_target, + with_compression=config.with_compression, + snapshot_backups=config.snapshot_backups, + endpoint=config.endpoint_url, + region=config.region, + verify_tls=config.verify_tls, + use_path_style=config.use_path_style, + access_key_id=config.credentials.access_key_id if config.credentials else None, + secret_access_key=config.credentials.secret_access_key if config.credentials else None, + bdb_lcpu_mask=bdb_lcpu_mask, + s3_lcpu_mask=s3_lcpu_mask, + s3_thread_pool_size=config.s3_thread_pool_size, + ) + rpc_client.bdev_lvol_s3_bdev(node.lvstore, name) + except RPCException as e: + raise RuntimeError( + f"Failed to attach S3 device {name} for bucket {config.bucket_name} " + f"on node {node.get_id()}") from e + + logger.info("Attached restore S3 device %s for bucket %s on node %s", + name, config.bucket_name, node.get_id()) + + +def delete_restore_s3_bdev(node: StorageNode, name: str) -> None: + """Detach a device created by :func:`create_restore_s3_bdev`. + + Best-effort by design: this runs on the restore's terminal paths, and a + failure to clean up must not turn a completed restore into a failed one. It + is logged rather than raised, because the consequence is a leaked device -- + which does block the lvstore from being destroyed, so it is worth noticing. + """ + try: + node.rpc_client().bdev_s3_delete(name) + except Exception as e: + # Deliberately broad and deliberately not re-raised: this runs on a + # restore's terminal paths, where the alternative to a leaked device is + # reporting a completed restore as failed. + logger.warning("Could not delete restore S3 device %s on node %s: %s", + name, node.get_id(), e) + else: + logger.info("Deleted restore S3 device %s on node %s", name, node.get_id()) + + +def _ensure_s3_bucket(config: BackupConfig, bucket_name): + try: + s3_client = backup_manifest.s3_client(config) + try: + s3_client.head_bucket(Bucket=bucket_name) + logger.info(f"S3 bucket already exists: {bucket_name}") + except ClientError as e: + error_code = int(e.response["Error"]["Code"]) + if error_code == 404: + s3_client.create_bucket(Bucket=bucket_name) + logger.info(f"S3 bucket created: {bucket_name}") + else: + raise + except BotoCoreError as e: + raise RuntimeError(f"Error ensuring S3 bucket {bucket_name} exists") from e + + +def create_s3_bdev(node: StorageNode, config: BackupConfig) -> None: + """Create the S3 bdev and attach it to a node's lvstore. + Called during cluster activate / node restart. + Args: + node: StorageNode with lvstore set + config: the cluster's validated backup configuration + """ + if not node.lvstore: + raise PreconditionError("Node does not have an lvstore") + + rpc_client = node.rpc_client() + s3_bdev_name = f"s3_{node.lvstore}" + + bdb_lcpu_mask, s3_lcpu_mask = _compute_s3_cpu_masks(node) + + # NO bdev_lvol_create_poller_group here: the lvstore-create poller group + # is created exactly ONCE per SPDK process lifetime — right after + # framework init in the add-node / restart-node flows, on the JC + # singleton's thread/core. This function used to re-call it with + # app_thread_mask (fix f0fed785, which predates the bring-up call from + # #938): a second creation with a different mask that either failed + # noisily on every activate or put the pollers on the wrong core. + + try: + _ensure_s3_bucket(config, config.bucket_name) + + rpc_client.bdev_s3_create( + name=s3_bdev_name, + bucket_name=config.bucket_name, + secondary_target=config.secondary_target, + with_compression=config.with_compression, + snapshot_backups=config.snapshot_backups, + endpoint=config.endpoint_url, + region=config.region, + verify_tls=config.verify_tls, + use_path_style=config.use_path_style, + access_key_id=config.credentials.access_key_id if config.credentials else None, + secret_access_key=config.credentials.secret_access_key if config.credentials else None, + bdb_lcpu_mask=bdb_lcpu_mask, + s3_lcpu_mask=s3_lcpu_mask, + s3_thread_pool_size=config.s3_thread_pool_size, + ) + + rpc_client.bdev_lvol_s3_bdev(node.lvstore, s3_bdev_name) + logger.info(f"S3 bdev created and attached: {s3_bdev_name} on node {node.get_id()}") + except (RPCException, RuntimeError) as e: + raise RuntimeError(f"Error S3 bdev on node {node.get_id()}") from e diff --git a/simplyblock_core/controllers/backup/manifest.py b/simplyblock_core/controllers/backup/manifest.py new file mode 100644 index 000000000..690b33583 --- /dev/null +++ b/simplyblock_core/controllers/backup/manifest.py @@ -0,0 +1,361 @@ +# coding=utf-8 +"""The self-describing part of a backup: a JSON manifest stored alongside its data. + +The data plane writes only opaque objects keyed ``{s3_id}/{mid}/{extent}``, and +nothing in them records which volume they came from, how they are encoded, or +which other backups they depend on. All of that used to live exclusively in the +originating cluster's FoundationDB, which is precisely what a disaster recovery +no longer has. + +A manifest closes that gap. It is written to the same bucket as the data it +describes, under a ``manifests/`` prefix -- a leading non-numeric segment, so it +cannot collide with the data plane's decimal keyspace. Given a bucket and +credentials for it, every backup in it can be enumerated, understood and +restored with no other input. + +Credentials are deliberately absent: a manifest says *where* the objects are and +*how* to read them, never how to authenticate. The reader supplies that. + +Each manifest describes exactly one backup and names only its immediate +predecessor. Chains are walked at read time by :func:`chain_of`, not stored: a +stored chain would have to be rewritten in every descendant's manifest each time +a merge folded a backup away, so a single merge would cost a write per descendant +and could half-fail, leaving the bucket advertising keys the data plane had +already unmapped. + +This document overlaps the ``Backup`` record in FoundationDB by design, and +substantially -- see the note on ``controller.build_manifest`` +for where the two genuinely differ and where they should be collapsed. +""" +import json +import logging +from typing import Iterable, List, Literal, Optional + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError +from pydantic import BaseModel, ConfigDict, model_validator + +from simplyblock_core.models.backup_config import BackupConfig, BackupLocation +from simplyblock_core.utils.secrets import unwrap_secret + + +logger = logging.getLogger() + +MANIFEST_PREFIX = "manifests/" + +#: Bumped when the manifest's meaning changes in a way an older reader would +#: misinterpret. A reader must refuse a version it does not know rather than +#: guess -- restoring from a misread manifest corrupts the volume silently. +MANIFEST_SCHEMA_VERSION = 1 + + +def manifest_key(backup_id: str) -> str: + return f"{MANIFEST_PREFIX}{backup_id}.json" + + +class Source(BaseModel): + """Where this backup came from. Provenance for an operator reading a bucket. + + Nothing may resolve configuration or keys through these -- that dependency + on the originating cluster is the whole problem being removed. + """ + model_config = ConfigDict(extra="forbid") + + cluster_id: str + node_id: str + + #: Absent when the cluster's own record of its name was no longer readable + #: at the time the manifest was written. + cluster_name: Optional[str] = None + + +class Volume(BaseModel): + """The shape of the volume this backup was taken from. + + Split in two by what is knowable. The identity and size come off the backup + record and are always present. The settings below them come off the live + volume, so they are absent together once that volume is deleted -- and + absent is not the same answer as ``0``, which for a QoS cap means + "unlimited" and for a priority class is a real class. + + Nothing reads the settings yet; restore still creates its volume with + hardcoded defaults. They are recorded anyway because a manifest is read + years after it is written, and a backup taken today cannot be given a shape + retroactively once its volume is gone. + """ + model_config = ConfigDict(extra="forbid") + + lvol_id: str + lvol_name: str + snapshot_id: str + snapshot_name: str + size: int + allowed_hosts: List[dict] = [] + + pool_name: Optional[str] = None + ha_type: Optional[str] = None + fabric: Optional[str] = None + lvol_priority_class: Optional[int] = None + max_size: Optional[int] = None + rw_ios_per_sec: Optional[int] = None + rw_mbytes_per_sec: Optional[int] = None + r_mbytes_per_sec: Optional[int] = None + w_mbytes_per_sec: Optional[int] = None + + +class KeyDescriptor(BaseModel): + """Where this backup's data encryption key lives. Never the key itself. + + Restoring an encrypted backup means reaching the KMS named here. The working + assumption is that a KMS is recoverable independently of the cluster that + used it -- a Vault deployment outlives one cluster, and the FoundationDB + behind LocalKMS is itself backed up -- so recording the dependency is enough, + and no key material has to travel with the ciphertext. + + Recording it as a document rather than as loose fields is also what leaves + room to change that assumption: a scheme that wraps the keys under an + operator-held secret adds a sibling field here and a branch in + _resolve_crypto_key, and touches nothing else. + """ + model_config = ConfigDict(extra="forbid") + + #: Which backend holds the key. Named rather than free text, because a reader + #: years later has to know which of the fields below mean anything. + kms: Literal["hashicorp_vault", "local"] + + dek_path: str + kek_name: str + + #: Vault only; absent for the local backend. + vault_base_url: Optional[str] = None + transit_mount: Optional[str] = None + kv_mount: Optional[str] = None + + +class Encryption(BaseModel): + """Whether this backup is ciphertext, and if so how to reach its key.""" + model_config = ConfigDict(extra="forbid") + + encrypted: bool + + #: Where the key lives. Required for an encrypted backup and meaningless + #: otherwise, so the two cannot disagree. + descriptor: Optional[KeyDescriptor] = None + + @model_validator(mode="after") + def _descriptor_matches_encrypted(self) -> "Encryption": + if self.encrypted and self.descriptor is None: + raise ValueError( + "An encrypted backup must record where its key lives; without " + "that, nothing can decrypt it and nothing can say why") + if not self.encrypted and self.descriptor is not None: + raise ValueError( + "An unencrypted backup has no key to describe") + return self + + +class DataPlane(BaseModel): + """How the objects are laid out, so a later format change is detectable.""" + model_config = ConfigDict(extra="forbid") + + #: Object key template. ``mid=1`` is metadata (``/1/0`` is the data plane's + #: own root record, ``/1/n`` the extent map); ``mid=0`` is a data cluster. + key_format: str = "{s3_id}/{mid}/{extent}" + + #: Object body size for both data and metadata objects. Absent when the + #: writing cluster's record was unreadable -- a reader then has to fall back + #: on the data plane's own default, which is why it is not silently 0. + cluster_size: Optional[int] = None + + +class BackupManifest(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: int = MANIFEST_SCHEMA_VERSION + backup_id: str + s3_id: int + created_at: int + completed_at: int + size: int + + #: The backup this one is a delta against, or absent when it is a full + #: backup and therefore the root of its chain. + #: + #: The chain itself is deliberately NOT stored. It is derivable by following + #: these links across the manifests in the bucket, and storing it would make + #: every merge invalidate the manifest of every descendant of the merged-away + #: backup -- so the write amplification of a merge would be the length of the + #: chain, and a partial failure would leave the bucket advertising object keys + #: the data plane had already unmapped. + prev_backup_id: Optional[str] = None + + location: BackupLocation + encryption: Encryption + source: Source + volume: Volume + dataplane: DataPlane + + +class ManifestError(Exception): + """A manifest could not be read, written, or understood.""" + + +def s3_client(config: BackupConfig): + """A boto3 client for a backup location. + + Credentials are passed only when configured; omitting them lets boto3 fall + back to its default provider chain (instance IAM role, environment, + profile), which is what an absent ``credentials`` means. + """ + return boto3.client("s3", + region_name=config.region, + endpoint_url=config.endpoint_url, + verify=config.verify_tls, + config=BotoConfig(s3={"addressing_style": "path" if config.use_path_style else "auto"}), + aws_access_key_id=( + unwrap_secret(config.credentials.access_key_id) + if config.credentials is not None else None), + aws_secret_access_key=( + unwrap_secret(config.credentials.secret_access_key) + if config.credentials is not None else None), + ) + + +def write(config: BackupConfig, manifest: BackupManifest) -> None: + """Store a manifest next to the data it describes. + + Raises: + ManifestError: The manifest could not be stored. Callers must treat this + as a failed backup: data in the bucket with no manifest is data + nobody can identify later. + """ + try: + s3_client(config).put_object( + Bucket=config.bucket_name, + Key=manifest_key(manifest.backup_id), + Body=manifest.model_dump_json().encode(), + ContentType="application/json", + ) + except (BotoCoreError, ClientError) as e: + raise ManifestError( + f"Failed to write manifest for backup {manifest.backup_id}") from e + + logger.info("Wrote manifest for backup %s to %s", + manifest.backup_id, config.bucket_name) + + +def read(config: BackupConfig, backup_id: str) -> BackupManifest: + """Load one manifest by backup id. + + Raises: + ManifestError: It is absent, unreadable, or a schema version this build + does not understand. + """ + try: + body = s3_client(config).get_object( + Bucket=config.bucket_name, Key=manifest_key(backup_id))["Body"].read() + except (BotoCoreError, ClientError) as e: + raise ManifestError( + f"Failed to read manifest for backup {backup_id} " + f"from {config.bucket_name}") from e + + return _parse(body, manifest_key(backup_id)) + + +def list_all(config: BackupConfig) -> List[BackupManifest]: + """Every manifest in the bucket, newest first. + + This is the disaster-recovery entry point: with a bucket and credentials it + answers "what is in here" without reference to any cluster. + + Raises: + ManifestError: The bucket could not be listed, or one of its manifests + could not be parsed. Deliberately not best-effort -- silently + omitting an unreadable backup from a recovery listing is how an + operator concludes their data is gone. + """ + client = s3_client(config) + manifests = [] + + try: + pages = client.get_paginator("list_objects_v2").paginate( + Bucket=config.bucket_name, Prefix=MANIFEST_PREFIX) + for page in pages: + for entry in page.get("Contents", []): + key = entry["Key"] + if not key.endswith(".json"): + continue + body = client.get_object(Bucket=config.bucket_name, Key=key)["Body"].read() + manifests.append(_parse(body, key)) + except (BotoCoreError, ClientError) as e: + raise ManifestError(f"Failed to list manifests in {config.bucket_name}") from e + + manifests.sort(key=lambda m: (m.created_at, m.backup_id), reverse=True) + return manifests + + +def delete(config: BackupConfig, backup_id: str) -> None: + """Remove a manifest. + + Only for a backup whose data is genuinely gone -- a merge folding it into + its successor. Deleting the manifest of a backup whose objects still exist + turns them into weight nobody can identify or reclaim. + """ + try: + s3_client(config).delete_object( + Bucket=config.bucket_name, Key=manifest_key(backup_id)) + except (BotoCoreError, ClientError) as e: + raise ManifestError(f"Failed to delete manifest for backup {backup_id}") from e + + +def _parse(body: bytes, key: str) -> BackupManifest: + try: + data = json.loads(body) + except ValueError as e: + raise ManifestError(f"Manifest {key} is not valid JSON") from e + + version = data.get("schema_version") + if version != MANIFEST_SCHEMA_VERSION: + # Refuse rather than guess: a manifest written by a newer control plane + # may mean something different by the same field names, and restoring + # from a misread manifest corrupts the volume without an error. + raise ManifestError( + f"Manifest {key} has schema version {version}, " + f"this build understands {MANIFEST_SCHEMA_VERSION}") + + try: + return BackupManifest.model_validate(data) + except ValueError as e: + raise ManifestError(f"Manifest {key} is malformed: {e}") from e + + +def chain_of(manifest: BackupManifest, + manifests: Iterable[BackupManifest]) -> List[BackupManifest]: + """The chain ending at ``manifest``, oldest first and including itself. + + Derived by following ``prev_backup_id`` through the manifests supplied, + rather than read from a list stored in each one. That keeps a merge a + two-object write -- republish the survivor, delete the merged-away one -- + instead of a rewrite of every descendant's manifest. + + Raises: + ManifestError: A link points at a backup that is not among the manifests + supplied, so the chain cannot be completed from them. Reported rather + than truncated: a short chain restores a volume with holes in it. + """ + by_id = {m.backup_id: m for m in manifests} + + chain = [manifest] + while (previous := chain[-1].prev_backup_id) is not None: + if previous not in by_id: + raise ManifestError( + f"Backup {chain[-1].backup_id} is a delta against {previous}, " + "which is not among the manifests given") + if previous in {m.backup_id for m in chain}: + raise ManifestError( + f"Backup {manifest.backup_id} has a cyclic chain at {previous}") + chain.append(by_id[previous]) + + chain.reverse() + return chain diff --git a/simplyblock_core/controllers/backup/policy.py b/simplyblock_core/controllers/backup/policy.py new file mode 100644 index 000000000..9112e1a15 --- /dev/null +++ b/simplyblock_core/controllers/backup/policy.py @@ -0,0 +1,340 @@ +# coding=utf-8 +"""Backup policies: retention limits, tiered schedules, and the merges they cause. + +A policy decides *when* a backup is taken and when two are folded together. What +a backup is and how it is written is :mod:`controller`'s business, which this +module calls into rather than reimplements. +""" +import logging +import re +import time +import uuid + +from simplyblock_core.controllers import tasks_controller +from simplyblock_core.controllers.backup.controller import ( + create_single_backup, get_latest_backup_for_lvol) +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment + +logger = logging.getLogger() + +db_controller = DBController() + + +def _parse_age_string(age_str): + """Parse age strings like '2d', '12h', '1w', '30m' into seconds.""" + match = re.match(r'^(\d+)([mhdw])$', age_str.strip()) + if not match: + raise ValueError(f"Invalid age format: {age_str}. Use e.g. 2d, 12h, 1w") + value = int(match.group(1)) + unit = match.group(2) + multipliers = {'m': 60, 'h': 3600, 'd': 86400, 'w': 604800} + return value * multipliers[unit] + + +def _parse_schedule(schedule_str): + """Parse schedule string like '15m,4 60m,11 24h,7' into list of (interval_seconds, keep_count) tuples. + Returns sorted list by interval ascending. Raises ValueError on invalid input.""" + if not schedule_str or not schedule_str.strip(): + return [] + tiers = [] + for part in schedule_str.strip().split(): + parts = part.split(',') + if len(parts) != 2: + raise ValueError(f"Invalid schedule tier: {part}. Expected format: , e.g. 15m,4") + interval_seconds = _parse_age_string(parts[0]) + try: + keep_count = int(parts[1]) + except ValueError: + raise ValueError(f"Invalid keep count in tier: {part}. Must be an integer.") + if keep_count < 1: + raise ValueError(f"Keep count must be >= 1 in tier: {part}") + tiers.append((interval_seconds, keep_count)) + tiers.sort(key=lambda t: t[0]) + # Validate intervals are strictly increasing + for i in range(1, len(tiers)): + if tiers[i][0] <= tiers[i - 1][0]: + raise ValueError("Schedule tier intervals must be strictly increasing") + return tiers + + +def add_policy(cluster_id, name, max_versions=0, max_age="", schedule=""): + """Create a new backup policy. + Returns (policy_id, error_message).""" + max_age_seconds = 0 + if max_age: + try: + max_age_seconds = _parse_age_string(max_age) + except ValueError as e: + return None, str(e) + + if schedule: + try: + _parse_schedule(schedule) + except ValueError as e: + return None, str(e) + + if max_versions <= 0 and max_age_seconds <= 0 and not schedule: + return None, "At least one of --versions, --age, or --schedule must be specified" + + # Check name uniqueness + for p in db_controller.get_backup_policies(cluster_id): + if p.policy_name == name: + return None, f"Policy name already exists: {name}" + + policy = BackupPolicy() + policy.uuid = str(uuid.uuid4()) + policy.cluster_id = cluster_id + policy.policy_name = name + policy.max_versions = max_versions + policy.max_age_seconds = max_age_seconds + policy.max_age_display = max_age + policy.backup_schedule = schedule + policy.status = BackupPolicy.STATUS_ACTIVE + policy.write_to_db() + + return policy.uuid, None + + +def remove_policy(policy_id): + """Remove a backup policy and all its attachments. + Returns (success, error_message).""" + try: + policy = db_controller.get_backup_policy_by_id(policy_id) + except KeyError as e: + return False, str(e) + + # Remove attachments + for att in db_controller.get_backup_policy_attachments(policy.cluster_id): + if att.policy_id == policy_id: + att.remove(db_controller.kv_store) + + policy.remove(db_controller.kv_store) + return True, None + + +def attach_policy(policy_id, target_type, target_id): + """Attach a backup policy to a pool or lvol. + Returns (attachment_id, error_message).""" + try: + policy = db_controller.get_backup_policy_by_id(policy_id) + except KeyError as e: + return None, str(e) + + if target_type not in ("pool", "lvol"): + return None, f"Invalid target_type: {target_type}. Use 'pool' or 'lvol'" + + # Validate target exists + try: + if target_type == "pool": + db_controller.get_pool_by_id(target_id) + else: + db_controller.get_lvol_by_id(target_id) + except KeyError as e: + return None, str(e) + + # Check if already attached + for att in db_controller.get_backup_policy_attachments(policy.cluster_id): + if att.policy_id == policy_id and att.target_type == target_type and att.target_id == target_id: + return att.uuid, None # already attached + + att = BackupPolicyAttachment() + att.uuid = str(uuid.uuid4()) + att.cluster_id = policy.cluster_id + att.policy_id = policy_id + att.target_type = target_type + att.target_id = target_id + att.write_to_db() + + return att.uuid, None + + +def detach_policy(policy_id, target_type, target_id): + """Detach a backup policy from a pool or lvol. + Returns (success, error_message).""" + try: + policy = db_controller.get_backup_policy_by_id(policy_id) + except KeyError as e: + return False, str(e) + + for att in db_controller.get_backup_policy_attachments(policy.cluster_id): + if att.policy_id == policy_id and att.target_type == target_type and att.target_id == target_id: + att.remove(db_controller.kv_store) + return True, None + + return False, "Attachment not found" + + +def list_policies(cluster_id=None): + """List all backup policies.""" + policies = db_controller.get_backup_policies(cluster_id) + data = [] + for p in policies: + data.append({ + "ID": p.uuid, + "Name": p.policy_name, + "Versions": p.max_versions if p.max_versions > 0 else "-", + "Max Age": p.max_age_display if p.max_age_display else "-", + "Schedule": p.backup_schedule if p.backup_schedule else "-", + "Status": p.status, + }) + return data + + +def evaluate_policy(lvol): + """Evaluate backup policy for an lvol and trigger merges if needed. + Called by the backup merge service.""" + policy = db_controller.get_policy_for_lvol(lvol) + if not policy: + return + + backups = db_controller.get_backups_by_lvol_id(lvol.get_id()) + completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] + if len(completed) < 2: + return + + completed.sort(key=lambda b: b.created_at) + now = int(time.time()) + + versions_exceeded = policy.max_versions > 0 and len(completed) > policy.max_versions + age_exceeded = False + if policy.max_age_seconds > 0 and completed: + oldest_age = now - completed[0].created_at + age_exceeded = oldest_age > policy.max_age_seconds + + # Either condition triggers a merge + if versions_exceeded or age_exceeded: + oldest = completed[0] + second = completed[1] + _trigger_merge(second, oldest) + + +def evaluate_schedule(lvol): + """Evaluate the backup schedule for an lvol and trigger auto-backups + tiered merges. + Called by the backup merge service.""" + policy = db_controller.get_policy_for_lvol(lvol) + if not policy or not policy.backup_schedule: + return + + try: + tiers = _parse_schedule(policy.backup_schedule) + except ValueError: + return + + if not tiers: + return + + now = int(time.time()) + + # Check if we need to create a new auto-backup based on the smallest tier interval + smallest_interval = tiers[0][0] + backups = db_controller.get_backups_by_lvol_id(lvol.get_id()) + completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] + pending_or_running = [b for b in backups if b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS)] + + # Don't create a new backup if one is already in progress + if not pending_or_running: + needs_backup = True + if completed: + completed.sort(key=lambda b: b.created_at, reverse=True) + latest = completed[0] + elapsed = now - latest.created_at + if elapsed < smallest_interval: + needs_backup = False + + if needs_backup: + _auto_backup_lvol(lvol) + return # Skip merge evaluation this cycle — let the backup complete first + + # Tiered merge: enforce keep_count per tier. + # Each tier covers an age range. Backups age from tier 0 (newest) + # into higher tiers. When a tier exceeds its keep_count, the oldest + # backup in that tier is merged into its successor. + # All tiers are evaluated each cycle so limits are maintained in parallel. + if len(completed) < 2: + return + + completed.sort(key=lambda b: b.created_at) + + # Don't merge while another merge is already in progress + merging = [b for b in backups if b.status == Backup.STATUS_MERGING] + if merging: + return + + for tier_idx, (interval, keep_count) in enumerate(tiers): + # Age boundaries for this tier + if tier_idx == 0: + lower_age = 0 + else: + lower_age = tiers[tier_idx - 1][0] + + if tier_idx + 1 < len(tiers): + upper_age = tiers[tier_idx + 1][0] + else: + upper_age = float('inf') + + tier_backups = [b for b in completed + if lower_age <= (now - b.created_at) < upper_age] + + if len(tier_backups) > keep_count: + tier_backups.sort(key=lambda b: b.created_at) + oldest = tier_backups[0] + second = tier_backups[1] + _trigger_merge(second, oldest) + return # One merge per cycle to avoid conflicts + + +def _auto_backup_lvol(lvol): + """Create an automatic snapshot + backup for scheduled backups. + + Unlike manual backup_snapshot() which walks the full ancestor chain, + auto-backups create a single snapshot and a single backup for it. + The prev_backup_id is set to the latest existing backup so the + incremental chain is maintained without re-backing all ancestors. + """ + from simplyblock_core.controllers import snapshot_controller + + # Resolve everything the backup needs BEFORE taking the snapshot. This used + # to create the snapshot first and discover afterwards that the node or + # cluster was unusable, leaving an orphaned auto_* snapshot behind on every + # scheduler tick. + node_id = lvol.node_id + try: + snode = db_controller.get_storage_node_by_id(node_id) + cluster_id = snode.cluster_id + location = db_controller.get_cluster_by_id(cluster_id).get_backup_config().location() + except (KeyError, ValueError) as e: + logger.warning(f"Auto-backup skipped for lvol {lvol.get_id()}: {e}") + return + + snap_name = f"auto_{lvol.lvol_name}_{int(time.time())}" + snap_id, error = snapshot_controller.add(lvol.get_id(), snap_name) + if error: + logger.warning(f"Auto-backup snapshot failed for lvol {lvol.get_id()}: {error}") + return + + try: + snapshot = db_controller.get_snapshot_by_id(snap_id) + except KeyError: + logger.warning(f"Auto-backup: snapshot {snap_id} not found after creation") + return + + prev_backup = get_latest_backup_for_lvol(lvol.get_id()) + create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup, location) + + +def _trigger_merge(keep_backup, old_backup): + """Trigger a merge of old_backup into keep_backup.""" + if old_backup.status != Backup.STATUS_COMPLETED: + return + if keep_backup.status != Backup.STATUS_COMPLETED: + return + + old_backup.status = Backup.STATUS_MERGING + old_backup.write_to_db() + + tasks_controller.add_backup_merge_task( + keep_backup.cluster_id, + keep_backup.node_id, + keep_backup.uuid, + old_backup.uuid) diff --git a/simplyblock_core/controllers/backup/validation.py b/simplyblock_core/controllers/backup/validation.py new file mode 100644 index 000000000..d11cfc6f7 --- /dev/null +++ b/simplyblock_core/controllers/backup/validation.py @@ -0,0 +1,112 @@ +# coding=utf-8 +"""When a backup chain can be restored, and when it cannot. + +Predicates rather than assertions, so a rule can answer a question as well as +block an operation -- ``location_holds_backups`` is a thing a caller may want to +know without being refused. :func:`require_restorable` is the one place that turns +a false answer into a refusal, so the wording an operator sees is written once +rather than at each entry point that enforces the same rules. +""" +from typing import Optional + +from simplyblock_core import constants +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup_config import BackupLocation + + +def chain_fits(length: int) -> bool: + """Whether a chain this long is accepted. + + The bound is the control plane's own; see BACKUP_MAX_CHAIN_LENGTH for what it + is a bound on. The data plane's own limit is higher and refuses rather than + overruns, so this is where a too-long chain is reported usefully. + """ + return length <= constants.BACKUP_MAX_CHAIN_LENGTH + + +def location_holds_backups(location: BackupLocation) -> bool: + """Whether backups written to this location could be read back. + + ``snapshot_backups=False`` selects the secondary-tiering object layout, whose + keys are ``{tiering_id}/{lpgi}``. The restore path addresses + ``{s3_id}/{mid}/{extent}``, so it can never find them. + """ + return location.snapshot_backups + + +def chain_is_coherent(backups, location: BackupLocation, + encrypted: Optional[bool] = None) -> bool: + """Whether these backups can be restored together. + + A restore reads clusters from every backup in the chain in one operation, + against one bucket, decrypting all of it with one key. So the chain has to + agree on where it lives, how it is encoded, and whether it is encrypted -- + nothing anywhere in the stack could express a chain split across two buckets + or half encrypted. + + ``encrypted`` folds in a backup that does not exist yet, which is the case at + creation time. + """ + if any(backup.get_location() != location for backup in backups): + return False + + variants = {backup.encrypted for backup in backups} + if encrypted is not None: + variants.add(encrypted) + return len(variants) <= 1 + + +def _describe_incoherence(backups, location: BackupLocation, + encrypted: Optional[bool]) -> str: + for backup in backups: + if backup.get_location() != location: + return ( + f"backup {backup.uuid} lives in bucket " + f"{backup.get_location().bucket_name}, but the rest of its chain " + f"is in {location.bucket_name}. A chain cannot span buckets or " + "encodings; start a new chain with a full backup") + + return ( + "a chain cannot mix encrypted and unencrypted backups: " + + ", ".join(f"{b.uuid}={'encrypted' if b.encrypted else 'plain'}" + for b in backups) + + (f", new backup={'encrypted' if encrypted else 'plain'}" + if encrypted is not None else "")) + + +def require_restorable(location: BackupLocation, backups=(), + chain_length: Optional[int] = None, + encrypted: Optional[bool] = None, + what: str = "This chain") -> None: + """Refuse a chain that could not be restored, naming the rule it breaks. + + Applied at creation, at import and at restore, because each is a point where + a chain could otherwise become unrestorable without anyone noticing -- and + each used to find out from whatever failed first, usually the data plane + mid-operation. + + Args: + chain_length: The eventual length, where it differs from ``len(backups)`` + -- at creation the ancestors are snapshots that have no backup yet. + encrypted: Whether the backup about to be created will be encrypted. + + Raises: + PreconditionError: One of the rules above does not hold. + """ + if not location_holds_backups(location): + raise PreconditionError( + f"Bucket {location.bucket_name} is configured with snapshot_backups " + "disabled, which selects the secondary-tiering object layout. " + "Backups cannot be written there.") + + length = len(backups) if chain_length is None else chain_length + if not chain_fits(length): + raise PreconditionError( + f"{what} is {length} backups long; the data plane accepts at most " + f"{constants.BACKUP_MAX_CHAIN_LENGTH}. Merge older backups to " + "shorten the chain, or start a new chain with a full backup.") + + if not chain_is_coherent(backups, location, encrypted): + raise PreconditionError( + f"{what} cannot be restored as a unit: " + + _describe_incoherence(backups, location, encrypted)) diff --git a/simplyblock_core/controllers/backup_controller.py b/simplyblock_core/controllers/backup_controller.py deleted file mode 100644 index 83ee41ef8..000000000 --- a/simplyblock_core/controllers/backup_controller.py +++ /dev/null @@ -1,1065 +0,0 @@ -# coding=utf-8 -import logging -import re -import time -import uuid -from typing import Optional - -import boto3 -from botocore.exceptions import BotoCoreError, ClientError - -from simplyblock_core.controllers import backup_events, tasks_controller -from simplyblock_core.db_controller import DBController -from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment -from simplyblock_core.models.storage_node import StorageNode -from simplyblock_core.kms import ( - KMSException, backup_dek_path, backup_kek_name, create_kms_connection, - lvol_dek_path, pool_kek_name, -) -from simplyblock_core.utils.secrets import unwrap_secret -from simplyblock_core.exceptions import PreconditionError -from simplyblock_core.rpc_client import RPCException - -logger = logging.getLogger() - -db_controller = DBController() - - -def _generate_backup_id(): - return str(uuid.uuid4()) - - -def _next_s3_id(cluster_id): - """Return the next cluster-wide unique s3_id (uint32) for data-plane RPCs.""" - max_id = 0 - for b in db_controller.get_backups(cluster_id): - if b.s3_id > max_id: - max_id = b.s3_id - return max_id + 1 - - -def _parse_age_string(age_str): - """Parse age strings like '2d', '12h', '1w', '30m' into seconds.""" - match = re.match(r'^(\d+)([mhdw])$', age_str.strip()) - if not match: - raise ValueError(f"Invalid age format: {age_str}. Use e.g. 2d, 12h, 1w") - value = int(match.group(1)) - unit = match.group(2) - multipliers = {'m': 60, 'h': 3600, 'd': 86400, 'w': 604800} - return value * multipliers[unit] - - -def _parse_schedule(schedule_str): - """Parse schedule string like '15m,4 60m,11 24h,7' into list of (interval_seconds, keep_count) tuples. - Returns sorted list by interval ascending. Raises ValueError on invalid input.""" - if not schedule_str or not schedule_str.strip(): - return [] - tiers = [] - for part in schedule_str.strip().split(): - parts = part.split(',') - if len(parts) != 2: - raise ValueError(f"Invalid schedule tier: {part}. Expected format: , e.g. 15m,4") - interval_seconds = _parse_age_string(parts[0]) - try: - keep_count = int(parts[1]) - except ValueError: - raise ValueError(f"Invalid keep count in tier: {part}. Must be an integer.") - if keep_count < 1: - raise ValueError(f"Keep count must be >= 1 in tier: {part}") - tiers.append((interval_seconds, keep_count)) - tiers.sort(key=lambda t: t[0]) - # Validate intervals are strictly increasing - for i in range(1, len(tiers)): - if tiers[i][0] <= tiers[i - 1][0]: - raise ValueError("Schedule tier intervals must be strictly increasing") - return tiers - - -def _write_s3_metadata(rpc_client, backup): - """Write backup metadata to the S3 metadata bucket. - This metadata is needed for cross-cluster recovery.""" - metadata = { - "backup_id": backup.uuid, - "lvol_id": backup.lvol_id, - "lvol_name": backup.lvol_name, - "snapshot_id": backup.snapshot_id, - "snapshot_name": backup.snapshot_name, - "node_id": backup.node_id, - "cluster_id": backup.cluster_id, - "prev_backup_id": backup.prev_backup_id, - "created_at": backup.created_at, - "size": backup.size, - "allowed_hosts": backup.allowed_hosts, - } - backup.s3_metadata = metadata - # The actual S3 metadata write is done via the data plane's S3 bdev. - # For now we store it in the backup object itself. - # In production, this would write to the metadata bucket via S3 API. - return metadata - - -def _get_latest_backup_for_lvol(lvol_id): - """Get the most recent non-failed backup for a given lvol. - - Includes pending/in-progress backups so that chain links are set - even when multiple backups are created in quick succession before - the earlier ones complete. - """ - backups = db_controller.get_backups_by_lvol_id(lvol_id) - valid = [b for b in backups if b.status in ( - Backup.STATUS_COMPLETED, Backup.STATUS_IN_PROGRESS, Backup.STATUS_PENDING)] - if not valid: - return None - valid.sort(key=lambda b: b.created_at, reverse=True) - return valid[0] - - -def _compute_s3_cpu_masks(node): - """Compute CPU masks for the S3 bdev. - Returns (bdb_lcpu_mask, s3_lcpu_mask): - bdb_lcpu_mask: app_thread core (SPDK lightweight thread, low overhead) - s3_lcpu_mask: all system vCPUs (no pinning — let Linux scheduler handle - the AWS SDK thread pool; the data plane default would - wrongly pin onto SPDK reactor cores) - """ - # SPDK thread for the bdev poller — reuse the app thread core - bdb_lcpu_mask = 0 - if node.app_thread_mask: - bdb_lcpu_mask = int(node.app_thread_mask, 16) - - # AWS SDK thread pool — set all system vCPU bits so threads are unconstrained - s3_lcpu_mask = (1 << node.cpu) - 1 if node.cpu > 0 else 0 - - return bdb_lcpu_mask, s3_lcpu_mask - - -def _s3_client(backup_config): - return boto3.client("s3", - aws_access_key_id=unwrap_secret(backup_config.get("access_key_id")), - aws_secret_access_key=unwrap_secret(backup_config.get("secret_access_key")), - endpoint_url=backup_config.get("local_endpoint"), - ) - -def _s3_bucket_exists(backup_config, bucket_name) -> bool: - try: - _s3_client(backup_config).head_bucket(Bucket=bucket_name) - return True - except ClientError as e: - error_code = int(e.response["Error"]["Code"]) - if error_code == 404: - return False - raise - - -def _ensure_s3_bucket(backup_config, bucket_name): - try: - s3_client = _s3_client(backup_config) - try: - s3_client.head_bucket(Bucket=bucket_name) - logger.info(f"S3 bucket already exists: {bucket_name}") - except ClientError as e: - error_code = int(e.response["Error"]["Code"]) - if error_code == 404: - s3_client.create_bucket(Bucket=bucket_name) - logger.info(f"S3 bucket created: {bucket_name}") - else: - raise - except BotoCoreError as e: - raise RuntimeError(f"Error ensuring S3 bucket {bucket_name} exists") from e - - -def create_s3_bdev(node, backup_config) -> None: - """Create the S3 bdev and attach it to a node's lvstore. - Called during cluster activate / node restart. - Args: - node: StorageNode with lvstore set - backup_config: dict from cluster.backup_config with S3/MinIO connection params - """ - if not node.lvstore: - raise PreconditionError("Node does not have an lvstore") - - rpc_client = node.rpc_client() - s3_bdev_name = f"s3_{node.lvstore}" - - bdb_lcpu_mask, s3_lcpu_mask = _compute_s3_cpu_masks(node) - - # NO bdev_lvol_create_poller_group here: the lvstore-create poller group - # is created exactly ONCE per SPDK process lifetime — right after - # framework init in the add-node / restart-node flows, on the JC - # singleton's thread/core. This function used to re-call it with - # app_thread_mask (fix f0fed785, which predates the bring-up call from - # #938): a second creation with a different mask that either failed - # noisily on every activate or put the pollers on the wrong core. - - try: - rpc_client.bdev_s3_create( - name=s3_bdev_name, - secondary_target=backup_config.get("secondary_target", 0), - with_compression=backup_config.get("with_compression", False), - snapshot_backups=backup_config.get("snapshot_backups", True), - local_testing=backup_config.get("local_testing", False), - local_endpoint=backup_config.get("local_endpoint", ""), - access_key_id=backup_config.get("access_key_id", ""), - secret_access_key=backup_config.get("secret_access_key", ""), - bdb_lcpu_mask=bdb_lcpu_mask, - s3_lcpu_mask=s3_lcpu_mask, - s3_thread_pool_size=backup_config.get("s3_thread_pool_size", 0), - ) - - bucket_name = backup_config.get("bucket_name", f"simplyblock-backup-{node.cluster_id}") - _ensure_s3_bucket(backup_config, bucket_name) - - rpc_client.bdev_s3_add_bucket_name(s3_bdev_name, bucket_name, allow_existing=True) - logger.info(f"S3 bdev bucket set: {bucket_name} on {s3_bdev_name}") - - rpc_client.bdev_lvol_s3_bdev(node.lvstore, s3_bdev_name) - logger.info(f"S3 bdev created and attached: {s3_bdev_name} on node {node.get_id()}") - except (RPCException, RuntimeError) as e: - raise RuntimeError(f"Error S3 bdev on node {node.get_id()}") from e - - -def _get_snapshot_chain(snapshot): - """Build the snapshot chain ending at this snapshot, oldest first. - - For cloned volumes, walks snap_ref_id upward. For regular volumes - (no snap_ref_id), collects all snapshots of the same lvol that were - created at or before this snapshot, ordered by created_at. - """ - if snapshot.snap_ref_id: - # Clone-based chain: walk snap_ref_id - chain = [snapshot] - current = snapshot - while current.snap_ref_id: - try: - parent = db_controller.get_snapshot_by_id(current.snap_ref_id) - chain.append(parent) - current = parent - except KeyError: - break - chain.reverse() # oldest first - return chain - - # Regular volume: all snapshots of the same lvol up to this one - lvol_id = snapshot.lvol.get_id() if snapshot.lvol else None - if not lvol_id: - return [snapshot] - - all_snaps = db_controller.get_snapshots_by_lvol_id(lvol_id) - # Filter to snapshots created at or before this one, sort oldest first - chain = [s for s in all_snaps if s.created_at <= snapshot.created_at] - chain.sort(key=lambda s: s.created_at) - return chain - - -def _snapshot_has_backup(snapshot_id): - """Check if a snapshot already has a non-failed backup.""" - backups = db_controller.get_backups_by_snapshot_id(snapshot_id) - return any(b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS, - Backup.STATUS_COMPLETED, Backup.STATUS_MERGED) for b in backups) - - -def _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup): - """Create a single backup record and task for one snapshot. - Returns the created Backup object.""" - backup_id = _generate_backup_id() - - backup = Backup() - backup.uuid = backup_id - backup.s3_id = _next_s3_id(cluster_id) - backup.cluster_id = cluster_id - backup.source_cluster_id = cluster_id # local backup - backup.lvol_id = lvol.get_id() - backup.lvol_name = lvol.lvol_name - backup.snapshot_id = snapshot.get_id() - backup.snapshot_name = snapshot.snap_name - backup.node_id = node_id - backup.pool_uuid = lvol.pool_uuid - backup.prev_backup_id = prev_backup.uuid if prev_backup else "" - backup.size = snapshot.size - backup.allowed_hosts = lvol.allowed_hosts - backup.created_at = int(time.time()) - backup.status = Backup.STATUS_PENDING - backup.encrypted = bool(lvol.crypto_bdev) - - if backup.encrypted: - cluster = db_controller.get_cluster_by_id(cluster_id) - with create_kms_connection(cluster) as kms: - kms.create_key_encryption_key(backup_kek_name(backup.uuid)) - kms.rekey_data_encryption_keys( - lvol_dek_path(cluster_id, lvol.get_id()), - pool_kek_name(lvol.pool_uuid), - backup_dek_path(cluster_id, backup.uuid), - backup_kek_name(backup.uuid), - ) - - backup.write_to_db() - - _write_s3_metadata(None, backup) - backup.write_to_db() - - backup_events.backup_created(cluster_id, node_id, backup) - tasks_controller.add_backup_task(backup) - - return backup - - -def backup_snapshot(snapshot_id, cluster_id=None): - """Create a backup from an existing snapshot. - - Walks the snapshot chain to ensure all ancestor snapshots are also - backed up, since a single snapshot backup is only a delta and cannot - be restored without its ancestors. - - Returns (backup_id, error_message) where backup_id is the ID of the - backup for the requested snapshot. - """ - try: - snapshot = db_controller.get_snapshot_by_id(snapshot_id) - except KeyError as e: - return None, str(e) - - # Block new backups when S3 source is switched to an external cluster - node_id = snapshot.lvol.node_id if snapshot.lvol else None - if node_id: - try: - snode = db_controller.get_storage_node_by_id(node_id) - if not is_local_backup_source(snode.cluster_id): - return None, ("Cannot create backups while backup source is " - "switched to an external cluster. Switch back " - "to local first.") - except KeyError: - pass - - if not snapshot.lvol: - return None, "Snapshot has no associated lvol" - - lvol = snapshot.lvol - node_id = lvol.node_id - try: - snode = db_controller.get_storage_node_by_id(node_id) - except KeyError as e: - return None, str(e) - - if snode.status != StorageNode.STATUS_ONLINE: - return None, f"Node {node_id} is not online (status: {snode.status})" - - if not cluster_id: - cluster_id = snode.cluster_id - - snap_chain = _get_snapshot_chain(snapshot) - chain_snapshot_ids = [snap.get_id() for snap in snap_chain] - acquired, existing_lock = db_controller.acquire_backup_chain_locks( - chain_snapshot_ids, snapshot_id, lvol.get_id()) - if not acquired: - lock_snapshot = getattr(existing_lock, "requested_snapshot_id", "") or getattr(existing_lock, "snapshot_id", "") - return None, ( - "A backup request is already preparing this snapshot chain" - + (f" (requested snapshot {lock_snapshot})" if lock_snapshot else "") - ) - - prev_backup = _get_latest_backup_for_lvol(lvol.get_id()) - final_backup_id = None - try: - # Walk the snapshot chain and back up all unbacked ancestors first - for snap in snap_chain: - if _snapshot_has_backup(snap.get_id()): - # Already backed up — update prev_backup pointer for chain linking - backups = db_controller.get_backups_by_snapshot_id(snap.get_id()) - existing = next( - (b for b in backups if b.status in ( - Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS, - Backup.STATUS_COMPLETED)), - None) - if existing: - prev_backup = existing - continue - - backup = _create_single_backup(snap, lvol, node_id, cluster_id, prev_backup) - time.sleep(1) - prev_backup = backup - if snap.get_id() == snapshot_id: - final_backup_id = backup.uuid - finally: - db_controller.release_backup_chain_locks(chain_snapshot_ids) - - if not final_backup_id: - # The target snapshot was already backed up - return None, f"Snapshot {snapshot_id} already has a backup" - - return final_backup_id, None - - -def restore_backup(backup_id: str, lvol_name: str, pool_id_or_name: str, - target_node_id: Optional[str] = None): - """Restore a backup chain into a new fully-accessible lvol. - - Creates the volume (with subsystem, listeners, namespace) via - lvol_controller.add_lvol_ha, then schedules an async task to - fill in the data from S3. The volume is in STATUS_RESTORING - until the data transfer completes. - - Args: - target_node_id: Optional node to restore onto. If not provided, a node - of the target cluster is auto-selected. Any node in the cluster - can restore any backup because S3 keys are node-agnostic - ({s3_id}/{mid_flag}/{extent}) and all nodes share the same - S3 bucket and credentials. - - Returns the uuid of the created volume. - """ - from simplyblock_core.controllers import lvol_controller - from simplyblock_core.models.lvol_model import LVol - - try: - backup = db_controller.get_backup_by_id(backup_id) - pool = db_controller.get_pool_by_id_or_name(pool_id_or_name) - cluster = db_controller.get_cluster_by_id(pool.cluster_id) - target_node = db_controller.get_storage_node_by_id(target_node_id) if target_node_id is not None else None - chain = db_controller.get_backup_chain(backup_id) - if (incomplete := [ - backup for backup in chain - if backup.status != Backup.STATUS_COMPLETED - ]): - raise PreconditionError("Incomplete backups in chain: " + ", ".join(backup.uuid for backup in incomplete)) - except KeyError as e: - raise PreconditionError(str(e)) from e - - # Verify the backup's source matches the active S3 source. - # If the backup came from an external cluster, the S3 bdev must be - # switched to that cluster's bucket before restoring. - backup_src = backup.source_cluster_id or backup.cluster_id - active_src = cluster.backup_source or cluster.uuid - if backup_src != active_src: - raise PreconditionError( - f"Backup source is {backup_src[:8]} but active S3 source " - f"is {active_src[:8]}. Use 'sbctl backup source-switch " - f"{backup_src}' first.") - - size = backup.size - if size <= 0: - raise PreconditionError("Backup has no size information") - - if target_node is not None: - if target_node.cluster_id != cluster.uuid: - raise PreconditionError( - f"Target node {target_node_id} belongs to cluster " - f"{target_node.cluster_id[:8]}, not {cluster.uuid[:8]}") - - if target_node.status != StorageNode.STATUS_ONLINE: - raise PreconditionError(f"Target node {target_node_id} is not online " - f"(status: {target_node.status})") - - if not target_node.lvstore: - raise PreconditionError( - f"Target node {target_node_id} has no lvstore (S3 bdev requires lvstore)") - - if backup.encrypted: - with create_kms_connection(cluster) as kms: - try: - crypto_key = kms.get_data_encryption_keys( - backup_dek_path(pool.cluster_id, backup.uuid), - backup_kek_name(backup.uuid), - ) - except KMSException as e: - raise RuntimeError("Failed to retrieve backup crypto keys") from e - else: - crypto_key = None - - logger.info(f"Backup allowed hosts: {backup.allowed_hosts}") - lvol_id, error = lvol_controller.add_lvol_ha( - name=lvol_name, - size=size, - pool_id_or_name=pool_id_or_name, - use_crypto=backup.encrypted, - max_size=0, - max_rw_iops=0, - max_rw_mbytes=0, - max_r_mbytes=0, - max_w_mbytes=0, - host_id_or_name=target_node_id, - ha_type="default", - crypto_key=crypto_key, - use_comp=False, - distr_vuid=0, - lvol_priority_class=0, - allowed_hosts=[h["nqn"] if isinstance(h, dict) else h - for h in (backup.allowed_hosts or [])] or None, - fabric="tcp", - ) - if error or not lvol_id: - raise RuntimeError(f"Failed to create restore volume: {error}") - - # Mark volume as restoring - try: - lvol = db_controller.get_lvol_by_id(lvol_id) - except KeyError as e: - raise RuntimeError(f"Volume created but not found in DB: {lvol_id}") from e - - lvol.status = LVol.STATUS_RESTORING - lvol.write_to_db() - - # The bdev name the data plane expects (e.g. LVS_7744/LVOL_12345) - bdev_name = f"{lvol.lvs_name}/{lvol.lvol_bdev}" - - # Data plane processes s3_ids in array order: the first entry's clusters - # take priority (skip-if-populated). Newest-first means the latest - # incremental data wins, with older backups filling any remaining gaps. - if not tasks_controller.add_backup_restore_task( - pool.cluster_id, lvol.node_id, backup_id, bdev_name, - [b.s3_id for b in reversed(chain)], lvol_id=lvol_id): - raise RuntimeError("Failed to create restore task") - - return lvol_id - - -def _cleanup_backup_kms_keys(backups): - encrypted = [b for b in backups if b.encrypted] - if not encrypted: - return - try: - cluster = db_controller.get_cluster_by_id(encrypted[0].cluster_id) - with create_kms_connection(cluster) as kms: - for b in encrypted: - try: - kms.delete_data_encryption_keys(backup_dek_path(b.cluster_id, b.uuid)) - kms.delete_key_encryption_key(backup_kek_name(b.uuid)) - except KMSException: - logger.exception(f"Failed to delete keys for backup {b.uuid}") - except (KMSException, KeyError): - logger.exception("Failed to clean up backup KMS keys") - - -def delete_backups(lvol_id): - """Delete all backups for a given lvol. - Returns (success, error_message).""" - backups = db_controller.get_backups_by_lvol_id(lvol_id) - if not backups: - return False, f"No backups found for lvol {lvol_id}" - - _cleanup_backup_kms_keys(backups) - - # Find node to run delete RPC on - completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] - if not completed: - # Just remove from DB - for b in backups: - b.remove(db_controller.kv_store) - return True, None - - node_id = completed[0].node_id - try: - snode = db_controller.get_storage_node_by_id(node_id) - except KeyError: - # Node gone, just clean up DB - for b in backups: - b.remove(db_controller.kv_store) - return True, None - - # Call S3 delete RPC (dummy for now) - if snode.status == StorageNode.STATUS_ONLINE: - rpc_client = snode.rpc_client() - s3_ids = [b.s3_id for b in completed] - try: - rpc_client.bdev_lvol_s3_delete(s3_ids) - except Exception as e: - logger.error(f"Error deleting S3 backups: {e}") - - cluster_id = completed[0].cluster_id - for b in backups: - backup_events.backup_deleted(cluster_id, node_id, b) - b.remove(db_controller.kv_store) - - return True, None - - -def list_backups(cluster_id=None): - """List all backups, optionally filtered by cluster.""" - backups = db_controller.get_backups(cluster_id) - backups = sorted(backups, key=lambda b: (b.created_at, b.uuid), reverse=True) - data = [] - for b in backups: - logger.debug(b) - source = b.source_cluster_id or b.cluster_id - is_external = source != b.cluster_id - entry = { - "ID": b.uuid, - "S3 ID": b.s3_id, - "LVol": b.lvol_name, - "Snapshot": b.snapshot_name, - "Node": b.node_id[:8] if b.node_id else "", - "Status": b.status, - "Prev": b.prev_backup_id[:8] if b.prev_backup_id else "-", - "Created": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(b.created_at)) if b.created_at else "", - "Source": source[:8] if is_external else "local", - } - data.append(entry) - return data - - -def export_backups(cluster_id=None, lvol_name=None): - """Export completed backup metadata as a list of dicts suitable for import - into another cluster via import_backups(). - - Returns a list of metadata dicts including s3_id, chain links, and size. - """ - backups = db_controller.get_backups(cluster_id) - completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] - if lvol_name: - completed = [b for b in completed if b.lvol_name == lvol_name] - - result = [] - for b in completed: - result.append({ - "backup_id": b.uuid, - "s3_id": b.s3_id, - "cluster_id": b.cluster_id, - "lvol_id": b.lvol_id, - "lvol_name": b.lvol_name, - "snapshot_id": b.snapshot_id, - "snapshot_name": b.snapshot_name, - "node_id": b.node_id, - "prev_backup_id": b.prev_backup_id, - "size": b.size, - "allowed_hosts": b.allowed_hosts, - "created_at": b.created_at, - }) - return result - - -def import_backups(s3_metadata_list, cluster_id=None): - """Import backup metadata from another cluster's S3 metadata. - - Backups are stored in the local cluster's DB namespace but keep their - original s3_ids (scoped to source_cluster_id). The source_cluster_id - field tracks which cluster originally created the backup. - - Args: - s3_metadata_list: list of dicts with backup metadata. - cluster_id: Target cluster to import into. Required for cross-cluster - restore so the backups are visible in the local cluster's DB. - - Raises: - PreconditionError: One of the given backup IDs is already known. Backup - lookups are not scoped by cluster, so a UUID reused across clusters - would make either record unaddressable. All IDs are checked before - the first record is written, so nothing is imported in that case. - """ - pending = {} - for meta in s3_metadata_list: - backup_id = meta.get("backup_id") - if not backup_id: - continue - - if backup_id in pending: - raise PreconditionError(f"Backup {backup_id} is listed more than once") - - try: - existing = db_controller.get_backup_by_id(backup_id) - except KeyError: - pending[backup_id] = meta - else: - raise PreconditionError(f"Backup {backup_id} already exists in cluster {existing.cluster_id}") - - for backup_id, meta in pending.items(): - source_cluster = meta.get("cluster_id", "") - target_cluster = cluster_id or source_cluster - - backup = Backup() - backup.uuid = backup_id - backup.s3_id = meta.get("s3_id", 0) - backup.cluster_id = target_cluster - backup.source_cluster_id = source_cluster - backup.lvol_id = meta.get("lvol_id", "") - backup.lvol_name = meta.get("lvol_name", "") - backup.snapshot_id = meta.get("snapshot_id", "") - backup.snapshot_name = meta.get("snapshot_name", "") - backup.node_id = meta.get("node_id", "") - backup.prev_backup_id = meta.get("prev_backup_id", "") - backup.size = meta.get("size", 0) - backup.allowed_hosts = meta.get("allowed_hosts", []) - backup.created_at = meta.get("created_at", 0) - backup.status = Backup.STATUS_COMPLETED - backup.s3_metadata = meta - backup.write_to_db() - - return len(pending) - - -def get_backup_sources(cluster_id): - """List all distinct backup sources (local + imported clusters). - - Returns a list of dicts with source_cluster_id, count, and whether - it is the currently active source. - """ - try: - cluster = db_controller.get_cluster_by_id(cluster_id) - except KeyError: - return [] - - backups = db_controller.get_backups(cluster_id) - sources = {} - for b in backups: - src = b.source_cluster_id or cluster_id - if src not in sources: - sources[src] = {"source_cluster_id": src, "count": 0, "is_local": src == cluster_id} - sources[src]["count"] += 1 - - active_source = cluster.backup_source or cluster_id - result = [] - for src_id, info in sources.items(): - info["active"] = (src_id == active_source) - result.append(info) - - # Always include local even if no backups - if cluster_id not in sources: - result.append({ - "source_cluster_id": cluster_id, - "count": 0, - "is_local": True, - "active": active_source == cluster_id, - }) - - return result - - -def switch_backup_source(cluster_id, source_cluster_id) -> None: - """Switch the active backup source for all nodes in the cluster. - - Reconfigures the S3 bdev on every node to read from the bucket - belonging to source_cluster_id. While switched to an external - source, new backups cannot be created. - - Args: - cluster_id: The local cluster ID. - source_cluster_id: The cluster ID whose S3 bucket to activate. - Use the local cluster_id (or "local") to switch back. - - Returns (success, error_message). - """ - try: - cluster = db_controller.get_cluster_by_id(cluster_id) - except KeyError as e: - raise PreconditionError("Precondition not met") from e - - if source_cluster_id == "local": - source_cluster_id = cluster_id - - # Determine the bucket name for the source cluster - backup_config = cluster.backup_config or {} - if source_cluster_id == cluster_id: - bucket_name = backup_config.get("bucket_name", - f"simplyblock-backup-{cluster_id}") - else: - bucket_name = f"simplyblock-backup-{source_cluster_id}" - - # Verify the bucket exists - try: - if not _s3_bucket_exists(backup_config, bucket_name): - raise PreconditionError(f"S3 bucket {bucket_name} does not exist") - except BotoCoreError as e: - raise RuntimeError(f"S3 bucket {bucket_name} not accessible: {e}") - - # Reconfigure S3 bdev bucket on all online nodes - nodes = db_controller.get_storage_nodes_by_cluster_id(cluster_id) - for node in nodes: - if node.status != StorageNode.STATUS_ONLINE or not node.lvstore: - continue - - rpc_client = node.rpc_client() - s3_bdev_name = f"s3_{node.lvstore}" - rpc_client.bdev_s3_add_bucket_name(s3_bdev_name, bucket_name, allow_existing=True) - logger.info(f"Switched S3 bucket to {bucket_name} on node {node.get_id()}") - - # Persist the active source in the cluster record. Atomic: the long - # per-node RPC loop above means a concurrent cluster.status change could be - # clobbered by a full write here (lost-update class — incident 2026-06-18). - db_controller.atomic_update( - db_controller.get_cluster_by_id(cluster_id), - lambda c, v=source_cluster_id: setattr(c, "backup_source", v)) - - -def is_local_backup_source(cluster_id): - """Check if the cluster is currently using its own local backup source.""" - try: - cluster = db_controller.get_cluster_by_id(cluster_id) - except KeyError: - return True - return not cluster.backup_source or cluster.backup_source == cluster_id - - -# ---- Backup Policy Management ---- - -def add_policy(cluster_id, name, max_versions=0, max_age="", schedule=""): - """Create a new backup policy. - Returns (policy_id, error_message).""" - max_age_seconds = 0 - if max_age: - try: - max_age_seconds = _parse_age_string(max_age) - except ValueError as e: - return None, str(e) - - if schedule: - try: - _parse_schedule(schedule) - except ValueError as e: - return None, str(e) - - if max_versions <= 0 and max_age_seconds <= 0 and not schedule: - return None, "At least one of --versions, --age, or --schedule must be specified" - - # Check name uniqueness - for p in db_controller.get_backup_policies(cluster_id): - if p.policy_name == name: - return None, f"Policy name already exists: {name}" - - policy = BackupPolicy() - policy.uuid = str(uuid.uuid4()) - policy.cluster_id = cluster_id - policy.policy_name = name - policy.max_versions = max_versions - policy.max_age_seconds = max_age_seconds - policy.max_age_display = max_age - policy.backup_schedule = schedule - policy.status = BackupPolicy.STATUS_ACTIVE - policy.write_to_db() - - return policy.uuid, None - - -def remove_policy(policy_id): - """Remove a backup policy and all its attachments. - Returns (success, error_message).""" - try: - policy = db_controller.get_backup_policy_by_id(policy_id) - except KeyError as e: - return False, str(e) - - # Remove attachments - for att in db_controller.get_backup_policy_attachments(policy.cluster_id): - if att.policy_id == policy_id: - att.remove(db_controller.kv_store) - - policy.remove(db_controller.kv_store) - return True, None - - -def attach_policy(policy_id, target_type, target_id): - """Attach a backup policy to a pool or lvol. - Returns (attachment_id, error_message).""" - try: - policy = db_controller.get_backup_policy_by_id(policy_id) - except KeyError as e: - return None, str(e) - - if target_type not in ("pool", "lvol"): - return None, f"Invalid target_type: {target_type}. Use 'pool' or 'lvol'" - - # Validate target exists - try: - if target_type == "pool": - db_controller.get_pool_by_id(target_id) - else: - db_controller.get_lvol_by_id(target_id) - except KeyError as e: - return None, str(e) - - # Check if already attached - for att in db_controller.get_backup_policy_attachments(policy.cluster_id): - if att.policy_id == policy_id and att.target_type == target_type and att.target_id == target_id: - return att.uuid, None # already attached - - att = BackupPolicyAttachment() - att.uuid = str(uuid.uuid4()) - att.cluster_id = policy.cluster_id - att.policy_id = policy_id - att.target_type = target_type - att.target_id = target_id - att.write_to_db() - - return att.uuid, None - - -def detach_policy(policy_id, target_type, target_id): - """Detach a backup policy from a pool or lvol. - Returns (success, error_message).""" - try: - policy = db_controller.get_backup_policy_by_id(policy_id) - except KeyError as e: - return False, str(e) - - for att in db_controller.get_backup_policy_attachments(policy.cluster_id): - if att.policy_id == policy_id and att.target_type == target_type and att.target_id == target_id: - att.remove(db_controller.kv_store) - return True, None - - return False, "Attachment not found" - - -def list_policies(cluster_id=None): - """List all backup policies.""" - policies = db_controller.get_backup_policies(cluster_id) - data = [] - for p in policies: - data.append({ - "ID": p.uuid, - "Name": p.policy_name, - "Versions": p.max_versions if p.max_versions > 0 else "-", - "Max Age": p.max_age_display if p.max_age_display else "-", - "Schedule": p.backup_schedule if p.backup_schedule else "-", - "Status": p.status, - }) - return data - - -def evaluate_policy(lvol): - """Evaluate backup policy for an lvol and trigger merges if needed. - Called by the backup merge service.""" - policy = db_controller.get_policy_for_lvol(lvol) - if not policy: - return - - backups = db_controller.get_backups_by_lvol_id(lvol.get_id()) - completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] - if len(completed) < 2: - return - - completed.sort(key=lambda b: b.created_at) - now = int(time.time()) - - versions_exceeded = policy.max_versions > 0 and len(completed) > policy.max_versions - age_exceeded = False - if policy.max_age_seconds > 0 and completed: - oldest_age = now - completed[0].created_at - age_exceeded = oldest_age > policy.max_age_seconds - - # Either condition triggers a merge - if versions_exceeded or age_exceeded: - oldest = completed[0] - second = completed[1] - _trigger_merge(second, oldest) - - -def evaluate_schedule(lvol): - """Evaluate the backup schedule for an lvol and trigger auto-backups + tiered merges. - Called by the backup merge service.""" - policy = db_controller.get_policy_for_lvol(lvol) - if not policy or not policy.backup_schedule: - return - - try: - tiers = _parse_schedule(policy.backup_schedule) - except ValueError: - return - - if not tiers: - return - - now = int(time.time()) - - # Check if we need to create a new auto-backup based on the smallest tier interval - smallest_interval = tiers[0][0] - backups = db_controller.get_backups_by_lvol_id(lvol.get_id()) - completed = [b for b in backups if b.status == Backup.STATUS_COMPLETED] - pending_or_running = [b for b in backups if b.status in (Backup.STATUS_PENDING, Backup.STATUS_IN_PROGRESS)] - - # Don't create a new backup if one is already in progress - if not pending_or_running: - needs_backup = True - if completed: - completed.sort(key=lambda b: b.created_at, reverse=True) - latest = completed[0] - elapsed = now - latest.created_at - if elapsed < smallest_interval: - needs_backup = False - - if needs_backup: - _auto_backup_lvol(lvol) - return # Skip merge evaluation this cycle — let the backup complete first - - # Tiered merge: enforce keep_count per tier. - # Each tier covers an age range. Backups age from tier 0 (newest) - # into higher tiers. When a tier exceeds its keep_count, the oldest - # backup in that tier is merged into its successor. - # All tiers are evaluated each cycle so limits are maintained in parallel. - if len(completed) < 2: - return - - completed.sort(key=lambda b: b.created_at) - - # Don't merge while another merge is already in progress - merging = [b for b in backups if b.status == Backup.STATUS_MERGING] - if merging: - return - - for tier_idx, (interval, keep_count) in enumerate(tiers): - # Age boundaries for this tier - if tier_idx == 0: - lower_age = 0 - else: - lower_age = tiers[tier_idx - 1][0] - - if tier_idx + 1 < len(tiers): - upper_age = tiers[tier_idx + 1][0] - else: - upper_age = float('inf') - - tier_backups = [b for b in completed - if lower_age <= (now - b.created_at) < upper_age] - - if len(tier_backups) > keep_count: - tier_backups.sort(key=lambda b: b.created_at) - oldest = tier_backups[0] - second = tier_backups[1] - _trigger_merge(second, oldest) - return # One merge per cycle to avoid conflicts - - -def _auto_backup_lvol(lvol): - """Create an automatic snapshot + backup for scheduled backups. - - Unlike manual backup_snapshot() which walks the full ancestor chain, - auto-backups create a single snapshot and a single backup for it. - The prev_backup_id is set to the latest existing backup so the - incremental chain is maintained without re-backing all ancestors. - """ - from simplyblock_core.controllers import snapshot_controller - snap_name = f"auto_{lvol.lvol_name}_{int(time.time())}" - snap_id, error = snapshot_controller.add(lvol.get_id(), snap_name) - if error: - logger.warning(f"Auto-backup snapshot failed for lvol {lvol.get_id()}: {error}") - return - - try: - snapshot = db_controller.get_snapshot_by_id(snap_id) - except KeyError: - logger.warning(f"Auto-backup: snapshot {snap_id} not found after creation") - return - - node_id = lvol.node_id - try: - snode = db_controller.get_storage_node_by_id(node_id) - except KeyError: - logger.warning(f"Auto-backup: node {node_id} not found") - return - - cluster_id = snode.cluster_id - prev_backup = _get_latest_backup_for_lvol(lvol.get_id()) - _create_single_backup(snapshot, lvol, node_id, cluster_id, prev_backup) - - -def _trigger_merge(keep_backup, old_backup): - """Trigger a merge of old_backup into keep_backup.""" - if old_backup.status != Backup.STATUS_COMPLETED: - return - if keep_backup.status != Backup.STATUS_COMPLETED: - return - - old_backup.status = Backup.STATUS_MERGING - old_backup.write_to_db() - - tasks_controller.add_backup_merge_task( - keep_backup.cluster_id, - keep_backup.node_id, - keep_backup.uuid, - old_backup.uuid) diff --git a/simplyblock_core/controllers/snapshot_controller.py b/simplyblock_core/controllers/snapshot_controller.py index acb23babe..e9b86bef6 100644 --- a/simplyblock_core/controllers/snapshot_controller.py +++ b/simplyblock_core/controllers/snapshot_controller.py @@ -800,7 +800,7 @@ def add(lvol_id, snapshot_name, backup=False, lock=True, all_snaps=None, all_lvo pass if backup: - from simplyblock_core.controllers import backup_controller + from simplyblock_core.controllers.backup import controller as backup_controller backup_id, backup_err = backup_controller.backup_snapshot(snap.uuid) if backup_err: logger.warning(f"Snapshot created but backup failed: {backup_err}") diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e92d3bdb5..5a7e233a1 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -1102,8 +1102,18 @@ def add_backup_task(backup): ) -def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids, lvol_id=""): - """Create the task that restores an S3 backup chain into a new lvol.""" +def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids, + lvol_id="", s3_bdev="", s3_config=None): + """Create the task that restores an S3 backup chain into a new lvol. + + Args: + s3_bdev: which S3 device on the node to read from. + s3_config: when set, the device named above does not exist yet and the + runner creates it from this configuration -- the case where the + backup lives in a bucket that is not the cluster's own. The runner + owns that device and deletes it, and scrubs this field, when the + restore reaches a terminal state. + """ return _add_task( JobSchedule.FN_BACKUP_RESTORE, cluster_id, @@ -1115,6 +1125,8 @@ def add_backup_restore_task(cluster_id, node_id, backup_id, lvol_name, chain_ids "lvol_name": lvol_name, "lvol_id": lvol_id, "chain_ids": chain_ids, + "s3_bdev": s3_bdev, + "s3_config": s3_config, }, ) diff --git a/simplyblock_core/db_controller.py b/simplyblock_core/db_controller.py index ea475de42..bddce38c1 100644 --- a/simplyblock_core/db_controller.py +++ b/simplyblock_core/db_controller.py @@ -1045,6 +1045,61 @@ def next_vuid(self) -> int: fdb.transactional(DBController._seed_vuid_tx)(self, self.kv_store, seed) return fdb.transactional(DBController._incr_vuid_tx)(self, self.kv_store) + # ---- s3_id allocation (monotonic sequence) ---- + # + # An s3_id names a backup's object keys in S3 ({s3_id}/{mid}/{extent}). The + # old allocator was max-plus-one over the local cluster's Backup records, + # which had three problems: it raced (two concurrent backups got the same + # id), it recycled the id of a deleted backup whose objects may still exist + # (nothing reclaims them -- bdev_lvol_s3_delete does not exist on the data + # plane), and after an import it counted foreign backups it should not have. + # A monotonic sequence makes reuse impossible by construction. + # + # Unlike vuid this space is NOT unbounded: the data plane packs s3_id into + # 30 bits and masks rather than validates, so callers must check + # BACKUP_MAX_S3_ID. 2^30 is ~1.07e9 backups per control plane. + _S3_ID_SEQ_KEY = b"sequence/s3_id" + + def _incr_s3_id_tx(self, tr): + raw = tr.get(DBController._S3_ID_SEQ_KEY).wait() + if not raw.present(): + return None + nxt = int(json.loads(raw)) + 1 + tr[DBController._S3_ID_SEQ_KEY] = json.dumps(nxt).encode() + return nxt + + def _seed_s3_id_tx(self, tr, seed): + # Only-if-absent CAS, as for vuid: the first allocator across all API + # workers seeds it; concurrent racers see it present and skip. + raw = tr.get(DBController._S3_ID_SEQ_KEY).wait() + if raw.present(): + return + tr[DBController._S3_ID_SEQ_KEY] = json.dumps(int(seed)).encode() + + def _max_existing_s3_id(self) -> int: + """Highest s3_id in use across every backup this control plane knows of. + + Read once to seed the counter on an upgraded cluster so the sequence + never reuses an id the old max-plus-one allocator handed out; never read + again. Deliberately unscoped by cluster -- imported backups keep their + originating cluster's ids, and seeding above those too costs nothing. + """ + return max((b.s3_id or 0 for b in self.get_backups()), default=0) + + def next_s3_id(self) -> int: + """Allocate the next globally-unique s3_id (monotonic, O(1)).""" + val = fdb.transactional(DBController._incr_s3_id_tx)(self, self.kv_store) + if val is None: + seed = self._max_existing_s3_id() + fdb.transactional(DBController._seed_s3_id_tx)(self, self.kv_store, seed) + val = fdb.transactional(DBController._incr_s3_id_tx)(self, self.kv_store) + + if val > constants.BACKUP_MAX_S3_ID: + raise ValueError( + f"s3_id space exhausted: {val} exceeds the data plane's " + f"{constants.BACKUP_MAX_S3_ID} limit") + return val + # ---- snapshot indexes (replace per-create cluster-wide scans) ---- # # Snapshot create used to read EVERY snapshot in the cluster on each request diff --git a/simplyblock_core/models/backup.py b/simplyblock_core/models/backup.py index fccee7da8..532bfeccd 100644 --- a/simplyblock_core/models/backup.py +++ b/simplyblock_core/models/backup.py @@ -2,6 +2,7 @@ import datetime from typing import List +from simplyblock_core.models.backup_config import BackupLocation from simplyblock_core.models.base_model import BaseModel @@ -35,19 +36,40 @@ class Backup(BaseModel): prev_backup_id: str = "" pool_uuid: str = "" size: int = 0 - source_cluster_id: str = "" # original cluster that created this backup created_at: int = 0 completed_at: int = 0 error_message: str = "" # Security params from the source lvol (for cross-cluster restore) allowed_hosts: List[dict] = [] - # S3 metadata written to metadata bucket - s3_metadata: dict = {} + #: Where this backup's objects live and how to interpret them, as a + #: ``BackupLocation``. Stored as a dict because ``BaseModel`` cannot nest + #: pydantic models; read it through :meth:`get_location`. + location: dict = {} encrypted: bool = False + #: Which KMS holds this backup's key, and under what path. A + #: ``backup_manifest.Encryption``; stored as a dict for the same reason + #: ``location`` is. Empty for an unencrypted backup. + encryption: dict = {} def get_id(self): return "%s/%s" % (self.cluster_id, self.uuid) + def get_location(self) -> BackupLocation: + """Validate and return where this backup's objects live. + + Raises: + ValueError: The backup predates self-describing locations, or its + recorded location is not valid. Either way it cannot be read + without knowing what wrote it. ``ValidationError`` is a + ``ValueError``, so one except clause covers both. + """ + if not self.location: + raise ValueError( + f"Backup {self.uuid} has no recorded location " + "(created before backups became self-describing)") + + return BackupLocation.model_validate(self.location) + def write_to_db(self, kv_store=None): self.updated_at = str(datetime.datetime.now(datetime.timezone.utc)) super().write_to_db(kv_store) diff --git a/simplyblock_core/models/backup_config.py b/simplyblock_core/models/backup_config.py new file mode 100644 index 000000000..b492f34bf --- /dev/null +++ b/simplyblock_core/models/backup_config.py @@ -0,0 +1,161 @@ +# coding=utf-8 +"""Typed configuration describing where backup objects live and how to read them. + +Two models, deliberately split so that "backups never carry credentials" is +enforced by the type system rather than by discipline: + +``BackupLocation`` + Everything needed to *find and interpret* a backup's objects. Safe to embed + in a backup record and in the S3 manifest. Cannot represent a secret. + +``BackupConfig`` + What a cluster is configured with: a location plus the credentials and + node-local tuning needed to act on it. Never leaves the control plane. + +Absence is expressed as ``None`` rather than as ``""`` or ``0``, so no caller has +to guess whether a field was configured or merely left at its default. Where the +AWS SDK has its own resolution chain -- credentials, region, endpoint -- absent +means "let it resolve", which is a real configuration rather than a gap. + +Both models serialize straight into the untyped ``dict`` fields the FoundationDB +records still use: a plain ``model_dump()`` is JSON-safe, while ``SecretStr`` +stays wrapped so the plaintext is produced only by ``BaseModel.write_to_db``'s +own ``unwrap_secrets`` pass, at the last possible moment. +""" +from enum import IntEnum +from typing import Any, Optional + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + HttpUrl, + SecretStr, + field_serializer, + model_validator, +) + + +class SecondaryTarget(IntEnum): + """The kind of secondary store, numbered as the data plane's RPC expects.""" + + S3 = 0 + FILESYSTEM = 1 + + +class S3Credentials(BaseModel): + """A static key pair. + + A pair rather than two independent fields, so "access key set, secret + missing" is unrepresentable instead of something a validator has to catch. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + access_key_id: SecretStr + secret_access_key: SecretStr + + +class BackupLocation(BaseModel): + """Where a backup's objects are, and how to interpret them. Never secret. + + Every field here affects whether the objects can be read back at all, which + is why the whole model is embedded in each backup rather than looked up from + the cluster that happened to create it. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + bucket_name: str + + #: Absent means the AWS SDK resolves it the way it resolves credentials: + #: from the environment, the profile, or instance metadata. Recording it is + #: better -- a manifest that names its region can be read from anywhere, + #: while one that does not depends on the reader's environment agreeing -- + #: but it is not required, because it is recoverable: bucket names are + #: globally unique, so S3 can be asked where a bucket lives, and every layer + #: below already treats an absent region this way (boto3's own resolution, + #: and `if (region && *region)` in the data plane's init_client). + region: Optional[str] = None + + #: Absent means the AWS SDK resolves the endpoint from the region. + endpoint: Optional[HttpUrl] = None + + secondary_target: SecondaryTarget = SecondaryTarget.S3 + with_compression: bool = False + + #: Selects the object key layout. ``True`` gives the backup layout + #: ``{s3_id}/{mid}/{extent}``; ``False`` gives the secondary-tiering layout + #: ``{tiering_id}/{lpgi}``, which cannot hold backups. + snapshot_backups: bool = True + + verify_tls: bool = True + use_path_style: bool = False + + @property + def endpoint_url(self) -> Optional[str]: + """The endpoint as the AWS SDK and boto3 want it, without a trailing slash.""" + return str(self.endpoint).rstrip("/") if self.endpoint is not None else None + + @field_serializer("endpoint", when_used="unless-none") + def _serialize_endpoint(self, endpoint: HttpUrl) -> str: + return self.endpoint_url # type: ignore[return-value] + + @field_serializer("secondary_target") + def _serialize_secondary_target(self, target: SecondaryTarget) -> int: + return int(target) + + @model_validator(mode="before") + @classmethod + def _migrate_legacy_keys(cls, data: Any) -> Any: + """Accept the untyped ``Cluster.backup_config`` dicts written before this model. + + Existing clusters and ``tests/perf/backup_config.json`` carry the shape + the data plane's RPC used to take directly. Mapping it here means no FDB + migration is needed. + """ + if not isinstance(data, dict): + return data + + data = dict(data) + + if (endpoint := data.pop("local_endpoint", None)) and "endpoint" not in data: + data["endpoint"] = endpoint + + access_key_id = data.pop("access_key_id", None) + secret_access_key = data.pop("secret_access_key", None) + if access_key_id and secret_access_key and "credentials" not in data: + data["credentials"] = { + "access_key_id": access_key_id, + "secret_access_key": secret_access_key, + } + + # `local_testing` bundled three separate decisions into one flag. It set + # plain HTTP, disabled certificate verification, forced path-style + # addressing and hardcoded us-east-1 (bdev_s3_impl.hpp init_client). + # Unpack it into the properties it actually stood for. + if data.pop("local_testing", False): + data.setdefault("verify_tls", False) + data.setdefault("use_path_style", True) + data.setdefault("region", "us-east-1") + + # 0 meant "let the data plane pick"; that is now an absent value. + if data.get("s3_thread_pool_size") == 0: + del data["s3_thread_pool_size"] + + return data + + +class BackupConfig(BackupLocation): + """A cluster's backup configuration: a location plus how to authenticate to it.""" + + #: Absent means the node's own IAM role / the AWS default provider chain. + credentials: Optional[S3Credentials] = None + + #: Absent means the data plane's own default (32 at the time of writing). + s3_thread_pool_size: Optional[int] = Field(default=None, ge=1) + + def location(self) -> BackupLocation: + return BackupLocation.model_validate( + self.model_dump(include=set(BackupLocation.model_fields)) + ) diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 82401eba9..7b2e9d626 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -1,10 +1,11 @@ # coding=utf-8 import os.path -from typing import List, Optional +from typing import Any, List, Mapping, Optional from pydantic import SecretStr from simplyblock_core import constants +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.base_model import BaseModel @@ -190,7 +191,6 @@ def is_topology_owned(self) -> bool: client_data_nic: str = "" max_fault_tolerance: int = 1 backup_config: dict = {} - backup_source: str = "" # active backup source cluster_id ("" = local) backup_timeout_seconds: int = 14400 # 4 hours default nvmf_base_port: int = 4420 rpc_base_port: int = 8080 @@ -224,6 +224,62 @@ def is_qos_set(self) -> bool: return True return False + def default_backup_bucket_name(self) -> str: + """The bucket this cluster backs up to when its configuration names none. + + A config that names no bucket gets the one the bucket name used to be + derived from at device-creation time, so a cluster configured before + ``bucket_name`` existed keeps addressing the bucket it has been writing + to. Callers that configure a bucket per cluster (``StorageCluster``'s + ``spec.backup`` has no bucket field at all) never name one, so this is + the normal case rather than a fallback. + """ + return f"simplyblock-backup-{self.uuid}" + + def _resolve_backup_config(self, config: Mapping[str, Any]) -> BackupConfig: + return BackupConfig.model_validate({ + "bucket_name": self.default_backup_bucket_name(), + **config, + }) + + def get_backup_config(self) -> BackupConfig: + """Validate and return this cluster's volume-backup configuration. + + ``backup_config`` stays an untyped dict on the record because + ``BaseModel`` cannot nest pydantic models; validating on read gives the + typing without an FDB migration. Stored configs are never rewritten by + reading them, so :meth:`default_backup_bucket_name` stays derived rather + than frozen into the record. + + Raises: + ValueError: The cluster has no backup configuration, or the stored + one is not valid. ``ValidationError`` is a ``ValueError``, so one + except clause covers both. + """ + if not self.backup_config: + raise ValueError(f"Cluster {self.get_id()} has no backup configuration") + + return self._resolve_backup_config(self.backup_config) + + def set_backup_config(self, config: Mapping[str, Any]) -> None: + """Validate a raw backup configuration and store it on this record. + + The only way to put a configuration on a cluster. Validating on write as + well as on read is what stops a configuration no cluster could act on + from being accepted at cluster-create and then failing activation on + every node, arbitrarily long after the mistake was made and with nothing + in the failure pointing back at it. + + What gets stored is what the caller passed, so an absent ``bucket_name`` + stays absent and keeps resolving through + :meth:`default_backup_bucket_name`. + + Raises: + ValueError: The configuration is not one this cluster could act on. + """ + self._resolve_backup_config(config) + self.backup_config = dict(config) + def get_backup_path(self, path=""): if self.backup_s3_bucket and self.backup_s3_cred: backup_path = f"blobstore://{self.backup_s3_cred}@s3.{self.backup_s3_region}.amazonaws.com/{path}?bucket={self.backup_s3_bucket}" \ diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index 1bba9177e..a3b23c48b 100644 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -2,7 +2,7 @@ import json from enum import IntEnum from json import JSONDecodeError -from typing import Any, Optional +from typing import Any, List, Optional import jsonschema import requests @@ -1752,7 +1752,7 @@ def bdev_lvol_transfer(self, name, offset, batch_size, bdev_name, operation="mig "operation": operation, }) - def bdev_lvol_transfer_stat(self, name): + def bdev_lvol_transfer_stat(self, name: str): """ Return transfer status for *name* (source composite bdev). @@ -1852,45 +1852,77 @@ def bdev_lvol_batch_transfer_final_step(self, lvol_names, lvol_ids, snapshot_nam # ---- S3 Backup RPCs ---- - def bdev_s3_create(self, name, secondary_target=0, with_compression=False, - snapshot_backups=True, local_testing=False, local_endpoint="", - access_key_id="", secret_access_key="", - bdb_lcpu_mask=0, s3_lcpu_mask=0, s3_thread_pool_size=0): - """Create the S3 bdev device. - Must be called before bdev_lvol_s3_bdev to attach it to an lvstore. + def bdev_s3_create(self, name: str, bucket_name: str, + secondary_target: int, with_compression: bool, + snapshot_backups: bool, verify_tls: bool = True, + use_path_style: bool = False, + region: Optional[str] = None, + endpoint: Optional[str] = None, + access_key_id: Optional[SecretStr] = None, + secret_access_key: Optional[SecretStr] = None, + bdb_lcpu_mask: Optional[int] = None, + s3_lcpu_mask: Optional[int] = None, + s3_thread_pool_size: Optional[int] = None): + """Create an S3 bdev for one bucket. + + A device serves exactly one bucket with one set of credentials, so + reading a second bucket means creating a second device. Attach it to an + lvstore with bdev_lvol_s3_bdev. + + What the device cannot function without has no default. What the data + plane has its own default for is Optional and omitted from the payload + when absent. Nothing here uses ``0`` or ``""`` to mean "unset": the data + plane already reads zero that way for the masks and the pool size + (bdev_s3_impl.cpp:1204, 1222, 1239), so an explicit 0 would behave as + absent while reading as a deliberate choice. + Args: - name: Bdev name - secondary_target: 0=S3, 1=FileSystem - with_compression: Enable ISA-L compression - snapshot_backups: Snapshot backup mode - local_testing: Use local endpoint (e.g. MinIO) - local_endpoint: Local endpoint URL - access_key_id: AWS access key (optional if using IAM roles) - secret_access_key: AWS secret key (optional if using IAM roles) - bdb_lcpu_mask: CPU mask for the SPDK thread of this bdev (uint64) - s3_lcpu_mask: CPU mask for the internal AWS S3 thread pool (uint64) - s3_thread_pool_size: AWS S3 thread pool size (default 32 on data plane) + bucket_name: the bucket this device reads and writes. A device + without one cannot service any I/O. + region: the bucket's region. Absent means the SDK resolves it, the + same way it resolves credentials -- the data plane leaves its + config untouched when this is not given. + secondary_target: 0=S3, 1=FileSystem. The caller holds a + SecondaryTarget enum and converts at this boundary. + with_compression: enable ISA-L compression. + snapshot_backups: selects the backup object layout + ({s3_id}/{mid}/{extent}) rather than the tiering one. + verify_tls: verify the endpoint's certificate. + use_path_style: path-style addressing, needed by MinIO and most + S3-compatible stores. + endpoint: an S3-compatible endpoint, e.g. "http://minio:9000". + Absent means the SDK resolves AWS's endpoint from the region. + access_key_id / secret_access_key: absent means the node's instance + role, via the SDK's default credential provider chain. Callers + derive them from an S3Credentials pair, so they arrive together + or not at all. + bdb_lcpu_mask: CPU mask for this bdev's SPDK thread. Absent lets the + data plane derive one from the app core mask. + s3_lcpu_mask: CPU mask for the AWS SDK thread pool. Absent leaves the + data plane's own choice, which pins onto SPDK reactor cores -- + so callers that care should compute one. + s3_thread_pool_size: AWS SDK thread pool size. Absent means the data + plane's default of 32. """ - params = { + params: dict[str, Any] = { "name": name, + "bucket_name": bucket_name, "secondary_target": secondary_target, "with_compression": with_compression, "snapshot_backups": snapshot_backups, - } - if local_testing: - params["local_testing"] = True - if local_endpoint: - params["local_endpoint"] = local_endpoint - if access_key_id: - params["access_key_id"] = access_key_id - if secret_access_key: - params["secret_access_key"] = secret_access_key - if bdb_lcpu_mask: - params["bdb_lcpu_mask"] = bdb_lcpu_mask - if s3_lcpu_mask: - params["s3_lcpu_mask"] = s3_lcpu_mask - if s3_thread_pool_size: - params["s3_thread_pool_size"] = s3_thread_pool_size + "verify_tls": verify_tls, + "use_path_style": use_path_style, + } + optional: dict[str, Any] = { + "region": region, + "endpoint": endpoint, + "access_key_id": access_key_id, + "secret_access_key": secret_access_key, + "bdb_lcpu_mask": bdb_lcpu_mask, + "s3_lcpu_mask": s3_lcpu_mask, + "s3_thread_pool_size": s3_thread_pool_size, + } + params.update({k: v for k, v in optional.items() if v is not None}) return self._request3("bdev_s3_create", **params) def bdev_lvol_create_poller_group(self, cpu_mask): @@ -1901,7 +1933,7 @@ def bdev_lvol_create_poller_group(self, cpu_mask): """ return self._request3("bdev_lvol_create_poller_group", cpu_mask=cpu_mask) - def bdev_lvol_s3_bdev(self, lvs_name, bdev_name): + def bdev_lvol_s3_bdev(self, lvs_name: str, bdev_name: str): """Attach an S3 bdev to the given lvstore. The S3 bdev must already exist (created via bdev_s3_create). Called once per lvstore at setup time (cluster activate, node restart).""" @@ -1910,37 +1942,23 @@ def bdev_lvol_s3_bdev(self, lvs_name, bdev_name): "s3_bdev": bdev_name, }) - def bdev_s3_add_bucket_name(self, name, bucket_name, allow_existing: bool = False): - """Register a bucket name with the S3 bdev. - Must be called after bdev_s3_create and before any backup/recovery operations. - Args: - name: S3 bdev name (e.g. 's3_LVS_1234') - bucket_name: S3/MinIO bucket name to use for data storage - Returns (result, error) tuple. - """ - try: - return self._request3( - "bdev_s3_add_bucket_name", - name=name, - bucket_name=bucket_name, - ) - except RPCRemoteError as e: - if allow_existing and e.code == -17: - logger.debug("Bucket %s already registered with %s", name, bucket_name) - return None - raise - - def bdev_lvol_s3_backup(self, s3_id, snapshot_names, cluster_batch=1): + def bdev_lvol_s3_backup(self, s3_id: int, snapshot_names: List[str], + s3_bdev: str, cluster_batch: int = 1): """Start an async backup of snapshots to S3. Args: - s3_id: unique backup identifier (uint32) - snapshot_names: list of snapshot composite bdev names + s3_id: unique backup identifier (uint32, < 2**30) + snapshot_names: snapshot composite bdev names, NEWEST first -- the + data plane unions their cluster maps first-writer-wins, so the + newest snapshot's allocation must be seen first. + s3_bdev: which S3 device to write through. Required: an lvstore may + carry several, and "the first" is ambiguous. cluster_batch: batch size in clusters (default 1) Returns RPC result (truthy on success). Poll with bdev_lvol_transfer_stat. """ params = { "s3_id": s3_id, "snapshot_names": snapshot_names, + "s3_bdev": s3_bdev, "cluster_batch": cluster_batch, } return self._request("bdev_lvol_s3_backup", params) @@ -1950,32 +1968,52 @@ def bdev_lvol_s3_backup(self, s3_id, snapshot_names, cluster_batch=1): # (pass snapshot bdev name) and recovery (pass target lvol name). # Merge has lvol=NULL on data plane so transfer_stat cannot poll it. - def bdev_lvol_s3_merge(self, s3_id, old_s3_id, cluster_batch, lvs_name=None): + def bdev_lvol_s3_merge(self, s3_id: int, old_s3_id: int, cluster_batch: int, + s3_bdev: str, lvs_name: Optional[str] = None): """Merge two backups: keep s3_id and merge old_s3_id into it. - This shortens the backup chain.""" - params = { + + This shortens the backup chain. Both backups must live in the bucket + served by s3_bdev -- a merge reads one and writes the other. + """ + params: dict[str, Any] = { "s3_id": s3_id, "old_s3_id": old_s3_id, "cluster_batch": cluster_batch, + "s3_bdev": s3_bdev, } if lvs_name: params["lvs_name"] = lvs_name return self._request("bdev_lvol_s3_merge", params) - def bdev_lvol_s3_recovery(self, lvol_name, s3_ids, cluster_batch): + def bdev_lvol_s3_recovery(self, lvol_name: str, s3_ids: List[int], + cluster_batch: int, s3_bdev: str): """Restore a chain of S3 backups into a new lvol. Args: lvol_name: target lvol name to restore into - s3_ids: list of S3 backup IDs (uint32) forming the chain (oldest first) + s3_ids: list of S3 backup IDs (uint32) forming the chain, NEWEST + first: the data plane claims each cluster for the first id that + offers it (prepare_s3_clusters is first-writer-wins), so the + newest backup's data must win. cluster_batch: batch size in clusters + s3_bdev: which S3 device to read from. Required: a restore from a + foreign bucket attaches a second device, so "the first" is + ambiguous exactly when it matters. """ return self._request("bdev_lvol_s3_recovery", { "lvol_name": lvol_name, "cluster_batch": cluster_batch, "s3_ids": s3_ids, + "s3_bdev": s3_bdev, }) - def bdev_lvol_s3_delete(self, s3_ids): + def bdev_s3_delete(self, name: str): + """Delete an S3 bdev. + + Used to release the device a restore attached for a foreign bucket. + """ + return self._request3("bdev_s3_delete", name=name) + + def bdev_lvol_s3_delete(self, s3_ids: List[int]): """Delete all S3 backups for the given IDs (list of uint32).""" # RPC still missing on data plane — use dummy return self._request("bdev_lvol_s3_delete", { diff --git a/simplyblock_core/services/tasks_runner_backup.py b/simplyblock_core/services/tasks_runner_backup.py index 8f0ed8af0..718f3a1f0 100644 --- a/simplyblock_core/services/tasks_runner_backup.py +++ b/simplyblock_core/services/tasks_runner_backup.py @@ -11,7 +11,12 @@ from simplyblock_core import constants, db_controller, utils from simplyblock_core.controllers import backup_events +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import device as backup_device +from simplyblock_core.controllers.backup.manifest import ManifestError +from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.storage_node import StorageNode @@ -81,7 +86,9 @@ def _run_backup(task): if backup.status == Backup.STATUS_PENDING: try: - ret = rpc_client.bdev_lvol_s3_backup(backup.s3_id, [snap_bdev_name], cluster_batch=16) + ret = rpc_client.bdev_lvol_s3_backup( + backup.s3_id, [snap_bdev_name], + backup_device.primary_s3_bdev_name(snode), cluster_batch=16) if not ret: _fail_backup(backup, task, "bdev_lvol_s3_backup RPC failed") return @@ -109,8 +116,19 @@ def _run_backup(task): if stat and isinstance(stat, dict): state = stat.get("transfer_state", "") if state == "Done": - backup.status = Backup.STATUS_COMPLETED backup.completed_at = int(time.time()) + + # Publish the manifest BEFORE marking the backup completed, so that + # COMPLETED implies "identifiable from the bucket alone". Data with + # no manifest is data nobody can attribute to a volume later, so a + # manifest failure fails the backup rather than leaving that behind. + try: + backup_controller.write_manifest(backup) + except (ManifestError, PreconditionError) as e: + _fail_backup(backup, task, f"Failed to publish backup manifest: {e}") + return + + backup.status = Backup.STATUS_COMPLETED backup.write_to_db() backup_events.backup_completed(backup.cluster_id, backup.node_id, backup) task.function_result = "Backup completed" @@ -179,6 +197,51 @@ def _set_lvol_restore_failed(task, reason): logger.warning(f"Restored lvol {lvol_id} not found in DB") +def _restore_s3_bdev(task, snode) -> str: + """The S3 device this restore reads from. + + A foreign bucket gets its own device, named from the backup id so a retry + re-derives the same name instead of leaking one per attempt. Otherwise the + node's own backup device already points at the right bucket. + """ + if task.function_params.get("s3_config"): + return backup_device.restore_s3_bdev_name( + task.function_params["backup_id"]) + return backup_device.primary_s3_bdev_name(snode) + + +def _ensure_restore_s3_bdev(task, snode) -> None: + """Create the foreign-bucket device if this restore needs one. + + Idempotent, and re-run on every attempt rather than once: a node restart + mid-restore takes the device with it, and the runner is the only component + positioned to put it back. + """ + config = task.function_params.get("s3_config") + if not config: + return + + backup_device.create_restore_s3_bdev( + snode, BackupConfig.model_validate(config), _restore_s3_bdev(task, snode)) + + +def _release_restore_s3_bdev(task, snode) -> None: + """Delete the foreign-bucket device and forget its credentials. + + Called on every terminal path. The credentials are scrubbed from the task + record because a task is retained for weeks after it finishes, and there is + no reason for another cluster's S3 keys to outlive the restore that needed + them. + """ + if not task.function_params.get("s3_config"): + return + + if snode is not None: + backup_device.delete_restore_s3_bdev(snode, _restore_s3_bdev(task, snode)) + + task.function_params["s3_config"] = None + + def _run_restore(task): backup_id = task.function_params.get("backup_id") lvol_name = task.function_params.get("lvol_name") @@ -209,19 +272,35 @@ def _run_restore(task): from simplyblock_core.models.lvol_model import LVol lvol = db.get_lvol_by_id(lvol_id) if lvol.status == LVol.STATUS_IN_DELETION: + _release_restore_s3_bdev(task, snode) task.function_result = f"Restore target {lvol_id} has been deleted" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) return except KeyError: + _release_restore_s3_bdev(task, snode) task.function_result = f"Restore target {lvol_id} no longer exists" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) return if not recovery_started: + # The device is established here, not when the restore was requested: + # only now is the node known (add_lvol_ha chooses it), and only the + # runner can put it back after a node restart wipes it mid-restore. + try: + _ensure_restore_s3_bdev(task, snode) + except (RuntimeError, ValueError) as e: + task.function_result = f"Could not attach the backup's bucket: {e}" + task.retry += 1 + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return + try: - ret = rpc_client.bdev_lvol_s3_recovery(lvol_name, chain_ids, cluster_batch=16) + ret = rpc_client.bdev_lvol_s3_recovery( + lvol_name, chain_ids, cluster_batch=16, + s3_bdev=_restore_s3_bdev(task, snode)) if not ret: task.function_result = "bdev_lvol_s3_recovery RPC failed" task.retry += 1 @@ -261,6 +340,7 @@ def _run_restore(task): task.cluster_id, node_id, backup, lvol_name) except KeyError: pass + _release_restore_s3_bdev(task, snode) task.function_result = f"Restore completed: {lvol_name}" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) @@ -279,6 +359,7 @@ def _run_restore(task): logger.warning( "Backup %s not found in DB; restore-failed event skipped for lvol %s", backup_id, lvol_name) + _release_restore_s3_bdev(task, snode) task.status = JobSchedule.STATUS_DONE else: task.retry += 1 @@ -331,7 +412,10 @@ def _run_merge(task): if not merge_started: try: - ret = rpc_client.bdev_lvol_s3_merge(keep_backup.s3_id, old_backup.s3_id, cluster_batch=16, lvs_name=snode.lvstore) + ret = rpc_client.bdev_lvol_s3_merge( + keep_backup.s3_id, old_backup.s3_id, cluster_batch=16, + s3_bdev=backup_device.primary_s3_bdev_name(snode), + lvs_name=snode.lvstore) if not ret: task.function_result = "bdev_lvol_s3_merge RPC failed" task.retry += 1 @@ -360,6 +444,24 @@ def _run_merge(task): old_backup.status = Backup.STATUS_MERGED old_backup.write_to_db() + # Two objects, and only two: the survivor's manifest, whose prev_backup_id + # just changed, and the merged-away one, which describes keys the data plane + # has unmapped. Every descendant's manifest stays valid because none of them + # names anything but its own immediate predecessor -- the chain is walked at + # read time rather than stored, precisely so a merge does not have to rewrite + # the whole line of descent and cannot half-succeed at it. + try: + backup_controller.write_manifest(keep_backup) + backup_controller.delete_manifest(old_backup) + except (ManifestError, PreconditionError) as e: + # The S3 merge already happened and is not reversible, so the task + # cannot be failed here -- retry the manifest work instead. + task.function_result = f"Merge done, manifest update failed: {e}" + task.retry += 1 + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return + task.function_result = "Merge completed" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) @@ -384,6 +486,12 @@ def _terminate_task(task, reason): pass elif task.function_name == JobSchedule.FN_BACKUP_RESTORE: _set_lvol_restore_failed(task, reason) + # This is the timeout / retry-ceiling path, so it is terminal too and + # owes the same cleanup as a normal failure. + try: + _release_restore_s3_bdev(task, db.get_storage_node_by_id(task.node_id)) + except KeyError: + _release_restore_s3_bdev(task, None) elif task.function_name == JobSchedule.FN_BACKUP_MERGE: old_bid = task.function_params.get("old_backup_id") if old_bid: diff --git a/simplyblock_core/services/tasks_runner_backup_merge.py b/simplyblock_core/services/tasks_runner_backup_merge.py index d28cfdc9a..45d09ea88 100644 --- a/simplyblock_core/services/tasks_runner_backup_merge.py +++ b/simplyblock_core/services/tasks_runner_backup_merge.py @@ -6,7 +6,7 @@ import time from simplyblock_core import constants, db_controller, utils -from simplyblock_core.controllers import backup_controller +from simplyblock_core.controllers.backup import policy as backup_policy from simplyblock_core.models.cluster import Cluster logger = utils.get_logger(__name__) @@ -31,11 +31,11 @@ def main(): lvols = db.get_lvols(cl.get_id()) for lvol in lvols: try: - backup_controller.evaluate_policy(lvol) + backup_policy.evaluate_policy(lvol) except Exception as e: logger.error(f"Error evaluating policy for lvol {lvol.get_id()}: {e}") try: - backup_controller.evaluate_schedule(lvol) + backup_policy.evaluate_schedule(lvol) except Exception as e: logger.error(f"Error evaluating schedule for lvol {lvol.get_id()}: {e}") except Exception as e: diff --git a/simplyblock_core/storage_node_ops.py b/simplyblock_core/storage_node_ops.py index dd3e7d6d9..3dc073864 100644 --- a/simplyblock_core/storage_node_ops.py +++ b/simplyblock_core/storage_node_ops.py @@ -4975,10 +4975,10 @@ def _abort_restart(reason): # Create S3 bdev for backup support (only if backup is configured) if cluster.backup_config: - from simplyblock_core.controllers import backup_controller + from simplyblock_core.controllers.backup import device as backup_device logger.info("Creating S3 bdev on restarted node") try: - backup_controller.create_s3_bdev(snode, cluster.backup_config) + backup_device.create_s3_bdev(snode, cluster.get_backup_config()) except Exception as e: logger.exception(str(e)) return False diff --git a/simplyblock_web/api/v2/_dtos.py b/simplyblock_web/api/v2/_dtos.py index 0e40fbf1d..11af748e1 100644 --- a/simplyblock_web/api/v2/_dtos.py +++ b/simplyblock_web/api/v2/_dtos.py @@ -19,6 +19,8 @@ from simplyblock_core.models.snapshot import SnapShot from simplyblock_core.models.storage_node import StorageNode from simplyblock_core.models.backup import Backup, BackupPolicy +from simplyblock_core.controllers.backup.manifest import BackupManifest +from simplyblock_core.models.backup_config import BackupConfig from simplyblock_core.models.stats import StatsObject from simplyblock_core.models.lvol_migration import LVolMigration from simplyblock_core.models.lvol_migration_group import LVolMigrationGroup @@ -502,6 +504,25 @@ def from_model( ) +#: A cluster's backup configuration as the API exchanges it, in both +#: directions: the request body of the PUT and the response body of the GET. +#: +#: An alias rather than a hand-copied duplicate, because the two shapes are +#: identical today and a copy would only drift. It is still a name of its own, so +#: the wire format can diverge from ``BackupConfig`` later by turning this into a +#: real class, without touching a single route signature. +BackupConfigDTO = BackupConfig + +#: A backup's manifest as the API exchanges it: the response body of +#: export/discover and the entries of an inline import. +#: +#: An alias for the same reason ``BackupConfigDTO`` is one -- except that here the +#: shapes have a reason to stay locked together, since the wire form of a manifest +#: is also its form in the bucket. Naming it separately still lets the API grow a +#: field the stored document does not have. +BackupManifestDTO = BackupManifest + + class BackupDTO(BaseModel): id: UUID s3_id: int @@ -516,7 +537,7 @@ class BackupDTO(BaseModel): allowed_hosts: List[dict] created_at: int completed_at: int - source_cluster_id: str + encrypted: bool @staticmethod def from_model(model: Backup): @@ -534,7 +555,7 @@ def from_model(model: Backup): allowed_hosts=model.allowed_hosts or [], created_at=model.created_at, completed_at=model.completed_at, - source_cluster_id=model.source_cluster_id or "", + encrypted=model.encrypted, ) diff --git a/simplyblock_web/api/v2/cluster/__init__.py b/simplyblock_web/api/v2/cluster/__init__.py index f94cf367e..728445ff2 100644 --- a/simplyblock_web/api/v2/cluster/__init__.py +++ b/simplyblock_web/api/v2/cluster/__init__.py @@ -3,7 +3,7 @@ from uuid import UUID from fastapi import APIRouter, HTTPException, Request, Response -from pydantic import BaseModel, Field, SecretStr, computed_field, model_validator +from pydantic import BaseModel, Field, computed_field, model_validator from pydantic.networks import AnyUrl, UrlConstraints from simplyblock_core.db_controller import DBController @@ -17,7 +17,7 @@ from .storage_node import api as storage_node_api from .subsystem import api as subsystem_api from .task import api as task_api -from .._dtos import ClusterDTO +from .._dtos import BackupConfigDTO, ClusterDTO from .. import util as util @@ -36,18 +36,6 @@ class _UpdateParams(BaseModel): restart: bool = Field(False) -class BackupConfigParams(BaseModel): - access_key_id: Optional[SecretStr] = None - secret_access_key: Optional[SecretStr] = None - local_endpoint: Optional[str] = None - bucket_name: Optional[str] = None - snapshot_backups: Optional[bool] = None - with_compression: Optional[bool] = None - secondary_target: Optional[int] = Field(default=None, ge=0) - local_testing: Optional[bool] = None - s3_thread_pool_size: Optional[int] = Field(default=None, ge=0) - - class HashicorpVaultSettings(BaseModel): base_url: Optional[Annotated[AnyUrl, UrlConstraints(allowed_schemes=["https"])]] = None transit_mount: str = "simplyblock/transit" @@ -82,7 +70,7 @@ class ClusterParams(BaseModel): nvmf_base_port: int = 4420 rpc_base_port: int = 8080 snode_api_port: int = 50001 - backup_config: Optional[BackupConfigParams] = None + backup_config: Optional[BackupConfigDTO] = None hashicorp_vault_settings: Optional[HashicorpVaultSettings] = None enable_failure_domain: bool = False @@ -115,6 +103,8 @@ def add(request: Request, parameters: ClusterParams, response_format: util.Creat params = parameters.model_dump(exclude_none=True) if "hashicorp_vault_settings" in params: params["hashicorp_vault_settings"] = ModelVaultSettings(params["hashicorp_vault_settings"]) + if parameters.backup_config is not None: + params["backup_config"] = parameters.backup_config.model_dump(exclude_none=True) cluster_id_or_false = cluster_ops.add_cluster(**params) except ValueError as e: raise HTTPException(status_code=409, detail=str(e)) @@ -156,6 +146,36 @@ def update(cluster: Cluster, parameters: UpdatableClusterParameters): return Response(status_code=204) +@instance_api.get('/backup-config', name='clusters:backup-config:get') +def get_backup_config(cluster: Cluster) -> BackupConfigDTO: + """The cluster's backup configuration, with credentials masked. + + The credentials are ``SecretStr``, which FastAPI's JSON serialization + renders as ``**********``. + """ + try: + return cluster.get_backup_config() + except ValueError as e: + raise HTTPException(404, str(e)) from e + + +@instance_api.put('/backup-config', name='clusters:backup-config:set', + status_code=204, responses={204: {"content": None}}) +def set_backup_config(cluster: Cluster, parameters: BackupConfigDTO) -> Response: + """Replace the cluster's backup configuration. + + Backup configuration used to be settable only at cluster-create time, which + left no way to correct or complete it -- notably no way to record a region + on a cluster created before it was mandatory. + + A full replacement rather than a patch: the fields interact (an endpoint + implies addressing style and TLS expectations), so merging half a config + into an existing one produces combinations nobody chose. + """ + cluster_ops.set_backup_config(cluster.get_id(), parameters) + return Response(status_code=204) + + @instance_api.delete('/', name='clusters:delete', status_code=204, responses={204: {"content": None}}) def delete(cluster: Cluster) -> Response: try: diff --git a/simplyblock_web/api/v2/cluster/backup.py b/simplyblock_web/api/v2/cluster/backup.py index be8147246..fcc812fb3 100644 --- a/simplyblock_web/api/v2/cluster/backup.py +++ b/simplyblock_web/api/v2/cluster/backup.py @@ -1,16 +1,19 @@ -from typing import List, Optional +from typing import List, Optional, Union from uuid import UUID from fastapi import APIRouter, HTTPException, Query, Request, Response -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from simplyblock_core.db_controller import DBController -from simplyblock_core.controllers import backup_controller +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import policy as backup_policy +from simplyblock_core.controllers.backup.manifest import ManifestError +from simplyblock_core.models.backup_config import S3Credentials from simplyblock_core.models.cluster import Cluster as ClusterModel from simplyblock_core.models.lvol_model import LVol from .._dependencies import BackupResource, Cluster, Policy -from .._dtos import BackupDTO, BackupPolicyDTO +from .._dtos import BackupConfigDTO, BackupDTO, BackupManifestDTO, BackupPolicyDTO from ..util import CreationResponseFormatParameter, creation_response @@ -53,28 +56,83 @@ class _RestoreParams(BaseModel): target_node_id: Optional[str] = None + #: Credentials for the backup's bucket, when that is not this cluster's own + #: -- the disaster-recovery case. Omit to use the nodes' instance role. + s3_credentials: Optional[S3Credentials] = None + @api.post('/restore', name='clusters:backups:restore', status_code=202) def restore_backup(cluster: Cluster, parameters: _RestoreParams): return {"lvol_id": backup_controller.restore_backup( - parameters.backup_id, parameters.lvol_name, parameters.pool, target_node_id=parameters.target_node_id)} + parameters.backup_id, parameters.lvol_name, parameters.pool, + target_node_id=parameters.target_node_id, + s3_credentials=parameters.s3_credentials)} + + +class _ImportManifests(BaseModel): + """Manifests carried in the request itself, e.g. from an export file.""" + model_config = ConfigDict(extra="forbid") + + metadata: List[BackupManifestDTO] + +class _ImportFromBucket(BaseModel): + """Import whatever a bucket turns out to contain. -class _ImportParams(BaseModel): - metadata: list[dict] + The disaster-recovery path: it needs a bucket and credentials for it, and + nothing from the cluster that wrote the backups. + """ + model_config = ConfigDict(extra="forbid") + + bucket: BackupConfigDTO + + +#: The two ways to name what to import. A union rather than one model with two +#: optional fields and a validator forbidding both/neither: pydantic then rejects +#: a malformed body itself, and the OpenAPI schema says "one of these two" rather +#: than "everything optional, good luck". +_ImportParams = Union[_ImportManifests, _ImportFromBucket] @api.post('/import', name='clusters:backups:import') def import_backups(cluster: Cluster, parameters: _ImportParams): - count = backup_controller.import_backups(parameters.metadata, cluster_id=cluster.get_id()) + try: + count = ( + backup_controller.import_from_bucket( + parameters.bucket, cluster_id=cluster.get_id()) + if isinstance(parameters, _ImportFromBucket) else + backup_controller.import_backups( + parameters.metadata, cluster_id=cluster.get_id()) + ) + except ManifestError as e: + # The bucket named in the request could not be read. 400 rather than + # 502: nothing here proxies for an upstream service, and what the caller + # supplied is the only thing that can be wrong from here. + raise HTTPException(400, str(e)) from e + except ValueError as e: + raise HTTPException(400, str(e)) from e return {"imported": count} +@api.post('/discover', name='clusters:backups:discover') +def discover_backups(parameters: BackupConfigDTO) -> List[BackupManifestDTO]: + """List the backups a bucket contains, without importing anything. + + A POST because it carries credentials, which have no business in a query + string. Takes no cluster state at all: this is what an operator runs when + the cluster that wrote the backups no longer exists. + """ + try: + return backup_controller.discover_backups(parameters) + except ManifestError as e: + raise HTTPException(400, str(e)) from e + + @api.get('/export', name='clusters:backups:export') def export_backups( cluster: Cluster, backup_id: Optional[str] = Query(None, description="Export only the chain containing this backup UUID"), lvol_name: Optional[str] = Query(None, description="Export all completed backups for this lvol name"), -): +) -> List[BackupManifestDTO]: lvol_name_filter = lvol_name if backup_id and not lvol_name_filter: try: @@ -82,25 +140,8 @@ def export_backups( lvol_name_filter = backup.lvol_name except KeyError: raise HTTPException(404, f"Backup {backup_id} not found") - data = backup_controller.export_backups( + return backup_controller.export_backups( cluster_id=cluster.get_id(), lvol_name=lvol_name_filter) - return data - - -class _BackupSourceSwitchParams(BaseModel): - source_cluster_id: str - - -@api.post('/source-switch', name='clusters:backups:source-switch') -def source_switch(cluster: Cluster, parameters: _BackupSourceSwitchParams): - backup_controller.switch_backup_source(cluster.get_id(), parameters.source_cluster_id) - return {"source_cluster_id": parameters.source_cluster_id} - - -@api.get('/sources', name='clusters:backups:sources') -def list_sources(cluster: Cluster): - sources = backup_controller.get_backup_sources(cluster.get_id()) - return sources def _lookup_lvol_in_cluster(volume_id: str, cluster: ClusterModel) -> LVol: @@ -155,7 +196,7 @@ class _PolicyCreateParams(BaseModel): @policy_api.post('/', name='clusters:backup-policies:create', status_code=201, responses={201: {"content": None}}) def create_policy(cluster: Cluster, parameters: _PolicyCreateParams) -> Response: - policy_id, error = backup_controller.add_policy( + policy_id, error = backup_policy.add_policy( cluster.get_id(), parameters.name, max_versions=parameters.versions or 0, max_age=parameters.age or "", @@ -179,7 +220,7 @@ def _validate_attachment_target(target_type: str, target_id: str, cluster: Clust @policy_api.delete('/{policy_id}', name='clusters:backup-policies:delete', status_code=204, responses={204: {"content": None}}) def delete_policy(cluster: Cluster, policy: Policy) -> Response: - success, error = backup_controller.remove_policy(policy.uuid) + success, error = backup_policy.remove_policy(policy.uuid) if error: raise HTTPException(400, error) return Response(status_code=204) @@ -193,7 +234,7 @@ class _AttachParams(BaseModel): @policy_api.post('/{policy_id}/attach', name='clusters:backup-policies:attach', status_code=201) def attach_policy(cluster: Cluster, policy: Policy, parameters: _AttachParams): _validate_attachment_target(parameters.target_type, parameters.target_id, cluster) - att_id, error = backup_controller.attach_policy( + att_id, error = backup_policy.attach_policy( policy.uuid, parameters.target_type, parameters.target_id) if error: raise HTTPException(400, error) @@ -203,7 +244,7 @@ def attach_policy(cluster: Cluster, policy: Policy, parameters: _AttachParams): @policy_api.post('/{policy_id}/detach', name='clusters:backup-policies:detach', status_code=204, responses={204: {"content": None}}) def detach_policy(cluster: Cluster, policy: Policy, parameters: _AttachParams) -> Response: _validate_attachment_target(parameters.target_type, parameters.target_id, cluster) - success, error = backup_controller.detach_policy( + success, error = backup_policy.detach_policy( policy.uuid, parameters.target_type, parameters.target_id) if error: raise HTTPException(400, error) diff --git a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py index d430a6c63..03b7fe469 100644 --- a/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py +++ b/simplyblock_web/api/v2/cluster/storage_pool/volume/__init__.py @@ -6,7 +6,8 @@ from simplyblock_core.db_controller import DBController from simplyblock_core import utils as core_utils -from simplyblock_core.controllers import backup_controller, lvol_controller, snapshot_controller +from simplyblock_core.controllers import lvol_controller, snapshot_controller +from simplyblock_core.controllers.backup import controller as backup_controller from simplyblock_core.models.lvol_model import LVol from ...._dependencies import Cluster, StoragePool, Volume diff --git a/tests/integration/expansion_sim/conftest.py b/tests/integration/expansion_sim/conftest.py index 85c604df1..b1f2d0b83 100644 --- a/tests/integration/expansion_sim/conftest.py +++ b/tests/integration/expansion_sim/conftest.py @@ -161,7 +161,7 @@ def patched_rpc_router(): "simplyblock_core.controllers.device_controller", "simplyblock_core.controllers.snapshot_controller", "simplyblock_core.controllers.pool_controller", - "simplyblock_core.controllers.backup_controller", + "simplyblock_core.controllers.backup.device", ] saved_rpc = install_rpc_router(target_modules) saved_fw = install_firewall_stub([ diff --git a/tests/integration/test_backup.py b/tests/integration/test_backup.py index cb733e230..0378d524a 100644 --- a/tests/integration/test_backup.py +++ b/tests/integration/test_backup.py @@ -24,6 +24,7 @@ import pytest +from simplyblock_core.controllers.backup.controller import backup_snapshot from simplyblock_core.db_controller import DBController from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup, BackupPolicy, BackupPolicyAttachment @@ -54,6 +55,22 @@ def _node(uuid="node-1", status=StorageNode.STATUS_ONLINE, lvstore="lvs_test", return n +def _backup_config(**overrides): + from simplyblock_core.models.backup_config import BackupConfig + return BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", + "region": "eu-central-1", + **overrides, + }) + + +def _cluster(uuid="cluster-1", **config_overrides): + c = Cluster() + c.uuid = uuid + c.backup_config = _backup_config(**config_overrides).model_dump(exclude_none=True) + return c + + def _backup(uuid="backup-1", lvol_id="lvol-1", status=Backup.STATUS_COMPLETED, node_id="node-1", cluster_id="cluster-1", prev_backup_id="", created_at=None, snapshot_id="snap-1", s3_id=1): @@ -119,7 +136,7 @@ def test_default_fields(self): self.assertEqual(b.size, 0) self.assertEqual(b.created_at, 0) self.assertEqual(b.completed_at, 0) - self.assertEqual(b.s3_metadata, {}) + self.assertEqual(b.location, {}) self.assertEqual(b.error_message, "") def test_status_constants(self): @@ -206,33 +223,33 @@ def test_backup_config_stored(self): class TestParseAgeString(unittest.TestCase): def test_minutes(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string("30m"), 1800) def test_hours(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string("12h"), 43200) def test_days(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string("2d"), 172800) def test_weeks(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string("1w"), 604800) def test_invalid_format(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string with self.assertRaises(ValueError): _parse_age_string("abc") def test_invalid_unit(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string with self.assertRaises(ValueError): _parse_age_string("5x") def test_whitespace(self): - from simplyblock_core.controllers.backup_controller import _parse_age_string + from simplyblock_core.controllers.backup.policy import _parse_age_string self.assertEqual(_parse_age_string(" 3d "), 259200) @@ -243,30 +260,32 @@ def test_whitespace(self): class TestComputeS3CpuMasks(unittest.TestCase): def test_masks_from_node(self): - from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks + from simplyblock_core.controllers.backup.device import _compute_s3_cpu_masks node = _node() # app_thread_mask="0x8", cpu=8 bdb, s3 = _compute_s3_cpu_masks(node) self.assertEqual(bdb, 0x8) # app thread core 3 self.assertEqual(s3, 0xFF) # all 8 vCPUs — no pinning def test_no_app_thread_mask(self): - from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks + from simplyblock_core.controllers.backup.device import _compute_s3_cpu_masks node = _node() node.app_thread_mask = "" bdb, s3 = _compute_s3_cpu_masks(node) - self.assertEqual(bdb, 0) # falls back to data plane default + # None, not 0: a zero mask selects no CPUs, and the RPC omits the + # parameter so the data plane derives one from the app core mask. + self.assertIsNone(bdb) self.assertEqual(s3, 0xFF) def test_no_cpu_count(self): - from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks + from simplyblock_core.controllers.backup.device import _compute_s3_cpu_masks node = _node() node.cpu = 0 bdb, s3 = _compute_s3_cpu_masks(node) self.assertEqual(bdb, 0x8) - self.assertEqual(s3, 0) # falls back to data plane default + self.assertIsNone(s3) # omitted; data plane picks def test_large_cpu_count(self): - from simplyblock_core.controllers.backup_controller import _compute_s3_cpu_masks + from simplyblock_core.controllers.backup.device import _compute_s3_cpu_masks node = _node() node.cpu = 32 bdb, s3 = _compute_s3_cpu_masks(node) @@ -279,38 +298,34 @@ def test_large_cpu_count(self): class TestCreateS3Bdev(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_success(self, MockRPC, mock_boto3_client): mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.return_value = (True, None) mock_rpc.bdev_lvol_s3_bdev.return_value = True mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() - config = {"secondary_target": 0, "with_compression": False, - "snapshot_backups": True} - create_s3_bdev(node, config) + create_s3_bdev(node, _backup_config()) mock_rpc.bdev_s3_create.assert_called_once() # Verify CPU masks: bdb_lcpu_mask=app_thread(0x8=8), s3_lcpu_mask=all 8 vCPUs(0xFF=255) _, kwargs = mock_rpc.bdev_s3_create.call_args self.assertEqual(kwargs["bdb_lcpu_mask"], 0x8) self.assertEqual(kwargs["s3_lcpu_mask"], 0xFF) - mock_rpc.bdev_s3_add_bucket_name.assert_called_once_with( - "s3_lvs_test", "simplyblock-backup-cluster-1", allow_existing=True) + self.assertEqual(kwargs["bucket_name"], "simplyblock-backup-cluster-1") mock_rpc.bdev_lvol_s3_bdev.assert_called_once_with("lvs_test", "s3_lvs_test") - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_no_lvstore(self, MockRPC, _mock_boto3_client): - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node(lvstore="") - with pytest.raises(Exception): - create_s3_bdev(node, {}) + with pytest.raises(PreconditionError): + create_s3_bdev(node, _backup_config()) MockRPC.assert_not_called() @@ -319,87 +334,131 @@ def test_bdev_s3_create_fails(self, MockRPC): mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = None - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() - with pytest.raises(Exception): - create_s3_bdev(node, {}) - mock_rpc.bdev_s3_add_bucket_name.assert_not_called() + with pytest.raises(RuntimeError): + create_s3_bdev(node, _backup_config()) mock_rpc.bdev_lvol_s3_bdev.assert_not_called() - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") - def test_bucket_name_fails(self, MockRPC, mock_boto3_client): - from simplyblock_core.rpc_client import RPCRemoteError + def test_bucket_is_a_create_parameter(self, MockRPC, mock_boto3_client): + """A device cannot exist without its bucket, so there is no window in + which one is attached to an lvstore with no bucket registered.""" mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.side_effect = RPCRemoteError("error", code=-1) - mock_s3 = mock_boto3_client.return_value - mock_s3.head_bucket.return_value = {} + mock_rpc.bdev_lvol_s3_bdev.return_value = True + mock_boto3_client.return_value.head_bucket.return_value = {} - from simplyblock_core.controllers.backup_controller import create_s3_bdev - node = _node() - with pytest.raises(Exception): - create_s3_bdev(node, {}) - mock_rpc.bdev_lvol_s3_bdev.assert_not_called() + from simplyblock_core.controllers.backup.device import create_s3_bdev + create_s3_bdev(_node(), _backup_config()) + + _, kwargs = mock_rpc.bdev_s3_create.call_args + assert kwargs["bucket_name"] == "simplyblock-backup-cluster-1" - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") + @patch("simplyblock_core.models.storage_node.RPCClient") + def test_a_cluster_that_configured_no_bucket_still_gets_a_device( + self, MockRPC, mock_boto3_client): + """Activation hands ``cluster.get_backup_config()`` straight to this + function for every node it brings up (``cluster_ops._finish_pass1_node``), + so a stored config that names no bucket -- what a cluster created through + the operator has, its CR having no bucket field -- failed the activation + rather than the backup that would have used the bucket.""" + mock_rpc = MockRPC.return_value + mock_rpc.bdev_s3_create.return_value = True + mock_rpc.bdev_lvol_s3_bdev.return_value = True + mock_boto3_client.return_value.head_bucket.return_value = {} + + cluster = Cluster() + cluster.uuid = "cluster-1" + cluster.backup_config = {"region": "eu-central-1"} + + from simplyblock_core.controllers.backup.device import create_s3_bdev + create_s3_bdev(_node(), cluster.get_backup_config()) + + _, kwargs = mock_rpc.bdev_s3_create.call_args + assert kwargs["bucket_name"] == "simplyblock-backup-cluster-1" + + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_attach_fails(self, MockRPC, mock_boto3_client): from simplyblock_core.rpc_client import RPCRemoteError mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.return_value = (True, None) mock_rpc.bdev_lvol_s3_bdev.side_effect = RPCRemoteError("attach failed", code=-1) mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() - with pytest.raises(Exception): - create_s3_bdev(node, {}) + with pytest.raises(RuntimeError): + create_s3_bdev(node, _backup_config()) - @patch("simplyblock_core.controllers.backup_controller.boto3.client") + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") @patch("simplyblock_core.models.storage_node.RPCClient") def test_local_testing_params(self, MockRPC, mock_boto3_client): mock_rpc = MockRPC.return_value mock_rpc.bdev_s3_create.return_value = True - mock_rpc.bdev_s3_add_bucket_name.return_value = (True, None) mock_rpc.bdev_lvol_s3_bdev.return_value = True mock_s3 = mock_boto3_client.return_value mock_s3.head_bucket.return_value = {} - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.models.backup_config import BackupConfig + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() - config = { - "secondary_target": 0, + # A genuine pre-BackupConfig dict: no region, local_testing standing in + # for four separate decisions. + create_s3_bdev(node, BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", "local_testing": True, "local_endpoint": "http://minio:9000", "access_key_id": "minioadmin", "secret_access_key": "minioadmin", - } - create_s3_bdev(node, config) + })) _, kwargs = mock_rpc.bdev_s3_create.call_args - self.assertTrue(kwargs.get("local_testing", False)) - self.assertEqual(kwargs.get("local_endpoint", ""), "http://minio:9000") - self.assertEqual(kwargs.get("access_key_id", ""), "minioadmin") - self.assertEqual(kwargs.get("secret_access_key", ""), "minioadmin") - mock_boto3_client.assert_called_once_with( - "s3", - aws_access_key_id="minioadmin", - aws_secret_access_key="minioadmin", - endpoint_url="http://minio:9000", - ) + self.assertEqual(kwargs["endpoint"], "http://minio:9000") + self.assertEqual(kwargs["region"], "us-east-1") + self.assertFalse(kwargs["verify_tls"]) + self.assertTrue(kwargs["use_path_style"]) + self.assertEqual(kwargs["access_key_id"].get_secret_value(), "minioadmin") + self.assertEqual(kwargs["secret_access_key"].get_secret_value(), "minioadmin") + + _, boto_kwargs = mock_boto3_client.call_args + self.assertEqual(boto_kwargs["aws_access_key_id"], "minioadmin") + self.assertEqual(boto_kwargs["aws_secret_access_key"], "minioadmin") + self.assertEqual(boto_kwargs["endpoint_url"], "http://minio:9000") + # local_testing unpacked into the properties it actually stood for. + self.assertEqual(boto_kwargs["region_name"], "us-east-1") + self.assertFalse(boto_kwargs["verify"]) + + @patch("simplyblock_core.controllers.backup.manifest.boto3.client") + @patch("simplyblock_core.models.storage_node.RPCClient") + def test_no_credentials_defers_to_the_provider_chain(self, MockRPC, mock_boto3_client): + """An absent key pair must mean "use the node's IAM role", not "send empty keys".""" + mock_rpc = MockRPC.return_value + mock_rpc.bdev_s3_create.return_value = True + mock_rpc.bdev_lvol_s3_bdev.return_value = True + mock_boto3_client.return_value.head_bucket.return_value = {} + + from simplyblock_core.controllers.backup.device import create_s3_bdev + create_s3_bdev(_node(), _backup_config()) + + _, boto_kwargs = mock_boto3_client.call_args + self.assertIsNone(boto_kwargs["aws_access_key_id"]) + self.assertIsNone(boto_kwargs["aws_secret_access_key"]) @patch("simplyblock_core.models.storage_node.RPCClient") def test_exception_handled(self, MockRPC): + from simplyblock_core.rpc_client import RPCException mock_rpc = MockRPC.return_value - mock_rpc.bdev_s3_create.side_effect = Exception("connection refused") + mock_rpc.bdev_s3_create.side_effect = RPCException("connection refused") - from simplyblock_core.controllers.backup_controller import create_s3_bdev + from simplyblock_core.controllers.backup.device import create_s3_bdev node = _node() - with pytest.raises(Exception): - create_s3_bdev(node, {}) + with pytest.raises(RuntimeError): + create_s3_bdev(node, _backup_config()) # =========================================================================== @@ -407,143 +466,121 @@ def test_exception_handled(self, MockRPC): # =========================================================================== class TestBackupSnapshot(unittest.TestCase): + """Real FDB: backup_snapshot reads cluster/node/snapshot state and writes Backups. - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.backup_events") - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_success(self, mock_db, _mock_local_source, mock_events, mock_tasks, _mock_write): - snap = _snapshot() - mock_db.get_snapshot_by_id.return_value = snap - mock_db.get_storage_node_by_id.return_value = _node() - mock_db.get_backups_by_lvol_id.return_value = [] - mock_db.get_backups.return_value = [] - mock_db.get_backups_by_snapshot_id.return_value = [] - mock_db.acquire_backup_chain_locks.return_value = (True, None) - mock_tasks.add_backup_task.return_value = "task-1" + Only what sits above the database is mocked -- the storage node's RPC client, + the task runner and event emission. + """ + + def setUp(self): + self.db = DBController() + _cluster().write_to_db(self.db.kv_store) + _node().write_to_db(self.db.kv_store) + + def _persist(self, snapshot): + snapshot.lvol.write_to_db(self.db.kv_store) + snapshot.write_to_db(self.db.kv_store) + return snapshot - from simplyblock_core.controllers.backup_controller import backup_snapshot - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", return_value=[snap]): + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.backup_events") + def test_success(self, mock_events, mock_tasks): + snap = self._persist(_snapshot()) + + with patch("simplyblock_core.controllers.backup.controller._get_snapshot_chain", + return_value=[snap]): backup_id, error = backup_snapshot("snap-1") - self.assertIsNotNone(backup_id) self.assertIsNone(error) + self.assertIsNotNone(backup_id) mock_tasks.add_backup_task.assert_called_once() - mock_events.backup_created.assert_called_once() - # Verify s3_id is assigned - created_backup = mock_events.backup_created.call_args[0][2] - self.assertEqual(created_backup.s3_id, 1) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_snapshot_not_found(self, mock_db): - mock_db.get_snapshot_by_id.side_effect = KeyError("not found") + stored = self.db.get_backup_by_id(backup_id) + self.assertEqual(stored.status, Backup.STATUS_PENDING) + self.assertGreater(stored.s3_id, 0) + # Self-describing from the moment it is created. + self.assertEqual(stored.get_location().bucket_name, + "simplyblock-backup-cluster-1") + + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.backup_events") + def test_incremental_backup(self, mock_events, mock_tasks): + snap = self._persist(_snapshot()) + prev = _backup(uuid="prev-backup", s3_id=3, snapshot_id="snap-0", + status=Backup.STATUS_COMPLETED) + prev.write_to_db(self.db.kv_store) + + with patch("simplyblock_core.controllers.backup.controller._get_snapshot_chain", + return_value=[snap]): + backup_id, error = backup_snapshot("snap-1") + + self.assertIsNone(error) + stored = self.db.get_backup_by_id(backup_id) + self.assertEqual(stored.prev_backup_id, "prev-backup") + # Monotonic allocation, so strictly above the existing backup's id. + self.assertGreater(stored.s3_id, 3) - from simplyblock_core.controllers.backup_controller import backup_snapshot + def test_snapshot_not_found(self): backup_id, error = backup_snapshot("missing") self.assertIsNone(backup_id) self.assertIn("not found", error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_no_lvol(self, mock_db): - snap = _snapshot() - snap.lvol = None - mock_db.get_snapshot_by_id.return_value = snap + def test_node_not_online(self): + _node(status=StorageNode.STATUS_OFFLINE).write_to_db(self.db.kv_store) + self._persist(_snapshot()) - from simplyblock_core.controllers.backup_controller import backup_snapshot backup_id, error = backup_snapshot("snap-1") self.assertIsNone(backup_id) - self.assertIn("no associated lvol", error.lower()) + self.assertIn("not online", error) - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_node_not_online(self, mock_db, _mock_local_source): - snap = _snapshot() - node = _node(status=StorageNode.STATUS_OFFLINE) - mock_db.get_snapshot_by_id.return_value = snap - mock_db.get_storage_node_by_id.return_value = node + def test_cluster_without_backup_config_is_refused(self): + """Refused before the chain lock, the KMS work or any task is created.""" + cluster = _cluster() + cluster.backup_config = {} + cluster.write_to_db(self.db.kv_store) + self._persist(_snapshot()) - from simplyblock_core.controllers.backup_controller import backup_snapshot backup_id, error = backup_snapshot("snap-1") self.assertIsNone(backup_id) - self.assertIn("not online", error) - - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.backup_events") - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_incremental_backup(self, mock_db, _mock_local_source, mock_events, mock_tasks, _mock_write): - snap = _snapshot() - prev = _backup(uuid="prev-backup", s3_id=3) - mock_db.get_snapshot_by_id.return_value = snap - mock_db.get_storage_node_by_id.return_value = _node() - mock_db.get_backups_by_lvol_id.return_value = [prev] - mock_db.get_backups.return_value = [prev] - mock_db.acquire_backup_chain_locks.return_value = (True, None) - mock_tasks.add_backup_task.return_value = "task-1" - - from simplyblock_core.controllers.backup_controller import backup_snapshot - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", return_value=[snap]): - backup_id, error = backup_snapshot("snap-1") - - self.assertIsNotNone(backup_id) - self.assertIsNone(error) - # Verify the backup object passed to events has prev_backup_id set - created_backup = mock_events.backup_created.call_args[0][2] - self.assertEqual(created_backup.prev_backup_id, "prev-backup") - self.assertEqual(created_backup.s3_id, 4) + self.assertIn("backup configuration", error) + self.assertEqual(self.db.get_backups(), []) - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.backup_events") - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_chain_backup_acquires_and_releases_lock(self, mock_db, _mock_local_source, mock_events, mock_tasks, _mock_write): - snap1 = _snapshot(uuid="snap-1") + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.controller.backup_events") + def test_chain_backup_acquires_and_releases_lock(self, mock_events, mock_tasks): + snap1 = self._persist(_snapshot(uuid="snap-1")) snap1.created_at = 1 - snap2 = _snapshot(uuid="snap-2") + snap2 = self._persist(_snapshot(uuid="snap-2")) snap2.created_at = 2 - snap2.lvol.uuid = "lvol-1" - mock_db.get_snapshot_by_id.return_value = snap2 - mock_db.get_storage_node_by_id.return_value = _node() - mock_db.get_backups_by_lvol_id.return_value = [] - mock_db.get_backups.return_value = [] - mock_db.get_backups_by_snapshot_id.return_value = [] - mock_db.acquire_backup_chain_locks.return_value = (True, None) - mock_tasks.add_backup_task.return_value = "task-1" - from simplyblock_core.controllers.backup_controller import backup_snapshot - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", return_value=[snap1, snap2]): + with patch("simplyblock_core.controllers.backup.controller._get_snapshot_chain", + return_value=[snap1, snap2]): backup_id, error = backup_snapshot("snap-2") - self.assertIsNotNone(backup_id) self.assertIsNone(error) - mock_db.acquire_backup_chain_locks.assert_called_once_with(["snap-1", "snap-2"], "snap-2", "lvol-1") - mock_db.release_backup_chain_locks.assert_called_once_with(["snap-1", "snap-2"]) + self.assertIsNotNone(backup_id) self.assertEqual(mock_tasks.add_backup_task.call_count, 2) - self.assertEqual(mock_events.backup_created.call_count, 2) - - @patch("simplyblock_core.controllers.backup_controller.is_local_backup_source", return_value=True) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_chain_backup_lock_conflict(self, mock_db, _mock_local_source): - snap = _snapshot(uuid="snap-4") - snap.created_at = 4 - existing_lock = MagicMock(requested_snapshot_id="snap-2", snapshot_id="snap-2") - mock_db.get_snapshot_by_id.return_value = snap - mock_db.get_storage_node_by_id.return_value = _node() - mock_db.acquire_backup_chain_locks.return_value = (False, existing_lock) + # Locks released, so a second request for the same chain can proceed. + self.assertIsNone(self.db.get_backup_chain_lock("snap-1")) + self.assertIsNone(self.db.get_backup_chain_lock("snap-2")) + + def test_chain_backup_lock_conflict(self): + snap = self._persist(_snapshot(uuid="snap-4")) + acquired, _ = self.db.acquire_backup_chain_locks(["snap-4"], "snap-2", "lvol-1") + self.assertTrue(acquired) - from simplyblock_core.controllers.backup_controller import backup_snapshot - with patch("simplyblock_core.controllers.backup_controller._get_snapshot_chain", return_value=[snap]): + with patch("simplyblock_core.controllers.backup.controller._get_snapshot_chain", + return_value=[snap]): backup_id, error = backup_snapshot("snap-4") self.assertIsNone(backup_id) self.assertIn("already preparing this snapshot chain", error) - mock_db.release_backup_chain_locks.assert_not_called() + # The conflicting holder's lock must survive. + self.assertIsNotNone(self.db.get_backup_chain_lock("snap-4")) + # =========================================================================== @@ -585,65 +622,79 @@ def test_release_backup_chain_locks_uses_unbound_method_with_db_handle(self, moc # =========================================================================== class TestRestoreBackup(unittest.TestCase): + """Real FDB: restore reads backup/pool/cluster state and creates a volume. - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_success(self, mock_db, mock_tasks): - cluster_uuid = "00000000-0000-0000-0000-000000000001" - backup = _backup(s3_id=5, cluster_id=cluster_uuid) - mock_db.get_backup_by_id.return_value = backup - mock_db.get_backup_chain.return_value = [backup] - mock_tasks.add_backup_restore_task.return_value = True + add_lvol_ha and the task runner are mocked -- they sit above the database + and drive RPC to storage nodes. + """ + + CLUSTER_ID = "00000000-0000-0000-0000-000000000001" - mock_cluster = MagicMock() - mock_cluster.uuid = cluster_uuid - mock_cluster.backup_source = "" - mock_db.get_cluster_by_id.return_value = mock_cluster + def setUp(self): + from simplyblock_core.models.pool import Pool + self.db = DBController() - # Mock the lvol created by add_lvol_ha - mock_lvol = MagicMock() - mock_lvol.node_id = "node-1" - mock_lvol.lvs_name = "lvs_test" - mock_lvol.lvol_bdev = "LVOL_123" - mock_lvol.write_to_db = MagicMock() - mock_db.get_lvol_by_id.return_value = mock_lvol + cluster = _cluster(uuid=self.CLUSTER_ID) + cluster.write_to_db(self.db.kv_store) - with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha") as mock_add_ha: - mock_add_ha.return_value = ("lvol-new", None) + pool = Pool() + pool.uuid = "pool-1" + pool.pool_name = "pool-1" # resolved by name: "pool-1" is not a UUID + pool.cluster_id = self.CLUSTER_ID + pool.write_to_db(self.db.kv_store) - from simplyblock_core.controllers.backup_controller import restore_backup + def _backup(self, **overrides): + backup = _backup(cluster_id=self.CLUSTER_ID, **overrides) + backup.location = _backup_config().location().model_dump(mode="json") + backup.write_to_db(self.db.kv_store) + return backup + + @patch("simplyblock_core.controllers.backup.controller.tasks_controller") + def test_success(self, mock_tasks): + self._backup(s3_id=5) + mock_tasks.add_backup_restore_task.return_value = True + + lvol = LVol() + lvol.uuid = "lvol-new" + lvol.node_id = "node-1" + lvol.lvs_name = "lvs_test" + lvol.lvol_bdev = "LVOL_123" + lvol.pool_uuid = "pool-1" + lvol.write_to_db(self.db.kv_store) + + with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha", + return_value=("lvol-new", None)): + from simplyblock_core.controllers.backup.controller import restore_backup result = restore_backup("backup-1", "restored_lvol", "pool-1") self.assertEqual(result, "lvol-new") - # Verify s3_id integers are passed, not UUIDs - call_args = mock_tasks.add_backup_restore_task.call_args - self.assertEqual(call_args[0][4], [5]) + # s3_id integers reach the data plane, not backup UUIDs. + self.assertEqual(mock_tasks.add_backup_restore_task.call_args[0][4], [5]) + self.assertEqual(self.db.get_lvol_by_id("lvol-new").status, LVol.STATUS_RESTORING) - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_backup_not_found(self, mock_db): - mock_db.get_backup_by_id.side_effect = KeyError("not found") - - from simplyblock_core.controllers.backup_controller import restore_backup + def test_backup_not_found(self): + from simplyblock_core.controllers.backup.controller import restore_backup with self.assertRaises(PreconditionError): restore_backup("missing", "lvol", "pool-1") - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_add_lvol_ha_fails(self, mock_db): - cluster_uuid = "00000000-0000-0000-0000-000000000001" - mock_db.get_backup_by_id.return_value = _backup(cluster_id=cluster_uuid) - mock_db.get_backup_chain.return_value = [_backup(cluster_id=cluster_uuid)] + def test_add_lvol_ha_fails(self): + self._backup() - mock_cluster = MagicMock() - mock_cluster.uuid = cluster_uuid - mock_cluster.backup_source = "" - mock_db.get_cluster_by_id.return_value = mock_cluster + with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha", + return_value=(None, "Pool not found")): + from simplyblock_core.controllers.backup.controller import restore_backup + with self.assertRaisesRegex(RuntimeError, "Failed to create restore volume"): + restore_backup("backup-1", "lvol", "pool-1") - with patch("simplyblock_core.controllers.lvol_controller.add_lvol_ha") as mock_add_ha: - mock_add_ha.return_value = (None, "Pool not found") + def test_incomplete_chain_is_refused(self): + self._backup(uuid="b-old", s3_id=1, status=Backup.STATUS_IN_PROGRESS) + self._backup(uuid="backup-1", s3_id=2, prev_backup_id="b-old") - from simplyblock_core.controllers.backup_controller import restore_backup - with self.assertRaisesRegex(RuntimeError, "Failed to create restore volume"): - restore_backup("backup-1", "lvol", "bad-pool") + from simplyblock_core.controllers.backup.controller import restore_backup + with self.assertRaisesRegex(PreconditionError, "Incomplete backups in chain"): + restore_backup("backup-1", "lvol", "pool-1") + + self.assertEqual(self.db.get_lvols(), []) # =========================================================================== @@ -652,9 +703,9 @@ def test_add_lvol_ha_fails(self, mock_db): class TestDeleteBackups(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.backup_events") + @patch("simplyblock_core.controllers.backup.controller.backup_events") @patch("simplyblock_core.models.storage_node.RPCClient") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_success(self, mock_db, MockRPC, mock_events): b1 = _backup(uuid="b-1") mock_db.get_backups_by_lvol_id.return_value = [b1] @@ -662,18 +713,18 @@ def test_success(self, mock_db, MockRPC, mock_events): b1.remove = MagicMock() MockRPC.return_value.bdev_lvol_s3_delete.return_value = True - from simplyblock_core.controllers.backup_controller import delete_backups + from simplyblock_core.controllers.backup.controller import delete_backups success, error = delete_backups("lvol-1") self.assertTrue(success) self.assertIsNone(error) b1.remove.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_no_backups(self, mock_db): mock_db.get_backups_by_lvol_id.return_value = [] - from simplyblock_core.controllers.backup_controller import delete_backups + from simplyblock_core.controllers.backup.controller import delete_backups success, error = delete_backups("lvol-1") self.assertFalse(success) @@ -686,34 +737,34 @@ def test_no_backups(self, mock_db): class TestListBackups(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_list_empty(self, mock_db): mock_db.get_backups.return_value = [] - from simplyblock_core.controllers.backup_controller import list_backups + from simplyblock_core.controllers.backup.controller import list_backups data = list_backups() self.assertEqual(data, []) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_list_with_backups(self, mock_db): b = _backup() mock_db.get_backups.return_value = [b] - from simplyblock_core.controllers.backup_controller import list_backups + from simplyblock_core.controllers.backup.controller import list_backups data = list_backups() self.assertEqual(len(data), 1) self.assertEqual(data[0]["ID"], "backup-1") self.assertEqual(data[0]["Status"], Backup.STATUS_COMPLETED) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.controller.db_controller") def test_list_sorted_newest_first_with_seconds(self, mock_db): older = _backup(uuid="older", created_at=1710000000) newer = _backup(uuid="newer", created_at=1710000005) mock_db.get_backups.return_value = [older, newer] - from simplyblock_core.controllers.backup_controller import list_backups + from simplyblock_core.controllers.backup.controller import list_backups data = list_backups() self.assertEqual([row["ID"] for row in data], ["newer", "older"]) @@ -727,38 +778,38 @@ def test_list_sorted_newest_first_with_seconds(self, mock_db): class TestPolicyAdd(unittest.TestCase): @patch.object(BackupPolicy, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_success(self, mock_db, _mock_write): mock_db.get_backup_policies.return_value = [] - from simplyblock_core.controllers.backup_controller import add_policy + from simplyblock_core.controllers.backup.policy import add_policy policy_id, error = add_policy("cluster-1", "daily", max_versions=5, max_age="2d") self.assertIsNotNone(policy_id) self.assertIsNone(error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_no_limits(self, mock_db): - from simplyblock_core.controllers.backup_controller import add_policy + from simplyblock_core.controllers.backup.policy import add_policy policy_id, error = add_policy("cluster-1", "empty", max_versions=0, max_age="") self.assertIsNone(policy_id) self.assertIn("must be specified", error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_duplicate_name(self, mock_db): existing = _policy(name="daily") mock_db.get_backup_policies.return_value = [existing] - from simplyblock_core.controllers.backup_controller import add_policy + from simplyblock_core.controllers.backup.policy import add_policy policy_id, error = add_policy("cluster-1", "daily", max_versions=5) self.assertIsNone(policy_id) self.assertIn("already exists", error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_invalid_age(self, mock_db): - from simplyblock_core.controllers.backup_controller import add_policy + from simplyblock_core.controllers.backup.policy import add_policy policy_id, error = add_policy("cluster-1", "test", max_age="invalid") self.assertIsNone(policy_id) @@ -767,25 +818,25 @@ def test_invalid_age(self, mock_db): class TestPolicyRemove(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_success(self, mock_db): p = _policy() p.remove = MagicMock() mock_db.get_backup_policy_by_id.return_value = p mock_db.get_backup_policy_attachments.return_value = [] - from simplyblock_core.controllers.backup_controller import remove_policy + from simplyblock_core.controllers.backup.policy import remove_policy success, error = remove_policy("policy-1") self.assertTrue(success) self.assertIsNone(error) p.remove.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_not_found(self, mock_db): mock_db.get_backup_policy_by_id.side_effect = KeyError("not found") - from simplyblock_core.controllers.backup_controller import remove_policy + from simplyblock_core.controllers.backup.policy import remove_policy success, error = remove_policy("missing") self.assertFalse(success) @@ -795,25 +846,25 @@ def test_not_found(self, mock_db): class TestPolicyAttach(unittest.TestCase): @patch.object(BackupPolicyAttachment, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_success(self, mock_db, _mock_write): p = _policy() mock_db.get_backup_policy_by_id.return_value = p mock_db.get_lvol_by_id.return_value = MagicMock() mock_db.get_backup_policy_attachments.return_value = [] - from simplyblock_core.controllers.backup_controller import attach_policy + from simplyblock_core.controllers.backup.policy import attach_policy att_id, error = attach_policy("policy-1", "lvol", "lvol-1") self.assertIsNotNone(att_id) self.assertIsNone(error) - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_invalid_target_type(self, mock_db): p = _policy() mock_db.get_backup_policy_by_id.return_value = p - from simplyblock_core.controllers.backup_controller import attach_policy + from simplyblock_core.controllers.backup.policy import attach_policy att_id, error = attach_policy("policy-1", "invalid", "target-1") self.assertIsNone(att_id) @@ -822,7 +873,7 @@ def test_invalid_target_type(self, mock_db): class TestPolicyDetach(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_success(self, mock_db): p = _policy() att = BackupPolicyAttachment() @@ -834,20 +885,20 @@ def test_success(self, mock_db): mock_db.get_backup_policy_by_id.return_value = p mock_db.get_backup_policy_attachments.return_value = [att] - from simplyblock_core.controllers.backup_controller import detach_policy + from simplyblock_core.controllers.backup.policy import detach_policy success, error = detach_policy("policy-1", "lvol", "lvol-1") self.assertTrue(success) self.assertIsNone(error) att.remove.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_not_found(self, mock_db): p = _policy() mock_db.get_backup_policy_by_id.return_value = p mock_db.get_backup_policy_attachments.return_value = [] - from simplyblock_core.controllers.backup_controller import detach_policy + from simplyblock_core.controllers.backup.policy import detach_policy success, error = detach_policy("policy-1", "lvol", "lvol-1") self.assertFalse(success) @@ -860,20 +911,20 @@ def test_not_found(self, mock_db): class TestEvaluatePolicy(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_no_policy(self, mock_db, mock_tasks): mock_db.get_policy_for_lvol.return_value = None - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_not_called() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_version_limit_exceeded(self, mock_db, mock_tasks, _mock_write): policy = _policy(max_versions=2, max_age_seconds=0) mock_db.get_policy_for_lvol.return_value = policy @@ -884,14 +935,14 @@ def test_version_limit_exceeded(self, mock_db, mock_tasks, _mock_write): b3 = _backup(uuid="b3", created_at=now - 100) mock_db.get_backups_by_lvol_id.return_value = [b1, b2, b3] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_under_version_limit(self, mock_db, mock_tasks): policy = _policy(max_versions=5, max_age_seconds=0) mock_db.get_policy_for_lvol.return_value = policy @@ -901,15 +952,15 @@ def test_under_version_limit(self, mock_db, mock_tasks): b2 = _backup(uuid="b2", created_at=now - 200) mock_db.get_backups_by_lvol_id.return_value = [b1, b2] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_not_called() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): policy = _policy(max_versions=0) policy.max_age_seconds = 3600 # 1 hour @@ -920,15 +971,15 @@ def test_age_limit_exceeded(self, mock_db, mock_tasks, _mock_write): b2 = _backup(uuid="b2", created_at=now - 100) mock_db.get_backups_by_lvol_id.return_value = [b1, b2] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_called_once() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): """When both versions and age are set, either limit can trigger a merge.""" policy = _policy(max_versions=3) @@ -940,15 +991,15 @@ def test_both_conditions_required(self, mock_db, mock_tasks, _mock_write): backups = [_backup(uuid=f"b{i}", created_at=now - (i * 60)) for i in range(4)] mock_db.get_backups_by_lvol_id.return_value = backups - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_called_once() @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_both_conditions_met(self, mock_db, mock_tasks, _mock_write): """When both limits set and both exceeded, merge triggers.""" policy = _policy(max_versions=2) @@ -961,98 +1012,40 @@ def test_both_conditions_met(self, mock_db, mock_tasks, _mock_write): b3 = _backup(uuid="b3", created_at=now - 100) mock_db.get_backups_by_lvol_id.return_value = [b1, b2, b3] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_fewer_than_two_backups(self, mock_db, mock_tasks): """Never merge with fewer than 2 completed backups.""" policy = _policy(max_versions=1) mock_db.get_policy_for_lvol.return_value = policy mock_db.get_backups_by_lvol_id.return_value = [_backup()] - from simplyblock_core.controllers.backup_controller import evaluate_policy + from simplyblock_core.controllers.backup.policy import evaluate_policy lvol = MagicMock() evaluate_policy(lvol) mock_tasks.add_backup_merge_task.assert_not_called() -# =========================================================================== -# 13. Import backups -# =========================================================================== - -class TestImportBackups(unittest.TestCase): - - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_import_new(self, mock_db, _mock_write): - mock_db.get_backup_by_id.side_effect = KeyError("not found") - mock_db.get_backups.return_value = [] - - from simplyblock_core.controllers.backup_controller import import_backups - count = import_backups([ - {"backup_id": "b-1", "lvol_id": "l-1", "cluster_id": "c-1"}, - {"backup_id": "b-2", "lvol_id": "l-1", "cluster_id": "c-1"}, - ]) - - self.assertEqual(count, 2) - - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_existing_fails(self, mock_db, mock_write): - existing = _backup(uuid="b-1") - mock_db.get_backup_by_id.side_effect = lambda bid: existing if bid == "b-1" else (_ for _ in ()).throw(KeyError()) - mock_db.get_backups.return_value = [existing] - - from simplyblock_core.controllers.backup_controller import import_backups - with self.assertRaises(PreconditionError): - import_backups([ - {"backup_id": "b-2", "lvol_id": "l-1", "cluster_id": "cluster-1"}, - {"backup_id": "b-1", "lvol_id": "l-1", "cluster_id": "cluster-1"}, - ]) - - mock_write.assert_not_called() - - @patch.object(Backup, 'write_to_db') - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_duplicate_in_metadata_fails(self, mock_db, mock_write): - mock_db.get_backup_by_id.side_effect = KeyError("not found") - mock_db.get_backups.return_value = [] - - from simplyblock_core.controllers.backup_controller import import_backups - with self.assertRaises(PreconditionError): - import_backups([ - {"backup_id": "b-1", "lvol_id": "l-1", "cluster_id": "cluster-1"}, - {"backup_id": "b-1", "lvol_id": "l-1", "cluster_id": "cluster-1"}, - ]) - - mock_write.assert_not_called() - - @patch("simplyblock_core.controllers.backup_controller.db_controller") - def test_skip_no_backup_id(self, mock_db): - from simplyblock_core.controllers.backup_controller import import_backups - count = import_backups([{"lvol_id": "l-1"}]) - self.assertEqual(count, 0) - - # =========================================================================== # 14. List policies # =========================================================================== class TestListPolicies(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.db_controller") + @patch("simplyblock_core.controllers.backup.policy.db_controller") def test_list_with_policies(self, mock_db): p = _policy(max_versions=5) p.max_age_display = "2d" mock_db.get_backup_policies.return_value = [p] - from simplyblock_core.controllers.backup_controller import list_policies + from simplyblock_core.controllers.backup.policy import list_policies data = list_policies() self.assertEqual(len(data), 1) @@ -1104,9 +1097,15 @@ def test_bdev_s3_create_exists(self): from simplyblock_core.rpc_client import RPCClient self.assertTrue(hasattr(RPCClient, 'bdev_s3_create')) - def test_bdev_s3_add_bucket_name_exists(self): + def test_bdev_s3_add_bucket_name_is_gone(self): + """The bucket is a create parameter now; the separate call was the + mechanism behind the non-functional source switch.""" + from simplyblock_core.rpc_client import RPCClient + self.assertFalse(hasattr(RPCClient, 'bdev_s3_add_bucket_name')) + + def test_bdev_s3_delete_exists(self): from simplyblock_core.rpc_client import RPCClient - self.assertTrue(hasattr(RPCClient, 'bdev_s3_add_bucket_name')) + self.assertTrue(hasattr(RPCClient, 'bdev_s3_delete')) def test_bdev_lvol_s3_bdev_exists(self): from simplyblock_core.rpc_client import RPCClient @@ -1198,20 +1197,28 @@ def test_backup_handlers_exist(self): # =========================================================================== -# 20. _write_s3_metadata +# 20. Backup.get_location # =========================================================================== -class TestWriteS3Metadata(unittest.TestCase): +class TestBackupLocationAccessor(unittest.TestCase): - def test_metadata_stored_on_backup(self): - from simplyblock_core.controllers.backup_controller import _write_s3_metadata + def test_recorded_location_round_trips(self): b = _backup() - meta = _write_s3_metadata(None, b) + b.location = {"bucket_name": "backups", "region": "eu-central-1"} + + location = b.get_location() + self.assertEqual(location.bucket_name, "backups") + self.assertEqual(location.region, "eu-central-1") + + def test_backup_without_a_location_raises(self): + with self.assertRaises(ValueError): + _backup().get_location() - self.assertEqual(meta["backup_id"], "backup-1") - self.assertEqual(meta["lvol_id"], "lvol-1") - self.assertEqual(meta["snapshot_id"], "snap-1") - self.assertEqual(b.s3_metadata, meta) + def test_invalid_location_raises_value_error(self): + b = _backup() + b.location = {"bucket_name": "backups"} # no region + with self.assertRaises(ValueError): + b.get_location() # =========================================================================== @@ -1220,9 +1227,9 @@ def test_metadata_stored_on_backup(self): class TestTriggerMerge(unittest.TestCase): - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") def test_trigger_merge_marks_old_as_merging(self, mock_tasks): - from simplyblock_core.controllers.backup_controller import _trigger_merge + from simplyblock_core.controllers.backup.policy import _trigger_merge keep = _backup(uuid="keep") old = _backup(uuid="old") old.write_to_db = MagicMock() @@ -1233,9 +1240,9 @@ def test_trigger_merge_marks_old_as_merging(self, mock_tasks): old.write_to_db.assert_called_once() mock_tasks.add_backup_merge_task.assert_called_once() - @patch("simplyblock_core.controllers.backup_controller.tasks_controller") + @patch("simplyblock_core.controllers.backup.policy.tasks_controller") def test_skip_if_not_completed(self, mock_tasks): - from simplyblock_core.controllers.backup_controller import _trigger_merge + from simplyblock_core.controllers.backup.policy import _trigger_merge keep = _backup(uuid="keep") old = _backup(uuid="old", status=Backup.STATUS_PENDING) diff --git a/tests/integration/test_backup_encryption.py b/tests/integration/test_backup_encryption.py new file mode 100644 index 000000000..c6a9f0167 --- /dev/null +++ b/tests/integration/test_backup_encryption.py @@ -0,0 +1,206 @@ +"""Encrypted backups: what is recorded about their key, and what a restore reaches. + +An encrypted backup is ciphertext in a bucket; the key is in a KMS. Nothing used +to record which KMS, so the dependency was implicit and only discovered during a +recovery. These tests pin down that it is written down now, that it never carries +key material, and that a restore which cannot reach the key fails instead of +producing a plaintext volume over ciphertext. +""" +import pytest + +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.kms import LocalKMS, backup_dek_path, backup_kek_name +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings + + +CLUSTER_ID = "cluster-1" +KEYS = ("a" * 64, "b" * 64) + + +def _config(**overrides): + return BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", + "region": "eu-central-1", + **overrides, + }) + + +@pytest.fixture +def db(): + return DBController() + + +def _cluster(db, **config_overrides): + c = Cluster() + c.uuid = CLUSTER_ID + c.backup_config = _config(**config_overrides).model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +def _descriptor(**overrides): + return { + "kms": "local", + "dek_path": backup_dek_path(CLUSTER_ID, "b-1"), + "kek_name": backup_kek_name("b-1"), + **overrides, + } + + +def _backup(db, uuid="b-1", encrypted=True, encryption=None): + b = Backup() + b.uuid = uuid + b.s3_id = 1 + b.cluster_id = CLUSTER_ID + b.lvol_id = "lvol-1" + b.lvol_name = "vol" + b.size = 4096 + b.status = Backup.STATUS_COMPLETED + b.location = _config().location().model_dump(exclude_none=True) + b.encrypted = encrypted + b.encryption = encryption or {} + b.write_to_db(db.kv_store) + return b + + +class TestKeyDescriptor: + + def test_local_kms_is_recorded_as_such(self, db): + cluster = _cluster(db) + backup = _backup(db) + + encryption = backup_controller._build_encryption(cluster, backup) + + assert encryption.descriptor.kms == "local" + + def test_vault_settings_are_recorded(self, db): + cluster = _cluster(db) + cluster.hashicorp_vault_settings = HashicorpVaultSettings() + cluster.hashicorp_vault_settings.base_url = "https://vault.example.com" + cluster.hashicorp_vault_settings.transit_mount = "sb/transit" + cluster.hashicorp_vault_settings.kv_mount = "sb/kv" + backup = _backup(db) + + encryption = backup_controller._build_encryption(cluster, backup) + + assert encryption.descriptor.kms == "hashicorp_vault" + assert encryption.descriptor.vault_base_url == "https://vault.example.com" + assert encryption.descriptor.transit_mount == "sb/transit" + assert encryption.descriptor.kv_mount == "sb/kv" + + def test_local_kms_records_no_vault_mounts(self, db): + """Absent rather than "" -- they mean nothing for this backend.""" + encryption = backup_controller._build_encryption(_cluster(db), _backup(db)) + + assert encryption.descriptor.vault_base_url is None + assert encryption.descriptor.transit_mount is None + assert encryption.descriptor.kv_mount is None + + def test_descriptor_points_at_the_key_path(self, db): + cluster = _cluster(db) + backup = _backup(db) + + encryption = backup_controller._build_encryption(cluster, backup) + + assert encryption.descriptor.dek_path == backup_dek_path(CLUSTER_ID, "b-1") + assert encryption.descriptor.kek_name == backup_kek_name("b-1") + + def test_descriptor_carries_no_key_material(self, db): + cluster = _cluster(db) + backup = _backup(db) + with LocalKMS(cluster) as kms: + kms.import_data_encryption_keys( + backup_dek_path(CLUSTER_ID, "b-1"), backup_kek_name("b-1"), KEYS) + + encryption = backup_controller._build_encryption(cluster, backup) + + assert KEYS[0] not in encryption.model_dump_json() + + def test_an_unknown_backend_is_refused_rather_than_guessed(self, db): + """Which fields of a descriptor mean anything depends on the backend.""" + with pytest.raises(ValueError): + backup_manifest.KeyDescriptor.model_validate(_descriptor(kms="something-new")) + + +class TestEncryptionDocument: + + def test_an_encrypted_backup_must_say_where_its_key_is(self): + """Without that, nothing can decrypt it and nothing can say why.""" + with pytest.raises(ValueError): + backup_manifest.Encryption(encrypted=True) + + def test_an_unencrypted_backup_has_no_key_to_describe(self): + with pytest.raises(ValueError): + backup_manifest.Encryption.model_validate( + {"encrypted": False, "descriptor": _descriptor()}) + + def test_manifest_cannot_disagree_with_the_backup(self, db): + """Backup.encrypted is authoritative; the stored sub-document is a copy. + + Two places recording the same fact can drift, and drift here means a + restore reads the wrong one. build_manifest overlays the authoritative + value so a manifest can never carry the stale copy. + """ + _cluster(db) + backup = _backup(db, encrypted=True, + encryption={"encrypted": False, "descriptor": _descriptor()}) + + manifest = backup_controller.build_manifest(backup) + + assert manifest.encryption.encrypted is True + + def test_survives_a_manifest_round_trip(self, db): + cluster = _cluster(db) + backup = _backup(db) + backup.encryption = backup_controller._build_encryption( + cluster, backup).model_dump(exclude_none=True) + backup.write_to_db(db.kv_store) + + parsed = backup_manifest.BackupManifest.model_validate( + backup_controller.build_manifest(backup).model_dump(mode="json")) + + assert parsed.encryption.descriptor.dek_path == backup_dek_path(CLUSTER_ID, "b-1") + + +class TestKeyResolutionOnRestore: + + def test_unencrypted_backup_needs_no_key(self, db): + cluster = _cluster(db) + assert backup_controller._resolve_crypto_key( + _backup(db, encrypted=False), cluster) is None + + def test_the_key_comes_from_the_kms_the_descriptor_names(self, db): + cluster = _cluster(db) + backup = _backup(db, encryption={"encrypted": True, "descriptor": _descriptor()}) + with LocalKMS(cluster) as kms: + kms.import_data_encryption_keys( + backup_dek_path(CLUSTER_ID, "b-1"), backup_kek_name("b-1"), KEYS) + + assert backup_controller._resolve_crypto_key(backup, cluster) == KEYS + + def test_unreachable_key_names_what_is_missing(self, db): + """The operator needs to know which cluster and KMS held the key.""" + cluster = _cluster(db) + backup = _backup(db, encryption={ + "encrypted": True, + "descriptor": _descriptor(dek_path="cluster/gone/backup/b-1")}) + + with pytest.raises(RuntimeError) as excinfo: + backup_controller._resolve_crypto_key(backup, cluster) + + message = str(excinfo.value) + assert "cluster/gone/backup/b-1" in message + assert "local" in message + + def test_encrypted_backup_with_no_encryption_record_is_refused(self, db): + """Rather than silently restoring a plaintext volume over ciphertext.""" + cluster = _cluster(db) + backup = _backup(db, encrypted=True, encryption={}) + + with pytest.raises(PreconditionError, match="records nothing about its"): + backup_controller._resolve_crypto_key(backup, cluster) diff --git a/tests/integration/test_backup_manifest_flow.py b/tests/integration/test_backup_manifest_flow.py new file mode 100644 index 000000000..30f72afdf --- /dev/null +++ b/tests/integration/test_backup_manifest_flow.py @@ -0,0 +1,302 @@ +"""Manifest assembly and the export -> import round-trip, against real FoundationDB. + +The point of a manifest is that a backup can be understood without the cluster +that wrote it, so the test that matters is: build manifests, throw the database +away, and rebuild usable Backup records from the manifests alone. + +Only boto3 is mocked -- it is an external service client. The database is real. +""" +from unittest.mock import patch + +import pytest + +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.pool import Pool + + +CLUSTER_ID = "cluster-1" + + +def _config(**overrides): + return BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", + "region": "eu-central-1", + **overrides, + }) + + +@pytest.fixture +def db(): + return DBController() + + +@pytest.fixture +def cluster(db): + c = Cluster() + c.uuid = CLUSTER_ID + c.cluster_name = "primary" + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +@pytest.fixture +def lvol(db): + pool = Pool() + pool.uuid = "pool-1" + pool.pool_name = "testpool" + pool.cluster_id = CLUSTER_ID + pool.write_to_db(db.kv_store) + + volume = LVol() + volume.uuid = "lvol-1" + volume.lvol_name = "vol" + volume.pool_uuid = "pool-1" + volume.pool_name = "testpool" + volume.node_id = "node-1" + volume.size = 4096 + volume.ha_type = "ha" + volume.fabric = "tcp" + volume.rw_ios_per_sec = 5000 + volume.max_size = 8192 + volume.write_to_db(db.kv_store) + return volume + + +def _backup(db, uuid, s3_id, prev="", **overrides): + b = Backup() + b.uuid = uuid + b.s3_id = s3_id + b.cluster_id = CLUSTER_ID + b.lvol_id = "lvol-1" + b.lvol_name = "vol" + b.snapshot_id = f"snap-{uuid}" + b.snapshot_name = f"snap_{uuid}" + b.node_id = "node-1" + b.pool_uuid = "pool-1" + b.prev_backup_id = prev + b.size = 4096 + b.created_at = 1000 + s3_id + b.completed_at = 2000 + s3_id + b.allowed_hosts = [{"nqn": "nqn.2024-01.io.test:host"}] + b.status = Backup.STATUS_COMPLETED + b.location = _config().location().model_dump(mode="json") + for key, value in overrides.items(): + setattr(b, key, value) + b.write_to_db(db.kv_store) + return b + + +class TestBuildManifest: + + def test_records_the_backup_location(self, db, cluster, lvol): + manifest = backup_controller.build_manifest(_backup(db, "b-1", 1)) + + assert manifest.location.bucket_name == "simplyblock-backup-cluster-1" + assert manifest.location.region == "eu-central-1" + + def test_records_only_the_immediate_predecessor(self, db, cluster, lvol): + """Not the whole chain: storing that would make a merge rewrite the + manifest of every descendant of the backup it folded away.""" + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, prev="b-1") + third = _backup(db, "b-3", 3, prev="b-2") + + manifest = backup_controller.build_manifest(third) + + assert manifest.prev_backup_id == "b-2" + + def test_a_full_backup_has_no_predecessor(self, db, cluster, lvol): + """Absent rather than "", so a chain root is a state and not a blank.""" + assert backup_controller.build_manifest(_backup(db, "b-1", 1)).prev_backup_id is None + + def test_the_chain_reconstructs_from_the_links(self, db, cluster, lvol): + """What the stored chain was for, done from the bucket's own contents.""" + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, prev="b-1") + _backup(db, "b-3", 3, prev="b-2") + manifests = [backup_controller.build_manifest(b) for b in db.get_backups()] + last = next(m for m in manifests if m.backup_id == "b-3") + + chain = backup_manifest.chain_of(last, manifests) + + assert [(m.backup_id, m.s3_id) for m in chain] == [ + ("b-1", 1), ("b-2", 2), ("b-3", 3)] + + def test_records_the_volume_shape(self, db, cluster, lvol): + """Restore currently hardcodes these; recording them is what lets it stop.""" + manifest = backup_controller.build_manifest(_backup(db, "b-1", 1)) + + assert manifest.volume.pool_name == "testpool" + assert manifest.volume.ha_type == "ha" + assert manifest.volume.rw_ios_per_sec == 5000 + assert manifest.volume.max_size == 8192 + assert manifest.volume.allowed_hosts == [{"nqn": "nqn.2024-01.io.test:host"}] + + def test_survives_a_deleted_volume(self, db, cluster, lvol): + """A backup outlives its volume; that must not stop the manifest.""" + backup = _backup(db, "b-1", 1) + lvol.remove(db.kv_store) + + manifest = backup_controller.build_manifest(backup) + + assert manifest.volume.lvol_name == "vol" # carried on the backup itself + assert manifest.volume.pool_name is None # only on the volume + # Absent, not 0 -- which for a QoS cap would read as "unlimited". + assert manifest.volume.rw_ios_per_sec is None + + def test_records_source_as_provenance(self, db, cluster, lvol): + manifest = backup_controller.build_manifest(_backup(db, "b-1", 1)) + + assert manifest.source.cluster_id == CLUSTER_ID + assert manifest.source.cluster_name == "primary" + assert manifest.source.node_id == "node-1" + + def test_records_the_object_layout(self, db, cluster, lvol): + manifest = backup_controller.build_manifest(_backup(db, "b-1", 1)) + + assert manifest.dataplane.key_format == "{s3_id}/{mid}/{extent}" + assert manifest.dataplane.cluster_size == cluster.page_size_in_blocks + + def test_backup_without_a_location_is_refused(self, db, cluster, lvol): + backup = _backup(db, "b-1", 1) + backup.location = {} + + with pytest.raises(ValueError): + backup_controller.build_manifest(backup) + + +class TestExportImportRoundTrip: + """Export, wipe the database, import -- the disaster-recovery shape.""" + + def test_backups_survive_losing_the_database(self, db, cluster, lvol): + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, prev="b-1", encrypted=True, + encryption={"encrypted": True, "descriptor": {"kms": "local"}}) + + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + for backup in db.get_backups(): + backup.remove(db.kv_store) + assert db.get_backups() == [] + + count = backup_controller.import_backups(exported, cluster_id="cluster-2") + + assert count == 2 + restored = db.get_backup_by_id("b-2") + assert restored.s3_id == 2 + assert restored.prev_backup_id == "b-1" + assert restored.cluster_id == "cluster-2" + assert restored.get_location().bucket_name == "simplyblock-backup-cluster-1" + assert restored.encrypted is True + + def test_encrypted_flag_survives(self, db, cluster, lvol): + """It used to be dropped, restoring a plaintext volume over ciphertext.""" + _backup(db, "b-1", 1, encrypted=True, + encryption={"encrypted": True, "descriptor": {"kms": "local"}}) + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + db.get_backup_by_id("b-1").remove(db.kv_store) + + backup_controller.import_backups(exported, cluster_id="cluster-2") + + assert db.get_backup_by_id("b-1").encrypted is True + + def test_export_emits_the_same_shape_as_the_bucket(self, db, cluster, lvol): + """One format, so a file and a bucket read are interchangeable.""" + _backup(db, "b-1", 1) + + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + + assert backup_manifest._parse( + exported[0].model_dump_json().encode(), "k") == exported[0] + + def test_only_completed_backups_are_exported(self, db, cluster, lvol): + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, status=Backup.STATUS_FAILED) + + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + + assert [m.backup_id for m in exported] == ["b-1"] + + def test_a_malformed_entry_never_reaches_the_controller(self, db, cluster, lvol): + """import_backups takes manifests, not dicts, so an unusable entry is + rejected by whoever read the bytes -- naming the file or the request.""" + with pytest.raises(ValueError): + backup_manifest.BackupManifest.model_validate( + {"backup_id": "b-9", "s3_id": "not-an-int"}) + + def test_duplicate_id_rejects_the_whole_batch(self, db, cluster, lvol): + _backup(db, "b-1", 1) + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + + with pytest.raises(PreconditionError, match="already exists"): + backup_controller.import_backups(exported, cluster_id="cluster-2") + + def test_same_id_listed_twice_rejects_the_whole_batch(self, db, cluster, lvol): + """Backup lookups are not cluster-scoped, so a reused uuid unaddresses both.""" + _backup(db, "b-1", 1) + exported = backup_controller.export_backups(cluster_id=CLUSTER_ID) + db.get_backup_by_id("b-1").remove(db.kv_store) + + with pytest.raises(ValueError, match="listed more than once"): + backup_controller.import_backups(exported + exported, cluster_id="cluster-2") + + assert db.get_backups() == [] + + def test_nothing_to_import_is_not_an_error(self, db, cluster, lvol): + assert backup_controller.import_backups([], cluster_id="cluster-2") == 0 + + +class TestBucketDiscovery: + """The path that needs nothing but a bucket and credentials.""" + + def test_discover_reads_every_manifest(self, db, cluster, lvol): + _backup(db, "b-1", 1) + _backup(db, "b-2", 2, prev="b-1") + manifests = [backup_controller.build_manifest(b) for b in db.get_backups()] + + with patch.object(backup_manifest, "list_all", return_value=manifests): + found = backup_controller.discover_backups(_config()) + + assert {m.backup_id for m in found} == {"b-1", "b-2"} + # Models, not dicts: rendering them is the caller's decision. + assert all(isinstance(m, backup_manifest.BackupManifest) for m in found) + + def test_import_from_bucket_needs_no_prior_records(self, db, cluster, lvol): + _backup(db, "b-1", 1) + manifests = [backup_controller.build_manifest(db.get_backup_by_id("b-1"))] + db.get_backup_by_id("b-1").remove(db.kv_store) + + with patch.object(backup_manifest, "list_all", return_value=manifests): + count = backup_controller.import_from_bucket(_config(), cluster_id="cluster-2") + + assert count == 1 + assert db.get_backup_by_id("b-1").cluster_id == "cluster-2" + + +class TestManifestPublication: + + def test_write_manifest_puts_the_object_in_the_backup_bucket(self, db, cluster, lvol): + backup = _backup(db, "b-1", 1) + + with patch.object(backup_manifest, "s3_client") as mock_client: + backup_controller.write_manifest(backup) + + _, kwargs = mock_client.return_value.put_object.call_args + assert kwargs["Bucket"] == "simplyblock-backup-cluster-1" + assert kwargs["Key"] == "manifests/b-1.json" + + def test_refuses_when_the_cluster_points_at_another_bucket(self, db, cluster, lvol): + """The cluster's credentials cannot be assumed to reach a foreign bucket.""" + backup = _backup(db, "b-1", 1) + backup.location = _config(bucket_name="somewhere-else").location().model_dump(mode="json") + backup.write_to_db(db.kv_store) + + with pytest.raises(PreconditionError, match="somewhere-else"): + backup_controller.write_manifest(backup) diff --git a/tests/integration/test_backup_restore_source.py b/tests/integration/test_backup_restore_source.py new file mode 100644 index 000000000..1f71e2333 --- /dev/null +++ b/tests/integration/test_backup_restore_source.py @@ -0,0 +1,241 @@ +"""Restoring from a bucket that is not the cluster's own, against real FoundationDB. + +This is the disaster-recovery path: the backup's own recorded location says +where its objects are, and the node gets a second S3 device pointed at that +bucket for the duration of the restore. + +The device's whole lifecycle belongs to the task runner, not to the restore +request -- the runner is the only component that knows which node the volume +landed on, and the only one that can put the device back after a node restart +mid-restore. +""" +from unittest.mock import MagicMock, patch + +import pytest + +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import device as backup_device +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig, S3Credentials +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.storage_node import StorageNode +from simplyblock_core.services import tasks_runner_backup + + +CLUSTER_ID = "cluster-1" +OWN_BUCKET = "simplyblock-backup-cluster-1" +FOREIGN_BUCKET = "someone-elses-bucket" + + +def _config(bucket=OWN_BUCKET, **overrides): + return BackupConfig.model_validate({ + "bucket_name": bucket, "region": "eu-central-1", **overrides}) + + +@pytest.fixture +def db(): + return DBController() + + +@pytest.fixture +def cluster(db): + c = Cluster() + c.uuid = CLUSTER_ID + c.backup_config = _config( + credentials={"access_key_id": "own", "secret_access_key": "own"} + ).model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +@pytest.fixture +def node(db): + n = StorageNode() + n.uuid = "node-1" + n.cluster_id = CLUSTER_ID + n.status = StorageNode.STATUS_ONLINE + n.lvstore = "lvs_test" + n.mgmt_ip = "10.0.0.1" + n.rpc_port = 5260 + n.app_thread_mask = "0x8" + n.cpu = 8 + n.write_to_db(db.kv_store) + return n + + +def _backup(db, bucket=OWN_BUCKET, uuid="b-1"): + b = Backup() + b.uuid = uuid + b.s3_id = 1 + b.cluster_id = CLUSTER_ID + b.lvol_id = "lvol-1" + b.size = 4096 + b.status = Backup.STATUS_COMPLETED + b.location = _config(bucket).location().model_dump(mode="json") + b.write_to_db(db.kv_store) + return b + + +class TestSourceSelection: + + def test_own_bucket_needs_no_new_device(self, db, cluster, node): + assert backup_controller.foreign_bucket_config( + _backup(db), cluster, None) is None + + def test_foreign_bucket_yields_its_own_config(self, db, cluster, node): + config = backup_controller.foreign_bucket_config( + _backup(db, FOREIGN_BUCKET), cluster, + S3Credentials(access_key_id="theirs", secret_access_key="theirs")) + + assert config is not None + assert config.bucket_name == FOREIGN_BUCKET + assert config.credentials.access_key_id.get_secret_value() == "theirs" + + def test_foreign_bucket_without_credentials_is_refused(self, db, cluster, node): + """The cluster's own static keys say nothing about someone else's bucket.""" + with pytest.raises(PreconditionError, match="not this cluster's own"): + backup_controller.foreign_bucket_config( + _backup(db, FOREIGN_BUCKET), cluster, None) + + def test_instance_role_cluster_may_reach_a_foreign_bucket(self, db, node): + """With no static credentials anywhere, the node's role is the only answer.""" + c = Cluster() + c.uuid = CLUSTER_ID + c.backup_config = _config().model_dump(exclude_none=True) # no credentials + c.write_to_db(db.kv_store) + + config = backup_controller.foreign_bucket_config( + _backup(db, FOREIGN_BUCKET), c, None) + + assert config.bucket_name == FOREIGN_BUCKET + assert config.credentials is None + + def test_explicit_credentials_override_the_own_bucket_shortcut(self, db, cluster, node): + """Restoring the cluster's own bucket with other credentials is legitimate.""" + config = backup_controller.foreign_bucket_config( + _backup(db), cluster, + S3Credentials(access_key_id="other", secret_access_key="other")) + + assert config is not None + assert config.bucket_name == OWN_BUCKET + + def test_device_name_is_derived_from_the_backup(self, db): + """Stable across retries, so an attempt cannot leak a device per try.""" + assert (backup_device.restore_s3_bdev_name("b-1234567890") + == backup_device.restore_s3_bdev_name("b-1234567890")) + assert backup_device.restore_s3_bdev_name( + "b-1234567890") != backup_device.restore_s3_bdev_name("c-1234567890") + + +def _restore_task(db, s3_config=None, **params): + task = JobSchedule() + task.uuid = "task-1" + task.cluster_id = CLUSTER_ID + task.node_id = "node-1" + task.function_name = JobSchedule.FN_BACKUP_RESTORE + task.status = JobSchedule.STATUS_RUNNING + task.function_params = { + "backup_id": "b-1", + "lvol_name": "lvs_test/LVOL_1", + "lvol_id": "", + "chain_ids": [1], + "s3_config": s3_config, + **params, + } + task.write_to_db(db.kv_store) + return task + + +class TestRunnerOwnsTheDevice: + + def test_own_bucket_creates_nothing(self, db, cluster, node): + task = _restore_task(db) + + with patch.object(backup_device, "create_restore_s3_bdev") as create: + tasks_runner_backup._ensure_restore_s3_bdev(task, node) + + create.assert_not_called() + + def test_foreign_bucket_device_is_created_by_the_runner(self, db, cluster, node): + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + with patch.object(backup_device, "create_restore_s3_bdev") as create: + tasks_runner_backup._ensure_restore_s3_bdev(task, node) + + _, kwargs = create.call_args + args = create.call_args[0] + assert args[0] is node + assert args[1].bucket_name == FOREIGN_BUCKET + assert args[2] == backup_device.restore_s3_bdev_name("b-1") + + def test_creation_is_idempotent_across_retries(self, db, cluster, node): + """A node restart mid-restore takes the device with it; the runner rebuilds it.""" + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + with patch.object(backup_device, "create_restore_s3_bdev") as create: + tasks_runner_backup._ensure_restore_s3_bdev(task, node) + tasks_runner_backup._ensure_restore_s3_bdev(task, node) + + assert create.call_count == 2 + assert {c[0][2] for c in create.call_args_list} == { + backup_device.restore_s3_bdev_name("b-1")} + + def test_release_deletes_the_device(self, db, cluster, node): + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + with patch.object(backup_device, "delete_restore_s3_bdev") as delete: + tasks_runner_backup._release_restore_s3_bdev(task, node) + + delete.assert_called_once_with( + node, backup_device.restore_s3_bdev_name("b-1")) + + def test_release_scrubs_the_credentials(self, db, cluster, node): + """A task record outlives the restore by weeks; foreign keys must not.""" + task = _restore_task(db, s3_config=_config( + FOREIGN_BUCKET, + credentials={"access_key_id": "theirs", "secret_access_key": "theirs"}, + ).model_dump(exclude_none=True)) + + with patch.object(backup_device, "delete_restore_s3_bdev"): + tasks_runner_backup._release_restore_s3_bdev(task, node) + + assert task.function_params["s3_config"] is None + + def test_release_is_a_noop_for_the_own_bucket(self, db, cluster, node): + """The node's own device is shared; a restore must never delete it.""" + task = _restore_task(db) + + with patch.object(backup_device, "delete_restore_s3_bdev") as delete: + tasks_runner_backup._release_restore_s3_bdev(task, node) + + delete.assert_not_called() + + def test_release_survives_a_missing_node(self, db, cluster): + """Cleanup runs on terminal paths, including ones reached because the node is gone.""" + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + tasks_runner_backup._release_restore_s3_bdev(task, None) + + assert task.function_params["s3_config"] is None + + def test_delete_failure_does_not_raise(self, db, cluster, node): + """A cleanup failure must not turn a completed restore into a failed one.""" + node.rpc_client = MagicMock() + node.rpc_client.return_value.bdev_s3_delete.side_effect = RuntimeError("gone") + + backup_controller.delete_restore_s3_bdev(node, "s3_restore_b-1") + + def test_recovery_names_the_device_it_reads_from(self, db, cluster, node): + task = _restore_task(db, s3_config=_config(FOREIGN_BUCKET).model_dump(exclude_none=True)) + + assert tasks_runner_backup._restore_s3_bdev(task, node) == \ + backup_device.restore_s3_bdev_name("b-1") + + def test_recovery_falls_back_to_the_nodes_own_device(self, db, cluster, node): + task = _restore_task(db) + + assert tasks_runner_backup._restore_s3_bdev(task, node) == \ + backup_device.primary_s3_bdev_name(node) diff --git a/tests/integration/test_backup_s3_id_allocation.py b/tests/integration/test_backup_s3_id_allocation.py new file mode 100644 index 000000000..9cfe5fabd --- /dev/null +++ b/tests/integration/test_backup_s3_id_allocation.py @@ -0,0 +1,73 @@ +"""s3_id allocation against real FoundationDB. + +An s3_id names a backup's object keys in S3 (``{s3_id}/{mid}/{extent}``), and +nothing on the data plane reclaims those objects. Reusing an id therefore aims a +new backup's writes at another backup's keys, so the properties worth pinning are +monotonicity and non-reuse -- not just "returns a number". +""" +import json + +import pytest + +from simplyblock_core import constants +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.backup import Backup + + +@pytest.fixture +def db(): + return DBController() + + +def _backup(uuid, cluster_id, s3_id): + backup = Backup() + backup.uuid = uuid + backup.cluster_id = cluster_id + backup.s3_id = s3_id + backup.status = Backup.STATUS_COMPLETED + return backup + + +class TestS3IdAllocation: + + def test_allocations_are_strictly_increasing(self, db): + allocated = [db.next_s3_id() for _ in range(5)] + assert allocated == sorted(allocated) + assert len(set(allocated)) == len(allocated) + + def test_first_allocation_is_usable(self, db): + """0 is rejected by the data plane (s3_id == 0 -> -EINVAL).""" + assert db.next_s3_id() > 0 + + def test_seeds_above_pre_existing_backups(self, db): + """Upgrade path: ids handed out by the old max-plus-one allocator must not repeat.""" + _backup("b-legacy", "cl-1", 42).write_to_db(db.kv_store) + + assert db.next_s3_id() > 42 + + def test_seeds_above_imported_foreign_backups(self, db): + """Imported backups keep their originating cluster's ids; seeding ignores cluster scope.""" + _backup("b-foreign", "cl-other", 900).write_to_db(db.kv_store) + + assert db.next_s3_id() > 900 + + def test_deleting_a_backup_does_not_recycle_its_id(self, db): + """The old allocator recycled the top id, aiming new writes at orphaned objects.""" + first = db.next_s3_id() + backup = _backup("b-1", "cl-1", first) + backup.write_to_db(db.kv_store) + backup.remove(db.kv_store) + + assert db.next_s3_id() > first + + def test_exhaustion_is_reported_not_wrapped(self, db): + """The data plane masks s3_id to 30 bits, so an overflow would silently alias.""" + db.kv_store[DBController._S3_ID_SEQ_KEY] = json.dumps( + constants.BACKUP_MAX_S3_ID).encode() + + with pytest.raises(ValueError, match="exhausted"): + db.next_s3_id() + + def test_max_s3_id_matches_the_data_plane_field_width(self): + """S3_ID_BITS is 30 in spdk_internal/lvolstore.h; a wider value aliases.""" + assert constants.BACKUP_MAX_S3_ID == (1 << 30) - 1 diff --git a/tests/integration/test_backup_validation.py b/tests/integration/test_backup_validation.py new file mode 100644 index 000000000..82434766c --- /dev/null +++ b/tests/integration/test_backup_validation.py @@ -0,0 +1,349 @@ +"""Preconditions on backup creation, restore and import, against real FoundationDB. + +Every rule here has the same shape: it must fire *before* any side effect. A +refused backup must leave no Backup record, no KMS key and no task; a refused +restore must leave no volume. The point is that a backup either is restorable or +was never created -- discovering the problem during a recovery is too late. +""" +from unittest.mock import patch + +import pytest + +from simplyblock_core import constants +from simplyblock_core.controllers.backup import controller as backup_controller +from simplyblock_core.controllers.backup import validation +from simplyblock_core.controllers.backup.manifest import BackupManifest +from simplyblock_core.db_controller import DBController +from simplyblock_core.exceptions import PreconditionError +from simplyblock_core.models.backup import Backup +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.lvol_model import LVol +from simplyblock_core.models.pool import Pool +from simplyblock_core.models.snapshot import SnapShot +from simplyblock_core.models.storage_node import StorageNode + + +CLUSTER_ID = "cluster-1" + + +def _config(**overrides): + return BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup-cluster-1", + "region": "eu-central-1", + **overrides, + }) + + +@pytest.fixture +def db(): + return DBController() + + +@pytest.fixture +def cluster(db): + c = Cluster() + c.uuid = CLUSTER_ID + c.backup_config = _config().model_dump(exclude_none=True) + c.write_to_db(db.kv_store) + return c + + +@pytest.fixture +def pool(db): + p = Pool() + p.uuid = "pool-1" + p.pool_name = "pool-1" # resolved by name: "pool-1" is not a UUID + p.cluster_id = CLUSTER_ID + p.write_to_db(db.kv_store) + return p + + +@pytest.fixture +def node(db): + n = StorageNode() + n.uuid = "node-1" + n.cluster_id = CLUSTER_ID + n.status = StorageNode.STATUS_ONLINE + n.lvstore = "lvs_test" + n.mgmt_ip = "10.0.0.1" + n.rpc_port = 5260 + n.write_to_db(db.kv_store) + return n + + +def _snapshot(db, uuid="snap-1", crypto=False): + volume = LVol() + volume.uuid = "lvol-1" + volume.lvol_name = "vol" + volume.node_id = "node-1" + volume.lvs_name = "lvs_test" + volume.pool_uuid = "pool-1" + volume.size = 4096 + if crypto: + volume.crypto_bdev = "crypto_lvol-1" + volume.write_to_db(db.kv_store) + + s = SnapShot() + s.uuid = uuid + s.snap_uuid = uuid + s.snap_name = uuid + s.snap_bdev = f"lvs_test/{uuid}" + s.size = 4096 + s.status = SnapShot.STATUS_ONLINE + s.lvol = volume + s.write_to_db(db.kv_store) + return s + + +def _backup(db, uuid, s3_id, snapshot_id, prev="", location=None, encrypted=False): + b = Backup() + b.uuid = uuid + b.s3_id = s3_id + b.cluster_id = CLUSTER_ID + b.lvol_id = "lvol-1" + b.lvol_name = "vol" + b.snapshot_id = snapshot_id + b.prev_backup_id = prev + b.size = 4096 + b.status = Backup.STATUS_COMPLETED + b.location = (location or _config().location()).model_dump(exclude_none=True) + b.encrypted = encrypted + b.write_to_db(db.kv_store) + return b + + +def _assert_no_side_effects(db): + assert db.get_backups() == [], "a refused backup must leave no record" + assert db.get_job_tasks(CLUSTER_ID) == [], "a refused backup must enqueue no task" + + +class TestBackupCreationPreconditions: + + def test_missing_backup_config_is_refused(self, db, cluster, node): + cluster.backup_config = {} + cluster.write_to_db(db.kv_store) + _snapshot(db) + + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "backup configuration" in error + _assert_no_side_effects(db) + + def test_tiering_layout_bucket_is_refused(self, db, cluster, node): + """snapshot_backups=False selects {tiering_id}/{lpgi}, which restore cannot read.""" + cluster.backup_config = _config(snapshot_backups=False).model_dump(exclude_none=True) + cluster.write_to_db(db.kv_store) + _snapshot(db) + + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "snapshot_backups disabled" in error + _assert_no_side_effects(db) + + def test_overlong_chain_is_refused(self, db, cluster, node): + """Longer than the data plane's fixed arrays, where it smashes the stack.""" + snap = _snapshot(db) + too_long = [snap] * (constants.BACKUP_MAX_CHAIN_LENGTH + 1) + + with patch.object(backup_controller, "_get_snapshot_chain", return_value=too_long): + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "data plane accepts at most" in error + _assert_no_side_effects(db) + + def test_chain_in_another_bucket_is_refused(self, db, cluster, node): + """The cluster's bucket changed since the ancestors were written.""" + snap = _snapshot(db) + _backup(db, "b-old", 1, snapshot_id="snap-0", + location=_config(bucket_name="the-old-bucket").location()) + + with patch.object(backup_controller, "_get_snapshot_chain", + return_value=[_named(snap, "snap-0"), snap]): + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "cannot span buckets" in error + assert db.get_backups() == [db.get_backup_by_id("b-old")] + + def test_encrypted_volume_over_a_plain_chain_is_refused(self, db, cluster, node): + snap = _snapshot(db, crypto=True) + _backup(db, "b-plain", 1, snapshot_id="snap-0", encrypted=False) + + with patch.object(backup_controller, "_get_snapshot_chain", + return_value=[_named(snap, "snap-0"), snap]): + backup_id, error = backup_controller.backup_snapshot("snap-1") + + assert backup_id is None + assert "cannot mix encrypted and unencrypted" in error + + def test_refusal_happens_before_the_chain_lock(self, db, cluster, node): + """Otherwise a refused request would block the next one.""" + cluster.backup_config = {} + cluster.write_to_db(db.kv_store) + _snapshot(db) + + backup_controller.backup_snapshot("snap-1") + + assert db.get_backup_chain_lock("snap-1") is None + + +def _named(snapshot, uuid): + """A shallow copy of a snapshot under a different id, for chain fixtures.""" + clone = SnapShot() + clone.from_dict(snapshot.to_dict()) + clone.uuid = uuid + clone.snap_uuid = uuid + return clone + + +class TestRestorePreconditions: + + def test_chain_spanning_buckets_is_refused(self, db, cluster, node, pool): + _backup(db, "b-1", 1, snapshot_id="snap-1", + location=_config(bucket_name="elsewhere").location()) + _backup(db, "b-2", 2, snapshot_id="snap-2", prev="b-1") + + with pytest.raises(PreconditionError, match="cannot span buckets"): + backup_controller.restore_backup("b-2", "restored", "pool-1") + + def test_chain_mixing_encryption_is_refused(self, db, cluster, node, pool): + _backup(db, "b-1", 1, snapshot_id="snap-1", encrypted=True) + _backup(db, "b-2", 2, snapshot_id="snap-2", prev="b-1", encrypted=False) + + with pytest.raises(PreconditionError, match="mix encrypted and unencrypted"): + backup_controller.restore_backup("b-2", "restored", "pool-1") + + def test_overlong_chain_is_refused(self, db, cluster, node, pool): + previous = "" + for index in range(constants.BACKUP_MAX_CHAIN_LENGTH + 1): + previous = _backup(db, f"b-{index}", index + 1, + snapshot_id=f"snap-{index}", prev=previous).uuid + + with pytest.raises(PreconditionError, match="data plane accepts at most"): + backup_controller.restore_backup(previous, "restored", "pool-1") + + def test_no_volume_is_created_when_a_precondition_fails(self, db, cluster, node, pool): + _backup(db, "b-1", 1, snapshot_id="snap-1", encrypted=True) + _backup(db, "b-2", 2, snapshot_id="snap-2", prev="b-1", encrypted=False) + + with pytest.raises(PreconditionError): + backup_controller.restore_backup("b-2", "restored", "pool-1") + + assert db.get_lvols() == [] + + +class TestImportPreconditions: + + def _manifest(self, backup_id, prev=None, s3_id=1, + bucket="simplyblock-backup-cluster-1"): + return BackupManifest.model_validate({ + "schema_version": 1, + "backup_id": backup_id, + "s3_id": s3_id, + "created_at": 100, + "completed_at": 200, + "size": 4096, + "encrypted": False, + "prev_backup_id": prev, + "location": _config(bucket_name=bucket).location().model_dump(mode="json"), + "source": {"cluster_id": CLUSTER_ID, "node_id": "node-1"}, + "volume": {"lvol_id": "lvol-1", "lvol_name": "vol", + "snapshot_id": f"snap-{backup_id}", "snapshot_name": "s", + "size": 4096}, + "dataplane": {}, + }) + + def _line(self, length, **overrides): + """A chain of `length` manifests, oldest first.""" + line, prev = [], None + for index in range(length): + line.append(self._manifest(f"b-{index}", prev=prev, s3_id=index + 1, + **overrides)) + prev = line[-1].backup_id + return line + + def test_incomplete_chain_is_refused(self, db, cluster): + """A delta whose ancestors are missing looks restorable until it is tried.""" + with pytest.raises(PreconditionError, match="neither in this import nor already known"): + backup_controller.import_backups( + [self._manifest("b-2", prev="b-1")], cluster_id=CLUSTER_ID) + + assert db.get_backups() == [] + + def test_chain_satisfied_within_the_batch_is_accepted(self, db, cluster): + count = backup_controller.import_backups(self._line(2), cluster_id=CLUSTER_ID) + + assert count == 2 + + def test_chain_satisfied_by_existing_records_is_accepted(self, db, cluster): + _backup(db, "b-1", 1, snapshot_id="snap-1") + + count = backup_controller.import_backups( + [self._manifest("b-2", prev="b-1")], cluster_id=CLUSTER_ID) + + assert count == 1 + + def test_chain_spanning_buckets_is_refused(self, db, cluster): + with pytest.raises(PreconditionError, match="different bucket or encoding"): + backup_controller.import_backups( + [self._manifest("b-1", bucket="elsewhere"), + self._manifest("b-2", prev="b-1")], + cluster_id=CLUSTER_ID) + + assert db.get_backups() == [] + + def test_overlong_chain_is_refused(self, db, cluster): + with pytest.raises(PreconditionError, match="data plane accepts at most"): + backup_controller.import_backups( + self._line(constants.BACKUP_MAX_CHAIN_LENGTH + 1), + cluster_id=CLUSTER_ID) + + assert db.get_backups() == [] + + def test_a_chain_lengthened_past_the_limit_by_existing_records_is_refused( + self, db, cluster): + """The ancestry already in the database counts towards the limit.""" + previous = "" + for index in range(constants.BACKUP_MAX_CHAIN_LENGTH): + previous = _backup(db, f"old-{index}", index + 1, + snapshot_id=f"snap-old-{index}", prev=previous).uuid + + with pytest.raises(PreconditionError, match="data plane accepts at most"): + backup_controller.import_backups( + [self._manifest("b-new", prev=previous, s3_id=999)], + cluster_id=CLUSTER_ID) + + def test_a_cyclic_chain_is_refused_rather_than_looping(self, db, cluster): + with pytest.raises(PreconditionError, match="cyclic"): + backup_controller.import_backups( + [self._manifest("b-1", prev="b-2"), self._manifest("b-2", prev="b-1")], + cluster_id=CLUSTER_ID) + + +class TestPredicates: + """The rules answer a yes/no question as well as blocking an operation.""" + + def test_chain_fits_at_the_limit(self): + assert validation.chain_fits(constants.BACKUP_MAX_CHAIN_LENGTH) + assert not validation.chain_fits(constants.BACKUP_MAX_CHAIN_LENGTH + 1) + + def test_a_tiering_bucket_holds_no_backups(self): + assert validation.location_holds_backups(_config().location()) + assert not validation.location_holds_backups( + _config(snapshot_backups=False).location()) + + def test_an_empty_chain_is_coherent(self): + assert validation.chain_is_coherent([], _config().location()) + + def test_coherence_covers_a_backup_that_does_not_exist_yet(self, db, cluster): + chain = [_backup(db, "b-1", 1, snapshot_id="snap-1", encrypted=False)] + + assert validation.chain_is_coherent(chain, _config().location()) + assert validation.chain_is_coherent( + chain, _config().location(), encrypted=False) + assert not validation.chain_is_coherent( + chain, _config().location(), encrypted=True) diff --git a/tests/unit/rpc/test_client.py b/tests/unit/rpc/test_client.py index 1a146cfac..246bb96e5 100644 --- a/tests/unit/rpc/test_client.py +++ b/tests/unit/rpc/test_client.py @@ -91,3 +91,4 @@ def test_subsystem_get_other_rpc_error_propagates(self, mock_req): if __name__ == "__main__": unittest.main() + diff --git a/tests/unit/test_api_dto_secrets.py b/tests/unit/test_api_dto_secrets.py index 95e7685f1..47df88432 100644 --- a/tests/unit/test_api_dto_secrets.py +++ b/tests/unit/test_api_dto_secrets.py @@ -3,8 +3,8 @@ from pydantic import SecretStr -from simplyblock_web.api.v2.cluster import BackupConfigParams -from simplyblock_web.api.v2._dtos import ClusterDTO, CapacityStatDTO +from simplyblock_core.models.backup_config import BackupConfig +from simplyblock_web.api.v2._dtos import BackupConfigDTO, ClusterDTO, CapacityStatDTO from uuid import uuid4 @@ -13,27 +13,63 @@ def _build_capacity(): return CapacityStatDTO.from_model(StatsObject()) +_BACKUP_CONFIG = { + "bucket_name": "backups", + "region": "eu-central-1", + "access_key_id": "AKID", + "secret_access_key": "SK", +} + + def test_backup_config_params_carry_secretstr(): - params = BackupConfigParams.model_validate({ - "access_key_id": "AKID", - "secret_access_key": "SK", - }) - assert isinstance(params.access_key_id, SecretStr) - assert isinstance(params.secret_access_key, SecretStr) - assert params.access_key_id.get_secret_value() == "AKID" - assert params.secret_access_key.get_secret_value() == "SK" + params = BackupConfigDTO.model_validate(_BACKUP_CONFIG) + assert params.credentials is not None + assert isinstance(params.credentials.access_key_id, SecretStr) + assert isinstance(params.credentials.secret_access_key, SecretStr) + assert params.credentials.access_key_id.get_secret_value() == "AKID" + assert params.credentials.secret_access_key.get_secret_value() == "SK" def test_backup_config_repr_masks_secret_values(): - params = BackupConfigParams.model_validate({ - "access_key_id": "AKID", - "secret_access_key": "SK", - }) - text = repr(params) + text = repr(BackupConfigDTO.model_validate(_BACKUP_CONFIG)) assert "AKID" not in text assert "SK" not in text +def test_backup_config_dump_keeps_secrets_wrapped(): + """write_to_db unwraps at the last moment; anything earlier leaks into logs.""" + stored = BackupConfig.model_validate(_BACKUP_CONFIG).model_dump(exclude_none=True) + assert isinstance(stored["credentials"]["access_key_id"], SecretStr) + assert "AKID" not in repr(stored) + + +def test_backup_config_dump_is_json_serializable(): + """Cluster.backup_config is a plain dict written through BaseModel to FDB. + + A python-mode dump is normally not JSON-safe. Field serializers on the two + offenders -- the URL and the enum -- are what make this hold without a + hand-written conversion step. + """ + from simplyblock_core.models.base_model import BaseModel as CoreBaseModel + + stored = BackupConfig.model_validate({ + **_BACKUP_CONFIG, "local_endpoint": "http://minio:9000", + }).model_dump(exclude_none=True) + + class _Holder(CoreBaseModel): + backup_config: dict = {} + + holder = _Holder() + holder.backup_config = stored + persisted = json.loads(json.dumps(holder.to_dict(unwrap_secrets=True))) + + assert stored["endpoint"] == "http://minio:9000" + assert stored["secondary_target"] == 0 + assert persisted["backup_config"]["credentials"]["access_key_id"] == "AKID" + assert BackupConfig.model_validate(persisted["backup_config"]).endpoint_url == \ + "http://minio:9000" + + def _build_cluster_dto(): return ClusterDTO( id=uuid4(), diff --git a/tests/unit/test_backup_cli.py b/tests/unit/test_backup_cli.py new file mode 100644 index 000000000..86441e32d --- /dev/null +++ b/tests/unit/test_backup_cli.py @@ -0,0 +1,113 @@ +"""The backup CLI's argument handling. + +Focused on the two helpers that turn command-line arguments into a +``BackupConfig``, because that is where a disaster-recovery operator's typing +becomes the thing that decides whether a bucket can be read at all. +""" +import pytest + +from simplyblock_cli import clibase +from simplyblock_core.models.backup_config import BackupConfig + + +class _Args: + """Stand-in for the argparse namespace, with only the attributes set.""" + + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + +class TestCredentials: + + def test_absent_keys_mean_the_instance_role(self): + """Not "empty credentials" -- that is what broke IAM roles in the first place.""" + assert clibase._s3_credentials(_Args()) is None + + def test_both_keys_are_carried(self): + credentials = clibase._s3_credentials( + _Args(access_key_id="AKIA", secret_access_key="shh")) + + assert credentials.access_key_id.get_secret_value() == "AKIA" + assert credentials.secret_access_key.get_secret_value() == "shh" + + @pytest.mark.parametrize("given", [ + {"access_key_id": "AKIA"}, + {"secret_access_key": "shh"}, + ]) + def test_half_a_pair_is_refused(self, given): + """Half a key pair reaches S3 as an authentication failure with no clue why.""" + with pytest.raises(ValueError, match="both"): + clibase._s3_credentials(_Args(**given)) + + +class TestBucketConfig: + + def _args(self, **overrides): + return _Args(**{"bucket": "backups", "region": "eu-central-1", **overrides}) + + def test_minimal_bucket(self): + config = clibase._bucket_config(self._args()) + + assert config.bucket_name == "backups" + assert config.region == "eu-central-1" + assert config.endpoint is None + assert config.credentials is None + assert config.verify_tls is True + assert config.use_path_style is False + + def test_endpoint_and_addressing(self): + config = clibase._bucket_config(self._args( + endpoint="http://minio:9000", path_style=True, no_verify_tls=True)) + + assert config.endpoint_url == "http://minio:9000" + assert config.use_path_style is True + assert config.verify_tls is False + + def test_empty_endpoint_is_absent_not_blank(self): + """argparse gives "" for an unset str; the model must not see that.""" + assert clibase._bucket_config(self._args(endpoint="")).endpoint is None + + def test_the_flag_is_negated_the_way_it_reads(self): + """--no-verify-tls sets verify_tls=False, not the other way round.""" + assert clibase._bucket_config(self._args()).verify_tls is True + assert clibase._bucket_config( + self._args(no_verify_tls=True)).verify_tls is False + + def test_an_omitted_region_defers_to_the_sdk(self): + """--region is optional, like the credentials: absent means "resolve it". + + Both spellings argparse can produce for an unsupplied option reach the + model as absence, rather than one of them becoming a region named "". + """ + assert clibase._bucket_config(_Args(bucket="backups", region=None)).region is None + assert clibase._bucket_config(_Args(bucket="backups", region="")).region is None + + def test_produces_a_usable_config(self): + config = clibase._bucket_config(self._args( + access_key_id="AKIA", secret_access_key="shh")) + + assert isinstance(config, BackupConfig) + assert config.location().bucket_name == "backups" + + +class TestRegisteredCommands: + + def _cli(self): + import sys + sys.argv = ['sbcli'] + from simplyblock_cli.cli import CLIWrapper + return CLIWrapper() + + def test_discover_is_registered(self): + """The disaster-recovery entry point: needs a bucket, not a cluster.""" + assert hasattr(self._cli(), 'init_backup__discover') + assert hasattr(clibase.CLIWrapperBase, 'backup__discover') + + def test_source_switch_is_gone(self): + """It never switched anything; see the data-plane bucket selection.""" + cli = self._cli() + assert not hasattr(cli, 'init_backup__source_switch') + assert not hasattr(cli, 'init_backup__source_list') + assert not hasattr(clibase.CLIWrapperBase, 'backup__source_switch') + assert not hasattr(clibase.CLIWrapperBase, 'backup__source_list') diff --git a/tests/unit/test_backup_config_model.py b/tests/unit/test_backup_config_model.py new file mode 100644 index 000000000..748492d0e --- /dev/null +++ b/tests/unit/test_backup_config_model.py @@ -0,0 +1,333 @@ +"""Unit tests for the typed backup configuration models.""" +import pytest +from pydantic import SecretStr, ValidationError + +from simplyblock_core.models.backup_config import ( + BackupConfig, + BackupLocation, + S3Credentials, + SecondaryTarget, +) +from simplyblock_core.models.cluster import Cluster + + +MINIMAL = {"bucket_name": "backups", "region": "eu-central-1"} + + +class TestBackupLocation: + def test_minimal_location(self): + location = BackupLocation.model_validate(MINIMAL) + assert location.bucket_name == "backups" + assert location.region == "eu-central-1" + assert location.endpoint is None + assert location.secondary_target is SecondaryTarget.S3 + assert location.snapshot_backups is True + assert location.verify_tls is True + assert location.use_path_style is False + + def test_a_bucket_name_is_mandatory(self): + """Nothing can invent one: a device without a bucket services no I/O.""" + with pytest.raises(ValidationError): + BackupLocation.model_validate( + {k: v for k, v in MINIMAL.items() if k != "bucket_name"}) + + def test_an_absent_region_defers_to_the_sdk(self): + """Like credentials and endpoint -- absent means "resolve it", not a gap.""" + location = BackupLocation.model_validate( + {k: v for k, v in MINIMAL.items() if k != "region"}) + + assert location.region is None + + def test_unknown_field_is_rejected(self): + with pytest.raises(ValidationError): + BackupLocation.model_validate({**MINIMAL, "buckt_name": "typo"}) + + def test_is_frozen(self): + location = BackupLocation.model_validate(MINIMAL) + with pytest.raises(ValidationError): + location.bucket_name = "other" + + def test_endpoint_url_strips_trailing_slash(self): + """pydantic normalises a bare authority to a trailing slash; the SDK wants it gone.""" + location = BackupLocation.model_validate({**MINIMAL, "endpoint": "http://minio:9000"}) + assert str(location.endpoint) == "http://minio:9000/" + assert location.endpoint_url == "http://minio:9000" + + def test_endpoint_url_is_none_when_unset(self): + assert BackupLocation.model_validate(MINIMAL).endpoint_url is None + + def test_equality_is_by_value(self): + """Chain-homogeneity checks compare locations directly.""" + assert BackupLocation.model_validate(MINIMAL) == BackupLocation.model_validate(MINIMAL) + assert BackupLocation.model_validate(MINIMAL) != BackupLocation.model_validate( + {**MINIMAL, "bucket_name": "other"} + ) + + +class TestSecondaryTarget: + def test_numbering_matches_the_data_plane_rpc(self): + """The members ARE the wire values; bdev_s3_create is passed them directly.""" + assert SecondaryTarget.S3 == 0 + assert SecondaryTarget.FILESYSTEM == 1 + + def test_unknown_value_is_rejected(self): + with pytest.raises(ValidationError): + BackupLocation.model_validate({**MINIMAL, "secondary_target": 7}) + + +class TestS3Credentials: + def test_half_a_pair_is_unrepresentable(self): + with pytest.raises(ValidationError): + S3Credentials.model_validate({"access_key_id": "AKIA"}) + + def test_secrets_are_masked_in_repr(self): + creds = S3Credentials( + access_key_id=SecretStr("AKIAEXAMPLE"), + secret_access_key=SecretStr("s3cr3t"), + ) + assert "AKIAEXAMPLE" not in repr(creds) + assert "s3cr3t" not in repr(creds) + + +class TestBackupConfig: + def test_optional_fields_default_to_none_not_sentinels(self): + config = BackupConfig.model_validate(MINIMAL) + assert config.credentials is None + assert config.s3_thread_pool_size is None + + def test_thread_pool_size_must_be_positive(self): + """Legacy 0 is rewritten to absent by the migrator; anything else below 1 is a bug.""" + with pytest.raises(ValidationError): + BackupConfig.model_validate({**MINIMAL, "s3_thread_pool_size": -1}) + + def test_location_drops_credentials(self): + config = BackupConfig.model_validate( + { + **MINIMAL, + "credentials": {"access_key_id": "AKIA", "secret_access_key": "s3cr3t"}, + "s3_thread_pool_size": 16, + } + ) + location = config.location() + + assert type(location) is BackupLocation + assert set(location.model_dump()) == set(BackupLocation.model_fields) + assert "credentials" not in location.model_dump() + assert "s3_thread_pool_size" not in location.model_dump() + + def test_location_preserves_every_interpretation_field(self): + config = BackupConfig.model_validate( + { + **MINIMAL, + "endpoint": "https://s3.example.com", + "with_compression": True, + "use_path_style": True, + "verify_tls": False, + "credentials": {"access_key_id": "AKIA", "secret_access_key": "s3cr3t"}, + } + ) + location = config.location() + + assert location.endpoint_url == "https://s3.example.com" + assert location.with_compression is True + assert location.use_path_style is True + assert location.verify_tls is False + + def test_secrets_are_masked_in_repr(self): + config = BackupConfig.model_validate( + {**MINIMAL, "credentials": { + "access_key_id": "AKIA", "secret_access_key": "s3cr3t"}} + ) + assert "AKIA" not in repr(config) + assert "s3cr3t" not in repr(config) + + +class TestLegacyMigration: + """The untyped dicts already stored on existing clusters must keep working.""" + + def test_local_endpoint_becomes_endpoint(self): + config = BackupConfig.model_validate( + {**MINIMAL, "local_endpoint": "http://minio:9000"} + ) + assert config.endpoint_url == "http://minio:9000" + + def test_flat_keys_become_a_credential_pair(self): + config = BackupConfig.model_validate( + {**MINIMAL, "access_key_id": "AKIA", "secret_access_key": "s3cr3t"} + ) + assert config.credentials is not None + assert config.credentials.access_key_id.get_secret_value() == "AKIA" + assert config.credentials.secret_access_key.get_secret_value() == "s3cr3t" + + def test_a_lone_access_key_does_not_produce_credentials(self): + config = BackupConfig.model_validate({**MINIMAL, "access_key_id": "AKIA"}) + assert config.credentials is None + + def test_local_testing_unpacks_into_the_properties_it_stood_for(self): + """It bundled scheme, TLS verification, addressing style and region into one flag.""" + config = BackupConfig.model_validate( + {"bucket_name": "backups", "local_testing": True, + "local_endpoint": "http://minio:9000"} + ) + assert config.verify_tls is False + assert config.use_path_style is True + assert config.region == "us-east-1" + + def test_explicit_values_win_over_local_testing_defaults(self): + config = BackupConfig.model_validate( + {"bucket_name": "backups", "region": "eu-west-1", + "local_testing": True, "verify_tls": True} + ) + assert config.region == "eu-west-1" + assert config.verify_tls is True + + def test_numeric_secondary_target_becomes_an_enum(self): + assert BackupConfig.model_validate( + {**MINIMAL, "secondary_target": 1} + ).secondary_target is SecondaryTarget.FILESYSTEM + + def test_zero_thread_pool_size_becomes_absent(self): + assert BackupConfig.model_validate( + {**MINIMAL, "s3_thread_pool_size": 0} + ).s3_thread_pool_size is None + + def test_full_legacy_dict(self): + """The shape from tests/perf/backup_config.json.""" + config = BackupConfig.model_validate({ + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin", + "local_endpoint": "http://127.0.0.1:9000", + "bucket_name": "simplyblock-backup", + "snapshot_backups": True, + "with_compression": False, + "secondary_target": 0, + "local_testing": True, + "s3_thread_pool_size": 0, + }) + + assert config.bucket_name == "simplyblock-backup" + assert config.region == "us-east-1" + assert config.endpoint_url == "http://127.0.0.1:9000" + assert config.secondary_target is SecondaryTarget.S3 + assert config.s3_thread_pool_size is None + assert config.credentials is not None + + def test_legacy_config_without_a_region_still_loads(self): + """No config written before this model has one, and they keep working.""" + config = BackupConfig.model_validate({ + "bucket_name": "simplyblock-backup", + "access_key_id": "AKIA", + "secret_access_key": "s3cr3t", + }) + + assert config.region is None + + +class TestClusterAccessor: + def test_returns_a_validated_config(self): + cluster = Cluster() + cluster.backup_config = dict(MINIMAL) + assert cluster.get_backup_config().bucket_name == "backups" + + def test_unconfigured_cluster_raises(self): + cluster = Cluster() + assert cluster.backup_config == {} + with pytest.raises(ValueError, match="no backup configuration"): + cluster.get_backup_config() + + def test_invalid_config_raises(self): + """ValidationError is a ValueError, so one except clause covers both cases.""" + cluster = Cluster() + cluster.backup_config = {"region": "eu-central-1", "endpoint": "minio:9000"} + with pytest.raises(ValueError): + cluster.get_backup_config() + + def test_a_config_without_a_bucket_gets_the_derived_one(self): + """The shape every operator-created cluster stores. + + ``StorageCluster.spec.backup`` has no bucket field (the operator's + ``utils.BackupConfig`` cannot express one), so the config that reaches + ``Cluster.backup_config`` names credentials and an endpoint and nothing + else. Rejecting it here fails cluster activation, which validates the + config for every node it brings up + (``cluster_ops._finish_pass1_node`` -> ``create_s3_bdev``). + """ + cluster = Cluster() + cluster.uuid = "7f4c1b2e-0000-4000-8000-000000000001" + cluster.backup_config = { + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin", + "local_endpoint": "http://minio:9000", + } + + config = cluster.get_backup_config() + + assert config.bucket_name == "simplyblock-backup-7f4c1b2e-0000-4000-8000-000000000001" + + def test_an_explicitly_configured_bucket_is_not_overridden(self): + cluster = Cluster() + cluster.uuid = "7f4c1b2e-0000-4000-8000-000000000001" + cluster.backup_config = {"bucket_name": "chosen-by-the-operator"} + + assert cluster.get_backup_config().bucket_name == "chosen-by-the-operator" + + def test_deriving_a_bucket_leaves_the_record_alone(self): + """Otherwise the next write of this cluster persists the derived name as + if someone had configured it, and renaming the derivation would orphan + every backup written before the rename.""" + cluster = Cluster() + cluster.uuid = "7f4c1b2e-0000-4000-8000-000000000001" + cluster.backup_config = {"region": "eu-central-1"} + + cluster.get_backup_config() + + assert cluster.backup_config == {"region": "eu-central-1"} + + +class TestClusterMutator: + """``Cluster.set_backup_config`` is the only way a config reaches a record. + + Before it existed, ``add_cluster`` stored whatever dict the caller handed it + -- a JSON file for ``sbctl cluster create --use-backup``, a request body for + the API -- and nothing looked at it until activation validated it once per + node. So a typo made at cluster-create surfaced as a failed activation. + """ + + def test_a_config_that_cannot_be_validated_is_refused(self): + cluster = Cluster() + + with pytest.raises(ValueError): + cluster.set_backup_config({**MINIMAL, "endpoint": "minio:9000"}) + + assert cluster.backup_config == {} + + def test_a_misspelled_field_is_refused(self): + """The failure this catches earliest: ``extra="forbid"`` turns a typo into + a rejection, but only where something validates. Stored unvalidated, a + misspelled key is a setting that silently does nothing.""" + cluster = Cluster() + + with pytest.raises(ValueError): + cluster.set_backup_config({"buckt_name": "backups"}) + + def test_a_config_without_a_bucket_is_accepted_and_stored_as_given(self): + """The bucket is the one field a valid config may omit, and omitting it + has to survive the round trip -- storing the derived name would freeze + this cluster's bucket into a record that never chose one.""" + cluster = Cluster() + cluster.uuid = "7f4c1b2e-0000-4000-8000-000000000001" + + cluster.set_backup_config({"region": "eu-central-1"}) + + assert cluster.backup_config == {"region": "eu-central-1"} + assert cluster.get_backup_config().bucket_name == cluster.default_backup_bucket_name() + + def test_a_stored_config_is_not_the_caller_dict(self): + """A caller that keeps mutating its dict must not be editing the record.""" + cluster = Cluster() + config = dict(MINIMAL) + + cluster.set_backup_config(config) + config["bucket_name"] = "somewhere-else" + + assert cluster.get_backup_config().bucket_name == "backups" diff --git a/tests/unit/test_backup_manifest.py b/tests/unit/test_backup_manifest.py new file mode 100644 index 000000000..7f797d07a --- /dev/null +++ b/tests/unit/test_backup_manifest.py @@ -0,0 +1,163 @@ +"""Manifest schema: serialization, version handling, and malformed input. + +Pure logic -- the S3 plumbing around it is exercised in the integration tier. +""" +import json + +import pytest + +from simplyblock_core.controllers.backup import manifest as backup_manifest +from simplyblock_core.controllers.backup.manifest import ( + BackupManifest, + DataPlane, + Encryption, + KeyDescriptor, + ManifestError, + MANIFEST_SCHEMA_VERSION, + Source, + Volume, +) +from simplyblock_core.models.backup_config import BackupLocation + + +LOCATION = {"bucket_name": "backups", "region": "eu-central-1"} + + +def _manifest(**overrides): + fields = { + "backup_id": "b-1", + "s3_id": 7, + "created_at": 100, + "completed_at": 200, + "size": 4096, + "encryption": Encryption(encrypted=False), + "location": BackupLocation.model_validate(LOCATION), + "source": Source(cluster_id="c-1", node_id="n-1"), + "volume": Volume(lvol_id="l-1", lvol_name="vol", snapshot_id="s-1", + snapshot_name="snap", size=4096), + "dataplane": DataPlane(), + } + fields.update(overrides) + return BackupManifest(**fields) + + +class TestManifestKey: + def test_key_cannot_collide_with_the_data_plane_keyspace(self): + """Data objects are {s3_id}/{mid}/{extent}, all decimal segments.""" + key = backup_manifest.manifest_key("b-1") + assert key.startswith("manifests/") + assert not key.split("/")[0].isdigit() + + +class TestSchema: + def test_round_trip(self): + original = _manifest(prev_backup_id="b-0", encryption=Encryption( + encrypted=True, + descriptor=KeyDescriptor(kms="local", dek_path="p", kek_name="k"))) + + restored = backup_manifest._parse(original.model_dump_json().encode(), "k") + + assert restored == original + + def test_serializes_to_plain_json(self): + data = json.loads(_manifest().model_dump_json()) + assert data["schema_version"] == MANIFEST_SCHEMA_VERSION + assert data["location"]["bucket_name"] == "backups" + + def test_carries_no_credential_field(self): + """A manifest sits next to the ciphertext; it must not carry keys.""" + data = json.loads(_manifest().model_dump_json()) + assert "credentials" not in data["location"] + assert "access_key_id" not in json.dumps(data) + + def test_unknown_field_is_rejected(self): + with pytest.raises(ValueError): + BackupManifest.model_validate({ + **json.loads(_manifest().model_dump_json()), "extra": 1}) + + def test_a_root_backup_has_no_predecessor(self): + """Absent, not "" -- a chain root is a state, not a missing value.""" + assert _manifest().prev_backup_id is None + + def test_the_chain_is_not_stored(self): + """It is derived from prev_backup_id, so a merge rewrites two objects + rather than every descendant's manifest.""" + assert "chain" not in json.loads(_manifest().model_dump_json()) + + def test_volume_settings_are_absent_rather_than_zero(self): + """0 is a real answer for a QoS cap -- it means unlimited.""" + volume = _manifest().volume + assert volume.rw_ios_per_sec is None + assert volume.pool_name is None + + +class TestChainOf: + def _line(self): + return [ + _manifest(backup_id="b-0", s3_id=1), + _manifest(backup_id="b-1", s3_id=2, prev_backup_id="b-0"), + _manifest(backup_id="b-2", s3_id=3, prev_backup_id="b-1"), + ] + + def test_walks_to_the_root_oldest_first(self): + line = self._line() + chain = backup_manifest.chain_of(line[-1], line) + assert [m.backup_id for m in chain] == ["b-0", "b-1", "b-2"] + + def test_a_full_backup_is_its_own_chain(self): + line = self._line() + assert backup_manifest.chain_of(line[0], line) == [line[0]] + + def test_order_does_not_matter(self): + line = self._line() + chain = backup_manifest.chain_of(line[-1], list(reversed(line))) + assert [m.backup_id for m in chain] == ["b-0", "b-1", "b-2"] + + def test_ignores_manifests_outside_the_chain(self): + line = self._line() + unrelated = _manifest(backup_id="other", s3_id=9) + chain = backup_manifest.chain_of(line[-1], line + [unrelated]) + assert [m.backup_id for m in chain] == ["b-0", "b-1", "b-2"] + + def test_a_missing_ancestor_is_reported_not_truncated(self): + """Truncating would restore a volume with holes in it.""" + line = self._line() + with pytest.raises(ManifestError, match="b-0"): + backup_manifest.chain_of(line[-1], line[1:]) + + def test_a_cycle_is_reported_rather_than_looping(self): + a = _manifest(backup_id="b-a", prev_backup_id="b-b") + b = _manifest(backup_id="b-b", prev_backup_id="b-a") + with pytest.raises(ManifestError, match="cyclic"): + backup_manifest.chain_of(a, [a, b]) + + +class TestParse: + def test_rejects_a_newer_schema_version(self): + """Guessing at an unknown schema restores a corrupt volume with no error.""" + data = json.loads(_manifest().model_dump_json()) + data["schema_version"] = MANIFEST_SCHEMA_VERSION + 1 + + with pytest.raises(ManifestError, match="schema version"): + backup_manifest._parse(json.dumps(data).encode(), "k") + + def test_rejects_a_missing_schema_version(self): + data = json.loads(_manifest().model_dump_json()) + del data["schema_version"] + + with pytest.raises(ManifestError, match="schema version"): + backup_manifest._parse(json.dumps(data).encode(), "k") + + def test_rejects_invalid_json(self): + with pytest.raises(ManifestError, match="not valid JSON"): + backup_manifest._parse(b"{not json", "k") + + def test_rejects_a_malformed_manifest(self): + with pytest.raises(ManifestError, match="malformed"): + backup_manifest._parse( + json.dumps({"schema_version": MANIFEST_SCHEMA_VERSION}).encode(), "k") + + def test_names_the_key_it_could_not_read(self): + """An operator sweeping a recovery bucket needs to know which object failed.""" + with pytest.raises(ManifestError, match="manifests/b-9.json"): + backup_manifest._parse(b"{not json", "manifests/b-9.json") diff --git a/tests/unit/test_backup_restore_node_selection.py b/tests/unit/test_backup_restore_node_selection.py index fa089b8b7..3fcf51880 100644 --- a/tests/unit/test_backup_restore_node_selection.py +++ b/tests/unit/test_backup_restore_node_selection.py @@ -13,6 +13,7 @@ from simplyblock_core.exceptions import PreconditionError from simplyblock_core.models.backup import Backup +from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.storage_node import StorageNode @@ -20,15 +21,18 @@ SOURCE_CLUSTER = "00000000-0000-0000-0000-00000000000f" -def _backup(node_id, cluster_id=TARGET_CLUSTER, source_cluster_id=""): +LOCATION = {"bucket_name": "backups", "region": "eu-central-1"} + + +def _backup(node_id, cluster_id=TARGET_CLUSTER): backup = Backup() backup.uuid = "backup-1" backup.s3_id = 5 backup.node_id = node_id backup.cluster_id = cluster_id - backup.source_cluster_id = source_cluster_id backup.size = 1024 backup.status = Backup.STATUS_COMPLETED + backup.location = dict(LOCATION) return backup @@ -43,16 +47,22 @@ def _node(uuid, cluster_id, status=StorageNode.STATUS_ONLINE, lvstore="lvs_test" @pytest.fixture def db(): - with patch("simplyblock_core.controllers.backup_controller.db_controller") as db: + with patch("simplyblock_core.controllers.backup.controller.db_controller") as db: pool = MagicMock() pool.cluster_id = TARGET_CLUSTER db.get_pool_by_id_or_name.return_value = pool - cluster = MagicMock() + # A real Cluster, not a mock: restore compares the backup's recorded + # location against the cluster's own configuration, and a mock compares + # unequal to everything. + cluster = Cluster() cluster.uuid = TARGET_CLUSTER - cluster.backup_source = "" + cluster.backup_config = dict(LOCATION) db.get_cluster_by_id.return_value = cluster + node = _node("target-node", TARGET_CLUSTER) + db.get_storage_node_by_id.return_value = node + lvol = MagicMock() lvol.node_id = "target-node" lvol.lvs_name = "lvs_test" @@ -70,13 +80,13 @@ def add_lvol_ha(): @pytest.fixture def tasks(): - with patch("simplyblock_core.controllers.backup_controller.tasks_controller") as tasks: + with patch("simplyblock_core.controllers.backup.controller.tasks_controller") as tasks: tasks.add_backup_restore_task.return_value = True yield tasks def _restore(**kwargs): - from simplyblock_core.controllers.backup_controller import restore_backup + from simplyblock_core.controllers.backup.controller import restore_backup return restore_backup("backup-1", "restored_lvol", "pool-1", **kwargs) @@ -84,29 +94,26 @@ class TestImplicitNode: def test_backup_node_is_not_used_for_placement(self, db, add_lvol_ha, tasks): """An imported backup's node_id points into the source cluster.""" - backup = _backup(node_id="source-cluster-node", source_cluster_id=SOURCE_CLUSTER) + backup = _backup(node_id="source-cluster-node") db.get_backup_by_id.return_value = backup db.get_backup_chain.return_value = [backup] - db.get_cluster_by_id.return_value.backup_source = SOURCE_CLUSTER assert _restore() == "lvol-new" assert not add_lvol_ha.call_args.kwargs["host_id_or_name"] def test_no_node_lookup_without_explicit_target(self, db, add_lvol_ha, tasks): - backup = _backup(node_id="source-cluster-node", source_cluster_id=SOURCE_CLUSTER) + backup = _backup(node_id="source-cluster-node") db.get_backup_by_id.return_value = backup db.get_backup_chain.return_value = [backup] - db.get_cluster_by_id.return_value.backup_source = SOURCE_CLUSTER _restore() db.get_storage_node_by_id.assert_not_called() def test_restore_task_targets_the_node_the_volume_landed_on(self, db, add_lvol_ha, tasks): - backup = _backup(node_id="source-cluster-node", source_cluster_id=SOURCE_CLUSTER) + backup = _backup(node_id="source-cluster-node") db.get_backup_by_id.return_value = backup db.get_backup_chain.return_value = [backup] - db.get_cluster_by_id.return_value.backup_source = SOURCE_CLUSTER _restore() @@ -171,13 +178,14 @@ def _local_backup(self, db): db.get_backup_by_id.return_value = backup db.get_backup_chain.return_value = [backup] - def test_source_mismatch_is_a_precondition(self, db, add_lvol_ha): - db.get_cluster_by_id.return_value.backup_source = SOURCE_CLUSTER - - with pytest.raises(PreconditionError, match="source-switch"): - _restore() + def test_a_backup_from_another_cluster_needs_no_switch(self, db, add_lvol_ha, tasks): + """It used to be refused unless the whole cluster had been re-pointed at + that cluster's bucket. Same bucket, so nothing to re-point.""" + backup = _backup(node_id="source-cluster-node") + db.get_backup_by_id.return_value = backup + db.get_backup_chain.return_value = [backup] - add_lvol_ha.assert_not_called() + assert _restore() == "lvol-new" def test_incomplete_chain_is_rejected_before_creating_a_volume(self, db, add_lvol_ha): db.get_backup_chain.return_value = [_backup(node_id="target-node")] diff --git a/tests/unit/test_client_secret_logging.py b/tests/unit/test_client_secret_logging.py index 77c5f727f..87662dea1 100644 --- a/tests/unit/test_client_secret_logging.py +++ b/tests/unit/test_client_secret_logging.py @@ -80,6 +80,59 @@ def test_rpc_client_response_body_logged_when_flag_on(rpc_client, caplog, monkey assert "RESPVALUE" in _captured_logs_text(caplog) +def _sent_params(client): + return json.loads(client._fake_session.post.call_args.kwargs["data"])["params"] + + +def test_bdev_s3_create_keys_reach_the_wire_but_not_the_log(rpc_client, caplog): + # bdev_s3_create is the only RPC carrying S3 keys, and it goes through + # _request3, which logs its parameter dict directly -- only a SecretStr + # masks there. + rpc_client._fake_session.post.return_value = _make_json_response({ + "jsonrpc": "2.0", "id": 1, "result": True, + }) + + with caplog.at_level(logging.DEBUG): + rpc_client.bdev_s3_create( + name="s3_lvs_test", bucket_name="bucket", region="eu-central-1", + secondary_target=0, with_compression=False, snapshot_backups=True, + access_key_id=SecretStr("AKIAEXAMPLE"), + secret_access_key=SecretStr("s3cr3t"), + ) + + params = _sent_params(rpc_client) + assert params["access_key_id"] == "AKIAEXAMPLE" + assert params["secret_access_key"] == "s3cr3t" + + logged = _captured_logs_text(caplog) + assert "AKIAEXAMPLE" not in logged + assert "s3cr3t" not in logged + assert "**********" in logged + + +def test_bdev_s3_create_omits_absent_credentials(rpc_client): + rpc_client._fake_session.post.return_value = _make_json_response({ + "jsonrpc": "2.0", "id": 1, "result": True, + }) + + rpc_client.bdev_s3_create( + name="s3_lvs_test", bucket_name="bucket", region="eu-central-1", + secondary_target=0, with_compression=False, snapshot_backups=True) + + params = _sent_params(rpc_client) + assert "access_key_id" not in params + assert "secret_access_key" not in params + + # Same for every other optional: the data plane reads 0 as "unset" for the + # masks and the pool size, so sending one would be indistinguishable from + # omitting it while reading as a deliberate choice. + for absent in ("endpoint", "bdb_lcpu_mask", "s3_lcpu_mask", "s3_thread_pool_size"): + assert absent not in params + + # ... and what is not optional is always present. + assert params["region"] == "eu-central-1" + + @pytest.fixture def snode_client(): with patch("simplyblock_core.snode_client.requests.session") as session_factory: diff --git a/tests/unit/test_imports.py b/tests/unit/test_imports.py index 1187d0570..039d7a0bd 100644 --- a/tests/unit/test_imports.py +++ b/tests/unit/test_imports.py @@ -35,7 +35,11 @@ "simplyblock_core.controllers.lvol_controller", "simplyblock_core.controllers.snapshot_controller", "simplyblock_core.controllers.migration_controller", - "simplyblock_core.controllers.backup_controller", + "simplyblock_core.controllers.backup.controller", + "simplyblock_core.controllers.backup.device", + "simplyblock_core.controllers.backup.manifest", + "simplyblock_core.controllers.backup.policy", + "simplyblock_core.controllers.backup.validation", "simplyblock_core.controllers.tasks_controller", # --- Models --- "simplyblock_core.models.lvol_migration", diff --git a/tests/unit/web/api/v2/conftest.py b/tests/unit/web/api/v2/conftest.py index 64ba57786..0a1534109 100644 --- a/tests/unit/web/api/v2/conftest.py +++ b/tests/unit/web/api/v2/conftest.py @@ -147,9 +147,15 @@ def snapshot_controller(monkeypatch): @pytest.fixture() def backup_controller(monkeypatch): + """One mock standing in for both halves of the backup package. + + The router reaches `controller` for backups and `policy` for policies; the + tests assert against a single object, so the same mock is installed as both. + """ mock = MagicMock() monkeypatch.setattr(volume_module, 'backup_controller', mock) monkeypatch.setattr(backup_module, 'backup_controller', mock) + monkeypatch.setattr(backup_module, 'backup_policy', mock) return mock diff --git a/tests/unit/web/api/v2/test_backup_endpoints.py b/tests/unit/web/api/v2/test_backup_endpoints.py index 59ac27868..17b0fff84 100644 --- a/tests/unit/web/api/v2/test_backup_endpoints.py +++ b/tests/unit/web/api/v2/test_backup_endpoints.py @@ -81,7 +81,164 @@ def test_restores_backup(self, client, db, cluster, backup_controller): assert response.status_code == 202 assert response.json() == {'lvol_id': VOLUME_ID} backup_controller.restore_backup.assert_called_once_with( - BACKUP_ID, 'restored-volume', 'pool-1', target_node_id=None) + BACKUP_ID, 'restored-volume', 'pool-1', target_node_id=None, + s3_credentials=None) + + def test_passes_bucket_credentials_through(self, client, db, cluster, + backup_controller): + """Restoring another cluster's bucket needs credentials for it.""" + backup_controller.restore_backup.return_value = VOLUME_ID + + response = client.post(f'{BASE}/restore', json={ + 'backup_id': BACKUP_ID, + 'lvol_name': 'restored-volume', + 'pool': 'pool-1', + 's3_credentials': {'access_key_id': 'AKIA', 'secret_access_key': 'shh'}, + }) + + assert response.status_code == 202 + credentials = backup_controller.restore_backup.call_args.kwargs['s3_credentials'] + assert credentials.access_key_id.get_secret_value() == 'AKIA' + + def test_a_precondition_error_is_not_mapped_here(self, client, db, cluster, + backup_controller): + """app.py maps PreconditionError to 400 for the whole API; a second, + disagreeing mapping in this one router made it inconsistent with itself. + + (This test app deliberately mounts only the routers, so an unhandled + exception surfaces here instead of reaching that handler.) + """ + import pytest + from simplyblock_core.exceptions import PreconditionError + backup_controller.restore_backup.side_effect = PreconditionError('node offline') + + with pytest.raises(PreconditionError): + client.post(f'{BASE}/restore', json={ + 'backup_id': BACKUP_ID, + 'lvol_name': 'restored-volume', + 'pool': 'pool-1', + }) + + +class TestImportBackups: + """The body is a union of two shapes, not one model with everything optional.""" + + _MANIFEST = { + 'schema_version': 1, + 'backup_id': BACKUP_ID, + 's3_id': 7, + 'created_at': 100, + 'completed_at': 200, + 'size': 4096, + 'encryption': {'encrypted': False}, + 'location': {'bucket_name': 'backups', 'region': 'eu-central-1'}, + 'source': {'cluster_id': CLUSTER_ID, 'node_id': 'node-1'}, + 'volume': {'lvol_id': VOLUME_ID, 'lvol_name': 'vol', + 'snapshot_id': SNAPSHOT_ID, 'snapshot_name': 'snap', + 'size': 4096}, + 'dataplane': {}, + } + + _BUCKET = {'bucket_name': 'backups', 'region': 'eu-central-1'} + + def test_inline_manifests_are_validated_by_the_body_type( + self, client, db, cluster, backup_controller): + backup_controller.import_backups.return_value = 1 + + response = client.post(f'{BASE}/import', json={'metadata': [self._MANIFEST]}) + + assert response.status_code == 200 + assert response.json() == {'imported': 1} + (manifests,), kwargs = backup_controller.import_backups.call_args + assert [m.backup_id for m in manifests] == [BACKUP_ID] + + def test_a_malformed_manifest_is_rejected_before_the_controller( + self, client, db, cluster, backup_controller): + response = client.post( + f'{BASE}/import', + json={'metadata': [{**self._MANIFEST, 's3_id': 'not-an-int'}]}) + + assert response.status_code == 422 + backup_controller.import_backups.assert_not_called() + + def test_a_bucket_reads_the_manifests_itself( + self, client, db, cluster, backup_controller): + backup_controller.import_from_bucket.return_value = 3 + + response = client.post(f'{BASE}/import', json={'bucket': self._BUCKET}) + + assert response.status_code == 200 + assert response.json() == {'imported': 3} + backup_controller.import_backups.assert_not_called() + + def test_naming_both_sources_is_rejected( + self, client, db, cluster, backup_controller): + """extra="forbid" on both arms is what makes the union decide.""" + response = client.post(f'{BASE}/import', json={ + 'metadata': [self._MANIFEST], 'bucket': self._BUCKET}) + + assert response.status_code == 422 + + def test_naming_neither_source_is_rejected( + self, client, db, cluster, backup_controller): + response = client.post(f'{BASE}/import', json={}) + + assert response.status_code == 422 + + def test_an_unreadable_bucket_is_a_bad_request_not_a_bad_gateway( + self, client, db, cluster, backup_controller): + """Nothing here proxies for S3, and the bucket came from the request.""" + from simplyblock_core.controllers.backup.manifest import ManifestError + backup_controller.import_from_bucket.side_effect = ManifestError('no such bucket') + + response = client.post(f'{BASE}/import', json={'bucket': self._BUCKET}) + + assert response.status_code == 400 + assert 'no such bucket' in response.json()['detail'] + + def test_a_precondition_error_is_not_mapped_here( + self, client, db, cluster, backup_controller): + """It used to become a 409, contradicting app.py, which maps every + PreconditionError to 400 for the whole API. The endpoint lets it through. + + (This test app deliberately mounts only the routers, so an unhandled + exception surfaces here instead of reaching that handler.) + """ + import pytest + from simplyblock_core.exceptions import PreconditionError + backup_controller.import_backups.side_effect = PreconditionError('already exists') + + with pytest.raises(PreconditionError): + client.post(f'{BASE}/import', json={'metadata': [self._MANIFEST]}) + + +class TestDiscoverBackups: + + def test_returns_the_manifests_the_bucket_holds( + self, client, db, backup_controller): + from simplyblock_core.controllers.backup.manifest import BackupManifest + backup_controller.discover_backups.return_value = [ + BackupManifest.model_validate(TestImportBackups._MANIFEST)] + + response = client.post( + f'{BASE}/discover', + json={'bucket_name': 'backups', 'region': 'eu-central-1'}) + + assert response.status_code == 200 + assert [entry['backup_id'] for entry in response.json()] == [BACKUP_ID] + + def test_credentials_are_masked_in_the_response_of_a_failure( + self, client, db, backup_controller): + from simplyblock_core.controllers.backup.manifest import ManifestError + backup_controller.discover_backups.side_effect = ManifestError('unreachable') + + response = client.post(f'{BASE}/discover', json={ + 'bucket_name': 'backups', 'region': 'eu-central-1', + 'credentials': {'access_key_id': 'AKIA', 'secret_access_key': 's3cr3t'}, + }) + + assert response.status_code == 400 + assert 's3cr3t' not in response.text class TestBackupPolicies: