diff --git a/.env.example b/.env.example index 631d98f..abedd8f 100644 --- a/.env.example +++ b/.env.example @@ -184,3 +184,13 @@ BACKUP_MAX_FILE_SIZE=1073741824 BACKUP_MAX_TOTAL_SIZE=10737418240 BACKUP_MAX_COMPRESSION_RATIO=200 BACKUP_MAX_MANIFEST_SIZE=10485760 +# Native Admin backup and restore are always available to administrators. +# Operational limits only; BACKUP_TEMP_DIR is a shared base outside DATA_DIR. +BACKUP_UPLOAD_MAX_SIZE=1073741824 +BACKUP_OPERATION_TIMEOUT=1800 +BACKUP_DOWNLOAD_TTL=600 +# BACKUP_TEMP_DIR=/tmp/webssh-backup-operations +RATELIMIT_BACKUP_CREATE=3 per hour +RATELIMIT_BACKUP_UPLOAD=5 per hour +RATELIMIT_BACKUP_DOWNLOAD=10 per hour +RATELIMIT_BACKUP_RESTORE=3 per hour diff --git a/README.md b/README.md index 646d203..1e0fda5 100644 --- a/README.md +++ b/README.md @@ -796,7 +796,81 @@ designed for a read-only snapshot and never migrates plaintext legacy keys. Its report omits key content, configured key names, filenames, paths, and the `SECRET_KEY`. -### Backup, Restore, and Secret Rotation +### Web Backup and Restore + +Administrators can create, download, verify, and restore backups from +**Administration > Backup & Restore**. This native feature is always present; +there are no feature flags that can silently disable backup or restore. + +Web backup uses SQLite's native backup API and briefly coordinates persistent +file writers while it captures the database and file-based stores. WebSSH stays +online during creation. The temporary snapshot is deleted immediately after the +verified ZIP has been created. The verified archive is kept in a private +directory outside `DATA_DIR` only until its one-time, session-bound download or +until its TTL expires. + +The archive includes the SQLite database, application settings, user profiles, +`known_hosts`, persisted application secret, SSH key metadata, encrypted private +keys, and the other persistent files covered by the CLI format. Runtime `logs/`, +`tmp/`, transient uploads, and incomplete transfer data remain excluded. New +web and CLI archives use format version 2 and are mutually compatible. Format +v2 records the WebSSH data-schema version, creation time, and producer in the +manifest. Existing format-v1 CLI archives remain supported as legacy schema 0 +backups. + +Archive verification and restore compatibility are separate decisions. A safe, +well-formed archive can be inspected even when it cannot be restored by the +running version. The Admin validation result shows the archive format, backup +and current data-schema versions, creation time, legacy status, and a +compatibility reason. Backups with the current schema are accepted. Older +schemas are accepted only when WebSSH has a complete registered migration path. +Backups with a newer schema are blocked server-side before restore preparation +and checked again before the destructive operation starts. Restoring a newer +backup into an older WebSSH release is not supported. + +To restore, upload an archive in the same Admin tab. WebSSH verifies its +manifest, checksums, sizes, members, compression limits, and format before it +shows a non-sensitive summary. Restore then requires two explicit confirmations, +the exact phrase `RESTORE`, and the current administrator password. The service +enters maintenance mode, rejects new writes and SSH sessions, closes active +runtime activity, creates an online-consistent emergency rollback archive, and +replaces the persistent state. All browser sessions are invalidated. + +After a successful restore, the process terminates intentionally. Docker +Compose and Portainer deployments using `restart: unless-stopped` restart the +container automatically. The Admin page reports the operation while possible; +a disconnect during the final step means the administrator should wait for +`/ready` and sign in again. An interrupted restore is detected on startup and +rolled back from the emergency archive. If both restore and rollback fail, +maintenance mode remains active and the operator must use the CLI restore path. +A successful confirmed CLI restore clears this recovery-only maintenance state; +the following application start removes the retained temporary rollback files. + +Backup archives are highly sensitive. HTTPS protects transport only; it does +not encrypt the downloaded ZIP at rest. Store downloads encrypted, off-host, +with administrator-only access, and dispose of them according to a retention +policy. + +Web restore is intentionally treated as a high-risk administrative operation, +not as a routine user action. Keep the Admin interface behind HTTPS and trusted +access controls, retain an encrypted off-host backup, and keep the offline CLI +restore procedure available when the web process or its current data schema +cannot start safely. + +Operational configuration: + +| Variable | Default | Purpose | +|----------|---------|---------| +| `BACKUP_UPLOAD_MAX_SIZE` | `1073741824` | Maximum streamed web upload size in bytes | +| `BACKUP_OPERATION_TIMEOUT` | `1800` | Operation and retained-status timeout in seconds | +| `BACKUP_DOWNLOAD_TTL` | `600` | TTL for generated downloads and verified uploads in seconds | +| `BACKUP_TEMP_DIR` | system temp + `webssh-backup-operations` | Private temporary base outside `DATA_DIR`; WebSSH creates an isolated namespace per resolved data directory | +| `RATELIMIT_BACKUP_CREATE` | `3 per hour` | Per-admin/IP creation rate | +| `RATELIMIT_BACKUP_UPLOAD` | `5 per hour` | Per-admin/IP upload rate | +| `RATELIMIT_BACKUP_DOWNLOAD` | `10 per hour` | Per-admin/IP download rate | +| `RATELIMIT_BACKUP_RESTORE` | `3 per hour` | Per-admin/IP restore-attempt rate | + +### CLI Backup, Restore, and Secret Rotation Run mutating maintenance commands only while every WebSSH application process that uses the data directory is stopped. Archives contain the database, user @@ -957,9 +1031,10 @@ private, and reserved targets after DNS resolution. - Restrict and encrypt backups of `DATA_DIR`; they contain account metadata, encrypted private keys, and may include the Docker-generated `SECRET_KEY`. Runtime logs and incomplete transfers are excluded. -- Stop all WebSSH processes before backup creation, restore, or secret rotation. - Verify archives before transferring or restoring them, and restart - immediately after a successful persisted-secret rotation. +- Use the Admin workflow for an online-consistent backup. Stop all WebSSH + processes before CLI backup creation, CLI restore, or secret rotation. Verify + archives before transferring or restoring them, and restart immediately after + a successful persisted-secret rotation. - Define a retention and secure-disposal policy for `DATA_DIR/deleted_users`. Account deletion quarantines those files to prevent numeric user-id reuse from exposing them, but does not wipe them automatically. diff --git a/app/__init__.py b/app/__init__.py index e957d69..ac6b11f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -42,6 +42,8 @@ def _initialize_persistent_storage(app): return config.DATA_DIR.mkdir(parents=True, exist_ok=True) + from .session_epoch import current_epoch + current_epoch() from .audit_logger import initialize_file_logging initialize_file_logging(config.DATA_DIR) with app.app_context(): @@ -74,6 +76,10 @@ def create_app( max_workers=config.BACKGROUND_WORKERS ) + from .maintenance_mode import is_active, recover_interrupted_restore + if initialize_storage: + recover_interrupted_restore() + for warning in config.SECURITY_CONFIG_WARNINGS: log_warning('Deployment security warning', warning=warning) @@ -135,6 +141,27 @@ def hide_disabled_admin_panel(): ): abort(404) + @app.before_request + def enforce_restore_maintenance_and_session_epoch(): + if is_active() and request.path not in { + '/health', + '/ready', + '/admin/api/backups/restore/status', + }: + return jsonify({ + 'error': 'WebSSH is in restore maintenance mode', + 'code': 'maintenance', + }), 503 + if initialize_storage and current_user.is_authenticated: + from .session_epoch import current_epoch + epoch = current_epoch() + stored_epoch = session.get('_auth_epoch') + if stored_epoch is None: + session['_auth_epoch'] = epoch + elif stored_epoch != epoch: + logout_user() + session.clear() + trusted_proxies = config.TRUSTED_PROXIES if trusted_proxies > 0: app.wsgi_app = ProxyFix( @@ -160,6 +187,8 @@ def hide_disabled_admin_panel(): app.config['SQLALCHEMY_DATABASE_URI'] = f'sqlite:///{config.DATA_DIR / "app.db"}' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False db.init_app(app) + from .backup_coordination import install_sqlalchemy_coordination + install_sqlalchemy_coordination() init_auth(app) from .request_limits import init_request_limits from .webauthn_routes import webauthn_blueprint @@ -167,6 +196,7 @@ def hide_disabled_admin_panel(): csrf.init_app(app) from .cli import register_cli from .audit_export import audit_export_blueprint + from .admin_backup import admin_backup_blueprint from .health import health_blueprint from .host_key_routes import host_key_blueprint from .oidc_routes import init_oidc, oidc_blueprint @@ -176,6 +206,7 @@ def hide_disabled_admin_panel(): if initialize_oidc: init_oidc(app) app.register_blueprint(audit_export_blueprint) + app.register_blueprint(admin_backup_blueprint) app.register_blueprint(health_blueprint) app.register_blueprint(host_key_blueprint) app.register_blueprint(oidc_blueprint) @@ -185,6 +216,12 @@ def hide_disabled_admin_panel(): if initialize_storage: _initialize_persistent_storage(app) if start_runtime: + from .backup_operations import backup_operations + backup_operations.cleanup_orphans() + app.extensions['runtime_lifecycle'].start_job( + 'backup-operation-cleanup', + backup_operations.cleanup_loop, + ) transfer_runtime_binding = transfer_manager.bind_runtime() app.extensions['runtime_lifecycle'].register_shutdown_callback( 'active_transfers', @@ -237,6 +274,9 @@ def add_security_headers(response): if not config.DEBUG: response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains' + if initialize_storage and session.get('_user_id') is not None: + from .session_epoch import current_epoch + session['_auth_epoch'] = current_epoch() return response from . import socket_events, command_manager, connection_pool diff --git a/app/admin_backup.py b/app/admin_backup.py new file mode 100644 index 0000000..bf9f5da --- /dev/null +++ b/app/admin_backup.py @@ -0,0 +1,485 @@ +"""Administrator HTTP API for backup creation, upload, and verification.""" + +from datetime import datetime, timezone +import logging +import os +from pathlib import Path +import secrets + +from flask import ( + Blueprint, + current_app, + jsonify, + request, + session, +) +from flask_login import current_user, login_required + +import config + +from .audit_logger import log_rate_limit_exceeded, log_security_event +from .auth import check_reauth_rate_limit +from .backup_coordination import OperationBusyError, operation_lock +from .backup_manager import ( + BackupIntegrityError, + evaluate_backup_compatibility, + verify_backup, +) +from .backup_operations import backup_operations +from .decorators import admin_required +from .online_backup import create_online_backup +from . import socketio + + +admin_backup_blueprint = Blueprint('admin_backup', __name__) +_UPLOAD_CHUNK_SIZE = 1024 * 1024 +_OPERATION_BUSY_MESSAGE = 'another backup or restore operation is active' +_UPLOAD_TOO_LARGE_MESSAGE = 'Backup upload is too large' + + +class _ArchiveDownloadStream: + """Stream one archive and invalidate it on EOF or disconnect.""" + + def __init__(self, record): + self._record = record + self._handle = record.archive_path.open('rb') + self._closed = False + + def __iter__(self): + return self + + def __next__(self): + chunk = self._handle.read(_UPLOAD_CHUNK_SIZE) + if chunk: + return chunk + self.close() + raise StopIteration + + def close(self): + if self._closed: + return + self._closed = True + self._handle.close() + backup_operations.remove(self._record.operation_id) + + +def _admin_session_id(): + value = session.get('_backup_admin_session_id') + if not isinstance(value, str) or len(value) < 32: + value = secrets.token_urlsafe(32) + session['_backup_admin_session_id'] = value + return value + + +def _rate_limited(endpoint, limit): + if not config.RATELIMIT_ENABLED: + return False + blocked = check_reauth_rate_limit( + current_user.id, + request.remote_addr or 'unknown', + endpoint, + limit, + ) + if blocked: + log_rate_limit_exceeded( + endpoint, + request.remote_addr or 'unknown', + user=current_user.username, + ) + return blocked + + +def _operation_payload(record): + return { + 'operation_id': record.operation_id, + 'kind': record.kind, + 'status': record.status, + 'size': record.size, + 'summary': record.summary, + 'error': record.error, + } + + +def _archive_summary(manifest): + compatibility = evaluate_backup_compatibility(manifest) + return { + 'format_version': manifest.format_version, + 'data_schema_version': compatibility.data_schema_version, + 'current_data_schema_version': ( + compatibility.current_data_schema_version + ), + 'created_at': manifest.created_at, + 'legacy': compatibility.legacy, + 'file_count': len(manifest.files), + 'total_uncompressed_size': sum(item.size for item in manifest.files), + 'compatible': compatibility.compatible, + 'compatibility_reason': compatibility.reason, + } + + +def _restore_compatibility(record): + manifest = verify_backup(record.archive_path) + return evaluate_backup_compatibility(manifest) + + +def _incompatible_restore_response(record): + try: + compatibility = _restore_compatibility(record) + except BackupIntegrityError: + return jsonify({'error': 'Backup archive is no longer valid'}), 409 + if not compatibility.compatible: + return jsonify({ + 'error': 'Backup is not compatible with this WebSSH version', + 'reason': compatibility.reason, + }), 409 + return None + + +def _no_store(response): + response.headers['Cache-Control'] = 'no-store' + response.headers['Pragma'] = 'no-cache' + response.headers['X-Content-Type-Options'] = 'nosniff' + return response + + +@admin_backup_blueprint.after_request +def add_backup_headers(response): + return _no_store(response) + + +@admin_backup_blueprint.post('/admin/api/backups') +@admin_required +@login_required +def create_backup_operation(): + if _rate_limited('backup_create', config.RATELIMIT_BACKUP_CREATE): + return jsonify({'error': 'Too many backup requests'}), 429 + try: + record = backup_operations.create( + 'created_backup', current_user.id, _admin_session_id() + ) + except OperationBusyError: + return jsonify({'error': _OPERATION_BUSY_MESSAGE}), 409 + + username = current_user.username + source_ip = request.remote_addr or 'unknown' + log_security_event( + 'BACKUP_CREATION_STARTED', user=username, ip=source_ip + ) + + def create_worker(cancel_event): + backup_operations.set_status(record.operation_id, 'running') + try: + manifest = create_online_backup(config.DATA_DIR, record.archive_path) + if cancel_event.is_set(): + backup_operations.remove(record.operation_id) + return + size = record.archive_path.stat().st_size + summary = _archive_summary(manifest) + backup_operations.set_status( + record.operation_id, + 'ready', + summary=summary, + size=size, + ttl=config.BACKUP_DOWNLOAD_TTL, + ) + log_security_event( + 'BACKUP_CREATION_SUCCEEDED', + user=username, + ip=source_ip, + size=size, + files=summary['file_count'], + ) + except Exception as error: + record.archive_path.unlink(missing_ok=True) + backup_operations.set_status( + record.operation_id, + 'failed', + error='Backup creation failed', + ttl=config.BACKUP_DOWNLOAD_TTL, + ) + log_security_event( + 'BACKUP_CREATION_FAILED', + level=logging.ERROR, + user=username, + ip=source_ip, + error_type=type(error).__name__, + ) + + try: + current_app.extensions['runtime_lifecycle'].start_job( + 'web-backup-create', + create_worker, + owner_id=current_user.id, + ) + except Exception: + backup_operations.remove(record.operation_id) + raise + return jsonify(_operation_payload(record)), 202 + + +@admin_backup_blueprint.get('/admin/api/backups/') +@admin_required +@login_required +def backup_operation_status(operation_id): + try: + record = backup_operations.get( + operation_id, current_user.id, _admin_session_id() + ) + except KeyError: + return jsonify({'error': 'Backup operation not found'}), 404 + return jsonify(_operation_payload(record)) + + +@admin_backup_blueprint.post('/admin/api/backups//download') +@admin_required +@login_required +def download_backup(operation_id): + if _rate_limited('backup_download', config.RATELIMIT_BACKUP_DOWNLOAD): + return jsonify({'error': 'Too many backup download requests'}), 429 + try: + record = backup_operations.claim_download( + operation_id, current_user.id, _admin_session_id() + ) + except KeyError: + return jsonify({'error': 'Backup operation not found'}), 404 + + filename = 'webssh-backup-' + datetime.now(timezone.utc).strftime( + '%Y%m%dT%H%M%SZ.zip' + ) + try: + stream = _ArchiveDownloadStream(record) + response = current_app.response_class( + stream, + mimetype='application/zip', + ) + response.headers['Content-Disposition'] = ( + f'attachment; filename="{filename}"' + ) + response.headers['Content-Length'] = str(record.size) + except Exception: + backup_operations.remove(record.operation_id) + raise + log_security_event( + 'BACKUP_DOWNLOADED', + user=current_user.username, + ip=request.remote_addr or 'unknown', + size=record.size, + ) + return response + + +def _stream_upload(destination: Path): + declared_size = request.content_length + if declared_size is not None and declared_size > config.BACKUP_UPLOAD_MAX_SIZE: + raise ValueError('Backup upload is too large') + total = 0 + descriptor = os.open( + destination, + os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, 'O_BINARY', 0), + 0o600, + ) + with os.fdopen(descriptor, 'wb') as handle: + while chunk := request.stream.read(_UPLOAD_CHUNK_SIZE): + total += len(chunk) + if total > config.BACKUP_UPLOAD_MAX_SIZE: + raise ValueError('Backup upload is too large') + handle.write(chunk) + handle.flush() + os.fsync(handle.fileno()) + if total < 4: + raise BackupIntegrityError('Backup upload is malformed') + return total + + +@admin_backup_blueprint.post('/admin/api/backups/upload') +@admin_required +@login_required +def upload_backup(): + if _rate_limited('backup_upload', config.RATELIMIT_BACKUP_UPLOAD): + return jsonify({'error': 'Too many backup upload requests'}), 429 + try: + record = backup_operations.create( + 'uploaded_backup', + current_user.id, + _admin_session_id(), + status='uploading', + ) + except OperationBusyError: + return jsonify({'error': _OPERATION_BUSY_MESSAGE}), 409 + + try: + size = _stream_upload(record.archive_path) + except ValueError: + backup_operations.remove(record.operation_id) + return jsonify({'error': _UPLOAD_TOO_LARGE_MESSAGE}), 413 + except Exception: + backup_operations.remove(record.operation_id) + return jsonify({'error': 'Backup upload failed'}), 400 + + username = current_user.username + source_ip = request.remote_addr or 'unknown' + log_security_event( + 'BACKUP_UPLOADED', user=username, ip=source_ip, size=size + ) + + def verify_worker(cancel_event): + backup_operations.set_status(record.operation_id, 'verifying', size=size) + try: + with operation_lock(): + manifest = verify_backup(record.archive_path) + if cancel_event.is_set(): + backup_operations.remove(record.operation_id) + return + summary = _archive_summary(manifest) + backup_operations.set_status( + record.operation_id, + 'verified', + summary=summary, + size=size, + ttl=config.BACKUP_DOWNLOAD_TTL, + ) + log_security_event( + 'BACKUP_VERIFICATION_SUCCEEDED', + user=username, + ip=source_ip, + size=size, + files=summary['file_count'], + ) + except Exception as error: + record.archive_path.unlink(missing_ok=True) + backup_operations.set_status( + record.operation_id, + 'failed', + error='Backup verification failed', + ttl=config.BACKUP_DOWNLOAD_TTL, + ) + log_security_event( + 'BACKUP_VERIFICATION_FAILED', + level=logging.WARNING, + user=username, + ip=source_ip, + error_type=type(error).__name__, + ) + + try: + current_app.extensions['runtime_lifecycle'].start_job( + 'web-backup-verify', + verify_worker, + owner_id=current_user.id, + ) + except Exception: + backup_operations.remove(record.operation_id) + raise + return jsonify(_operation_payload(record)), 202 + + +@admin_backup_blueprint.post('/admin/api/backups//cancel') +@admin_required +@login_required +def cancel_backup_operation(operation_id): + try: + record = backup_operations.get( + operation_id, current_user.id, _admin_session_id() + ) + except KeyError: + return jsonify({'error': 'Backup operation not found'}), 404 + if record.status in { + 'pending', 'running', 'uploading', 'verifying', 'downloading', + 'restoring', + }: + return jsonify({'error': 'Active operation cannot be cancelled safely'}), 409 + backup_operations.remove(operation_id) + return jsonify({'ok': True}) + + +@admin_backup_blueprint.post( + '/admin/api/backups//restore/prepare' +) +@admin_required +@login_required +def prepare_restore(operation_id): + data = request.get_json(silent=True) or {} + if data.get('acknowledge_sensitive_restore') is not True: + return jsonify({'error': 'Restore acknowledgement is required'}), 400 + try: + record = backup_operations.get( + operation_id, current_user.id, _admin_session_id() + ) + except KeyError: + return jsonify({'error': 'Backup operation not found'}), 404 + if record.kind != 'uploaded_backup' or record.status != 'verified': + return jsonify({'error': 'A verified upload is required'}), 409 + incompatible = _incompatible_restore_response(record) + if incompatible is not None: + return incompatible + try: + token = backup_operations.prepare_restore( + operation_id, current_user.id, _admin_session_id() + ) + except KeyError: + return jsonify({'error': 'A verified upload is required'}), 409 + return jsonify({ + 'confirmation_token': token, + 'confirmation_phrase': 'RESTORE', + 'warning': 'Restore replaces the current persistent state.', + }) + + +@admin_backup_blueprint.post('/admin/api/backups//restore') +@admin_required +@login_required +def restore_uploaded_backup(operation_id): + if _rate_limited('backup_restore', config.RATELIMIT_BACKUP_RESTORE): + return jsonify({'error': 'Too many restore attempts'}), 429 + data = request.get_json(silent=True) or {} + password = data.get('password') + if ( + data.get('confirm_destructive_restore') is not True + or data.get('confirmation_phrase') != 'RESTORE' + or not isinstance(password, str) + ): + return jsonify({'error': 'Explicit restore confirmation is required'}), 400 + from .auth import password_exceeds_bcrypt_limit + if password_exceeds_bcrypt_limit(password) or not current_user.check_password(password): + log_security_event( + 'RESTORE_REAUTH_FAILED', + level=logging.WARNING, + user=current_user.username, + ip=request.remote_addr or 'unknown', + ) + return jsonify({'error': 'Password confirmation failed'}), 403 + try: + record = backup_operations.get( + operation_id, current_user.id, _admin_session_id() + ) + except KeyError: + return jsonify({'error': 'Backup operation not found'}), 404 + incompatible = _incompatible_restore_response(record) + if incompatible is not None: + return incompatible + try: + record = backup_operations.begin_restore( + operation_id, + current_user.id, + _admin_session_id(), + data.get('confirmation_token'), + ) + except KeyError: + return jsonify({'error': 'Restore confirmation expired or invalid'}), 409 + + username = current_user.username + source_ip = request.remote_addr or 'unknown' + log_security_event('RESTORE_STARTED', user=username, ip=source_ip) + from .restore_service import start_restore + start_restore(current_app._get_current_object(), socketio, record, + username, source_ip) + return jsonify(_operation_payload(record)), 202 + + +@admin_backup_blueprint.get('/admin/api/backups/restore/status') +@admin_required +@login_required +def restore_status(): + from .maintenance_mode import public_status + + return jsonify(public_status()) diff --git a/app/backup_coordination.py b/app/backup_coordination.py new file mode 100644 index 0000000..eb41aa1 --- /dev/null +++ b/app/backup_coordination.py @@ -0,0 +1,245 @@ +"""Process and thread coordination for backup-sensitive persistent state.""" + +from contextlib import contextmanager +from dataclasses import dataclass +from hashlib import sha256 +import os +from pathlib import Path +import stat +import threading +import time +from uuid import uuid4 + +import config + + +class OperationBusyError(RuntimeError): + """Raised when another backup-sensitive operation owns the process lock.""" + + +@dataclass(frozen=True) +class OperationToken: + value: str + + +class _SnapshotBarrier: + def __init__(self): + self._condition = threading.Condition(threading.Lock()) + self._readers = 0 + self._writer = False + self._waiting_writers = 0 + self._local = threading.local() + + @contextmanager + def persistent_write(self): + depth = getattr(self._local, 'write_depth', 0) + if depth: + self._local.write_depth = depth + 1 + try: + yield + finally: + self._local.write_depth -= 1 + return + + with self._condition: + while self._writer or self._waiting_writers: + self._condition.wait() + self._readers += 1 + self._local.write_depth = 1 + try: + yield + finally: + self._local.write_depth = 0 + with self._condition: + self._readers -= 1 + if self._readers == 0: + self._condition.notify_all() + + @contextmanager + def snapshot(self): + if getattr(self._local, 'write_depth', 0): + raise RuntimeError('snapshot barrier cannot begin during a write') + with self._condition: + self._waiting_writers += 1 + try: + while self._writer or self._readers: + self._condition.wait() + self._writer = True + finally: + self._waiting_writers -= 1 + try: + yield + finally: + with self._condition: + self._writer = False + self._condition.notify_all() + + +_barrier = _SnapshotBarrier() +_operation_local = threading.local() +_sqlalchemy_guard = threading.Lock() +_sqlalchemy_installed = False + + +def _paths_overlap(left: Path, right: Path) -> bool: + return left == right or left.is_relative_to(right) or right.is_relative_to(left) + + +def ensure_backup_temp_dir() -> Path: + """Create the private, per-DATA_DIR operation namespace.""" + root = Path(config.BACKUP_TEMP_DIR).expanduser() + if root.exists() and root.is_symlink(): + raise RuntimeError('BACKUP_TEMP_DIR must not be a symbolic link') + root = root.resolve(strict=False) + protected = ( + Path(config.DATA_DIR).resolve(strict=False), + (Path(config.BASE_DIR) / 'static').resolve(strict=False), + (Path(config.DATA_DIR) / 'logs').resolve(strict=False), + ) + if any(_paths_overlap(root, path) for path in protected): + raise RuntimeError( + 'BACKUP_TEMP_DIR must be outside DATA_DIR, static, and logs' + ) + root.mkdir(parents=True, exist_ok=True, mode=0o700) + metadata = root.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise RuntimeError('BACKUP_TEMP_DIR must be a real directory') + try: + os.chmod(root, 0o700) + except OSError: + if os.name != 'nt': + raise + + data_dir = Path(config.DATA_DIR).expanduser().resolve(strict=False) + identity = os.path.normcase(str(data_dir)).encode('utf-8') + namespace = root / f'instance-{sha256(identity).hexdigest()[:32]}' + if namespace.exists() and namespace.is_symlink(): + raise RuntimeError('backup operation namespace must not be a symbolic link') + namespace.mkdir(mode=0o700, exist_ok=True) + metadata = namespace.lstat() + if not stat.S_ISDIR(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode): + raise RuntimeError('backup operation namespace must be a real directory') + try: + os.chmod(namespace, 0o700) + except OSError: + if os.name != 'nt': + raise + return namespace.resolve(strict=True) + + +def _try_lock(descriptor: int) -> bool: + if os.name == 'nt': + import msvcrt + + os.lseek(descriptor, 0, os.SEEK_SET) + try: + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) + except OSError: + return False + return True + + import fcntl + + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False + return True + + +def _unlock(descriptor: int) -> None: + if os.name == 'nt': + import msvcrt + + os.lseek(descriptor, 0, os.SEEK_SET) + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + return + + import fcntl + + fcntl.flock(descriptor, fcntl.LOCK_UN) + + +@contextmanager +def operation_lock(*, timeout=None, held_token=None): + """Hold the cross-process lock for one backup-sensitive operation.""" + active = getattr(_operation_local, 'token', None) + if active is not None: + if held_token != active: + raise RuntimeError('nested operation requires its active token') + _operation_local.depth += 1 + try: + yield active + finally: + _operation_local.depth -= 1 + return + + root = ensure_backup_temp_dir() + lock_path = root / 'operation.lock' + flags = os.O_CREAT | os.O_RDWR | getattr(os, 'O_NOFOLLOW', 0) + descriptor = os.open(lock_path, flags, 0o600) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise RuntimeError('backup operation lock must be a regular file') + if os.fstat(descriptor).st_size < 1: + os.write(descriptor, b'0') + os.fsync(descriptor) + deadline = time.monotonic() + ( + config.BACKUP_OPERATION_TIMEOUT if timeout is None else timeout + ) + while not _try_lock(descriptor): + if time.monotonic() >= deadline: + raise OperationBusyError( + 'another backup or restore operation is active' + ) + time.sleep(0.05) + + token = OperationToken(uuid4().hex) + _operation_local.token = token + _operation_local.depth = 1 + try: + yield token + finally: + _operation_local.depth = 0 + _operation_local.token = None + _unlock(descriptor) + finally: + os.close(descriptor) + + +def persistent_write(): + """Coordinate one persistent mutation with online snapshots.""" + return _barrier.persistent_write() + + +def snapshot_barrier(): + """Pause coordinated writes while a consistent snapshot is captured.""" + return _barrier.snapshot() + + +def install_sqlalchemy_coordination() -> None: + """Hold the persistent-write barrier around SQLAlchemy commits.""" + global _sqlalchemy_installed + with _sqlalchemy_guard: + if _sqlalchemy_installed: + return + from sqlalchemy import event + from sqlalchemy.orm import Session + + def acquire(session, *_args): + if '_webssh_persistent_write' in session.info: + return + context = persistent_write() + context.__enter__() + session.info['_webssh_persistent_write'] = context + + def release(session, *_args): + context = session.info.pop('_webssh_persistent_write', None) + if context is not None: + context.__exit__(None, None, None) + + event.listen(Session, 'before_commit', acquire) + event.listen(Session, 'after_commit', release) + event.listen(Session, 'after_rollback', release) + event.listen(Session, 'after_soft_rollback', release) + _sqlalchemy_installed = True diff --git a/app/backup_manager.py b/app/backup_manager.py index 813f6df..a3aa912 100644 --- a/app/backup_manager.py +++ b/app/backup_manager.py @@ -1,10 +1,12 @@ """Verified backup and restore operations for the WebSSH data directory.""" from dataclasses import dataclass +from datetime import datetime, timezone import hashlib import json import os from pathlib import Path, PurePosixPath +import sqlite3 import stat import tempfile import zipfile @@ -13,9 +15,14 @@ from .storage_utils import atomic_copy_file, fsync_parent_directory -_FORMAT_VERSION = 1 +_FORMAT_VERSION = 2 +_LEGACY_FORMAT_VERSION = 1 +_CURRENT_DATA_SCHEMA_VERSION = 1 +_DATA_SCHEMA_MIGRATIONS = {0: 1} +_PRODUCER = 'webssh' _MANIFEST_NAME = 'manifest.json' _DATA_PREFIX = 'data/' +_DATABASE_PATH = 'app.db' _EXCLUDED_TOP_LEVEL_DIRECTORIES = {'logs', 'tmp'} @@ -34,6 +41,18 @@ class BackupFile: class BackupManifest: format_version: int files: tuple[BackupFile, ...] + data_schema_version: int = 0 + created_at: str | None = None + producer: str | None = None + + +@dataclass(frozen=True) +class BackupCompatibility: + compatible: bool + legacy: bool + data_schema_version: int + current_data_schema_version: int + reason: str def _safe_relative_path(value): @@ -50,34 +69,78 @@ def _safe_relative_path(value): def _manifest_payload(manifest): + document = { + 'files': [ + { + 'path': item.path, + 'sha256': item.sha256, + 'size': item.size, + } + for item in manifest.files + ], + 'format_version': manifest.format_version, + } + if manifest.format_version == _FORMAT_VERSION: + document.update({ + 'created_at': manifest.created_at, + 'data_schema_version': manifest.data_schema_version, + 'producer': manifest.producer, + }) return json.dumps( - { - 'files': [ - { - 'path': item.path, - 'sha256': item.sha256, - 'size': item.size, - } - for item in manifest.files - ], - 'format_version': manifest.format_version, - }, + document, sort_keys=True, separators=(',', ':'), ).encode('utf-8') +def _valid_utc_timestamp(value): + if not isinstance(value, str) or not value.endswith('Z'): + return False + try: + parsed = datetime.fromisoformat(value[:-1] + '+00:00') + except ValueError: + return False + return parsed.tzinfo == timezone.utc + + def _parse_manifest(payload): try: document = json.loads(payload.decode('utf-8')) except (UnicodeError, json.JSONDecodeError) as exc: raise BackupIntegrityError('backup manifest is invalid') from exc + if not isinstance(document, dict): + raise BackupIntegrityError('backup manifest is incompatible') + format_version = document.get('format_version') + if type(format_version) is not int: + raise BackupIntegrityError('backup manifest is incompatible') + if format_version == _LEGACY_FORMAT_VERSION: + expected_keys = {'files', 'format_version'} + data_schema_version = 0 + created_at = None + producer = None + elif format_version == _FORMAT_VERSION: + expected_keys = { + 'created_at', + 'data_schema_version', + 'files', + 'format_version', + 'producer', + } + data_schema_version = document.get('data_schema_version') + created_at = document.get('created_at') + producer = document.get('producer') + if ( + type(data_schema_version) is not int + or data_schema_version < 0 + or producer != _PRODUCER + or not _valid_utc_timestamp(created_at) + ): + raise BackupIntegrityError('backup manifest is incompatible') + else: + raise BackupIntegrityError('backup manifest is incompatible') if ( - not isinstance(document, dict) - or set(document) != {'files', 'format_version'} - or type(document['format_version']) is not int - or document['format_version'] != _FORMAT_VERSION - or not isinstance(document['files'], list) + set(document) != expected_keys + or not isinstance(document.get('files'), list) ): raise BackupIntegrityError('backup manifest is incompatible') @@ -105,7 +168,66 @@ def _parse_manifest(payload): raise BackupIntegrityError( 'backup manifest paths must be unique and sorted' ) - return BackupManifest(document['format_version'], tuple(files)) + return BackupManifest( + format_version, + tuple(files), + data_schema_version, + created_at, + producer, + ) + + +def evaluate_backup_compatibility(manifest): + data_schema_version = manifest.data_schema_version + legacy = manifest.format_version == _LEGACY_FORMAT_VERSION + common = { + 'legacy': legacy, + 'data_schema_version': data_schema_version, + 'current_data_schema_version': _CURRENT_DATA_SCHEMA_VERSION, + } + if data_schema_version > _CURRENT_DATA_SCHEMA_VERSION: + return BackupCompatibility( + compatible=False, + reason='backup data schema is newer than this WebSSH version', + **common, + ) + + cursor = data_schema_version + visited = set() + while cursor < _CURRENT_DATA_SCHEMA_VERSION: + if cursor in visited: + break + visited.add(cursor) + next_version = _DATA_SCHEMA_MIGRATIONS.get(cursor) + if ( + type(next_version) is not int + or next_version <= cursor + or next_version > _CURRENT_DATA_SCHEMA_VERSION + ): + break + cursor = next_version + if cursor != _CURRENT_DATA_SCHEMA_VERSION: + return BackupCompatibility( + compatible=False, + reason='no complete migration path for backup data schema', + **common, + ) + if legacy: + reason = 'legacy archive can be migrated' + elif data_schema_version < _CURRENT_DATA_SCHEMA_VERSION: + reason = 'backup data schema can be migrated' + else: + reason = 'backup data schema is current' + return BackupCompatibility(compatible=True, reason=reason, **common) + + +def require_restore_compatible(manifest): + compatibility = evaluate_backup_compatibility(manifest) + if not compatibility.compatible: + raise BackupIntegrityError( + f'backup is not restore compatible: {compatibility.reason}' + ) + return compatibility def _regular_zip_info(name): @@ -161,7 +283,7 @@ def _copy_regular_file(source, destination): return digest.hexdigest(), size -def _stage_source(data_dir, stage): +def _stage_source(data_dir, stage, excluded_relative_paths=frozenset()): files = [] for current_root, directory_names, file_names in os.walk( data_dir, @@ -183,14 +305,20 @@ def _stage_source(data_dir, stage): for file_name in file_names: source = current / file_name relative = source.relative_to(data_dir).as_posix() + if relative in excluded_relative_paths: + continue _safe_relative_path(relative) staged = stage / relative digest, size = _copy_regular_file(source, staged) files.append(BackupFile(relative, digest, size)) - return BackupManifest(_FORMAT_VERSION, tuple(sorted( - files, - key=lambda item: item.path, - ))) + created_at = datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z') + return BackupManifest( + _FORMAT_VERSION, + tuple(sorted(files, key=lambda item: item.path)), + _CURRENT_DATA_SCHEMA_VERSION, + created_at, + _PRODUCER, + ) def _write_archive(stage, archive, manifest): @@ -334,6 +462,67 @@ def _read_manifest(backup, members): return _parse_manifest(payload) +def _validate_webssh_database(path: Path) -> None: + connection = None + try: + uri = path.resolve(strict=True).as_uri() + '?mode=ro&immutable=1' + connection = sqlite3.connect(uri, uri=True) + connection.execute('PRAGMA query_only = ON') + connection.execute('PRAGMA trusted_schema = OFF') + if connection.execute('PRAGMA quick_check').fetchall() != [('ok',)]: + raise BackupIntegrityError('backup database is invalid') + tables = { + row[0] + for row in connection.execute( + "SELECT name FROM sqlite_schema WHERE type = 'table'" + ) + } + if 'users' not in tables: + raise BackupIntegrityError('backup database is not a WebSSH database') + user_columns = { + row[1] for row in connection.execute('PRAGMA table_info(users)') + } + if not {'id', 'username', 'password_hash'} <= user_columns: + raise BackupIntegrityError('backup database is not a WebSSH database') + except BackupIntegrityError: + raise + except (OSError, sqlite3.DatabaseError) as exc: + raise BackupIntegrityError('backup database is invalid') from exc + finally: + if connection is not None: + connection.close() + + +def _verify_database_member(backup, info, expected_size) -> None: + descriptor, temporary_name = tempfile.mkstemp( + prefix='.webssh-backup-database-', + suffix='.db', + ) + temporary = Path(temporary_name) + try: + size = 0 + with os.fdopen(descriptor, 'wb') as destination: + descriptor = None + with backup.open(info, 'r') as source: + while chunk := source.read(1024 * 1024): + size += len(chunk) + if size > expected_size: + raise BackupIntegrityError( + 'backup database exceeds its declared size' + ) + destination.write(chunk) + destination.flush() + os.fsync(destination.fileno()) + os.chmod(temporary, 0o600) + if size != expected_size: + raise BackupIntegrityError('backup database size is invalid') + _validate_webssh_database(temporary) + finally: + if descriptor is not None: + os.close(descriptor) + temporary.unlink(missing_ok=True) + + def verify_backup(archive): try: with zipfile.ZipFile(Path(archive), 'r') as backup: @@ -347,6 +536,12 @@ def verify_backup(archive): raise BackupIntegrityError( 'backup members do not match the manifest' ) + database = next( + (item for item in manifest.files if item.path == _DATABASE_PATH), + None, + ) + if database is None: + raise BackupIntegrityError('backup database is missing') total_size = 0 for item in manifest.files: info = members[_DATA_PREFIX + item.path] @@ -373,6 +568,11 @@ def verify_backup(archive): raise BackupIntegrityError( f'backup checksum mismatch for {item.path}' ) + _verify_database_member( + backup, + members[_DATA_PREFIX + _DATABASE_PATH], + database.size, + ) return manifest except BackupIntegrityError: raise @@ -547,6 +747,7 @@ def restore_backup(archive, data_dir): ) data_dir = data_dir.resolve(strict=False) manifest = verify_backup(archive) + require_restore_compatible(manifest) _validate_restore_targets(data_dir, manifest) existing_paths = _existing_persistent_files(data_dir) manifest_paths = { diff --git a/app/backup_operations.py b/app/backup_operations.py new file mode 100644 index 0000000..54015a8 --- /dev/null +++ b/app/backup_operations.py @@ -0,0 +1,257 @@ +"""Private, session-bound lifecycle for web backup archives.""" + +from dataclasses import dataclass, field +from pathlib import Path +import os +import secrets +import shutil +import stat +import threading +import time + +import config + +from .backup_coordination import ( + OperationBusyError, + ensure_backup_temp_dir, + operation_lock, +) + + +_ACTIVE_STATUSES = frozenset({ + 'pending', 'running', 'uploading', 'verifying', 'downloading', 'restoring', +}) + + +@dataclass +class BackupOperation: + operation_id: str + kind: str + owner_id: int + session_id: str + directory: Path + archive_path: Path + status: str + created_at: float + expires_at: float + size: int = 0 + summary: dict | None = None + error: str | None = None + metadata: dict = field(default_factory=dict) + + +def _is_reparse_point(metadata) -> bool: + attributes = getattr(metadata, 'st_file_attributes', 0) + reparse = getattr(stat, 'FILE_ATTRIBUTE_REPARSE_POINT', 0x400) + return bool(attributes & reparse) + + +def _remove_private_tree(root: Path, directory: Path) -> None: + root = root.resolve(strict=True) + directory = Path(directory) + try: + metadata = directory.lstat() + except FileNotFoundError: + return + resolved = directory.resolve(strict=True) + if ( + resolved.parent != root + or stat.S_ISLNK(metadata.st_mode) + or _is_reparse_point(metadata) + or not stat.S_ISDIR(metadata.st_mode) + ): + raise RuntimeError('refusing unsafe backup operation cleanup') + shutil.rmtree(resolved) + + +class BackupOperationRegistry: + def __init__(self): + self._lock = threading.RLock() + self._records = {} + self._root = None + + def _operation_root(self) -> Path: + configured_root = ensure_backup_temp_dir() + if self._root != configured_root: + if self._records: + raise RuntimeError( + 'BACKUP_TEMP_DIR changed while operations were active' + ) + self._root = configured_root + return self._root + + def _cleanup_expired_locked(self, now=None): + now = time.time() if now is None else now + expired = [ + operation_id + for operation_id, record in self._records.items() + if record.expires_at <= now and record.status not in _ACTIVE_STATUSES + ] + for operation_id in expired: + self._remove_locked(operation_id) + + def create(self, kind, owner_id, session_id, status='pending'): + with self._lock: + self._cleanup_expired_locked() + if any( + record.status in _ACTIVE_STATUSES + for record in self._records.values() + ): + raise OperationBusyError( + 'another backup or restore operation is active' + ) + root = self._operation_root() + for _attempt in range(4): + operation_id = secrets.token_urlsafe(32) + directory = root / f'operation-{operation_id}' + try: + directory.mkdir(mode=0o700) + except FileExistsError: + continue + break + else: + raise RuntimeError('could not allocate backup operation') + now = time.time() + record = BackupOperation( + operation_id=operation_id, + kind=kind, + owner_id=int(owner_id), + session_id=str(session_id), + directory=directory, + archive_path=directory / 'archive.zip', + status=status, + created_at=now, + expires_at=now + config.BACKUP_OPERATION_TIMEOUT, + ) + self._records[operation_id] = record + return record + + def get(self, operation_id, owner_id, session_id): + with self._lock: + self._cleanup_expired_locked() + record = self._records.get(str(operation_id)) + if ( + record is None + or record.owner_id != int(owner_id) + or not secrets.compare_digest(record.session_id, str(session_id)) + ): + raise KeyError(operation_id) + return record + + def set_status(self, operation_id, status, *, summary=None, size=None, + error=None, ttl=None): + with self._lock: + record = self._records.get(operation_id) + if record is None: + return None + record.status = status + if summary is not None: + record.summary = dict(summary) + if size is not None: + record.size = int(size) + record.error = error + if ttl is not None: + record.expires_at = time.time() + ttl + return record + + def claim_download(self, operation_id, owner_id, session_id): + with self._lock: + record = self.get(operation_id, owner_id, session_id) + if ( + record.kind != 'created_backup' + or record.status != 'ready' + or not record.archive_path.is_file() + ): + raise KeyError(operation_id) + record.status = 'downloading' + return record + + def begin_restore(self, operation_id, owner_id, session_id, + confirmation_token): + with self._lock: + record = self.get(operation_id, owner_id, session_id) + expected = record.metadata.get('restore_confirmation_token') + expires = record.metadata.get('restore_confirmation_expires', 0) + if ( + record.kind != 'uploaded_backup' + or record.status != 'verified' + or not isinstance(expected, str) + or not secrets.compare_digest(expected, str(confirmation_token)) + or expires <= time.time() + or any( + candidate.operation_id != record.operation_id + and candidate.status in _ACTIVE_STATUSES + for candidate in self._records.values() + ) + ): + raise KeyError(operation_id) + record.metadata.clear() + record.status = 'restoring' + record.expires_at = time.time() + config.BACKUP_OPERATION_TIMEOUT + return record + + def prepare_restore(self, operation_id, owner_id, session_id, ttl=300): + with self._lock: + record = self.get(operation_id, owner_id, session_id) + if record.kind != 'uploaded_backup' or record.status != 'verified': + raise KeyError(operation_id) + token = secrets.token_urlsafe(32) + record.metadata = { + 'restore_confirmation_token': token, + 'restore_confirmation_expires': time.time() + ttl, + } + return token + + def remove(self, operation_id): + with self._lock: + self._remove_locked(operation_id) + + def _remove_locked(self, operation_id): + record = self._records.pop(operation_id, None) + if record is not None: + _remove_private_tree(self._operation_root(), record.directory) + + def cleanup_orphans(self): + root = self._operation_root() + from .maintenance_mode import protected_operation_directory_name + protected_name = protected_operation_directory_name() + try: + lock_context = operation_lock(timeout=0) + lock_context.__enter__() + except OperationBusyError: + return + try: + with self._lock: + active_directories = { + record.directory.resolve(strict=False) + for record in self._records.values() + } + for child in root.iterdir(): + if ( + not child.name.startswith('operation-') + or child.name == protected_name + or child.resolve(strict=False) in active_directories + ): + continue + _remove_private_tree(root, child) + finally: + lock_context.__exit__(None, None, None) + + def cleanup_expired(self): + with self._lock: + self._cleanup_expired_locked() + + def cleanup_loop(self, cancel_event): + interval = max(5, min(60, config.BACKUP_DOWNLOAD_TTL // 2)) + while not cancel_event.wait(interval): + self.cleanup_expired() + + def close(self, _deadline=None): + with self._lock: + operation_ids = tuple(self._records) + for operation_id in operation_ids: + self._remove_locked(operation_id) + return () + + +backup_operations = BackupOperationRegistry() diff --git a/app/cli.py b/app/cli.py index 670e8ff..707d085 100644 --- a/app/cli.py +++ b/app/cli.py @@ -196,6 +196,7 @@ def backup_cli(): @click.option('--confirm-offline', is_flag=True) def backup_create(destination, confirm_offline): """Create and verify a backup while WebSSH is stopped.""" + from .backup_coordination import operation_lock from .backup_manager import create_backup _require_offline_confirmation(confirm_offline) @@ -205,7 +206,8 @@ def backup_create(destination, confirm_offline): f'webssh-backup-{timestamp}.zip' ) try: - manifest = create_backup(config.DATA_DIR, destination) + with operation_lock(): + manifest = create_backup(config.DATA_DIR, destination) except Exception as exc: raise click.ClickException(str(exc)) from exc _audit_operation( @@ -214,7 +216,8 @@ def backup_create(destination, confirm_offline): ) click.echo( f'Backup created and verified: {destination} ' - f'({len(manifest.files)} files)' + f'({len(manifest.files)} files, format v{manifest.format_version}, ' + f'data schema {manifest.data_schema_version})' ) @@ -225,7 +228,7 @@ def backup_create(destination, confirm_offline): ) def backup_verify(archive): """Verify archive structure and every recorded checksum.""" - from .backup_manager import verify_backup + from .backup_manager import evaluate_backup_compatibility, verify_backup try: manifest = verify_backup(archive) @@ -235,7 +238,21 @@ def backup_verify(archive): 'BACKUP_VERIFY_SUCCESS', file_count=len(manifest.files), ) - click.echo(f'Backup verified ({len(manifest.files)} files).') + compatibility = evaluate_backup_compatibility(manifest) + restore_status = ( + 'restore compatible (legacy)' + if compatibility.compatible and compatibility.legacy + else ( + 'restore compatible' + if compatibility.compatible + else 'restore incompatible' + ) + ) + click.echo( + f'Backup verified ({len(manifest.files)} files, ' + f'format v{manifest.format_version}, ' + f'data schema {manifest.data_schema_version}, {restore_status}).' + ) @backup_cli.command('restore') @@ -246,15 +263,19 @@ def backup_verify(archive): @click.option('--confirm-offline', is_flag=True) def backup_restore(archive, confirm_offline): """Restore a verified backup while WebSSH is stopped.""" + from .backup_coordination import operation_lock from .backup_manager import restore_backup _require_offline_confirmation(confirm_offline) - db.session.remove() - db.engine.dispose() try: - restore_backup(archive, config.DATA_DIR) + with operation_lock(): + db.session.remove() + db.engine.dispose() + restore_backup(archive, config.DATA_DIR) except Exception as exc: raise click.ClickException(str(exc)) from exc + from .maintenance_mode import clear_failed_status_after_cli_restore + clear_failed_status_after_cli_restore() _audit_operation('BACKUP_RESTORE_SUCCESS') click.echo('Backup restored and verified.') diff --git a/app/decorators.py b/app/decorators.py index 241a988..7f2c2c4 100644 --- a/app/decorators.py +++ b/app/decorators.py @@ -1,5 +1,5 @@ from functools import wraps -from flask_socketio import disconnect +from flask_socketio import disconnect, emit from flask import request, abort from flask_login import current_user import config @@ -42,6 +42,15 @@ def handle_event(data, current_user=None): """ @wraps(f) def decorated_function(*args, **kwargs): + from .maintenance_mode import is_active + if is_active(): + payload = { + 'success': False, + 'error': 'WebSSH is in restore maintenance mode', + 'code': 'maintenance', + } + emit('error', payload) + return payload socket_sid = request.sid user = get_user_from_socket(socket_sid) if not user: diff --git a/app/health.py b/app/health.py index 520cec2..911871e 100644 --- a/app/health.py +++ b/app/health.py @@ -46,6 +46,9 @@ def health(): @health_blueprint.get('/ready') def ready(): failed = [] + from .maintenance_mode import is_active + if is_active(): + failed.append('maintenance') lifecycle = current_app.extensions.get('runtime_lifecycle') if lifecycle is None or not lifecycle.accepting_work(): failed.append('runtime') diff --git a/app/maintenance_mode.py b/app/maintenance_mode.py new file mode 100644 index 0000000..c00e970 --- /dev/null +++ b/app/maintenance_mode.py @@ -0,0 +1,231 @@ +"""Restore maintenance and restart-surviving operation status.""" + +from hashlib import sha256 +import json +import os +from pathlib import Path, PurePath +import tempfile +import threading +import time + +import config + +from .backup_coordination import ensure_backup_temp_dir, operation_lock + + +_lock = threading.RLock() +_state = None +_state_path = None +_STATUS_NAME = 'restore-status.json' + + +def _status_path() -> Path: + return ensure_backup_temp_dir() / _STATUS_NAME + + +def _data_fingerprint() -> str: + value = str(Path(config.DATA_DIR).resolve(strict=False)).encode('utf-8') + return sha256(value).hexdigest() + + +def _write(document) -> None: + global _state, _state_path + root = ensure_backup_temp_dir() + target = root / _STATUS_NAME + temporary = None + payload = json.dumps( + document, sort_keys=True, separators=(',', ':') + ).encode('utf-8') + try: + with tempfile.NamedTemporaryFile( + mode='wb', dir=root, prefix='.restore-status-', delete=False + ) as handle: + temporary = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary, 0o600) + os.replace(temporary, target) + from .storage_utils import fsync_parent_directory + fsync_parent_directory(target) + temporary = None + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + _state = dict(document) + _state_path = target.resolve(strict=False) + + +def _read(): + global _state, _state_path + path = _status_path().resolve(strict=False) + if _state is not None and _state_path == path: + return dict(_state) + try: + payload = path.read_bytes() + document = json.loads(payload.decode('utf-8')) + except FileNotFoundError: + return None + except (OSError, UnicodeError, json.JSONDecodeError): + return { + 'state': 'rollback_failed', + 'message': 'Restore status is unreadable', + 'updated_at': time.time(), + } + if not isinstance(document, dict): + return None + _state = document + _state_path = path + return dict(document) + + +def is_active() -> bool: + document = _read() + return bool(document and document.get('state') in { + 'preparing', 'in_progress', 'rollback_failed', + }) + + +def begin_preparing(operation_id: str) -> None: + _write({ + 'state': 'preparing', + 'operation_id': str(operation_id), + 'data_fingerprint': _data_fingerprint(), + 'message': 'Preparing restore safety snapshot', + 'updated_at': time.time(), + }) + + +def mark_in_progress(operation_id: str, rollback_relative: str) -> None: + path = PurePath(rollback_relative) + if path.is_absolute() or '..' in path.parts or len(path.parts) != 2: + raise ValueError('unsafe rollback reference') + _write({ + 'state': 'in_progress', + 'operation_id': str(operation_id), + 'data_fingerprint': _data_fingerprint(), + 'rollback_relative': path.as_posix(), + 'message': 'Restoring persistent state', + 'updated_at': time.time(), + }) + + +def mark_succeeded(operation_id: str) -> None: + _write({ + 'state': 'succeeded', + 'operation_id': str(operation_id), + 'message': 'Restore completed; application restart requested', + 'updated_at': time.time(), + }) + + +def mark_failed(operation_id: str, message: str, *, rollback_failed=False) -> None: + previous = _read() if rollback_failed else None + document = { + 'state': 'rollback_failed' if rollback_failed else 'failed', + 'operation_id': str(operation_id), + 'message': str(message)[:256], + 'updated_at': time.time(), + } + if ( + rollback_failed + and previous + and previous.get('operation_id') == str(operation_id) + ): + for key in ('data_fingerprint', 'rollback_relative'): + if key in previous: + document[key] = previous[key] + _write(document) + + +def protected_operation_directory_name(): + """Return the rollback-failure directory that startup must preserve.""" + document = _read() + if not document or document.get('state') != 'rollback_failed': + return None + operation_id = str(document.get('operation_id') or '') + path = PurePath(operation_id) + if ( + not operation_id + or path.is_absolute() + or len(path.parts) != 1 + or operation_id in {'.', '..'} + ): + return None + return f'operation-{operation_id}' + + +def clear_failed_status_after_cli_restore() -> None: + """Leave recovery maintenance after a successful explicit CLI restore.""" + global _state, _state_path + document = _read() + if not document or document.get('state') != 'rollback_failed': + return + path = _status_path() + path.unlink(missing_ok=True) + from .storage_utils import fsync_parent_directory + fsync_parent_directory(path) + _state = None + _state_path = None + + +def public_status(): + document = _read() + if document is None: + return {'state': 'idle', 'message': None} + if ( + document.get('state') not in {'preparing', 'in_progress', 'rollback_failed'} + and time.time() - float(document.get('updated_at', 0)) + > config.BACKUP_OPERATION_TIMEOUT + ): + try: + _status_path().unlink() + except FileNotFoundError: + pass + global _state, _state_path + _state = None + _state_path = None + return {'state': 'idle', 'message': None} + return { + 'state': document.get('state', 'failed'), + 'message': document.get('message'), + } + + +def recover_interrupted_restore() -> None: + """Rollback an interrupted restore before database initialization.""" + document = _read() + if not document or document.get('state') not in {'preparing', 'in_progress'}: + return + operation_id = str(document.get('operation_id') or 'unknown') + if document.get('data_fingerprint') != _data_fingerprint(): + mark_failed(operation_id, 'Interrupted restore belongs to another data directory') + return + if document.get('state') == 'preparing': + mark_failed(operation_id, 'Restore stopped before persistent state changed') + return + + relative = PurePath(str(document.get('rollback_relative') or '')) + if relative.is_absolute() or '..' in relative.parts or len(relative.parts) != 2: + mark_failed( + operation_id, + 'Interrupted restore has no safe rollback reference', + rollback_failed=True, + ) + return + rollback = ensure_backup_temp_dir() / Path(*relative.parts) + try: + from .backup_manager import restore_backup + from .session_epoch import reset_cache, rotate_epoch + + with operation_lock(): + restore_backup(rollback, config.DATA_DIR) + reset_cache() + rotate_epoch() + mark_failed(operation_id, 'Interrupted restore rolled back automatically') + except Exception: + mark_failed( + operation_id, + 'Interrupted restore rollback failed; use the CLI recovery path', + rollback_failed=True, + ) diff --git a/app/online_backup.py b/app/online_backup.py new file mode 100644 index 0000000..c8f4107 --- /dev/null +++ b/app/online_backup.py @@ -0,0 +1,73 @@ +"""Online-consistent backup creation for the running WebSSH application.""" + +import os +from pathlib import Path +import sqlite3 +import stat +import tempfile + +import config + +from .backup_coordination import ( + ensure_backup_temp_dir, + operation_lock, + snapshot_barrier, +) +from .backup_manager import ( + BackupIntegrityError, + _stage_source, + create_backup, +) + + +_SQLITE_RUNTIME_FILES = frozenset({ + 'app.db', + 'app.db-journal', + 'app.db-shm', + 'app.db-wal', +}) + + +def _snapshot_sqlite(source_path: Path, destination_path: Path) -> None: + source_path = Path(source_path) + metadata = source_path.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + raise BackupIntegrityError('SQLite source must be a regular file') + + source = sqlite3.connect(str(source_path), timeout=30) + destination = sqlite3.connect(str(destination_path), timeout=30) + try: + source.execute('PRAGMA query_only = ON') + source.backup(destination, pages=256, sleep=0.01) + result = destination.execute('PRAGMA quick_check').fetchone() + if result != ('ok',): + raise BackupIntegrityError('SQLite snapshot integrity check failed') + finally: + destination.close() + source.close() + os.chmod(destination_path, 0o600) + + +def create_online_backup(data_dir, destination, *, held_token=None): + """Create and verify a current-format archive without copying live SQLite.""" + data_dir = Path(data_dir).resolve(strict=True) + destination = Path(destination) + if destination.resolve(strict=False).is_relative_to(data_dir): + raise ValueError('backup destination must be outside DATA_DIR') + + operation_root = ensure_backup_temp_dir() + with operation_lock(held_token=held_token): + with tempfile.TemporaryDirectory( + dir=operation_root, + prefix='snapshot-', + ) as working_directory: + snapshot = Path(working_directory) / 'data' + snapshot.mkdir(mode=0o700) + with snapshot_barrier(): + _stage_source( + data_dir, + snapshot, + excluded_relative_paths=_SQLITE_RUNTIME_FILES, + ) + _snapshot_sqlite(data_dir / 'app.db', snapshot / 'app.db') + return create_backup(snapshot, destination) diff --git a/app/request_limits.py b/app/request_limits.py index 30a0722..65ca14a 100644 --- a/app/request_limits.py +++ b/app/request_limits.py @@ -16,7 +16,10 @@ "transfers", "webauthn", }) -_STREAMING_ENDPOINTS = frozenset({"transfers.upload_transfer"}) +_STREAMING_ENDPOINTS = frozenset({ + "admin_backup.upload_backup", + "transfers.upload_transfer", +}) def _policy_for_request(): @@ -24,6 +27,8 @@ def _policy_for_request(): limit = config.MAX_RECOVERY_JSON_SIZE elif request.blueprint == "webauthn": limit = config.MAX_WEBAUTHN_JSON_SIZE + elif request.endpoint == "admin_backup.upload_backup": + limit = config.BACKUP_UPLOAD_MAX_SIZE elif request.endpoint in _STREAMING_ENDPOINTS: limit = config.MAX_UPLOAD_SIZE else: diff --git a/app/restore_service.py b/app/restore_service.py new file mode 100644 index 0000000..a7a0141 --- /dev/null +++ b/app/restore_service.py @@ -0,0 +1,162 @@ +"""Transactional web restore orchestration and reliable process restart.""" + +import os +from pathlib import Path +import signal +import sqlite3 +import threading +import time + +import config + +from . import connection_pool, ssh_manager +from .audit_logger import log_security_event +from .backup_coordination import operation_lock +from .backup_manager import restore_backup +from .backup_operations import backup_operations +from .online_backup import create_online_backup +from .maintenance_mode import ( + begin_preparing, + mark_failed, + mark_in_progress, + mark_succeeded, +) +from .session_epoch import reset_cache, rotate_epoch + + +def _close_active_ssh_sessions(): + with ssh_manager.sessions_lock: + session_ids = tuple(ssh_manager.sessions) + for session_id in session_ids: + ssh_manager.close_session(session_id, kill_tmux=False) + + +def _disconnect_sockets(socketio): + server = getattr(socketio, 'server', None) + manager = getattr(server, 'manager', None) + if server is None or manager is None: + return + try: + participants = tuple(manager.get_participants('/', None)) + except Exception: + return + for participant in participants: + sid = participant[0] if isinstance(participant, tuple) else participant + try: + server.disconnect(sid, namespace='/') + except Exception: + continue + + +def _clear_restored_runtime_sessions(database_path: Path): + connection = sqlite3.connect(str(database_path), timeout=30) + try: + connection.execute('DELETE FROM socket_sessions') + connection.execute('DELETE FROM ssh_sessions') + connection.commit() + finally: + connection.close() + + +def request_process_restart(delay=1.0): + time.sleep(delay) + if 'gunicorn' in os.environ.get('SERVER_SOFTWARE', '').lower(): + os.kill(os.getppid(), signal.SIGTERM) + os._exit(0) + + +def _perform_restore(app, socketio, record, username, source_ip, + restart_callback): + operation_id = record.operation_id + rollback_archive = record.directory / 'rollback.zip' + restart_required = False + runtime_stopped = False + rollback_available = False + rollback_failed = False + try: + with operation_lock() as token: + begin_preparing(operation_id) + lifecycle = app.extensions['runtime_lifecycle'] + shutdown = lifecycle.begin_shutdown( + config.RUNTIME_SHUTDOWN_GRACE_SECONDS + ) + runtime_stopped = True + if shutdown.remaining: + raise RuntimeError( + 'runtime activity did not stop before restore' + ) + _close_active_ssh_sessions() + connection_pool.temp_connection_pool.close_all_connections() + _disconnect_sockets(socketio) + + with app.app_context(): + from .models import db + db.session.remove() + db.engine.dispose() + + create_online_backup( + config.DATA_DIR, + rollback_archive, + held_token=token, + ) + rollback_available = True + relative = rollback_archive.relative_to( + record.directory.parent + ).as_posix() + mark_in_progress(operation_id, relative) + + restore_backup(record.archive_path, config.DATA_DIR) + reset_cache() + rotate_epoch() + _clear_restored_runtime_sessions(Path(config.DATA_DIR) / 'app.db') + mark_succeeded(operation_id) + log_security_event( + 'RESTORE_SUCCEEDED', user=username, ip=source_ip + ) + restart_required = True + except Exception as restore_error: + if rollback_available: + try: + restore_backup(rollback_archive, config.DATA_DIR) + reset_cache() + rotate_epoch() + _clear_restored_runtime_sessions(Path(config.DATA_DIR) / 'app.db') + restart_required = True + except Exception: + rollback_failed = True + mark_failed( + operation_id, + ( + 'Restore failed and emergency rollback failed; use CLI recovery' + if rollback_failed + else ( + 'Restore failed; original state was restored' + if rollback_available + else 'Restore failed before persistent state changed' + ) + ), + rollback_failed=rollback_failed, + ) + log_security_event( + 'RESTORE_FAILED', + user=username, + ip=source_ip, + error_type=type(restore_error).__name__, + rollback_failed=rollback_failed, + ) + finally: + if not rollback_failed: + backup_operations.remove(operation_id) + if restart_required or runtime_stopped: + restart_callback() + +def start_restore(app, socketio, record, username, source_ip, + restart_callback=request_process_restart): + thread = threading.Thread( + target=_perform_restore, + args=(app, socketio, record, username, source_ip, restart_callback), + name='webssh-restore', + daemon=False, + ) + thread.start() + return thread diff --git a/app/secret_rotation.py b/app/secret_rotation.py index 09dacfb..45099a0 100644 --- a/app/secret_rotation.py +++ b/app/secret_rotation.py @@ -229,7 +229,7 @@ def _rollback_key_files(originals): raise RuntimeError('secret rotation rollback failed') from rollback_error -def rotate_secret(old_secret, new_secret, data_dir): +def _rotate_secret_locked(old_secret, new_secret, data_dir): _validate_secret(old_secret, 'old') _validate_secret(new_secret, 'new') if hmac.compare_digest(old_secret, new_secret): @@ -333,3 +333,12 @@ def rotate_secret(old_secret, new_secret, data_dir): ) from rollback_error return RotationReport(len(key_files), backup_path) + + +def rotate_secret(old_secret, new_secret, data_dir): + """Rotate the persisted secret without racing backup or restore.""" + from .backup_coordination import operation_lock, persistent_write + + with operation_lock(): + with persistent_write(): + return _rotate_secret_locked(old_secret, new_secret, data_dir) diff --git a/app/session_epoch.py b/app/session_epoch.py new file mode 100644 index 0000000..6ad846a --- /dev/null +++ b/app/session_epoch.py @@ -0,0 +1,61 @@ +"""Server-side generation used to invalidate every browser login session.""" + +from pathlib import Path +import secrets +import threading + +import config + +from .storage_utils import atomic_write_bytes + + +_lock = threading.RLock() +_cached = None +_cached_path = None + + +def _path() -> Path: + return Path(config.DATA_DIR) / 'session_epoch' + + +def _valid(value: str) -> bool: + return ( + len(value) == 64 + and all(character in '0123456789abcdef' for character in value) + ) + + +def current_epoch() -> str: + global _cached, _cached_path + with _lock: + path = _path().resolve(strict=False) + if _cached is not None and _cached_path == path: + return _cached + try: + value = path.read_text(encoding='ascii').strip() + except FileNotFoundError: + return rotate_epoch() + if not _valid(value): + raise RuntimeError('persisted session epoch is invalid') + _cached = value + _cached_path = path + return value + + +def rotate_epoch() -> str: + global _cached, _cached_path + with _lock: + value = secrets.token_hex(32) + path = _path() + path.parent.mkdir(parents=True, exist_ok=True) + atomic_write_bytes(path, (value + '\n').encode('ascii'), mode=0o600) + _cached = value + _cached_path = path.resolve(strict=False) + return value + + +def reset_cache() -> None: + global _cached, _cached_path + with _lock: + _cached = None + _cached_path = None diff --git a/app/storage_utils.py b/app/storage_utils.py index 165a557..8007e59 100644 --- a/app/storage_utils.py +++ b/app/storage_utils.py @@ -16,6 +16,7 @@ from typing import Callable, TypeVar from .storage_errors import StorageCorruptionError +from .backup_coordination import persistent_write T = TypeVar('T') @@ -24,6 +25,44 @@ _locks_guard = threading.Lock() +class _CoordinatedStorageLock: + """Preserve the Lock API while coordinating its mutation cycles.""" + + def __init__(self): + self._lock = threading.Lock() + self._write_context = None + + def acquire(self, *args, **kwargs): + acquired = self._lock.acquire(*args, **kwargs) + if not acquired: + return False + context = persistent_write() + try: + context.__enter__() + except Exception: + self._lock.release() + raise + self._write_context = context + return True + + def release(self): + context = self._write_context + self._write_context = None + if context is not None: + context.__exit__(None, None, None) + self._lock.release() + + def locked(self): + return self._lock.locked() + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.release() + + def safe_reference_name(value): """Return bounded printable display text for cross-store references.""" value = value if isinstance(value, str) else '' @@ -43,7 +82,7 @@ def storage_lock(key): with _locks_guard: lock = _locks.get(key) if lock is None: - lock = threading.Lock() + lock = _CoordinatedStorageLock() _locks[key] = lock return lock @@ -125,30 +164,31 @@ def atomic_write_bytes(path: Path, payload: bytes, mode: int = 0o600) -> None: If the final directory fsync fails, the exception is surfaced even though ``os.replace`` has already made the new file active. """ - path = Path(path) - temporary_path = None - try: - with tempfile.NamedTemporaryFile( - mode='wb', - dir=path.parent, - prefix=f'.{path.name}.', - suffix='.tmp', - delete=False, - ) as handle: - temporary_path = Path(handle.name) - handle.write(payload) - handle.flush() - os.fsync(handle.fileno()) - os.chmod(temporary_path, mode) - os.replace(temporary_path, path) + with persistent_write(): + path = Path(path) temporary_path = None - fsync_parent_directory(path) - finally: - if temporary_path is not None: - try: - temporary_path.unlink() - except FileNotFoundError: - pass + try: + with tempfile.NamedTemporaryFile( + mode='wb', + dir=path.parent, + prefix=f'.{path.name}.', + suffix='.tmp', + delete=False, + ) as handle: + temporary_path = Path(handle.name) + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary_path, mode) + os.replace(temporary_path, path) + temporary_path = None + fsync_parent_directory(path) + finally: + if temporary_path is not None: + try: + temporary_path.unlink() + except FileNotFoundError: + pass def atomic_copy_file( @@ -157,6 +197,11 @@ def atomic_copy_file( mode: int = 0o600, ) -> None: """Durably stream a regular file into an atomic destination replace.""" + with persistent_write(): + _atomic_copy_file(source, destination, mode) + + +def _atomic_copy_file(source: Path, destination: Path, mode: int) -> None: source = Path(source) destination = Path(destination) source_stat = source.lstat() diff --git a/app/user_lifecycle.py b/app/user_lifecycle.py index 1631b67..91e252f 100644 --- a/app/user_lifecycle.py +++ b/app/user_lifecycle.py @@ -7,6 +7,7 @@ from .audit_logger import log_error, log_info, log_warning from .models import db, SocketSession, SSHSession from . import connection_pool, ssh_manager +from .backup_coordination import persistent_write def revoke_user_access(user_id, socketio_instance=None): @@ -143,24 +144,25 @@ def restore_quarantined_user_data(original, quarantined): def delete_user_account(user, socketio_instance=None): """Revoke a user and delete their row without exposing retained files.""" - user_id = int(user.id) - revoke_user_access(user_id, socketio_instance) - original = quarantined = None + with persistent_write(): + user_id = int(user.id) + revoke_user_access(user_id, socketio_instance) + original = quarantined = None - try: - original, quarantined = quarantine_user_data(user_id) - db.session.delete(user) - db.session.commit() - except Exception: - db.session.rollback() try: - restore_quarantined_user_data(original, quarantined) - except Exception as restore_error: - log_error( - "Failed to restore quarantined user data", - user_id=user_id, - error=str(restore_error), - ) - raise + original, quarantined = quarantine_user_data(user_id) + db.session.delete(user) + db.session.commit() + except Exception: + db.session.rollback() + try: + restore_quarantined_user_data(original, quarantined) + except Exception as restore_error: + log_error( + "Failed to restore quarantined user data", + user_id=user_id, + error=str(restore_error), + ) + raise - return quarantined + return quarantined diff --git a/config.py b/config.py index 1bbf279..a97ddd7 100644 --- a/config.py +++ b/config.py @@ -1,6 +1,7 @@ import os import secrets import ipaddress +import tempfile from pathlib import Path from datetime import timedelta from urllib.parse import urlsplit @@ -146,6 +147,17 @@ def _non_negative_int_env(name, default): BACKUP_MAX_MANIFEST_SIZE = _positive_int_env( 'BACKUP_MAX_MANIFEST_SIZE', 10 * 1024 * 1024 ) +BACKUP_UPLOAD_MAX_SIZE = _positive_int_env( + 'BACKUP_UPLOAD_MAX_SIZE', 1024 * 1024 * 1024 +) +BACKUP_OPERATION_TIMEOUT = _positive_int_env( + 'BACKUP_OPERATION_TIMEOUT', 1800 +) +BACKUP_DOWNLOAD_TTL = _positive_int_env('BACKUP_DOWNLOAD_TTL', 600) +BACKUP_TEMP_DIR = Path(os.environ.get( + 'BACKUP_TEMP_DIR', + Path(tempfile.gettempdir()) / 'webssh-backup-operations', +)) # Atomic, in-process resource quotas. Per-user defaults remain below their @@ -393,6 +405,18 @@ def _csv_env(name): RATELIMIT_LOGIN_LIMIT = os.environ.get('RATELIMIT_LOGIN_LIMIT', '5 per minute') RATELIMIT_DEFAULT = os.environ.get('RATELIMIT_DEFAULT', '200 per hour') RATELIMIT_REAUTH = os.environ.get('RATELIMIT_REAUTH', '5 per minute') +RATELIMIT_BACKUP_CREATE = os.environ.get( + 'RATELIMIT_BACKUP_CREATE', '3 per hour' +) +RATELIMIT_BACKUP_UPLOAD = os.environ.get( + 'RATELIMIT_BACKUP_UPLOAD', '5 per hour' +) +RATELIMIT_BACKUP_DOWNLOAD = os.environ.get( + 'RATELIMIT_BACKUP_DOWNLOAD', '10 per hour' +) +RATELIMIT_BACKUP_RESTORE = os.environ.get( + 'RATELIMIT_BACKUP_RESTORE', '3 per hour' +) # Per-user limit on SSH connection attempts via WebSocket (ssh_connect / # quick_connect). Prevents an authenticated user from abusing the server as an # unthrottled SSH brute-force / port-scan proxy against third-party hosts. diff --git a/docker-compose.yml b/docker-compose.yml index 46dbffc..8d40507 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,6 +90,17 @@ services: # but remains off here because homelabs commonly connect to private IPs. # - BLOCK_INTERNAL_SSH=true + # === Native web backup and restore === + # Always available to administrators. Restore always requires two + # confirmations plus the current administrator password. Temporary + # snapshots and uploaded archives use an instance-specific namespace + # outside /app/data and expire. + # Override operational limits only when the defaults do not fit. + # - BACKUP_UPLOAD_MAX_SIZE=1073741824 + # - BACKUP_OPERATION_TIMEOUT=1800 + # - BACKUP_DOWNLOAD_TTL=600 + # - BACKUP_TEMP_DIR=/tmp/webssh-backup-operations + # === Passkeys and OIDC (optional, disabled by default) === # Passkeys require the exact public browser domain and origin. # - WEBAUTHN_ENABLED=true diff --git a/static/css/admin.css b/static/css/admin.css index 195663e..11ec38d 100644 --- a/static/css/admin.css +++ b/static/css/admin.css @@ -202,6 +202,75 @@ line-height: 1.5; } +.admin-backup-warning { + display: flex; + align-items: flex-start; + gap: 12px; + margin-bottom: 18px; + padding: 14px 16px; + border: 1px solid #f59e0b; + border-radius: 10px; + background: color-mix(in srgb, #f59e0b 12%, var(--bg-secondary)); +} + +.admin-backup-warning p, +.admin-backup-destructive { + margin: 0; + line-height: 1.55; +} + +.admin-backup-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(min(100%, 320px), 1fr)); + gap: 16px; +} + +.admin-backup-grid .admin-setting-row { + max-width: none; +} + +.admin-backup-grid h2 { + margin: 0; + font-size: 18px; +} + +.admin-backup-actions { + margin: 16px 0 0; +} + +.admin-backup-status { + min-height: 1.5em; + margin: 12px 0 0; + color: var(--text-secondary); + overflow-wrap: anywhere; +} + +.admin-backup-status.error, +.admin-backup-destructive { + color: #f43f5e; +} + +.admin-backup-summary dl { + display: grid; + gap: 8px; + margin: 14px 0; +} + +.admin-backup-summary dl div { + display: flex; + justify-content: space-between; + gap: 16px; +} + +.admin-backup-summary dt { + color: var(--text-muted); +} + +.admin-backup-summary dd { + margin: 0; + font-weight: 600; +} + .admin-oidc-list { display: grid; gap: 8px; diff --git a/static/js/admin.js b/static/js/admin.js index 62ea100..7ad86fe 100644 --- a/static/js/admin.js +++ b/static/js/admin.js @@ -62,11 +62,12 @@ document.querySelectorAll('.admin-tab').forEach(x => x.classList.remove('active')); tab.classList.add('active'); const name = tab.dataset.tab; - ['users', 'audit', 'settings'].forEach(n => { + ['users', 'audit', 'settings', 'backup'].forEach(n => { document.getElementById('tab-' + n)?.classList.toggle('hidden', n !== name); }); if (name === 'audit') { loadAudit(); } if (name === 'settings') { loadSettings(); } + if (name === 'backup') { loadRestoreStatus(); } }); }); } @@ -520,6 +521,321 @@ document.getElementById('globalHostKeyRefresh')?.addEventListener('click', loadGlobalHostKeys); } + // ---- Backup and restore ---- + const backupState = { + createdOperationId: null, + uploadedOperationId: null, + confirmationToken: null, + createPollGeneration: 0, + uploadPollGeneration: 0 + }; + const restoreStatusFlow = window.WebSSHRestoreStatus.createRestoreStatusFlow({ + storage: window.sessionStorage, + fetchStatus: () => api('/admin/api/backups/restore/status'), + checkReady: async () => { + const response = await fetch(`${APP_ROOT}/ready`, { + cache: 'no-store', + credentials: 'same-origin' + }); + return response.ok; + }, + present: status => { + const failed = ['failed', 'rollback_failed'].includes(status.state); + const message = status.message || status.state; + setBackupStatus('restoreGlobalStatus', message, failed); + if (status.state === 'succeeded') { + notify(message, 'success'); + } else if (failed) { + notify(message, 'error'); + } + }, + presentRestarting: () => setBackupStatus( + 'restoreGlobalStatus', + t('backup.restarting', 'WebSSH is restarting. Sign in again when the service is ready.') + ), + reload: () => window.location.reload() + }); + + function setBackupStatus(id, message, error) { + const target = document.getElementById(id); + if (!target) { return; } + target.textContent = message; + target.classList.toggle('error', !!error); + } + + function formatBytes(value) { + const size = Number(value) || 0; + if (size < 1024) { return `${size} B`; } + if (size < 1024 * 1024) { return `${(size / 1024).toFixed(1)} KiB`; } + if (size < 1024 * 1024 * 1024) { + return `${(size / (1024 * 1024)).toFixed(1)} MiB`; + } + return `${(size / (1024 * 1024 * 1024)).toFixed(1)} GiB`; + } + + async function pollBackupOperation(operationId, generation, generationKey, onComplete) { + if (!operationId || generation !== backupState[generationKey]) { return; } + try { + const record = await api(`/admin/api/backups/${operationId}`); + if (generation !== backupState[generationKey]) { return; } + if (['ready', 'verified', 'failed'].includes(record.status)) { + onComplete(record); + return; + } + onComplete(record, true); + setTimeout(() => pollBackupOperation( + operationId, generation, generationKey, onComplete + ), 900); + } catch (error) { + onComplete({ status: 'failed', error: error.message }); + } + } + + async function createBackup() { + const button = document.getElementById('backupCreateBtn'); + const download = document.getElementById('backupDownloadBtn'); + button.disabled = true; + download.disabled = true; + backupState.createdOperationId = null; + setBackupStatus('backupCreateStatus', t('backup.creating', 'Creating and verifying backup...')); + try { + const record = await api('/admin/api/backups', { method: 'POST' }); + backupState.createdOperationId = record.operation_id; + const generation = ++backupState.createPollGeneration; + pollBackupOperation(record.operation_id, generation, 'createPollGeneration', (update, pending) => { + if (pending) { + setBackupStatus('backupCreateStatus', t('backup.creating', 'Creating and verifying backup...')); + return; + } + button.disabled = false; + if (update.status === 'ready') { + download.disabled = false; + setBackupStatus( + 'backupCreateStatus', + t('backup.ready', 'Backup verified and ready for one-time download.') + ); + notify(t('backup.ready', 'Backup verified and ready for one-time download.'), 'success'); + } else { + setBackupStatus('backupCreateStatus', update.error || t('backup.failed', 'Backup operation failed.'), true); + } + }); + } catch (error) { + button.disabled = false; + setBackupStatus('backupCreateStatus', error.message, true); + notify(error.message, 'error'); + } + } + + function downloadBackup() { + const operationId = backupState.createdOperationId; + const button = document.getElementById('backupDownloadBtn'); + if (!operationId) { return; } + button.disabled = true; + const form = document.createElement('form'); + const frame = document.createElement('iframe'); + frame.name = `backup-download-${Date.now()}`; + frame.hidden = true; + form.method = 'POST'; + form.action = `${APP_ROOT}/admin/api/backups/${operationId}/download`; + form.target = frame.name; + form.hidden = true; + const csrf = document.createElement('input'); + csrf.type = 'hidden'; + csrf.name = 'csrf_token'; + csrf.value = CSRF; + form.appendChild(csrf); + document.body.appendChild(frame); + document.body.appendChild(form); + form.submit(); + form.remove(); + backupState.createdOperationId = null; + setBackupStatus('backupCreateStatus', t('backup.downloaded', 'Download started; the server copy is one-time use.')); + } + + function renderBackupSummary(summary) { + const panel = document.getElementById('backupValidationPanel'); + panel.hidden = false; + document.getElementById('backupFormatVersion').textContent = summary.format_version; + document.getElementById('backupDataSchemaVersion').textContent = summary.data_schema_version; + document.getElementById('backupCurrentDataSchemaVersion').textContent = summary.current_data_schema_version; + document.getElementById('backupCreatedAt').textContent = summary.created_at + ? fmtDate(summary.created_at) + : t('backup.notRecorded', 'Not recorded'); + document.getElementById('backupLegacy').textContent = summary.legacy + ? t('common.yes', 'Yes') + : t('common.no', 'No'); + document.getElementById('backupFileCount').textContent = summary.file_count; + document.getElementById('backupTotalSize').textContent = formatBytes(summary.total_uncompressed_size); + document.getElementById('backupCompatible').textContent = summary.compatible + ? t('common.yes', 'Yes') + : t('common.no', 'No'); + const reasonKeys = { + 'backup data schema is current': 'backup.compatibilityCurrent', + 'legacy archive can be migrated': 'backup.compatibilityLegacy', + 'backup data schema can be migrated': 'backup.compatibilityMigratable', + 'backup data schema is newer than this WebSSH version': 'backup.compatibilityFuture', + 'no complete migration path for backup data schema': 'backup.compatibilityNoMigration' + }; + const reasonKey = reasonKeys[summary.compatibility_reason]; + document.getElementById('backupCompatibilityReason').textContent = reasonKey + ? t(reasonKey, summary.compatibility_reason) + : summary.compatibility_reason; + document.getElementById('backupRestoreBtn').disabled = summary.compatible !== true; + } + + async function uploadBackup() { + const input = document.getElementById('backupUploadFile'); + const button = document.getElementById('backupUploadBtn'); + const file = input.files?.[0]; + if (!file) { + notify(t('backup.selectFile', 'Select a ZIP backup first.'), 'error'); + return; + } + button.disabled = true; + document.getElementById('backupValidationPanel').hidden = true; + document.getElementById('backupRestoreBtn').disabled = true; + setBackupStatus('backupUploadStatus', t('backup.verifying', 'Uploading and verifying backup...')); + try { + const response = await fetch(`${APP_ROOT}/admin/api/backups/upload`, { + method: 'POST', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/zip', + 'X-CSRFToken': CSRF + }, + body: file + }); + const record = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(record.error || `Upload failed (${response.status})`); + } + backupState.uploadedOperationId = record.operation_id; + const generation = ++backupState.uploadPollGeneration; + pollBackupOperation(record.operation_id, generation, 'uploadPollGeneration', (update, pending) => { + if (pending) { + setBackupStatus('backupUploadStatus', t('backup.verifying', 'Uploading and verifying backup...')); + return; + } + button.disabled = false; + input.value = ''; + if (update.status === 'verified') { + renderBackupSummary(update.summary); + setBackupStatus('backupUploadStatus', t('backup.verified', 'Backup verified successfully.')); + notify(t('backup.verified', 'Backup verified successfully.'), 'success'); + } else { + backupState.uploadedOperationId = null; + setBackupStatus('backupUploadStatus', update.error || t('backup.failed', 'Backup operation failed.'), true); + } + }); + } catch (error) { + button.disabled = false; + setBackupStatus('backupUploadStatus', error.message, true); + notify(error.message, 'error'); + } + } + + function showModal(id, show) { + const modal = document.getElementById(id); + modal?.classList.toggle('show', show); + modal?.setAttribute('aria-hidden', show ? 'false' : 'true'); + } + + function closeRestoreModals() { + showModal('restoreFirstConfirmModal', false); + showModal('restoreSecondConfirmModal', false); + document.getElementById('restoreFirstAcknowledge').checked = false; + document.getElementById('restoreFinalAcknowledge').checked = false; + document.getElementById('restorePhrase').value = ''; + document.getElementById('restorePassword').value = ''; + backupState.confirmationToken = null; + } + + async function continueRestoreConfirmation() { + if (!document.getElementById('restoreFirstAcknowledge').checked) { + notify(t('backup.ackRequired', 'Acknowledge the restore impact first.'), 'error'); + return; + } + try { + const result = await api( + `/admin/api/backups/${backupState.uploadedOperationId}/restore/prepare`, + { + method: 'POST', + body: { acknowledge_sensitive_restore: true } + } + ); + backupState.confirmationToken = result.confirmation_token; + showModal('restoreFirstConfirmModal', false); + showModal('restoreSecondConfirmModal', true); + document.getElementById('restorePhrase').focus(); + } catch (error) { + notify(error.message, 'error'); + } + } + + async function startRestore() { + const password = document.getElementById('restorePassword'); + const body = { + confirmation_token: backupState.confirmationToken, + confirmation_phrase: document.getElementById('restorePhrase').value, + password: password.value, + confirm_destructive_restore: document.getElementById('restoreFinalAcknowledge').checked + }; + try { + await api( + `/admin/api/backups/${backupState.uploadedOperationId}/restore`, + { method: 'POST', body } + ); + restoreStatusFlow.markPending(); + closeRestoreModals(); + document.getElementById('backupRestoreBtn').disabled = true; + setBackupStatus( + 'restoreGlobalStatus', + t('backup.restoreStarted', 'Restore started. WebSSH is entering maintenance mode and will restart.') + ); + restoreStatusFlow.poll(); + } catch (error) { + notify(error.message, 'error'); + } finally { + password.value = ''; + body.password = ''; + } + } + + async function loadRestoreStatus() { + try { + const status = await api('/admin/api/backups/restore/status'); + if (status.state !== 'idle') { + setBackupStatus('restoreGlobalStatus', status.message || status.state, + ['failed', 'rollback_failed'].includes(status.state)); + } + } catch (error) { + if (!/401|log in/i.test(error.message)) { + setBackupStatus('restoreGlobalStatus', error.message, true); + } + } + } + + function initBackupRestore() { + document.getElementById('backupCreateBtn')?.addEventListener('click', createBackup); + document.getElementById('backupDownloadBtn')?.addEventListener('click', downloadBackup); + document.getElementById('backupUploadBtn')?.addEventListener('click', uploadBackup); + document.getElementById('backupRestoreBtn')?.addEventListener('click', () => { + if (backupState.uploadedOperationId) { + showModal('restoreFirstConfirmModal', true); + document.getElementById('restoreFirstAcknowledge').focus(); + } + }); + document.getElementById('restoreFirstContinue')?.addEventListener('click', continueRestoreConfirmation); + document.getElementById('restoreStartBtn')?.addEventListener('click', startRestore); + ['restoreFirstCancel', 'restoreFirstCancelButton', 'restoreSecondCancel', 'restoreSecondCancelButton'] + .forEach(id => document.getElementById(id)?.addEventListener('click', closeRestoreModals)); + if (restoreStatusFlow.isPending()) { + restoreStatusFlow.resume(); + } else { + loadRestoreStatus(); + } + } + async function loadGlobalHostKeys() { const body = document.getElementById('globalHostKeyList'); if (!body) { return; } @@ -574,6 +890,7 @@ initUsers(); initAudit(); initSettings(); + initBackupRestore(); loadUsers(); loadGlobalHostKeys(); }); diff --git a/static/js/i18n.js b/static/js/i18n.js index 835bace..a925b55 100644 --- a/static/js/i18n.js +++ b/static/js/i18n.js @@ -221,6 +221,51 @@ const translations = { 'admin.confirmTargetUsername': 'Confirm target username', 'admin.stableOidcSubject': 'Stable OIDC subject', 'admin.continue': 'Continue', + 'backup.title': 'Backup & Restore', + 'backup.sensitiveWarning': 'Backup archives contain passwords, application secrets, SSH key metadata, and encrypted private keys. Store every downloaded archive encrypted and off-host.', + 'backup.create': 'Create backup', + 'backup.createHint': 'Create an online-consistent snapshot while WebSSH remains available.', + 'backup.download': 'Download backup', + 'backup.statusIdle': 'No operation running.', + 'backup.upload': 'Upload and verify', + 'backup.uploadHint': 'Select a WebSSH ZIP backup. The archive is retained temporarily and can only be used by this administrator session.', + 'backup.verify': 'Upload and verify', + 'backup.validation': 'Validation result', + 'backup.formatVersion': 'Format version', + 'backup.files': 'Files', + 'backup.totalSize': 'Uncompressed size', + 'backup.compatible': 'Compatible', + 'backup.dataSchemaVersion': 'Backup data schema', + 'backup.currentDataSchemaVersion': 'Current data schema', + 'backup.createdAt': 'Created at', + 'backup.legacy': 'Legacy archive', + 'backup.compatibilityReason': 'Compatibility', + 'backup.notRecorded': 'Not recorded', + 'backup.compatibilityCurrent': 'The backup data schema is current.', + 'backup.compatibilityLegacy': 'Legacy v1 archive; a migration path is available.', + 'backup.compatibilityMigratable': 'The older backup data schema can be migrated.', + 'backup.compatibilityFuture': 'This backup was created by a newer data schema and cannot be restored here.', + 'backup.compatibilityNoMigration': 'No complete migration path is available for this backup.', + 'backup.restoreWarning': 'Restore replaces the complete persistent state, invalidates all sessions, enters maintenance mode, and restarts WebSSH.', + 'backup.restore': 'Restore verified backup', + 'backup.firstConfirm': 'Confirm restore impact', + 'backup.firstAcknowledge': 'I understand that the current persistent state will be replaced.', + 'backup.cancel': 'Cancel', + 'backup.secondConfirm': 'Final destructive confirmation', + 'backup.typeRestore': 'Type RESTORE', + 'backup.password': 'Administrator password', + 'backup.destructiveConfirm': 'Permanently replace the current persistent state now.', + 'backup.startRestore': 'Start restore', + 'backup.creating': 'Creating and verifying backup...', + 'backup.ready': 'Backup verified and ready for one-time download.', + 'backup.failed': 'Backup operation failed.', + 'backup.downloaded': 'Download started; the server copy is one-time use.', + 'backup.selectFile': 'Select a ZIP backup first.', + 'backup.verifying': 'Uploading and verifying backup...', + 'backup.verified': 'Backup verified successfully.', + 'backup.ackRequired': 'Acknowledge the restore impact first.', + 'backup.restoreStarted': 'Restore started. WebSSH is entering maintenance mode and will restart.', + 'backup.restarting': 'WebSSH is restarting. Sign in again when the service is ready.', 'admin.oneTimeRecoveryCodes': 'One-time recovery codes', 'admin.copyRecoveryCodesNow': 'Copy these codes now. They will not be shown again.', 'admin.loading': 'Loading...', @@ -438,6 +483,8 @@ const translations = { 'common.cancel': 'Cancel', 'common.save': 'Save', 'common.close': 'Close', + 'common.yes': 'Yes', + 'common.no': 'No', 'common.tip': 'Tip', 'common.optional': 'Optional', 'common.apply': 'Apply', @@ -690,6 +737,51 @@ const translations = { 'admin.confirmTargetUsername': 'Xác nhận tên người dùng đích', 'admin.stableOidcSubject': 'Chủ thể OIDC ổn định', 'admin.continue': 'Tiếp tục', + 'backup.title': 'Sao lưu và khôi phục', + 'backup.sensitiveWarning': 'Bản sao lưu chứa mật khẩu, bí mật ứng dụng, siêu dữ liệu khóa SSH và khóa riêng được mã hóa. Hãy lưu trữ bản tải xuống ở dạng mã hóa và ngoài máy chủ.', + 'backup.create': 'Tạo bản sao lưu', + 'backup.createHint': 'Tạo ảnh chụp nhất quán trực tuyến trong khi WebSSH vẫn hoạt động.', + 'backup.download': 'Tải bản sao lưu', + 'backup.statusIdle': 'Không có thao tác nào đang chạy.', + 'backup.upload': 'Tải lên và xác minh', + 'backup.uploadHint': 'Chọn bản sao lưu ZIP của WebSSH. Tệp chỉ được giữ tạm thời cho phiên quản trị này.', + 'backup.verify': 'Tải lên và xác minh', + 'backup.validation': 'Kết quả xác minh', + 'backup.formatVersion': 'Phiên bản định dạng', + 'backup.files': 'Tệp', + 'backup.totalSize': 'Kích thước chưa nén', + 'backup.compatible': 'Tương thích', + 'backup.dataSchemaVersion': 'Lược đồ dữ liệu bản sao lưu', + 'backup.currentDataSchemaVersion': 'Lược đồ dữ liệu hiện tại', + 'backup.createdAt': 'Thời điểm tạo', + 'backup.legacy': 'Bản lưu kiểu cũ', + 'backup.compatibilityReason': 'Khả năng tương thích', + 'backup.notRecorded': 'Không được ghi lại', + 'backup.compatibilityCurrent': 'Lược đồ dữ liệu của bản sao lưu đang là phiên bản hiện tại.', + 'backup.compatibilityLegacy': 'Bản lưu v1 kiểu cũ; có sẵn đường dẫn di chuyển.', + 'backup.compatibilityMigratable': 'Có thể di chuyển lược đồ dữ liệu cũ của bản sao lưu.', + 'backup.compatibilityFuture': 'Bản sao lưu này dùng lược đồ dữ liệu mới hơn và không thể khôi phục tại đây.', + 'backup.compatibilityNoMigration': 'Không có đường dẫn di chuyển đầy đủ cho bản sao lưu này.', + 'backup.restoreWarning': 'Khôi phục sẽ thay thế toàn bộ dữ liệu bền vững, vô hiệu hóa mọi phiên, bật chế độ bảo trì và khởi động lại WebSSH.', + 'backup.restore': 'Khôi phục bản sao đã xác minh', + 'backup.firstConfirm': 'Xác nhận tác động khôi phục', + 'backup.firstAcknowledge': 'Tôi hiểu trạng thái bền vững hiện tại sẽ bị thay thế.', + 'backup.cancel': 'Hủy', + 'backup.secondConfirm': 'Xác nhận phá hủy cuối cùng', + 'backup.typeRestore': 'Nhập RESTORE', + 'backup.password': 'Mật khẩu quản trị viên', + 'backup.destructiveConfirm': 'Thay thế vĩnh viễn trạng thái bền vững hiện tại ngay bây giờ.', + 'backup.startRestore': 'Bắt đầu khôi phục', + 'backup.creating': 'Đang tạo và xác minh bản sao lưu...', + 'backup.ready': 'Bản sao lưu đã xác minh và sẵn sàng tải xuống một lần.', + 'backup.failed': 'Thao tác sao lưu thất bại.', + 'backup.downloaded': 'Đã bắt đầu tải xuống; bản sao trên máy chủ chỉ dùng một lần.', + 'backup.selectFile': 'Trước tiên hãy chọn bản sao lưu ZIP.', + 'backup.verifying': 'Đang tải lên và xác minh bản sao lưu...', + 'backup.verified': 'Xác minh bản sao lưu thành công.', + 'backup.ackRequired': 'Trước tiên hãy xác nhận tác động khôi phục.', + 'backup.restoreStarted': 'Đã bắt đầu khôi phục. WebSSH đang vào chế độ bảo trì và sẽ khởi động lại.', + 'backup.restarting': 'WebSSH đang khởi động lại. Hãy đăng nhập lại khi dịch vụ sẵn sàng.', 'admin.oneTimeRecoveryCodes': 'Mã khôi phục dùng một lần', 'admin.copyRecoveryCodesNow': 'Sao chép các mã này ngay. Chúng sẽ không được hiển thị lại.', 'admin.loading': 'Đang tải...', @@ -944,6 +1036,8 @@ const translations = { 'common.cancel': 'Hủy', 'common.save': 'Lưu', 'common.close': 'Đóng', + 'common.yes': 'Có', + 'common.no': 'Không', 'common.tip': 'Mẹo', 'common.optional': 'Tùy chọn', 'common.apply': 'Áp dụng', @@ -1232,6 +1326,51 @@ const translations = { 'admin.confirmTargetUsername': 'Zielbenutzernamen bestätigen', 'admin.stableOidcSubject': 'Stabile OIDC-Subjektkennung', 'admin.continue': 'Weiter', + 'backup.title': 'Backup & Wiederherstellung', + 'backup.sensitiveWarning': 'Backup-Archive enthalten Passwörter, Anwendungsgeheimnisse, SSH-Schlüsselmetadaten und verschlüsselte private Schlüssel. Speichere jedes heruntergeladene Archiv verschlüsselt und außerhalb des Hosts.', + 'backup.create': 'Backup erstellen', + 'backup.createHint': 'Erstellt einen online-konsistenten Snapshot, während WebSSH verfügbar bleibt.', + 'backup.download': 'Backup herunterladen', + 'backup.statusIdle': 'Kein Vorgang aktiv.', + 'backup.upload': 'Hochladen und prüfen', + 'backup.uploadHint': 'Wähle ein WebSSH-ZIP-Backup. Das Archiv wird nur temporär und nur für diese Admin-Sitzung aufbewahrt.', + 'backup.verify': 'Hochladen und prüfen', + 'backup.validation': 'Prüfergebnis', + 'backup.formatVersion': 'Formatversion', + 'backup.files': 'Dateien', + 'backup.totalSize': 'Unkomprimierte Größe', + 'backup.compatible': 'Kompatibel', + 'backup.dataSchemaVersion': 'Backup-Datenschema', + 'backup.currentDataSchemaVersion': 'Aktuelles Datenschema', + 'backup.createdAt': 'Erstellt am', + 'backup.legacy': 'Legacy-Archiv', + 'backup.compatibilityReason': 'Kompatibilität', + 'backup.notRecorded': 'Nicht aufgezeichnet', + 'backup.compatibilityCurrent': 'Das Backup-Datenschema ist aktuell.', + 'backup.compatibilityLegacy': 'Legacy-v1-Archiv; ein Migrationspfad ist vorhanden.', + 'backup.compatibilityMigratable': 'Das ältere Backup-Datenschema kann migriert werden.', + 'backup.compatibilityFuture': 'Dieses Backup verwendet ein neueres Datenschema und kann hier nicht wiederhergestellt werden.', + 'backup.compatibilityNoMigration': 'Für dieses Backup ist kein vollständiger Migrationspfad vorhanden.', + 'backup.restoreWarning': 'Die Wiederherstellung ersetzt den gesamten persistenten Zustand, beendet alle Sitzungen, aktiviert den Wartungsmodus und startet WebSSH neu.', + 'backup.restore': 'Geprüftes Backup wiederherstellen', + 'backup.firstConfirm': 'Auswirkungen bestätigen', + 'backup.firstAcknowledge': 'Ich verstehe, dass der aktuelle persistente Zustand ersetzt wird.', + 'backup.cancel': 'Abbrechen', + 'backup.secondConfirm': 'Letzte destruktive Bestätigung', + 'backup.typeRestore': 'RESTORE eingeben', + 'backup.password': 'Administratorpasswort', + 'backup.destructiveConfirm': 'Den aktuellen persistenten Zustand jetzt dauerhaft ersetzen.', + 'backup.startRestore': 'Wiederherstellung starten', + 'backup.creating': 'Backup wird erstellt und geprüft...', + 'backup.ready': 'Backup geprüft und zum einmaligen Download bereit.', + 'backup.failed': 'Backup-Vorgang fehlgeschlagen.', + 'backup.downloaded': 'Download gestartet; die Serverkopie kann nur einmal verwendet werden.', + 'backup.selectFile': 'Wähle zuerst ein ZIP-Backup aus.', + 'backup.verifying': 'Backup wird hochgeladen und geprüft...', + 'backup.verified': 'Backup erfolgreich geprüft.', + 'backup.ackRequired': 'Bestätige zuerst die Auswirkungen der Wiederherstellung.', + 'backup.restoreStarted': 'Wiederherstellung gestartet. WebSSH wechselt in den Wartungsmodus und startet neu.', + 'backup.restarting': 'WebSSH startet neu. Melde dich erneut an, sobald der Dienst bereit ist.', 'admin.oneTimeRecoveryCodes': 'Einmalige Wiederherstellungscodes', 'admin.copyRecoveryCodesNow': 'Kopiere diese Codes jetzt. Sie werden nicht erneut angezeigt.', 'admin.loading': 'Wird geladen...', @@ -1461,6 +1600,8 @@ const translations = { 'common.cancel': 'Abbrechen', 'common.save': 'Speichern', 'common.close': 'Schließen', + 'common.yes': 'Ja', + 'common.no': 'Nein', 'common.tip': 'Tipp', 'common.optional': 'Optional', 'common.apply': 'Anwenden', @@ -1699,6 +1840,51 @@ const translations = { 'admin.confirmTargetUsername': "Confirmer le nom d’utilisateur cible", 'admin.stableOidcSubject': 'Sujet OIDC stable', 'admin.continue': 'Continuer', + 'backup.title': 'Sauvegarde et restauration', + 'backup.sensitiveWarning': 'Les archives contiennent des mots de passe, des secrets applicatifs, des métadonnées de clés SSH et des clés privées chiffrées. Conservez chaque archive téléchargée chiffrée et hors de l’hôte.', + 'backup.create': 'Créer une sauvegarde', + 'backup.createHint': 'Crée un instantané cohérent en ligne pendant que WebSSH reste disponible.', + 'backup.download': 'Télécharger la sauvegarde', + 'backup.statusIdle': 'Aucune opération en cours.', + 'backup.upload': 'Téléverser et vérifier', + 'backup.uploadHint': 'Sélectionnez une sauvegarde ZIP WebSSH. Elle est conservée temporairement pour cette session administrateur uniquement.', + 'backup.verify': 'Téléverser et vérifier', + 'backup.validation': 'Résultat de la validation', + 'backup.formatVersion': 'Version du format', + 'backup.files': 'Fichiers', + 'backup.totalSize': 'Taille décompressée', + 'backup.compatible': 'Compatible', + 'backup.dataSchemaVersion': 'Schéma de données de la sauvegarde', + 'backup.currentDataSchemaVersion': 'Schéma de données actuel', + 'backup.createdAt': 'Créée le', + 'backup.legacy': 'Archive héritée', + 'backup.compatibilityReason': 'Compatibilité', + 'backup.notRecorded': 'Non enregistré', + 'backup.compatibilityCurrent': 'Le schéma de données de la sauvegarde est actuel.', + 'backup.compatibilityLegacy': 'Archive v1 héritée ; un chemin de migration est disponible.', + 'backup.compatibilityMigratable': 'L’ancien schéma de données peut être migré.', + 'backup.compatibilityFuture': 'Cette sauvegarde utilise un schéma plus récent et ne peut pas être restaurée ici.', + 'backup.compatibilityNoMigration': 'Aucun chemin de migration complet n’est disponible pour cette sauvegarde.', + 'backup.restoreWarning': 'La restauration remplace tout l’état persistant, invalide les sessions, active la maintenance et redémarre WebSSH.', + 'backup.restore': 'Restaurer la sauvegarde vérifiée', + 'backup.firstConfirm': 'Confirmer les conséquences', + 'backup.firstAcknowledge': 'Je comprends que l’état persistant actuel sera remplacé.', + 'backup.cancel': 'Annuler', + 'backup.secondConfirm': 'Confirmation destructive finale', + 'backup.typeRestore': 'Saisissez RESTORE', + 'backup.password': 'Mot de passe administrateur', + 'backup.destructiveConfirm': 'Remplacer définitivement l’état persistant actuel maintenant.', + 'backup.startRestore': 'Démarrer la restauration', + 'backup.creating': 'Création et vérification de la sauvegarde...', + 'backup.ready': 'Sauvegarde vérifiée et prête pour un téléchargement unique.', + 'backup.failed': 'Échec de l’opération de sauvegarde.', + 'backup.downloaded': 'Téléchargement démarré ; la copie serveur est à usage unique.', + 'backup.selectFile': 'Sélectionnez d’abord une sauvegarde ZIP.', + 'backup.verifying': 'Téléversement et vérification de la sauvegarde...', + 'backup.verified': 'Sauvegarde vérifiée avec succès.', + 'backup.ackRequired': 'Confirmez d’abord les conséquences de la restauration.', + 'backup.restoreStarted': 'Restauration démarrée. WebSSH passe en maintenance et va redémarrer.', + 'backup.restarting': 'WebSSH redémarre. Reconnectez-vous lorsque le service est prêt.', 'admin.oneTimeRecoveryCodes': 'Codes de récupération à usage unique', 'admin.copyRecoveryCodesNow': "Copiez ces codes maintenant. Ils ne seront plus affichés.", 'admin.loading': 'Chargement...', @@ -1974,6 +2160,8 @@ const translations = { 'common.cancel': 'Annuler', 'common.save': 'Enregistrer', 'common.close': 'Fermer', + 'common.yes': 'Oui', + 'common.no': 'Non', 'common.tip': 'Astuce', 'common.optional': 'Optionnel', 'common.apply': 'Appliquer', @@ -2203,6 +2391,51 @@ const translations = { 'admin.confirmTargetUsername': 'Confirmar nombre de usuario de destino', 'admin.stableOidcSubject': 'Sujeto OIDC estable', 'admin.continue': 'Continuar', + 'backup.title': 'Copia de seguridad y restauración', + 'backup.sensitiveWarning': 'Las copias contienen contraseñas, secretos de la aplicación, metadatos de claves SSH y claves privadas cifradas. Guarda cada archivo descargado cifrado y fuera del host.', + 'backup.create': 'Crear copia de seguridad', + 'backup.createHint': 'Crea una instantánea coherente en línea mientras WebSSH sigue disponible.', + 'backup.download': 'Descargar copia', + 'backup.statusIdle': 'No hay ninguna operación en curso.', + 'backup.upload': 'Subir y verificar', + 'backup.uploadHint': 'Selecciona una copia ZIP de WebSSH. Se conserva temporalmente solo para esta sesión de administrador.', + 'backup.verify': 'Subir y verificar', + 'backup.validation': 'Resultado de validación', + 'backup.formatVersion': 'Versión del formato', + 'backup.files': 'Archivos', + 'backup.totalSize': 'Tamaño sin comprimir', + 'backup.compatible': 'Compatible', + 'backup.dataSchemaVersion': 'Esquema de datos de la copia', + 'backup.currentDataSchemaVersion': 'Esquema de datos actual', + 'backup.createdAt': 'Creada el', + 'backup.legacy': 'Archivo heredado', + 'backup.compatibilityReason': 'Compatibilidad', + 'backup.notRecorded': 'No registrado', + 'backup.compatibilityCurrent': 'El esquema de datos de la copia está actualizado.', + 'backup.compatibilityLegacy': 'Archivo v1 heredado; existe una ruta de migración.', + 'backup.compatibilityMigratable': 'El esquema de datos anterior se puede migrar.', + 'backup.compatibilityFuture': 'Esta copia usa un esquema más reciente y no se puede restaurar aquí.', + 'backup.compatibilityNoMigration': 'No existe una ruta de migración completa para esta copia.', + 'backup.restoreWarning': 'La restauración reemplaza todo el estado persistente, invalida las sesiones, activa el mantenimiento y reinicia WebSSH.', + 'backup.restore': 'Restaurar copia verificada', + 'backup.firstConfirm': 'Confirmar el impacto', + 'backup.firstAcknowledge': 'Entiendo que se reemplazará el estado persistente actual.', + 'backup.cancel': 'Cancelar', + 'backup.secondConfirm': 'Confirmación destructiva final', + 'backup.typeRestore': 'Escribe RESTORE', + 'backup.password': 'Contraseña del administrador', + 'backup.destructiveConfirm': 'Reemplazar permanentemente el estado persistente actual ahora.', + 'backup.startRestore': 'Iniciar restauración', + 'backup.creating': 'Creando y verificando la copia...', + 'backup.ready': 'Copia verificada y lista para una única descarga.', + 'backup.failed': 'La operación de copia de seguridad falló.', + 'backup.downloaded': 'Descarga iniciada; la copia del servidor es de un solo uso.', + 'backup.selectFile': 'Selecciona primero una copia ZIP.', + 'backup.verifying': 'Subiendo y verificando la copia...', + 'backup.verified': 'Copia verificada correctamente.', + 'backup.ackRequired': 'Confirma primero el impacto de la restauración.', + 'backup.restoreStarted': 'Restauración iniciada. WebSSH entra en mantenimiento y se reiniciará.', + 'backup.restarting': 'WebSSH se está reiniciando. Vuelve a iniciar sesión cuando el servicio esté listo.', 'admin.oneTimeRecoveryCodes': 'Códigos de recuperación de un solo uso', 'admin.copyRecoveryCodesNow': 'Copia estos códigos ahora. No volverán a mostrarse.', 'admin.loading': 'Cargando...', @@ -2478,6 +2711,8 @@ const translations = { 'common.cancel': 'Cancelar', 'common.save': 'Guardar', 'common.close': 'Cerrar', + 'common.yes': 'Sí', + 'common.no': 'No', 'common.tip': 'Consejo', 'common.optional': 'Opcional', 'common.apply': 'Aplicar', @@ -2707,6 +2942,51 @@ const translations = { 'admin.confirmTargetUsername': '确认目标用户名', 'admin.stableOidcSubject': '稳定 OIDC 主体', 'admin.continue': '继续', + 'backup.title': '备份与恢复', + 'backup.sensitiveWarning': '备份包含密码、应用密钥、SSH 密钥元数据和加密私钥。请加密保存下载的归档，并存放在主机之外。', + 'backup.create': '创建备份', + 'backup.createHint': '在 WebSSH 保持可用时创建在线一致性快照。', + 'backup.download': '下载备份', + 'backup.statusIdle': '当前没有运行中的操作。', + 'backup.upload': '上传并验证', + 'backup.uploadHint': '选择 WebSSH ZIP 备份。归档仅为当前管理员会话临时保留。', + 'backup.verify': '上传并验证', + 'backup.validation': '验证结果', + 'backup.formatVersion': '格式版本', + 'backup.files': '文件', + 'backup.totalSize': '未压缩大小', + 'backup.compatible': '兼容', + 'backup.dataSchemaVersion': '备份数据架构', + 'backup.currentDataSchemaVersion': '当前数据架构', + 'backup.createdAt': '创建时间', + 'backup.legacy': '旧版归档', + 'backup.compatibilityReason': '兼容性', + 'backup.notRecorded': '未记录', + 'backup.compatibilityCurrent': '备份数据架构为当前版本。', + 'backup.compatibilityLegacy': '旧版 v1 归档；存在可用的迁移路径。', + 'backup.compatibilityMigratable': '可以迁移较旧的备份数据架构。', + 'backup.compatibilityFuture': '此备份使用较新的数据架构，无法在此恢复。', + 'backup.compatibilityNoMigration': '此备份没有完整的迁移路径。', + 'backup.restoreWarning': '恢复将替换全部持久状态、使所有会话失效、进入维护模式并重启 WebSSH。', + 'backup.restore': '恢复已验证的备份', + 'backup.firstConfirm': '确认恢复影响', + 'backup.firstAcknowledge': '我了解当前持久状态将被替换。', + 'backup.cancel': '取消', + 'backup.secondConfirm': '最终破坏性确认', + 'backup.typeRestore': '输入 RESTORE', + 'backup.password': '管理员密码', + 'backup.destructiveConfirm': '立即永久替换当前持久状态。', + 'backup.startRestore': '开始恢复', + 'backup.creating': '正在创建并验证备份...', + 'backup.ready': '备份已验证，可供一次性下载。', + 'backup.failed': '备份操作失败。', + 'backup.downloaded': '下载已开始；服务器副本只能使用一次。', + 'backup.selectFile': '请先选择 ZIP 备份。', + 'backup.verifying': '正在上传并验证备份...', + 'backup.verified': '备份验证成功。', + 'backup.ackRequired': '请先确认恢复影响。', + 'backup.restoreStarted': '恢复已开始。WebSSH 正在进入维护模式并将重启。', + 'backup.restarting': 'WebSSH 正在重启。服务就绪后请重新登录。', 'admin.oneTimeRecoveryCodes': '一次性恢复代码', 'admin.copyRecoveryCodesNow': '请立即复制这些代码。之后不会再次显示。', 'admin.loading': '正在加载...', @@ -2973,6 +3253,8 @@ const translations = { 'common.cancel': '取消', 'common.save': '保存', 'common.close': '关闭', + 'common.yes': '是', + 'common.no': '否', 'common.tip': '提示', 'common.optional': '可选', 'common.apply': '应用', diff --git a/static/js/restore-status-flow.js b/static/js/restore-status-flow.js new file mode 100644 index 0000000..b5637c9 --- /dev/null +++ b/static/js/restore-status-flow.js @@ -0,0 +1,116 @@ +(function (root, factory) { + const api = factory(); + if (typeof module === 'object' && module.exports) { + module.exports = api; + } + if (root) { + root.WebSSHRestoreStatus = api; + } +})(typeof window !== 'undefined' ? window : globalThis, function () { + 'use strict'; + + const DEFAULT_STORAGE_KEY = 'webssh.restore.pending'; + const ACTIVE_STATES = new Set(['preparing', 'in_progress']); + const TERMINAL_STATES = new Set([ + 'succeeded', 'failed', 'rollback_failed' + ]); + + function createRestoreStatusFlow(options) { + const storage = options.storage; + const storageKey = options.storageKey || DEFAULT_STORAGE_KEY; + const schedule = options.schedule || setTimeout; + const pollInterval = options.pollInterval || 900; + const readyInterval = options.readyInterval || 1000; + let memoryPending = false; + + function isPending() { + try { + return memoryPending || storage.getItem(storageKey) === '1'; + } catch (_error) { + return memoryPending; + } + } + + function markPending() { + memoryPending = true; + try { + storage.setItem(storageKey, '1'); + } catch (_error) { + // Continue polling in memory when browser storage is blocked. + } + } + + function clearPending() { + memoryPending = false; + try { + storage.removeItem(storageKey); + } catch (_error) { + // A storage failure must not hide the server result. + } + } + + function schedulePoll() { + schedule(poll, pollInterval); + } + + function scheduleReadyCheck() { + schedule(waitUntilReady, readyInterval); + } + + async function waitUntilReady() { + try { + if (await options.checkReady()) { + options.reload(); + return; + } + } catch (_error) { + // The process is expected to be unavailable during restart. + } + scheduleReadyCheck(); + } + + async function poll() { + let status; + try { + status = await options.fetchStatus(); + if (!status || typeof status.state !== 'string') { + throw new TypeError('invalid restore status response'); + } + } catch (_error) { + options.presentRestarting(); + scheduleReadyCheck(); + return; + } + + if (TERMINAL_STATES.has(status.state)) { + clearPending(); + options.present(status); + return; + } + if (ACTIVE_STATES.has(status.state)) { + options.present(status); + schedulePoll(); + return; + } + if (status.state === 'idle' && isPending()) { + schedulePoll(); + return; + } + options.present(status); + } + + function resume() { + return isPending() ? poll() : Promise.resolve(false); + } + + return { + clearPending, + isPending, + markPending, + poll, + resume + }; + } + + return { createRestoreStatusFlow }; +}); diff --git a/templates/admin.html b/templates/admin.html index 4eec3b6..34e7258 100644 --- a/templates/admin.html +++ b/templates/admin.html @@ -32,6 +32,7 @@

Admin Panel ⚙️Audit Logs {% endif %} +
@@ -124,6 +125,54 @@

Admin Panel ⚙️ {% endif %}

+ + {% endif %} + + + +
+ diff --git a/tests/js/restore-status-flow.test.js b/tests/js/restore-status-flow.test.js new file mode 100644 index 0000000..de3a1e9 --- /dev/null +++ b/tests/js/restore-status-flow.test.js @@ -0,0 +1,107 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const { + createRestoreStatusFlow +} = require('../../static/js/restore-status-flow.js'); + +function memoryStorage() { + const values = new Map(); + return { + getItem: key => values.has(key) ? values.get(key) : null, + setItem: (key, value) => values.set(key, String(value)), + removeItem: key => values.delete(key) + }; +} + +test('restore status survives idle race, restart, and reauthentication reload', async () => { + const storage = memoryStorage(); + const scheduled = []; + const presented = []; + let statusAttempt = 0; + let readyAttempt = 0; + let reloads = 0; + + const flow = createRestoreStatusFlow({ + storage, + schedule: callback => scheduled.push(callback), + fetchStatus: async () => { + statusAttempt += 1; + if (statusAttempt === 1) { + return { state: 'idle', message: null }; + } + if (statusAttempt === 2) { + return { state: 'preparing', message: 'Preparing restore' }; + } + throw new TypeError('fetch failed'); + }, + checkReady: async () => { + readyAttempt += 1; + return readyAttempt === 2; + }, + present: status => presented.push(status.state), + presentRestarting: () => presented.push('restarting'), + reload: () => { reloads += 1; } + }); + + flow.markPending(); + await flow.poll(); + assert.deepEqual(presented, []); + assert.equal(scheduled.length, 1); + + await scheduled.shift()(); + assert.deepEqual(presented, ['preparing']); + assert.equal(scheduled.length, 1); + + await scheduled.shift()(); + assert.deepEqual(presented, ['preparing', 'restarting']); + assert.equal(scheduled.length, 1); + + await scheduled.shift()(); + assert.equal(reloads, 0); + assert.equal(scheduled.length, 1); + await scheduled.shift()(); + assert.equal(reloads, 1); + assert.equal(flow.isPending(), true); + + const afterLogin = createRestoreStatusFlow({ + storage, + schedule: callback => scheduled.push(callback), + fetchStatus: async () => ({ + state: 'succeeded', + message: 'Restore completed' + }), + checkReady: async () => true, + present: status => presented.push(status.state), + presentRestarting: () => presented.push('restarting'), + reload: () => { reloads += 1; } + }); + + assert.equal(afterLogin.isPending(), true); + await afterLogin.resume(); + assert.equal(afterLogin.isPending(), false); + assert.deepEqual(presented, ['preparing', 'restarting', 'succeeded']); +}); + +test('restore polling continues when session storage is unavailable', async () => { + const scheduled = []; + const storage = { + getItem: () => { throw new Error('storage blocked'); }, + setItem: () => { throw new Error('storage blocked'); }, + removeItem: () => { throw new Error('storage blocked'); } + }; + const flow = createRestoreStatusFlow({ + storage, + schedule: callback => scheduled.push(callback), + fetchStatus: async () => ({ state: 'idle', message: null }), + checkReady: async () => false, + present: () => {}, + presentRestarting: () => {}, + reload: () => {} + }); + + assert.doesNotThrow(() => flow.markPending()); + assert.equal(flow.isPending(), true); + await flow.poll(); + assert.equal(scheduled.length, 1); +}); diff --git a/tests/test_admin_backup.py b/tests/test_admin_backup.py new file mode 100644 index 0000000..85ae410 --- /dev/null +++ b/tests/test_admin_backup.py @@ -0,0 +1,395 @@ +import json +import sqlite3 +import time +from pathlib import Path +import zipfile + +import pytest + +from app.backup_manager import create_backup + + +def _create_user(app, username, *, admin): + from app.auth import register_user + from app.models import db + + with app.app_context(): + user, error = register_user(username, 'password123') + assert error is None + user.is_admin = admin + db.session.commit() + return user.id + + +def _login(client, username): + response = client.post('/login', data={ + 'username': username, + 'password': 'password123', + }) + assert response.status_code == 302 + + +def _wait_for_status(client, operation_id, expected, timeout=5): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + response = client.get(f'/admin/api/backups/{operation_id}') + if response.status_code == 200 and response.json['status'] in expected: + return response + time.sleep(0.03) + raise AssertionError(f'operation did not reach {expected}') + + +@pytest.fixture +def isolated_operations(app, monkeypatch, tmp_path): + import config + from app.backup_operations import backup_operations + + backup_operations.close() + monkeypatch.setattr(config, 'BACKUP_TEMP_DIR', tmp_path / 'operations') + monkeypatch.setattr(config, 'BACKUP_DOWNLOAD_TTL', 60) + monkeypatch.setattr(config, 'BACKUP_OPERATION_TIMEOUT', 60) + monkeypatch.setattr(config, 'RATELIMIT_ENABLED', False) + yield backup_operations + backup_operations.close() + + +def _valid_archive(tmp_path): + source = tmp_path / 'source' + source.mkdir() + database = sqlite3.connect(source / 'app.db') + try: + database.execute( + 'CREATE TABLE users (' + 'id INTEGER PRIMARY KEY, ' + 'username TEXT NOT NULL, ' + 'password_hash TEXT NOT NULL' + ')' + ) + database.commit() + finally: + database.close() + (source / 'settings.json').write_text('{}', encoding='utf-8') + archive = tmp_path / 'upload.zip' + create_backup(source, archive) + return archive + + +def _archive_with_data_schema(source, destination, data_schema_version): + with zipfile.ZipFile(source, 'r') as archive: + entries = { + info.filename: archive.read(info) + for info in archive.infolist() + } + manifest = json.loads(entries['manifest.json']) + manifest['data_schema_version'] = data_schema_version + entries['manifest.json'] = json.dumps( + manifest, sort_keys=True, separators=(',', ':') + ).encode('utf-8') + with zipfile.ZipFile(destination, 'w', compression=zipfile.ZIP_DEFLATED) as archive: + for name, payload in entries.items(): + archive.writestr(name, payload) + return destination + + +def _archive_as_legacy_v1(source, destination): + with zipfile.ZipFile(source, 'r') as archive: + entries = { + info.filename: archive.read(info) + for info in archive.infolist() + } + manifest = json.loads(entries['manifest.json']) + manifest = { + 'files': manifest['files'], + 'format_version': 1, + } + entries['manifest.json'] = json.dumps( + manifest, sort_keys=True, separators=(',', ':') + ).encode('utf-8') + with zipfile.ZipFile( + destination, 'w', compression=zipfile.ZIP_DEFLATED + ) as archive: + for name, payload in entries.items(): + archive.writestr(name, payload) + return destination + + +def test_backup_endpoints_require_admin(app, client, isolated_operations): + _create_user(app, 'normal_backup_user', admin=False) + + assert client.post('/admin/api/backups').status_code == 302 + _login(client, 'normal_backup_user') + assert client.post('/admin/api/backups').status_code == 403 + assert client.post('/admin/api/backups/upload', data=b'PK').status_code == 403 + + +@pytest.mark.parametrize( + 'endpoint', + ('/admin/api/backups', '/admin/api/backups/upload'), +) +def test_busy_backup_response_does_not_expose_exception_details( + app, client, isolated_operations, monkeypatch, endpoint +): + from app.backup_coordination import OperationBusyError + + _create_user(app, 'busy_backup_admin', admin=True) + _login(client, 'busy_backup_admin') + + def reject_operation(*_args, **_kwargs): + raise OperationBusyError('sensitive/server/path') + + monkeypatch.setattr(isolated_operations, 'create', reject_operation) + + response = client.post(endpoint, data=b'PK\x03\x04') + + assert response.status_code == 409 + assert response.json == { + 'error': 'another backup or restore operation is active' + } + assert 'sensitive' not in response.get_data(as_text=True) + + +def test_upload_limit_response_does_not_expose_exception_details( + app, client, isolated_operations, monkeypatch +): + import app.admin_backup as admin_backup + + _create_user(app, 'limited_upload_admin', admin=True) + _login(client, 'limited_upload_admin') + + def reject_upload(_destination): + raise ValueError('sensitive/server/path') + + monkeypatch.setattr(admin_backup, '_stream_upload', reject_upload) + + response = client.post('/admin/api/backups/upload', data=b'PK\x03\x04') + + assert response.status_code == 413 + assert response.json == {'error': 'Backup upload is too large'} + assert 'sensitive' not in response.get_data(as_text=True) + assert not isolated_operations._records + + +def test_admin_can_create_and_one_time_download_online_backup( + app, client, isolated_operations +): + _create_user(app, 'backup_admin', admin=True) + _login(client, 'backup_admin') + + created = client.post('/admin/api/backups') + assert created.status_code == 202 + operation_id = created.json['operation_id'] + ready = _wait_for_status(client, operation_id, {'ready', 'failed'}) + assert ready.json['status'] == 'ready' + + download = client.post( + f'/admin/api/backups/{operation_id}/download', buffered=False + ) + assert download.status_code == 200 + assert download.mimetype == 'application/zip' + assert download.headers['Cache-Control'] == 'no-store' + assert download.headers['X-Content-Type-Options'] == 'nosniff' + assert b''.join(download.response).startswith(b'PK') + assert operation_id not in isolated_operations._records + download.close() + assert client.post( + f'/admin/api/backups/{operation_id}/download' + ).status_code == 404 + + +def test_interrupted_download_invalidates_server_archive( + app, client, isolated_operations +): + _create_user(app, 'disconnect_backup_admin', admin=True) + _login(client, 'disconnect_backup_admin') + created = client.post('/admin/api/backups') + operation_id = created.json['operation_id'] + ready = _wait_for_status(client, operation_id, {'ready', 'failed'}) + assert ready.json['status'] == 'ready' + + download = client.post( + f'/admin/api/backups/{operation_id}/download', buffered=False + ) + download.close() + + assert operation_id not in isolated_operations._records + + +def test_uploaded_backup_is_session_bound_and_requires_two_step_reauth( + app, client, isolated_operations, tmp_path, monkeypatch +): + _create_user(app, 'restore_admin', admin=True) + archive = _valid_archive(tmp_path) + _login(client, 'restore_admin') + + uploaded = client.post( + '/admin/api/backups/upload', + data=archive.read_bytes(), + content_type='application/zip', + ) + assert uploaded.status_code == 202 + operation_id = uploaded.json['operation_id'] + verified = _wait_for_status(client, operation_id, {'verified', 'failed'}) + assert verified.json['status'] == 'verified' + assert set(verified.json['summary']) == { + 'compatibility_reason', 'compatible', 'created_at', + 'current_data_schema_version', 'data_schema_version', 'file_count', + 'format_version', 'legacy', 'total_uncompressed_size', + } + + other_session = app.test_client() + _login(other_session, 'restore_admin') + assert other_session.get( + f'/admin/api/backups/{operation_id}' + ).status_code == 404 + + first = client.post( + f'/admin/api/backups/{operation_id}/restore/prepare', + json={'acknowledge_sensitive_restore': True}, + ) + assert first.status_code == 200 + token = first.json['confirmation_token'] + assert client.post( + f'/admin/api/backups/{operation_id}/restore', + json={ + 'confirmation_token': token, + 'confirmation_phrase': 'RESTORE', + 'confirm_destructive_restore': True, + 'password': 'wrong-password', + }, + ).status_code == 403 + + started = [] + import app.restore_service as restore_service + monkeypatch.setattr( + restore_service, + 'start_restore', + lambda app, socketio, record, username, source_ip: started.append(record), + ) + response = client.post( + f'/admin/api/backups/{operation_id}/restore', + json={ + 'confirmation_token': token, + 'confirmation_phrase': 'RESTORE', + 'confirm_destructive_restore': True, + 'password': 'password123', + }, + ) + assert response.status_code == 202 + assert len(started) == 1 + assert started[0].status == 'restoring' + + +def test_future_schema_is_verified_but_blocked_at_both_restore_gates( + app, client, isolated_operations, tmp_path, monkeypatch +): + user_id = _create_user(app, 'future_restore_admin', admin=True) + current = _valid_archive(tmp_path) + future = _archive_with_data_schema(current, tmp_path / 'future.zip', 2) + _login(client, 'future_restore_admin') + + uploaded = client.post( + '/admin/api/backups/upload', + data=future.read_bytes(), + content_type='application/zip', + ) + operation_id = uploaded.json['operation_id'] + verified = _wait_for_status(client, operation_id, {'verified', 'failed'}) + + assert verified.json['status'] == 'verified' + assert verified.json['summary']['compatible'] is False + assert verified.json['summary']['data_schema_version'] == 2 + assert verified.json['summary']['current_data_schema_version'] == 1 + assert verified.json['summary']['compatibility_reason'] == ( + 'backup data schema is newer than this WebSSH version' + ) + assert client.post( + f'/admin/api/backups/{operation_id}/restore/prepare', + json={'acknowledge_sensitive_restore': True}, + ).status_code == 409 + + with client.session_transaction() as browser_session: + session_id = browser_session['_backup_admin_session_id'] + token = isolated_operations.prepare_restore( + operation_id, user_id, session_id + ) + started = [] + import app.restore_service as restore_service + monkeypatch.setattr( + restore_service, + 'start_restore', + lambda *args: started.append(args), + ) + response = client.post( + f'/admin/api/backups/{operation_id}/restore', + json={ + 'confirmation_token': token, + 'confirmation_phrase': 'RESTORE', + 'confirm_destructive_restore': True, + 'password': 'password123', + }, + ) + + assert response.status_code == 409 + assert started == [] + + +def test_legacy_v1_upload_remains_restore_compatible( + app, client, isolated_operations, tmp_path +): + _create_user(app, 'legacy_restore_admin', admin=True) + current = _valid_archive(tmp_path) + legacy = _archive_as_legacy_v1(current, tmp_path / 'legacy-v1.zip') + _login(client, 'legacy_restore_admin') + + uploaded = client.post( + '/admin/api/backups/upload', + data=legacy.read_bytes(), + content_type='application/zip', + ) + operation_id = uploaded.json['operation_id'] + verified = _wait_for_status(client, operation_id, {'verified', 'failed'}) + + assert verified.json['status'] == 'verified' + assert verified.json['summary']['format_version'] == 1 + assert verified.json['summary']['data_schema_version'] == 0 + assert verified.json['summary']['legacy'] is True + assert verified.json['summary']['compatible'] is True + prepared = client.post( + f'/admin/api/backups/{operation_id}/restore/prepare', + json={'acknowledge_sensitive_restore': True}, + ) + assert prepared.status_code == 200 + + +def test_upload_limit_and_csrf_are_enforced( + app, client, isolated_operations, monkeypatch +): + import config + + _create_user(app, 'bounded_backup_admin', admin=True) + _login(client, 'bounded_backup_admin') + monkeypatch.setattr(config, 'BACKUP_UPLOAD_MAX_SIZE', 4) + oversized = client.post( + '/admin/api/backups/upload', + data=b'PK123', + content_type='application/zip', + ) + assert oversized.status_code == 413 + assert not tuple(Path(config.BACKUP_TEMP_DIR).glob('operation-*')) + + app.config['WTF_CSRF_ENABLED'] = True + assert client.post('/admin/api/backups').status_code == 400 + + +def test_admin_backup_ui_is_native_and_has_destructive_confirmations(): + template = Path('templates/admin.html').read_text(encoding='utf-8') + + assert 'data-tab="backup"' in template + assert 'restoreFirstConfirmModal' in template + assert 'restoreSecondConfirmModal' in template + assert 'restorePassword' in template + assert 'backupDataSchemaVersion' in template + assert 'backupCurrentDataSchemaVersion' in template + assert 'backupCreatedAt' in template + assert 'backupLegacy' in template + assert 'backupCompatibilityReason' in template diff --git a/tests/test_backup_manager.py b/tests/test_backup_manager.py index 71f7303..c2f4f69 100644 --- a/tests/test_backup_manager.py +++ b/tests/test_backup_manager.py @@ -2,12 +2,14 @@ import hashlib import os from pathlib import Path +import sqlite3 import subprocess import sys import zipfile import pytest +import app.backup_manager as backup_manager from app.backup_manager import ( BackupIntegrityError, create_backup, @@ -172,7 +174,6 @@ def test_backup_verify_missing_archive_does_not_initialize_storage(tmp_path): def _write_representative_data(data_dir): files = { - 'app.db': b'SQLite format 3\x00representative database', 'app_settings.json': b'{"registration_enabled": false}', 'known_hosts': b'ssh.example ssh-ed25519 AAAA-test\n', 'secret_key': b'persisted-secret\n', @@ -186,9 +187,41 @@ def _write_representative_data(data_dir): path = data_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(payload) + database_path = data_dir / 'app.db' + if not database_path.exists(): + connection = sqlite3.connect(database_path) + try: + connection.execute( + 'CREATE TABLE users (' + 'id INTEGER PRIMARY KEY, ' + 'username TEXT NOT NULL, ' + 'password_hash TEXT NOT NULL' + ')' + ) + connection.commit() + finally: + connection.close() + files['app.db'] = database_path.read_bytes() return files +def _webssh_database_bytes(tmp_path): + database_path = tmp_path / 'valid-webssh.db' + connection = sqlite3.connect(database_path) + try: + connection.execute( + 'CREATE TABLE users (' + 'id INTEGER PRIMARY KEY, ' + 'username TEXT NOT NULL, ' + 'password_hash TEXT NOT NULL' + ')' + ) + connection.commit() + finally: + connection.close() + return database_path.read_bytes() + + def _snapshot(directory): return { path.relative_to(directory).as_posix(): path.read_bytes() @@ -209,9 +242,17 @@ def _corrupt_archive_member(source, destination, member): destination_zip.writestr(name, payload) -def _write_manifest_archive(archive, files): +def _write_manifest_archive( + archive, + files, + *, + format_version=1, + data_schema_version=None, + created_at='2026-08-03T12:00:00Z', + producer='webssh', +): manifest = { - 'format_version': 1, + 'format_version': format_version, 'files': [ { 'path': path, @@ -221,6 +262,12 @@ def _write_manifest_archive(archive, files): for path, payload in sorted(files.items()) ], } + if format_version == 2: + manifest.update({ + 'created_at': created_at, + 'data_schema_version': data_schema_version, + 'producer': producer, + }) with zipfile.ZipFile( archive, 'w', @@ -234,6 +281,250 @@ def _write_manifest_archive(archive, files): backup.writestr(f'data/{path}', payload) +def test_cli_verify_reports_legacy_restore_compatibility(tmp_path): + archive = tmp_path / 'legacy-v1.zip' + _write_manifest_archive(archive, {'app.db': _webssh_database_bytes(tmp_path)}) + + result = _maintenance_cli( + tmp_path / 'unused-data', 'backup', 'verify', str(archive) + ) + + assert result.returncode == 0, result.stderr + assert 'format v1' in result.stdout + assert 'data schema 0' in result.stdout + assert 'restore compatible (legacy)' in result.stdout + + +def test_cli_verify_reports_future_schema_without_accepting_restore(tmp_path): + archive = tmp_path / 'future-v2.zip' + _write_manifest_archive( + archive, + {'app.db': _webssh_database_bytes(tmp_path)}, + format_version=2, + data_schema_version=2, + ) + + result = _maintenance_cli( + tmp_path / 'unused-data', 'backup', 'verify', str(archive) + ) + + assert result.returncode == 0, result.stderr + assert 'format v2' in result.stdout + assert 'data schema 2' in result.stdout + assert 'restore incompatible' in result.stdout + + +def test_new_backup_records_v2_compatibility_metadata(tmp_path): + data_dir = tmp_path / 'data' + data_dir.mkdir() + _write_representative_data(data_dir) + archive = tmp_path / 'backup.zip' + + manifest = create_backup(data_dir, archive) + + assert manifest.format_version == 2 + assert manifest.data_schema_version == 1 + assert manifest.producer == 'webssh' + assert manifest.created_at.endswith('Z') + + +def test_v1_backup_is_legacy_and_migratable(tmp_path): + archive = tmp_path / 'legacy-v1.zip' + _write_manifest_archive( + archive, + {'app.db': _webssh_database_bytes(tmp_path)}, + format_version=1, + ) + + manifest = verify_backup(archive) + compatibility = backup_manager.evaluate_backup_compatibility(manifest) + + assert manifest.data_schema_version == 0 + assert compatibility.compatible is True + assert compatibility.legacy is True + assert compatibility.reason == 'legacy archive can be migrated' + + +def test_future_data_schema_verifies_but_is_not_restore_compatible(tmp_path): + archive = tmp_path / 'future-v2.zip' + _write_manifest_archive( + archive, + {'app.db': _webssh_database_bytes(tmp_path)}, + format_version=2, + data_schema_version=2, + ) + + manifest = verify_backup(archive) + compatibility = backup_manager.evaluate_backup_compatibility(manifest) + + assert compatibility.compatible is False + assert compatibility.reason == 'backup data schema is newer than this WebSSH version' + + +def test_future_data_schema_is_rejected_before_restore_mutates_data(tmp_path): + archive = tmp_path / 'future-v2.zip' + _write_manifest_archive( + archive, + {'app.db': _webssh_database_bytes(tmp_path)}, + format_version=2, + data_schema_version=2, + ) + restore_dir = tmp_path / 'restore' + restore_dir.mkdir() + sentinel = restore_dir / 'keep.txt' + sentinel.write_bytes(b'keep-current-state') + + with pytest.raises(BackupIntegrityError, match='newer'): + restore_backup(archive, restore_dir) + + assert sentinel.read_bytes() == b'keep-current-state' + assert not (restore_dir / 'app.db').exists() + + +@pytest.mark.parametrize( + 'files', + ( + {}, + {'app.db': b'not-a-sqlite-database'}, + ), +) +def test_restore_rejects_missing_or_invalid_database_before_mutation( + tmp_path, files +): + archive = tmp_path / 'invalid-database.zip' + _write_manifest_archive( + archive, + files, + format_version=2, + data_schema_version=1, + ) + restore_dir = tmp_path / 'restore' + restore_dir.mkdir() + sentinel = restore_dir / 'keep.txt' + sentinel.write_bytes(b'keep-current-state') + + with pytest.raises(BackupIntegrityError, match='database'): + restore_backup(archive, restore_dir) + + assert sentinel.read_bytes() == b'keep-current-state' + assert _snapshot(restore_dir) == {'keep.txt': b'keep-current-state'} + + +def test_restore_rejects_non_webssh_sqlite_database_before_mutation(tmp_path): + unrelated_database = tmp_path / 'unrelated.db' + connection = sqlite3.connect(unrelated_database) + try: + connection.execute('CREATE TABLE notes (value TEXT)') + connection.commit() + finally: + connection.close() + archive = tmp_path / 'unrelated-database.zip' + _write_manifest_archive( + archive, + {'app.db': unrelated_database.read_bytes()}, + format_version=2, + data_schema_version=1, + ) + restore_dir = tmp_path / 'restore' + restore_dir.mkdir() + sentinel = restore_dir / 'keep.txt' + sentinel.write_bytes(b'keep-current-state') + + with pytest.raises(BackupIntegrityError, match='WebSSH database'): + restore_backup(archive, restore_dir) + + assert _snapshot(restore_dir) == {'keep.txt': b'keep-current-state'} + + +@pytest.mark.parametrize('command', ('create', 'restore')) +def test_backup_cli_mutations_reject_a_cross_process_operation_lock( + app, tmp_path, monkeypatch, command +): + import config + from app.backup_coordination import operation_lock + + data_dir = Path(app.config['DATA_DIR']) + _write_representative_data(data_dir) + archive = tmp_path / 'locked-operation.zip' + create_backup(data_dir, archive) + destination = tmp_path / 'must-not-exist.zip' + monkeypatch.setattr(config, 'DATA_DIR', data_dir) + monkeypatch.setattr(config, 'BACKUP_TEMP_DIR', tmp_path / 'operations') + monkeypatch.setattr(config, 'BACKUP_OPERATION_TIMEOUT', 1) + arguments = ( + ('backup', 'create', '--destination', str(destination), '--confirm-offline') + if command == 'create' + else ('backup', 'restore', str(archive), '--confirm-offline') + ) + + with operation_lock(): + result = _maintenance_cli( + data_dir, + *arguments, + environment_overrides={ + 'BACKUP_TEMP_DIR': str(config.BACKUP_TEMP_DIR), + 'BACKUP_OPERATION_TIMEOUT': '1', + }, + ) + + assert result.returncode != 0 + assert 'another backup or restore operation is active' in result.stderr + assert not destination.exists() + + +def test_missing_data_migration_step_is_not_restore_compatible(monkeypatch): + monkeypatch.setattr(backup_manager, '_CURRENT_DATA_SCHEMA_VERSION', 3) + monkeypatch.setattr(backup_manager, '_DATA_SCHEMA_MIGRATIONS', {1: 2}) + manifest = backup_manager.BackupManifest( + format_version=2, + files=(), + data_schema_version=1, + created_at='2026-08-03T12:00:00Z', + producer='webssh', + ) + + compatibility = backup_manager.evaluate_backup_compatibility(manifest) + + assert compatibility.compatible is False + assert compatibility.reason == 'no complete migration path for backup data schema' + + +@pytest.mark.parametrize( + ('producer', 'created_at'), + ( + ('another-product', '2026-08-03T12:00:00Z'), + ('webssh', '2026-08-03T12:00:00'), + ), +) +def test_v2_manifest_rejects_untrusted_compatibility_metadata( + tmp_path, producer, created_at +): + archive = tmp_path / 'untrusted-v2.zip' + _write_manifest_archive( + archive, + {'app.db': b'data'}, + format_version=2, + data_schema_version=1, + created_at=created_at, + producer=producer, + ) + + with pytest.raises(BackupIntegrityError, match='incompatible'): + verify_backup(archive) + + +def test_unknown_manifest_format_is_rejected(tmp_path): + archive = tmp_path / 'unknown-format.zip' + _write_manifest_archive( + archive, + {'app.db': b'data'}, + format_version=3, + ) + + with pytest.raises(BackupIntegrityError, match='incompatible'): + verify_backup(archive) + + def test_create_verify_and_restore_round_trip(tmp_path): data_dir = tmp_path / 'data' data_dir.mkdir() @@ -244,7 +535,8 @@ def test_create_verify_and_restore_round_trip(tmp_path): verified = verify_backup(archive) assert created == verified - assert created.format_version == 1 + assert created.format_version == 2 + assert backup_manager.evaluate_backup_compatibility(created).compatible assert tuple(item.path for item in created.files) == tuple( sorted(expected_files) ) diff --git a/tests/test_online_backup.py b/tests/test_online_backup.py new file mode 100644 index 0000000..aeb9909 --- /dev/null +++ b/tests/test_online_backup.py @@ -0,0 +1,162 @@ +import sqlite3 +import threading +import time +import zipfile + +import pytest + +from app.backup_coordination import ( + OperationBusyError, + ensure_backup_temp_dir, + operation_lock, + persistent_write, + snapshot_barrier, +) +from app.backup_manager import verify_backup +from app.online_backup import create_online_backup + + +def _configure_temp_root(monkeypatch, tmp_path): + import config + + root = tmp_path / 'operations' + monkeypatch.setattr(config, 'BACKUP_TEMP_DIR', root) + monkeypatch.setattr(config, 'BACKUP_OPERATION_TIMEOUT', 2) + return root + + +def test_backup_temp_root_is_namespaced_by_data_directory( + tmp_path, monkeypatch +): + import config + + configured_root = tmp_path / 'operations' + first_data_dir = tmp_path / 'first-data' + second_data_dir = tmp_path / 'second-data' + monkeypatch.setattr(config, 'BACKUP_TEMP_DIR', configured_root) + + monkeypatch.setattr(config, 'DATA_DIR', first_data_dir) + first_root = ensure_backup_temp_dir() + monkeypatch.setattr(config, 'DATA_DIR', second_data_dir) + second_root = ensure_backup_temp_dir() + + assert first_root != second_root + assert first_root.parent == configured_root.resolve() + assert second_root.parent == configured_root.resolve() + + +def test_online_backup_uses_valid_sqlite_snapshot_and_discards_worktree( + tmp_path, monkeypatch +): + operation_root = _configure_temp_root(monkeypatch, tmp_path) + data_dir = tmp_path / 'data' + data_dir.mkdir() + database = sqlite3.connect(data_dir / 'app.db') + database.execute('PRAGMA journal_mode=WAL') + database.execute( + 'CREATE TABLE users (' + 'id INTEGER PRIMARY KEY, ' + 'username TEXT NOT NULL, ' + 'password_hash TEXT NOT NULL' + ')' + ) + database.execute('CREATE TABLE records (id INTEGER PRIMARY KEY, value TEXT)') + database.executemany( + 'INSERT INTO records(value) VALUES (?)', + ((f'value-{index}',) for index in range(2000)), + ) + database.commit() + database.close() + (data_dir / 'settings.json').write_text('{"enabled":true}', encoding='utf-8') + (data_dir / 'logs').mkdir() + (data_dir / 'logs' / 'audit.log').write_text('excluded', encoding='utf-8') + (data_dir / 'tmp').mkdir() + (data_dir / 'tmp' / 'upload.bin').write_bytes(b'excluded') + archive = tmp_path / 'backup.zip' + + manifest = create_online_backup(data_dir, archive) + + assert manifest == verify_backup(archive) + assert not tuple(operation_root.glob('snapshot-*')) + assert 'settings.json' in {item.path for item in manifest.files} + assert not any(item.path.startswith(('logs/', 'tmp/')) for item in manifest.files) + extracted_database = tmp_path / 'snapshot.db' + with zipfile.ZipFile(archive) as backup: + extracted_database.write_bytes(backup.read('data/app.db')) + snapshot = sqlite3.connect(extracted_database) + try: + assert snapshot.execute('PRAGMA quick_check').fetchone() == ('ok',) + assert snapshot.execute('SELECT COUNT(*) FROM records').fetchone() == (2000,) + finally: + snapshot.close() + + +def test_snapshot_barrier_waits_for_persistent_writes(): + entered = threading.Event() + finished = threading.Event() + + def writer(): + with persistent_write(): + entered.set() + finished.set() + + with snapshot_barrier(): + thread = threading.Thread(target=writer) + thread.start() + assert not entered.wait(0.1) + assert finished.wait(1) + thread.join(timeout=1) + + +def test_sqlalchemy_commit_waits_for_online_snapshot(app): + from app.models import User, db + + committed = threading.Event() + + def database_writer(): + with app.app_context(): + db.session.add(User( + username='snapshot-writer', + password_hash='not-used-in-this-test', + )) + db.session.commit() + committed.set() + + with snapshot_barrier(): + thread = threading.Thread(target=database_writer) + thread.start() + assert not committed.wait(0.1) + assert committed.wait(2) + thread.join(timeout=1) + + +def test_process_operation_lock_rejects_concurrent_operation( + tmp_path, monkeypatch +): + _configure_temp_root(monkeypatch, tmp_path) + outcome = [] + + def contender(): + try: + with operation_lock(timeout=0.1): + outcome.append('acquired') + except OperationBusyError: + outcome.append('busy') + + with operation_lock(): + thread = threading.Thread(target=contender) + thread.start() + thread.join(timeout=1) + assert outcome == ['busy'] + + +def test_online_backup_rejects_destination_inside_data_dir( + tmp_path, monkeypatch +): + _configure_temp_root(monkeypatch, tmp_path) + data_dir = tmp_path / 'data' + data_dir.mkdir() + sqlite3.connect(data_dir / 'app.db').close() + + with pytest.raises(ValueError, match='outside DATA_DIR'): + create_online_backup(data_dir, data_dir / 'backup.zip') diff --git a/tests/test_restore_web.py b/tests/test_restore_web.py new file mode 100644 index 0000000..91c0d99 --- /dev/null +++ b/tests/test_restore_web.py @@ -0,0 +1,164 @@ +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace + +import config + + +def _configure_temp_root(monkeypatch, tmp_path): + import config + import app.maintenance_mode as maintenance + + monkeypatch.setattr(config, 'BACKUP_TEMP_DIR', tmp_path / 'operations') + monkeypatch.setattr(config, 'DATA_DIR', tmp_path / 'data') + config.DATA_DIR.mkdir() + maintenance._state = None + maintenance._state_path = None + return maintenance + + +def test_maintenance_status_survives_memory_reset_and_blocks_writes( + app, client, monkeypatch, tmp_path +): + maintenance = _configure_temp_root(monkeypatch, tmp_path) + maintenance.begin_preparing('restore-test') + status_path = maintenance._status_path() + + assert status_path.is_file() + assert not status_path.is_relative_to(Path(config.DATA_DIR)) + maintenance._state = None + maintenance._state_path = None + assert maintenance.public_status()['state'] == 'preparing' + assert client.post('/api/upload').status_code == 503 + assert client.get('/ready').status_code == 503 + + maintenance.mark_failed('restore-test', 'cancelled safely') + assert not maintenance.is_active() + + +def test_session_epoch_invalidates_existing_login(app, client): + from app.auth import register_user + from app.models import db + from app.session_epoch import rotate_epoch + + with app.app_context(): + user, error = register_user('epoch_admin', 'password123') + assert error is None + user.is_admin = True + db.session.commit() + assert client.post('/login', data={ + 'username': 'epoch_admin', 'password': 'password123' + }).status_code == 302 + assert client.get('/admin').status_code == 200 + + rotate_epoch() + + response = client.get('/admin') + assert response.status_code == 302 + assert '/login?next=' in response.headers['Location'] + + +def test_rollback_failure_archive_survives_orphan_cleanup( + monkeypatch, tmp_path +): + maintenance = _configure_temp_root(monkeypatch, tmp_path) + from app.backup_operations import BackupOperationRegistry + + root = maintenance._status_path().parent + operation = root / 'operation-preserved' + operation.mkdir() + (operation / 'rollback.zip').write_bytes(b'emergency') + maintenance.begin_preparing('preserved') + maintenance.mark_in_progress('preserved', 'operation-preserved/rollback.zip') + maintenance.mark_failed( + 'preserved', + 'Restore and rollback failed', + rollback_failed=True, + ) + + BackupOperationRegistry().cleanup_orphans() + + assert (operation / 'rollback.zip').read_bytes() == b'emergency' + maintenance.clear_failed_status_after_cli_restore() + assert maintenance.public_status()['state'] == 'idle' + + +def test_failed_restore_runs_emergency_rollback_and_restarts( + app, monkeypatch, tmp_path +): + import config + import app.restore_service as service + + data_dir = tmp_path / 'data' + data_dir.mkdir() + monkeypatch.setattr(config, 'DATA_DIR', data_dir) + operation_dir = tmp_path / 'operation' + operation_dir.mkdir() + archive = operation_dir / 'archive.zip' + archive.write_bytes(b'uploaded') + record = SimpleNamespace( + operation_id='restore-failure', + directory=operation_dir, + archive_path=archive, + ) + events = [] + + @contextmanager + def fake_operation_lock(): + yield object() + + monkeypatch.setattr(service, 'operation_lock', fake_operation_lock) + restore_app = SimpleNamespace( + extensions={ + 'runtime_lifecycle': SimpleNamespace( + begin_shutdown=lambda grace: ( + events.append('shutdown') + or SimpleNamespace(remaining=()) + ) + ) + }, + app_context=app.app_context, + ) + monkeypatch.setattr(service, '_close_active_ssh_sessions', lambda: None) + monkeypatch.setattr( + service.connection_pool.temp_connection_pool, + 'close_all_connections', + lambda: None, + ) + monkeypatch.setattr(service, '_disconnect_sockets', lambda socketio: None) + + def create_rollback(data, destination, held_token=None): + destination.write_bytes(b'rollback') + events.append('snapshot') + + monkeypatch.setattr(service, 'create_online_backup', create_rollback) + restores = [] + + def restore(source, destination): + restores.append(Path(source).name) + if Path(source) == archive: + raise RuntimeError('restore failed') + + monkeypatch.setattr(service, 'restore_backup', restore) + monkeypatch.setattr(service, 'reset_cache', lambda: None) + monkeypatch.setattr(service, 'rotate_epoch', lambda: None) + monkeypatch.setattr(service, '_clear_restored_runtime_sessions', lambda path: None) + monkeypatch.setattr(service, 'begin_preparing', lambda operation_id: events.append('preparing')) + monkeypatch.setattr(service, 'mark_in_progress', lambda *args: events.append('restore')) + monkeypatch.setattr(service, 'mark_failed', lambda *args, **kwargs: events.append(('failed', kwargs))) + monkeypatch.setattr(service.backup_operations, 'remove', lambda operation_id: events.append('cleanup')) + monkeypatch.setattr(service, 'log_security_event', lambda name, **kwargs: events.append(name)) + + service._perform_restore( + restore_app, + SimpleNamespace(), + record, + 'admin', + '127.0.0.1', + lambda: events.append('restart'), + ) + + assert restores == ['archive.zip', 'rollback.zip'] + assert ('failed', {'rollback_failed': False}) in events + assert 'RESTORE_FAILED' in events + assert events[-2:] == ['cleanup', 'restart'] diff --git a/tests/test_runtime_lifecycle.py b/tests/test_runtime_lifecycle.py index c8e9c46..8ee098f 100644 --- a/tests/test_runtime_lifecycle.py +++ b/tests/test_runtime_lifecycle.py @@ -739,6 +739,7 @@ def test_app_owns_and_stops_all_permanent_cleanup_jobs(app): assert report.remaining == () assert report.cancelled == ( + ('backup-operation-cleanup', None), ('inactive_socket_session_cleanup', None), ('idle_ssh_session_cleanup', None), ('temporary_connection_cleanup', None), diff --git a/tests/test_secret_rotation.py b/tests/test_secret_rotation.py index 9ca7101..852cdda 100644 --- a/tests/test_secret_rotation.py +++ b/tests/test_secret_rotation.py @@ -1,5 +1,6 @@ import json from pathlib import Path +import sqlite3 import uuid from cryptography.fernet import Fernet @@ -42,7 +43,18 @@ def _rotation_data(tmp_path): }), encoding='utf-8', ) - (data_dir / 'app.db').write_bytes(b'database') + database = sqlite3.connect(data_dir / 'app.db') + try: + database.execute( + 'CREATE TABLE users (' + 'id INTEGER PRIMARY KEY, ' + 'username TEXT NOT NULL, ' + 'password_hash TEXT NOT NULL' + ')' + ) + database.commit() + finally: + database.close() return data_dir, old_secret, new_secret, plaintexts