From bc6ef2fe025440351bf87a56f1b8c87302be37ce Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 27 Jul 2026 12:51:51 +0000 Subject: [PATCH 1/4] db: Adds alembic scaffolding for DB migrations oslo_db has deprecated sqlalchemy-migrate in favor of alembic, and the old facade no longer exists in current oslo.db releases. This adds the alembic config, env, and template scaffolding as the first step of porting coriolis-dbsync to alembic. The implementation is based on OpenStack Nova and Neutron. --- coriolis/db/sqlalchemy/alembic/README.rst | 16 ++++ coriolis/db/sqlalchemy/alembic/alembic.ini | 49 +++++++++++ coriolis/db/sqlalchemy/alembic/env.py | 80 ++++++++++++++++++ coriolis/db/sqlalchemy/alembic/script.py.mako | 27 ++++++ .../tests/db/sqlalchemy/alembic/__init__.py | 0 .../tests/db/sqlalchemy/alembic/test_env.py | 82 +++++++++++++++++++ requirements.txt | 1 + 7 files changed, 255 insertions(+) create mode 100644 coriolis/db/sqlalchemy/alembic/README.rst create mode 100644 coriolis/db/sqlalchemy/alembic/alembic.ini create mode 100644 coriolis/db/sqlalchemy/alembic/env.py create mode 100644 coriolis/db/sqlalchemy/alembic/script.py.mako create mode 100644 coriolis/tests/db/sqlalchemy/alembic/__init__.py create mode 100644 coriolis/tests/db/sqlalchemy/alembic/test_env.py diff --git a/coriolis/db/sqlalchemy/alembic/README.rst b/coriolis/db/sqlalchemy/alembic/README.rst new file mode 100644 index 000000000..2cfa5bf80 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/README.rst @@ -0,0 +1,16 @@ +Migrations for the main database +================================ + +This directory contains migrations for the coriolis database. These are implemented +using `alembic`__, a lightweight database migration tool designed for usage +with `SQLAlchemy`__. + +The best place to start understanding Alembic is with its own `tutorial`__. You +can also play around with the :command:`alembic` command:: + + $ alembic --help + +.. __: https://alembic.sqlalchemy.org/en/latest/ +.. __: https://www.sqlalchemy.org/ +.. __: https://alembic.sqlalchemy.org/en/latest/tutorial.html + diff --git a/coriolis/db/sqlalchemy/alembic/alembic.ini b/coriolis/db/sqlalchemy/alembic/alembic.ini new file mode 100644 index 000000000..ce0b7f04b --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/alembic.ini @@ -0,0 +1,49 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# default to an empty string because the migration cli will +# extract the correct value and set it programmatically before alembic is fully +# invoked. +sqlalchemy.url = +path_separator = space + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/coriolis/db/sqlalchemy/alembic/env.py b/coriolis/db/sqlalchemy/alembic/env.py new file mode 100644 index 000000000..5fe936ef8 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/env.py @@ -0,0 +1,80 @@ +# Copyright 2026 Cloudbase Solutions Srl +# All Rights Reserved. + +from alembic import context +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from coriolis.db.sqlalchemy import models + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# this is the MetaData object for the various models in the database. +target_metadata = models.BASE.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL and not an Engine, though an + Engine is acceptable here as well. By skipping the Engine creation we + don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + render_as_batch=True, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine and associate a connection + with the context. + """ + connectable = config.attributes.get("connection", None) + + if connectable is not None: + context.configure( + connection=connectable, + target_metadata=target_metadata, + render_as_batch=True, + ) + + with context.begin_transaction(): + context.run_migrations() + return + + # only create Engine if we don't have a Connection from the outside. + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/coriolis/db/sqlalchemy/alembic/script.py.mako b/coriolis/db/sqlalchemy/alembic/script.py.mako new file mode 100644 index 000000000..d8c585872 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/script.py.mako @@ -0,0 +1,27 @@ +# Copyright ${create_date.year} Cloudbase Solutions Srl +# All Rights Reserved. + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from alembic import op +import sqlalchemy +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/coriolis/tests/db/sqlalchemy/alembic/__init__.py b/coriolis/tests/db/sqlalchemy/alembic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/coriolis/tests/db/sqlalchemy/alembic/test_env.py b/coriolis/tests/db/sqlalchemy/alembic/test_env.py new file mode 100644 index 000000000..2b226bcec --- /dev/null +++ b/coriolis/tests/db/sqlalchemy/alembic/test_env.py @@ -0,0 +1,82 @@ +# Copyright 2026 Cloudbase Solutions Srl +# All Rights Reserved. + +import importlib +import sys +from unittest import mock + +from coriolis.tests import test_base + +ENV_MODULE_NAME = "coriolis.db.sqlalchemy.alembic.env" + + +class AlembicEnvTestCase(test_base.CoriolisBaseTestCase): + """Test suite for the Coriolis Alembic 'env.py' migration script.""" + + def setUp(self): + super(AlembicEnvTestCase, self).setUp() + self.addCleanup(sys.modules.pop, ENV_MODULE_NAME, None) + + def _import_env(self, offline_mode, connection=None): + # run_migrations_online or run_migrations_offline runs on module + # import, so we need to set up what we need beforehand. + sys.modules.pop(ENV_MODULE_NAME, None) + + mock_context = mock.MagicMock() + mock_context.is_offline_mode.return_value = offline_mode + mock_context.config.attributes.get.return_value = connection + with mock.patch("alembic.context", mock_context): + env = importlib.import_module(ENV_MODULE_NAME) + + return env, mock_context + + def test_offline_mode(self): + env, mock_context = self._import_env(offline_mode=True) + + self.assertIs(env.config, mock_context.config) + mock_context.config.get_main_option.assert_called_once_with( + "sqlalchemy.url") + mock_context.configure.assert_called_once_with( + url=mock_context.config.get_main_option.return_value, + target_metadata=env.target_metadata, + render_as_batch=True, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + mock_context.begin_transaction.assert_called_once_with() + mock_context.run_migrations.assert_called_once_with() + + def test_online_mode_with_existing_connection(self): + env, mock_context = self._import_env( + offline_mode=False, connection=mock.sentinel.connection) + + mock_context.config.attributes.get.assert_called_once_with( + "connection", None) + mock_context.configure.assert_called_once_with( + connection=mock.sentinel.connection, + target_metadata=env.target_metadata, + render_as_batch=True, + ) + mock_context.begin_transaction.assert_called_once_with() + mock_context.run_migrations.assert_called_once_with() + + @mock.patch("sqlalchemy.engine_from_config") + def test_online_mode_creates_engine(self, mock_engine_from_config): + env, mock_context = self._import_env(offline_mode=False) + + mock_engine_from_config.assert_called_once_with( + mock_context.config.get_section.return_value, + prefix="sqlalchemy.", + poolclass=env.pool.NullPool, + ) + mock_connectable = mock_engine_from_config.return_value + mock_connectable.connect.assert_called_once_with() + mock_connection = ( + mock_connectable.connect.return_value.__enter__.return_value) + mock_context.configure.assert_called_once_with( + connection=mock_connection, + target_metadata=env.target_metadata, + render_as_batch=True, + ) + mock_context.begin_transaction.assert_called_once_with() + mock_context.run_migrations.assert_called_once_with() diff --git a/requirements.txt b/requirements.txt index 4783a4e43..7b0fc092c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ setuptools>=65.0.0,<82 # pkg_resources removed in 82; required by sqlalchemy-migrate +alembic keystoneauth1 keystonemiddleware Jinja2 From 822f396f37dfa571fe93890d2023eb9305801927 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 27 Jul 2026 15:07:52 +0000 Subject: [PATCH 2/4] db: Move sqlalchemy-migrate scripts to alembic/versions Relocates the existing migrate_repo scripts under coriolis/db/sqlalchemy/alembic/versions/ using git move, so their history is preserved. --- .../sqlalchemy/{migrate_repo => alembic}/versions/001_initial.py | 0 .../{migrate_repo => alembic}/versions/002_adds_endpoints.py | 0 .../{migrate_repo => alembic}/versions/003_adds_notes.py | 0 .../versions/004_adds_replica_schedules.py | 0 .../versions/005_adds_transfer_result.py | 0 .../{migrate_repo => alembic}/versions/006_adds_network_map.py | 0 .../versions/007_adds_storage_mappings.py | 0 .../versions/008_adds_source_environment.py | 0 .../versions/009_migrate_info_to_blob.py} | 0 .../{migrate_repo => alembic}/versions/010_adds_reservation_id.py | 0 .../{migrate_repo => alembic}/versions/011_adds_execution_type.py | 0 .../versions/012_adds_migration_sync_fields.py | 0 .../{migrate_repo => alembic}/versions/013_adds_task_index.py | 0 .../versions/014_adds_worker_service_regions.py | 0 .../versions/015_adds_action_last_execution_status.py | 0 .../versions/016_adds_minion_vm_pools.py | 0 .../versions/017_adds_user_scripts_column.py | 0 .../versions/018_adds_task_progress_idices.py | 0 .../versions/019_add_replica_scenario_field.py | 0 .../{migrate_repo => alembic}/versions/020_rename_tables.py | 0 .../versions/021_add_deployment_defaults.py | 0 .../versions/022_adds_auto_deploy_column.py | 0 .../{migrate_repo => alembic}/versions/023_add_deployer_id.py | 0 .../versions/024_add_clustered_to_base_transfer_action.py | 0 .../db/sqlalchemy/{migrate_repo => alembic}/versions/__init__.py | 0 25 files changed, 0 insertions(+), 0 deletions(-) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/001_initial.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/002_adds_endpoints.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/003_adds_notes.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/004_adds_replica_schedules.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/005_adds_transfer_result.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/006_adds_network_map.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/007_adds_storage_mappings.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/008_adds_source_environment.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo/versions/009_Migrate_info_to_blob.py => alembic/versions/009_migrate_info_to_blob.py} (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/010_adds_reservation_id.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/011_adds_execution_type.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/012_adds_migration_sync_fields.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/013_adds_task_index.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/014_adds_worker_service_regions.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/015_adds_action_last_execution_status.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/016_adds_minion_vm_pools.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/017_adds_user_scripts_column.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/018_adds_task_progress_idices.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/019_add_replica_scenario_field.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/020_rename_tables.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/021_add_deployment_defaults.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/022_adds_auto_deploy_column.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/023_add_deployer_id.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/024_add_clustered_to_base_transfer_action.py (100%) rename coriolis/db/sqlalchemy/{migrate_repo => alembic}/versions/__init__.py (100%) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py b/coriolis/db/sqlalchemy/alembic/versions/001_initial.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py rename to coriolis/db/sqlalchemy/alembic/versions/001_initial.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py b/coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py rename to coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py b/coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py rename to coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py b/coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py rename to coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py b/coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py rename to coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py b/coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py rename to coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py b/coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py rename to coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py b/coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py rename to coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py b/coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py rename to coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py b/coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py rename to coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py b/coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py rename to coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py b/coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py rename to coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py b/coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py rename to coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py b/coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py rename to coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py b/coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py rename to coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py b/coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py rename to coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py b/coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py rename to coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py b/coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py rename to coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py b/coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py rename to coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py b/coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py rename to coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py b/coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py rename to coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py b/coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py rename to coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py b/coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py rename to coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/024_add_clustered_to_base_transfer_action.py b/coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/024_add_clustered_to_base_transfer_action.py rename to coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/__init__.py b/coriolis/db/sqlalchemy/alembic/versions/__init__.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/__init__.py rename to coriolis/db/sqlalchemy/alembic/versions/__init__.py From b578137f62a18201c4b96ac12531e18aec74bc93 Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 27 Jul 2026 15:11:26 +0000 Subject: [PATCH 3/4] db: Port database migration scripts from sqlalchemy-migrate to alembic Updates database migrations to alembic. Most updates are straight-forward and repetitive, which some exceptions: 001, 016, 018, 020. In some scripts (e.g.: 001), the code was shifted around to account for the proper table creation order (e.g.: table Y might have a foreign key from table X). Scripts 014 and 016 deviate from the alembic db migration script pattern in order to keep diffs to a minimum. Downgrades are left unimplemented (raise NotImplementedError), matching the original scripts which never defined a downgrade() either. --- .../alembic/versions/001_initial.py | 114 +++++++++--------- .../alembic/versions/002_adds_endpoints.py | 53 ++++---- .../alembic/versions/003_adds_notes.py | 26 ++-- .../versions/004_adds_replica_schedules.py | 37 +++--- .../versions/005_adds_transfer_result.py | 26 ++-- .../alembic/versions/006_adds_network_map.py | 26 ++-- .../versions/007_adds_storage_mappings.py | 26 ++-- .../versions/008_adds_source_environment.py | 26 ++-- .../versions/009_migrate_info_to_blob.py | 26 ++-- .../versions/010_adds_reservation_id.py | 26 ++-- .../versions/011_adds_execution_type.py | 26 ++-- .../012_adds_migration_sync_fields.py | 34 ++++-- .../alembic/versions/013_adds_task_index.py | 26 ++-- .../014_adds_worker_service_regions.py | 36 ++++-- .../015_adds_action_last_execution_status.py | 30 +++-- .../versions/016_adds_minion_vm_pools.py | 67 +++++----- .../versions/017_adds_user_scripts_column.py | 28 +++-- .../versions/018_adds_task_progress_idices.py | 48 +++++--- .../019_add_replica_scenario_field.py | 29 +++-- .../alembic/versions/020_rename_tables.py | 84 ++++++++----- .../versions/021_add_deployment_defaults.py | 35 ++++-- .../versions/022_adds_auto_deploy_column.py | 30 +++-- .../alembic/versions/023_add_deployer_id.py | 37 +++--- ...4_add_clustered_to_base_transfer_action.py | 30 +++-- 24 files changed, 589 insertions(+), 337 deletions(-) diff --git a/coriolis/db/sqlalchemy/alembic/versions/001_initial.py b/coriolis/db/sqlalchemy/alembic/versions/001_initial.py index 71b6c534d..68302fc33 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/001_initial.py +++ b/coriolis/db/sqlalchemy/alembic/versions/001_initial.py @@ -1,17 +1,28 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +"""initial + +Revision ID: 001 +Revises: +Create Date: 2016-01-15 22:28:25.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "001" +down_revision = None +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, +def upgrade(): + op.create_table( + 'base_transfer_action', sqlalchemy.Column("base_id", sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -31,8 +42,18 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - migration = sqlalchemy.Table( - 'migration', meta, + op.create_table( + 'replica', + sqlalchemy.Column("id", sqlalchemy.String(36), + sqlalchemy.ForeignKey( + 'base_transfer_action.base_id'), + primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8' + ) + + op.create_table( + 'migration', sqlalchemy.Column("id", sqlalchemy.String(36), sqlalchemy.ForeignKey( 'base_transfer_action.base_id'), @@ -44,8 +65,26 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - task = sqlalchemy.Table( - 'task', meta, sqlalchemy.Column( + op.create_table( + 'tasks_execution', + sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, + default=lambda: str(uuid.uuid4())), + sqlalchemy.Column('created_at', sqlalchemy.DateTime), + sqlalchemy.Column('updated_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted', sqlalchemy.String(36)), + sqlalchemy.Column("action_id", sqlalchemy.String(36), + sqlalchemy.ForeignKey( + 'base_transfer_action.base_id'), + nullable=False), + sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False), + sqlalchemy.Column("number", sqlalchemy.Integer, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8' + ) + + op.create_table( + 'task', sqlalchemy.Column( 'id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -76,26 +115,8 @@ def upgrade(migrate_engine): sqlalchemy.Column("on_error", sqlalchemy.Boolean, nullable=True), mysql_engine='InnoDB', mysql_charset='utf8') - tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column("action_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - nullable=False), - sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False), - sqlalchemy.Column("number", sqlalchemy.Integer, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - task_progress_update = sqlalchemy.Table( - 'task_progress_update', meta, + op.create_table( + 'task_progress_update', sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -112,8 +133,8 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - task_events = sqlalchemy.Table( - 'task_event', meta, + op.create_table( + 'task_event', sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -129,31 +150,8 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - replica = sqlalchemy.Table( - 'replica', meta, - sqlalchemy.Column("id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - primary_key=True), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - tables = ( - base_transfer_action, - replica, - migration, - tasks_execution, - task, - task_progress_update, - task_events, - ) - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise +def downgrade(): + # Downgrades were never supported by the original sqlalchemy-migrate + # migrations this revision chain was ported from. + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py b/coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py index 4320a1197..18d57359d 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py +++ b/coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py @@ -1,17 +1,28 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +"""adds endpoints + +Revision ID: 002 +Revises: 001 +Create Date: 2017-03-22 22:17:09.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "002" +down_revision = "001" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - endpoint = sqlalchemy.Table( - 'endpoint', meta, +def upgrade(): + op.create_table( + 'endpoint', sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -29,37 +40,27 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - tables = ( - endpoint, - ) - - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise - - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - # NOTE(alexpilotti) delete all records in base_transfer_action # before performing this migration origin_endpoint_id = sqlalchemy.Column( "origin_endpoint_id", sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), nullable=False) - base_transfer_action.create_column(origin_endpoint_id) + op.add_column("base_transfer_action", origin_endpoint_id) destination_endpoint_id = sqlalchemy.Column( "destination_endpoint_id", sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), nullable=False) - base_transfer_action.create_column(destination_endpoint_id) + op.add_column("base_transfer_action", destination_endpoint_id) destination_environment = sqlalchemy.Column( "destination_environment", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(destination_environment) + op.add_column("base_transfer_action", destination_environment) + + op.drop_column("base_transfer_action", "origin") + op.drop_column("base_transfer_action", "destination") + - base_transfer_action.drop_column("origin") - base_transfer_action.drop_column("destination") +def downgrade(): + # Downgrades were never supported by the original sqlalchemy-migrate + # migrations this revision chain was ported from. + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py b/coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py index 40556f25a..cad0c5be7 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py +++ b/coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py @@ -1,17 +1,29 @@ # Copyright 2017 Cloudbase Solutions Srl # All Rights Reserved. -import sqlalchemy +"""adds notes + +Revision ID: 003 +Revises: 002 +Create Date: 2017-05-04 13:47:52.000000 +""" +from alembic import op +import sqlalchemy -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +# revision identifiers, used by Alembic. +revision = "003" +down_revision = "002" +branch_labels = None +depends_on = None - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) +def upgrade(): notes = sqlalchemy.Column( "notes", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(notes) + op.add_column("base_transfer_action", notes) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py b/coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py index 8ae2fa444..7a39801e5 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py +++ b/coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py @@ -1,17 +1,25 @@ +"""adds replica schedules + +Revision ID: 004 +Revises: 003 +Create Date: 2017-11-04 18:17:31.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "004" +down_revision = "003" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - sqlalchemy.Table( - 'replica', meta, autoload=True) - - replica_schedules = sqlalchemy.Table( - 'replica_schedules', meta, +def upgrade(): + op.create_table( + 'replica_schedules', sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -32,15 +40,6 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - tables = ( - replica_schedules, - ) - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py b/coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py index 7805fd4a2..843172191 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py +++ b/coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py @@ -1,17 +1,29 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. +"""adds transfer result + +Revision ID: 005 +Revises: 004 +Create Date: 2018-09-18 18:33:14.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "005" +down_revision = "004" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): # add 'transfer_result' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - transfer_result = sqlalchemy.Column( "transfer_result", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(transfer_result) + op.add_column("base_transfer_action", transfer_result) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py b/coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py index 460b5f43b..8ae25518e 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py +++ b/coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py @@ -1,17 +1,29 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. +"""adds network map + +Revision ID: 006 +Revises: 005 +Create Date: 2018-11-01 16:43:05.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "006" +down_revision = "005" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): # add 'network_map' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - network_map = sqlalchemy.Column( "network_map", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(network_map) + op.add_column("base_transfer_action", network_map) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py b/coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py index c99963293..668e86ed3 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py +++ b/coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py @@ -1,17 +1,29 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. +"""adds storage mappings + +Revision ID: 007 +Revises: 006 +Create Date: 2018-11-13 13:37:09.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "007" +down_revision = "006" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): # add 'storage_mappings' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - storage_mappings = sqlalchemy.Column( "storage_mappings", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(storage_mappings) + op.add_column("base_transfer_action", storage_mappings) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py b/coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py index c04096d62..0367a4261 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py +++ b/coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py @@ -1,17 +1,29 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. +"""adds source environment + +Revision ID: 008 +Revises: 007 +Create Date: 2018-11-27 18:48:35.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "008" +down_revision = "007" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): # add 'source_environment' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - source_environment = sqlalchemy.Column( "source_environment", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(source_environment) + op.add_column("base_transfer_action", source_environment) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py b/coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py index 1814a815d..8efb4d45e 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py +++ b/coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py @@ -1,12 +1,24 @@ -import sqlalchemy +"""migrate info to blob + +Revision ID: 009 +Revises: 008 +Create Date: 2019-03-23 00:15:49.000000 +""" + +from alembic import op from sqlalchemy import types +# revision identifiers, used by Alembic. +revision = "009" +down_revision = "008" +branch_labels = None +depends_on = None + -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): + op.alter_column( + 'base_transfer_action', 'info', type_=types.LargeBinary(4294967295)) - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - base_transfer_action.c.info.alter(type=types.LargeBinary(4294967295)) +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py b/coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py index 0bd370d13..289029e62 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py +++ b/coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py @@ -1,17 +1,29 @@ # Copyright 2018 Cloudbase Solutions Srl # All Rights Reserved. +"""adds reservation id + +Revision ID: 010 +Revises: 009 +Create Date: 2019-04-26 22:47:10.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "010" +down_revision = "009" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): # add 'reservation_id' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - reservation_id = sqlalchemy.Column( "reservation_id", sqlalchemy.String(36), nullable=True) - base_transfer_action.create_column(reservation_id) + op.add_column("base_transfer_action", reservation_id) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py b/coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py index 44e835435..e8be1e1b5 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py +++ b/coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py @@ -1,17 +1,29 @@ # Copyright 2019 Cloudbase Solutions Srl # All Rights Reserved. +"""adds execution type + +Revision ID: 011 +Revises: 010 +Create Date: 2019-08-15 07:30:35.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "011" +down_revision = "010" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): # add 'type' column to 'tasks_execution': - tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, autoload=True) - execution_type = sqlalchemy.Column( "type", sqlalchemy.String(20)) - tasks_execution.create_column(execution_type) + op.add_column("tasks_execution", execution_type) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py b/coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py index d278324c8..de08ca2ed 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py +++ b/coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py @@ -1,19 +1,31 @@ -import sqlalchemy +"""adds migration sync fields + +Revision ID: 012 +Revises: 011 +Create Date: 2019-10-16 15:40:42.000000 +""" +from alembic import op +import sqlalchemy -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +# revision identifiers, used by Alembic. +revision = "012" +down_revision = "011" +branch_labels = None +depends_on = None - migration = sqlalchemy.Table( - 'migration', meta, autoload=True) +def upgrade(): shutdown_instances = sqlalchemy.Column( "shutdown_instances", sqlalchemy.Boolean, - nullable=False, default=False) - migration.create_column(shutdown_instances) + nullable=False, server_default=sqlalchemy.false()) + op.add_column("migration", shutdown_instances) replication_count = sqlalchemy.Column( - "replication_count", sqlalchemy.Integer, default=0, - nullable=False) - migration.create_column(replication_count) + "replication_count", sqlalchemy.Integer, + nullable=False, server_default="0") + op.add_column("migration", replication_count) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py b/coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py index fb841d7d8..e4075152e 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py +++ b/coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py @@ -1,13 +1,25 @@ -import sqlalchemy +"""adds task index + +Revision ID: 013 +Revises: 012 +Create Date: 2019-10-18 19:35:20.000000 +""" +from alembic import op +import sqlalchemy -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +# revision identifiers, used by Alembic. +revision = "013" +down_revision = "012" +branch_labels = None +depends_on = None - task = sqlalchemy.Table('task', meta, autoload=True) +def upgrade(): index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) + "index", sqlalchemy.Integer, nullable=False, server_default="0") + op.add_column("task", index) + - task.create_column(index) +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py b/coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py index 4682a974a..781eb8801 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py +++ b/coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py @@ -1,17 +1,31 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +"""adds worker service regions + +Revision ID: 014 +Revises: 013 +Create Date: 2020-07-28 18:21:57.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "014" +down_revision = "013" +branch_labels = None +depends_on = None + -def upgrade(migrate_engine): +def upgrade(): meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - sqlalchemy.Table( - 'endpoint', meta, autoload=True) + # load 'endpoint' into meta so the 'endpoint_id' foreign key below can + # be resolved against it. + sqlalchemy.Table('endpoint', meta, autoload_with=op.get_bind()) tables = [] @@ -122,11 +136,9 @@ def upgrade(migrate_engine): mysql_engine='InnoDB', mysql_charset='utf8')) - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise + for table in tables: + table.create(bind=op.get_bind()) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py b/coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py index 27c3a8b18..3bcb6054b 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py +++ b/coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py @@ -1,20 +1,30 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +"""adds action last execution status + +Revision ID: 015 +Revises: 014 +Create Date: 2020-08-27 20:38:25.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "015" +down_revision = "014" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): # add 'last_execution_status' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - last_execution_status = sqlalchemy.Column( "last_execution_status", sqlalchemy.String(255), - default=lambda: "UNEXECUTED", nullable=False) - base_transfer_action.create_column(last_execution_status) + nullable=False, server_default="UNEXECUTED") + op.add_column("base_transfer_action", last_execution_status) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py b/coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py index 6c48212bb..6fc1790e8 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py +++ b/coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py @@ -1,24 +1,39 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +"""adds minion vm pools + +Revision ID: 016 +Revises: 015 +Create Date: 2020-08-26 07:10:03.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "016" +down_revision = "015" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") +def upgrade(): + meta = sqlalchemy.MetaData() # extend tasks execution 'type' column: - tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, autoload=True) - tasks_execution.c.type.alter(type=sqlalchemy.String(255)) + op.alter_column( + 'tasks_execution', 'type', + existing_type=sqlalchemy.String(20), + existing_nullable=True, + type_=sqlalchemy.String(255), + nullable=True) + + # load 'endpoint' into meta so the 'endpoint_id' foreign key below can + # be resolved against it. + sqlalchemy.Table('endpoint', meta, autoload_with=op.get_bind()) tables = [] @@ -153,30 +168,26 @@ def upgrade(migrate_engine): "message", sqlalchemy.Text, nullable=True), mysql_engine='InnoDB', mysql_charset='utf8')) + for table in tables: + table.create(bind=op.get_bind()) + # add the pool option properties for the transfer: origin_minion_pool_id = sqlalchemy.Column( "origin_minion_pool_id", sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=True) + op.add_column("base_transfer_action", origin_minion_pool_id) + destination_minion_pool_id = sqlalchemy.Column( "destination_minion_pool_id", sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=True) + op.add_column("base_transfer_action", destination_minion_pool_id) + instance_osmorphing_minion_pool_mappings = sqlalchemy.Column( "instance_osmorphing_minion_pool_mappings", sqlalchemy.Text, - nullable=False, default='{}') - - created_columns = [] - try: - for index, table in enumerate(tables): - table.create() - for col in [ - origin_minion_pool_id, destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings]: - base_transfer_action.create_column(col) - created_columns.append(col) - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - for col in created_columns: - base_transfer_action.drop_column(col) - meta.drop_all(tables=tables[:index]) - raise + nullable=False, server_default='{}') + op.add_column( + "base_transfer_action", instance_osmorphing_minion_pool_mappings) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py b/coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py index f860fd72c..7d9083b77 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py +++ b/coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py @@ -1,19 +1,29 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +"""adds user scripts column + +Revision ID: 017 +Revises: 016 +Create Date: 2021-01-26 19:16:44.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "017" +down_revision = "016" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +def upgrade(): # add 'user_scripts' column to 'base_transfer_action': - base_transfer = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - user_scripts = sqlalchemy.Column( "user_scripts", sqlalchemy.Text, nullable=True) - base_transfer.create_column(user_scripts) + op.add_column("base_transfer_action", user_scripts) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py b/coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py index 6e6101461..33b0abaee 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py +++ b/coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py @@ -1,21 +1,41 @@ +"""adds task progress idices + +Revision ID: 018 +Revises: 017 +Create Date: 2021-01-07 17:33:05.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "018" +down_revision = "017" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - task_event = sqlalchemy.Table('task_event', meta, autoload=True) +def upgrade(): event_index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) - task_event.create_column(event_index) + "index", sqlalchemy.Integer, nullable=False, server_default="0") + op.add_column("task_event", event_index) - task_progress_update = sqlalchemy.Table( - 'task_progress_update', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") progress_index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) - task_progress_update.create_column(progress_index) - task_progress_update.c.current_step.alter(type=sqlalchemy.BigInteger) - task_progress_update.c.total_steps.alter(type=sqlalchemy.BigInteger) + "index", sqlalchemy.Integer, nullable=False, server_default="0") + op.add_column("task_progress_update", progress_index) + op.alter_column( + "task_progress_update", "current_step", + existing_type=sqlalchemy.Integer, + existing_nullable=False, + type_=sqlalchemy.BigInteger, + nullable=False) + op.alter_column( + "task_progress_update", "total_steps", + existing_type=sqlalchemy.Integer, + existing_nullable=True, + type_=sqlalchemy.BigInteger, + nullable=True) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py b/coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py index edd16a5d9..7be084f37 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py +++ b/coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py @@ -1,20 +1,29 @@ # Copyright 2024 Cloudbase Solutions Srl # All Rights Reserved. -import sqlalchemy +"""add replica scenario field + +Revision ID: 019 +Revises: 018 +Create Date: 2024-04-09 14:42:14.000000 +""" +from alembic import op +import sqlalchemy -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine +# revision identifiers, used by Alembic. +revision = "019" +down_revision = "018" +branch_labels = None +depends_on = None - replica = sqlalchemy.Table( - 'replica', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") +def upgrade(): replica_scenario = sqlalchemy.Column( "scenario", sqlalchemy.String(255), nullable=False, - default="replica") + server_default="replica") + op.add_column("replica", replica_scenario) + - replica.create_column(replica_scenario) +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py b/coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py index a6bed3b3c..7d289d426 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py +++ b/coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py @@ -1,12 +1,40 @@ +"""rename tables + +Revision ID: 020 +Revises: 019 +Create Date: 2024-10-29 15:26:01.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "020" +down_revision = "019" +branch_labels = None +depends_on = None + +# NOTE(dvincze): Update models polymorphic identity. Due to the model code +# changes, this cannot be done using the ORM; raw SQL is required. +_TYPE_RENAMES = ( + ("base_transfer_action", "replica", "transfer"), + ("tasks_execution", "replica_execution", "transfer_execution"), + ("tasks_execution", "replica_disks_delete", "transfer_disks_delete"), + ("tasks_execution", "replica_deploy", "deployment"), + ("tasks_execution", "replica_update", "transfer_update"), +) -def upgrade(migrate_engine): + +def upgrade(): meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - replica = sqlalchemy.Table('replica', meta, autoload=True) - replica.rename('transfer') + op.rename_table('replica', 'transfer') + + # load 'base_transfer_action' and 'transfer' into meta so the foreign + # keys below can be resolved against them. + sqlalchemy.Table( + 'base_transfer_action', meta, autoload_with=op.get_bind()) + sqlalchemy.Table('transfer', meta, autoload_with=op.get_bind()) deployment = sqlalchemy.Table( 'deployment', meta, @@ -19,32 +47,22 @@ def upgrade(migrate_engine): nullable=False), mysql_engine="InnoDB", mysql_charset="utf8") - try: - deployment.create() - except Exception: - deployment.drop() - raise - - replica_schedule = sqlalchemy.Table( - 'replica_schedules', meta, autoload=True) - replica_schedule.rename('transfer_schedules') - replica_schedule.c.replica_id.alter(name='transfer_id') - - # NOTE(dvincze): Update models polymorphic identity - # Due to the model code changes, this cannot be done using the ORM. - # Had to resort to using raw SQL statements. - with migrate_engine.connect() as conn: - conn.execute( - 'UPDATE base_transfer_action SET type = "transfer" ' - 'WHERE type = "replica";') - conn.execute('UPDATE tasks_execution SET type = "transfer_execution"' - 'WHERE type = "replica_execution"') - conn.execute( - 'UPDATE tasks_execution SET type = "transfer_disks_delete"' - 'WHERE type = "replica_disks_delete"') - conn.execute( - 'UPDATE tasks_execution SET type = "deployment"' - 'WHERE type = "replica_deploy"') - conn.execute( - 'UPDATE tasks_execution SET type = "transfer_update"' - 'WHERE type = "replica_update"') + deployment.create(bind=op.get_bind()) + + op.rename_table('replica_schedules', 'transfer_schedules') + op.alter_column( + 'transfer_schedules', 'replica_id', + new_column_name='transfer_id', + existing_type=sqlalchemy.String(36), + existing_nullable=False) + + for table, old_type, new_type in _TYPE_RENAMES: + op.execute( + sqlalchemy.text( + f"UPDATE {table} SET type = :new_type " + f"WHERE type = :old_type" + ).bindparams(new_type=new_type, old_type=old_type)) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py b/coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py index 95a5b1684..e2bac0d8a 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py +++ b/coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py @@ -1,17 +1,30 @@ +"""add deployment defaults + +Revision ID: 021 +Revises: 020 +Create Date: 2025-01-29 19:13:39.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "021" +down_revision = "020" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - base_transfer = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") +def upgrade(): clone_disks = sqlalchemy.Column( - "clone_disks", sqlalchemy.Boolean, nullable=False, default=True) - base_transfer.create_column(clone_disks) + "clone_disks", sqlalchemy.Boolean, nullable=False, + server_default=sqlalchemy.true()) + op.add_column("base_transfer_action", clone_disks) skip_os_morphing = sqlalchemy.Column( - "skip_os_morphing", sqlalchemy.Boolean, nullable=False, default=False) - base_transfer.create_column(skip_os_morphing) + "skip_os_morphing", sqlalchemy.Boolean, nullable=False, + server_default=sqlalchemy.false()) + op.add_column("base_transfer_action", skip_os_morphing) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py b/coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py index 63a7cfb0b..081acf940 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py +++ b/coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py @@ -1,14 +1,26 @@ +"""adds auto deploy column + +Revision ID: 022 +Revises: 021 +Create Date: 2025-01-30 13:54:39.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "022" +down_revision = "021" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - transfer_schedule = sqlalchemy.Table( - 'transfer_schedules', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") +def upgrade(): auto_deploy = sqlalchemy.Column( - 'auto_deploy', sqlalchemy.Boolean, nullable=False, default=False) - transfer_schedule.create_column(auto_deploy) + 'auto_deploy', sqlalchemy.Boolean, nullable=False, + server_default=sqlalchemy.false()) + op.add_column("transfer_schedules", auto_deploy) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py b/coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py index 37729f55b..dd9b2381c 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py +++ b/coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py @@ -1,25 +1,28 @@ +"""add deployer id + +Revision ID: 023 +Revises: 022 +Create Date: 2025-02-06 14:42:29.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "023" +down_revision = "022" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - deployment = sqlalchemy.Table( - 'deployment', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") +def upgrade(): deployer_id = sqlalchemy.Column( 'deployer_id', sqlalchemy.String(36), nullable=True) + op.add_column("deployment", deployer_id) trust_id = sqlalchemy.Column( 'trust_id', sqlalchemy.String(255), nullable=True) - created_columns = [] - try: - deployment.create_column(deployer_id) - created_columns.append(deployer_id) - deployment.create_column(trust_id) - created_columns.append(trust_id) - except Exception: - for c in created_columns: - deployment.drop_column(c) - raise + op.add_column("deployment", trust_id) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py b/coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py index 9e834e85d..761814ae4 100644 --- a/coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py +++ b/coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py @@ -1,20 +1,36 @@ # Copyright 2026 Cloudbase Solutions Srl # All Rights Reserved. +"""add clustered to base transfer action + +Revision ID: 024 +Revises: 023 +Create Date: 2026-04-09 04:03:23.000000 +""" + +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "024" +down_revision = "023" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - base_transfer = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - if 'clustered' in base_transfer.c: +def upgrade(): + inspector = sqlalchemy.inspect(op.get_bind()) + columns = [c['name'] for c in inspector.get_columns( + 'base_transfer_action')] + if 'clustered' in columns: return # server_default so existing rows get a value when the column is added # (MySQL stores booleans as TINYINT). clustered = sqlalchemy.Column( 'clustered', sqlalchemy.Boolean, nullable=False, server_default=sqlalchemy.text('0')) - base_transfer.create_column(clustered) + op.add_column("base_transfer_action", clustered) + + +def downgrade(): + raise NotImplementedError() From ad33e4063bbfb76a238e58fd872597e89a37c71e Mon Sep 17 00:00:00 2001 From: Claudiu Belu Date: Mon, 27 Jul 2026 17:47:34 +0000 Subject: [PATCH 4/4] db: Switch coriolis-dbsync from sqlalchemy-migrate to alembic Rewrites coriolis/db/sqlalchemy/migration.py to drive the alembic revision chain added instead of oslo_db.sqlalchemy.migration, which no longer exists in current oslo.db releases. db_sync auto-detects a pre-existing sqlalchemy-migrate `migrate_version` table left over from the old migrate_repo: if it's at the final version (24), it stamps the database onto the equivalent alembic revision instead of re-running already-applied migrations, then proceeds to upgrade normally. If the database is at any other legacy version, an exception will be raised, mentioning that upgrades must be finished with the previous release first. The now-empty migrate_repo/ is removed entirely. The oslo.db upper constraint is removed. --- coriolis/db/sqlalchemy/api.py | 8 +- .../db/sqlalchemy/migrate_repo/__init__.py | 0 coriolis/db/sqlalchemy/migrate_repo/manage.py | 9 -- .../db/sqlalchemy/migrate_repo/migrate.cfg | 25 ---- coriolis/db/sqlalchemy/migration.py | 73 +++++++++-- coriolis/tests/db/sqlalchemy/test_api.py | 31 ++++- .../tests/db/sqlalchemy/test_migration.py | 115 ++++++++++++------ requirements.txt | 2 +- 8 files changed, 167 insertions(+), 96 deletions(-) delete mode 100644 coriolis/db/sqlalchemy/migrate_repo/__init__.py delete mode 100644 coriolis/db/sqlalchemy/migrate_repo/manage.py delete mode 100644 coriolis/db/sqlalchemy/migrate_repo/migrate.cfg diff --git a/coriolis/db/sqlalchemy/api.py b/coriolis/db/sqlalchemy/api.py index 822972268..21e056d2a 100644 --- a/coriolis/db/sqlalchemy/api.py +++ b/coriolis/db/sqlalchemy/api.py @@ -39,9 +39,11 @@ def get_backend(): def db_sync(engine, version=None): """Migrate the database to `version` or the most recent version.""" - if version is not None and int(version) < db_version(engine): - raise exception.CoriolisException( - _("Cannot migrate to lower schema version.")) + if version is not None: + current_version = db_version(engine) + if current_version is not None and int(version) < int(current_version): + raise exception.CoriolisException( + _("Cannot migrate to lower schema version.")) return migration.db_sync(engine, version=version) diff --git a/coriolis/db/sqlalchemy/migrate_repo/__init__.py b/coriolis/db/sqlalchemy/migrate_repo/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/coriolis/db/sqlalchemy/migrate_repo/manage.py b/coriolis/db/sqlalchemy/migrate_repo/manage.py deleted file mode 100644 index 48fb3c431..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/manage.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Cloudbase Solutions Srl -# All Rights Reserved. - -from migrate.versioning.shell import main - -if __name__ == '__main__': - main(debug='False') diff --git a/coriolis/db/sqlalchemy/migrate_repo/migrate.cfg b/coriolis/db/sqlalchemy/migrate_repo/migrate.cfg deleted file mode 100644 index 75db1dffd..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=coriolis - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering=False diff --git a/coriolis/db/sqlalchemy/migration.py b/coriolis/db/sqlalchemy/migration.py index f05ceea14..847e26bcd 100644 --- a/coriolis/db/sqlalchemy/migration.py +++ b/coriolis/db/sqlalchemy/migration.py @@ -3,25 +3,74 @@ import os -from oslo_db.sqlalchemy import migration as oslo_migration +from alembic import command +from alembic import config as alembic_config +from alembic.runtime import migration as alembic_migration +import sqlalchemy -INIT_VERSION = 0 +from coriolis import exception +from coriolis.i18n import _ + +ALEMBIC_DIR = os.path.join(os.path.dirname(__file__), "alembic") +ALEMBIC_INI_PATH = os.path.join(ALEMBIC_DIR, "alembic.ini") + +# The final version stamped by the old sqlalchemy-migrate migrate_repo +# (migrate_repo/versions/024_add_clustered_to_base_transfer_action.py), and +# the alembic revision it corresponds to. +LEGACY_VERSION_TABLE = "migrate_version" +LEGACY_FINAL_VERSION = 24 +LEGACY_FINAL_ALEMBIC_REVISION = "024" + + +def _get_alembic_config(): + config = alembic_config.Config(ALEMBIC_INI_PATH) + config.set_main_option("script_location", ALEMBIC_DIR) + return config + + +def _stamp_legacy_database_if_needed(engine, config): + """Transition a sqlalchemy-migrate managed database to alembic. + + If this database was previously managed by the old sqlalchemy-migrate + based migrate_repo, stamp it onto the equivalent alembic revision + instead of re-running the already-applied DDL. + """ + inspector = sqlalchemy.inspect(engine) + if LEGACY_VERSION_TABLE not in inspector.get_table_names(): + return + + with engine.connect() as conn: + legacy_version = conn.execute( + sqlalchemy.text( + f"SELECT version FROM {LEGACY_VERSION_TABLE}")).scalar() + + if legacy_version != LEGACY_FINAL_VERSION: + raise exception.CoriolisException( + _("This database was last migrated using the legacy " + "sqlalchemy-migrate based coriolis-dbsync (version %(cur)s), " + "but only the final version (%(final)s) can be transitioned " + "to alembic. Please run coriolis-dbsync from a previous " + "Coriolis release first to reach version %(final)s.") % { + "cur": legacy_version, "final": LEGACY_FINAL_VERSION}) + + config.attributes["connection"] = engine.connect() + command.stamp(config, LEGACY_FINAL_ALEMBIC_REVISION) def db_sync(engine, version=None): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_sync(engine, path, version, - init_version=INIT_VERSION) + config = _get_alembic_config() + _stamp_legacy_database_if_needed(engine, config) + config.attributes["connection"] = engine.connect() + return command.upgrade(config, version or "head") def db_version(engine): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_version(engine, path, INIT_VERSION) + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_revision() def db_version_control(engine, version=None): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_version_control(engine, path, version) + config = _get_alembic_config() + config.attributes["connection"] = engine.connect() + return command.stamp(config, version or "head") diff --git a/coriolis/tests/db/sqlalchemy/test_api.py b/coriolis/tests/db/sqlalchemy/test_api.py index de79b5d86..de4eda7ac 100644 --- a/coriolis/tests/db/sqlalchemy/test_api.py +++ b/coriolis/tests/db/sqlalchemy/test_api.py @@ -94,9 +94,9 @@ def test_db_sync_version( mock_db_version, mock_db_sync ): - mock_db_version.return_value = 1 + mock_db_version.return_value = "001" - result = api.db_sync(mock.sentinel.engine, version=1) + result = api.db_sync(mock.sentinel.engine, version="002") self.assertEqual( mock_db_sync.return_value, @@ -104,20 +104,39 @@ def test_db_sync_version( ) mock_db_version.assert_called_once_with(mock.sentinel.engine) mock_db_sync.assert_called_once_with( - mock.sentinel.engine, version=1) + mock.sentinel.engine, version="002") + + @mock.patch.object(migration, 'db_sync') + @mock.patch.object(api, 'db_version') + def test_db_sync_version_no_current_version( + self, + mock_db_version, + mock_db_sync + ): + mock_db_version.return_value = None + + result = api.db_sync(mock.sentinel.engine, version="001") + + self.assertEqual( + mock_db_sync.return_value, + result + ) + mock_db_version.assert_called_once_with(mock.sentinel.engine) + mock_db_sync.assert_called_once_with( + mock.sentinel.engine, version="001") @mock.patch.object(api, 'db_version') def test_db_sync_version_raise( self, - mock_db_version + mock_db_version, ): - mock_db_version.return_value = 2 + mock_db_version.return_value = "003" self.assertRaises( exception.CoriolisException, api.db_sync, mock.sentinel.engine, - version=1 + version="001" ) mock_db_version.assert_called_once_with(mock.sentinel.engine) diff --git a/coriolis/tests/db/sqlalchemy/test_migration.py b/coriolis/tests/db/sqlalchemy/test_migration.py index 6f35ad59c..642e7b891 100644 --- a/coriolis/tests/db/sqlalchemy/test_migration.py +++ b/coriolis/tests/db/sqlalchemy/test_migration.py @@ -1,61 +1,96 @@ # Copyright 2024 Cloudbase Solutions Srl # All Rights Reserved. -import os from unittest import mock -from oslo_db.sqlalchemy import migration as oslo_migration - from coriolis.db.sqlalchemy import migration +from coriolis import exception from coriolis.tests import test_base class DatabaseSqlalchemyMigrationTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis Database Sqlalchemy migration.""" - @mock.patch.object(os.path, 'abspath') - @mock.patch.object(oslo_migration, 'db_sync') - def test_db_sync(self, mock_db_sync, mock_abspath): - mock_abspath.return_value = "/abspath" + @mock.patch.object(migration, "sqlalchemy") + def test_stamp_legacy_database_if_needed_no_table(self, mock_sqlalchemy): + mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [ + "foo"] + mock_engine = mock.MagicMock() + mock_config = mock.MagicMock() - result = migration.db_sync(mock.sentinel.engine, mock.sentinel.version) + migration._stamp_legacy_database_if_needed(mock_engine, mock_config) - self.assertEqual( - mock_db_sync.return_value, - result - ) - mock_db_sync.assert_called_once_with( - mock.sentinel.engine, - "/abspath/migrate_repo", - mock.sentinel.version, - init_version=0 - ) + mock_engine.connect.assert_not_called() - @mock.patch.object(os.path, 'abspath') - @mock.patch.object(oslo_migration, 'db_version') - def test_db_version(self, mock_db_version, mock_abspath): - mock_abspath.return_value = "/abspath" + @mock.patch.object(migration, "sqlalchemy") + def test_stamp_legacy_database_if_needed_raises(self, mock_sqlalchemy): + mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [ + migration.LEGACY_VERSION_TABLE] + mock_engine = mock.MagicMock() + mock_conn = mock_engine.connect.return_value.__enter__.return_value + mock_conn.execute.return_value.scalar.return_value = 1 + mock_config = mock.MagicMock() - result = migration.db_version(mock.sentinel.engine) + self.assertRaises( + exception.CoriolisException, + migration._stamp_legacy_database_if_needed, + mock_engine, mock_config) - self.assertEqual(mock_db_version.return_value, result) - mock_db_version.assert_called_once_with( - mock.sentinel.engine, - "/abspath/migrate_repo", - 0 - ) + @mock.patch.object(migration, "command") + @mock.patch.object(migration, "sqlalchemy") + def test_stamp_legacy_database_if_needed_final_version( + self, mock_sqlalchemy, mock_command, + ): + mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [ + migration.LEGACY_VERSION_TABLE] + mock_engine = mock.MagicMock() + mock_conn = mock_engine.connect.return_value.__enter__.return_value + mock_conn.execute.return_value.scalar.return_value = ( + migration.LEGACY_FINAL_VERSION) + mock_config = mock.MagicMock() - @mock.patch.object(os.path, 'abspath') - @mock.patch.object(oslo_migration, 'db_version_control') - def test_db_version_control(self, mock_db_version_control, mock_abspath): - mock_abspath.return_value = "/abspath" + migration._stamp_legacy_database_if_needed(mock_engine, mock_config) - result = migration.db_version_control( - mock.sentinel.engine, mock.sentinel.version) + mock_command.stamp.assert_called_once_with( + mock_config, migration.LEGACY_FINAL_ALEMBIC_REVISION) + + @mock.patch.object(migration, "_stamp_legacy_database_if_needed") + @mock.patch.object(migration, "command") + @mock.patch.object(migration, "_get_alembic_config") + def test_db_sync( + self, mock_get_config, mock_command, mock_stamp_legacy, + ): + mock_engine = mock.MagicMock() + + result = migration.db_sync(mock_engine, mock.sentinel.version) - self.assertEqual(mock_db_version_control.return_value, result) - mock_db_version_control.assert_called_once_with( - mock.sentinel.engine, - "/abspath/migrate_repo", - mock.sentinel.version + self.assertEqual(mock_command.upgrade.return_value, result) + mock_get_config.assert_called_once_with() + mock_stamp_legacy.assert_called_once_with( + mock_engine, mock_get_config.return_value) + mock_command.upgrade.assert_called_once_with( + mock_get_config.return_value, mock.sentinel.version) + + @mock.patch.object(migration.alembic_migration, "MigrationContext") + def test_db_version(self, mock_migration_context): + mock_engine = mock.MagicMock() + + result = migration.db_version(mock_engine) + + mock_conn = mock_engine.connect.return_value.__enter__.return_value + mock_configure = mock_migration_context.configure + mock_configure.assert_called_once_with(mock_conn) + self.assertEqual( + mock_configure.return_value.get_current_revision.return_value, + result, ) + + @mock.patch.object(migration, "command") + @mock.patch.object(migration, "_get_alembic_config") + def test_db_version_control(self, mock_get_config, mock_command): + result = migration.db_version_control( + mock.MagicMock(), mock.sentinel.version) + + self.assertEqual(mock_command.stamp.return_value, result) + mock_command.stamp.assert_called_once_with( + mock_get_config.return_value, mock.sentinel.version) diff --git a/requirements.txt b/requirements.txt index 7b0fc092c..4bab04a03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ oslo.cache oslo.concurrency oslo.config<9.8.0 oslo.context<6.0.0 -oslo.db<=12.3.2 +oslo.db oslo.i18n oslo.log oslo.messaging==12.2.0