diff --git a/coriolis/db/sqlalchemy/alembic/README.rst b/coriolis/db/sqlalchemy/alembic/README.rst new file mode 100644 index 000000000..2cfa5bf80 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/README.rst @@ -0,0 +1,16 @@ +Migrations for the main database +================================ + +This directory contains migrations for the coriolis database. These are implemented +using `alembic`__, a lightweight database migration tool designed for usage +with `SQLAlchemy`__. + +The best place to start understanding Alembic is with its own `tutorial`__. You +can also play around with the :command:`alembic` command:: + + $ alembic --help + +.. __: https://alembic.sqlalchemy.org/en/latest/ +.. __: https://www.sqlalchemy.org/ +.. __: https://alembic.sqlalchemy.org/en/latest/tutorial.html + diff --git a/coriolis/db/sqlalchemy/alembic/alembic.ini b/coriolis/db/sqlalchemy/alembic/alembic.ini new file mode 100644 index 000000000..ce0b7f04b --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/alembic.ini @@ -0,0 +1,49 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# default to an empty string because the migration cli will +# extract the correct value and set it programmatically before alembic is fully +# invoked. +sqlalchemy.url = +path_separator = space + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/coriolis/db/sqlalchemy/alembic/env.py b/coriolis/db/sqlalchemy/alembic/env.py new file mode 100644 index 000000000..5fe936ef8 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/env.py @@ -0,0 +1,80 @@ +# Copyright 2026 Cloudbase Solutions Srl +# All Rights Reserved. + +from alembic import context +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from coriolis.db.sqlalchemy import models + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# this is the MetaData object for the various models in the database. +target_metadata = models.BASE.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL and not an Engine, though an + Engine is acceptable here as well. By skipping the Engine creation we + don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + render_as_batch=True, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine and associate a connection + with the context. + """ + connectable = config.attributes.get("connection", None) + + if connectable is not None: + context.configure( + connection=connectable, + target_metadata=target_metadata, + render_as_batch=True, + ) + + with context.begin_transaction(): + context.run_migrations() + return + + # only create Engine if we don't have a Connection from the outside. + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/coriolis/db/sqlalchemy/alembic/script.py.mako b/coriolis/db/sqlalchemy/alembic/script.py.mako new file mode 100644 index 000000000..d8c585872 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/script.py.mako @@ -0,0 +1,27 @@ +# Copyright ${create_date.year} Cloudbase Solutions Srl +# All Rights Reserved. + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from alembic import op +import sqlalchemy +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py b/coriolis/db/sqlalchemy/alembic/versions/001_initial.py similarity index 84% rename from coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py rename to coriolis/db/sqlalchemy/alembic/versions/001_initial.py index 71b6c534d..68302fc33 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/001_initial.py +++ b/coriolis/db/sqlalchemy/alembic/versions/001_initial.py @@ -1,17 +1,28 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +"""initial + +Revision ID: 001 +Revises: +Create Date: 2016-01-15 22:28:25.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "001" +down_revision = None +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, +def upgrade(): + op.create_table( + 'base_transfer_action', sqlalchemy.Column("base_id", sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -31,8 +42,18 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - migration = sqlalchemy.Table( - 'migration', meta, + op.create_table( + 'replica', + sqlalchemy.Column("id", sqlalchemy.String(36), + sqlalchemy.ForeignKey( + 'base_transfer_action.base_id'), + primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8' + ) + + op.create_table( + 'migration', sqlalchemy.Column("id", sqlalchemy.String(36), sqlalchemy.ForeignKey( 'base_transfer_action.base_id'), @@ -44,8 +65,26 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - task = sqlalchemy.Table( - 'task', meta, sqlalchemy.Column( + op.create_table( + 'tasks_execution', + sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, + default=lambda: str(uuid.uuid4())), + sqlalchemy.Column('created_at', sqlalchemy.DateTime), + sqlalchemy.Column('updated_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), + sqlalchemy.Column('deleted', sqlalchemy.String(36)), + sqlalchemy.Column("action_id", sqlalchemy.String(36), + sqlalchemy.ForeignKey( + 'base_transfer_action.base_id'), + nullable=False), + sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False), + sqlalchemy.Column("number", sqlalchemy.Integer, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8' + ) + + op.create_table( + 'task', sqlalchemy.Column( 'id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -76,26 +115,8 @@ def upgrade(migrate_engine): sqlalchemy.Column("on_error", sqlalchemy.Boolean, nullable=True), mysql_engine='InnoDB', mysql_charset='utf8') - tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted', sqlalchemy.String(36)), - sqlalchemy.Column("action_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - nullable=False), - sqlalchemy.Column("status", sqlalchemy.String(100), nullable=False), - sqlalchemy.Column("number", sqlalchemy.Integer, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - task_progress_update = sqlalchemy.Table( - 'task_progress_update', meta, + op.create_table( + 'task_progress_update', sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -112,8 +133,8 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - task_events = sqlalchemy.Table( - 'task_event', meta, + op.create_table( + 'task_event', sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -129,31 +150,8 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - replica = sqlalchemy.Table( - 'replica', meta, - sqlalchemy.Column("id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - primary_key=True), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - tables = ( - base_transfer_action, - replica, - migration, - tasks_execution, - task, - task_progress_update, - task_events, - ) - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise +def downgrade(): + # Downgrades were never supported by the original sqlalchemy-migrate + # migrations this revision chain was ported from. + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py b/coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py similarity index 64% rename from coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py rename to coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py index 4320a1197..18d57359d 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/002_adds_endpoints.py +++ b/coriolis/db/sqlalchemy/alembic/versions/002_adds_endpoints.py @@ -1,17 +1,28 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +"""adds endpoints + +Revision ID: 002 +Revises: 001 +Create Date: 2017-03-22 22:17:09.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "002" +down_revision = "001" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - endpoint = sqlalchemy.Table( - 'endpoint', meta, +def upgrade(): + op.create_table( + 'endpoint', sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -29,37 +40,27 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - tables = ( - endpoint, - ) - - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise - - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - # NOTE(alexpilotti) delete all records in base_transfer_action # before performing this migration origin_endpoint_id = sqlalchemy.Column( "origin_endpoint_id", sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), nullable=False) - base_transfer_action.create_column(origin_endpoint_id) + op.add_column("base_transfer_action", origin_endpoint_id) destination_endpoint_id = sqlalchemy.Column( "destination_endpoint_id", sqlalchemy.String(36), sqlalchemy.ForeignKey('endpoint.id'), nullable=False) - base_transfer_action.create_column(destination_endpoint_id) + op.add_column("base_transfer_action", destination_endpoint_id) destination_environment = sqlalchemy.Column( "destination_environment", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(destination_environment) + op.add_column("base_transfer_action", destination_environment) + + op.drop_column("base_transfer_action", "origin") + op.drop_column("base_transfer_action", "destination") + - base_transfer_action.drop_column("origin") - base_transfer_action.drop_column("destination") +def downgrade(): + # Downgrades were never supported by the original sqlalchemy-migrate + # migrations this revision chain was ported from. + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py b/coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py new file mode 100644 index 000000000..cad0c5be7 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/003_adds_notes.py @@ -0,0 +1,29 @@ +# Copyright 2017 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds notes + +Revision ID: 003 +Revises: 002 +Create Date: 2017-05-04 13:47:52.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "003" +down_revision = "002" +branch_labels = None +depends_on = None + + +def upgrade(): + notes = sqlalchemy.Column( + "notes", sqlalchemy.Text, nullable=True) + + op.add_column("base_transfer_action", notes) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py b/coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py similarity index 65% rename from coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py rename to coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py index 8ae2fa444..7a39801e5 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/004_adds_replica_schedules.py +++ b/coriolis/db/sqlalchemy/alembic/versions/004_adds_replica_schedules.py @@ -1,17 +1,25 @@ +"""adds replica schedules + +Revision ID: 004 +Revises: 003 +Create Date: 2017-11-04 18:17:31.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "004" +down_revision = "003" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - sqlalchemy.Table( - 'replica', meta, autoload=True) - - replica_schedules = sqlalchemy.Table( - 'replica_schedules', meta, +def upgrade(): + op.create_table( + 'replica_schedules', sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, default=lambda: str(uuid.uuid4())), sqlalchemy.Column('created_at', sqlalchemy.DateTime), @@ -32,15 +40,6 @@ def upgrade(migrate_engine): mysql_charset='utf8' ) - tables = ( - replica_schedules, - ) - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py b/coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py new file mode 100644 index 000000000..843172191 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/005_adds_transfer_result.py @@ -0,0 +1,29 @@ +# Copyright 2018 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds transfer result + +Revision ID: 005 +Revises: 004 +Create Date: 2018-09-18 18:33:14.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "005" +down_revision = "004" +branch_labels = None +depends_on = None + + +def upgrade(): + # add 'transfer_result' column to 'base_transfer_action': + transfer_result = sqlalchemy.Column( + "transfer_result", sqlalchemy.Text, nullable=True) + op.add_column("base_transfer_action", transfer_result) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py b/coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py new file mode 100644 index 000000000..8ae25518e --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/006_adds_network_map.py @@ -0,0 +1,29 @@ +# Copyright 2018 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds network map + +Revision ID: 006 +Revises: 005 +Create Date: 2018-11-01 16:43:05.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "006" +down_revision = "005" +branch_labels = None +depends_on = None + + +def upgrade(): + # add 'network_map' column to 'base_transfer_action': + network_map = sqlalchemy.Column( + "network_map", sqlalchemy.Text, nullable=True) + op.add_column("base_transfer_action", network_map) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py b/coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py new file mode 100644 index 000000000..668e86ed3 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/007_adds_storage_mappings.py @@ -0,0 +1,29 @@ +# Copyright 2018 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds storage mappings + +Revision ID: 007 +Revises: 006 +Create Date: 2018-11-13 13:37:09.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "007" +down_revision = "006" +branch_labels = None +depends_on = None + + +def upgrade(): + # add 'storage_mappings' column to 'base_transfer_action': + storage_mappings = sqlalchemy.Column( + "storage_mappings", sqlalchemy.Text, nullable=True) + op.add_column("base_transfer_action", storage_mappings) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py b/coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py new file mode 100644 index 000000000..0367a4261 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/008_adds_source_environment.py @@ -0,0 +1,29 @@ +# Copyright 2018 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds source environment + +Revision ID: 008 +Revises: 007 +Create Date: 2018-11-27 18:48:35.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "008" +down_revision = "007" +branch_labels = None +depends_on = None + + +def upgrade(): + # add 'source_environment' column to 'base_transfer_action': + source_environment = sqlalchemy.Column( + "source_environment", sqlalchemy.Text, nullable=True) + op.add_column("base_transfer_action", source_environment) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py b/coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py new file mode 100644 index 000000000..8efb4d45e --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/009_migrate_info_to_blob.py @@ -0,0 +1,24 @@ +"""migrate info to blob + +Revision ID: 009 +Revises: 008 +Create Date: 2019-03-23 00:15:49.000000 +""" + +from alembic import op +from sqlalchemy import types + +# revision identifiers, used by Alembic. +revision = "009" +down_revision = "008" +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + 'base_transfer_action', 'info', type_=types.LargeBinary(4294967295)) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py b/coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py new file mode 100644 index 000000000..289029e62 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/010_adds_reservation_id.py @@ -0,0 +1,29 @@ +# Copyright 2018 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds reservation id + +Revision ID: 010 +Revises: 009 +Create Date: 2019-04-26 22:47:10.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "010" +down_revision = "009" +branch_labels = None +depends_on = None + + +def upgrade(): + # add 'reservation_id' column to 'base_transfer_action': + reservation_id = sqlalchemy.Column( + "reservation_id", sqlalchemy.String(36), nullable=True) + op.add_column("base_transfer_action", reservation_id) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py b/coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py new file mode 100644 index 000000000..e8be1e1b5 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/011_adds_execution_type.py @@ -0,0 +1,29 @@ +# Copyright 2019 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds execution type + +Revision ID: 011 +Revises: 010 +Create Date: 2019-08-15 07:30:35.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "011" +down_revision = "010" +branch_labels = None +depends_on = None + + +def upgrade(): + # add 'type' column to 'tasks_execution': + execution_type = sqlalchemy.Column( + "type", sqlalchemy.String(20)) + op.add_column("tasks_execution", execution_type) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py b/coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py new file mode 100644 index 000000000..de08ca2ed --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/012_adds_migration_sync_fields.py @@ -0,0 +1,31 @@ +"""adds migration sync fields + +Revision ID: 012 +Revises: 011 +Create Date: 2019-10-16 15:40:42.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "012" +down_revision = "011" +branch_labels = None +depends_on = None + + +def upgrade(): + shutdown_instances = sqlalchemy.Column( + "shutdown_instances", sqlalchemy.Boolean, + nullable=False, server_default=sqlalchemy.false()) + op.add_column("migration", shutdown_instances) + + replication_count = sqlalchemy.Column( + "replication_count", sqlalchemy.Integer, + nullable=False, server_default="0") + op.add_column("migration", replication_count) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py b/coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py new file mode 100644 index 000000000..e4075152e --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/013_adds_task_index.py @@ -0,0 +1,25 @@ +"""adds task index + +Revision ID: 013 +Revises: 012 +Create Date: 2019-10-18 19:35:20.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "013" +down_revision = "012" +branch_labels = None +depends_on = None + + +def upgrade(): + index = sqlalchemy.Column( + "index", sqlalchemy.Integer, nullable=False, server_default="0") + op.add_column("task", index) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py b/coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py similarity index 89% rename from coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py rename to coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py index 4682a974a..781eb8801 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/014_adds_worker_service_regions.py +++ b/coriolis/db/sqlalchemy/alembic/versions/014_adds_worker_service_regions.py @@ -1,17 +1,31 @@ # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. +"""adds worker service regions + +Revision ID: 014 +Revises: 013 +Create Date: 2020-07-28 18:21:57.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "014" +down_revision = "013" +branch_labels = None +depends_on = None + -def upgrade(migrate_engine): +def upgrade(): meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - sqlalchemy.Table( - 'endpoint', meta, autoload=True) + # load 'endpoint' into meta so the 'endpoint_id' foreign key below can + # be resolved against it. + sqlalchemy.Table('endpoint', meta, autoload_with=op.get_bind()) tables = [] @@ -122,11 +136,9 @@ def upgrade(migrate_engine): mysql_engine='InnoDB', mysql_charset='utf8')) - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise + for table in tables: + table.create(bind=op.get_bind()) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py b/coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py new file mode 100644 index 000000000..3bcb6054b --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/015_adds_action_last_execution_status.py @@ -0,0 +1,30 @@ +# Copyright 2020 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds action last execution status + +Revision ID: 015 +Revises: 014 +Create Date: 2020-08-27 20:38:25.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "015" +down_revision = "014" +branch_labels = None +depends_on = None + + +def upgrade(): + # add 'last_execution_status' column to 'base_transfer_action': + last_execution_status = sqlalchemy.Column( + "last_execution_status", sqlalchemy.String(255), + nullable=False, server_default="UNEXECUTED") + op.add_column("base_transfer_action", last_execution_status) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py b/coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py similarity index 86% rename from coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py rename to coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py index 6c48212bb..6fc1790e8 100644 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/016_adds_minion_vm_pools.py +++ b/coriolis/db/sqlalchemy/alembic/versions/016_adds_minion_vm_pools.py @@ -1,24 +1,39 @@ # Copyright 2020 Cloudbase Solutions Srl # All Rights Reserved. +"""adds minion vm pools + +Revision ID: 016 +Revises: 015 +Create Date: 2020-08-26 07:10:03.000000 +""" + import uuid +from alembic import op import sqlalchemy +# revision identifiers, used by Alembic. +revision = "016" +down_revision = "015" +branch_labels = None +depends_on = None -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") +def upgrade(): + meta = sqlalchemy.MetaData() # extend tasks execution 'type' column: - tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, autoload=True) - tasks_execution.c.type.alter(type=sqlalchemy.String(255)) + op.alter_column( + 'tasks_execution', 'type', + existing_type=sqlalchemy.String(20), + existing_nullable=True, + type_=sqlalchemy.String(255), + nullable=True) + + # load 'endpoint' into meta so the 'endpoint_id' foreign key below can + # be resolved against it. + sqlalchemy.Table('endpoint', meta, autoload_with=op.get_bind()) tables = [] @@ -153,30 +168,26 @@ def upgrade(migrate_engine): "message", sqlalchemy.Text, nullable=True), mysql_engine='InnoDB', mysql_charset='utf8')) + for table in tables: + table.create(bind=op.get_bind()) + # add the pool option properties for the transfer: origin_minion_pool_id = sqlalchemy.Column( "origin_minion_pool_id", sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=True) + op.add_column("base_transfer_action", origin_minion_pool_id) + destination_minion_pool_id = sqlalchemy.Column( "destination_minion_pool_id", sqlalchemy.String(36), sqlalchemy.ForeignKey('minion_pool.id'), nullable=True) + op.add_column("base_transfer_action", destination_minion_pool_id) + instance_osmorphing_minion_pool_mappings = sqlalchemy.Column( "instance_osmorphing_minion_pool_mappings", sqlalchemy.Text, - nullable=False, default='{}') - - created_columns = [] - try: - for index, table in enumerate(tables): - table.create() - for col in [ - origin_minion_pool_id, destination_minion_pool_id, - instance_osmorphing_minion_pool_mappings]: - base_transfer_action.create_column(col) - created_columns.append(col) - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - for col in created_columns: - base_transfer_action.drop_column(col) - meta.drop_all(tables=tables[:index]) - raise + nullable=False, server_default='{}') + op.add_column( + "base_transfer_action", instance_osmorphing_minion_pool_mappings) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py b/coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py new file mode 100644 index 000000000..7d9083b77 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/017_adds_user_scripts_column.py @@ -0,0 +1,29 @@ +# Copyright 2020 Cloudbase Solutions Srl +# All Rights Reserved. + +"""adds user scripts column + +Revision ID: 017 +Revises: 016 +Create Date: 2021-01-26 19:16:44.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "017" +down_revision = "016" +branch_labels = None +depends_on = None + + +def upgrade(): + # add 'user_scripts' column to 'base_transfer_action': + user_scripts = sqlalchemy.Column( + "user_scripts", sqlalchemy.Text, nullable=True) + op.add_column("base_transfer_action", user_scripts) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py b/coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py new file mode 100644 index 000000000..33b0abaee --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/018_adds_task_progress_idices.py @@ -0,0 +1,41 @@ +"""adds task progress idices + +Revision ID: 018 +Revises: 017 +Create Date: 2021-01-07 17:33:05.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "018" +down_revision = "017" +branch_labels = None +depends_on = None + + +def upgrade(): + event_index = sqlalchemy.Column( + "index", sqlalchemy.Integer, nullable=False, server_default="0") + op.add_column("task_event", event_index) + + progress_index = sqlalchemy.Column( + "index", sqlalchemy.Integer, nullable=False, server_default="0") + op.add_column("task_progress_update", progress_index) + op.alter_column( + "task_progress_update", "current_step", + existing_type=sqlalchemy.Integer, + existing_nullable=False, + type_=sqlalchemy.BigInteger, + nullable=False) + op.alter_column( + "task_progress_update", "total_steps", + existing_type=sqlalchemy.Integer, + existing_nullable=True, + type_=sqlalchemy.BigInteger, + nullable=True) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py b/coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py new file mode 100644 index 000000000..7be084f37 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/019_add_replica_scenario_field.py @@ -0,0 +1,29 @@ +# Copyright 2024 Cloudbase Solutions Srl +# All Rights Reserved. + +"""add replica scenario field + +Revision ID: 019 +Revises: 018 +Create Date: 2024-04-09 14:42:14.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "019" +down_revision = "018" +branch_labels = None +depends_on = None + + +def upgrade(): + replica_scenario = sqlalchemy.Column( + "scenario", sqlalchemy.String(255), nullable=False, + server_default="replica") + op.add_column("replica", replica_scenario) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py b/coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py new file mode 100644 index 000000000..7d289d426 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/020_rename_tables.py @@ -0,0 +1,68 @@ +"""rename tables + +Revision ID: 020 +Revises: 019 +Create Date: 2024-10-29 15:26:01.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "020" +down_revision = "019" +branch_labels = None +depends_on = None + +# NOTE(dvincze): Update models polymorphic identity. Due to the model code +# changes, this cannot be done using the ORM; raw SQL is required. +_TYPE_RENAMES = ( + ("base_transfer_action", "replica", "transfer"), + ("tasks_execution", "replica_execution", "transfer_execution"), + ("tasks_execution", "replica_disks_delete", "transfer_disks_delete"), + ("tasks_execution", "replica_deploy", "deployment"), + ("tasks_execution", "replica_update", "transfer_update"), +) + + +def upgrade(): + meta = sqlalchemy.MetaData() + + op.rename_table('replica', 'transfer') + + # load 'base_transfer_action' and 'transfer' into meta so the foreign + # keys below can be resolved against them. + sqlalchemy.Table( + 'base_transfer_action', meta, autoload_with=op.get_bind()) + sqlalchemy.Table('transfer', meta, autoload_with=op.get_bind()) + + deployment = sqlalchemy.Table( + 'deployment', meta, + sqlalchemy.Column("id", sqlalchemy.String(36), + sqlalchemy.ForeignKey( + 'base_transfer_action.base_id'), + primary_key=True), + sqlalchemy.Column("transfer_id", sqlalchemy.String(36), + sqlalchemy.ForeignKey('transfer.id'), + nullable=False), + mysql_engine="InnoDB", + mysql_charset="utf8") + deployment.create(bind=op.get_bind()) + + op.rename_table('replica_schedules', 'transfer_schedules') + op.alter_column( + 'transfer_schedules', 'replica_id', + new_column_name='transfer_id', + existing_type=sqlalchemy.String(36), + existing_nullable=False) + + for table, old_type, new_type in _TYPE_RENAMES: + op.execute( + sqlalchemy.text( + f"UPDATE {table} SET type = :new_type " + f"WHERE type = :old_type" + ).bindparams(new_type=new_type, old_type=old_type)) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py b/coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py new file mode 100644 index 000000000..e2bac0d8a --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/021_add_deployment_defaults.py @@ -0,0 +1,30 @@ +"""add deployment defaults + +Revision ID: 021 +Revises: 020 +Create Date: 2025-01-29 19:13:39.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "021" +down_revision = "020" +branch_labels = None +depends_on = None + + +def upgrade(): + clone_disks = sqlalchemy.Column( + "clone_disks", sqlalchemy.Boolean, nullable=False, + server_default=sqlalchemy.true()) + op.add_column("base_transfer_action", clone_disks) + skip_os_morphing = sqlalchemy.Column( + "skip_os_morphing", sqlalchemy.Boolean, nullable=False, + server_default=sqlalchemy.false()) + op.add_column("base_transfer_action", skip_os_morphing) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py b/coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py new file mode 100644 index 000000000..081acf940 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/022_adds_auto_deploy_column.py @@ -0,0 +1,26 @@ +"""adds auto deploy column + +Revision ID: 022 +Revises: 021 +Create Date: 2025-01-30 13:54:39.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "022" +down_revision = "021" +branch_labels = None +depends_on = None + + +def upgrade(): + auto_deploy = sqlalchemy.Column( + 'auto_deploy', sqlalchemy.Boolean, nullable=False, + server_default=sqlalchemy.false()) + op.add_column("transfer_schedules", auto_deploy) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py b/coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py new file mode 100644 index 000000000..dd9b2381c --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/023_add_deployer_id.py @@ -0,0 +1,28 @@ +"""add deployer id + +Revision ID: 023 +Revises: 022 +Create Date: 2025-02-06 14:42:29.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "023" +down_revision = "022" +branch_labels = None +depends_on = None + + +def upgrade(): + deployer_id = sqlalchemy.Column( + 'deployer_id', sqlalchemy.String(36), nullable=True) + op.add_column("deployment", deployer_id) + trust_id = sqlalchemy.Column( + 'trust_id', sqlalchemy.String(255), nullable=True) + op.add_column("deployment", trust_id) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py b/coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py new file mode 100644 index 000000000..761814ae4 --- /dev/null +++ b/coriolis/db/sqlalchemy/alembic/versions/024_add_clustered_to_base_transfer_action.py @@ -0,0 +1,36 @@ +# Copyright 2026 Cloudbase Solutions Srl +# All Rights Reserved. + +"""add clustered to base transfer action + +Revision ID: 024 +Revises: 023 +Create Date: 2026-04-09 04:03:23.000000 +""" + +from alembic import op +import sqlalchemy + +# revision identifiers, used by Alembic. +revision = "024" +down_revision = "023" +branch_labels = None +depends_on = None + + +def upgrade(): + inspector = sqlalchemy.inspect(op.get_bind()) + columns = [c['name'] for c in inspector.get_columns( + 'base_transfer_action')] + if 'clustered' in columns: + return + # server_default so existing rows get a value when the column is added + # (MySQL stores booleans as TINYINT). + clustered = sqlalchemy.Column( + 'clustered', sqlalchemy.Boolean, nullable=False, + server_default=sqlalchemy.text('0')) + op.add_column("base_transfer_action", clustered) + + +def downgrade(): + raise NotImplementedError() diff --git a/coriolis/db/sqlalchemy/migrate_repo/__init__.py b/coriolis/db/sqlalchemy/alembic/versions/__init__.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/__init__.py rename to coriolis/db/sqlalchemy/alembic/versions/__init__.py diff --git a/coriolis/db/sqlalchemy/api.py b/coriolis/db/sqlalchemy/api.py index 822972268..21e056d2a 100644 --- a/coriolis/db/sqlalchemy/api.py +++ b/coriolis/db/sqlalchemy/api.py @@ -39,9 +39,11 @@ def get_backend(): def db_sync(engine, version=None): """Migrate the database to `version` or the most recent version.""" - if version is not None and int(version) < db_version(engine): - raise exception.CoriolisException( - _("Cannot migrate to lower schema version.")) + if version is not None: + current_version = db_version(engine) + if current_version is not None and int(version) < int(current_version): + raise exception.CoriolisException( + _("Cannot migrate to lower schema version.")) return migration.db_sync(engine, version=version) diff --git a/coriolis/db/sqlalchemy/migrate_repo/manage.py b/coriolis/db/sqlalchemy/migrate_repo/manage.py deleted file mode 100644 index 48fb3c431..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/manage.py +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Cloudbase Solutions Srl -# All Rights Reserved. - -from migrate.versioning.shell import main - -if __name__ == '__main__': - main(debug='False') diff --git a/coriolis/db/sqlalchemy/migrate_repo/migrate.cfg b/coriolis/db/sqlalchemy/migrate_repo/migrate.cfg deleted file mode 100644 index 75db1dffd..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=coriolis - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering=False diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py b/coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py deleted file mode 100644 index 40556f25a..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/003_adds_notes.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2017 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - - notes = sqlalchemy.Column( - "notes", sqlalchemy.Text, nullable=True) - - base_transfer_action.create_column(notes) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py b/coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py deleted file mode 100644 index 7805fd4a2..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/005_adds_transfer_result.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2018 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # add 'transfer_result' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - - transfer_result = sqlalchemy.Column( - "transfer_result", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(transfer_result) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py b/coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py deleted file mode 100644 index 460b5f43b..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/006_adds_network_map.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2018 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # add 'network_map' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - - network_map = sqlalchemy.Column( - "network_map", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(network_map) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py b/coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py deleted file mode 100644 index c99963293..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/007_adds_storage_mappings.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2018 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # add 'storage_mappings' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - - storage_mappings = sqlalchemy.Column( - "storage_mappings", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(storage_mappings) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py b/coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py deleted file mode 100644 index c04096d62..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/008_adds_source_environment.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2018 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # add 'source_environment' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - - source_environment = sqlalchemy.Column( - "source_environment", sqlalchemy.Text, nullable=True) - base_transfer_action.create_column(source_environment) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py b/coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py deleted file mode 100644 index 1814a815d..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/009_Migrate_info_to_blob.py +++ /dev/null @@ -1,12 +0,0 @@ -import sqlalchemy -from sqlalchemy import types - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - - base_transfer_action.c.info.alter(type=types.LargeBinary(4294967295)) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py b/coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py deleted file mode 100644 index 0bd370d13..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/010_adds_reservation_id.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2018 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # add 'reservation_id' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - - reservation_id = sqlalchemy.Column( - "reservation_id", sqlalchemy.String(36), nullable=True) - base_transfer_action.create_column(reservation_id) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py b/coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py deleted file mode 100644 index 44e835435..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/011_adds_execution_type.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2019 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # add 'type' column to 'tasks_execution': - tasks_execution = sqlalchemy.Table( - 'tasks_execution', meta, autoload=True) - - execution_type = sqlalchemy.Column( - "type", sqlalchemy.String(20)) - tasks_execution.create_column(execution_type) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py b/coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py deleted file mode 100644 index d278324c8..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/012_adds_migration_sync_fields.py +++ /dev/null @@ -1,19 +0,0 @@ -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - migration = sqlalchemy.Table( - 'migration', meta, autoload=True) - - shutdown_instances = sqlalchemy.Column( - "shutdown_instances", sqlalchemy.Boolean, - nullable=False, default=False) - migration.create_column(shutdown_instances) - - replication_count = sqlalchemy.Column( - "replication_count", sqlalchemy.Integer, default=0, - nullable=False) - migration.create_column(replication_count) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py b/coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py deleted file mode 100644 index fb841d7d8..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/013_adds_task_index.py +++ /dev/null @@ -1,13 +0,0 @@ -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - task = sqlalchemy.Table('task', meta, autoload=True) - - index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) - - task.create_column(index) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py b/coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py deleted file mode 100644 index 27c3a8b18..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/015_adds_action_last_execution_status.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2020 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # add 'last_execution_status' column to 'base_transfer_action': - base_transfer_action = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - - last_execution_status = sqlalchemy.Column( - "last_execution_status", sqlalchemy.String(255), - default=lambda: "UNEXECUTED", nullable=False) - base_transfer_action.create_column(last_execution_status) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py b/coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py deleted file mode 100644 index f860fd72c..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/017_adds_user_scripts_column.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2020 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - # add 'user_scripts' column to 'base_transfer_action': - base_transfer = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - - user_scripts = sqlalchemy.Column( - "user_scripts", sqlalchemy.Text, nullable=True) - base_transfer.create_column(user_scripts) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py b/coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py deleted file mode 100644 index 6e6101461..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/018_adds_task_progress_idices.py +++ /dev/null @@ -1,21 +0,0 @@ -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - task_event = sqlalchemy.Table('task_event', meta, autoload=True) - event_index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) - task_event.create_column(event_index) - - task_progress_update = sqlalchemy.Table( - 'task_progress_update', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - progress_index = sqlalchemy.Column( - "index", sqlalchemy.Integer, default=0, nullable=False) - task_progress_update.create_column(progress_index) - task_progress_update.c.current_step.alter(type=sqlalchemy.BigInteger) - task_progress_update.c.total_steps.alter(type=sqlalchemy.BigInteger) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py b/coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py deleted file mode 100644 index edd16a5d9..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/019_add_replica_scenario_field.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - replica = sqlalchemy.Table( - 'replica', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - - replica_scenario = sqlalchemy.Column( - "scenario", sqlalchemy.String(255), nullable=False, - default="replica") - - replica.create_column(replica_scenario) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py b/coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py deleted file mode 100644 index a6bed3b3c..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/020_rename_tables.py +++ /dev/null @@ -1,50 +0,0 @@ -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - replica = sqlalchemy.Table('replica', meta, autoload=True) - replica.rename('transfer') - - deployment = sqlalchemy.Table( - 'deployment', meta, - sqlalchemy.Column("id", sqlalchemy.String(36), - sqlalchemy.ForeignKey( - 'base_transfer_action.base_id'), - primary_key=True), - sqlalchemy.Column("transfer_id", sqlalchemy.String(36), - sqlalchemy.ForeignKey('transfer.id'), - nullable=False), - mysql_engine="InnoDB", - mysql_charset="utf8") - try: - deployment.create() - except Exception: - deployment.drop() - raise - - replica_schedule = sqlalchemy.Table( - 'replica_schedules', meta, autoload=True) - replica_schedule.rename('transfer_schedules') - replica_schedule.c.replica_id.alter(name='transfer_id') - - # NOTE(dvincze): Update models polymorphic identity - # Due to the model code changes, this cannot be done using the ORM. - # Had to resort to using raw SQL statements. - with migrate_engine.connect() as conn: - conn.execute( - 'UPDATE base_transfer_action SET type = "transfer" ' - 'WHERE type = "replica";') - conn.execute('UPDATE tasks_execution SET type = "transfer_execution"' - 'WHERE type = "replica_execution"') - conn.execute( - 'UPDATE tasks_execution SET type = "transfer_disks_delete"' - 'WHERE type = "replica_disks_delete"') - conn.execute( - 'UPDATE tasks_execution SET type = "deployment"' - 'WHERE type = "replica_deploy"') - conn.execute( - 'UPDATE tasks_execution SET type = "transfer_update"' - 'WHERE type = "replica_update"') diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py b/coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py deleted file mode 100644 index 95a5b1684..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/021_add_deployment_defaults.py +++ /dev/null @@ -1,17 +0,0 @@ -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - base_transfer = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - clone_disks = sqlalchemy.Column( - "clone_disks", sqlalchemy.Boolean, nullable=False, default=True) - base_transfer.create_column(clone_disks) - skip_os_morphing = sqlalchemy.Column( - "skip_os_morphing", sqlalchemy.Boolean, nullable=False, default=False) - base_transfer.create_column(skip_os_morphing) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py b/coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py deleted file mode 100644 index 63a7cfb0b..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/022_adds_auto_deploy_column.py +++ /dev/null @@ -1,14 +0,0 @@ -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - transfer_schedule = sqlalchemy.Table( - 'transfer_schedules', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - auto_deploy = sqlalchemy.Column( - 'auto_deploy', sqlalchemy.Boolean, nullable=False, default=False) - transfer_schedule.create_column(auto_deploy) diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py b/coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py deleted file mode 100644 index 37729f55b..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/023_add_deployer_id.py +++ /dev/null @@ -1,25 +0,0 @@ -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - deployment = sqlalchemy.Table( - 'deployment', meta, autoload=True, - mysql_engine="InnoDB", - mysql_charset="utf8") - deployer_id = sqlalchemy.Column( - 'deployer_id', sqlalchemy.String(36), nullable=True) - trust_id = sqlalchemy.Column( - 'trust_id', sqlalchemy.String(255), nullable=True) - created_columns = [] - try: - deployment.create_column(deployer_id) - created_columns.append(deployer_id) - deployment.create_column(trust_id) - created_columns.append(trust_id) - except Exception: - for c in created_columns: - deployment.drop_column(c) - raise diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/024_add_clustered_to_base_transfer_action.py b/coriolis/db/sqlalchemy/migrate_repo/versions/024_add_clustered_to_base_transfer_action.py deleted file mode 100644 index 9e834e85d..000000000 --- a/coriolis/db/sqlalchemy/migrate_repo/versions/024_add_clustered_to_base_transfer_action.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2026 Cloudbase Solutions Srl -# All Rights Reserved. - -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - base_transfer = sqlalchemy.Table( - 'base_transfer_action', meta, autoload=True) - if 'clustered' in base_transfer.c: - return - # server_default so existing rows get a value when the column is added - # (MySQL stores booleans as TINYINT). - clustered = sqlalchemy.Column( - 'clustered', sqlalchemy.Boolean, nullable=False, - server_default=sqlalchemy.text('0')) - base_transfer.create_column(clustered) diff --git a/coriolis/db/sqlalchemy/migration.py b/coriolis/db/sqlalchemy/migration.py index f05ceea14..847e26bcd 100644 --- a/coriolis/db/sqlalchemy/migration.py +++ b/coriolis/db/sqlalchemy/migration.py @@ -3,25 +3,74 @@ import os -from oslo_db.sqlalchemy import migration as oslo_migration +from alembic import command +from alembic import config as alembic_config +from alembic.runtime import migration as alembic_migration +import sqlalchemy -INIT_VERSION = 0 +from coriolis import exception +from coriolis.i18n import _ + +ALEMBIC_DIR = os.path.join(os.path.dirname(__file__), "alembic") +ALEMBIC_INI_PATH = os.path.join(ALEMBIC_DIR, "alembic.ini") + +# The final version stamped by the old sqlalchemy-migrate migrate_repo +# (migrate_repo/versions/024_add_clustered_to_base_transfer_action.py), and +# the alembic revision it corresponds to. +LEGACY_VERSION_TABLE = "migrate_version" +LEGACY_FINAL_VERSION = 24 +LEGACY_FINAL_ALEMBIC_REVISION = "024" + + +def _get_alembic_config(): + config = alembic_config.Config(ALEMBIC_INI_PATH) + config.set_main_option("script_location", ALEMBIC_DIR) + return config + + +def _stamp_legacy_database_if_needed(engine, config): + """Transition a sqlalchemy-migrate managed database to alembic. + + If this database was previously managed by the old sqlalchemy-migrate + based migrate_repo, stamp it onto the equivalent alembic revision + instead of re-running the already-applied DDL. + """ + inspector = sqlalchemy.inspect(engine) + if LEGACY_VERSION_TABLE not in inspector.get_table_names(): + return + + with engine.connect() as conn: + legacy_version = conn.execute( + sqlalchemy.text( + f"SELECT version FROM {LEGACY_VERSION_TABLE}")).scalar() + + if legacy_version != LEGACY_FINAL_VERSION: + raise exception.CoriolisException( + _("This database was last migrated using the legacy " + "sqlalchemy-migrate based coriolis-dbsync (version %(cur)s), " + "but only the final version (%(final)s) can be transitioned " + "to alembic. Please run coriolis-dbsync from a previous " + "Coriolis release first to reach version %(final)s.") % { + "cur": legacy_version, "final": LEGACY_FINAL_VERSION}) + + config.attributes["connection"] = engine.connect() + command.stamp(config, LEGACY_FINAL_ALEMBIC_REVISION) def db_sync(engine, version=None): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_sync(engine, path, version, - init_version=INIT_VERSION) + config = _get_alembic_config() + _stamp_legacy_database_if_needed(engine, config) + config.attributes["connection"] = engine.connect() + return command.upgrade(config, version or "head") def db_version(engine): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_version(engine, path, INIT_VERSION) + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_revision() def db_version_control(engine, version=None): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_version_control(engine, path, version) + config = _get_alembic_config() + config.attributes["connection"] = engine.connect() + return command.stamp(config, version or "head") diff --git a/coriolis/db/sqlalchemy/migrate_repo/versions/__init__.py b/coriolis/tests/db/sqlalchemy/alembic/__init__.py similarity index 100% rename from coriolis/db/sqlalchemy/migrate_repo/versions/__init__.py rename to coriolis/tests/db/sqlalchemy/alembic/__init__.py diff --git a/coriolis/tests/db/sqlalchemy/alembic/test_env.py b/coriolis/tests/db/sqlalchemy/alembic/test_env.py new file mode 100644 index 000000000..2b226bcec --- /dev/null +++ b/coriolis/tests/db/sqlalchemy/alembic/test_env.py @@ -0,0 +1,82 @@ +# Copyright 2026 Cloudbase Solutions Srl +# All Rights Reserved. + +import importlib +import sys +from unittest import mock + +from coriolis.tests import test_base + +ENV_MODULE_NAME = "coriolis.db.sqlalchemy.alembic.env" + + +class AlembicEnvTestCase(test_base.CoriolisBaseTestCase): + """Test suite for the Coriolis Alembic 'env.py' migration script.""" + + def setUp(self): + super(AlembicEnvTestCase, self).setUp() + self.addCleanup(sys.modules.pop, ENV_MODULE_NAME, None) + + def _import_env(self, offline_mode, connection=None): + # run_migrations_online or run_migrations_offline runs on module + # import, so we need to set up what we need beforehand. + sys.modules.pop(ENV_MODULE_NAME, None) + + mock_context = mock.MagicMock() + mock_context.is_offline_mode.return_value = offline_mode + mock_context.config.attributes.get.return_value = connection + with mock.patch("alembic.context", mock_context): + env = importlib.import_module(ENV_MODULE_NAME) + + return env, mock_context + + def test_offline_mode(self): + env, mock_context = self._import_env(offline_mode=True) + + self.assertIs(env.config, mock_context.config) + mock_context.config.get_main_option.assert_called_once_with( + "sqlalchemy.url") + mock_context.configure.assert_called_once_with( + url=mock_context.config.get_main_option.return_value, + target_metadata=env.target_metadata, + render_as_batch=True, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + mock_context.begin_transaction.assert_called_once_with() + mock_context.run_migrations.assert_called_once_with() + + def test_online_mode_with_existing_connection(self): + env, mock_context = self._import_env( + offline_mode=False, connection=mock.sentinel.connection) + + mock_context.config.attributes.get.assert_called_once_with( + "connection", None) + mock_context.configure.assert_called_once_with( + connection=mock.sentinel.connection, + target_metadata=env.target_metadata, + render_as_batch=True, + ) + mock_context.begin_transaction.assert_called_once_with() + mock_context.run_migrations.assert_called_once_with() + + @mock.patch("sqlalchemy.engine_from_config") + def test_online_mode_creates_engine(self, mock_engine_from_config): + env, mock_context = self._import_env(offline_mode=False) + + mock_engine_from_config.assert_called_once_with( + mock_context.config.get_section.return_value, + prefix="sqlalchemy.", + poolclass=env.pool.NullPool, + ) + mock_connectable = mock_engine_from_config.return_value + mock_connectable.connect.assert_called_once_with() + mock_connection = ( + mock_connectable.connect.return_value.__enter__.return_value) + mock_context.configure.assert_called_once_with( + connection=mock_connection, + target_metadata=env.target_metadata, + render_as_batch=True, + ) + mock_context.begin_transaction.assert_called_once_with() + mock_context.run_migrations.assert_called_once_with() diff --git a/coriolis/tests/db/sqlalchemy/test_api.py b/coriolis/tests/db/sqlalchemy/test_api.py index de79b5d86..de4eda7ac 100644 --- a/coriolis/tests/db/sqlalchemy/test_api.py +++ b/coriolis/tests/db/sqlalchemy/test_api.py @@ -94,9 +94,9 @@ def test_db_sync_version( mock_db_version, mock_db_sync ): - mock_db_version.return_value = 1 + mock_db_version.return_value = "001" - result = api.db_sync(mock.sentinel.engine, version=1) + result = api.db_sync(mock.sentinel.engine, version="002") self.assertEqual( mock_db_sync.return_value, @@ -104,20 +104,39 @@ def test_db_sync_version( ) mock_db_version.assert_called_once_with(mock.sentinel.engine) mock_db_sync.assert_called_once_with( - mock.sentinel.engine, version=1) + mock.sentinel.engine, version="002") + + @mock.patch.object(migration, 'db_sync') + @mock.patch.object(api, 'db_version') + def test_db_sync_version_no_current_version( + self, + mock_db_version, + mock_db_sync + ): + mock_db_version.return_value = None + + result = api.db_sync(mock.sentinel.engine, version="001") + + self.assertEqual( + mock_db_sync.return_value, + result + ) + mock_db_version.assert_called_once_with(mock.sentinel.engine) + mock_db_sync.assert_called_once_with( + mock.sentinel.engine, version="001") @mock.patch.object(api, 'db_version') def test_db_sync_version_raise( self, - mock_db_version + mock_db_version, ): - mock_db_version.return_value = 2 + mock_db_version.return_value = "003" self.assertRaises( exception.CoriolisException, api.db_sync, mock.sentinel.engine, - version=1 + version="001" ) mock_db_version.assert_called_once_with(mock.sentinel.engine) diff --git a/coriolis/tests/db/sqlalchemy/test_migration.py b/coriolis/tests/db/sqlalchemy/test_migration.py index 6f35ad59c..642e7b891 100644 --- a/coriolis/tests/db/sqlalchemy/test_migration.py +++ b/coriolis/tests/db/sqlalchemy/test_migration.py @@ -1,61 +1,96 @@ # Copyright 2024 Cloudbase Solutions Srl # All Rights Reserved. -import os from unittest import mock -from oslo_db.sqlalchemy import migration as oslo_migration - from coriolis.db.sqlalchemy import migration +from coriolis import exception from coriolis.tests import test_base class DatabaseSqlalchemyMigrationTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis Database Sqlalchemy migration.""" - @mock.patch.object(os.path, 'abspath') - @mock.patch.object(oslo_migration, 'db_sync') - def test_db_sync(self, mock_db_sync, mock_abspath): - mock_abspath.return_value = "/abspath" + @mock.patch.object(migration, "sqlalchemy") + def test_stamp_legacy_database_if_needed_no_table(self, mock_sqlalchemy): + mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [ + "foo"] + mock_engine = mock.MagicMock() + mock_config = mock.MagicMock() - result = migration.db_sync(mock.sentinel.engine, mock.sentinel.version) + migration._stamp_legacy_database_if_needed(mock_engine, mock_config) - self.assertEqual( - mock_db_sync.return_value, - result - ) - mock_db_sync.assert_called_once_with( - mock.sentinel.engine, - "/abspath/migrate_repo", - mock.sentinel.version, - init_version=0 - ) + mock_engine.connect.assert_not_called() - @mock.patch.object(os.path, 'abspath') - @mock.patch.object(oslo_migration, 'db_version') - def test_db_version(self, mock_db_version, mock_abspath): - mock_abspath.return_value = "/abspath" + @mock.patch.object(migration, "sqlalchemy") + def test_stamp_legacy_database_if_needed_raises(self, mock_sqlalchemy): + mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [ + migration.LEGACY_VERSION_TABLE] + mock_engine = mock.MagicMock() + mock_conn = mock_engine.connect.return_value.__enter__.return_value + mock_conn.execute.return_value.scalar.return_value = 1 + mock_config = mock.MagicMock() - result = migration.db_version(mock.sentinel.engine) + self.assertRaises( + exception.CoriolisException, + migration._stamp_legacy_database_if_needed, + mock_engine, mock_config) - self.assertEqual(mock_db_version.return_value, result) - mock_db_version.assert_called_once_with( - mock.sentinel.engine, - "/abspath/migrate_repo", - 0 - ) + @mock.patch.object(migration, "command") + @mock.patch.object(migration, "sqlalchemy") + def test_stamp_legacy_database_if_needed_final_version( + self, mock_sqlalchemy, mock_command, + ): + mock_sqlalchemy.inspect.return_value.get_table_names.return_value = [ + migration.LEGACY_VERSION_TABLE] + mock_engine = mock.MagicMock() + mock_conn = mock_engine.connect.return_value.__enter__.return_value + mock_conn.execute.return_value.scalar.return_value = ( + migration.LEGACY_FINAL_VERSION) + mock_config = mock.MagicMock() - @mock.patch.object(os.path, 'abspath') - @mock.patch.object(oslo_migration, 'db_version_control') - def test_db_version_control(self, mock_db_version_control, mock_abspath): - mock_abspath.return_value = "/abspath" + migration._stamp_legacy_database_if_needed(mock_engine, mock_config) - result = migration.db_version_control( - mock.sentinel.engine, mock.sentinel.version) + mock_command.stamp.assert_called_once_with( + mock_config, migration.LEGACY_FINAL_ALEMBIC_REVISION) + + @mock.patch.object(migration, "_stamp_legacy_database_if_needed") + @mock.patch.object(migration, "command") + @mock.patch.object(migration, "_get_alembic_config") + def test_db_sync( + self, mock_get_config, mock_command, mock_stamp_legacy, + ): + mock_engine = mock.MagicMock() + + result = migration.db_sync(mock_engine, mock.sentinel.version) - self.assertEqual(mock_db_version_control.return_value, result) - mock_db_version_control.assert_called_once_with( - mock.sentinel.engine, - "/abspath/migrate_repo", - mock.sentinel.version + self.assertEqual(mock_command.upgrade.return_value, result) + mock_get_config.assert_called_once_with() + mock_stamp_legacy.assert_called_once_with( + mock_engine, mock_get_config.return_value) + mock_command.upgrade.assert_called_once_with( + mock_get_config.return_value, mock.sentinel.version) + + @mock.patch.object(migration.alembic_migration, "MigrationContext") + def test_db_version(self, mock_migration_context): + mock_engine = mock.MagicMock() + + result = migration.db_version(mock_engine) + + mock_conn = mock_engine.connect.return_value.__enter__.return_value + mock_configure = mock_migration_context.configure + mock_configure.assert_called_once_with(mock_conn) + self.assertEqual( + mock_configure.return_value.get_current_revision.return_value, + result, ) + + @mock.patch.object(migration, "command") + @mock.patch.object(migration, "_get_alembic_config") + def test_db_version_control(self, mock_get_config, mock_command): + result = migration.db_version_control( + mock.MagicMock(), mock.sentinel.version) + + self.assertEqual(mock_command.stamp.return_value, result) + mock_command.stamp.assert_called_once_with( + mock_get_config.return_value, mock.sentinel.version) diff --git a/requirements.txt b/requirements.txt index 4783a4e43..4bab04a03 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ setuptools>=65.0.0,<82 # pkg_resources removed in 82; required by sqlalchemy-migrate +alembic keystoneauth1 keystonemiddleware Jinja2 @@ -13,7 +14,7 @@ oslo.cache oslo.concurrency oslo.config<9.8.0 oslo.context<6.0.0 -oslo.db<=12.3.2 +oslo.db oslo.i18n oslo.log oslo.messaging==12.2.0