From 53c5a4a231b2196562d0117a10f1cef1866467c2 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 13 Nov 2026 08:01:59 +0000 Subject: [PATCH 1/9] chore(django-google-spanner): batch DDL statements during create_test_db Scope autocommit-disabled DDL batching in DatabaseSchemaEditor to DatabaseCreation.create_test_db() so test database schema setup sends queued CREATE TABLE, index, and foreign key statements in batched UpdateDatabaseDdl requests while keeping autocommit enabled during test execution. --- .../django_spanner/creation.py | 6 +- .../django_spanner/schema.py | 24 +++ .../tests/mockserver_tests/test_basics.py | 147 +++++++++++++++++- .../unit/django_spanner/test_creation.py | 66 ++++++++ .../tests/unit/django_spanner/test_schema.py | 90 +++++++++++ 5 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 packages/django-google-spanner/tests/unit/django_spanner/test_creation.py diff --git a/packages/django-google-spanner/django_spanner/creation.py b/packages/django-google-spanner/django_spanner/creation.py index 219240195c5f..9cafabb8b777 100644 --- a/packages/django-google-spanner/django_spanner/creation.py +++ b/packages/django-google-spanner/django_spanner/creation.py @@ -48,7 +48,11 @@ def create_test_db(self, *args, **kwargs): # by a developer running the tests locally. if os.environ.get("RUNNING_SPANNER_BACKEND_TESTS") == "1": self.mark_skips() - super().create_test_db(*args, **kwargs) + self.connection._batch_ddl_in_schema_editor = True + try: + return super().create_test_db(*args, **kwargs) + finally: + self.connection._batch_ddl_in_schema_editor = False def _create_test_db(self, verbosity, autoclobber, keepdb=False): """ diff --git a/packages/django-google-spanner/django_spanner/schema.py b/packages/django-google-spanner/django_spanner/schema.py index 3ed8d8359777..c7f991031b56 100644 --- a/packages/django-google-spanner/django_spanner/schema.py +++ b/packages/django-google-spanner/django_spanner/schema.py @@ -57,6 +57,30 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): # sql_create_inline_fk = "CONSTRAINT FK_%(to_table)s_%(to_column)s_%(from_table)s_%(from_column)s FOREIGN KEY (%(from_column_norm)s) REFERENCES %(to_table_norm)s (%(to_column_norm)s)" # noqa sql_create_inline_fk = None + def __enter__(self): + super().__enter__() + self._restore_autocommit = False + if ( + getattr(self.connection, "_batch_ddl_in_schema_editor", False) + and not self.collect_sql + and not self.connection.in_atomic_block + and self.connection.get_autocommit() + ): + self.connection.set_autocommit(False) + self._restore_autocommit = True + return self + + def __exit__(self, exc_type, exc_value, traceback): + try: + super().__exit__(exc_type, exc_value, traceback) + finally: + if getattr(self, "_restore_autocommit", False): + try: + if exc_type is not None: + self.connection.rollback() + finally: + self.connection.set_autocommit(True) + def create_model(self, model): """ Create a table and any accompanying indexes or unique constraints for diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index e728c1e3570e..dfe59000d472 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -16,9 +16,15 @@ from decimal import Decimal from unittest import mock +import django import google.cloud.spanner_v1.types.result_set as result_set import google.cloud.spanner_v1.types.type as spanner_type -from django.db import connection, models +from django.core.management.commands.migrate import Command as MigrateCommand +from django.db import connection, models, transaction +from django.db.migrations.migration import Migration +from django.db.migrations.operations.models import CreateModel +from django.db.migrations.operations.special import RunPython +from django.db.migrations.state import ProjectState from google.cloud.spanner_v1 import ( BatchCreateSessionsRequest, CommitRequest, @@ -404,3 +410,142 @@ def test_insert_and_fetch_json_field(self): # filter(value=None) request -> also queries JSON null ("null", TypeCode.JSON) self.assertEqual(exec_requests[14].param_types["a0"].code, TypeCode.JSON) self.assertEqual(exec_requests[14].params["a0"], "null") + + def test_sync_apps_batches_ddl_when_create_test_db_flag_enabled(self): + cmd = MigrateCommand() + cmd.verbosity = 0 + with mock.patch.object( + connection.introspection, "table_names", return_value=[] + ): + connection._batch_ddl_in_schema_editor = True + try: + cmd.sync_apps(connection, ["auth", "contenttypes", "sessions", "sites"]) + finally: + connection._batch_ddl_in_schema_editor = False + + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 1) + self.assertEqual(len(ddl_requests[0].statements), 32) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + def test_default_schema_editor_executes_ddl_per_statement(self): + class LocalAuthor(models.Model): + name = models.CharField(max_length=100, unique=True) + + class Meta: + app_label = "tests" + + with connection.schema_editor() as editor: + editor.create_model(LocalAuthor) + + # Without _batch_ddl_in_schema_editor, CREATE TABLE and CREATE UNIQUE INDEX + # execute as 2 separate UpdateDatabaseDdlRequest calls. + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 2) + self.assertTrue(transaction.get_autocommit()) + + def test_migration_with_runpython_and_deferred_sql_under_create_test_db_flag(self): + add_select1_result() + + def run_query(apps, schema_editor): + with schema_editor.connection.cursor() as cursor: + cursor.execute("select 1") + + mig = Migration("0001_batch_runpython", "tests") + mig.operations = [ + CreateModel( + name="BatchModelOne", + fields=[ + ("id", models.BigAutoField(primary_key=True)), + ("code", models.CharField(max_length=50, unique=True)), + ], + ), + RunPython(run_query), + CreateModel( + name="BatchModelTwo", + fields=[ + ("id", models.BigAutoField(primary_key=True)), + ("title", models.CharField(max_length=50, unique=True)), + ], + ), + ] + state = ProjectState() + + connection._batch_ddl_in_schema_editor = True + try: + with connection.schema_editor(atomic=mig.atomic) as editor: + mig.apply(state, editor) + finally: + connection._batch_ddl_in_schema_editor = False + + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 2) + self.assertEqual(len(ddl_requests[0].statements), 1) + self.assertEqual(len(ddl_requests[1].statements), 3) + self.assertTrue( + any(isinstance(r, CommitRequest) for r in self.spanner_service.requests) + ) + self.assertTrue(transaction.get_autocommit()) + + def test_schema_editor_exception_discards_queued_ddl_when_batching(self): + class BrokenModel(models.Model): + name = models.CharField(max_length=100) + + class Meta: + app_label = "tests" + + connection._batch_ddl_in_schema_editor = True + try: + with self.assertRaises(RuntimeError): + with connection.schema_editor() as editor: + editor.create_model(BrokenModel) + raise RuntimeError("abort schema block") + finally: + connection._batch_ddl_in_schema_editor = False + + self.assertEqual(len(self.database_admin_service.requests), 0) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + def test_create_test_db_end_to_end_batches_syncdb_ddls(self): + orig_name = connection.settings_dict["NAME"] + + def fake_migrate_syncdb(*args, **kwargs): + cmd = MigrateCommand() + cmd.verbosity = 0 + with mock.patch.object( + connection.introspection, "table_names", return_value=[] + ): + cmd.sync_apps(connection, ["sites", "sessions"]) + + def fake_call_command(name, *args, **kwargs): + if name == "migrate": + fake_migrate_syncdb(*args, **kwargs) + + try: + with ( + mock.patch.object( + connection.creation, + "_create_test_db", + return_value="test_django_test_db", + ), + mock.patch( + "django.core.management.call_command", + side_effect=fake_call_command, + ), + ): + create_kwargs = {"verbosity": 0, "autoclobber": True} + if django.VERSION < (6, 0): + create_kwargs["serialize"] = False + db_name = connection.creation.create_test_db(**create_kwargs) + + self.assertEqual(db_name, "django_test_db") + self.assertFalse(getattr(connection, "_batch_ddl_in_schema_editor", False)) + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 1) + # sites (table + unique index) + sessions (table + expire_date index) = 4 DDLs + self.assertEqual(len(ddl_requests[0].statements), 4) + self.assertTrue(transaction.get_autocommit()) + finally: + connection.settings_dict["NAME"] = orig_name diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py b/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py new file mode 100644 index 000000000000..15cd18ee6207 --- /dev/null +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py @@ -0,0 +1,66 @@ +# Copyright 2026 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +import os +from unittest import mock + +from django.db.backends.base.creation import BaseDatabaseCreation + +from django_spanner.creation import DatabaseCreation +from tests.unit.django_spanner.simple_test import SpannerSimpleTestClass + + +class TestCreation(SpannerSimpleTestClass): + def test_create_test_db_enables_ddl_batching_and_returns_db_name(self): + creation = DatabaseCreation(self.connection) + observed_flags = [] + + def fake_super_create_test_db(*args, **kwargs): + observed_flags.append( + getattr(self.connection, "_batch_ddl_in_schema_editor", False) + ) + return "test_database_id" + + with mock.patch.object( + BaseDatabaseCreation, + "create_test_db", + side_effect=fake_super_create_test_db, + ): + result = creation.create_test_db(verbosity=0, autoclobber=True) + + self.assertEqual(result, "test_database_id") + self.assertEqual(observed_flags, [True]) + self.assertFalse(self.connection._batch_ddl_in_schema_editor) + + def test_create_test_db_resets_ddl_batching_on_error(self): + creation = DatabaseCreation(self.connection) + + with mock.patch.object( + BaseDatabaseCreation, + "create_test_db", + side_effect=RuntimeError("boom"), + ): + with self.assertRaises(RuntimeError): + creation.create_test_db(verbosity=0, autoclobber=True) + + self.assertFalse(self.connection._batch_ddl_in_schema_editor) + + def test_create_test_db_marks_skips_when_env_set(self): + creation = DatabaseCreation(self.connection) + + with ( + mock.patch.dict(os.environ, {"RUNNING_SPANNER_BACKEND_TESTS": "1"}), + mock.patch.object(creation, "mark_skips") as mock_mark_skips, + mock.patch.object( + BaseDatabaseCreation, + "create_test_db", + return_value="test_database_id", + ), + ): + result = creation.create_test_db(verbosity=0, autoclobber=True) + + self.assertEqual(result, "test_database_id") + mock_mark_skips.assert_called_once_with() diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py index ea4888e6f4a8..befc674274f9 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py @@ -517,3 +517,93 @@ def test_skip_default_generated_and_db_default(self): normal_field = mock.MagicMock(generated=False, db_default=None) self.assertTrue(schema_editor.skip_default(normal_field)) + + def test_schema_editor_batches_ddl_when_flag_enabled(self): + """Toggles set_autocommit(False) -> set_autocommit(True) around deferred_sql.""" + events = [] + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn._batch_ddl_in_schema_editor = True + mock_conn.set_autocommit.side_effect = lambda val: events.append( + ("set_autocommit", val) + ) + + editor = DatabaseSchemaEditor(mock_conn) + editor.execute = mock.MagicMock( + side_effect=lambda sql, params=None: events.append(("execute", sql)) + ) + + with editor: + editor.execute("CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None) + editor.deferred_sql.append("CREATE INDEX idx_t1 ON t1 (id)") + + self.assertEqual( + events, + [ + ("set_autocommit", False), + ("execute", "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)"), + ("execute", "CREATE INDEX idx_t1 ON t1 (id)"), + ("set_autocommit", True), + ], + ) + + def test_schema_editor_rolls_back_on_exception_when_batching(self): + """Rolls back queued DDLs before restoring autocommit on exception.""" + events = [] + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn._batch_ddl_in_schema_editor = True + mock_conn.rollback.side_effect = lambda: events.append("rollback") + mock_conn.set_autocommit.side_effect = lambda val: events.append( + ("set_autocommit", val) + ) + + with self.assertRaises(ValueError): + with DatabaseSchemaEditor(mock_conn): + raise ValueError("ddl failure") + + self.assertEqual( + events, + [ + ("set_autocommit", False), + "rollback", + ("set_autocommit", True), + ], + ) + + def test_schema_editor_skips_autocommit_toggle_by_default(self): + """Does not toggle autocommit when _batch_ddl_in_schema_editor is False/unset.""" + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn._batch_ddl_in_schema_editor = False + + with DatabaseSchemaEditor(mock_conn): + pass + + mock_conn.set_autocommit.assert_not_called() + + def test_schema_editor_skips_autocommit_toggle_for_collect_sql_or_atomic_or_manual( + self, + ): + """Skips toggling when collect_sql=True, in_atomic_block=True, or autocommit=False.""" + for collect_sql, in_atomic, is_autocommit in [ + (True, False, True), + (False, True, True), + (False, False, False), + ]: + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = in_atomic + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = is_autocommit + mock_conn._batch_ddl_in_schema_editor = True + + with DatabaseSchemaEditor(mock_conn, collect_sql=collect_sql): + pass + + mock_conn.set_autocommit.assert_not_called() From 8862e994e03f0763019e11d4fa357e2c78ae049e Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 13 Nov 2026 08:42:36 +0000 Subject: [PATCH 2/9] Addressed comments --- .../django_spanner/schema.py | 9 ++- .../tests/mockserver_tests/test_basics.py | 36 ++++++++++ .../tests/unit/django_spanner/test_schema.py | 69 ++++++++++++++++++- 3 files changed, 111 insertions(+), 3 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/schema.py b/packages/django-google-spanner/django_spanner/schema.py index c7f991031b56..a28e7a117836 100644 --- a/packages/django-google-spanner/django_spanner/schema.py +++ b/packages/django-google-spanner/django_spanner/schema.py @@ -71,12 +71,17 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): + clean_exit = False try: - super().__exit__(exc_type, exc_value, traceback) + res = super().__exit__(exc_type, exc_value, traceback) + clean_exit = (exc_type is None) or bool(res) + return res finally: if getattr(self, "_restore_autocommit", False): try: - if exc_type is not None: + if clean_exit: + self.connection.commit() + else: self.connection.rollback() finally: self.connection.set_autocommit(True) diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index dfe59000d472..951f7dd18c88 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -549,3 +549,39 @@ def fake_call_command(name, *args, **kwargs): self.assertTrue(transaction.get_autocommit()) finally: connection.settings_dict["NAME"] = orig_name + + def test_schema_editor_deferred_sql_exception_discards_queued_ddl(self): + connection._batch_ddl_in_schema_editor = True + try: + with self.assertRaises(ValueError): + with connection.schema_editor() as editor: + editor.create_model(Singer) + editor.deferred_sql.append( + "CREATE INDEX idx_s ON tests_singer (first_name); SELECT 1" + ) + finally: + connection._batch_ddl_in_schema_editor = False + + self.assertEqual(len(self.database_admin_service.requests), 0) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + def test_schema_editor_server_side_ddl_error_restores_autocommit(self): + from google.api_core.exceptions import InvalidArgument + + connection.ensure_connection() + connection._batch_ddl_in_schema_editor = True + try: + with mock.patch.object( + connection.connection.database, + "update_ddl", + side_effect=InvalidArgument("400 Invalid DDL syntax"), + ): + with self.assertRaises(InvalidArgument): + with connection.schema_editor() as editor: + editor.create_model(Singer) + finally: + connection._batch_ddl_in_schema_editor = False + + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py index befc674274f9..19fe70524206 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py @@ -519,13 +519,14 @@ def test_skip_default_generated_and_db_default(self): self.assertTrue(schema_editor.skip_default(normal_field)) def test_schema_editor_batches_ddl_when_flag_enabled(self): - """Toggles set_autocommit(False) -> set_autocommit(True) around deferred_sql.""" + """Toggles set_autocommit(False), commits, and restores set_autocommit(True).""" events = [] mock_conn = mock.MagicMock() mock_conn.in_atomic_block = False mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = True mock_conn._batch_ddl_in_schema_editor = True + mock_conn.commit.side_effect = lambda: events.append("commit") mock_conn.set_autocommit.side_effect = lambda val: events.append( ("set_autocommit", val) ) @@ -545,6 +546,7 @@ def test_schema_editor_batches_ddl_when_flag_enabled(self): ("set_autocommit", False), ("execute", "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)"), ("execute", "CREATE INDEX idx_t1 ON t1 (id)"), + "commit", ("set_autocommit", True), ], ) @@ -575,6 +577,71 @@ def test_schema_editor_rolls_back_on_exception_when_batching(self): ], ) + def test_schema_editor_rolls_back_on_deferred_sql_exception_when_batching(self): + """Rolls back when super().__exit__() raises Exception or BaseException.""" + for exc_cls in (RuntimeError, KeyboardInterrupt): + events = [] + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn._batch_ddl_in_schema_editor = True + mock_conn.commit.side_effect = lambda: events.append("commit") + mock_conn.rollback.side_effect = lambda: events.append("rollback") + mock_conn.set_autocommit.side_effect = lambda val: events.append( + ("set_autocommit", val) + ) + + editor = DatabaseSchemaEditor(mock_conn) + editor.execute = mock.MagicMock(side_effect=exc_cls("deferred failure")) + + with self.assertRaises(exc_cls): + with editor: + editor.deferred_sql.append("CREATE INDEX idx_bad ON t1 (id)") + + self.assertEqual( + events, + [ + ("set_autocommit", False), + "rollback", + ("set_autocommit", True), + ], + ) + + def test_schema_editor_restores_autocommit_when_commit_raises(self): + """Restores set_autocommit(True) even if commit() raises a DDL error.""" + events = [] + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn._batch_ddl_in_schema_editor = True + + def failing_commit(): + events.append("commit") + raise RuntimeError("server DDL error") + + mock_conn.commit.side_effect = failing_commit + mock_conn.set_autocommit.side_effect = lambda val: events.append( + ("set_autocommit", val) + ) + + editor = DatabaseSchemaEditor(mock_conn) + editor.execute = mock.MagicMock() + + with self.assertRaises(RuntimeError): + with editor: + pass + + self.assertEqual( + events, + [ + ("set_autocommit", False), + "commit", + ("set_autocommit", True), + ], + ) + def test_schema_editor_skips_autocommit_toggle_by_default(self): """Does not toggle autocommit when _batch_ddl_in_schema_editor is False/unset.""" mock_conn = mock.MagicMock() From 68a3a2a28a25eba2fcfbc27179edc9393dfcb402 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 13 Nov 2026 10:28:50 +0000 Subject: [PATCH 3/9] Addressed comments --- packages/django-google-spanner/README.rst | 24 ++++++++++ .../django_spanner/base.py | 1 + .../django_spanner/creation.py | 6 +-- .../django_spanner/schema.py | 5 ++- .../django_test_suite_5.2.sh | 6 +++ .../django_test_suite_6.0.sh | 6 +++ .../django-google-spanner/docs/README.rst | 24 ++++++++++ .../docs/limitations-spanner.rst | 9 ++-- .../django-google-spanner/foreign_key_test.sh | 3 ++ packages/django-google-spanner/noxfile.py | 2 +- packages/django-google-spanner/setup.py | 2 +- .../tests/mockserver_tests/test_basics.py | 44 ++++++++++--------- .../tests/unit/django_spanner/test_base.py | 4 +- .../unit/django_spanner/test_creation.py | 27 ++---------- .../tests/unit/django_spanner/test_schema.py | 35 ++++++++------- 15 files changed, 125 insertions(+), 73 deletions(-) diff --git a/packages/django-google-spanner/README.rst b/packages/django-google-spanner/README.rst index 9e3e8cbd50ab..8bb77001949d 100644 --- a/packages/django-google-spanner/README.rst +++ b/packages/django-google-spanner/README.rst @@ -170,6 +170,30 @@ A transaction is automatically started if you define an To disable transactions in autocommit mode, set the flag `ALLOW_TRANSACTIONS_IN_AUTO_COMMIT` to False in your ``settings.py`` file. +DDL batching +~~~~~~~~~~~~ + +By default, ``DatabaseSchemaEditor`` executes each DDL statement individually. +To batch DDL statements generated within a ``schema_editor()`` context (such as +during ``manage.py migrate`` or test database creation) into batched +``UpdateDatabaseDdl`` requests, set ``'batch_ddl': True`` in ``OPTIONS`` +(defaults to ``False``): + +.. code:: python + + DATABASES = { + 'default': { + 'ENGINE': 'django_spanner', + 'PROJECT': '$PROJECT', + 'INSTANCE': '$INSTANCE', + 'NAME': '$DATABASE', + 'OPTIONS': { + 'batch_ddl': True, + }, + } + } + + Set credentials and project environment variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You'll need to download a service account JSON key file and point to it using an environment variable: diff --git a/packages/django-google-spanner/django_spanner/base.py b/packages/django-google-spanner/django_spanner/base.py index 8e488e41f4f4..cec94539d018 100644 --- a/packages/django-google-spanner/django_spanner/base.py +++ b/packages/django-google-spanner/django_spanner/base.py @@ -182,6 +182,7 @@ def get_new_connection(self, conn_params): conn_params.pop("instance", None) conn_params.pop("instance_id", None) conn_params.pop("client", None) + conn_params.pop("batch_ddl", None) # Ensure client is initialized instance = self.instance return self.Database.connect( diff --git a/packages/django-google-spanner/django_spanner/creation.py b/packages/django-google-spanner/django_spanner/creation.py index 9cafabb8b777..2998ab38a733 100644 --- a/packages/django-google-spanner/django_spanner/creation.py +++ b/packages/django-google-spanner/django_spanner/creation.py @@ -48,11 +48,7 @@ def create_test_db(self, *args, **kwargs): # by a developer running the tests locally. if os.environ.get("RUNNING_SPANNER_BACKEND_TESTS") == "1": self.mark_skips() - self.connection._batch_ddl_in_schema_editor = True - try: - return super().create_test_db(*args, **kwargs) - finally: - self.connection._batch_ddl_in_schema_editor = False + return super().create_test_db(*args, **kwargs) def _create_test_db(self, verbosity, autoclobber, keepdb=False): """ diff --git a/packages/django-google-spanner/django_spanner/schema.py b/packages/django-google-spanner/django_spanner/schema.py index a28e7a117836..923a9a489fe2 100644 --- a/packages/django-google-spanner/django_spanner/schema.py +++ b/packages/django-google-spanner/django_spanner/schema.py @@ -60,8 +60,11 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): def __enter__(self): super().__enter__() self._restore_autocommit = False + batch_ddl = self.connection.settings_dict.get("OPTIONS", {}).get( + "batch_ddl", False + ) if ( - getattr(self.connection, "_batch_ddl_in_schema_editor", False) + batch_ddl and not self.collect_sql and not self.connection.in_atomic_block and self.connection.get_autocommit() diff --git a/packages/django-google-spanner/django_test_suite_5.2.sh b/packages/django-google-spanner/django_test_suite_5.2.sh index 52e489561f31..9e2141c9143d 100755 --- a/packages/django-google-spanner/django_test_suite_5.2.sh +++ b/packages/django-google-spanner/django_test_suite_5.2.sh @@ -47,12 +47,18 @@ DATABASES = { 'PROJECT': "$PROJECT", 'INSTANCE': "$INSTANCE", 'NAME': "$TEST_DBNAME", + 'OPTIONS': { + 'batch_ddl': True, + }, }, 'other': { 'ENGINE': 'django_spanner', 'PROJECT': "$PROJECT", 'INSTANCE': "$INSTANCE", 'NAME': "$TEST_DBNAME_OTHER", + 'OPTIONS': { + 'batch_ddl': True, + }, }, } USE_TZ = False diff --git a/packages/django-google-spanner/django_test_suite_6.0.sh b/packages/django-google-spanner/django_test_suite_6.0.sh index eeab3d35e62d..03414cf7893c 100755 --- a/packages/django-google-spanner/django_test_suite_6.0.sh +++ b/packages/django-google-spanner/django_test_suite_6.0.sh @@ -49,12 +49,18 @@ DATABASES = { 'PROJECT': "$PROJECT", 'INSTANCE': "$INSTANCE", 'NAME': "$TEST_DBNAME", + 'OPTIONS': { + 'batch_ddl': True, + }, }, 'other': { 'ENGINE': 'django_spanner', 'PROJECT': "$PROJECT", 'INSTANCE': "$INSTANCE", 'NAME': "$TEST_DBNAME_OTHER", + 'OPTIONS': { + 'batch_ddl': True, + }, }, } USE_TZ = False diff --git a/packages/django-google-spanner/docs/README.rst b/packages/django-google-spanner/docs/README.rst index 9e3e8cbd50ab..8bb77001949d 100644 --- a/packages/django-google-spanner/docs/README.rst +++ b/packages/django-google-spanner/docs/README.rst @@ -170,6 +170,30 @@ A transaction is automatically started if you define an To disable transactions in autocommit mode, set the flag `ALLOW_TRANSACTIONS_IN_AUTO_COMMIT` to False in your ``settings.py`` file. +DDL batching +~~~~~~~~~~~~ + +By default, ``DatabaseSchemaEditor`` executes each DDL statement individually. +To batch DDL statements generated within a ``schema_editor()`` context (such as +during ``manage.py migrate`` or test database creation) into batched +``UpdateDatabaseDdl`` requests, set ``'batch_ddl': True`` in ``OPTIONS`` +(defaults to ``False``): + +.. code:: python + + DATABASES = { + 'default': { + 'ENGINE': 'django_spanner', + 'PROJECT': '$PROJECT', + 'INSTANCE': '$INSTANCE', + 'NAME': '$DATABASE', + 'OPTIONS': { + 'batch_ddl': True, + }, + } + } + + Set credentials and project environment variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You'll need to download a service account JSON key file and point to it using an environment variable: diff --git a/packages/django-google-spanner/docs/limitations-spanner.rst b/packages/django-google-spanner/docs/limitations-spanner.rst index 75cbbebaf6b3..94fc26ed22e2 100644 --- a/packages/django-google-spanner/docs/limitations-spanner.rst +++ b/packages/django-google-spanner/docs/limitations-spanner.rst @@ -20,9 +20,10 @@ it inherently. The workaround is to use python-spanner objects and run Mutations Batch DDL ~~~~~~~~~ -Migrations (creation and updation) of tables in Django are sequential operations. -Django doesn't inherently support doing batch DDL operations. However, customers -can use python-spanner objects to run batch DDL for unmanaged tables in Django. +By default, migrations (creation and update of tables) in Django execute DDL +statements sequentially. You can opt in to batching DDL statements within each +``schema_editor()`` context by setting ``'OPTIONS': {'batch_ddl': True}`` in +``DATABASES``, or use python-spanner objects directly for unmanaged tables. Stale Reads ~~~~~~~~~~~ @@ -91,7 +92,7 @@ Workaround * - Mutations - Python-spanner database objects can be used to run Mutation queries. `Link `__. * - Batch DDL - - Using unmanaged tables in Django. `Link `__ and Directly using Python spanner objects to execute batch DDL statements. `Link `__. + - Set ``'OPTIONS': {'batch_ddl': True}`` in ``DATABASES`` to batch DDL statements within ``schema_editor()``, or use unmanaged tables (`Link `__) with python-spanner objects directly (`Link `__). * - Stale Reads - Python-spanner database objects can be used to perform stale reads. `Link `__. * - Interleaving diff --git a/packages/django-google-spanner/foreign_key_test.sh b/packages/django-google-spanner/foreign_key_test.sh index 23feb5fdab2d..3469114cd38c 100644 --- a/packages/django-google-spanner/foreign_key_test.sh +++ b/packages/django-google-spanner/foreign_key_test.sh @@ -44,6 +44,9 @@ DATABASES = { 'PROJECT': '$GOOGLE_CLOUD_PROJECT', 'INSTANCE': '$SPANNER_TEST_INSTANCE', 'NAME': 'test_foreign_keys', + 'OPTIONS': { + 'batch_ddl': True, + }, } } " >> foreign_keys/settings.py diff --git a/packages/django-google-spanner/noxfile.py b/packages/django-google-spanner/noxfile.py index 1889aa16ef87..f6ea5270883a 100644 --- a/packages/django-google-spanner/noxfile.py +++ b/packages/django-google-spanner/noxfile.py @@ -63,7 +63,7 @@ ] UNIT_TEST_MOCKSERVER_DEPENDENCIES = [ - "google-cloud-spanner>=3.70.0", + "google-cloud-spanner>=3.72.0", "sqlparse>=0.4.4", ] diff --git a/packages/django-google-spanner/setup.py b/packages/django-google-spanner/setup.py index e5c88ef94c02..811cb0093997 100644 --- a/packages/django-google-spanner/setup.py +++ b/packages/django-google-spanner/setup.py @@ -21,7 +21,7 @@ # (django_spanner/__init__.py currently enforces SUPPORTED_DJANGO_VERSIONS = [(6, 0), (5, 2)]) dependencies = [ "sqlparse >= 0.3.0", - "google-cloud-spanner >= 3.70.0", + "google-cloud-spanner >= 3.72.0", "django >= 5.2, < 6.1", ] extras = { diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index 951f7dd18c88..fad9ea731f15 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -411,17 +411,19 @@ def test_insert_and_fetch_json_field(self): self.assertEqual(exec_requests[14].param_types["a0"].code, TypeCode.JSON) self.assertEqual(exec_requests[14].params["a0"], "null") - def test_sync_apps_batches_ddl_when_create_test_db_flag_enabled(self): + def test_sync_apps_batches_ddl_when_batch_ddl_option_enabled(self): cmd = MigrateCommand() cmd.verbosity = 0 - with mock.patch.object( - connection.introspection, "table_names", return_value=[] - ): - connection._batch_ddl_in_schema_editor = True - try: + connection.close() + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with mock.patch.object( + connection.introspection, "table_names", return_value=[] + ): cmd.sync_apps(connection, ["auth", "contenttypes", "sessions", "sites"]) - finally: - connection._batch_ddl_in_schema_editor = False + self.assertNotIn("batch_ddl", connection.connection._connection_variables) + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) ddl_requests = self.database_admin_service.requests self.assertEqual(len(ddl_requests), 1) @@ -439,13 +441,13 @@ class Meta: with connection.schema_editor() as editor: editor.create_model(LocalAuthor) - # Without _batch_ddl_in_schema_editor, CREATE TABLE and CREATE UNIQUE INDEX + # Without OPTIONS["batch_ddl"], CREATE TABLE and CREATE UNIQUE INDEX # execute as 2 separate UpdateDatabaseDdlRequest calls. ddl_requests = self.database_admin_service.requests self.assertEqual(len(ddl_requests), 2) self.assertTrue(transaction.get_autocommit()) - def test_migration_with_runpython_and_deferred_sql_under_create_test_db_flag(self): + def test_migration_with_runpython_and_deferred_sql_with_batch_ddl_option(self): add_select1_result() def run_query(apps, schema_editor): @@ -472,12 +474,12 @@ def run_query(apps, schema_editor): ] state = ProjectState() - connection._batch_ddl_in_schema_editor = True + connection.settings_dict["OPTIONS"]["batch_ddl"] = True try: with connection.schema_editor(atomic=mig.atomic) as editor: mig.apply(state, editor) finally: - connection._batch_ddl_in_schema_editor = False + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) ddl_requests = self.database_admin_service.requests self.assertEqual(len(ddl_requests), 2) @@ -495,21 +497,22 @@ class BrokenModel(models.Model): class Meta: app_label = "tests" - connection._batch_ddl_in_schema_editor = True + connection.settings_dict["OPTIONS"]["batch_ddl"] = True try: with self.assertRaises(RuntimeError): with connection.schema_editor() as editor: editor.create_model(BrokenModel) raise RuntimeError("abort schema block") finally: - connection._batch_ddl_in_schema_editor = False + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) self.assertEqual(len(self.database_admin_service.requests), 0) self.assertTrue(transaction.get_autocommit()) self.assertTrue(connection.connection.autocommit) - def test_create_test_db_end_to_end_batches_syncdb_ddls(self): + def test_create_test_db_end_to_end_batches_syncdb_ddls_when_batch_ddl_enabled(self): orig_name = connection.settings_dict["NAME"] + connection.settings_dict["OPTIONS"]["batch_ddl"] = True def fake_migrate_syncdb(*args, **kwargs): cmd = MigrateCommand() @@ -541,17 +544,18 @@ def fake_call_command(name, *args, **kwargs): db_name = connection.creation.create_test_db(**create_kwargs) self.assertEqual(db_name, "django_test_db") - self.assertFalse(getattr(connection, "_batch_ddl_in_schema_editor", False)) + self.assertFalse(hasattr(connection, "_batch_ddl_in_schema_editor")) ddl_requests = self.database_admin_service.requests self.assertEqual(len(ddl_requests), 1) # sites (table + unique index) + sessions (table + expire_date index) = 4 DDLs self.assertEqual(len(ddl_requests[0].statements), 4) self.assertTrue(transaction.get_autocommit()) finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) connection.settings_dict["NAME"] = orig_name def test_schema_editor_deferred_sql_exception_discards_queued_ddl(self): - connection._batch_ddl_in_schema_editor = True + connection.settings_dict["OPTIONS"]["batch_ddl"] = True try: with self.assertRaises(ValueError): with connection.schema_editor() as editor: @@ -560,7 +564,7 @@ def test_schema_editor_deferred_sql_exception_discards_queued_ddl(self): "CREATE INDEX idx_s ON tests_singer (first_name); SELECT 1" ) finally: - connection._batch_ddl_in_schema_editor = False + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) self.assertEqual(len(self.database_admin_service.requests), 0) self.assertTrue(transaction.get_autocommit()) @@ -570,7 +574,7 @@ def test_schema_editor_server_side_ddl_error_restores_autocommit(self): from google.api_core.exceptions import InvalidArgument connection.ensure_connection() - connection._batch_ddl_in_schema_editor = True + connection.settings_dict["OPTIONS"]["batch_ddl"] = True try: with mock.patch.object( connection.connection.database, @@ -581,7 +585,7 @@ def test_schema_editor_server_side_ddl_error_restores_autocommit(self): with connection.schema_editor() as editor: editor.create_model(Singer) finally: - connection._batch_ddl_in_schema_editor = False + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) self.assertTrue(transaction.get_autocommit()) self.assertTrue(connection.connection.autocommit) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_base.py b/packages/django-google-spanner/tests/unit/django_spanner/test_base.py index f6595688646b..6253d257739b 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_base.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_base.py @@ -46,12 +46,12 @@ def test_get_connection_params(self): def test_get_new_connection(self): self.db_wrapper.Database = mock_database = mock.MagicMock() mock_database.connect = mock_connection = mock.MagicMock() - conn_params = {"test_param": "dummy"} + conn_params = {"test_param": "dummy", "batch_ddl": True} self.db_wrapper.get_new_connection(conn_params) mock_connection.assert_called_once_with( self.INSTANCE_ID, client=mock_database.connect.call_args[1]["client"], - **conn_params, + test_param="dummy", ) def test_init_connection_state(self): diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py b/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py index 15cd18ee6207..6388fe0e88ab 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py @@ -14,39 +14,18 @@ class TestCreation(SpannerSimpleTestClass): - def test_create_test_db_enables_ddl_batching_and_returns_db_name(self): + def test_create_test_db_returns_db_name_without_private_flag(self): creation = DatabaseCreation(self.connection) - observed_flags = [] - - def fake_super_create_test_db(*args, **kwargs): - observed_flags.append( - getattr(self.connection, "_batch_ddl_in_schema_editor", False) - ) - return "test_database_id" with mock.patch.object( BaseDatabaseCreation, "create_test_db", - side_effect=fake_super_create_test_db, + return_value="test_database_id", ): result = creation.create_test_db(verbosity=0, autoclobber=True) self.assertEqual(result, "test_database_id") - self.assertEqual(observed_flags, [True]) - self.assertFalse(self.connection._batch_ddl_in_schema_editor) - - def test_create_test_db_resets_ddl_batching_on_error(self): - creation = DatabaseCreation(self.connection) - - with mock.patch.object( - BaseDatabaseCreation, - "create_test_db", - side_effect=RuntimeError("boom"), - ): - with self.assertRaises(RuntimeError): - creation.create_test_db(verbosity=0, autoclobber=True) - - self.assertFalse(self.connection._batch_ddl_in_schema_editor) + self.assertFalse(hasattr(self.connection, "_batch_ddl_in_schema_editor")) def test_create_test_db_marks_skips_when_env_set(self): creation = DatabaseCreation(self.connection) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py index 19fe70524206..5eb78822fb60 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py @@ -518,14 +518,14 @@ def test_skip_default_generated_and_db_default(self): normal_field = mock.MagicMock(generated=False, db_default=None) self.assertTrue(schema_editor.skip_default(normal_field)) - def test_schema_editor_batches_ddl_when_flag_enabled(self): + def test_schema_editor_batches_ddl_when_option_enabled(self): """Toggles set_autocommit(False), commits, and restores set_autocommit(True).""" events = [] mock_conn = mock.MagicMock() mock_conn.in_atomic_block = False mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = True - mock_conn._batch_ddl_in_schema_editor = True + mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} mock_conn.commit.side_effect = lambda: events.append("commit") mock_conn.set_autocommit.side_effect = lambda val: events.append( ("set_autocommit", val) @@ -558,7 +558,7 @@ def test_schema_editor_rolls_back_on_exception_when_batching(self): mock_conn.in_atomic_block = False mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = True - mock_conn._batch_ddl_in_schema_editor = True + mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} mock_conn.rollback.side_effect = lambda: events.append("rollback") mock_conn.set_autocommit.side_effect = lambda val: events.append( ("set_autocommit", val) @@ -585,7 +585,7 @@ def test_schema_editor_rolls_back_on_deferred_sql_exception_when_batching(self): mock_conn.in_atomic_block = False mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = True - mock_conn._batch_ddl_in_schema_editor = True + mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} mock_conn.commit.side_effect = lambda: events.append("commit") mock_conn.rollback.side_effect = lambda: events.append("rollback") mock_conn.set_autocommit.side_effect = lambda val: events.append( @@ -615,7 +615,7 @@ def test_schema_editor_restores_autocommit_when_commit_raises(self): mock_conn.in_atomic_block = False mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = True - mock_conn._batch_ddl_in_schema_editor = True + mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} def failing_commit(): events.append("commit") @@ -643,17 +643,22 @@ def failing_commit(): ) def test_schema_editor_skips_autocommit_toggle_by_default(self): - """Does not toggle autocommit when _batch_ddl_in_schema_editor is False/unset.""" - mock_conn = mock.MagicMock() - mock_conn.in_atomic_block = False - mock_conn.features.can_rollback_ddl = False - mock_conn.get_autocommit.return_value = True - mock_conn._batch_ddl_in_schema_editor = False + """Does not toggle autocommit when OPTIONS['batch_ddl'] is False or unset.""" + for settings_dict in ( + {}, + {"OPTIONS": {}}, + {"OPTIONS": {"batch_ddl": False}}, + ): + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn.settings_dict = settings_dict - with DatabaseSchemaEditor(mock_conn): - pass + with DatabaseSchemaEditor(mock_conn): + pass - mock_conn.set_autocommit.assert_not_called() + mock_conn.set_autocommit.assert_not_called() def test_schema_editor_skips_autocommit_toggle_for_collect_sql_or_atomic_or_manual( self, @@ -668,7 +673,7 @@ def test_schema_editor_skips_autocommit_toggle_for_collect_sql_or_atomic_or_manu mock_conn.in_atomic_block = in_atomic mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = is_autocommit - mock_conn._batch_ddl_in_schema_editor = True + mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} with DatabaseSchemaEditor(mock_conn, collect_sql=collect_sql): pass From d039c5efce168d1a8d8b0f27d5463e9f709963b8 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 13 Nov 2026 12:15:32 +0000 Subject: [PATCH 4/9] Addressed comments --- .../django_spanner/schema.py | 44 +++++++++- .../tests/mockserver_tests/test_basics.py | 64 ++++++++++++++- .../tests/unit/django_spanner/test_schema.py | 81 +++++++++++++++++++ 3 files changed, 185 insertions(+), 4 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/schema.py b/packages/django-google-spanner/django_spanner/schema.py index 923a9a489fe2..21cb9d53f32a 100644 --- a/packages/django-google-spanner/django_spanner/schema.py +++ b/packages/django-google-spanner/django_spanner/schema.py @@ -7,9 +7,22 @@ import uuid import django -from django.db import NotSupportedError +from django.db import ( + IntegrityError, + NotSupportedError, + OperationalError, + ProgrammingError, +) from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.models.fields import NOT_PROVIDED +from google.api_core.exceptions import ( + AlreadyExists, + FailedPrecondition, + GoogleAPICallError, + InternalServerError, + InvalidArgument, + OutOfRange, +) from django_spanner import USE_EMULATOR from django_spanner._opentelemetry_tracing import trace_call @@ -57,6 +70,20 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): # sql_create_inline_fk = "CONSTRAINT FK_%(to_table)s_%(to_column)s_%(from_table)s_%(from_column)s FOREIGN KEY (%(from_column_norm)s) REFERENCES %(to_table_norm)s (%(to_column_norm)s)" # noqa sql_create_inline_fk = None + def __init__(self, connection, collect_sql=False, atomic=True): + self._atomic = atomic + super().__init__(connection, collect_sql=collect_sql, atomic=atomic) + + def _commit_ddls(self): + try: + self.connection.commit() + except (AlreadyExists, FailedPrecondition, OutOfRange) as exc: + raise IntegrityError(getattr(exc, "details", exc)) from exc + except InvalidArgument as exc: + raise ProgrammingError(getattr(exc, "details", exc)) from exc + except (InternalServerError, GoogleAPICallError) as exc: + raise OperationalError(getattr(exc, "details", exc)) from exc + def __enter__(self): super().__enter__() self._restore_autocommit = False @@ -65,6 +92,7 @@ def __enter__(self): ) if ( batch_ddl + and getattr(self, "_atomic", True) and not self.collect_sql and not self.connection.in_atomic_block and self.connection.get_autocommit() @@ -83,12 +111,24 @@ def __exit__(self, exc_type, exc_value, traceback): if getattr(self, "_restore_autocommit", False): try: if clean_exit: - self.connection.commit() + self._commit_ddls() else: self.connection.rollback() finally: self.connection.set_autocommit(True) + def execute(self, sql, params=()): + if ( + getattr(self, "_restore_autocommit", False) + and not self.collect_sql + and getattr( + self.connection.connection, "_spanner_transaction_started", False + ) + is True + ): + self._commit_ddls() + return super().execute(sql, params) + def create_model(self, model): """ Create a table and any accompanying indexes or unique constraints for diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index fad9ea731f15..7dfadfe58b1e 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -571,7 +571,8 @@ def test_schema_editor_deferred_sql_exception_discards_queued_ddl(self): self.assertTrue(connection.connection.autocommit) def test_schema_editor_server_side_ddl_error_restores_autocommit(self): - from google.api_core.exceptions import InvalidArgument + from django.db import IntegrityError, ProgrammingError + from google.api_core.exceptions import FailedPrecondition, InvalidArgument connection.ensure_connection() connection.settings_dict["OPTIONS"]["batch_ddl"] = True @@ -581,7 +582,19 @@ def test_schema_editor_server_side_ddl_error_restores_autocommit(self): "update_ddl", side_effect=InvalidArgument("400 Invalid DDL syntax"), ): - with self.assertRaises(InvalidArgument): + with self.assertRaises(ProgrammingError): + with connection.schema_editor() as editor: + editor.create_model(Singer) + + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + with mock.patch.object( + connection.connection.database, + "update_ddl", + side_effect=FailedPrecondition("400 Duplicate name in schema"), + ): + with self.assertRaises(IntegrityError): with connection.schema_editor() as editor: editor.create_model(Singer) finally: @@ -589,3 +602,50 @@ def test_schema_editor_server_side_ddl_error_restores_autocommit(self): self.assertTrue(transaction.get_autocommit()) self.assertTrue(connection.connection.autocommit) + + def test_schema_editor_add_field_with_default_commits_rw_tx_before_alter_column( + self, + ): + add_update_count("UPDATE tests_singer SET stage_name=@a0 WHERE 1=1", 2) + field = models.CharField(max_length=100, default="anon") + field.set_attributes_from_name("stage_name") + + event_log = [] + orig_update_ddl = self.database_admin_service.UpdateDatabaseDdl + + def logged_update_ddl(request, context): + event_log.append(("ddl", list(request.statements))) + return orig_update_ddl(request, context) + + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with connection.schema_editor() as editor: + editor.add_field(Singer, field) + self.assertFalse(connection.connection._spanner_transaction_started) + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 2) + self.assertTrue( + any(isinstance(r, CommitRequest) for r in self.spanner_service.requests) + ) + self.assertTrue(transaction.get_autocommit()) + + def test_schema_editor_atomic_false_skips_batch_ddl(self): + class NonAtomicModel(models.Model): + code = models.CharField(max_length=50, unique=True) + + class Meta: + app_label = "tests" + + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with connection.schema_editor(atomic=False) as editor: + self.assertTrue(transaction.get_autocommit()) + editor.create_model(NonAtomicModel) + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + self.assertEqual(len(self.database_admin_service.requests), 2) + self.assertTrue(transaction.get_autocommit()) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py index 5eb78822fb60..3775cae4e2db 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py @@ -679,3 +679,84 @@ def test_schema_editor_skips_autocommit_toggle_for_collect_sql_or_atomic_or_manu pass mock_conn.set_autocommit.assert_not_called() + + def test_schema_editor_skips_autocommit_toggle_when_atomic_false(self): + """Skips batching when schema_editor(atomic=False) is requested.""" + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} + + with DatabaseSchemaEditor(mock_conn, atomic=False): + pass + + mock_conn.set_autocommit.assert_not_called() + + def test_schema_editor_execute_commits_open_rw_tx_before_next_statement(self): + """Commits an active Spanner RW transaction before executing subsequent DDL.""" + from django.db.backends.base.schema import BaseDatabaseSchemaEditor + + events = [] + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} + mock_conn.connection._spanner_transaction_started = True + + def fake_commit(): + events.append("commit") + mock_conn.connection._spanner_transaction_started = False + + mock_conn.commit.side_effect = fake_commit + mock_conn.set_autocommit.side_effect = lambda val: events.append( + ("set_autocommit", val) + ) + + with mock.patch.object( + BaseDatabaseSchemaEditor, + "execute", + side_effect=lambda sql, params=(): events.append(("super_execute", sql)), + ): + with DatabaseSchemaEditor(mock_conn) as editor: + editor.execute("ALTER TABLE t1 ALTER COLUMN c1 INT64 NOT NULL", None) + + self.assertEqual( + events, + [ + ("set_autocommit", False), + "commit", + ("super_execute", "ALTER TABLE t1 ALTER COLUMN c1 INT64 NOT NULL"), + "commit", + ("set_autocommit", True), + ], + ) + + def test_schema_editor_maps_google_api_errors_to_django_database_errors(self): + """Maps GoogleAPICallError subclasses on commit() to django.db DatabaseErrors.""" + from django.db import IntegrityError, OperationalError, ProgrammingError + from google.api_core.exceptions import ( + FailedPrecondition, + InternalServerError, + InvalidArgument, + ) + + cases = [ + (FailedPrecondition("Duplicate name"), IntegrityError), + (InvalidArgument("Invalid syntax"), ProgrammingError), + (InternalServerError("Internal failure"), OperationalError), + ] + for api_exc, expected_django_exc in cases: + mock_conn = mock.MagicMock() + mock_conn.in_atomic_block = False + mock_conn.features.can_rollback_ddl = False + mock_conn.get_autocommit.return_value = True + mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} + mock_conn.commit.side_effect = api_exc + + with self.assertRaises(expected_django_exc): + with DatabaseSchemaEditor(mock_conn): + pass + + mock_conn.set_autocommit.assert_called_with(True) From c7239036b0ebb6dd4ef813162c2ef69c212eca3f Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 13 Nov 2026 12:47:07 +0000 Subject: [PATCH 5/9] Addressed comments --- .../django_spanner/base.py | 11 + .../django_spanner/schema.py | 79 +++-- .../tests/mockserver_tests/mock_spanner.py | 7 + .../tests/mockserver_tests/test_basics.py | 91 ++++++ .../tests/unit/django_spanner/test_schema.py | 278 ++++++++++-------- 5 files changed, 324 insertions(+), 142 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/base.py b/packages/django-google-spanner/django_spanner/base.py index cec94539d018..9a1bffd38233 100644 --- a/packages/django-google-spanner/django_spanner/base.py +++ b/packages/django-google-spanner/django_spanner/base.py @@ -210,6 +210,17 @@ def create_cursor(self, name=None): """ return self.connection.cursor() + def set_autocommit( + self, autocommit, force_begin_transaction_with_broken_autocommit=False + ): + editor = getattr(self, "_active_ddl_batch_editor", None) + if editor is not None and editor._ddl_batch_active: + editor._exit_ddl_batch(clean=True) + super().set_autocommit( + autocommit, + force_begin_transaction_with_broken_autocommit=force_begin_transaction_with_broken_autocommit, + ) + def _set_autocommit(self, autocommit): """Set the Spanner transaction autocommit flag. diff --git a/packages/django-google-spanner/django_spanner/schema.py b/packages/django-google-spanner/django_spanner/schema.py index 21cb9d53f32a..3d2ff8748331 100644 --- a/packages/django-google-spanner/django_spanner/schema.py +++ b/packages/django-google-spanner/django_spanner/schema.py @@ -23,6 +23,8 @@ InvalidArgument, OutOfRange, ) +from google.cloud.spanner_dbapi import parse_utils +from google.cloud.spanner_dbapi.parsed_statement import StatementType from django_spanner import USE_EMULATOR from django_spanner._opentelemetry_tracing import trace_call @@ -76,7 +78,7 @@ def __init__(self, connection, collect_sql=False, atomic=True): def _commit_ddls(self): try: - self.connection.commit() + self.connection.connection.commit() except (AlreadyExists, FailedPrecondition, OutOfRange) as exc: raise IntegrityError(getattr(exc, "details", exc)) from exc except InvalidArgument as exc: @@ -84,9 +86,46 @@ def _commit_ddls(self): except (InternalServerError, GoogleAPICallError) as exc: raise OperationalError(getattr(exc, "details", exc)) from exc + def _enter_ddl_batch(self): + if not self._ddl_batch_active: + self.connection.ensure_connection() + self.connection._set_autocommit(False) + self._ddl_batch_active = True + + def _exit_ddl_batch(self, clean): + if not self._ddl_batch_active: + return + try: + if clean: + self._commit_ddls() + else: + self.connection.connection.rollback() + finally: + self._ddl_batch_active = False + self.connection._set_autocommit(True) + + def _ddl_execution_wrapper(self, execute, sql, params, many, context): + is_ddl = False + if ( + not many + and sql is not None + and not self.connection.in_atomic_block + and self.connection.get_autocommit() + ): + parsed = parse_utils.classify_statement(str(sql)) + is_ddl = parsed is not None and parsed.statement_type == StatementType.DDL + + if is_ddl: + self._enter_ddl_batch() + elif self._ddl_batch_active: + self._exit_ddl_batch(clean=True) + + return execute(sql, params, many, context) + def __enter__(self): super().__enter__() - self._restore_autocommit = False + self._ddl_batch_active = False + self._execute_wrapper_cm = None batch_ddl = self.connection.settings_dict.get("OPTIONS", {}).get( "batch_ddl", False ) @@ -96,9 +135,13 @@ def __enter__(self): and not self.collect_sql and not self.connection.in_atomic_block and self.connection.get_autocommit() + and getattr(self.connection, "_active_ddl_batch_editor", None) is None ): - self.connection.set_autocommit(False) - self._restore_autocommit = True + self.connection._active_ddl_batch_editor = self + self._execute_wrapper_cm = self.connection.execute_wrapper( + self._ddl_execution_wrapper + ) + self._execute_wrapper_cm.__enter__() return self def __exit__(self, exc_type, exc_value, traceback): @@ -108,26 +151,14 @@ def __exit__(self, exc_type, exc_value, traceback): clean_exit = (exc_type is None) or bool(res) return res finally: - if getattr(self, "_restore_autocommit", False): - try: - if clean_exit: - self._commit_ddls() - else: - self.connection.rollback() - finally: - self.connection.set_autocommit(True) - - def execute(self, sql, params=()): - if ( - getattr(self, "_restore_autocommit", False) - and not self.collect_sql - and getattr( - self.connection.connection, "_spanner_transaction_started", False - ) - is True - ): - self._commit_ddls() - return super().execute(sql, params) + try: + self._exit_ddl_batch(clean=clean_exit) + finally: + if getattr(self, "_execute_wrapper_cm", None) is not None: + self._execute_wrapper_cm.__exit__(exc_type, exc_value, traceback) + self._execute_wrapper_cm = None + if getattr(self.connection, "_active_ddl_batch_editor", None) is self: + self.connection._active_ddl_batch_editor = None def create_model(self, model): """ diff --git a/packages/django-google-spanner/tests/mockserver_tests/mock_spanner.py b/packages/django-google-spanner/tests/mockserver_tests/mock_spanner.py index 3f43fff0423b..70e70694195a 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/mock_spanner.py +++ b/packages/django-google-spanner/tests/mockserver_tests/mock_spanner.py @@ -35,10 +35,14 @@ class MockSpanner: def __init__(self): self.results = {} + self.errors = {} def add_result(self, sql: str, result: result_set.ResultSet): self.results[sql.lower().strip()] = result + def add_error(self, sql: str, code: grpc.StatusCode, details: str): + self.errors[sql.lower().strip()] = (code, details) + def get_result(self, sql: str) -> result_set.ResultSet: result = self.results.get(sql.lower().strip()) if result is None: @@ -135,6 +139,9 @@ def ExecuteStreamingSql(self, request: ExecuteSqlRequest, context): started_transaction = self.__create_transaction( request.session, request.transaction.begin ) + error = self.mock_spanner.errors.get(request.sql.lower().strip()) + if error is not None: + context.abort(error[0], error[1]) partials = self.mock_spanner.get_result_as_partial_result_sets( request.sql, started_transaction ) diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index 7dfadfe58b1e..081377cec685 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -30,6 +30,7 @@ CommitRequest, CreateSessionRequest, ExecuteSqlRequest, + RollbackRequest, TypeCode, ) @@ -649,3 +650,93 @@ class Meta: self.assertEqual(len(self.database_admin_service.requests), 2) self.assertTrue(transaction.get_autocommit()) + + def test_schema_editor_nested_atomic_integrity_error_rolls_back_and_recovers(self): + import grpc + from django.db import IntegrityError + + add_select1_result() + failing_sql = "UPDATE tests_singer SET first_name=@a0 WHERE id=@a1" + self.spanner_service.mock_spanner.add_error( + failing_sql, + grpc.StatusCode.ALREADY_EXISTS, + "UNIQUE violation on index", + ) + + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with connection.schema_editor(): + self.assertTrue(transaction.get_autocommit()) + with self.assertRaises(IntegrityError): + with transaction.atomic(): + with connection.cursor() as cursor: + cursor.execute( + "UPDATE tests_singer SET first_name=%s WHERE id=%s", + ["dup", 1], + ) + self.assertFalse(connection.needs_rollback) + with connection.cursor() as cursor: + cursor.execute("select 1") + self.assertEqual(cursor.fetchone()[0], 1) + finally: + self.spanner_service.mock_spanner.errors.pop(failing_sql.lower(), None) + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + self.assertTrue( + any(isinstance(r, RollbackRequest) for r in self.spanner_service.requests) + ) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + def test_migration_create_model_then_runpython_flushes_ddl_before_atomic(self): + add_update_count("UPDATE tests_singer SET first_name=@a0 WHERE id=@a1", 1) + events = [] + + def run_atomic_dml(apps, schema_editor): + events.append( + ( + "runpython_state", + schema_editor.connection.in_atomic_block, + schema_editor.connection.commit_on_exit, + len(self.database_admin_service.requests), + ) + ) + with schema_editor.connection.cursor() as cursor: + cursor.execute( + "UPDATE tests_singer SET first_name=%s WHERE id=%s", + ["updated", 1], + ) + + mig = Migration("0001_create_then_runpython", "tests") + mig.operations = [ + CreateModel( + name="BatchModelBeforeRunPython", + fields=[ + ("id", models.BigAutoField(primary_key=True)), + ("code", models.CharField(max_length=50, unique=True)), + ], + ), + RunPython(run_atomic_dml), + ] + state = ProjectState() + + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with connection.schema_editor(atomic=mig.atomic) as editor: + mig.apply(state, editor) + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + # CreateModel's CREATE TABLE was flushed before RunPython's atomic() block began, + # and RunPython ran inside a real outermost atomic block (commit_on_exit=True). + self.assertEqual(events, [("runpython_state", True, True, 1)]) + # Deferred CREATE UNIQUE INDEX flushed on schema_editor.__exit__. + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 2) + self.assertEqual(len(ddl_requests[0].statements), 1) + self.assertEqual(len(ddl_requests[1].statements), 1) + self.assertTrue( + any(isinstance(r, CommitRequest) for r in self.spanner_service.requests) + ) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py index 3775cae4e2db..1987f3ef2d8d 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py @@ -518,62 +518,143 @@ def test_skip_default_generated_and_db_default(self): normal_field = mock.MagicMock(generated=False, db_default=None) self.assertTrue(schema_editor.skip_default(normal_field)) - def test_schema_editor_batches_ddl_when_option_enabled(self): - """Toggles set_autocommit(False), commits, and restores set_autocommit(True).""" - events = [] + def _make_batch_ddl_mock_conn(self, events, batch_ddl=True): + import contextlib + mock_conn = mock.MagicMock() mock_conn.in_atomic_block = False mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = True - mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} - mock_conn.commit.side_effect = lambda: events.append("commit") - mock_conn.set_autocommit.side_effect = lambda val: events.append( - ("set_autocommit", val) + mock_conn._active_ddl_batch_editor = None + mock_conn.settings_dict = ( + {"OPTIONS": {"batch_ddl": batch_ddl}} if batch_ddl is not None else {} ) - - editor = DatabaseSchemaEditor(mock_conn) - editor.execute = mock.MagicMock( - side_effect=lambda sql, params=None: events.append(("execute", sql)) + wrappers = [] + + @contextlib.contextmanager + def fake_execute_wrapper(wrapper): + wrappers.append(wrapper) + try: + yield + finally: + wrappers.remove(wrapper) + + mock_conn.execute_wrapper.side_effect = fake_execute_wrapper + mock_conn.connection.commit.side_effect = lambda: events.append("dbapi_commit") + mock_conn.connection.rollback.side_effect = lambda: events.append( + "dbapi_rollback" + ) + mock_conn._set_autocommit.side_effect = lambda val: events.append( + ("_set_autocommit", val) ) - with editor: + def run_cursor_execute(sql, params=None): + def base_exec(s, p, many, ctx): + events.append(("cursor_execute", s)) + + def make_wrapped_executor(w, nxt): + def wrapped(s, p, m, c): + return w(nxt, s, p, m, c) + + return wrapped + + executor = base_exec + for wrapper in reversed(wrappers): + executor = make_wrapped_executor(wrapper, executor) + return executor(sql, params, False, {"connection": mock_conn}) + + cursor_cm = mock.MagicMock() + cursor_cm.__enter__.return_value.execute.side_effect = run_cursor_execute + mock_conn.cursor.return_value = cursor_cm + return mock_conn + + def test_schema_editor_batches_ddl_lazily_when_option_enabled(self): + """Lazily toggles _set_autocommit(False) on DDL, commits, and restores _set_autocommit(True).""" + events = [] + mock_conn = self._make_batch_ddl_mock_conn(events) + + with DatabaseSchemaEditor(mock_conn) as editor: + # Entering schema_editor alone does not disable autocommit before any DDL runs. + self.assertEqual(events, []) editor.execute("CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None) editor.deferred_sql.append("CREATE INDEX idx_t1 ON t1 (id)") self.assertEqual( events, [ - ("set_autocommit", False), - ("execute", "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)"), - ("execute", "CREATE INDEX idx_t1 ON t1 (id)"), - "commit", - ("set_autocommit", True), + ("_set_autocommit", False), + ( + "cursor_execute", + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", + ), + ("cursor_execute", "CREATE INDEX idx_t1 ON t1 (id)"), + "dbapi_commit", + ("_set_autocommit", True), ], ) + mock_conn.set_autocommit.assert_not_called() - def test_schema_editor_rolls_back_on_exception_when_batching(self): - """Rolls back queued DDLs before restoring autocommit on exception.""" + def test_schema_editor_noop_without_ddl_does_not_toggle_autocommit(self): + """Does not toggle DB-API autocommit when no DDL statement executes.""" events = [] - mock_conn = mock.MagicMock() - mock_conn.in_atomic_block = False - mock_conn.features.can_rollback_ddl = False - mock_conn.get_autocommit.return_value = True - mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} - mock_conn.rollback.side_effect = lambda: events.append("rollback") - mock_conn.set_autocommit.side_effect = lambda val: events.append( - ("set_autocommit", val) + mock_conn = self._make_batch_ddl_mock_conn(events) + + with DatabaseSchemaEditor(mock_conn) as editor: + editor.execute("SELECT 1", None) + + self.assertEqual(events, [("cursor_execute", "SELECT 1")]) + mock_conn._set_autocommit.assert_not_called() + mock_conn.set_autocommit.assert_not_called() + + def test_schema_editor_flushes_ddl_before_non_ddl_and_resumes_batching(self): + """Flushes queued DDLs and restores DB-API autocommit before DML/DQL, then resumes on next DDL.""" + events = [] + mock_conn = self._make_batch_ddl_mock_conn(events) + + with DatabaseSchemaEditor(mock_conn) as editor: + editor.execute("ALTER TABLE t1 ADD COLUMN c1 INT64", None) + editor.execute("UPDATE t1 SET c1 = %s WHERE 1=1", (0,)) + editor.execute("ALTER TABLE t1 ALTER COLUMN c1 INT64 NOT NULL", None) + editor.deferred_sql.append("CREATE INDEX idx_t1_c1 ON t1 (c1)") + + self.assertEqual( + events, + [ + ("_set_autocommit", False), + ("cursor_execute", "ALTER TABLE t1 ADD COLUMN c1 INT64"), + "dbapi_commit", + ("_set_autocommit", True), + ("cursor_execute", "UPDATE t1 SET c1 = %s WHERE 1=1"), + ("_set_autocommit", False), + ("cursor_execute", "ALTER TABLE t1 ALTER COLUMN c1 INT64 NOT NULL"), + ("cursor_execute", "CREATE INDEX idx_t1_c1 ON t1 (c1)"), + "dbapi_commit", + ("_set_autocommit", True), + ], ) + def test_schema_editor_rolls_back_on_exception_when_batching(self): + """Rolls back queued DDLs on DB-API connection before restoring autocommit on exception.""" + events = [] + mock_conn = self._make_batch_ddl_mock_conn(events) + with self.assertRaises(ValueError): - with DatabaseSchemaEditor(mock_conn): + with DatabaseSchemaEditor(mock_conn) as editor: + editor.execute( + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None + ) raise ValueError("ddl failure") self.assertEqual( events, [ - ("set_autocommit", False), - "rollback", - ("set_autocommit", True), + ("_set_autocommit", False), + ( + "cursor_execute", + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", + ), + "dbapi_rollback", + ("_set_autocommit", True), ], ) @@ -581,69 +662,68 @@ def test_schema_editor_rolls_back_on_deferred_sql_exception_when_batching(self): """Rolls back when super().__exit__() raises Exception or BaseException.""" for exc_cls in (RuntimeError, KeyboardInterrupt): events = [] - mock_conn = mock.MagicMock() - mock_conn.in_atomic_block = False - mock_conn.features.can_rollback_ddl = False - mock_conn.get_autocommit.return_value = True - mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} - mock_conn.commit.side_effect = lambda: events.append("commit") - mock_conn.rollback.side_effect = lambda: events.append("rollback") - mock_conn.set_autocommit.side_effect = lambda val: events.append( - ("set_autocommit", val) - ) + mock_conn = self._make_batch_ddl_mock_conn(events) editor = DatabaseSchemaEditor(mock_conn) - editor.execute = mock.MagicMock(side_effect=exc_cls("deferred failure")) + + def fail_on_index(sql, params=None, exc_cls=exc_cls, orig=editor.execute): + if "idx_bad" in sql: + raise exc_cls("deferred failure") + return orig(sql, params) with self.assertRaises(exc_cls): with editor: + editor.execute( + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None + ) + editor.execute = fail_on_index editor.deferred_sql.append("CREATE INDEX idx_bad ON t1 (id)") self.assertEqual( events, [ - ("set_autocommit", False), - "rollback", - ("set_autocommit", True), + ("_set_autocommit", False), + ( + "cursor_execute", + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", + ), + "dbapi_rollback", + ("_set_autocommit", True), ], ) def test_schema_editor_restores_autocommit_when_commit_raises(self): - """Restores set_autocommit(True) even if commit() raises a DDL error.""" + """Restores _set_autocommit(True) even if DB-API commit() raises a DDL error.""" events = [] - mock_conn = mock.MagicMock() - mock_conn.in_atomic_block = False - mock_conn.features.can_rollback_ddl = False - mock_conn.get_autocommit.return_value = True - mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} + mock_conn = self._make_batch_ddl_mock_conn(events) def failing_commit(): - events.append("commit") + events.append("dbapi_commit") raise RuntimeError("server DDL error") - mock_conn.commit.side_effect = failing_commit - mock_conn.set_autocommit.side_effect = lambda val: events.append( - ("set_autocommit", val) - ) - - editor = DatabaseSchemaEditor(mock_conn) - editor.execute = mock.MagicMock() + mock_conn.connection.commit.side_effect = failing_commit with self.assertRaises(RuntimeError): - with editor: - pass + with DatabaseSchemaEditor(mock_conn) as editor: + editor.execute( + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None + ) self.assertEqual( events, [ - ("set_autocommit", False), - "commit", - ("set_autocommit", True), + ("_set_autocommit", False), + ( + "cursor_execute", + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", + ), + "dbapi_commit", + ("_set_autocommit", True), ], ) def test_schema_editor_skips_autocommit_toggle_by_default(self): - """Does not toggle autocommit when OPTIONS['batch_ddl'] is False or unset.""" + """Does not install wrapper or toggle autocommit when OPTIONS['batch_ddl'] is False or unset.""" for settings_dict in ( {}, {"OPTIONS": {}}, @@ -658,12 +738,13 @@ def test_schema_editor_skips_autocommit_toggle_by_default(self): with DatabaseSchemaEditor(mock_conn): pass - mock_conn.set_autocommit.assert_not_called() + mock_conn.execute_wrapper.assert_not_called() + mock_conn._set_autocommit.assert_not_called() def test_schema_editor_skips_autocommit_toggle_for_collect_sql_or_atomic_or_manual( self, ): - """Skips toggling when collect_sql=True, in_atomic_block=True, or autocommit=False.""" + """Skips batching when collect_sql=True, in_atomic_block=True, or autocommit=False.""" for collect_sql, in_atomic, is_autocommit in [ (True, False, True), (False, True, True), @@ -678,7 +759,8 @@ def test_schema_editor_skips_autocommit_toggle_for_collect_sql_or_atomic_or_manu with DatabaseSchemaEditor(mock_conn, collect_sql=collect_sql): pass - mock_conn.set_autocommit.assert_not_called() + mock_conn.execute_wrapper.assert_not_called() + mock_conn._set_autocommit.assert_not_called() def test_schema_editor_skips_autocommit_toggle_when_atomic_false(self): """Skips batching when schema_editor(atomic=False) is requested.""" @@ -691,47 +773,8 @@ def test_schema_editor_skips_autocommit_toggle_when_atomic_false(self): with DatabaseSchemaEditor(mock_conn, atomic=False): pass - mock_conn.set_autocommit.assert_not_called() - - def test_schema_editor_execute_commits_open_rw_tx_before_next_statement(self): - """Commits an active Spanner RW transaction before executing subsequent DDL.""" - from django.db.backends.base.schema import BaseDatabaseSchemaEditor - - events = [] - mock_conn = mock.MagicMock() - mock_conn.in_atomic_block = False - mock_conn.features.can_rollback_ddl = False - mock_conn.get_autocommit.return_value = True - mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} - mock_conn.connection._spanner_transaction_started = True - - def fake_commit(): - events.append("commit") - mock_conn.connection._spanner_transaction_started = False - - mock_conn.commit.side_effect = fake_commit - mock_conn.set_autocommit.side_effect = lambda val: events.append( - ("set_autocommit", val) - ) - - with mock.patch.object( - BaseDatabaseSchemaEditor, - "execute", - side_effect=lambda sql, params=(): events.append(("super_execute", sql)), - ): - with DatabaseSchemaEditor(mock_conn) as editor: - editor.execute("ALTER TABLE t1 ALTER COLUMN c1 INT64 NOT NULL", None) - - self.assertEqual( - events, - [ - ("set_autocommit", False), - "commit", - ("super_execute", "ALTER TABLE t1 ALTER COLUMN c1 INT64 NOT NULL"), - "commit", - ("set_autocommit", True), - ], - ) + mock_conn.execute_wrapper.assert_not_called() + mock_conn._set_autocommit.assert_not_called() def test_schema_editor_maps_google_api_errors_to_django_database_errors(self): """Maps GoogleAPICallError subclasses on commit() to django.db DatabaseErrors.""" @@ -748,15 +791,14 @@ def test_schema_editor_maps_google_api_errors_to_django_database_errors(self): (InternalServerError("Internal failure"), OperationalError), ] for api_exc, expected_django_exc in cases: - mock_conn = mock.MagicMock() - mock_conn.in_atomic_block = False - mock_conn.features.can_rollback_ddl = False - mock_conn.get_autocommit.return_value = True - mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} - mock_conn.commit.side_effect = api_exc + events = [] + mock_conn = self._make_batch_ddl_mock_conn(events) + mock_conn.connection.commit.side_effect = api_exc with self.assertRaises(expected_django_exc): - with DatabaseSchemaEditor(mock_conn): - pass + with DatabaseSchemaEditor(mock_conn) as editor: + editor.execute( + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None + ) - mock_conn.set_autocommit.assert_called_with(True) + mock_conn._set_autocommit.assert_called_with(True) From fd57ab74da5804e53a79c98227caa06fa5f426aa Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 13 Nov 2026 17:53:35 +0000 Subject: [PATCH 6/9] Addressed comments --- .../django_spanner/creation.py | 2 +- .../tests/mockserver_tests/test_basics.py | 4 +- .../unit/django_spanner/test_creation.py | 45 ------------------- 3 files changed, 2 insertions(+), 49 deletions(-) delete mode 100644 packages/django-google-spanner/tests/unit/django_spanner/test_creation.py diff --git a/packages/django-google-spanner/django_spanner/creation.py b/packages/django-google-spanner/django_spanner/creation.py index 2998ab38a733..219240195c5f 100644 --- a/packages/django-google-spanner/django_spanner/creation.py +++ b/packages/django-google-spanner/django_spanner/creation.py @@ -48,7 +48,7 @@ def create_test_db(self, *args, **kwargs): # by a developer running the tests locally. if os.environ.get("RUNNING_SPANNER_BACKEND_TESTS") == "1": self.mark_skips() - return super().create_test_db(*args, **kwargs) + super().create_test_db(*args, **kwargs) def _create_test_db(self, verbosity, autoclobber, keepdb=False): """ diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index 081377cec685..da79dfa3cb8f 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -542,10 +542,8 @@ def fake_call_command(name, *args, **kwargs): create_kwargs = {"verbosity": 0, "autoclobber": True} if django.VERSION < (6, 0): create_kwargs["serialize"] = False - db_name = connection.creation.create_test_db(**create_kwargs) + connection.creation.create_test_db(**create_kwargs) - self.assertEqual(db_name, "django_test_db") - self.assertFalse(hasattr(connection, "_batch_ddl_in_schema_editor")) ddl_requests = self.database_admin_service.requests self.assertEqual(len(ddl_requests), 1) # sites (table + unique index) + sessions (table + expire_date index) = 4 DDLs diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py b/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py deleted file mode 100644 index 6388fe0e88ab..000000000000 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_creation.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2026 Google LLC -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd - -import os -from unittest import mock - -from django.db.backends.base.creation import BaseDatabaseCreation - -from django_spanner.creation import DatabaseCreation -from tests.unit.django_spanner.simple_test import SpannerSimpleTestClass - - -class TestCreation(SpannerSimpleTestClass): - def test_create_test_db_returns_db_name_without_private_flag(self): - creation = DatabaseCreation(self.connection) - - with mock.patch.object( - BaseDatabaseCreation, - "create_test_db", - return_value="test_database_id", - ): - result = creation.create_test_db(verbosity=0, autoclobber=True) - - self.assertEqual(result, "test_database_id") - self.assertFalse(hasattr(self.connection, "_batch_ddl_in_schema_editor")) - - def test_create_test_db_marks_skips_when_env_set(self): - creation = DatabaseCreation(self.connection) - - with ( - mock.patch.dict(os.environ, {"RUNNING_SPANNER_BACKEND_TESTS": "1"}), - mock.patch.object(creation, "mark_skips") as mock_mark_skips, - mock.patch.object( - BaseDatabaseCreation, - "create_test_db", - return_value="test_database_id", - ), - ): - result = creation.create_test_db(verbosity=0, autoclobber=True) - - self.assertEqual(result, "test_database_id") - mock_mark_skips.assert_called_once_with() From 937152146f3b6b7df89de776e8ba159fe0868eb9 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Mon, 16 Nov 2026 07:06:48 +0000 Subject: [PATCH 7/9] Addressed comments --- .../django_spanner/base.py | 11 -- .../django_spanner/schema.py | 154 ++++++++-------- .../tests/mockserver_tests/test_basics.py | 10 +- .../tests/unit/django_spanner/test_schema.py | 170 +++++++++--------- 4 files changed, 171 insertions(+), 174 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/base.py b/packages/django-google-spanner/django_spanner/base.py index 9a1bffd38233..cec94539d018 100644 --- a/packages/django-google-spanner/django_spanner/base.py +++ b/packages/django-google-spanner/django_spanner/base.py @@ -210,17 +210,6 @@ def create_cursor(self, name=None): """ return self.connection.cursor() - def set_autocommit( - self, autocommit, force_begin_transaction_with_broken_autocommit=False - ): - editor = getattr(self, "_active_ddl_batch_editor", None) - if editor is not None and editor._ddl_batch_active: - editor._exit_ddl_batch(clean=True) - super().set_autocommit( - autocommit, - force_begin_transaction_with_broken_autocommit=force_begin_transaction_with_broken_autocommit, - ) - def _set_autocommit(self, autocommit): """Set the Spanner transaction autocommit flag. diff --git a/packages/django-google-spanner/django_spanner/schema.py b/packages/django-google-spanner/django_spanner/schema.py index 3d2ff8748331..699321bbc199 100644 --- a/packages/django-google-spanner/django_spanner/schema.py +++ b/packages/django-google-spanner/django_spanner/schema.py @@ -73,92 +73,102 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_create_inline_fk = None def __init__(self, connection, collect_sql=False, atomic=True): - self._atomic = atomic super().__init__(connection, collect_sql=collect_sql, atomic=atomic) - - def _commit_ddls(self): - try: - self.connection.connection.commit() - except (AlreadyExists, FailedPrecondition, OutOfRange) as exc: - raise IntegrityError(getattr(exc, "details", exc)) from exc - except InvalidArgument as exc: - raise ProgrammingError(getattr(exc, "details", exc)) from exc - except (InternalServerError, GoogleAPICallError) as exc: - raise OperationalError(getattr(exc, "details", exc)) from exc - - def _enter_ddl_batch(self): - if not self._ddl_batch_active: - self.connection.ensure_connection() - self.connection._set_autocommit(False) - self._ddl_batch_active = True - - def _exit_ddl_batch(self, clean): - if not self._ddl_batch_active: - return - try: - if clean: - self._commit_ddls() - else: - self.connection.connection.rollback() - finally: - self._ddl_batch_active = False - self.connection._set_autocommit(True) - - def _ddl_execution_wrapper(self, execute, sql, params, many, context): - is_ddl = False - if ( - not many - and sql is not None - and not self.connection.in_atomic_block - and self.connection.get_autocommit() - ): - parsed = parse_utils.classify_statement(str(sql)) - is_ddl = parsed is not None and parsed.statement_type == StatementType.DDL - - if is_ddl: - self._enter_ddl_batch() - elif self._ddl_batch_active: - self._exit_ddl_batch(clean=True) - - return execute(sql, params, many, context) + # BaseDatabaseSchemaEditor sets self.atomic = False because Spanner sets + # can_rollback_ddl = False. Preserve the caller's atomic flag to know + # whether DDL batching is permitted for this schema editor session. + self.batch_ddl = bool( + atomic + and not collect_sql + and self.connection.settings_dict.get("OPTIONS", {}).get("batch_ddl", False) + ) def __enter__(self): super().__enter__() - self._ddl_batch_active = False - self._execute_wrapper_cm = None - batch_ddl = self.connection.settings_dict.get("OPTIONS", {}).get( - "batch_ddl", False - ) - if ( - batch_ddl - and getattr(self, "_atomic", True) + # Disable DDL batching if the connection is already inside an atomic + # block or manual transaction mode when entering the schema editor. + self.batch_ddl = ( + self.batch_ddl and not self.collect_sql and not self.connection.in_atomic_block and self.connection.get_autocommit() - and getattr(self.connection, "_active_ddl_batch_editor", None) is None - ): - self.connection._active_ddl_batch_editor = self - self._execute_wrapper_cm = self.connection.execute_wrapper( - self._ddl_execution_wrapper - ) - self._execute_wrapper_cm.__enter__() + ) return self def __exit__(self, exc_type, exc_value, traceback): - clean_exit = False + has_exception = exc_type is not None try: - res = super().__exit__(exc_type, exc_value, traceback) - clean_exit = (exc_type is None) or bool(res) - return res + # Executes any statements queued in self.deferred_sql via self.execute(). + return super().__exit__(exc_type, exc_value, traceback) + except BaseException: + has_exception = True + raise finally: + if self.batch_ddl: + if has_exception: + if ( + self.connection.connection is not None + and not self.connection.connection.autocommit + ): + try: + # Discard buffered DDL statements in DB-API before + # restoring autocommit=True (otherwise setting + # autocommit=True would commit them). + self.connection.connection.rollback() + finally: + self.connection.connection.autocommit = True + else: + # Flush all buffered DDL statements in a single batch. + self.flush_ddl_batch() + + def execute(self, sql, params=()): + # Bypass DDL batching when disabled, collecting SQL for sqlmigrate, + # or running inside an explicit transaction block. + if ( + not self.batch_ddl + or self.collect_sql + or self.connection.in_atomic_block + or not self.connection.get_autocommit() + ): + return super().execute(sql, params) + + parsed = parse_utils.classify_statement(str(sql)) + if parsed is None or parsed.statement_type != StatementType.DDL: + # Flush any prior buffered DDLs before running a non-DDL statement + # (e.g., an UPDATE setting default values during add_field). + self.flush_ddl_batch() + return super().execute(sql, params) + + # Turn off DB-API autocommit before executing DDL so spanner_dbapi + # buffers consecutive DDL statements instead of running each immediately. + self.connection.ensure_connection() + if self.connection.connection.autocommit: + self.connection.connection.autocommit = False + return super().execute(sql, params) + + def flush_ddl_batch(self): + """Flush buffered DDL statements by restoring DB-API autocommit=True.""" + if ( + self.connection.connection is not None + and not self.connection.connection.autocommit + ): try: - self._exit_ddl_batch(clean=clean_exit) + # Transitioning autocommit from False to True calls commit() in + # DB-API, which flushes all buffered DDL statements and clears + # _ddl_statements in its finally block. + self.connection.connection.autocommit = True + except (AlreadyExists, FailedPrecondition, OutOfRange) as exc: + raise IntegrityError(getattr(exc, "details", exc)) from exc + except InvalidArgument as exc: + raise ProgrammingError(getattr(exc, "details", exc)) from exc + except (InternalServerError, GoogleAPICallError) as exc: + raise OperationalError(getattr(exc, "details", exc)) from exc finally: - if getattr(self, "_execute_wrapper_cm", None) is not None: - self._execute_wrapper_cm.__exit__(exc_type, exc_value, traceback) - self._execute_wrapper_cm = None - if getattr(self.connection, "_active_ddl_batch_editor", None) is self: - self.connection._active_ddl_batch_editor = None + # If commit() raised inside the autocommit setter before + # _autocommit was updated to True, restore autocommit=True + # (_ddl_statements is already cleared by DB-API commit()). + if not self.connection.connection.autocommit: + self.connection.connection.autocommit = True def create_model(self, model): """ diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index da79dfa3cb8f..2a33121ce9f5 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -691,6 +691,11 @@ def test_migration_create_model_then_runpython_flushes_ddl_before_atomic(self): events = [] def run_atomic_dml(apps, schema_editor): + with schema_editor.connection.cursor() as cursor: + cursor.execute( + "UPDATE tests_singer SET first_name=%s WHERE id=%s", + ["updated", 1], + ) events.append( ( "runpython_state", @@ -699,11 +704,6 @@ def run_atomic_dml(apps, schema_editor): len(self.database_admin_service.requests), ) ) - with schema_editor.connection.cursor() as cursor: - cursor.execute( - "UPDATE tests_singer SET first_name=%s WHERE id=%s", - ["updated", 1], - ) mig = Migration("0001_create_then_runpython", "tests") mig.operations = [ diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py index 1987f3ef2d8d..539129230795 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py @@ -518,50 +518,44 @@ def test_skip_default_generated_and_db_default(self): normal_field = mock.MagicMock(generated=False, db_default=None) self.assertTrue(schema_editor.skip_default(normal_field)) - def _make_batch_ddl_mock_conn(self, events, batch_ddl=True): - import contextlib + def _make_batch_ddl_mock_conn( + self, events, batch_ddl=True, flush_exc=None, rollback_exc=None + ): + class FakeDBAPIConn: + def __init__(self): + self._ac = True + self.flush_exc = flush_exc + self.rollback_exc = rollback_exc + + @property + def autocommit(self): + return self._ac + + @autocommit.setter + def autocommit(self, value): + events.append(("autocommit", value)) + if value and not self._ac and self.flush_exc is not None: + exc = self.flush_exc + self.flush_exc = None + raise exc + self._ac = value + + def rollback(self): + events.append("rollback") + if self.rollback_exc is not None: + raise self.rollback_exc mock_conn = mock.MagicMock() mock_conn.in_atomic_block = False mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = True - mock_conn._active_ddl_batch_editor = None mock_conn.settings_dict = ( {"OPTIONS": {"batch_ddl": batch_ddl}} if batch_ddl is not None else {} ) - wrappers = [] - - @contextlib.contextmanager - def fake_execute_wrapper(wrapper): - wrappers.append(wrapper) - try: - yield - finally: - wrappers.remove(wrapper) - - mock_conn.execute_wrapper.side_effect = fake_execute_wrapper - mock_conn.connection.commit.side_effect = lambda: events.append("dbapi_commit") - mock_conn.connection.rollback.side_effect = lambda: events.append( - "dbapi_rollback" - ) - mock_conn._set_autocommit.side_effect = lambda val: events.append( - ("_set_autocommit", val) - ) + mock_conn.connection = FakeDBAPIConn() def run_cursor_execute(sql, params=None): - def base_exec(s, p, many, ctx): - events.append(("cursor_execute", s)) - - def make_wrapped_executor(w, nxt): - def wrapped(s, p, m, c): - return w(nxt, s, p, m, c) - - return wrapped - - executor = base_exec - for wrapper in reversed(wrappers): - executor = make_wrapped_executor(wrapper, executor) - return executor(sql, params, False, {"connection": mock_conn}) + events.append(("cursor_execute", sql)) cursor_cm = mock.MagicMock() cursor_cm.__enter__.return_value.execute.side_effect = run_cursor_execute @@ -569,29 +563,29 @@ def wrapped(s, p, m, c): return mock_conn def test_schema_editor_batches_ddl_lazily_when_option_enabled(self): - """Lazily toggles _set_autocommit(False) on DDL, commits, and restores _set_autocommit(True).""" + """Lazily sets connection.autocommit=False on DDL and restores connection.autocommit=True on exit.""" events = [] mock_conn = self._make_batch_ddl_mock_conn(events) with DatabaseSchemaEditor(mock_conn) as editor: - # Entering schema_editor alone does not disable autocommit before any DDL runs. self.assertEqual(events, []) editor.execute("CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None) + self.assertFalse(mock_conn.connection.autocommit) editor.deferred_sql.append("CREATE INDEX idx_t1 ON t1 (id)") self.assertEqual( events, [ - ("_set_autocommit", False), + ("autocommit", False), ( "cursor_execute", "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", ), ("cursor_execute", "CREATE INDEX idx_t1 ON t1 (id)"), - "dbapi_commit", - ("_set_autocommit", True), + ("autocommit", True), ], ) + self.assertTrue(mock_conn.connection.autocommit) mock_conn.set_autocommit.assert_not_called() def test_schema_editor_noop_without_ddl_does_not_toggle_autocommit(self): @@ -603,11 +597,11 @@ def test_schema_editor_noop_without_ddl_does_not_toggle_autocommit(self): editor.execute("SELECT 1", None) self.assertEqual(events, [("cursor_execute", "SELECT 1")]) - mock_conn._set_autocommit.assert_not_called() + self.assertTrue(mock_conn.connection.autocommit) mock_conn.set_autocommit.assert_not_called() def test_schema_editor_flushes_ddl_before_non_ddl_and_resumes_batching(self): - """Flushes queued DDLs and restores DB-API autocommit before DML/DQL, then resumes on next DDL.""" + """Flushes queued DDLs via autocommit=True before DML/DQL, then resumes batching on next DDL.""" events = [] mock_conn = self._make_batch_ddl_mock_conn(events) @@ -620,21 +614,19 @@ def test_schema_editor_flushes_ddl_before_non_ddl_and_resumes_batching(self): self.assertEqual( events, [ - ("_set_autocommit", False), + ("autocommit", False), ("cursor_execute", "ALTER TABLE t1 ADD COLUMN c1 INT64"), - "dbapi_commit", - ("_set_autocommit", True), + ("autocommit", True), ("cursor_execute", "UPDATE t1 SET c1 = %s WHERE 1=1"), - ("_set_autocommit", False), + ("autocommit", False), ("cursor_execute", "ALTER TABLE t1 ALTER COLUMN c1 INT64 NOT NULL"), ("cursor_execute", "CREATE INDEX idx_t1_c1 ON t1 (c1)"), - "dbapi_commit", - ("_set_autocommit", True), + ("autocommit", True), ], ) def test_schema_editor_rolls_back_on_exception_when_batching(self): - """Rolls back queued DDLs on DB-API connection before restoring autocommit on exception.""" + """Calls connection.rollback() and restores connection.autocommit=True on exception.""" events = [] mock_conn = self._make_batch_ddl_mock_conn(events) @@ -648,18 +640,19 @@ def test_schema_editor_rolls_back_on_exception_when_batching(self): self.assertEqual( events, [ - ("_set_autocommit", False), + ("autocommit", False), ( "cursor_execute", "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", ), - "dbapi_rollback", - ("_set_autocommit", True), + "rollback", + ("autocommit", True), ], ) + self.assertTrue(mock_conn.connection.autocommit) def test_schema_editor_rolls_back_on_deferred_sql_exception_when_batching(self): - """Rolls back when super().__exit__() raises Exception or BaseException.""" + """Calls connection.rollback() and restores autocommit=True when super().__exit__() raises.""" for exc_cls in (RuntimeError, KeyboardInterrupt): events = [] mock_conn = self._make_batch_ddl_mock_conn(events) @@ -682,26 +675,23 @@ def fail_on_index(sql, params=None, exc_cls=exc_cls, orig=editor.execute): self.assertEqual( events, [ - ("_set_autocommit", False), + ("autocommit", False), ( "cursor_execute", "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", ), - "dbapi_rollback", - ("_set_autocommit", True), + "rollback", + ("autocommit", True), ], ) + self.assertTrue(mock_conn.connection.autocommit) def test_schema_editor_restores_autocommit_when_commit_raises(self): - """Restores _set_autocommit(True) even if DB-API commit() raises a DDL error.""" + """Restores connection.autocommit=True even if setting autocommit=True raises.""" events = [] - mock_conn = self._make_batch_ddl_mock_conn(events) - - def failing_commit(): - events.append("dbapi_commit") - raise RuntimeError("server DDL error") - - mock_conn.connection.commit.side_effect = failing_commit + mock_conn = self._make_batch_ddl_mock_conn( + events, flush_exc=RuntimeError("server DDL error") + ) with self.assertRaises(RuntimeError): with DatabaseSchemaEditor(mock_conn) as editor: @@ -712,18 +702,19 @@ def failing_commit(): self.assertEqual( events, [ - ("_set_autocommit", False), + ("autocommit", False), ( "cursor_execute", "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", ), - "dbapi_commit", - ("_set_autocommit", True), + ("autocommit", True), + ("autocommit", True), ], ) + self.assertTrue(mock_conn.connection.autocommit) def test_schema_editor_skips_autocommit_toggle_by_default(self): - """Does not install wrapper or toggle autocommit when OPTIONS['batch_ddl'] is False or unset.""" + """Sets batch_ddl=False when OPTIONS['batch_ddl'] is False or unset.""" for settings_dict in ( {}, {"OPTIONS": {}}, @@ -735,11 +726,8 @@ def test_schema_editor_skips_autocommit_toggle_by_default(self): mock_conn.get_autocommit.return_value = True mock_conn.settings_dict = settings_dict - with DatabaseSchemaEditor(mock_conn): - pass - - mock_conn.execute_wrapper.assert_not_called() - mock_conn._set_autocommit.assert_not_called() + with DatabaseSchemaEditor(mock_conn) as editor: + self.assertFalse(editor.batch_ddl) def test_schema_editor_skips_autocommit_toggle_for_collect_sql_or_atomic_or_manual( self, @@ -756,11 +744,8 @@ def test_schema_editor_skips_autocommit_toggle_for_collect_sql_or_atomic_or_manu mock_conn.get_autocommit.return_value = is_autocommit mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} - with DatabaseSchemaEditor(mock_conn, collect_sql=collect_sql): - pass - - mock_conn.execute_wrapper.assert_not_called() - mock_conn._set_autocommit.assert_not_called() + with DatabaseSchemaEditor(mock_conn, collect_sql=collect_sql) as editor: + self.assertFalse(editor.batch_ddl) def test_schema_editor_skips_autocommit_toggle_when_atomic_false(self): """Skips batching when schema_editor(atomic=False) is requested.""" @@ -770,14 +755,11 @@ def test_schema_editor_skips_autocommit_toggle_when_atomic_false(self): mock_conn.get_autocommit.return_value = True mock_conn.settings_dict = {"OPTIONS": {"batch_ddl": True}} - with DatabaseSchemaEditor(mock_conn, atomic=False): - pass - - mock_conn.execute_wrapper.assert_not_called() - mock_conn._set_autocommit.assert_not_called() + with DatabaseSchemaEditor(mock_conn, atomic=False) as editor: + self.assertFalse(editor.batch_ddl) def test_schema_editor_maps_google_api_errors_to_django_database_errors(self): - """Maps GoogleAPICallError subclasses on commit() to django.db DatabaseErrors.""" + """Maps GoogleAPICallError subclasses on autocommit=True flush to django.db DatabaseErrors.""" from django.db import IntegrityError, OperationalError, ProgrammingError from google.api_core.exceptions import ( FailedPrecondition, @@ -792,8 +774,7 @@ def test_schema_editor_maps_google_api_errors_to_django_database_errors(self): ] for api_exc, expected_django_exc in cases: events = [] - mock_conn = self._make_batch_ddl_mock_conn(events) - mock_conn.connection.commit.side_effect = api_exc + mock_conn = self._make_batch_ddl_mock_conn(events, flush_exc=api_exc) with self.assertRaises(expected_django_exc): with DatabaseSchemaEditor(mock_conn) as editor: @@ -801,4 +782,21 @@ def test_schema_editor_maps_google_api_errors_to_django_database_errors(self): "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None ) - mock_conn._set_autocommit.assert_called_with(True) + self.assertTrue(mock_conn.connection.autocommit) + + def test_schema_editor_restores_autocommit_when_rollback_raises(self): + """Restores connection.autocommit=True via finally even if rollback() raises.""" + events = [] + mock_conn = self._make_batch_ddl_mock_conn( + events, + rollback_exc=RuntimeError("rollback error"), + ) + + with self.assertRaisesRegex(RuntimeError, "rollback error"): + with DatabaseSchemaEditor(mock_conn) as editor: + editor.execute( + "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", None + ) + raise ValueError("ddl failure") + + self.assertTrue(mock_conn.connection.autocommit) From ce40c6269d9757ae16af1756783c5daa06549b2a Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Mon, 16 Nov 2026 07:35:06 +0000 Subject: [PATCH 8/9] Addressed comments --- .../django_spanner/base.py | 11 ++ .../django_spanner/schema.py | 67 +++++++---- .../tests/mockserver_tests/test_basics.py | 113 ++++++++++++++++++ .../tests/unit/django_spanner/test_schema.py | 20 +++- 4 files changed, 188 insertions(+), 23 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/base.py b/packages/django-google-spanner/django_spanner/base.py index cec94539d018..fc34f501fad6 100644 --- a/packages/django-google-spanner/django_spanner/base.py +++ b/packages/django-google-spanner/django_spanner/base.py @@ -217,6 +217,12 @@ def _set_autocommit(self, autocommit): :param autocommit: The new value of the autocommit flag. """ with self.wrap_database_errors: + if not autocommit and not self.connection.autocommit: + # If DatabaseSchemaEditor set DB-API autocommit=False to buffer + # DDL statements while Django's autocommit remained True, flush + # the buffered DDLs by restoring DB-API autocommit=True before + # disabling autocommit via transaction.set_autocommit(False). + self.connection.autocommit = True self.connection.autocommit = autocommit async def _a_set_autocommit(self, autocommit): @@ -245,6 +251,11 @@ def _start_transaction_under_autocommit(self): """ Start a transaction explicitly in autocommit mode. """ + if not self.connection.autocommit: + # Flush any DDLs buffered by DatabaseSchemaEditor before starting + # an explicit transaction block (e.g., transaction.atomic()). + with self.wrap_database_errors: + self.connection.autocommit = True if self.allow_transactions_in_auto_commit: self.connection.cursor().execute("BEGIN") else: diff --git a/packages/django-google-spanner/django_spanner/schema.py b/packages/django-google-spanner/django_spanner/schema.py index 699321bbc199..c98abbd42076 100644 --- a/packages/django-google-spanner/django_spanner/schema.py +++ b/packages/django-google-spanner/django_spanner/schema.py @@ -82,6 +82,7 @@ def __init__(self, connection, collect_sql=False, atomic=True): and not collect_sql and self.connection.settings_dict.get("OPTIONS", {}).get("batch_ddl", False) ) + self.execute_wrapper_cm = None def __enter__(self): super().__enter__() @@ -93,6 +94,16 @@ def __enter__(self): and not self.connection.in_atomic_block and self.connection.get_autocommit() ) + if self.batch_ddl: + # Register a connection-level execute_wrapper so that all SQL run + # inside the schema_editor context—including direct ORM queries + # (e.g., Model.objects.create()) and introspection queries that + # bypass schema_editor.execute()—flushes any buffered DDLs before + # executing non-DDL statements. + self.execute_wrapper_cm = self.connection.execute_wrapper( + self.ddl_batch_wrapper + ) + self.execute_wrapper_cm.__enter__() return self def __exit__(self, exc_type, exc_value, traceback): @@ -104,24 +115,30 @@ def __exit__(self, exc_type, exc_value, traceback): has_exception = True raise finally: - if self.batch_ddl: - if has_exception: - if ( - self.connection.connection is not None - and not self.connection.connection.autocommit - ): - try: - # Discard buffered DDL statements in DB-API before - # restoring autocommit=True (otherwise setting - # autocommit=True would commit them). - self.connection.connection.rollback() - finally: - self.connection.connection.autocommit = True - else: - # Flush all buffered DDL statements in a single batch. - self.flush_ddl_batch() + try: + if self.batch_ddl: + if has_exception: + if ( + self.connection.connection is not None + and not self.connection.connection.autocommit + ): + try: + # Discard buffered DDL statements in DB-API before + # restoring autocommit=True (otherwise setting + # autocommit=True would commit them). + self.connection.connection.rollback() + finally: + self.connection.connection.autocommit = True + else: + # Flush all buffered DDL statements in a single batch. + self.flush_ddl_batch() + finally: + if self.execute_wrapper_cm is not None: + self.execute_wrapper_cm.__exit__(exc_type, exc_value, traceback) + self.execute_wrapper_cm = None - def execute(self, sql, params=()): + def ddl_batch_wrapper(self, execute, sql, params, many, context): + """Wrap cursor execution to batch consecutive DDL statements.""" # Bypass DDL batching when disabled, collecting SQL for sqlmigrate, # or running inside an explicit transaction block. if ( @@ -130,21 +147,27 @@ def execute(self, sql, params=()): or self.connection.in_atomic_block or not self.connection.get_autocommit() ): - return super().execute(sql, params) + return execute(sql, params, many, context) - parsed = parse_utils.classify_statement(str(sql)) + parsed = ( + parse_utils.classify_statement(str(sql)) + if not many and sql is not None + else None + ) if parsed is None or parsed.statement_type != StatementType.DDL: # Flush any prior buffered DDLs before running a non-DDL statement - # (e.g., an UPDATE setting default values during add_field). + # (e.g., an ORM query, introspection query, or UPDATE in add_field) + # so non-DDL statements run in autocommit=True mode without leaving + # an uncommitted Spanner read-write transaction open. self.flush_ddl_batch() - return super().execute(sql, params) + return execute(sql, params, many, context) # Turn off DB-API autocommit before executing DDL so spanner_dbapi # buffers consecutive DDL statements instead of running each immediately. self.connection.ensure_connection() if self.connection.connection.autocommit: self.connection.connection.autocommit = False - return super().execute(sql, params) + return execute(sql, params, many, context) def flush_ddl_batch(self): """Flush buffered DDL statements by restoring DB-API autocommit=True.""" diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index 2a33121ce9f5..4b6a6c836041 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -738,3 +738,116 @@ def run_atomic_dml(apps, schema_editor): ) self.assertTrue(transaction.get_autocommit()) self.assertTrue(connection.connection.autocommit) + + def test_schema_editor_orm_insert_between_create_model_and_add_field(self): + """Flushes buffered DDL and runs ORM insert in autocommit=True before subsequent add_field DDL.""" + add_single_result( + "INSERT INTO tests_singer " + "(id, first_name, last_name) " + "VALUES (@a0, @a1, @a2) " + "THEN RETURN id", + "id", + TypeCode.INT64, + [("1",)], + ) + add_update_count("UPDATE tests_singer SET stage_name=@a0 WHERE 1=1", 1) + + field = models.CharField(max_length=100, default="anon") + field.set_attributes_from_name("stage_name") + + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with connection.schema_editor() as editor: + editor.create_model(Singer) + # Direct ORM insert inside schema_editor block bypasses editor.execute() + # and must flush CREATE TABLE and commit its own insert in autocommit=True. + Singer.objects.create(first_name="Jane", last_name="Doe") + self.assertTrue(connection.connection.autocommit) + self.assertFalse(connection.connection._spanner_transaction_started) + editor.add_field(Singer, field) + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 3) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + def test_schema_editor_atomic_rollback_after_create_model_preserves_flushed_ddl( + self, + ): + """Flushes buffered DDL on entering transaction.atomic() so an atomic() rollback does not discard prior DDL.""" + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with connection.schema_editor() as editor: + editor.create_model(Singer) + # Entering transaction.atomic() must flush CREATE TABLE before BEGIN, + # so rolling back the atomic block before any SQL runs does not wipe out + # CREATE TABLE. + with self.assertRaises(RuntimeError): + with transaction.atomic(): + self.assertEqual(len(self.database_admin_service.requests), 1) + raise RuntimeError("abort atomic block before any SQL") + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 1) + self.assertEqual(len(ddl_requests[0].statements), 1) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + def test_schema_editor_manual_set_autocommit_false_after_create_model_flushes_ddl( + self, + ): + """Flushes buffered DDL on transaction.set_autocommit(False) so subsequent rollback() preserves prior DDL.""" + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with connection.schema_editor() as editor: + editor.create_model(Singer) + transaction.set_autocommit(False) + try: + self.assertEqual(len(self.database_admin_service.requests), 1) + transaction.rollback() + finally: + transaction.set_autocommit(True) + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + ddl_requests = self.database_admin_service.requests + self.assertEqual(len(ddl_requests), 1) + self.assertEqual(len(ddl_requests[0].statements), 1) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + def test_schema_editor_inside_outer_atomic_block_skips_batch_ddl(self): + """Disables batch_ddl when schema_editor is entered inside an outer transaction.atomic() block.""" + add_select1_result() + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with transaction.atomic(): + with connection.schema_editor() as editor: + self.assertFalse(editor.batch_ddl) + with connection.cursor() as cursor: + cursor.execute("select 1") + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + self.assertTrue( + any(isinstance(r, CommitRequest) for r in self.spanner_service.requests) + ) + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + def test_schema_editor_collect_sql_skips_batch_ddl(self): + """Disables batch_ddl and collects SQL without RPCs when collect_sql=True.""" + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with connection.schema_editor(collect_sql=True) as editor: + self.assertFalse(editor.batch_ddl) + editor.create_model(Singer) + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + self.assertEqual(len(self.database_admin_service.requests), 0) + self.assertEqual(len(editor.collected_sql), 1) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py index 539129230795..01109f71345b 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py @@ -553,10 +553,28 @@ def rollback(self): {"OPTIONS": {"batch_ddl": batch_ddl}} if batch_ddl is not None else {} ) mock_conn.connection = FakeDBAPIConn() + wrappers = [] - def run_cursor_execute(sql, params=None): + def fake_execute_wrapper(wrapper): + cm = mock.MagicMock() + cm.__enter__.side_effect = lambda: wrappers.append(wrapper) + cm.__exit__.side_effect = lambda *args: ( + wrappers.remove(wrapper) if wrapper in wrappers else None + ) + return cm + + mock_conn.execute_wrapper.side_effect = fake_execute_wrapper + + def base_execute(sql, params, many, context): events.append(("cursor_execute", sql)) + def run_cursor_execute(sql, params=None): + if wrappers: + return wrappers[-1]( + base_execute, sql, params, False, {"connection": mock_conn} + ) + return base_execute(sql, params, False, {"connection": mock_conn}) + cursor_cm = mock.MagicMock() cursor_cm.__enter__.return_value.execute.side_effect = run_cursor_execute mock_conn.cursor.return_value = cursor_cm From e24525568e9a09f5991f27bc918a9a31356751e9 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian Date: Fri, 20 Nov 2026 08:25:29 +0000 Subject: [PATCH 9/9] Addressed comments --- .../django_spanner/base.py | 41 +++++++++--- .../django_spanner/schema.py | 39 +---------- .../tests/mockserver_tests/test_basics.py | 66 +++++++++++++++++++ .../tests/unit/django_spanner/test_schema.py | 24 +++++-- 4 files changed, 120 insertions(+), 50 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/base.py b/packages/django-google-spanner/django_spanner/base.py index fc34f501fad6..884aa40aeb82 100644 --- a/packages/django-google-spanner/django_spanner/base.py +++ b/packages/django-google-spanner/django_spanner/base.py @@ -7,7 +7,16 @@ import os from asgiref.sync import sync_to_async +from django.db import IntegrityError, OperationalError, ProgrammingError from django.db.backends.base.base import BaseDatabaseWrapper +from google.api_core.exceptions import ( + AlreadyExists, + FailedPrecondition, + GoogleAPICallError, + InternalServerError, + InvalidArgument, + OutOfRange, +) from google.cloud import spanner, spanner_dbapi from .client import DatabaseClient @@ -210,19 +219,34 @@ def create_cursor(self, name=None): """ return self.connection.cursor() + def flush_ddl_batch(self): + """Flush any DDL statements buffered on the underlying DB-API connection.""" + if self.connection is not None and not self.connection.autocommit: + try: + self.connection.run_prior_DDL_statements() + except (AlreadyExists, FailedPrecondition, OutOfRange) as exc: + raise IntegrityError(getattr(exc, "details", exc)) from exc + except InvalidArgument as exc: + raise ProgrammingError(getattr(exc, "details", exc)) from exc + except (InternalServerError, GoogleAPICallError) as exc: + raise OperationalError(getattr(exc, "details", exc)) from exc + finally: + if self.autocommit: + self.connection.autocommit = True + def _set_autocommit(self, autocommit): """Set the Spanner transaction autocommit flag. :type autocommit: bool :param autocommit: The new value of the autocommit flag. """ + if not autocommit and self.autocommit and not self.connection.autocommit: + # If DatabaseSchemaEditor set DB-API autocommit=False to buffer + # DDL statements while Django's autocommit remained True, flush + # the buffered DDLs via run_prior_DDL_statements() before disabling + # autocommit via transaction.set_autocommit(False). + self.flush_ddl_batch() with self.wrap_database_errors: - if not autocommit and not self.connection.autocommit: - # If DatabaseSchemaEditor set DB-API autocommit=False to buffer - # DDL statements while Django's autocommit remained True, flush - # the buffered DDLs by restoring DB-API autocommit=True before - # disabling autocommit via transaction.set_autocommit(False). - self.connection.autocommit = True self.connection.autocommit = autocommit async def _a_set_autocommit(self, autocommit): @@ -251,11 +275,10 @@ def _start_transaction_under_autocommit(self): """ Start a transaction explicitly in autocommit mode. """ - if not self.connection.autocommit: + if self.autocommit and not self.connection.autocommit: # Flush any DDLs buffered by DatabaseSchemaEditor before starting # an explicit transaction block (e.g., transaction.atomic()). - with self.wrap_database_errors: - self.connection.autocommit = True + self.flush_ddl_batch() if self.allow_transactions_in_auto_commit: self.connection.cursor().execute("BEGIN") else: diff --git a/packages/django-google-spanner/django_spanner/schema.py b/packages/django-google-spanner/django_spanner/schema.py index c98abbd42076..20fe783c3787 100644 --- a/packages/django-google-spanner/django_spanner/schema.py +++ b/packages/django-google-spanner/django_spanner/schema.py @@ -7,22 +7,9 @@ import uuid import django -from django.db import ( - IntegrityError, - NotSupportedError, - OperationalError, - ProgrammingError, -) +from django.db import NotSupportedError from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django.db.models.fields import NOT_PROVIDED -from google.api_core.exceptions import ( - AlreadyExists, - FailedPrecondition, - GoogleAPICallError, - InternalServerError, - InvalidArgument, - OutOfRange, -) from google.cloud.spanner_dbapi import parse_utils from google.cloud.spanner_dbapi.parsed_statement import StatementType @@ -170,28 +157,8 @@ def ddl_batch_wrapper(self, execute, sql, params, many, context): return execute(sql, params, many, context) def flush_ddl_batch(self): - """Flush buffered DDL statements by restoring DB-API autocommit=True.""" - if ( - self.connection.connection is not None - and not self.connection.connection.autocommit - ): - try: - # Transitioning autocommit from False to True calls commit() in - # DB-API, which flushes all buffered DDL statements and clears - # _ddl_statements in its finally block. - self.connection.connection.autocommit = True - except (AlreadyExists, FailedPrecondition, OutOfRange) as exc: - raise IntegrityError(getattr(exc, "details", exc)) from exc - except InvalidArgument as exc: - raise ProgrammingError(getattr(exc, "details", exc)) from exc - except (InternalServerError, GoogleAPICallError) as exc: - raise OperationalError(getattr(exc, "details", exc)) from exc - finally: - # If commit() raised inside the autocommit setter before - # _autocommit was updated to True, restore autocommit=True - # (_ddl_statements is already cleared by DB-API commit()). - if not self.connection.connection.autocommit: - self.connection.connection.autocommit = True + """Flush buffered DDL statements via run_prior_DDL_statements() and restore autocommit.""" + self.connection.flush_ddl_batch() def create_model(self, model): """ diff --git a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py index 4b6a6c836041..efa12cbdb7a3 100644 --- a/packages/django-google-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/django-google-spanner/tests/mockserver_tests/test_basics.py @@ -851,3 +851,69 @@ def test_schema_editor_collect_sql_skips_batch_ddl(self): self.assertEqual(len(self.database_admin_service.requests), 0) self.assertEqual(len(editor.collected_sql), 1) + + def test_manual_set_autocommit_false_twice_does_not_commit(self): + """Calling transaction.set_autocommit(False) twice in manual transaction mode must not commit active DML.""" + add_single_result( + "INSERT INTO tests_singer " + "(id, first_name, last_name) " + "VALUES (@a0, @a1, @a2) " + "THEN RETURN id", + "id", + TypeCode.INT64, + [("1",)], + ) + transaction.set_autocommit(False) + try: + Singer.objects.create(first_name="Alice", last_name="Smith") + # Calling set_autocommit(False) again while already in manual transaction mode + transaction.set_autocommit(False) + transaction.rollback() + finally: + transaction.set_autocommit(True) + + self.assertFalse( + any(isinstance(r, CommitRequest) for r in self.spanner_service.requests) + ) + self.assertTrue( + any(isinstance(r, RollbackRequest) for r in self.spanner_service.requests) + ) + + def test_schema_editor_server_ddl_error_on_atomic_flush_raises_programming_error( + self, + ): + """Maps server DDL errors flushed on entering atomic() or set_autocommit(False) to Django DatabaseErrors.""" + from django.db import IntegrityError, ProgrammingError + from google.api_core.exceptions import FailedPrecondition, InvalidArgument + + connection.ensure_connection() + connection.settings_dict["OPTIONS"]["batch_ddl"] = True + try: + with mock.patch.object( + connection.connection.database, + "update_ddl", + side_effect=InvalidArgument("400 Invalid DDL syntax"), + ): + with self.assertRaises(ProgrammingError): + with connection.schema_editor() as editor: + editor.create_model(Singer) + with transaction.atomic(): + pass + + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) + + with mock.patch.object( + connection.connection.database, + "update_ddl", + side_effect=FailedPrecondition("400 Duplicate name in schema"), + ): + with self.assertRaises(IntegrityError): + with connection.schema_editor() as editor: + editor.create_model(Singer) + transaction.set_autocommit(False) + finally: + connection.settings_dict["OPTIONS"].pop("batch_ddl", None) + + self.assertTrue(transaction.get_autocommit()) + self.assertTrue(connection.connection.autocommit) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py index 01109f71345b..421637f467dd 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_schema.py @@ -521,6 +521,8 @@ def test_skip_default_generated_and_db_default(self): def _make_batch_ddl_mock_conn( self, events, batch_ddl=True, flush_exc=None, rollback_exc=None ): + from django_spanner.base import DatabaseWrapper + class FakeDBAPIConn: def __init__(self): self._ac = True @@ -534,11 +536,14 @@ def autocommit(self): @autocommit.setter def autocommit(self, value): events.append(("autocommit", value)) - if value and not self._ac and self.flush_exc is not None: + self._ac = value + + def run_prior_DDL_statements(self): + events.append("run_prior_DDL_statements") + if self.flush_exc is not None: exc = self.flush_exc self.flush_exc = None raise exc - self._ac = value def rollback(self): events.append("rollback") @@ -546,6 +551,7 @@ def rollback(self): raise self.rollback_exc mock_conn = mock.MagicMock() + mock_conn.autocommit = True mock_conn.in_atomic_block = False mock_conn.features.can_rollback_ddl = False mock_conn.get_autocommit.return_value = True @@ -553,6 +559,11 @@ def rollback(self): {"OPTIONS": {"batch_ddl": batch_ddl}} if batch_ddl is not None else {} ) mock_conn.connection = FakeDBAPIConn() + + def call_flush_ddl_batch(): + return DatabaseWrapper.flush_ddl_batch(mock_conn) + + mock_conn.flush_ddl_batch.side_effect = call_flush_ddl_batch wrappers = [] def fake_execute_wrapper(wrapper): @@ -600,6 +611,7 @@ def test_schema_editor_batches_ddl_lazily_when_option_enabled(self): "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", ), ("cursor_execute", "CREATE INDEX idx_t1 ON t1 (id)"), + "run_prior_DDL_statements", ("autocommit", True), ], ) @@ -619,7 +631,7 @@ def test_schema_editor_noop_without_ddl_does_not_toggle_autocommit(self): mock_conn.set_autocommit.assert_not_called() def test_schema_editor_flushes_ddl_before_non_ddl_and_resumes_batching(self): - """Flushes queued DDLs via autocommit=True before DML/DQL, then resumes batching on next DDL.""" + """Flushes queued DDLs via run_prior_DDL_statements() before DML/DQL, then resumes batching on next DDL.""" events = [] mock_conn = self._make_batch_ddl_mock_conn(events) @@ -634,11 +646,13 @@ def test_schema_editor_flushes_ddl_before_non_ddl_and_resumes_batching(self): [ ("autocommit", False), ("cursor_execute", "ALTER TABLE t1 ADD COLUMN c1 INT64"), + "run_prior_DDL_statements", ("autocommit", True), ("cursor_execute", "UPDATE t1 SET c1 = %s WHERE 1=1"), ("autocommit", False), ("cursor_execute", "ALTER TABLE t1 ALTER COLUMN c1 INT64 NOT NULL"), ("cursor_execute", "CREATE INDEX idx_t1_c1 ON t1 (c1)"), + "run_prior_DDL_statements", ("autocommit", True), ], ) @@ -705,7 +719,7 @@ def fail_on_index(sql, params=None, exc_cls=exc_cls, orig=editor.execute): self.assertTrue(mock_conn.connection.autocommit) def test_schema_editor_restores_autocommit_when_commit_raises(self): - """Restores connection.autocommit=True even if setting autocommit=True raises.""" + """Restores connection.autocommit=True even if run_prior_DDL_statements() raises.""" events = [] mock_conn = self._make_batch_ddl_mock_conn( events, flush_exc=RuntimeError("server DDL error") @@ -725,7 +739,7 @@ def test_schema_editor_restores_autocommit_when_commit_raises(self): "cursor_execute", "CREATE TABLE t1 (id INT64 NOT NULL) PRIMARY KEY(id)", ), - ("autocommit", True), + "run_prior_DDL_statements", ("autocommit", True), ], )