Skip to content
Open
24 changes: 24 additions & 0 deletions packages/django-google-spanner/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,30 @@ A transaction is automatically started if you define an
To disable transactions in autocommit mode, set the flag `ALLOW_TRANSACTIONS_IN_AUTO_COMMIT` to False in your ``settings.py`` file.


DDL batching
~~~~~~~~~~~~

By default, ``DatabaseSchemaEditor`` executes each DDL statement individually.
To batch DDL statements generated within a ``schema_editor()`` context (such as
during ``manage.py migrate`` or test database creation) into batched
``UpdateDatabaseDdl`` requests, set ``'batch_ddl': True`` in ``OPTIONS``
(defaults to ``False``):

.. code:: python

DATABASES = {
'default': {
'ENGINE': 'django_spanner',
'PROJECT': '$PROJECT',
'INSTANCE': '$INSTANCE',
'NAME': '$DATABASE',
'OPTIONS': {
'batch_ddl': True,
},
}
}


Set credentials and project environment variables
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You'll need to download a service account JSON key file and point to it using an environment variable:
Expand Down
35 changes: 35 additions & 0 deletions packages/django-google-spanner/django_spanner/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,16 @@
import os

from asgiref.sync import sync_to_async
from django.db import IntegrityError, OperationalError, ProgrammingError
from django.db.backends.base.base import BaseDatabaseWrapper
from google.api_core.exceptions import (
AlreadyExists,
FailedPrecondition,
GoogleAPICallError,
InternalServerError,
InvalidArgument,
OutOfRange,
)
from google.cloud import spanner, spanner_dbapi

from .client import DatabaseClient
Expand Down Expand Up @@ -182,6 +191,7 @@ def get_new_connection(self, conn_params):
conn_params.pop("instance", None)
conn_params.pop("instance_id", None)
conn_params.pop("client", None)
conn_params.pop("batch_ddl", None)
# Ensure client is initialized
instance = self.instance
return self.Database.connect(
Expand Down Expand Up @@ -209,12 +219,33 @@ def create_cursor(self, name=None):
"""
return self.connection.cursor()

def flush_ddl_batch(self):
"""Flush any DDL statements buffered on the underlying DB-API connection."""
if self.connection is not None and not self.connection.autocommit:
try:
self.connection.run_prior_DDL_statements()
except (AlreadyExists, FailedPrecondition, OutOfRange) as exc:
raise IntegrityError(getattr(exc, "details", exc)) from exc
except InvalidArgument as exc:
raise ProgrammingError(getattr(exc, "details", exc)) from exc
except (InternalServerError, GoogleAPICallError) as exc:
raise OperationalError(getattr(exc, "details", exc)) from exc
finally:
if self.autocommit:
self.connection.autocommit = True

def _set_autocommit(self, autocommit):
"""Set the Spanner transaction autocommit flag.

:type autocommit: bool
:param autocommit: The new value of the autocommit flag.
"""
if not autocommit and self.autocommit and not self.connection.autocommit:
# If DatabaseSchemaEditor set DB-API autocommit=False to buffer
# DDL statements while Django's autocommit remained True, flush
# the buffered DDLs via run_prior_DDL_statements() before disabling
# autocommit via transaction.set_autocommit(False).
self.flush_ddl_batch()
with self.wrap_database_errors:
self.connection.autocommit = autocommit

Expand Down Expand Up @@ -244,6 +275,10 @@ def _start_transaction_under_autocommit(self):
"""
Start a transaction explicitly in autocommit mode.
"""
if self.autocommit and not self.connection.autocommit:
# Flush any DDLs buffered by DatabaseSchemaEditor before starting
# an explicit transaction block (e.g., transaction.atomic()).
self.flush_ddl_batch()
if self.allow_transactions_in_auto_commit:
self.connection.cursor().execute("BEGIN")
else:
Expand Down
103 changes: 103 additions & 0 deletions packages/django-google-spanner/django_spanner/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
from django.db import NotSupportedError
from django.db.backends.base.schema import BaseDatabaseSchemaEditor
from django.db.models.fields import NOT_PROVIDED
from google.cloud.spanner_dbapi import parse_utils
from google.cloud.spanner_dbapi.parsed_statement import StatementType

from django_spanner._opentelemetry_tracing import trace_call

Expand Down Expand Up @@ -56,6 +58,107 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
# sql_create_inline_fk = "CONSTRAINT FK_%(to_table)s_%(to_column)s_%(from_table)s_%(from_column)s FOREIGN KEY (%(from_column_norm)s) REFERENCES %(to_table_norm)s (%(to_column_norm)s)" # noqa
sql_create_inline_fk = None

def __init__(self, connection, collect_sql=False, atomic=True):
super().__init__(connection, collect_sql=collect_sql, atomic=atomic)
# BaseDatabaseSchemaEditor sets self.atomic = False because Spanner sets
# can_rollback_ddl = False. Preserve the caller's atomic flag to know
# whether DDL batching is permitted for this schema editor session.
self.batch_ddl = bool(
atomic
and not collect_sql
and self.connection.settings_dict.get("OPTIONS", {}).get("batch_ddl", False)
)
self.execute_wrapper_cm = None

def __enter__(self):
super().__enter__()
# Disable DDL batching if the connection is already inside an atomic
# block or manual transaction mode when entering the schema editor.
self.batch_ddl = (
self.batch_ddl
and not self.collect_sql
and not self.connection.in_atomic_block
and self.connection.get_autocommit()
)
if self.batch_ddl:
# Register a connection-level execute_wrapper so that all SQL run
# inside the schema_editor context—including direct ORM queries
# (e.g., Model.objects.create()) and introspection queries that
# bypass schema_editor.execute()—flushes any buffered DDLs before
# executing non-DDL statements.
self.execute_wrapper_cm = self.connection.execute_wrapper(
self.ddl_batch_wrapper
)
self.execute_wrapper_cm.__enter__()
return self

def __exit__(self, exc_type, exc_value, traceback):
has_exception = exc_type is not None
try:
# Executes any statements queued in self.deferred_sql via self.execute().
return super().__exit__(exc_type, exc_value, traceback)
except BaseException:
has_exception = True
raise
finally:
try:
if self.batch_ddl:
if has_exception:
if (
self.connection.connection is not None
and not self.connection.connection.autocommit
):
try:
# Discard buffered DDL statements in DB-API before
# restoring autocommit=True (otherwise setting
# autocommit=True would commit them).
self.connection.connection.rollback()
finally:
self.connection.connection.autocommit = True
else:
# Flush all buffered DDL statements in a single batch.
self.flush_ddl_batch()
finally:
if self.execute_wrapper_cm is not None:
self.execute_wrapper_cm.__exit__(exc_type, exc_value, traceback)
self.execute_wrapper_cm = None

def ddl_batch_wrapper(self, execute, sql, params, many, context):
"""Wrap cursor execution to batch consecutive DDL statements."""
# Bypass DDL batching when disabled, collecting SQL for sqlmigrate,
# or running inside an explicit transaction block.
if (
not self.batch_ddl
or self.collect_sql
or self.connection.in_atomic_block
or not self.connection.get_autocommit()
):
return execute(sql, params, many, context)

parsed = (
parse_utils.classify_statement(str(sql))
if not many and sql is not None
else None
)
if parsed is None or parsed.statement_type != StatementType.DDL:
# Flush any prior buffered DDLs before running a non-DDL statement
# (e.g., an ORM query, introspection query, or UPDATE in add_field)
# so non-DDL statements run in autocommit=True mode without leaving
# an uncommitted Spanner read-write transaction open.
self.flush_ddl_batch()
return execute(sql, params, many, context)

# Turn off DB-API autocommit before executing DDL so spanner_dbapi
# buffers consecutive DDL statements instead of running each immediately.
self.connection.ensure_connection()
if self.connection.connection.autocommit:
self.connection.connection.autocommit = False
return execute(sql, params, many, context)

def flush_ddl_batch(self):
"""Flush buffered DDL statements via run_prior_DDL_statements() and restore autocommit."""
self.connection.flush_ddl_batch()

def create_model(self, model):
"""
Create a table and any accompanying indexes or unique constraints for
Expand Down
6 changes: 6 additions & 0 deletions packages/django-google-spanner/django_test_suite_5.2.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,18 @@ DATABASES = {
'PROJECT': "$PROJECT",
'INSTANCE': "$INSTANCE",
'NAME': "$TEST_DBNAME",
'OPTIONS': {
'batch_ddl': True,
},
},
'other': {
'ENGINE': 'django_spanner',
'PROJECT': "$PROJECT",
'INSTANCE': "$INSTANCE",
'NAME': "$TEST_DBNAME_OTHER",
'OPTIONS': {
'batch_ddl': True,
},
},
}
USE_TZ = False
Expand Down
6 changes: 6 additions & 0 deletions packages/django-google-spanner/django_test_suite_6.0.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,18 @@ DATABASES = {
'PROJECT': "$PROJECT",
'INSTANCE': "$INSTANCE",
'NAME': "$TEST_DBNAME",
'OPTIONS': {
'batch_ddl': True,
},
},
'other': {
'ENGINE': 'django_spanner',
'PROJECT': "$PROJECT",
'INSTANCE': "$INSTANCE",
'NAME': "$TEST_DBNAME_OTHER",
'OPTIONS': {
'batch_ddl': True,
},
},
}
USE_TZ = False
Expand Down
24 changes: 24 additions & 0 deletions packages/django-google-spanner/docs/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,30 @@ A transaction is automatically started if you define an
To disable transactions in autocommit mode, set the flag `ALLOW_TRANSACTIONS_IN_AUTO_COMMIT` to False in your ``settings.py`` file.


DDL batching
~~~~~~~~~~~~

By default, ``DatabaseSchemaEditor`` executes each DDL statement individually.
To batch DDL statements generated within a ``schema_editor()`` context (such as
during ``manage.py migrate`` or test database creation) into batched
``UpdateDatabaseDdl`` requests, set ``'batch_ddl': True`` in ``OPTIONS``
(defaults to ``False``):

.. code:: python

DATABASES = {
'default': {
'ENGINE': 'django_spanner',
'PROJECT': '$PROJECT',
'INSTANCE': '$INSTANCE',
'NAME': '$DATABASE',
'OPTIONS': {
'batch_ddl': True,
},
}
}


Set credentials and project environment variables
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You'll need to download a service account JSON key file and point to it using an environment variable:
Expand Down
9 changes: 5 additions & 4 deletions packages/django-google-spanner/docs/limitations-spanner.rst
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ it inherently. The workaround is to use python-spanner objects and run Mutations

Batch DDL
~~~~~~~~~
Migrations (creation and updation) of tables in Django are sequential operations.
Django doesn't inherently support doing batch DDL operations. However, customers
can use python-spanner objects to run batch DDL for unmanaged tables in Django.
By default, migrations (creation and update of tables) in Django execute DDL
statements sequentially. You can opt in to batching DDL statements within each
``schema_editor()`` context by setting ``'OPTIONS': {'batch_ddl': True}`` in
``DATABASES``, or use python-spanner objects directly for unmanaged tables.

Stale Reads
~~~~~~~~~~~
Expand Down Expand Up @@ -91,7 +92,7 @@ Workaround
* - Mutations
- Python-spanner database objects can be used to run Mutation queries. `Link <https://cloud.google.com/spanner/docs/modify-mutation-api#python>`__.
* - Batch DDL
- Using unmanaged tables in Django. `Link <https://docs.djangoproject.com/en/4.0/ref/models/options/#managed>`__ and Directly using Python spanner objects to execute batch DDL statements. `Link <https://cloud.google.com/spanner/docs/getting-started/python#create_a_database>`__.
- Set ``'OPTIONS': {'batch_ddl': True}`` in ``DATABASES`` to batch DDL statements within ``schema_editor()``, or use unmanaged tables (`Link <https://docs.djangoproject.com/en/5.2/ref/models/options/#managed>`__) with python-spanner objects directly (`Link <https://cloud.google.com/spanner/docs/getting-started/python#create_a_database>`__).
* - Stale Reads
- Python-spanner database objects can be used to perform stale reads. `Link <https://cloud.google.com/spanner/docs/reads#python>`__.
* - Interleaving
Expand Down
3 changes: 3 additions & 0 deletions packages/django-google-spanner/foreign_key_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ DATABASES = {
'PROJECT': '$GOOGLE_CLOUD_PROJECT',
'INSTANCE': '$SPANNER_TEST_INSTANCE',
'NAME': 'test_foreign_keys',
'OPTIONS': {
'batch_ddl': True,
},
}
}
" >> foreign_keys/settings.py
Expand Down
2 changes: 1 addition & 1 deletion packages/django-google-spanner/noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
]

UNIT_TEST_MOCKSERVER_DEPENDENCIES = [
"google-cloud-spanner>=3.70.0",
"google-cloud-spanner>=3.72.0",
"sqlparse>=0.4.4",
]

Expand Down
2 changes: 1 addition & 1 deletion packages/django-google-spanner/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
# (django_spanner/__init__.py currently enforces SUPPORTED_DJANGO_VERSIONS = [(6, 0), (5, 2)])
dependencies = [
"sqlparse >= 0.3.0",
"google-cloud-spanner >= 3.70.0",
"google-cloud-spanner >= 3.72.0",
"django >= 5.2, < 6.1",
]
extras = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,14 @@
class MockSpanner:
def __init__(self):
self.results = {}
self.errors = {}

def add_result(self, sql: str, result: result_set.ResultSet):
self.results[sql.lower().strip()] = result

def add_error(self, sql: str, code: grpc.StatusCode, details: str):
self.errors[sql.lower().strip()] = (code, details)

def get_result(self, sql: str) -> result_set.ResultSet:
result = self.results.get(sql.lower().strip())
if result is None:
Expand Down Expand Up @@ -135,6 +139,9 @@ def ExecuteStreamingSql(self, request: ExecuteSqlRequest, context):
started_transaction = self.__create_transaction(
request.session, request.transaction.begin
)
error = self.mock_spanner.errors.get(request.sql.lower().strip())
if error is not None:
context.abort(error[0], error[1])
partials = self.mock_spanner.get_result_as_partial_result_sets(
request.sql, started_transaction
)
Expand Down
Loading
Loading