From 984f435d0d10c45ad39f36906bc011f5e385656d Mon Sep 17 00:00:00 2001 From: P SRIKAR NAIDU Date: Sat, 7 Nov 2026 10:59:31 +0530 Subject: [PATCH 1/6] Add explicit database handle for sync access --- morango/sync/db.py | 95 +++++++++++++++++++++-------- tests/testapp/tests/helpers.py | 14 +++++ tests/testapp/tests/sync/test_db.py | 71 +++++++++++++++++++++ 3 files changed, 154 insertions(+), 26 deletions(-) diff --git a/morango/sync/db.py b/morango/sync/db.py index aa4f4c96..17ea9a99 100644 --- a/morango/sync/db.py +++ b/morango/sync/db.py @@ -1,38 +1,81 @@ -import logging from contextlib import contextmanager from django.db import connection +from django.db import connections +from django.db import DEFAULT_DB_ALIAS from django.db import transaction +from django.db.models import F +from django.utils.functional import cached_property +from morango.models.core import InstanceIDModel from morango.sync.backends.utils import load_backend from morango.sync.utils import lock_partitions -logger = logging.getLogger(__name__) -DBBackend = load_backend(connection) +class SyncDatabase: + """Database access for sync, with routed ORM access when no alias is supplied.""" + def __init__(self, alias=None): + self.alias = alias -@contextmanager -def begin_transaction(sync_filter, isolated=False, shared_lock=False): - """ - Starts a transaction, sets the transaction isolation level to repeatable read, and locks - affected partitions - - :param sync_filter: The filter for filtering applicable records of the sync - :type sync_filter: morango.models.certificates.Filter|None - :param isolated: Whether to alter the transaction isolation to repeatable-read - :type isolated: bool - :param shared_lock: Whether the advisory lock should be exclusive or shared - :type shared_lock: bool - """ - if isolated: - # when isolation is requested, we modify the transaction isolation of the connection for the - # duration of the transaction - with DBBackend._set_transaction_repeatable_read(): - with transaction.atomic(savepoint=False): - lock_partitions(DBBackend, sync_filter=sync_filter, shared=shared_lock) + @classmethod + def default(cls): + return cls() + + @property + def connection(self): + alias = DEFAULT_DB_ALIAS if self.alias is None else self.alias + return connections[alias] + + @cached_property + def backend(self): + # The default backend is also exposed as a module global. Django's + # connection proxy resolves the default connection on the calling thread. + return load_backend(connection if self.alias is None else self.connection) + + def manager(self, model): + manager = model._default_manager + if self.alias is None: + return manager + return manager.db_manager(self.alias) + + def atomic(self, savepoint=True): + return transaction.atomic(using=self.connection.alias, savepoint=savepoint) + + @contextmanager + def begin_transaction(self, sync_filter, isolated=False, shared_lock=False): + """Start a transaction and lock the affected partitions.""" + if isolated: + with self.backend._set_transaction_repeatable_read(): + with self.atomic(savepoint=False): + lock_partitions(self.backend, sync_filter=sync_filter, shared=shared_lock) + yield + else: + with self.atomic(): + lock_partitions(self.backend, sync_filter=sync_filter, shared=shared_lock) yield - else: - with transaction.atomic(): - lock_partitions(DBBackend, sync_filter=sync_filter, shared=shared_lock) - yield + + def cursor(self): + return self.connection.cursor() + + def current_instance_and_increment_counter(self): + """Increment the current instance, which must already exist on a pinned alias.""" + if self.alias is None: + return InstanceIDModel.get_current_instance_and_increment_counter() + + with self.atomic(): + instances = self.manager(InstanceIDModel) + instance = instances.get(current=True) + instances.filter(pk=instance.pk).update(counter=F("counter") + 1) + instance.refresh_from_db(using=self.alias, fields=["counter"]) + return instance + + +DBBackend = SyncDatabase.default().backend + + +def begin_transaction(sync_filter, isolated=False, shared_lock=False): + """Start a sync transaction on the default database.""" + return SyncDatabase.default().begin_transaction( + sync_filter, isolated=isolated, shared_lock=shared_lock + ) diff --git a/tests/testapp/tests/helpers.py b/tests/testapp/tests/helpers.py index 0a5a69d7..af8c4fdc 100644 --- a/tests/testapp/tests/helpers.py +++ b/tests/testapp/tests/helpers.py @@ -5,10 +5,13 @@ import json import socket import uuid +from contextlib import contextmanager import factory import mock from django.core.serializers.json import DjangoJSONEncoder +from django.db import connections +from django.db import DEFAULT_DB_ALIAS from django.test.testcases import LiveServerTestCase from django.utils import timezone from facility_profile.models import Facility @@ -36,6 +39,17 @@ from .compat import EnvironmentVarGuard +@contextmanager +def assert_no_default_db_queries(): + """Fail with the SQL if anything executes on the default connection.""" + + def reject_query(execute, sql, params, many, context): + raise AssertionError("Unexpected query on the default database: {}".format(sql)) + + with connections[DEFAULT_DB_ALIAS].execute_wrapper(reject_query): + yield + + def get_free_tcp_port(): # sandboxing can cause 0 to fail (which is to try for any open port), so fallback to a hardcoded # list afterward diff --git a/tests/testapp/tests/sync/test_db.py b/tests/testapp/tests/sync/test_db.py index 4882939f..cfffb63d 100644 --- a/tests/testapp/tests/sync/test_db.py +++ b/tests/testapp/tests/sync/test_db.py @@ -5,22 +5,93 @@ import pytest from django.conf import settings from django.db import connection +from django.db import connections +from django.db import transaction from django.test import override_settings +from django.test import TestCase from django.test import TransactionTestCase from django.utils import timezone from morango.models.certificates import Filter +from morango.models.core import DatabaseIDModel +from morango.models.core import InstanceIDModel from morango.models.core import Store from morango.models.core import SyncSession from morango.models.core import TransferSession +from morango.sync.backends.sqlite import SQLWrapper as SQLiteWrapper from morango.sync.backends.utils import load_backend from morango.sync.db import begin_transaction +from morango.sync.db import SyncDatabase +from ..helpers import assert_no_default_db_queries from ..helpers import create_buffer_and_store_dummy_data DBBackend = load_backend(connection) +class SyncDatabaseTestCase(TestCase): + def test_default_manager(self): + self.assertIs(SyncDatabase.default().manager(Store), Store._default_manager) + + def test_default_counter(self): + instance, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True) + counter = instance.counter + updated = SyncDatabase.default().current_instance_and_increment_counter() + self.assertEqual(updated.pk, instance.pk) + self.assertEqual(updated.counter, counter + 1) + + def test_guard_rejects_default_query(self): + with self.assertRaisesRegex(AssertionError, "SELECT COUNT"): + with assert_no_default_db_queries(): + Store.objects.using("default").count() + Store.objects.using("default").count() + + def test_guard_rejects_default_savepoint(self): + with self.assertRaisesRegex(AssertionError, "SAVEPOINT"): + with assert_no_default_db_queries(): + with transaction.atomic(): + pass + + +@pytest.mark.skipif("default2" not in settings.DATABASES, reason="requires a second database") +class PinnedSyncDatabaseTestCase(TransactionTestCase): + databases = {"default", "default2"} + + def test_pinned_manager_and_backend(self): + db = SyncDatabase("default2") + self.assertEqual(db.manager(Store).db, "default2") + self.assertIsInstance(db.backend, SQLiteWrapper) + self.assertIs(db.backend, db.backend) + self.assertIs(db.backend.connection, connections["default2"]) + + def test_pinned_transaction_and_cursor(self): + db = SyncDatabase("default2") + with assert_no_default_db_queries(), db.begin_transaction(None, isolated=True): + self.assertFalse(connection.in_atomic_block) + self.assertTrue(connections["default2"].in_atomic_block) + self.assertEqual(db.manager(Store).count(), 0) + with db.cursor() as cursor: + cursor.execute("SELECT 1") + self.assertEqual(cursor.fetchone(), (1,)) + self.assertFalse(connections["default2"].in_atomic_block) + + def test_pinned_counter_uses_alias_with_default_instance_cached(self): + default_instance, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True) + counter = default_instance.counter + instance = InstanceIDModel.objects.get(pk=default_instance.pk) + database = instance.database + db = SyncDatabase("default2") + # Seed a database identity without invoking the routed model save methods. + with assert_no_default_db_queries(): + db.manager(DatabaseIDModel).bulk_create([database]) + db.manager(InstanceIDModel).bulk_create([instance]) + updated = db.current_instance_and_increment_counter() + self.assertEqual(updated.counter, counter + 1) + self.assertEqual(db.manager(InstanceIDModel).get(pk=updated.pk).counter, counter + 1) + default_instance.refresh_from_db() + self.assertEqual(default_instance.counter, counter) + + def _concurrent_store_write(thread_event, store_id): while not thread_event.is_set(): sleep(0.1) From 75988a6c24a12c9fce3f9bb3929a1a2122111f1b Mon Sep 17 00:00:00 2001 From: P SRIKAR NAIDU Date: Sat, 7 Nov 2026 15:50:57 +0530 Subject: [PATCH 2/6] Address sync database handle review --- morango/sync/db.py | 22 +++++++++++------ tests/testapp/tests/helpers.py | 19 ++++++++++++--- tests/testapp/tests/sync/test_db.py | 38 +++++++++++++++++++++++------ 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/morango/sync/db.py b/morango/sync/db.py index 17ea9a99..214ff2ff 100644 --- a/morango/sync/db.py +++ b/morango/sync/db.py @@ -1,11 +1,11 @@ from contextlib import contextmanager +from threading import local from django.db import connection from django.db import connections from django.db import DEFAULT_DB_ALIAS from django.db import transaction from django.db.models import F -from django.utils.functional import cached_property from morango.models.core import InstanceIDModel from morango.sync.backends.utils import load_backend @@ -17,6 +17,7 @@ class SyncDatabase: def __init__(self, alias=None): self.alias = alias + self._backends = local() @classmethod def default(cls): @@ -27,11 +28,16 @@ def connection(self): alias = DEFAULT_DB_ALIAS if self.alias is None else self.alias return connections[alias] - @cached_property + @property def backend(self): - # The default backend is also exposed as a module global. Django's - # connection proxy resolves the default connection on the calling thread. - return load_backend(connection if self.alias is None else self.connection) + if self.alias is None: + return DBBackend + + conn = self.connection + backend = getattr(self._backends, "backend", None) + if backend is None or backend.connection is not conn: + backend = self._backends.backend = load_backend(conn) + return backend def manager(self, model): manager = model._default_manager @@ -71,11 +77,13 @@ def current_instance_and_increment_counter(self): return instance -DBBackend = SyncDatabase.default().backend +# Django's default connection proxy resolves the connection on the calling thread. +DBBackend = load_backend(connection) +_default = SyncDatabase.default() def begin_transaction(sync_filter, isolated=False, shared_lock=False): """Start a sync transaction on the default database.""" - return SyncDatabase.default().begin_transaction( + return _default.begin_transaction( sync_filter, isolated=isolated, shared_lock=shared_lock ) diff --git a/tests/testapp/tests/helpers.py b/tests/testapp/tests/helpers.py index af8c4fdc..ba23d18f 100644 --- a/tests/testapp/tests/helpers.py +++ b/tests/testapp/tests/helpers.py @@ -42,12 +42,23 @@ @contextmanager def assert_no_default_db_queries(): """Fail with the SQL if anything executes on the default connection.""" + rejected_queries = [] def reject_query(execute, sql, params, many, context): - raise AssertionError("Unexpected query on the default database: {}".format(sql)) - - with connections[DEFAULT_DB_ALIAS].execute_wrapper(reject_query): - yield + error = AssertionError("Unexpected query on the default database: {}".format(sql)) + rejected_queries.append(error) + raise error + + try: + with connections[DEFAULT_DB_ALIAS].execute_wrapper(reject_query): + yield + except Exception as error: + if not rejected_queries or error in rejected_queries: + raise + raise AssertionError("\n".join(map(str, rejected_queries))) from error + else: + if rejected_queries: + raise AssertionError("\n".join(map(str, rejected_queries))) def get_free_tcp_port(): diff --git a/tests/testapp/tests/sync/test_db.py b/tests/testapp/tests/sync/test_db.py index cfffb63d..f796ce22 100644 --- a/tests/testapp/tests/sync/test_db.py +++ b/tests/testapp/tests/sync/test_db.py @@ -1,5 +1,6 @@ import threading import uuid +from concurrent.futures import ThreadPoolExecutor from time import sleep import pytest @@ -52,6 +53,12 @@ def test_guard_rejects_default_savepoint(self): with transaction.atomic(): pass + def test_guard_reports_a_caught_query_violation(self): + with self.assertRaisesRegex(AssertionError, "SELECT COUNT"): + with assert_no_default_db_queries(): + with self.assertRaises(AssertionError): + Store.objects.using("default").count() + @pytest.mark.skipif("default2" not in settings.DATABASES, reason="requires a second database") class PinnedSyncDatabaseTestCase(TransactionTestCase): @@ -64,16 +71,31 @@ def test_pinned_manager_and_backend(self): self.assertIs(db.backend, db.backend) self.assertIs(db.backend.connection, connections["default2"]) + def test_pinned_backend_uses_calling_threads_connection(self): + db = SyncDatabase("default2") + main_backend = db.backend + + def get_worker_backend(): + backend = db.backend + self.assertIs(backend.connection, connections["default2"]) + return backend + + with ThreadPoolExecutor(max_workers=1) as executor: + self.assertIsNot(executor.submit(get_worker_backend).result(), main_backend) + self.assertIs(db.backend, main_backend) + def test_pinned_transaction_and_cursor(self): db = SyncDatabase("default2") - with assert_no_default_db_queries(), db.begin_transaction(None, isolated=True): - self.assertFalse(connection.in_atomic_block) - self.assertTrue(connections["default2"].in_atomic_block) - self.assertEqual(db.manager(Store).count(), 0) - with db.cursor() as cursor: - cursor.execute("SELECT 1") - self.assertEqual(cursor.fetchone(), (1,)) - self.assertFalse(connections["default2"].in_atomic_block) + for isolated in (False, True): + with self.subTest(isolated=isolated): + with assert_no_default_db_queries(), db.begin_transaction(None, isolated=isolated): + self.assertFalse(connection.in_atomic_block) + self.assertTrue(connections["default2"].in_atomic_block) + self.assertEqual(db.manager(Store).count(), 0) + with db.cursor() as cursor: + cursor.execute("SELECT 1") + self.assertEqual(cursor.fetchone(), (1,)) + self.assertFalse(connections["default2"].in_atomic_block) def test_pinned_counter_uses_alias_with_default_instance_cached(self): default_instance, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True) From 16b6853a2f4950e83b84ab69394b1faef31d5a1d Mon Sep 17 00:00:00 2001 From: P SRIKAR NAIDU Date: Sat, 7 Nov 2026 15:52:47 +0530 Subject: [PATCH 3/6] Apply Ruff formatting to sync database helper --- morango/sync/db.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/morango/sync/db.py b/morango/sync/db.py index 214ff2ff..f6913670 100644 --- a/morango/sync/db.py +++ b/morango/sync/db.py @@ -84,6 +84,4 @@ def current_instance_and_increment_counter(self): def begin_transaction(sync_filter, isolated=False, shared_lock=False): """Start a sync transaction on the default database.""" - return _default.begin_transaction( - sync_filter, isolated=isolated, shared_lock=shared_lock - ) + return _default.begin_transaction(sync_filter, isolated=isolated, shared_lock=shared_lock) From f4131c70d476c4647cc65a465812183d8fac4e64 Mon Sep 17 00:00:00 2001 From: P SRIKAR NAIDU Date: Mon, 9 Nov 2026 09:27:27 +0530 Subject: [PATCH 4/6] Address sync database documentation and test review --- morango/sync/db.py | 18 ++++++++++++++++-- tests/testapp/tests/helpers.py | 7 ++++++- tests/testapp/tests/sync/test_db.py | 1 + 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/morango/sync/db.py b/morango/sync/db.py index f6913670..3d503d02 100644 --- a/morango/sync/db.py +++ b/morango/sync/db.py @@ -50,7 +50,14 @@ def atomic(self, savepoint=True): @contextmanager def begin_transaction(self, sync_filter, isolated=False, shared_lock=False): - """Start a transaction and lock the affected partitions.""" + """ + Start a transaction and lock the affected partitions. + + :param sync_filter: The sync filter, or None to lock all partitions + :type sync_filter: morango.models.certificates.Filter | None + :param isolated: Whether to set repeatable-read isolation for the transaction + :param shared_lock: Whether to acquire shared rather than exclusive locks + """ if isolated: with self.backend._set_transaction_repeatable_read(): with self.atomic(savepoint=False): @@ -83,5 +90,12 @@ def current_instance_and_increment_counter(self): def begin_transaction(sync_filter, isolated=False, shared_lock=False): - """Start a sync transaction on the default database.""" + """ + Start a sync transaction on the default database and lock affected partitions. + + :param sync_filter: The sync filter, or None to lock all partitions + :type sync_filter: morango.models.certificates.Filter | None + :param isolated: Whether to set repeatable-read isolation for the transaction + :param shared_lock: Whether to acquire shared rather than exclusive locks + """ return _default.begin_transaction(sync_filter, isolated=isolated, shared_lock=shared_lock) diff --git a/tests/testapp/tests/helpers.py b/tests/testapp/tests/helpers.py index ba23d18f..09aad312 100644 --- a/tests/testapp/tests/helpers.py +++ b/tests/testapp/tests/helpers.py @@ -41,7 +41,12 @@ @contextmanager def assert_no_default_db_queries(): - """Fail with the SQL if anything executes on the default connection.""" + """Report SQL sent to the default connection by the calling thread. + + A top-level PostgreSQL transaction emits no SQL through Django's execute + wrapper. Check ``connection.in_atomic_block`` separately when verifying + which connection opened the transaction. + """ rejected_queries = [] def reject_query(execute, sql, params, many, context): diff --git a/tests/testapp/tests/sync/test_db.py b/tests/testapp/tests/sync/test_db.py index f796ce22..88031880 100644 --- a/tests/testapp/tests/sync/test_db.py +++ b/tests/testapp/tests/sync/test_db.py @@ -61,6 +61,7 @@ def test_guard_reports_a_caught_query_violation(self): @pytest.mark.skipif("default2" not in settings.DATABASES, reason="requires a second database") +@override_settings(DATABASE_ROUTERS=[]) class PinnedSyncDatabaseTestCase(TransactionTestCase): databases = {"default", "default2"} From 89403317f03ceeef26aea14a27c1060bbb310334 Mon Sep 17 00:00:00 2001 From: P SRIKAR NAIDU Date: Mon, 9 Nov 2026 09:27:31 +0530 Subject: [PATCH 5/6] Identify legacy instances by their original UUID --- morango/models/core.py | 6 +++++- tests/testapp/tests/test_uuid_utilities.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/morango/models/core.py b/morango/models/core.py index 75fa500d..0df6416a 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -34,6 +34,7 @@ from morango.models.fields.uuids import UUIDModelMixin from morango.models.fsic_utils import remove_redundant_instance_counters from morango.models.manager import SyncableModelManager +from morango.models.utils import _calculate_0_4_uuid from morango.models.utils import get_0_4_system_parameters from morango.models.utils import get_0_5_mac_address from morango.models.utils import get_0_5_system_id @@ -157,7 +158,10 @@ def get_or_create_current_instance(cls, clear_cache=False): database_id=DatabaseIDModel.get_or_create_current_database_id().id ) try: - instance = InstanceIDModel.objects.get(current=True, **kwargs) + # The legacy fields can also match a modern instance when no MAC is available. + instance = InstanceIDModel.objects.get( + current=True, id=_calculate_0_4_uuid(kwargs), **kwargs + ) cls._cached_instance = instance return instance, False except InstanceIDModel.DoesNotExist: diff --git a/tests/testapp/tests/test_uuid_utilities.py b/tests/testapp/tests/test_uuid_utilities.py index 58d7b393..b9f2da39 100644 --- a/tests/testapp/tests/test_uuid_utilities.py +++ b/tests/testapp/tests/test_uuid_utilities.py @@ -185,6 +185,21 @@ def test_envvar_overrides(self, *args): self.assertEqual(instance.id, expected_id) + @mock.patch("uuid.getnode", return_value=0x010000000000) + @mock.patch("ifcfg.interfaces", return_value={}) + def test_system_id_change_without_a_mac_address(self, *args): + with EnvironmentVarGuard() as env: + env["MORANGO_NODE_ID"] = "" + env["MORANGO_SYSTEM_ID"] = "initial-system" + initial, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True) + self.assertEqual(initial.node_id, "") + + env["MORANGO_SYSTEM_ID"] = "replacement-system" + replacement, created = InstanceIDModel.get_or_create_current_instance(clear_cache=True) + + self.assertTrue(created) + self.assertNotEqual(initial.id, replacement.id) + @mock.patch( "ifcfg.interfaces", return_value={"eth0": {"device": "eth0", "ether": "a0:aa:aa:aa:aa"}}, From fb59b94ceddf900d8e4a173a71f181c0170a1d9d Mon Sep 17 00:00:00 2001 From: P SRIKAR NAIDU Date: Tue, 10 Nov 2026 06:57:09 +0530 Subject: [PATCH 6/6] Revert "Identify legacy instances by their original UUID" This reverts commit 89403317f03ceeef26aea14a27c1060bbb310334. --- morango/models/core.py | 6 +----- tests/testapp/tests/test_uuid_utilities.py | 15 --------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/morango/models/core.py b/morango/models/core.py index 0df6416a..75fa500d 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -34,7 +34,6 @@ from morango.models.fields.uuids import UUIDModelMixin from morango.models.fsic_utils import remove_redundant_instance_counters from morango.models.manager import SyncableModelManager -from morango.models.utils import _calculate_0_4_uuid from morango.models.utils import get_0_4_system_parameters from morango.models.utils import get_0_5_mac_address from morango.models.utils import get_0_5_system_id @@ -158,10 +157,7 @@ def get_or_create_current_instance(cls, clear_cache=False): database_id=DatabaseIDModel.get_or_create_current_database_id().id ) try: - # The legacy fields can also match a modern instance when no MAC is available. - instance = InstanceIDModel.objects.get( - current=True, id=_calculate_0_4_uuid(kwargs), **kwargs - ) + instance = InstanceIDModel.objects.get(current=True, **kwargs) cls._cached_instance = instance return instance, False except InstanceIDModel.DoesNotExist: diff --git a/tests/testapp/tests/test_uuid_utilities.py b/tests/testapp/tests/test_uuid_utilities.py index b9f2da39..58d7b393 100644 --- a/tests/testapp/tests/test_uuid_utilities.py +++ b/tests/testapp/tests/test_uuid_utilities.py @@ -185,21 +185,6 @@ def test_envvar_overrides(self, *args): self.assertEqual(instance.id, expected_id) - @mock.patch("uuid.getnode", return_value=0x010000000000) - @mock.patch("ifcfg.interfaces", return_value={}) - def test_system_id_change_without_a_mac_address(self, *args): - with EnvironmentVarGuard() as env: - env["MORANGO_NODE_ID"] = "" - env["MORANGO_SYSTEM_ID"] = "initial-system" - initial, _ = InstanceIDModel.get_or_create_current_instance(clear_cache=True) - self.assertEqual(initial.node_id, "") - - env["MORANGO_SYSTEM_ID"] = "replacement-system" - replacement, created = InstanceIDModel.get_or_create_current_instance(clear_cache=True) - - self.assertTrue(created) - self.assertNotEqual(initial.id, replacement.id) - @mock.patch( "ifcfg.interfaces", return_value={"eth0": {"device": "eth0", "ether": "a0:aa:aa:aa:aa"}},