OpenConceptLab/ocl_issues#927 Redis clustering

Author: rkorytkowski

core/common/backends.py

@@ -1,7 +1,21 @@
+from celery_once.backends import Redis
 from django.conf import settings
 from django.contrib.auth.backends import ModelBackend
 from mozilla_django_oidc.auth import OIDCAuthenticationBackend
 from pydash import get
+from redis import Sentinel
+
+
+class QueueOnceRedisSentinelBackend(Redis):
+    def __init__(self, backend_settings):
+        self._sentinel = Sentinel(backend_settings['sentinels'])
+        self._sentinel_master = backend_settings['sentinels_master']
+        self.blocking_timeout = backend_settings.get("blocking_timeout", 1)
+        self.blocking = backend_settings.get("blocking", False)
+
+    @property
+    def redis(self):
+        return self._sentinel.master_for(self._sentinel_master)
 
 
 class OCLOIDCAuthenticationBackend(OIDCAuthenticationBackend):

core/settings.py

@@ -306,12 +306,27 @@
 REDIS_HOST = os.environ.get('REDIS_HOST', 'redis')
 REDIS_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}"
 
+REDIS_SENTINELS = os.environ.get('REDIS_SENTINELS', None)
+REDIS_SENTINELS_MASTER = os.environ.get('REDIS_SENTINELS_MASTER', 'default')
+REDIS_SENTINELS_LIST = []
+
 # django cache
 if ENV and ENV not in ['ci']:
+    OPTIONS = {}
+    if REDIS_SENTINELS:
+        for REDIS_SENTINEL in REDIS_SENTINELS.split(','):
+            SENTINEL = REDIS_SENTINEL.split(':')
+            REDIS_SENTINELS_LIST.append((SENTINEL[0], int(SENTINEL[1])))
+        OPTIONS.update({
+            'CLIENT_CLASS': 'django_redis.client.SentinelClient',
+            'SENTINELS': REDIS_SENTINELS_LIST
+        })
+
     CACHES = {
         'default': {
-            'BACKEND': 'django.core.cache.backends.redis.RedisCache',
+            'BACKEND': 'django_redis.cache.RedisCache',
             'LOCATION': REDIS_URL,
+            'OPTIONS': OPTIONS
         }
     }
 
@@ -337,25 +352,44 @@
     'core.common.tasks.populate_indexes': {'queue': 'indexing'},
     'core.common.tasks.rebuild_indexes': {'queue': 'indexing'}
 }
-CELERY_RESULT_BACKEND = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
-CELERY_RESULT_EXTENDED = True
+
+CELERY_BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 259200}  # 72 hours, the longest ETA
 CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS = {
     'retry_policy': {
         'timeout': 10.0
     }
 }
+
+if REDIS_SENTINELS:
+    CELERY_RESULT_BACKEND = ''
+    for REDIS_SENTINEL in REDIS_SENTINELS.split(','):
+        CELERY_RESULT_BACKEND = CELERY_RESULT_BACKEND + f'sentinel://{REDIS_SENTINEL}/0;'
+        CELERY_BROKER_TRANSPORT_OPTIONS.update({'master_name': REDIS_SENTINELS_MASTER})
+        CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS.update({'master_name': REDIS_SENTINELS_MASTER})
+else:
+    CELERY_RESULT_BACKEND = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+
+CELERY_RESULT_EXTENDED = True
 CELERY_RESULT_EXPIRES = 259200  # 72 hours
 CELERY_BROKER_URL = CELERY_RESULT_BACKEND
 CELERY_BROKER_POOL_LIMIT = 50  # should be adjusted considering the number of threads
 CELERY_BROKER_CONNECTION_TIMEOUT = 10.0
-CELERY_BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 259200}  # 72 hours, the lon
 CELERY_ACCEPT_CONTENT = ['application/json']
-CELERY_ONCE = {
-    'backend': 'celery_once.backends.Redis',
-    'settings': {
-        'url': CELERY_RESULT_BACKEND,
+if REDIS_SENTINELS:
+    CELERY_ONCE = {
+        'backend': 'core.common.backends.QueueOnceRedisSentinelBackend',
+        'settings': {
+            'sentinels': REDIS_SENTINELS_LIST,
+            'sentinels_master': REDIS_SENTINELS_MASTER
+        }
+    }
+else:
+    CELERY_ONCE = {
+        'backend': 'celery_once.backends.Redis',
+        'settings': {
+            'url': CELERY_RESULT_BACKEND,
+        }
     }
-}
 CELERYBEAT_SCHEDULE = {
     'healthcheck-every-minute': {
         'task': 'core.common.tasks.beat_healthcheck',

docker-compose.yml

@@ -12,10 +12,14 @@ services:
     volumes:
       - postgres-data:/usr/share/postgres/data
   redis:
-    image: redis:6.0.6-alpine
+    image: bitnami/redis:7.0.12
     restart: "always"
+    environment:
+      - ALLOW_EMPTY_PASSWORD=yes
     healthcheck:
       test: "redis-cli ping"
+    volumes:
+      - redis-data:/bitnami/redis/data
   api:
     image: openconceptlab/oclapi2:${ENVIRONMENT-production}
     restart: "always"
@@ -171,3 +175,4 @@ services:
 volumes:
   postgres-data:
   es-data:
+  redis-data:

requirements.txt

@@ -18,6 +18,7 @@ celery[redis]==5.2.7
 celery_once==3.0.1
 git+https://github.com/snyaggarwal/flower # use until https://github.com/mher/flower/issues/1231 issue is resolved
 redis==4.5.5
+django-redis==5.3.0 # needed for redis sentinel support
 kombu==5.3.1
 django-elasticsearch-dsl==7.3
 drf-yasg==1.21.5