diff --git a/framework/celery_tasks/__init__.py b/framework/celery_tasks/__init__.py index ac68d10f192..ba1cb7b45f9 100644 --- a/framework/celery_tasks/__init__.py +++ b/framework/celery_tasks/__init__.py @@ -1,5 +1,6 @@ """Asynchronous task queue module.""" from celery import Celery +from celery.signals import beat_init, worker_init from celery.utils.log import get_task_logger from sentry_sdk import init, configure_scope @@ -8,6 +9,7 @@ from sentry_sdk.integrations.flask import FlaskIntegration from website.settings import SENTRY_DSN, VERSION, CeleryConfig +from framework.celery_tasks import checks app = Celery() app.config_from_object(CeleryConfig) @@ -24,6 +26,45 @@ scope.set_tag('App', 'celery') +@worker_init.connect +def check_routing(sender, **kwargs): + """Sent once the worker has imported CeleryConfig.imports, so the registry has every task. + """ + from framework import sentry + + tasks = checks.get_registered_tasks(sender.app) + for message, found in ( + ('Tasks sent to the default queue without a route naming it', + checks.find_tasks_sent_to_default_queue_implicitly(sender.app, tasks)), + ('Routes to queues missing from CeleryConfig.task_queues', + checks.find_routes_to_undeclared_queues(sender.app)), + ('Routes matching no registered task', + checks.find_routes_matching_no_task(sender.app, tasks)), + ('Tasks setting routing attributes instead of using CeleryConfig.task_routes', + checks.find_tasks_setting_routing_attributes(tasks)), + ): + if found: + sentry.log_message(f'[Celery Config Check] {message}', skip_session=True, extra_data={'found': found}) + + +@beat_init.connect +def check_periodic_tasks(sender, **kwargs): + """Sent once beat has imported CeleryConfig.imports and synced CeleryConfig.beat_schedule to the database, + so the enabled PeriodicTask rows are every entry beat will run. + """ + from django_celery_beat.models import PeriodicTask + from framework import sentry + + enabled = PeriodicTask.objects.enabled().values_list('name', 'task') + unregistered = checks.find_periodic_tasks_with_unregistered_task(sender.app, enabled) + if unregistered: + sentry.log_message( + '[Celery Config Check] Enabled periodic tasks running unregistered Celery tasks', + skip_session=True, + extra_data={'found': unregistered}, + ) + + @app.task def error_handler(task_id, task_name): """logs detailed message about tasks that raise exceptions diff --git a/framework/celery_tasks/checks.py b/framework/celery_tasks/checks.py new file mode 100644 index 00000000000..b408779de7d --- /dev/null +++ b/framework/celery_tasks/checks.py @@ -0,0 +1,64 @@ +"""Checks of the Celery routing and beat schedule, which framework.celery_tasks reports to Sentry when a worker or beat starts. + +They run against the settings of the environment the process starts in, which the tests in +osf_tests/test_celery_routing.py don't see: e.g. a deployed local.py or the periodic tasks stored in the database. +""" +from fnmatch import fnmatchcase + +# Set on the task itself (e.g. @app.task(queue=...)), these take precedence over task_routes +ROUTING_ATTRIBUTES = ('queue', 'exchange', 'routing_key') + + +def get_registered_tasks(app): + return {name: task for name, task in app.tasks.items() if not name.startswith('celery.')} + + +def find_tasks_sent_to_default_queue_implicitly(app, tasks): + # Celery silently sends a task to task_default_queue when no route matches it or its matching route + # is empty or names no queue, so a task may only end up there when its route names that queue + default_queue = app.conf.task_default_queue + router = app.amqp.router + implicit = [] + for name, task in sorted(tasks.items()): + # Resolve the queue like Task.apply_async() does: the task's own options merged over its route + queue = router.route(dict(task._get_exec_options()), name).get('queue') + route = router.lookup_route(name) or {} + if (queue is None or queue.name == default_queue) and route.get('queue') != default_queue: + implicit.append(name) + return implicit + + +def find_routes_to_undeclared_queues(app): + # A queue missing from task_queues is created on send (task_create_missing_queues), with no worker consuming it + declared = {queue.name for queue in app.conf.task_queues} + return { + entry: route.get('queue') + for entry, route in app.conf.task_routes.items() + if route.get('queue') not in declared + } + + +def find_routes_matching_no_task(app, tasks): + # Celery matches task_routes entries against task names as fnmatch globs, so an entry that + # matches no registered task routes nothing: the task is gone, renamed, or not in CeleryConfig.imports + return [ + entry for entry in app.conf.task_routes + if not any(fnmatchcase(name, entry) for name in tasks) + ] + + +def find_tasks_setting_routing_attributes(tasks): + # Only catches routing set on the task decorator (e.g. @app.task(queue=...)). Overrides passed at call time, + # e.g. task.apply_async(queue=...), live in the calling code and take precedence over both, but aren't visible here + overrides = {} + for name, task in sorted(tasks.items()): + attributes = {attr: getattr(task, attr) for attr in ROUTING_ATTRIBUTES if getattr(task, attr, None) is not None} + if attributes: + overrides[name] = attributes + return overrides + + +def find_periodic_tasks_with_unregistered_task(app, periodic_tasks): + # periodic_tasks are (entry name, task name) pairs, e.g. from CeleryConfig.beat_schedule or the database. + # Beat sends tasks by name, so otherwise an unregistered one is only noticed when a worker discards it + return {name: task for name, task in periodic_tasks if task not in app.tasks} diff --git a/osf_tests/test_celery_routing.py b/osf_tests/test_celery_routing.py index 5f5a3e5a07a..27efdbee528 100644 --- a/osf_tests/test_celery_routing.py +++ b/osf_tests/test_celery_routing.py @@ -1,13 +1,9 @@ -"""Tests for the Celery task routing configured in website.settings.CeleryConfig.""" +"""Tests for the Celery task routing and beat schedule configured in website.settings.CeleryConfig.""" import importlib -from fnmatch import fnmatchcase import pytest -from framework.celery_tasks import app as celery_app - -# Set on the task itself (e.g. @app.task(queue=...)), these take precedence over task_routes -ROUTING_ATTRIBUTES = ('queue', 'exchange', 'routing_key') +from framework.celery_tasks import app as celery_app, checks @pytest.fixture(scope='module') @@ -16,51 +12,33 @@ def registered_tasks(): # Not loader.import_default_modules(): Celery's Django fixup would also run the Django system checks for module in (*celery_app.conf.imports, *celery_app.conf.include): importlib.import_module(module) - return {name: task for name, task in celery_app.tasks.items() if not name.startswith('celery.')} + return checks.get_registered_tasks(celery_app) def test_every_task_has_a_route(registered_tasks): - # Celery silently sends a task to task_default_queue when no route matches it or its matching route - # is empty or names no queue, so a task may only end up there when its route names that queue - default_queue = celery_app.conf.task_default_queue - router = celery_app.amqp.router - implicit = [] - for name, task in sorted(registered_tasks.items()): - # Resolve the queue like Task.apply_async() does: the task's own options merged over its route - queue = router.route(dict(task._get_exec_options()), name).get('queue') - route = router.lookup_route(name) or {} - if (queue is None or queue.name == default_queue) and route.get('queue') != default_queue: - implicit.append(name) + implicit = checks.find_tasks_sent_to_default_queue_implicitly(celery_app, registered_tasks) assert not implicit, f'Tasks sent to the default queue without a route naming it: {implicit}' def test_every_route_targets_a_declared_queue(): - # A queue missing from task_queues is created on send (task_create_missing_queues), with no worker consuming it - declared = {queue.name for queue in celery_app.conf.task_queues} - undeclared = { - entry: route.get('queue', None) - for entry, route in celery_app.conf.task_routes.items() - if route.get('queue', None) not in declared - } + undeclared = checks.find_routes_to_undeclared_queues(celery_app) assert not undeclared, f'Routes to queues missing from CeleryConfig.task_queues: {undeclared}' def test_every_route_matches_a_registered_task(registered_tasks): - # Celery matches task_routes entries against task names as fnmatch globs, so an entry that - # matches no registered task routes nothing: the task is gone, renamed, or not in CeleryConfig.imports - unmatched = [ - entry for entry in celery_app.conf.task_routes - if not any(fnmatchcase(name, entry) for name in registered_tasks) - ] + unmatched = checks.find_routes_matching_no_task(celery_app, registered_tasks) assert not unmatched, f'Routes matching no registered task: {unmatched}' def test_no_task_sets_routing_attributes(registered_tasks): - # Only catches routing set on the task decorator (e.g. @app.task(queue=...)). Overrides passed at call time, - # e.g. task.apply_async(queue=...), live in the calling code and take precedence over both, but aren't visible here - overrides = {} - for name, task in sorted(registered_tasks.items()): - attributes = {attr: getattr(task, attr) for attr in ROUTING_ATTRIBUTES if getattr(task, attr, None) is not None} - if attributes: - overrides[name] = attributes + overrides = checks.find_tasks_setting_routing_attributes(registered_tasks) assert not overrides, f'Tasks setting routing attributes instead of using CeleryConfig.task_routes: {overrides}' + + +@pytest.mark.usefixtures('registered_tasks') +def test_every_beat_schedule_entry_runs_a_registered_task(): + # Checked against all of celery_app.tasks + # Entries may run Celery's built-in tasks, e.g. celery.backend_cleanup + entries = ((name, entry['task']) for name, entry in celery_app.conf.beat_schedule.items()) + unregistered = checks.find_periodic_tasks_with_unregistered_task(celery_app, entries) + assert not unregistered, f'CeleryConfig.beat_schedule entries running unregistered tasks: {unregistered}'