Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions framework/celery_tasks/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Asynchronous task queue module."""
from celery import Celery
from celery.signals import beat_init, worker_init
from celery.utils.log import get_task_logger

from sentry_sdk import init, configure_scope
Expand All @@ -8,6 +9,7 @@
from sentry_sdk.integrations.flask import FlaskIntegration

from website.settings import SENTRY_DSN, VERSION, CeleryConfig
from framework.celery_tasks import checks

app = Celery()
app.config_from_object(CeleryConfig)
Expand All @@ -24,6 +26,45 @@
scope.set_tag('App', 'celery')


@worker_init.connect
def check_routing(sender, **kwargs):
"""Sent once the worker has imported CeleryConfig.imports, so the registry has every task.
"""
from framework import sentry

tasks = checks.get_registered_tasks(sender.app)
for message, found in (
('Tasks sent to the default queue without a route naming it',
checks.find_tasks_sent_to_default_queue_implicitly(sender.app, tasks)),
('Routes to queues missing from CeleryConfig.task_queues',
checks.find_routes_to_undeclared_queues(sender.app)),
('Routes matching no registered task',
checks.find_routes_matching_no_task(sender.app, tasks)),
('Tasks setting routing attributes instead of using CeleryConfig.task_routes',
checks.find_tasks_setting_routing_attributes(tasks)),
):
if found:
sentry.log_message(f'[Celery Config Check] {message}', skip_session=True, extra_data={'found': found})


@beat_init.connect
def check_periodic_tasks(sender, **kwargs):
"""Sent once beat has imported CeleryConfig.imports and synced CeleryConfig.beat_schedule to the database,
so the enabled PeriodicTask rows are every entry beat will run.
"""
from django_celery_beat.models import PeriodicTask
from framework import sentry

enabled = PeriodicTask.objects.enabled().values_list('name', 'task')
unregistered = checks.find_periodic_tasks_with_unregistered_task(sender.app, enabled)
if unregistered:
sentry.log_message(
'[Celery Config Check] Enabled periodic tasks running unregistered Celery tasks',
skip_session=True,
extra_data={'found': unregistered},
)


@app.task
def error_handler(task_id, task_name):
"""logs detailed message about tasks that raise exceptions
Expand Down
64 changes: 64 additions & 0 deletions framework/celery_tasks/checks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
"""Checks of the Celery routing and beat schedule, which framework.celery_tasks reports to Sentry when a worker or beat starts.

They run against the settings of the environment the process starts in, which the tests in
osf_tests/test_celery_routing.py don't see: e.g. a deployed local.py or the periodic tasks stored in the database.
"""
from fnmatch import fnmatchcase

# Set on the task itself (e.g. @app.task(queue=...)), these take precedence over task_routes
ROUTING_ATTRIBUTES = ('queue', 'exchange', 'routing_key')


def get_registered_tasks(app):
return {name: task for name, task in app.tasks.items() if not name.startswith('celery.')}


def find_tasks_sent_to_default_queue_implicitly(app, tasks):
# Celery silently sends a task to task_default_queue when no route matches it or its matching route
# is empty or names no queue, so a task may only end up there when its route names that queue
default_queue = app.conf.task_default_queue
router = app.amqp.router
implicit = []
for name, task in sorted(tasks.items()):
# Resolve the queue like Task.apply_async() does: the task's own options merged over its route
queue = router.route(dict(task._get_exec_options()), name).get('queue')
route = router.lookup_route(name) or {}
if (queue is None or queue.name == default_queue) and route.get('queue') != default_queue:
implicit.append(name)
return implicit


def find_routes_to_undeclared_queues(app):
# A queue missing from task_queues is created on send (task_create_missing_queues), with no worker consuming it
declared = {queue.name for queue in app.conf.task_queues}
return {
entry: route.get('queue')
for entry, route in app.conf.task_routes.items()
if route.get('queue') not in declared
}


def find_routes_matching_no_task(app, tasks):
# Celery matches task_routes entries against task names as fnmatch globs, so an entry that
# matches no registered task routes nothing: the task is gone, renamed, or not in CeleryConfig.imports
return [
entry for entry in app.conf.task_routes
if not any(fnmatchcase(name, entry) for name in tasks)
]


def find_tasks_setting_routing_attributes(tasks):
# Only catches routing set on the task decorator (e.g. @app.task(queue=...)). Overrides passed at call time,
# e.g. task.apply_async(queue=...), live in the calling code and take precedence over both, but aren't visible here
overrides = {}
for name, task in sorted(tasks.items()):
attributes = {attr: getattr(task, attr) for attr in ROUTING_ATTRIBUTES if getattr(task, attr, None) is not None}
if attributes:
overrides[name] = attributes
return overrides


def find_periodic_tasks_with_unregistered_task(app, periodic_tasks):
# periodic_tasks are (entry name, task name) pairs, e.g. from CeleryConfig.beat_schedule or the database.
# Beat sends tasks by name, so otherwise an unregistered one is only noticed when a worker discards it
return {name: task for name, task in periodic_tasks if task not in app.tasks}
54 changes: 16 additions & 38 deletions osf_tests/test_celery_routing.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
"""Tests for the Celery task routing configured in website.settings.CeleryConfig."""
"""Tests for the Celery task routing and beat schedule configured in website.settings.CeleryConfig."""
import importlib
from fnmatch import fnmatchcase

import pytest

from framework.celery_tasks import app as celery_app

# Set on the task itself (e.g. @app.task(queue=...)), these take precedence over task_routes
ROUTING_ATTRIBUTES = ('queue', 'exchange', 'routing_key')
from framework.celery_tasks import app as celery_app, checks


@pytest.fixture(scope='module')
Expand All @@ -16,51 +12,33 @@ def registered_tasks():
# Not loader.import_default_modules(): Celery's Django fixup would also run the Django system checks
for module in (*celery_app.conf.imports, *celery_app.conf.include):
importlib.import_module(module)
return {name: task for name, task in celery_app.tasks.items() if not name.startswith('celery.')}
return checks.get_registered_tasks(celery_app)


def test_every_task_has_a_route(registered_tasks):
# Celery silently sends a task to task_default_queue when no route matches it or its matching route
# is empty or names no queue, so a task may only end up there when its route names that queue
default_queue = celery_app.conf.task_default_queue
router = celery_app.amqp.router
implicit = []
for name, task in sorted(registered_tasks.items()):
# Resolve the queue like Task.apply_async() does: the task's own options merged over its route
queue = router.route(dict(task._get_exec_options()), name).get('queue')
route = router.lookup_route(name) or {}
if (queue is None or queue.name == default_queue) and route.get('queue') != default_queue:
implicit.append(name)
implicit = checks.find_tasks_sent_to_default_queue_implicitly(celery_app, registered_tasks)
assert not implicit, f'Tasks sent to the default queue without a route naming it: {implicit}'


def test_every_route_targets_a_declared_queue():
# A queue missing from task_queues is created on send (task_create_missing_queues), with no worker consuming it
declared = {queue.name for queue in celery_app.conf.task_queues}
undeclared = {
entry: route.get('queue', None)
for entry, route in celery_app.conf.task_routes.items()
if route.get('queue', None) not in declared
}
undeclared = checks.find_routes_to_undeclared_queues(celery_app)
assert not undeclared, f'Routes to queues missing from CeleryConfig.task_queues: {undeclared}'


def test_every_route_matches_a_registered_task(registered_tasks):
# Celery matches task_routes entries against task names as fnmatch globs, so an entry that
# matches no registered task routes nothing: the task is gone, renamed, or not in CeleryConfig.imports
unmatched = [
entry for entry in celery_app.conf.task_routes
if not any(fnmatchcase(name, entry) for name in registered_tasks)
]
unmatched = checks.find_routes_matching_no_task(celery_app, registered_tasks)
assert not unmatched, f'Routes matching no registered task: {unmatched}'


def test_no_task_sets_routing_attributes(registered_tasks):
# Only catches routing set on the task decorator (e.g. @app.task(queue=...)). Overrides passed at call time,
# e.g. task.apply_async(queue=...), live in the calling code and take precedence over both, but aren't visible here
overrides = {}
for name, task in sorted(registered_tasks.items()):
attributes = {attr: getattr(task, attr) for attr in ROUTING_ATTRIBUTES if getattr(task, attr, None) is not None}
if attributes:
overrides[name] = attributes
overrides = checks.find_tasks_setting_routing_attributes(registered_tasks)
assert not overrides, f'Tasks setting routing attributes instead of using CeleryConfig.task_routes: {overrides}'


@pytest.mark.usefixtures('registered_tasks')
def test_every_beat_schedule_entry_runs_a_registered_task():
# Checked against all of celery_app.tasks
# Entries may run Celery's built-in tasks, e.g. celery.backend_cleanup
entries = ((name, entry['task']) for name, entry in celery_app.conf.beat_schedule.items())
unregistered = checks.find_periodic_tasks_with_unregistered_task(celery_app, entries)
assert not unregistered, f'CeleryConfig.beat_schedule entries running unregistered tasks: {unregistered}'
Loading