Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 113 additions & 7 deletions admin/base/celery_panel.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,30 @@
from collections import defaultdict
from datetime import timedelta
from fnmatch import fnmatchcase
from functools import cache

from django.contrib import messages
from django.shortcuts import redirect, render
from django.http import HttpResponseBadRequest
from django.core.paginator import Paginator
from django.db import transaction
from django.db.models import Count, Q
from django.forms import formset_factory
from django.shortcuts import get_object_or_404, redirect, render
from django.utils import timezone
from django.views.decorators.http import require_http_methods
from dj_celery_panel import views
from dj_celery_panel.celery_utils import CeleryQueuesInspectBackend, QueueDetailPage, QueueListPage
from dj_celery_panel.conf import panel_config

from framework.celery_tasks import app as celery_app
from osf.models.celery_test import CeleryTestSuite, CeleryTestTask
from osf.celery_stress_test.control import cancel_suite, finish_publishing
from osf.celery_stress_test.metrics import suite_metrics
from osf.celery_stress_test.scenarios import reconcile_suites
from osf.celery_stress_test.tasks import start_celery_stress_test
from .forms import CeleryStressTestForm, WorkloadForm
from django.forms import formset_factory
from .forms import CeleryStressTestForm, WorkloadForm, CeleryScenarioForm
from osf.celery_stress_test.presets import PRESETS, resolve
from website import settings


WorkloadFormSet = formset_factory(
Expand Down Expand Up @@ -136,6 +148,7 @@ def get_unmatched_routing_entries_by_queue(self):
return unmatched_entries


@transaction.non_atomic_requests
@panel_config.permission_required('celery_stress_test')
@require_http_methods(['GET', 'POST'])
def celery_stress_test_view(request):
Expand All @@ -146,6 +159,32 @@ def celery_stress_test_view(request):
request,
current_tab='celery_stress_test',
)
preset = request.POST.get('preset', request.GET.get('preset', 'load'))
if preset not in PRESETS:
return HttpResponseBadRequest('Unknown suite preset.')
if preset != 'load':
form = CeleryScenarioForm(request.POST if request.method == 'POST' else None,
suite_type=preset, initial=PRESETS[preset]['defaults'])
if request.method == 'POST' and form.is_valid():
config = resolve(preset, form.cleaned_data)
if preset == 'workflow':
try:
celery_app.backend.ensure_chords_allowed()
except (NotImplementedError, AttributeError) as exc:
form.add_error(None, f'This result backend does not support chords: {exc}')
if not form.errors:
suite = CeleryTestSuite.objects.create(name=config['name'], config=config, created_by=request.user,
deadline_at=timezone.now() + timedelta(seconds=config['deadline_seconds']))
try:
start_celery_stress_test.apply_async(args=[str(suite.id)],
task_id=str(suite.coordinator_task_id), retry=False)
except Exception as exc:
finish_publishing(suite.id, f'Coordinator publication failed: {exc}')
messages.error(request, 'Failed to submit the suite.')
return redirect('celery_stress_suite_detail', suite_id=suite.id)
context.update({'form': form, 'presets': PRESETS, 'selected_preset': preset})
return render(request, 'admin/dj_celery_panel/scenario_testing.html', context)
context.update({'presets': PRESETS, 'selected_preset': preset})

if request.method == 'POST':
form = CeleryStressTestForm(request.POST)
Expand All @@ -154,6 +193,8 @@ def celery_stress_test_view(request):
if form.is_valid() and workload_formset.is_valid():
config = {
**form.cleaned_data,
'suite_type': 'load',
'preset_version': 1,
'workloads': [
workload_form.cleaned_data
for workload_form in workload_formset
Expand All @@ -167,20 +208,32 @@ def celery_stress_test_view(request):
request,
'Please configure at least one workload.',
)
elif config['total_duration'] * min(
config['max_total_rate'],
sum(workload['rate'] for workload in config['workloads']),
) > settings.CELERY_STRESS_TEST_MAX_TASKS:
form.add_error(None, f'This run exceeds the {settings.CELERY_STRESS_TEST_MAX_TASKS:,} task limit.')
else:
suite = CeleryTestSuite.objects.create(
name=config['name'], config=config, created_by=request.user,
deadline_at=timezone.now() + timedelta(seconds=config['total_duration'] + config['drain_timeout'] + 60),
)
Comment thread
Ostap-Zherebetskyi marked this conversation as resolved.
Resolved
try:
result = start_celery_stress_test.delay(config)
except Exception:
start_celery_stress_test.apply_async(
args=[str(suite.id)], task_id=str(suite.coordinator_task_id), retry=False,
)
except Exception as error:
finish_publishing(suite.id, f'Coordinator publication failed: {error}')
messages.error(
request,
'Failed to start the stress test. Please try again.',
)
else:
messages.success(
request,
f"Stress test submitted successfully. Task ID: {result.id}",
f"Stress test submitted successfully: {suite.name}",
)
return redirect(request.path)
return redirect('celery_stress_suite_detail', suite_id=suite.id)
else:
form = CeleryStressTestForm()
workload_formset = WorkloadFormSet()
Expand All @@ -195,3 +248,56 @@ def celery_stress_test_view(request):
'admin/dj_celery_panel/celery_stress_testing.html',
context,
)


@panel_config.permission_required('celery_stress_test')
@require_http_methods(['GET'])
def celery_stress_suites(request):
context = panel_config.get_context(request, current_tab='celery_stress_test')
suites = CeleryTestSuite.objects.annotate(
task_count=Count('tasks'),
succeeded_count=Count('tasks', filter=Q(tasks__status=CeleryTestTask.Status.SUCCEEDED)),
failed_count=Count('tasks', filter=Q(tasks__status=CeleryTestTask.Status.FAILED)),
)
context.update({
'active_suites': suites.filter(status__in=CeleryTestSuite.ACTIVE_STATUSES),
'history': Paginator(suites.exclude(status__in=CeleryTestSuite.ACTIVE_STATUSES), 25).get_page(request.GET.get('page')),
})
return render(request, 'admin/dj_celery_panel/stress_suites.html', context)


@panel_config.permission_required('celery_stress_test')
@require_http_methods(['GET'])
def celery_stress_suite_detail(request, suite_id):
suite = get_object_or_404(CeleryTestSuite, pk=suite_id)
context = panel_config.get_context(request, current_tab='celery_stress_test')
context.update({
'suite': suite,
'metrics': suite_metrics(suite),
'task_page': Paginator(suite.tasks.order_by('-submitted_at', '-id'), 50).get_page(request.GET.get('page')),
})
return render(request, 'admin/dj_celery_panel/stress_suite_detail.html', context)


@transaction.non_atomic_requests
@panel_config.permission_required('celery_stress_test')
@require_http_methods(['POST'])
def celery_stress_suite_cancel(request, suite_id):
get_object_or_404(CeleryTestSuite, pk=suite_id)
suite, warning = cancel_suite(suite_id, request.user, celery_app)
if warning:
messages.warning(request, warning)
elif suite.cancel_requested_at:
messages.success(request, 'Cancellation requested. Publishing stops; queued load tasks are revoked where possible and expire at the deadline. Running load tasks finish naturally.')
else:
messages.info(request, 'This suite has already finished.')
return redirect('celery_stress_suite_detail', suite_id=suite.id)


@transaction.non_atomic_requests
@panel_config.permission_required('celery_stress_test')
@require_http_methods(['POST'])
def celery_stress_suites_reconcile(request):
reconcile_suites.run()
messages.success(request, 'Checked overdue suites and reconciled their status.')
return redirect('celery_stress_test')
57 changes: 47 additions & 10 deletions admin/base/forms.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from django import forms
from django.core.validators import validate_email
from framework.celery_tasks import app as celery_app
from website import settings


class GuidForm(forms.Form):
Expand Down Expand Up @@ -27,6 +29,9 @@ class ArchiveRegistrationWithPigeonForm(forms.Form):


class CeleryStressTestForm(forms.Form):
name = forms.CharField(max_length=100, initial='Celery load test')
drain_timeout = forms.IntegerField(min_value=10, max_value=3600, initial=120,
help_text='Maximum time to wait for load tasks after publication ends.')
total_duration = forms.IntegerField(
label='Duration (seconds)',
min_value=1,
Expand All @@ -41,6 +46,11 @@ class CeleryStressTestForm(forms.Form):
)


def stress_test_queue_choices():
declared_queues = {queue.name for queue in celery_app.conf.task_queues}
return [(name, name) for name in settings.CELERY_STRESS_TEST_QUEUES if name in declared_queues]


class WorkloadForm(forms.Form):
name = forms.CharField(
max_length=100,
Expand All @@ -54,7 +64,7 @@ class WorkloadForm(forms.Form):
],
initial='io',
)
queue = forms.ChoiceField(choices=[])
queue = forms.ChoiceField(choices=stress_test_queue_choices)

rate = forms.IntegerField(
label='Task rate (tasks/second)',
Expand All @@ -77,15 +87,6 @@ class WorkloadForm(forms.Form):
required=False,
)

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)

allowed_queues = ('high', 'med', 'low', 'celery') # TODO: Replace with settings.CELERY_QUEUES

self.fields['queue'].choices = [
(queue, queue) for queue in allowed_queues
]

def clean(self):
cleaned_data = super().clean()
mode = cleaned_data.get('mode')
Expand All @@ -97,3 +98,39 @@ def clean(self):
self.add_error('cpu_iterations', 'Specify the CPU iteration count.')

return cleaned_data


class CeleryScenarioForm(forms.Form):
name = forms.CharField(max_length=100, initial='Celery correctness test')
queue = forms.ChoiceField(choices=stress_test_queue_choices)
deadline_seconds = forms.IntegerField(min_value=10, max_value=3600, initial=120)
retry_delay = forms.IntegerField(min_value=1, max_value=60, initial=2)
max_retries = forms.IntegerField(min_value=2, max_value=10, initial=3)
tasks_per_scenario = forms.IntegerField(min_value=1, max_value=100, initial=1,
help_text='Number of tasks for each of the three retry scenarios (3 × this value total).')
transient_failures = forms.IntegerField(min_value=1, max_value=10, initial=2,
help_text='Failures before a transient task succeeds; must not exceed maximum retries.')
workflow_count = forms.IntegerField(min_value=1, max_value=100, initial=1,
help_text='Number of independent chain, group and chord sets.')
chain_length = forms.IntegerField(min_value=2, max_value=100, initial=2)
group_size = forms.IntegerField(min_value=2, max_value=100, initial=3)
chord_size = forms.IntegerField(min_value=2, max_value=100, initial=3,
help_text='Header task count; each chord also has one final callback.')

def __init__(self, *args, suite_type, **kwargs):
super().__init__(*args, **kwargs)
excluded = (('workflow_count', 'chain_length', 'group_size', 'chord_size') if suite_type == 'retry'
else ('tasks_per_scenario', 'transient_failures', 'retry_delay', 'max_retries'))
for name in excluded:
del self.fields[name]

def clean(self):
data = super().clean()
if 'max_retries' in data and data.get('deadline_seconds', 0) <= data['max_retries'] * data.get('retry_delay', 0):
self.add_error('deadline_seconds', 'Allow more time than the configured retry delays.')
if data.get('transient_failures', 0) > data.get('max_retries', 0):
self.add_error('transient_failures', 'Must not exceed maximum retries.')
total = data.get('workflow_count', 0) * (data.get('chain_length', 0) + data.get('group_size', 0) + data.get('chord_size', 0) + 1)
if total > 500:
self.add_error('workflow_count', 'Choose sizes producing at most 500 tasks in total.')
return data
6 changes: 5 additions & 1 deletion admin/base/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@
re_path(r'^unprotected_views/$', views.unprotected_views, name='unprotected_views'),
re_path(r'^middleware_check/$', views.admin_auth_middleware_check, name='middleware_check'),
re_path(r'^admin/dj-celery-panel/$', celery_panel.overview),
re_path(r'^admin/dj-celery-panel/celery_stress_test/$', celery_panel.celery_stress_test_view, name='celery_stress_test'),
re_path(r'^admin/dj-celery-panel/celery_stress_test/$', celery_panel.celery_stress_suites, name='celery_stress_test'),
re_path(r'^admin/dj-celery-panel/celery_stress_test/reconcile/$', celery_panel.celery_stress_suites_reconcile, name='celery_stress_suites_reconcile'),
re_path(r'^admin/dj-celery-panel/celery_stress_test/new/$', celery_panel.celery_stress_test_view, name='celery_stress_test_new'),
re_path(r'^admin/dj-celery-panel/celery_stress_test/(?P<suite_id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/$', celery_panel.celery_stress_suite_detail, name='celery_stress_suite_detail'),
re_path(r'^admin/dj-celery-panel/celery_stress_test/(?P<suite_id>[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/cancel/$', celery_panel.celery_stress_suite_cancel, name='celery_stress_suite_cancel'),
re_path(r'^admin/dj-celery-panel/', include('dj_celery_panel.urls')),
re_path(r'^admin/', admin.site.urls),
re_path(r'^provider_asset_files/', include('admin.provider_asset_files.urls', namespace='provider_asset_files')),
Expand Down
Loading
Loading