diff --git a/admin/base/celery_panel.py b/admin/base/celery_panel.py index 5a9c2927ce8..ba012d0a8c7 100644 --- a/admin/base/celery_panel.py +++ b/admin/base/celery_panel.py @@ -1,18 +1,30 @@ from collections import defaultdict +from datetime import timedelta from fnmatch import fnmatchcase from functools import cache from django.contrib import messages -from django.shortcuts import redirect, render +from django.http import HttpResponseBadRequest +from django.core.paginator import Paginator +from django.db import transaction +from django.db.models import Count, Q +from django.forms import formset_factory +from django.shortcuts import get_object_or_404, redirect, render +from django.utils import timezone from django.views.decorators.http import require_http_methods from dj_celery_panel import views from dj_celery_panel.celery_utils import CeleryQueuesInspectBackend, QueueDetailPage, QueueListPage from dj_celery_panel.conf import panel_config from framework.celery_tasks import app as celery_app +from osf.models.celery_test import CeleryTestSuite, CeleryTestTask +from osf.celery_stress_test.control import cancel_suite, finish_publishing +from osf.celery_stress_test.metrics import suite_metrics +from osf.celery_stress_test.scenarios import reconcile_suites from osf.celery_stress_test.tasks import start_celery_stress_test -from .forms import CeleryStressTestForm, WorkloadForm -from django.forms import formset_factory +from .forms import CeleryStressTestForm, WorkloadForm, CeleryScenarioForm +from osf.celery_stress_test.presets import PRESETS, resolve +from website import settings WorkloadFormSet = formset_factory( @@ -136,6 +148,7 @@ def get_unmatched_routing_entries_by_queue(self): return unmatched_entries +@transaction.non_atomic_requests @panel_config.permission_required('celery_stress_test') @require_http_methods(['GET', 'POST']) def celery_stress_test_view(request): @@ -146,6 +159,32 @@ def celery_stress_test_view(request): request, current_tab='celery_stress_test', ) + preset = request.POST.get('preset', request.GET.get('preset', 'load')) + if preset not in PRESETS: + return HttpResponseBadRequest('Unknown suite preset.') + if preset != 'load': + form = CeleryScenarioForm(request.POST if request.method == 'POST' else None, + suite_type=preset, initial=PRESETS[preset]['defaults']) + if request.method == 'POST' and form.is_valid(): + config = resolve(preset, form.cleaned_data) + if preset == 'workflow': + try: + celery_app.backend.ensure_chords_allowed() + except (NotImplementedError, AttributeError) as exc: + form.add_error(None, f'This result backend does not support chords: {exc}') + if not form.errors: + suite = CeleryTestSuite.objects.create(name=config['name'], config=config, created_by=request.user, + deadline_at=timezone.now() + timedelta(seconds=config['deadline_seconds'])) + try: + start_celery_stress_test.apply_async(args=[str(suite.id)], + task_id=str(suite.coordinator_task_id), retry=False) + except Exception as exc: + finish_publishing(suite.id, f'Coordinator publication failed: {exc}') + messages.error(request, 'Failed to submit the suite.') + return redirect('celery_stress_suite_detail', suite_id=suite.id) + context.update({'form': form, 'presets': PRESETS, 'selected_preset': preset}) + return render(request, 'admin/dj_celery_panel/scenario_testing.html', context) + context.update({'presets': PRESETS, 'selected_preset': preset}) if request.method == 'POST': form = CeleryStressTestForm(request.POST) @@ -154,6 +193,8 @@ def celery_stress_test_view(request): if form.is_valid() and workload_formset.is_valid(): config = { **form.cleaned_data, + 'suite_type': 'load', + 'preset_version': 1, 'workloads': [ workload_form.cleaned_data for workload_form in workload_formset @@ -167,10 +208,22 @@ def celery_stress_test_view(request): request, 'Please configure at least one workload.', ) + elif config['total_duration'] * min( + config['max_total_rate'], + sum(workload['rate'] for workload in config['workloads']), + ) > settings.CELERY_STRESS_TEST_MAX_TASKS: + form.add_error(None, f'This run exceeds the {settings.CELERY_STRESS_TEST_MAX_TASKS:,} task limit.') else: + suite = CeleryTestSuite.objects.create( + name=config['name'], config=config, created_by=request.user, + deadline_at=timezone.now() + timedelta(seconds=config['total_duration'] + config['drain_timeout'] + 60), + ) try: - result = start_celery_stress_test.delay(config) - except Exception: + start_celery_stress_test.apply_async( + args=[str(suite.id)], task_id=str(suite.coordinator_task_id), retry=False, + ) + except Exception as error: + finish_publishing(suite.id, f'Coordinator publication failed: {error}') messages.error( request, 'Failed to start the stress test. Please try again.', @@ -178,9 +231,9 @@ def celery_stress_test_view(request): else: messages.success( request, - f"Stress test submitted successfully. Task ID: {result.id}", + f"Stress test submitted successfully: {suite.name}", ) - return redirect(request.path) + return redirect('celery_stress_suite_detail', suite_id=suite.id) else: form = CeleryStressTestForm() workload_formset = WorkloadFormSet() @@ -195,3 +248,56 @@ def celery_stress_test_view(request): 'admin/dj_celery_panel/celery_stress_testing.html', context, ) + + +@panel_config.permission_required('celery_stress_test') +@require_http_methods(['GET']) +def celery_stress_suites(request): + context = panel_config.get_context(request, current_tab='celery_stress_test') + suites = CeleryTestSuite.objects.annotate( + task_count=Count('tasks'), + succeeded_count=Count('tasks', filter=Q(tasks__status=CeleryTestTask.Status.SUCCEEDED)), + failed_count=Count('tasks', filter=Q(tasks__status=CeleryTestTask.Status.FAILED)), + ) + context.update({ + 'active_suites': suites.filter(status__in=CeleryTestSuite.ACTIVE_STATUSES), + 'history': Paginator(suites.exclude(status__in=CeleryTestSuite.ACTIVE_STATUSES), 25).get_page(request.GET.get('page')), + }) + return render(request, 'admin/dj_celery_panel/stress_suites.html', context) + + +@panel_config.permission_required('celery_stress_test') +@require_http_methods(['GET']) +def celery_stress_suite_detail(request, suite_id): + suite = get_object_or_404(CeleryTestSuite, pk=suite_id) + context = panel_config.get_context(request, current_tab='celery_stress_test') + context.update({ + 'suite': suite, + 'metrics': suite_metrics(suite), + 'task_page': Paginator(suite.tasks.order_by('-submitted_at', '-id'), 50).get_page(request.GET.get('page')), + }) + return render(request, 'admin/dj_celery_panel/stress_suite_detail.html', context) + + +@transaction.non_atomic_requests +@panel_config.permission_required('celery_stress_test') +@require_http_methods(['POST']) +def celery_stress_suite_cancel(request, suite_id): + get_object_or_404(CeleryTestSuite, pk=suite_id) + suite, warning = cancel_suite(suite_id, request.user, celery_app) + if warning: + messages.warning(request, warning) + elif suite.cancel_requested_at: + messages.success(request, 'Cancellation requested. Publishing stops; queued load tasks are revoked where possible and expire at the deadline. Running load tasks finish naturally.') + else: + messages.info(request, 'This suite has already finished.') + return redirect('celery_stress_suite_detail', suite_id=suite.id) + + +@transaction.non_atomic_requests +@panel_config.permission_required('celery_stress_test') +@require_http_methods(['POST']) +def celery_stress_suites_reconcile(request): + reconcile_suites.run() + messages.success(request, 'Checked overdue suites and reconciled their status.') + return redirect('celery_stress_test') diff --git a/admin/base/forms.py b/admin/base/forms.py index 11c9386693f..3c04490cc11 100644 --- a/admin/base/forms.py +++ b/admin/base/forms.py @@ -1,5 +1,7 @@ from django import forms from django.core.validators import validate_email +from framework.celery_tasks import app as celery_app +from website import settings class GuidForm(forms.Form): @@ -27,6 +29,9 @@ class ArchiveRegistrationWithPigeonForm(forms.Form): class CeleryStressTestForm(forms.Form): + name = forms.CharField(max_length=100, initial='Celery load test') + drain_timeout = forms.IntegerField(min_value=10, max_value=3600, initial=120, + help_text='Maximum time to wait for load tasks after publication ends.') total_duration = forms.IntegerField( label='Duration (seconds)', min_value=1, @@ -41,6 +46,11 @@ class CeleryStressTestForm(forms.Form): ) +def stress_test_queue_choices(): + declared_queues = {queue.name for queue in celery_app.conf.task_queues} + return [(name, name) for name in settings.CELERY_STRESS_TEST_QUEUES if name in declared_queues] + + class WorkloadForm(forms.Form): name = forms.CharField( max_length=100, @@ -54,7 +64,7 @@ class WorkloadForm(forms.Form): ], initial='io', ) - queue = forms.ChoiceField(choices=[]) + queue = forms.ChoiceField(choices=stress_test_queue_choices) rate = forms.IntegerField( label='Task rate (tasks/second)', @@ -77,15 +87,6 @@ class WorkloadForm(forms.Form): required=False, ) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - allowed_queues = ('high', 'med', 'low', 'celery') # TODO: Replace with settings.CELERY_QUEUES - - self.fields['queue'].choices = [ - (queue, queue) for queue in allowed_queues - ] - def clean(self): cleaned_data = super().clean() mode = cleaned_data.get('mode') @@ -97,3 +98,39 @@ def clean(self): self.add_error('cpu_iterations', 'Specify the CPU iteration count.') return cleaned_data + + +class CeleryScenarioForm(forms.Form): + name = forms.CharField(max_length=100, initial='Celery correctness test') + queue = forms.ChoiceField(choices=stress_test_queue_choices) + deadline_seconds = forms.IntegerField(min_value=10, max_value=3600, initial=120) + retry_delay = forms.IntegerField(min_value=1, max_value=60, initial=2) + max_retries = forms.IntegerField(min_value=2, max_value=10, initial=3) + tasks_per_scenario = forms.IntegerField(min_value=1, max_value=100, initial=1, + help_text='Number of tasks for each of the three retry scenarios (3 × this value total).') + transient_failures = forms.IntegerField(min_value=1, max_value=10, initial=2, + help_text='Failures before a transient task succeeds; must not exceed maximum retries.') + workflow_count = forms.IntegerField(min_value=1, max_value=100, initial=1, + help_text='Number of independent chain, group and chord sets.') + chain_length = forms.IntegerField(min_value=2, max_value=100, initial=2) + group_size = forms.IntegerField(min_value=2, max_value=100, initial=3) + chord_size = forms.IntegerField(min_value=2, max_value=100, initial=3, + help_text='Header task count; each chord also has one final callback.') + + def __init__(self, *args, suite_type, **kwargs): + super().__init__(*args, **kwargs) + excluded = (('workflow_count', 'chain_length', 'group_size', 'chord_size') if suite_type == 'retry' + else ('tasks_per_scenario', 'transient_failures', 'retry_delay', 'max_retries')) + for name in excluded: + del self.fields[name] + + def clean(self): + data = super().clean() + if 'max_retries' in data and data.get('deadline_seconds', 0) <= data['max_retries'] * data.get('retry_delay', 0): + self.add_error('deadline_seconds', 'Allow more time than the configured retry delays.') + if data.get('transient_failures', 0) > data.get('max_retries', 0): + self.add_error('transient_failures', 'Must not exceed maximum retries.') + total = data.get('workflow_count', 0) * (data.get('chain_length', 0) + data.get('group_size', 0) + data.get('chord_size', 0) + 1) + if total > 500: + self.add_error('workflow_count', 'Choose sizes producing at most 500 tasks in total.') + return data diff --git a/admin/base/urls.py b/admin/base/urls.py index 533fedabc9b..6ee17942247 100644 --- a/admin/base/urls.py +++ b/admin/base/urls.py @@ -15,7 +15,11 @@ re_path(r'^unprotected_views/$', views.unprotected_views, name='unprotected_views'), re_path(r'^middleware_check/$', views.admin_auth_middleware_check, name='middleware_check'), re_path(r'^admin/dj-celery-panel/$', celery_panel.overview), - re_path(r'^admin/dj-celery-panel/celery_stress_test/$', celery_panel.celery_stress_test_view, name='celery_stress_test'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/$', celery_panel.celery_stress_suites, name='celery_stress_test'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/reconcile/$', celery_panel.celery_stress_suites_reconcile, name='celery_stress_suites_reconcile'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/new/$', celery_panel.celery_stress_test_view, name='celery_stress_test_new'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/$', celery_panel.celery_stress_suite_detail, name='celery_stress_suite_detail'), + re_path(r'^admin/dj-celery-panel/celery_stress_test/(?P[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/cancel/$', celery_panel.celery_stress_suite_cancel, name='celery_stress_suite_cancel'), re_path(r'^admin/dj-celery-panel/', include('dj_celery_panel.urls')), re_path(r'^admin/', admin.site.urls), re_path(r'^provider_asset_files/', include('admin.provider_asset_files.urls', namespace='provider_asset_files')), diff --git a/admin/templates/admin/dj_celery_panel/celery_stress_testing.html b/admin/templates/admin/dj_celery_panel/celery_stress_testing.html index 1d600ded786..eb39e3a5938 100644 --- a/admin/templates/admin/dj_celery_panel/celery_stress_testing.html +++ b/admin/templates/admin/dj_celery_panel/celery_stress_testing.html @@ -2,122 +2,224 @@ {% load i18n static dcr_icons %} {% block panel_page_content %} -
-
-
- {% dcr_icon "cog" %} -
-
-

{% trans "Stress Testing" %}

-

- {% trans "Run stress tests on your Celery workers to evaluate their performance and stability under load." %} -

+ {% include "admin/dj_celery_panel/suite_presets.html" %} +
+
+
+ {% dcr_icon "cog" %} +
+
+

{% trans "Stress Testing" %}

+

+ {% trans "Run stress tests on your Celery workers to evaluate their performance and stability under load." %} +

+
-
-
- {% csrf_token %} +

{% trans "All test suites" %}

+ + {% csrf_token %} +

{{ form.drain_timeout.label_tag }} {{ form.drain_timeout }} {{ form.drain_timeout.errors }} {{ form.drain_timeout.help_text }}

- {% if form.non_field_errors %} -
- {{ form.non_field_errors }} -
- {% endif %} - - {# Global configuration #} -
-
-

{% trans "Global Configuration" %}

-

- {% trans "Configure the duration and maximum publishing rate for the entire test suite." %} -

-
+ {% if form.non_field_errors %} +
+ {{ form.non_field_errors }} +
+ {% endif %} -
-
+ {# Global configuration #} +
+
+

{% trans "Global Configuration" %}

+

+ {% trans "Configure the duration and maximum publishing rate for the entire test suite." %} +

+
+ +
- - {{ form.total_duration }} - {% if form.total_duration.help_text %} - {{ form.total_duration.help_text }} - {% endif %} - {{ form.total_duration.errors }} + {{ form.name.label_tag }} {{ form.name }} {{ form.name.errors }}
+
+
+ + {{ form.total_duration }} + {% if form.total_duration.help_text %} + {{ form.total_duration.help_text }} + {% endif %} + {{ form.total_duration.errors }} +
-
- - {{ form.max_total_rate }} - {% if form.max_total_rate.help_text %} - {{ form.max_total_rate.help_text }} - {% endif %} - {{ form.max_total_rate.errors }} +
+ + {{ form.max_total_rate }} + {% if form.max_total_rate.help_text %} + {{ form.max_total_rate.help_text }} + {% endif %} + {{ form.max_total_rate.errors }} +
-
-
- - {# Workload configuration #} -
-
-
-

{% trans "Workloads" %}

-

- {% trans "Define the workloads to publish to your Celery queues." %} -

-
+
- -
- {{ workload_formset.max_num }} + {# Workload configuration #} +
+
+
+

{% trans "Workloads" %}

+

+ {% trans "Define the workloads to publish to your Celery queues." %} +

+
+ + +
+ {{ workload_formset.max_num }} +
-
- {{ workload_formset.management_form }} + {{ workload_formset.management_form }} - {% if workload_formset.non_form_errors %} -
- {{ workload_formset.non_form_errors }} -
- {% endif %} + {% if workload_formset.non_form_errors %} +
+ {{ workload_formset.non_form_errors }} +
+ {% endif %} + +
+ {% for workload_form in workload_formset %} +
+
+
+

+ {% blocktrans with number=forloop.counter %} + Workload {{ number }} + {% endblocktrans %} +

+

+ {% trans "Configure the workload type, destination queue, and publishing rate." %} +

+
-
- {% for workload_form in workload_formset %} -
-
-
-

- {% blocktrans with number=forloop.counter %} - Workload {{ number }} - {% endblocktrans %} -

-

- {% trans "Configure the workload type, destination queue, and publishing rate." %} -

+
- -
+
+ {{ workload_form.id }} -
- {{ workload_form.id }} + {% if workload_form.non_field_errors %} +
+ {{ workload_form.non_field_errors }} +
+ {% endif %} + +
+
+ + {{ workload_form.name }} + {{ workload_form.name.errors }} +
+ +
+ + {{ workload_form.mode }} + {{ workload_form.mode.errors }} +
- {% if workload_form.non_field_errors %} -
- {{ workload_form.non_field_errors }} +
+ + {{ workload_form.queue }} + {{ workload_form.queue.errors }} +
+ +
+ + {{ workload_form.rate }} + {{ workload_form.rate.errors }} +
- {% endif %} + +
+

{% trans "Workload Arguments" %}

+ +
+
+ + {{ workload_form.io_seconds }} + {{ workload_form.io_seconds.errors }} + + {% trans "Time spent waiting in I/O and mixed modes." %} + +
+
+ +
+
+ + {{ workload_form.cpu_iterations }} + {{ workload_form.cpu_iterations.errors }} + + {% trans "Number of calculation iterations in CPU and mixed modes." %} + +
+
+
+
+
+ {% empty %} +

+ {% trans "No workloads configured. Add a workload to get started." %} +

+ {% endfor %} +
+
+ + {# Hidden template used by JavaScript to add new workloads #} + + +
+
- +
- {# Hidden template used by JavaScript to add new workloads #} -