diff --git a/api/subscriptions/views.py b/api/subscriptions/views.py index 953e2a66aec..b23fccb2804 100644 --- a/api/subscriptions/views.py +++ b/api/subscriptions/views.py @@ -1,4 +1,4 @@ -from django.db.models import Value, When, Case, OuterRef, Subquery +from django.db.models import Value, When, Case, OuterRef, Subquery, F from django.db.models.fields import CharField, IntegerField from django.db.models.functions import Concat, Cast from django.contrib.contenttypes.models import ContentType @@ -46,6 +46,10 @@ class SubscriptionList(JSONAPIBaseView, generics.ListAPIView, ListFilterMixin): def get_queryset(self): user_guid = self.request.user._id + provider_ct = ContentType.objects.get_by_natural_key(app_label='osf', model='abstractprovider') + node_ct = ContentType.objects.get_by_natural_key(app_label='osf', model='abstractnode') + user_ct = ContentType.objects.get_by_natural_key(app_label='osf', model='osfuser') + node_subquery = AbstractNode.objects.filter( id=Cast(OuterRef('object_id'), IntegerField()), ).values('guids___id')[:1] @@ -60,11 +64,13 @@ def get_queryset(self): NotificationType.Type.ADDON_FILE_REMOVED.value, NotificationType.Type.FOLDER_CREATED.value, ] - _global_reviews = [ + _global_reviews_provider = [ NotificationType.Type.PROVIDER_NEW_PENDING_SUBMISSIONS.value, NotificationType.Type.PROVIDER_REVIEWS_SUBMISSION_CONFIRMATION.value, NotificationType.Type.PROVIDER_REVIEWS_RESUBMISSION_CONFIRMATION.value, NotificationType.Type.PROVIDER_NEW_PENDING_WITHDRAW_REQUESTS.value, + ] + _global_reviews_user = [ NotificationType.Type.REVIEWS_SUBMISSION_STATUS.value, ] _node_file_updated = [ @@ -80,27 +86,31 @@ def get_queryset(self): ] qs = NotificationSubscription.objects.filter( - notification_type__name__in=[ - NotificationType.Type.USER_FILE_UPDATED.value, - NotificationType.Type.NODE_FILE_UPDATED.value, - NotificationType.Type.PROVIDER_NEW_PENDING_SUBMISSIONS.value, - ] + _global_reviews + _global_file_updated + _node_file_updated, + notification_type__name__in=_global_reviews_provider + _global_reviews_user + _global_file_updated + _node_file_updated, user=self.request.user, ).annotate( event_name=Case( When( notification_type__name__in=_node_file_updated, + content_type=node_ct, then=Value('files_updated'), ), When( notification_type__name__in=_global_file_updated, + content_type=user_ct, then=Value('global_file_updated'), ), When( - notification_type__name__in=_global_reviews, + notification_type__name__in=_global_reviews_provider, + content_type=provider_ct, + then=Value('global_reviews'), + ), + When( + notification_type__name__in=_global_reviews_user, + content_type=user_ct, then=Value('global_reviews'), ), - default=Value('notification_type__name'), + default=F('notification_type__name'), ), legacy_id=Case( When( @@ -112,10 +122,16 @@ def get_queryset(self): then=Value(f'{user_guid}_global_file_updated'), ), When( - notification_type__name__in=_global_reviews, + notification_type__name__in=_global_reviews_provider, + content_type=provider_ct, + then=Value(f'{user_guid}_global_reviews'), + ), + When( + notification_type__name__in=_global_reviews_user, + content_type=user_ct, then=Value(f'{user_guid}_global_reviews'), ), - default=Value('notification_type__name'), + default=F('notification_type__name'), ), ).distinct('legacy_id') diff --git a/api_tests/guids/views/test_guid_detail.py b/api_tests/guids/views/test_guid_detail.py index 4a6db0fbc35..f52ff10e215 100644 --- a/api_tests/guids/views/test_guid_detail.py +++ b/api_tests/guids/views/test_guid_detail.py @@ -12,6 +12,7 @@ PrivateLinkFactory, ) from website.settings import API_DOMAIN +from tests.utils import capture_notifications @pytest.mark.django_db @@ -32,13 +33,14 @@ def registration(self): @pytest.fixture() def versioned_preprint(self, user): preprint = PreprintFactory(reviews_workflow='pre-moderation') - PreprintFactory.create_version( - create_from=preprint, - creator=user, - final_machine_state='accepted', - is_published=True, - set_doi=False - ) + with capture_notifications(): + PreprintFactory.create_version( + create_from=preprint, + creator=user, + final_machine_state='accepted', + is_published=True, + set_doi=False + ) return preprint def test_redirects(self, app, project, registration, user): diff --git a/api_tests/mailhog/provider/test_preprints.py b/api_tests/mailhog/provider/test_preprints.py index db514c87c34..96d8a8c099c 100644 --- a/api_tests/mailhog/provider/test_preprints.py +++ b/api_tests/mailhog/provider/test_preprints.py @@ -2,7 +2,7 @@ from osf import features from framework.auth.core import Auth -from osf.models import NotificationType, Notification +from osf.models import NotificationType from osf_tests.factories import ( ProjectFactory, AuthUserFactory, @@ -12,7 +12,6 @@ from osf.utils.permissions import WRITE from tests.base import OsfTestCase from tests.utils import get_mailhog_messages, delete_mailhog_messages, capture_notifications, assert_emails -from notifications.tasks import send_users_instant_digest_email class TestPreprintConfirmationEmails(OsfTestCase): @@ -35,13 +34,7 @@ def test_creator_gets_email(self): assert len(notifications['emits']) == 1 assert notifications['emits'][0]['type'] == NotificationType.Type.PROVIDER_REVIEWS_SUBMISSION_CONFIRMATION messages = get_mailhog_messages() - assert not messages['items'] - assert Notification.objects.all() - with capture_notifications(passthrough=True) as notifications: - send_users_instant_digest_email.delay() - - messages = get_mailhog_messages() - assert messages['count'] == len(notifications['emits']) + assert_emails(messages, notifications) delete_mailhog_messages() with capture_notifications(passthrough=True) as notifications: @@ -49,11 +42,6 @@ def test_creator_gets_email(self): assert len(notifications['emits']) == 1 assert notifications['emits'][0]['type'] == NotificationType.Type.PROVIDER_REVIEWS_SUBMISSION_CONFIRMATION messages = get_mailhog_messages() - assert not messages['items'] - with capture_notifications(passthrough=True) as notifications: - send_users_instant_digest_email.delay() - massages = get_mailhog_messages() - assert massages['count'] == len(notifications['emits']) - assert_emails(massages, notifications) + assert_emails(messages, notifications) delete_mailhog_messages() diff --git a/api_tests/mailhog/provider/test_submissions.py b/api_tests/mailhog/provider/test_submissions.py index caa0abe71b0..b9d26e155df 100644 --- a/api_tests/mailhog/provider/test_submissions.py +++ b/api_tests/mailhog/provider/test_submissions.py @@ -21,7 +21,7 @@ from osf.models import NotificationType from osf.migrations import update_provider_auth_groups -from tests.utils import capture_notifications, get_mailhog_messages, delete_mailhog_messages +from tests.utils import capture_notifications, get_mailhog_messages, delete_mailhog_messages, assert_emails @pytest.mark.django_db @@ -85,8 +85,7 @@ def test_get_registration_actions(self, app, registration_actions_url, registrat assert notifications['emits'][0]['type'] == NotificationType.Type.PROVIDER_NEW_PENDING_WITHDRAW_REQUESTS assert notifications['emits'][1]['type'] == NotificationType.Type.PROVIDER_NEW_PENDING_WITHDRAW_REQUESTS messages = get_mailhog_messages() - assert messages['count'] == 1 - assert messages['items'][0]['Content']['Headers']['To'][0] == registration.creator.username + assert_emails(messages, notifications) delete_mailhog_messages() diff --git a/api_tests/notifications/test_notification_digest.py b/api_tests/notifications/test_notification_digest.py index 70433f983c5..b5592b2148e 100644 --- a/api_tests/notifications/test_notification_digest.py +++ b/api_tests/notifications/test_notification_digest.py @@ -18,7 +18,7 @@ def add_notification_subscription(user, notification_type, frequency, subscribed Create a NotificationSubscription for a user. If the notification type corresponds to a subscribed_object, set subscribed_object to get the provider. """ - from osf.models import NotificationSubscription, AbstractProvider + from osf.models import NotificationSubscription kwargs = { 'user': user, 'notification_type': NotificationType.objects.get(name=notification_type), @@ -26,10 +26,7 @@ def add_notification_subscription(user, notification_type, frequency, subscribed } if subscribed_object is not None: kwargs['object_id'] = subscribed_object.id - if isinstance(subscribed_object, AbstractProvider): - kwargs['content_type'] = ContentType.objects.get_for_model(subscribed_object, for_concrete_model=False) if subscribed_object else None - else: - kwargs['content_type'] = ContentType.objects.get_for_model(subscribed_object) if subscribed_object else None + kwargs['content_type'] = ContentType.objects.get_for_model(subscribed_object) if subscription is not None: kwargs['object_id'] = subscription.id kwargs['content_type'] = ContentType.objects.get_for_model(subscription) diff --git a/api_tests/notifications/test_notifications_cleanup.py b/api_tests/notifications/test_notifications_cleanup.py new file mode 100644 index 00000000000..848b142b740 --- /dev/null +++ b/api_tests/notifications/test_notifications_cleanup.py @@ -0,0 +1,189 @@ +import pytest +from osf.models import Notification, NotificationType, EmailTask, NotificationSubscription +from notifications.tasks import ( + notifications_cleanup_task +) +from osf_tests.factories import AuthUserFactory +from website.settings import NOTIFICATIONS_CLEANUP_AGE +from django.utils import timezone +from datetime import timedelta + +def create_notification(subscription, sent_date=None): + return Notification.objects.create( + subscription=subscription, + event_context={}, + sent=sent_date + ) + +def create_email_task(user, created_date): + et = EmailTask.objects.create( + task_id=f'test-{created_date.timestamp()}', + user=user, + status='SUCCESS', + ) + et.created_at = created_date + et.save() + return et + +@pytest.mark.django_db +class TestNotificationCleanUpTask: + + @pytest.fixture() + def user(self): + return AuthUserFactory() + + @pytest.fixture() + def notification_type(self): + return NotificationType.objects.get_or_create( + name='Test Notification', + subject='Hello', + template='Sample Template', + )[0] + + @pytest.fixture() + def subscription(self, user, notification_type): + return NotificationSubscription.objects.get_or_create( + user=user, + notification_type=notification_type, + message_frequency='daily', + )[0] + + def test_dry_run_does_not_delete_records(self, user, subscription): + now = timezone.now() + + old_notification = create_notification( + subscription, + sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + old_email_task = create_email_task( + user, + created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + + notifications_cleanup_task(dry_run=True) + + assert Notification.objects.filter(id=old_notification.id).exists() + assert EmailTask.objects.filter(id=old_email_task.id).exists() + + def test_deletes_old_notifications_and_email_tasks(self, user, subscription): + now = timezone.now() + + old_notification = create_notification( + subscription, + sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + new_notification = create_notification( + subscription, + sent_date=now - timedelta(days=10), + ) + + old_email_task = create_email_task( + user, + created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + new_email_task = create_email_task( + user, + created_date=now - timedelta(days=10), + ) + + notifications_cleanup_task() + + assert not Notification.objects.filter(id=old_notification.id).exists() + assert Notification.objects.filter(id=new_notification.id).exists() + + assert not EmailTask.objects.filter(id=old_email_task.id).exists() + assert EmailTask.objects.filter(id=new_email_task.id).exists() + + def test_records_at_cutoff_are_not_deleted(self, user, subscription): + now = timezone.now() + cutoff = now - NOTIFICATIONS_CLEANUP_AGE + timedelta(hours=1) + + notification = create_notification( + subscription, + sent_date=cutoff, + ) + email_task = create_email_task( + user, + created_date=cutoff, + ) + + notifications_cleanup_task() + + assert Notification.objects.filter(id=notification.id).exists() + assert EmailTask.objects.filter(id=email_task.id).exists() + + def test_cleanup_when_only_notifications_exist(self, user, subscription): + now = timezone.now() + + notification = create_notification( + subscription, + sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + + notifications_cleanup_task() + + assert not Notification.objects.filter(id=notification.id).exists() + + def test_cleanup_when_only_email_tasks_exist(self, user, subscription): + now = timezone.now() + + email_task = create_email_task( + user, + created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + + notifications_cleanup_task() + + assert not EmailTask.objects.filter(id=email_task.id).exists() + + def test_task_is_idempotent(self, user, subscription): + now = timezone.now() + + create_notification( + subscription, + sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + create_email_task( + user, + created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + + notifications_cleanup_task() + notifications_cleanup_task() + + assert Notification.objects.count() == 0 + assert EmailTask.objects.count() == 0 + + def test_recent_records_are_not_deleted(self, user, subscription): + now = timezone.now() + + create_notification( + subscription, + sent_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + create_email_task( + user, + created_date=now - NOTIFICATIONS_CLEANUP_AGE - timedelta(days=1), + ) + create_notification( + subscription, + sent_date=now, + ) + create_email_task( + user, + created_date=now, + ) + + notifications_cleanup_task() + + assert Notification.objects.count() == 1 + assert EmailTask.objects.count() == 1 + + def test_not_sent_notifications_are_not_deleted(self, user, subscription): + create_notification(subscription) + create_notification(subscription) + create_notification(subscription) + + notifications_cleanup_task() + + assert Notification.objects.count() == 3 diff --git a/api_tests/notifications/test_notifications_db_transaction.py b/api_tests/notifications/test_notifications_db_transaction.py index dc09dd46487..2d2d7365494 100644 --- a/api_tests/notifications/test_notifications_db_transaction.py +++ b/api_tests/notifications/test_notifications_db_transaction.py @@ -1,12 +1,14 @@ +from django.db import reset_queries, connection +from django.utils import timezone + import pytest + +from osf.models import Notification, NotificationType, NotificationSubscription from osf_tests.factories import ( AuthUserFactory, NotificationTypeFactory ) -from datetime import datetime -from osf.models import Notification, NotificationType, NotificationSubscription from tests.utils import capture_notifications -from django.db import reset_queries, connection @pytest.mark.django_db @@ -47,12 +49,21 @@ def test_emit_without_saving(self, user_one, test_notification_type): ).exists() def test_emit_frequency_none(self, user_one, test_notification_type): + assert not Notification.objects.filter( + subscription__notification_type=test_notification_type, + fake_sent=True + ).exists() + time_before = timezone.now() test_notification_type.emit( user=user_one, event_context={'notifications': 'test template for Test notification'}, message_frequency='none' ) - assert Notification.objects.filter( + time_after = timezone.now() + notifications = Notification.objects.filter( subscription__notification_type=test_notification_type, - sent=datetime(1000, 1, 1) - ).exists() + fake_sent=True + ) + assert notifications.exists() + assert notifications.count() == 1 + assert time_before < notifications.first().sent < time_after diff --git a/api_tests/share/test_share_preprint.py b/api_tests/share/test_share_preprint.py index cf0c8a3d92d..118abf3105b 100644 --- a/api_tests/share/test_share_preprint.py +++ b/api_tests/share/test_share_preprint.py @@ -18,6 +18,7 @@ from website import settings from website.preprints.tasks import on_preprint_updated from ._utils import expect_preprint_ingest_request +from tests.utils import capture_notifications @pytest.mark.django_db @@ -72,54 +73,59 @@ def test_save_unpublished_not_called(self, mock_share_responses, preprint): preprint.save() def test_save_published_called(self, mock_share_responses, preprint, user, auth): - with expect_preprint_ingest_request(mock_share_responses, preprint): - preprint.set_published(True, auth=auth, save=True) + with capture_notifications(): + with expect_preprint_ingest_request(mock_share_responses, preprint): + preprint.set_published(True, auth=auth, save=True) # This covers an edge case where a preprint is forced back to unpublished # that it sends the information back to share def test_save_unpublished_called_forced(self, mock_share_responses, auth, preprint): - with expect_preprint_ingest_request(mock_share_responses, preprint): - preprint.set_published(True, auth=auth, save=True) - with expect_preprint_ingest_request(mock_share_responses, preprint, delete=True): - preprint.is_published = False - preprint.save(**{'force_update': True}) + with capture_notifications(): + with expect_preprint_ingest_request(mock_share_responses, preprint): + preprint.set_published(True, auth=auth, save=True) + with expect_preprint_ingest_request(mock_share_responses, preprint, delete=True): + preprint.is_published = False + preprint.save(**{'force_update': True}) def test_save_published_subject_change_called(self, mock_share_responses, auth, preprint, subject, subject_two): - preprint.set_published(True, auth=auth, save=True) - with expect_preprint_ingest_request(mock_share_responses, preprint): - preprint.set_subjects([[subject_two._id]], auth=auth) + with capture_notifications(): + preprint.set_published(True, auth=auth, save=True) + with expect_preprint_ingest_request(mock_share_responses, preprint): + preprint.set_subjects([[subject_two._id]], auth=auth) def test_save_unpublished_subject_change_not_called(self, mock_share_responses, auth, preprint, subject_two): with expect_preprint_ingest_request(mock_share_responses, preprint, delete=True): preprint.set_subjects([[subject_two._id]], auth=auth) def test_send_to_share_is_true(self, mock_share_responses, auth, preprint): - preprint.set_published(True, auth=auth, save=True) - with expect_preprint_ingest_request(mock_share_responses, preprint): - on_preprint_updated(preprint._id, saved_fields=['title']) + with capture_notifications(): + preprint.set_published(True, auth=auth, save=True) + with expect_preprint_ingest_request(mock_share_responses, preprint): + on_preprint_updated(preprint._id, saved_fields=['title']) def test_preprint_contributor_changes_updates_preprints_share(self, mock_share_responses, user, auth): - preprint = PreprintFactory(is_published=True, creator=user) - preprint.set_published(True, auth=auth, save=True) - user2 = AuthUserFactory() + with capture_notifications(): + preprint = PreprintFactory(is_published=True, creator=user) + preprint.set_published(True, auth=auth, save=True) + user2 = AuthUserFactory() - with expect_preprint_ingest_request(mock_share_responses, preprint): - preprint.add_contributor(contributor=user2, auth=auth, save=True) + with expect_preprint_ingest_request(mock_share_responses, preprint): + preprint.add_contributor(contributor=user2, auth=auth, save=True) - with expect_preprint_ingest_request(mock_share_responses, preprint): - preprint.move_contributor(contributor=user, index=0, auth=auth, save=True) + with expect_preprint_ingest_request(mock_share_responses, preprint): + preprint.move_contributor(contributor=user, index=0, auth=auth, save=True) - data = [{'id': user._id, 'permissions': ADMIN, 'visible': True}, - {'id': user2._id, 'permissions': WRITE, 'visible': False}] + data = [{'id': user._id, 'permissions': ADMIN, 'visible': True}, + {'id': user2._id, 'permissions': WRITE, 'visible': False}] - with expect_preprint_ingest_request(mock_share_responses, preprint): - preprint.manage_contributors(data, auth=auth, save=True) + with expect_preprint_ingest_request(mock_share_responses, preprint): + preprint.manage_contributors(data, auth=auth, save=True) - with expect_preprint_ingest_request(mock_share_responses, preprint): - preprint.update_contributor(user2, READ, True, auth=auth, save=True) + with expect_preprint_ingest_request(mock_share_responses, preprint): + preprint.update_contributor(user2, READ, True, auth=auth, save=True) - with expect_preprint_ingest_request(mock_share_responses, preprint): - preprint.remove_contributor(contributor=user2, auth=auth) + with expect_preprint_ingest_request(mock_share_responses, preprint): + preprint.remove_contributor(contributor=user2, auth=auth) @pytest.mark.skip('Synchronous retries not supported if celery >=5.0') def test_call_async_update_on_500_failure(self, mock_share_responses, preprint, auth): @@ -129,10 +135,11 @@ def test_call_async_update_on_500_failure(self, mock_share_responses, preprint, preprint.update_search() def test_no_call_async_update_on_400_failure(self, mock_share_responses, preprint, auth): - mock_share_responses.replace(responses.POST, shtrove_ingest_url(), status=400) - preprint.set_published(True, auth=auth, save=True) - with expect_preprint_ingest_request(mock_share_responses, preprint, count=1, error_response=True): - preprint.update_search() + with capture_notifications(): + mock_share_responses.replace(responses.POST, shtrove_ingest_url(), status=400) + preprint.set_published(True, auth=auth, save=True) + with expect_preprint_ingest_request(mock_share_responses, preprint, count=1, error_response=True): + preprint.update_search() def test_delete_from_share(self, mock_share_responses): preprint = PreprintFactory() diff --git a/notifications/listeners.py b/notifications/listeners.py index 62ad487b358..97eba256c53 100644 --- a/notifications/listeners.py +++ b/notifications/listeners.py @@ -102,7 +102,8 @@ def reviews_withdraw_requests_notification_moderators(self, timestamp, context, NotificationType.Type.PROVIDER_NEW_PENDING_WITHDRAW_REQUESTS.instance.emit( subscribed_object=provider, user=user, - event_context=context + event_context=context, + is_digest=True, ) diff --git a/notifications/tasks.py b/notifications/tasks.py index 84a825088f2..2f4cf004ba6 100644 --- a/notifications/tasks.py +++ b/notifications/tasks.py @@ -1,7 +1,6 @@ import itertools from calendar import monthrange -from datetime import date, datetime -from django.contrib.contenttypes.models import ContentType +from datetime import date from django.db import connection from django.utils import timezone from django.core.validators import EmailValidator @@ -34,10 +33,8 @@ def safe_render_notification(notifications, email_task): email_task.save() failed_notifications.append(notification.id) # Mark notifications that failed to render as fake sent - # Use 1000/12/31 to distinguish itself from another type of fake sent 1000/1/1 + notification.mark_sent(fake_sent=True) log_message(f'Error rendering notification, mark as fake sent: [notification_id={notification.id}]') - notification.sent = datetime(1000, 12, 31) - notification.save() continue rendered_notifications.append(rendered) @@ -102,9 +99,10 @@ def send_user_email_task(self, user_id, notification_ids, **kwargs): notifications_qs = notifications_qs.exclude(id__in=failed_notifications) if not rendered_notifications: + email_task.status = 'SUCCESS' if email_task.error_message: logger.error(f'Partial success for send_user_email_task for user {user_id}. Task id: {self.request.id}. Errors: {email_task.error_message}') - email_task.status = 'SUCCESS' + email_task.status = 'PARTIAL_SUCCESS' email_task.save() return @@ -123,10 +121,10 @@ def send_user_email_task(self, user_id, notification_ids, **kwargs): notifications_qs.update(sent=timezone.now()) email_task.status = 'SUCCESS' - email_task.save() - if email_task.error_message: logger.error(f'Partial success for send_user_email_task for user {user_id}. Task id: {self.request.id}. Errors: {email_task.error_message}') + email_task.status = 'PARTIAL_SUCCESS' + email_task.save() except Exception as e: retry_count = self.request.retries @@ -177,32 +175,32 @@ def send_moderator_email_task(self, user_id, notification_ids, provider_content_ notifications_qs = notifications_qs.exclude(id__in=failed_notifications) if not rendered_notifications: + email_task.status = 'SUCCESS' if email_task.error_message: logger.error(f'Partial success for send_moderator_email_task for user {user_id}. Task id: {self.request.id}. Errors: {email_task.error_message}') - email_task.status = 'SUCCESS' + email_task.status = 'PARTIAL_SUCCESS' email_task.save() return - ProviderModel = ContentType.objects.get_for_id(provider_content_type_id).model_class() try: - provider = ProviderModel.objects.get(id=provider_id) + provider = AbstractProvider.objects.get(id=provider_id) except AbstractProvider.DoesNotExist: - log_message(f'Provider with id {provider_id} does not exist for model {ProviderModel.name}') + log_message(f'Provider with id {provider_id} does not exist for model {provider.type}') email_task.status = 'FAILURE' - email_task.error_message = f'Provider with id {provider_id} does not exist for model {ProviderModel.name}' + email_task.error_message = f'Provider with id {provider_id} does not exist for model {provider.type}' email_task.save() return except AttributeError as err: - log_message(f'Error retrieving provider with id {provider_id} for model {ProviderModel.name}: {err}') + log_message(f'Error retrieving provider with id {provider_id} for model {provider.type}: {err}') email_task.status = 'FAILURE' - email_task.error_message = f'Error retrieving provider with id {provider_id} for model {ProviderModel.name}: {err}' + email_task.error_message = f'Error retrieving provider with id {provider_id} for model {provider.type}: {err}' email_task.save() return if provider is None: - log_message(f'Provider with id {provider_id} does not exist for model {ProviderModel.name}') + log_message(f'Provider with id {provider_id} does not exist for model {provider.type}') email_task.status = 'FAILURE' - email_task.error_message = f'Provider with id {provider_id} does not exist for model {ProviderModel.name}' + email_task.error_message = f'Provider with id {provider_id} does not exist for model {provider.type}' email_task.save() return @@ -211,10 +209,10 @@ def send_moderator_email_task(self, user_id, notification_ids, provider_content_ current_admins = provider.get_group('admin') if current_admins is None or not current_admins.user_set.filter(id=user.id).exists(): log_message(f"User is not a moderator for provider {provider._id} - notifications will be marked as sent.") - email_task.status = 'FAILURE' + email_task.status = 'AUTO_FIXED' email_task.error_message = f'User is not a moderator for provider {provider._id}' email_task.save() - notifications_qs.update(sent=datetime(1000, 1, 1)) + notifications_qs.update(sent=timezone.now(), fake_sent=True) return additional_context = {} @@ -274,10 +272,10 @@ def send_moderator_email_task(self, user_id, notification_ids, provider_content_ notifications_qs.update(sent=timezone.now()) email_task.status = 'SUCCESS' - email_task.save() - if email_task.error_message: logger.error(f'Partial success for send_moderator_email_task for user {user_id}. Task id: {self.request.id}. Errors: {email_task.error_message}') + email_task.status = 'PARTIAL_SUCCESS' + email_task.save() except Exception as e: retry_count = self.request.retries @@ -490,3 +488,22 @@ def send_no_addon_email(self, dry_run=False, **kwargs): pass else: notification.mark_sent() + + +@celery_app.task(bind=True, name='notifications.tasks.notifications_cleanup_task') +def notifications_cleanup_task(self, dry_run=False, **kwargs): + """Remove old notifications and email tasks from the database.""" + + cutoff_date = timezone.now() - settings.NOTIFICATIONS_CLEANUP_AGE + old_notifications = Notification.objects.filter(sent__lt=cutoff_date) + old_email_tasks = EmailTask.objects.filter(created_at__lt=cutoff_date) + + if dry_run: + notifications_count = old_notifications.count() + email_tasks_count = old_email_tasks.count() + logger.info(f'[Dry Run] Notifications Cleanup Task: {notifications_count} notifications and {email_tasks_count} email tasks would be deleted.') + return + + deleted_notifications_count, _ = old_notifications.delete() + deleted_email_tasks_count, _ = old_email_tasks.delete() + logger.info(f'Notifications Cleanup Task: Deleted {deleted_notifications_count} notifications and {deleted_email_tasks_count} email tasks older than {cutoff_date}.') diff --git a/osf/admin.py b/osf/admin.py index 2f5698b2aca..d9fed50b7ff 100644 --- a/osf/admin.py +++ b/osf/admin.py @@ -367,7 +367,7 @@ class EmailTaskAdmin(admin.ModelAdmin): @admin.register(Notification) class NotificationAdmin(admin.ModelAdmin): - list_display = ('user', 'notification_type_name', 'sent', 'seen') + list_display = ('user', 'notification_type_name', 'sent', 'fake_sent') list_filter = ('sent',) search_fields = ('subscription__notification_type__name', 'subscription__user__username') list_per_page = 50 diff --git a/osf/migrations/0036_notification_refactor_post_release.py b/osf/migrations/0036_notification_refactor_post_release.py new file mode 100644 index 00000000000..d65fe5b82bb --- /dev/null +++ b/osf/migrations/0036_notification_refactor_post_release.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.17 on 2026-01-27 21:03 + +from django.db import migrations, models +import osf.utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('osf', '0035_merge_20251215_1451'), + ] + + operations = [ + migrations.RemoveField( + model_name='notification', + name='seen', + ), + migrations.AddField( + model_name='notification', + name='fake_sent', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='osfuser', + name='no_login_email_last_sent', + field=osf.utils.fields.NonNaiveDateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name='emailtask', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('NO_USER_FOUND', 'No User Found'), ('USER_DISABLED', 'User Disabled'), ('STARTED', 'Started'), ('SUCCESS', 'Success'), ('FAILURE', 'Failure'), ('RETRY', 'Retry'), ('PARTIAL_SUCCESS', 'Partial Success'), ('AUTO_FIXED', 'Auto Fixed')], default='PENDING', max_length=20), + ), + migrations.AlterUniqueTogether( + name='notificationsubscription', + unique_together={('notification_type', 'user', 'content_type', 'object_id', '_is_digest')}, + ), + ] diff --git a/osf/models/email_task.py b/osf/models/email_task.py index f89f2285e5c..12def4c8c12 100644 --- a/osf/models/email_task.py +++ b/osf/models/email_task.py @@ -9,6 +9,8 @@ class EmailTask(models.Model): ('SUCCESS', 'Success'), ('FAILURE', 'Failure'), ('RETRY', 'Retry'), + ('PARTIAL_SUCCESS', 'Partial Success'), + ('AUTO_FIXED', 'Auto Fixed'), ) task_id = models.CharField(max_length=255, unique=True) diff --git a/osf/models/mixins.py b/osf/models/mixins.py index 2604b7d32cb..53573bd45b4 100644 --- a/osf/models/mixins.py +++ b/osf/models/mixins.py @@ -1084,7 +1084,7 @@ def add_to_group(self, user, group): for subscription in self.DEFAULT_SUBSCRIPTIONS: NotificationSubscription.objects.get_or_create( user=user, - content_type=ContentType.objects.get_for_model(self, for_concrete_model=False), + content_type=ContentType.objects.get_for_model(self), object_id=self.id, notification_type=subscription.instance, message_frequency='instantly', @@ -1102,7 +1102,7 @@ def remove_from_group(self, user, group, unsubscribe=True): NotificationSubscription.objects.filter( notification_type=subscription.instance, user=user, - content_type=ContentType.objects.get_for_model(self, for_concrete_model=False), + content_type=ContentType.objects.get_for_model(self), object_id=self.id, ).delete() diff --git a/osf/models/notification.py b/osf/models/notification.py index aab9f2b0f0e..533a05a4e97 100644 --- a/osf/models/notification.py +++ b/osf/models/notification.py @@ -16,8 +16,8 @@ class Notification(models.Model): ) event_context: dict = models.JSONField() sent = models.DateTimeField(null=True, blank=True) - seen = models.DateTimeField(null=True, blank=True) created = models.DateTimeField(auto_now_add=True) + fake_sent = models.BooleanField(default=False) def send( self, @@ -56,14 +56,13 @@ def send( if save: self.mark_sent() - def mark_sent(self) -> None: + def mark_sent(self, fake_sent=False) -> None: + update_fields = ['sent'] self.sent = timezone.now() - self.save(update_fields=['sent']) - - def mark_seen(self) -> None: - raise NotImplementedError('mark_seen must be implemented by subclasses.') - # self.seen = timezone.now() - # self.save(update_fields=['seen']) + if fake_sent: + update_fields.append('fake_sent') + self.fake_sent = True + self.save(update_fields=update_fields) def render(self) -> str: """Render the notification message using the event context.""" diff --git a/osf/models/notification_subscription.py b/osf/models/notification_subscription.py index 6a4a27533b5..1e38f4cbdff 100644 --- a/osf/models/notification_subscription.py +++ b/osf/models/notification_subscription.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime +from django.utils import timezone from django.db import models from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType @@ -59,6 +59,7 @@ class Meta: verbose_name = 'Notification Subscription' verbose_name_plural = 'Notification Subscriptions' db_table = 'osf_notificationsubscription_v2' + unique_together = ('notification_type', 'user', 'content_type', 'object_id', '_is_digest') def emit( self, @@ -126,7 +127,8 @@ def emit( Notification.objects.create( subscription=self, event_context=event_context, - sent=None if self.message_frequency != 'none' else datetime(1000, 1, 1), + sent=timezone.now() if self.message_frequency == 'none' else None, + fake_sent=True if self.message_frequency == 'none' else False, ) @property diff --git a/osf/models/notification_type.py b/osf/models/notification_type.py index fbbe2524f99..109e758c8da 100644 --- a/osf/models/notification_type.py +++ b/osf/models/notification_type.py @@ -151,6 +151,30 @@ def instance(self): obj, created = NotificationType.objects.get_or_create(name=self.value) return obj + @property + def is_digest_type(self): + digest_types = { + # User types + NotificationType.Type.USER_NO_ADDON.value, + # File types + NotificationType.Type.ADDON_FILE_COPIED.value, + NotificationType.Type.ADDON_FILE_MOVED.value, + NotificationType.Type.ADDON_FILE_RENAMED.value, + NotificationType.Type.FILE_ADDED.value, + NotificationType.Type.FILE_REMOVED.value, + NotificationType.Type.FILE_UPDATED.value, + NotificationType.Type.FOLDER_CREATED.value, + NotificationType.Type.NODE_FILE_UPDATED.value, + NotificationType.Type.USER_FILE_UPDATED.value, + + # Review types + NotificationType.Type.COLLECTION_SUBMISSION_SUBMITTED.value, + NotificationType.Type.PROVIDER_NEW_PENDING_SUBMISSIONS.value, + NotificationType.Type.PROVIDER_NEW_PENDING_WITHDRAW_REQUESTS.value, + NotificationType.Type.REVIEWS_SUBMISSION_STATUS.value, + } + return self.name in digest_types + notification_interval_choices = ArrayField( base_field=models.CharField(max_length=32), default=get_default_frequency_choices, @@ -202,13 +226,14 @@ def emit( used. """ from osf.models.notification_subscription import NotificationSubscription - from osf.models.provider import AbstractProvider - # use concrete model for AbstractProvider to specifically get the provider content type - if isinstance(subscribed_object, AbstractProvider): - content_type = ContentType.objects.get_for_model(subscribed_object, for_concrete_model=False) if subscribed_object else None - else: - content_type = ContentType.objects.get_for_model(subscribed_object) if subscribed_object else None + if is_digest != self.is_digest_type: + sentry.log_message(f'NotificationType.emit called with is_digest={is_digest} for ' + f'NotificationType {self.name} which has is_digest_type={self.is_digest_type}' + 'is_digest value will be overridden.') + is_digest = self.is_digest_type + + content_type = ContentType.objects.get_for_model(subscribed_object) if subscribed_object else None if message_frequency is None: message_frequency = self.get_group_frequency_or_default(user, subscribed_object, content_type) diff --git a/osf/models/preprint.py b/osf/models/preprint.py index 7a61b31db06..9938a0e7147 100644 --- a/osf/models/preprint.py +++ b/osf/models/preprint.py @@ -1094,7 +1094,6 @@ def _send_preprint_confirmation(self, auth): 'document_type': self.provider.preprint_word, 'notify_comment': not self.provider.reviews_comments_private }, - is_digest=True ) # FOLLOWING BEHAVIOR NOT SPECIFIC TO PREPRINTS diff --git a/osf/models/user.py b/osf/models/user.py index a8cbf66d5b3..8b8c0ede502 100644 --- a/osf/models/user.py +++ b/osf/models/user.py @@ -249,6 +249,7 @@ class OSFUser(DirtyFieldsMixin, GuidMixin, BaseModel, AbstractBaseUser, Permissi # } email_last_sent = NonNaiveDateTimeField(null=True, blank=True) + no_login_email_last_sent = NonNaiveDateTimeField(null=True, blank=True) change_password_last_attempt = NonNaiveDateTimeField(null=True, blank=True) # Logs number of times user attempted to change their password where their # old password was invalid diff --git a/osf_tests/management_commands/test_migrate_notifications.py b/osf_tests/management_commands/test_migrate_notifications.py index 1ea906d0d17..3e37f83167d 100644 --- a/osf_tests/management_commands/test_migrate_notifications.py +++ b/osf_tests/management_commands/test_migrate_notifications.py @@ -347,7 +347,7 @@ def test_provider_subscription_copy_group_frequency(self, user, node, provider): nt = NotificationSubscription.objects.get( user=user, notification_type__name=NotificationType.Type.PROVIDER_NEW_PENDING_SUBMISSIONS, - content_type=ContentType.objects.get_for_model(provider, for_concrete_model=False), + content_type=ContentType.objects.get_for_model(provider), object_id=provider.id, ) assert nt.message_frequency == 'none' diff --git a/scripts/populate_notification_subscriptions.py b/scripts/populate_notification_subscriptions.py deleted file mode 100644 index 557b9f2a47d..00000000000 --- a/scripts/populate_notification_subscriptions.py +++ /dev/null @@ -1,113 +0,0 @@ -import django -django.setup() - -from website.app import init_app -init_app(routes=False) - -from framework.celery_tasks import app as celery_app -from django.contrib.contenttypes.models import ContentType -from django.db.models import Count, F, OuterRef, Subquery, IntegerField, CharField -from django.db.models.functions import Cast -from osf.models import OSFUser, Node, NotificationSubscription, NotificationType - - -@celery_app.task(name='scripts.populate_notification_subscriptions') -def populate_notification_subscriptions(): - created = 0 - user_file_nt = NotificationType.Type.USER_FILE_UPDATED.instance - review_nt = NotificationType.Type.REVIEWS_SUBMISSION_STATUS.instance - node_file_nt = NotificationType.Type.NODE_FILE_UPDATED.instance - - user_ct = ContentType.objects.get_for_model(OSFUser) - node_ct = ContentType.objects.get_for_model(Node) - - reviews_qs = OSFUser.objects.exclude(subscriptions__notification_type__name=NotificationType.Type.REVIEWS_SUBMISSION_STATUS).distinct('id') - files_qs = OSFUser.objects.exclude(subscriptions__notification_type__name=NotificationType.Type.USER_FILE_UPDATED).distinct('id') - - node_notifications_sq = ( - NotificationSubscription.objects.filter( - content_type=node_ct, - notification_type=node_file_nt, - object_id=Cast(OuterRef('pk'), CharField()), - ).values( - 'object_id' - ).annotate( - cnt=Count('id') - ).values('cnt')[:1] - ) - - nodes_qs = ( - Node.objects - .annotate( - contributors_count=Count('_contributors', distinct=True), - notifications_count=Subquery( - node_notifications_sq, - output_field=IntegerField(), - ), - ).exclude(contributors_count=F('notifications_count')) - ) - - print(f"Creating REVIEWS_SUBMISSION_STATUS subscriptions for {reviews_qs.count()} users.") - for id, user in enumerate(reviews_qs, 1): - print(f"Processing user {id} / {reviews_qs.count()}") - try: - _, is_created = NotificationSubscription.objects.get_or_create( - notification_type=review_nt, - user=user, - content_type=user_ct, - object_id=user.id, - defaults={ - 'message_frequency': 'none', - }, - ) - if is_created: - created += 1 - except Exception as exeption: - print(exeption) - continue - - print(f"Creating USER_FILE_UPDATED subscriptions for {files_qs.count()} users.") - for id, user in enumerate(files_qs, 1): - print(f"Processing user {id} / {files_qs.count()}") - try: - _, is_created = NotificationSubscription.objects.get_or_create( - notification_type=user_file_nt, - user=user, - content_type=user_ct, - object_id=user.id, - defaults={ - '_is_digest': True, - 'message_frequency': 'none', - }, - ) - if is_created: - created += 1 - except Exception as exeption: - print(exeption) - continue - - print(f"Creating NODE_FILE_UPDATED subscriptions for {nodes_qs.count()} nodes.") - for id, node in enumerate(nodes_qs, 1): - print(f"Processing node {id} / {nodes_qs.count()}") - for contributor in node.contributors.all(): - try: - _, is_created = NotificationSubscription.objects.get_or_create( - notification_type=node_file_nt, - user=contributor, - content_type=node_ct, - object_id=node.id, - defaults={ - '_is_digest': True, - 'message_frequency': 'none', - }, - ) - if is_created: - created += 1 - except Exception as exeption: - print(exeption) - continue - - print(f"Created {created} subscriptions") - -if __name__ == '__main__': - populate_notification_subscriptions.delay() diff --git a/scripts/remove_after_use/merge_notification_subscription_provider_ct.py b/scripts/remove_after_use/merge_notification_subscription_provider_ct.py new file mode 100644 index 00000000000..50da93b2669 --- /dev/null +++ b/scripts/remove_after_use/merge_notification_subscription_provider_ct.py @@ -0,0 +1,31 @@ +import django +django.setup() + +from website.app import init_app +init_app(routes=False) + +from django.contrib.contenttypes.models import ContentType +from framework.celery_tasks import app as celery_app +from osf.models import NotificationSubscription + + +@celery_app.task(name='scripts.remove_after_use.merge_notification_subscription_provider_ct') +def merge_notification_subscription_provider_ct(): + + abstract_provider_ct = ContentType.objects.get_by_natural_key('osf', 'abstractprovider') + + provider_ct_list = [ + ContentType.objects.get_by_natural_key('osf', 'preprintprovider'), + ContentType.objects.get_by_natural_key('osf', 'registrationprovider'), + ContentType.objects.get_by_natural_key('osf', 'collectionprovider'), + ] + subscriptions = NotificationSubscription.objects.filter( + content_type__in=provider_ct_list + ) + subscriptions.update( + content_type=abstract_provider_ct + ) + + +if __name__ == '__main__': + merge_notification_subscription_provider_ct.delay() diff --git a/scripts/remove_after_use/populate_notification_subscriptions_node_file_updated.py b/scripts/remove_after_use/populate_notification_subscriptions_node_file_updated.py new file mode 100644 index 00000000000..560eb51ee88 --- /dev/null +++ b/scripts/remove_after_use/populate_notification_subscriptions_node_file_updated.py @@ -0,0 +1,128 @@ +import django +django.setup() + +from website.app import init_app +init_app(routes=False) + +from datetime import datetime +from framework.celery_tasks import app as celery_app +from django.contrib.contenttypes.models import ContentType +from django.db.models import Count, F, OuterRef, Subquery, IntegerField, CharField +from django.db.models.functions import Cast, Coalesce +from osf.models import Node, NotificationSubscription, NotificationType + + +@celery_app.task(name='scripts.remove_after_use.populate_notification_subscriptions_node_file_updated') +def populate_notification_subscriptions_node_file_updated(batch_size: int = 1000): + print('---Starting NODE_FILE_UPDATED subscriptions population script----') + global_start = datetime.now() + + node_file_nt = NotificationType.Type.NODE_FILE_UPDATED + + node_ct = ContentType.objects.get_for_model(Node) + + node_notifications_sq = ( + NotificationSubscription.objects.filter( + content_type=node_ct, + notification_type=node_file_nt.instance, + object_id=Cast(OuterRef('pk'), CharField()), + ).values( + 'object_id' + ).annotate( + cnt=Count('id') + ).values('cnt')[:1] + ) + + nodes_qs = ( + Node.objects + .filter(is_deleted=False) + .annotate( + contributors_count=Count('_contributors', distinct=True), + notifications_count=Coalesce( + Subquery( + node_notifications_sq, + output_field=IntegerField(), + ), + 0 + ), + ).exclude(contributors_count=F('notifications_count')) + ).iterator(chunk_size=batch_size) + + items_to_create = [] + total_created = 0 + batch_start = datetime.now() + count_nodes = 0 + count_contributors = 0 + for node in nodes_qs: + count_nodes += 1 + for contributor in node.contributors.all(): + count_contributors += 1 + items_to_create.append( + NotificationSubscription( + notification_type=node_file_nt.instance, + user=contributor, + content_type=node_ct, + object_id=node.id, + _is_digest=True, + message_frequency='none', + ) + ) + if len(items_to_create) >= batch_size: + print(f'Creating batch of {len(items_to_create)} subscriptions...') + try: + NotificationSubscription.objects.bulk_create( + items_to_create, + batch_size=batch_size, + ignore_conflicts=True, + ) + total_created += len(items_to_create) + items_to_create = [] + except Exception as exeption: + print(f"Error during bulk_create: {exeption}") + continue + finally: + items_to_create.clear() + batch_end = datetime.now() + print(f'Batch took {batch_end - batch_start}') + + if count_contributors % batch_size == 0: + print(f'Processed {count_nodes} nodes with {count_contributors} contributors, created {total_created} subscriptions') + batch_start = datetime.now() + + if items_to_create: + final_batch_start = datetime.now() + print(f'Creating final batch of {len(items_to_create)} subscriptions...') + try: + NotificationSubscription.objects.bulk_create( + items_to_create, + batch_size=batch_size, + ignore_conflicts=True, + ) + total_created += len(items_to_create) + except Exception as exeption: + print(f"Error during bulk_create: {exeption}") + final_batch_end = datetime.now() + print(f'Final batch took {final_batch_end - final_batch_start}') + + global_end = datetime.now() + print(f'Total time for NODE_FILE_UPDATED subscription population: {global_end - global_start}') + print(f'Created {total_created} subscriptions.') + print('----Creation finished----') + +@celery_app.task(name='scripts.remove_after_use.update_notification_subscriptions_node_file_updated') +def update_notification_subscriptions_node_file_updated(): + print('---Starting NODE_FILE_UPDATED subscriptions update script----') + + node_file_nt = NotificationType.Type.NODE_FILE_UPDATED + + updated_start = datetime.now() + updated = ( + NotificationSubscription.objects.filter( + notification_type__name=node_file_nt, + _is_digest=False, + ) + .update(_is_digest=True) + ) + updated_end = datetime.now() + print(f'Updated {updated} subscriptions. Took time: {updated_end - updated_start}') + print('Update finished.') diff --git a/scripts/remove_after_use/populate_notification_subscriptions_user_global_file_updated.py b/scripts/remove_after_use/populate_notification_subscriptions_user_global_file_updated.py new file mode 100644 index 00000000000..f34f1620fd8 --- /dev/null +++ b/scripts/remove_after_use/populate_notification_subscriptions_user_global_file_updated.py @@ -0,0 +1,111 @@ +import django +django.setup() + +from website.app import init_app +init_app(routes=False) + +from django.utils import timezone +from dateutil.relativedelta import relativedelta +from datetime import datetime +from framework.celery_tasks import app as celery_app +from django.contrib.contenttypes.models import ContentType +from osf.models import OSFUser, NotificationSubscription, NotificationType + +@celery_app.task(name='scripts.remove_after_use.populate_notification_subscriptions_user_global_file_updated') +def populate_notification_subscriptions_user_global_file_updated(per_last_years: int | None= None, batch_size: int = 1000): + print('---Starting USER_FILE_UPDATED subscriptions population script----') + global_start = datetime.now() + + user_file_updated_nt = NotificationType.Type.USER_FILE_UPDATED + user_ct = ContentType.objects.get_for_model(OSFUser) + if per_last_years: + from_date = timezone.now() - relativedelta(years=per_last_years) + user_qs = (OSFUser.objects + .filter(date_last_login__gte=from_date) + .exclude(subscriptions__notification_type__name=user_file_updated_nt) + .distinct('id') + .order_by('id') + .iterator(chunk_size=batch_size) + ) + else: + user_qs = (OSFUser.objects + .exclude(subscriptions__notification_type__name=user_file_updated_nt) + .distinct('id') + .order_by('id') + .iterator(chunk_size=batch_size) + ) + + items_to_create = [] + total_created = 0 + + batch_start = datetime.now() + for count, user in enumerate(user_qs, 1): + items_to_create.append( + NotificationSubscription( + notification_type=user_file_updated_nt.instance, + user=user, + content_type=user_ct, + object_id=user.id, + _is_digest=True, + message_frequency='none', + ) + ) + if len(items_to_create) >= batch_size: + print(f'Creating batch of {len(items_to_create)} subscriptions...') + try: + NotificationSubscription.objects.bulk_create( + items_to_create, + batch_size=batch_size, + ignore_conflicts=True, + ) + total_created += len(items_to_create) + except Exception as e: + print(f'Error during bulk_create: {e}') + finally: + items_to_create.clear() + batch_end = datetime.now() + print(f'Batch took {batch_end - batch_start}') + + if count % batch_size == 0: + print(f'Processed {count}, created {total_created}') + batch_start = datetime.now() + + if items_to_create: + final_batch_start = datetime.now() + print(f'Creating final batch of {len(items_to_create)} subscriptions...') + try: + NotificationSubscription.objects.bulk_create( + items_to_create, + batch_size=batch_size, + ignore_conflicts=True, + ) + total_created += len(items_to_create) + except Exception as e: + print(f'Error during bulk_create: {e}') + final_batch_end = datetime.now() + print(f'Final batch took {final_batch_end - final_batch_start}') + + global_end = datetime.now() + print(f'Total time for USER_FILE_UPDATED subscription population: {global_end - global_start}') + print(f'Created {total_created} subscriptions.') + print('----Creation finished----') + +@celery_app.task(name='scripts.remove_after_use.update_notification_subscriptions_user_global_file_updated') +def update_notification_subscriptions_user_global_file_updated(): + print('---Starting USER_FILE_UPDATED subscriptions updating script----') + + user_file_updated_nt = NotificationType.Type.USER_FILE_UPDATED + + update_start = datetime.now() + updated = ( + NotificationSubscription.objects + .filter( + notification_type__name=user_file_updated_nt, + _is_digest=False, + ) + .update(_is_digest=True) + ) + update_end = datetime.now() + + print(f'Updated {updated} subscriptions. Took time: {update_end - update_start}') + print('Update finished.') diff --git a/scripts/remove_after_use/populate_notification_subscriptions_user_global_reviews.py b/scripts/remove_after_use/populate_notification_subscriptions_user_global_reviews.py new file mode 100644 index 00000000000..02492cd2d4a --- /dev/null +++ b/scripts/remove_after_use/populate_notification_subscriptions_user_global_reviews.py @@ -0,0 +1,104 @@ +import django +django.setup() + +from website.app import init_app +init_app(routes=False) + +from django.utils import timezone +from dateutil.relativedelta import relativedelta +from datetime import datetime +from framework.celery_tasks import app as celery_app +from django.contrib.contenttypes.models import ContentType +from osf.models import OSFUser, NotificationSubscription, NotificationType + + +@celery_app.task(name='scripts.remove_after_use.populate_notification_subscriptions_user_global_reviews') +def populate_notification_subscriptions_user_global_reviews(per_last_years: int | None = None, batch_size: int = 1000): + print('---Starting REVIEWS_SUBMISSION_STATUS subscriptions population script----') + global_start = datetime.now() + + review_nt = NotificationType.Type.REVIEWS_SUBMISSION_STATUS + user_ct = ContentType.objects.get_for_model(OSFUser) + if per_last_years: + from_date = timezone.now() - relativedelta(years=per_last_years) + user_qs = OSFUser.objects.filter(date_last_login__gte=from_date).exclude( + subscriptions__notification_type__name=review_nt.instance + ).distinct('id') + else: + user_qs = OSFUser.objects.exclude( + subscriptions__notification_type__name=review_nt.instance + ).distinct('id') + + items_to_create = [] + total_created = 0 + + batch_start = datetime.now() + for count, user in enumerate(user_qs, 1): + items_to_create.append( + NotificationSubscription( + notification_type=review_nt.instance, + user=user, + content_type=user_ct, + object_id=user.id, + _is_digest=True, + message_frequency='none', + ) + ) + if len(items_to_create) >= batch_size: + print(f'Creating batch of {len(items_to_create)} subscriptions...') + try: + NotificationSubscription.objects.bulk_create( + items_to_create, + batch_size=batch_size, + ignore_conflicts=True, + ) + total_created += len(items_to_create) + except Exception as e: + print(f'Error during bulk_create: {e}') + finally: + items_to_create.clear() + batch_end = datetime.now() + print(f'Batch took {batch_end - batch_start}') + + if count % batch_size == 0: + print(f'Processed {count}, created {total_created}') + batch_start = datetime.now() + + if items_to_create: + final_batch_start = datetime.now() + print(f'Creating final batch of {len(items_to_create)} subscriptions...') + try: + NotificationSubscription.objects.bulk_create( + items_to_create, + batch_size=batch_size, + ignore_conflicts=True, + ) + total_created += len(items_to_create) + except Exception as e: + print(f'Error during bulk_create: {e}') + final_batch_end = datetime.now() + print(f'Final batch took {final_batch_end - final_batch_start}') + + global_end = datetime.now() + print(f'Total time for REVIEWS_SUBMISSION_STATUS subscription population: {global_end - global_start}') + print(f'Created {total_created} subscriptions.') + print('----Creation finished----') + +@celery_app.task(name='scripts.remove_after_use.update_notification_subscriptions_user_global_reviews') +def update_notification_subscriptions_user_global_reviews(): + print('---Starting REVIEWS_SUBMISSION_STATUS subscriptions updating script----') + + review_nt = NotificationType.Type.REVIEWS_SUBMISSION_STATUS + + updated_start = datetime.now() + updated = ( + NotificationSubscription.objects.filter( + notification_type__name=review_nt, + _is_digest=False, + ) + .update(_is_digest=True) + ) + updated_end = datetime.now() + + print(f'Updated {updated} subscriptions. Took time: {updated_end - updated_start}') + print('Update finished.') diff --git a/scripts/triggered_mails.py b/scripts/triggered_mails.py index 4b56d12c5df..8d06245f9e6 100644 --- a/scripts/triggered_mails.py +++ b/scripts/triggered_mails.py @@ -60,31 +60,29 @@ def find_inactive_users_without_enqueued_or_sent_no_login(): Match your original inactivity rules, but exclude users who already have a no_login EmailTask either pending, started, retrying, or already sent successfully. """ + now = timezone.now() - # Subquery: Is there already a not-yet-failed/aborted EmailTask for this user with our prefix? - existing_no_login = EmailTask.objects.filter( - user_id=OuterRef('pk'), - task_id__startswith=NO_LOGIN_PREFIX, - status__in=['PENDING', 'STARTED', 'RETRY', 'SUCCESS'], - ) cutoff_query = Q(date_last_login__gte=settings.NO_LOGIN_EMAIL_CUTOFF - settings.NO_LOGIN_WAIT_TIME) if settings.NO_LOGIN_EMAIL_CUTOFF else Q() base_q = OSFUser.objects.filter( cutoff_query, is_active=True, ).filter( Q( - date_last_login__lt=timezone.now() - settings.NO_LOGIN_WAIT_TIME, + date_last_login__lt=now - settings.NO_LOGIN_WAIT_TIME, # NOT tagged osf4m ) & ~Q(tags__name='osf4m') | Q( - date_last_login__lt=timezone.now() - settings.NO_LOGIN_OSF4M_WAIT_TIME, + date_last_login__lt=now - settings.NO_LOGIN_OSF4M_WAIT_TIME, tags__name='osf4m' ) ).distinct() - # Exclude users who already have a task for this email type - return base_q.annotate(_has_task=Exists(existing_no_login)).filter(_has_task=False) + # Exclude users who have already received a no-login email recently + return base_q.filter( + Q(no_login_email_last_sent__isnull=True) | + Q(no_login_email_last_sent__lt=now - settings.NO_LOGIN_WAIT_TIME) + ) @celery_app.task(name='scripts.triggered_no_login_email') @@ -133,6 +131,8 @@ def send_no_login_email(email_task_id: int): ) email_task.status = 'SUCCESS' email_task.save() + user.no_login_email_last_sent = timezone.now() + user.save() except Exception as exc: # noqa: BLE001 logger.exception(f'EmailTask {email_task.id}: error while sending') diff --git a/tests/test_triggered_mails.py b/tests/test_triggered_mails.py index c482338ccff..946c6a5e84b 100644 --- a/tests/test_triggered_mails.py +++ b/tests/test_triggered_mails.py @@ -22,7 +22,7 @@ def _inactive_time(): """Make a timestamp that is definitely 'inactive' regardless of threshold settings.""" # Very conservative: 12 weeks ago - return timezone.now() - timedelta(weeks=12) + return timezone.now() - timedelta(weeks=52) def _recent_time(): @@ -114,21 +114,15 @@ def test_finder_returns_two_inactive_when_none_queued(self): assert ids == {u1.id, u2.id} def test_finder_excludes_users_with_existing_task(self): - """Inactive users but one already has a no_login EmailTask -> excluded.""" + """Inactive users but one already has a no_login_email_last_sent -> excluded.""" u1 = UserFactory(fullname='Jalen Hurts') u2 = UserFactory(fullname='Jason Kelece') u1.date_last_login = _inactive_time() u2.date_last_login = _inactive_time() + u2.no_login_email_last_sent = timezone.now() u1.save() u2.save() - # Pretend u2 already had this email flow (SUCCESS qualifies for exclusion) - EmailTask.objects.create( - task_id=f"{NO_LOGIN_PREFIX}existing-success", - user=u2, - status='SUCCESS', - ) - users = list(find_inactive_users_without_enqueued_or_sent_no_login()) ids = {u.id for u in users} assert ids == {u1.id} # u2 excluded because of existing task diff --git a/website/notifications/__init__.py b/website/notifications/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/website/notifications/constants.py b/website/notifications/constants.py deleted file mode 100644 index ce3c9db4315..00000000000 --- a/website/notifications/constants.py +++ /dev/null @@ -1,39 +0,0 @@ -NODE_SUBSCRIPTIONS_AVAILABLE = { - 'file_updated': 'Files updated' -} - -# Note: if the subscription starts with 'global_', it will be treated like a default -# subscription. If no notification type has been assigned, the user subscription -# will default to 'email_transactional'. -USER_SUBSCRIPTIONS_AVAILABLE = { - 'global_file_updated': 'Files updated', - 'global_reviews': 'Preprint submissions updated' -} - -PROVIDER_SUBSCRIPTIONS_AVAILABLE = { - 'new_pending_submissions': 'New preprint submissions for moderators to review.' -} - -# Note: the python value None mean inherit from parent -NOTIFICATION_TYPES = { - 'email_transactional': 'Email when a change occurs', - 'email_digest': 'Daily email digest of all changes to this project', - 'none': 'None' -} - -# Formatted file provider names for notification emails -PROVIDERS = { - 'osfstorage': 'OSF Storage', - 'boa': 'Boa', - 'box': 'Box', - 'dataverse': 'Dataverse', - 'dropbox': 'Dropbox', - 'figshare': 'figshare', - 'github': 'GitHub', - 'gitlab': 'GitLab', - 'bitbucket': 'Bitbucket', - 'googledrive': 'Google Drive', - 'owncloud': 'ownCloud', - 'onedrive': 'Microsoft OneDrive', - 's3': 'Amazon S3' -} diff --git a/website/notifications/emails.py b/website/notifications/emails.py deleted file mode 100644 index e61c64e660a..00000000000 --- a/website/notifications/emails.py +++ /dev/null @@ -1,243 +0,0 @@ -from django.apps import apps - -from babel import dates, core, Locale - -from osf.models import AbstractNode, NotificationDigest, NotificationSubscription -from osf.utils.permissions import ADMIN, READ -from website import mails -from website.notifications import constants -from website.notifications import utils -from website.util import web_url_for - - -def notify(event, user, node, timestamp, **context): - """Retrieve appropriate ***subscription*** and passe user list - - :param event: event that triggered the notification - :param user: user who triggered notification - :param node: instance of Node - :param timestamp: time event happened - :param context: optional variables specific to templates - target_user: used with comment_replies - :return: List of user ids notifications were sent to - """ - sent_users = [] - # The user who the current comment is a reply to - target_user = context.get('target_user', None) - exclude = context.get('exclude', []) - # do not notify user who initiated the emails - exclude.append(user._id) - - event_type = utils.find_subscription_type(event) - if target_user and event_type in constants.USER_SUBSCRIPTIONS_AVAILABLE: - # global user - subscriptions = get_user_subscriptions(target_user, event_type) - else: - # local project user - subscriptions = compile_subscriptions(node, event_type, event) - - for notification_type in subscriptions: - if notification_type == 'none' or not subscriptions[notification_type]: - continue - # Remove excluded ids from each notification type - subscriptions[notification_type] = [guid for guid in subscriptions[notification_type] if guid not in exclude] - - # If target, they get a reply email and are removed from the general email - if target_user and target_user._id in subscriptions[notification_type]: - subscriptions[notification_type].remove(target_user._id) - store_emails([target_user._id], notification_type, 'comment_replies', user, node, timestamp, **context) - sent_users.append(target_user._id) - - if subscriptions[notification_type]: - store_emails(subscriptions[notification_type], notification_type, event_type, user, node, timestamp, **context) - sent_users.extend(subscriptions[notification_type]) - return sent_users - -def notify_mentions(event, user, node, timestamp, **context): - OSFUser = apps.get_model('osf', 'OSFUser') - recipient_ids = context.get('new_mentions', []) - recipients = OSFUser.objects.filter(guids___id__in=recipient_ids) - sent_users = notify_global_event(event, user, node, timestamp, recipients, context=context) - return sent_users - -def notify_global_event(event, sender_user, node, timestamp, recipients, template=None, context=None): - event_type = utils.find_subscription_type(event) - sent_users = [] - if not context: - context = {} - - for recipient in recipients: - subscriptions = get_user_subscriptions(recipient, event_type) - context['is_creator'] = recipient == node.creator - if node.provider: - context['has_psyarxiv_chronos_text'] = node.has_permission(recipient, ADMIN) and 'psyarxiv' in node.provider.name.lower() - for notification_type in subscriptions: - if (notification_type != 'none' and subscriptions[notification_type] and recipient._id in subscriptions[notification_type]): - store_emails([recipient._id], notification_type, event, sender_user, node, timestamp, template=template, **context) - sent_users.append(recipient._id) - - return sent_users - - -def store_emails(recipient_ids, notification_type, event, user, node, timestamp, abstract_provider=None, template=None, **context): - """Store notification emails - - Emails are sent via celery beat as digests - :param recipient_ids: List of user ids to send mail to. - :param notification_type: from constants.Notification_types - :param event: event that triggered notification - :param user: user who triggered the notification - :param node: instance of Node - :param timestamp: time event happened - :param context: - :return: -- - """ - OSFUser = apps.get_model('osf', 'OSFUser') - - if notification_type == 'none': - return - - # If `template` is not specified, default to using a template with name `event` - template = f'{template or event}.html.mako' - - # user whose action triggered email sending - context['user'] = user - node_lineage_ids = get_node_lineage(node) if node else [] - - for recipient_id in recipient_ids: - if recipient_id == user._id: - continue - recipient = OSFUser.load(recipient_id) - if recipient.is_disabled: - continue - context['localized_timestamp'] = localize_timestamp(timestamp, recipient) - context['recipient'] = recipient - message = mails.render_message(template, **context) - digest = NotificationDigest( - timestamp=timestamp, - send_type=notification_type, - event=event, - user=recipient, - message=message, - node_lineage=node_lineage_ids, - provider=abstract_provider - ) - digest.save() - - -def compile_subscriptions(node, event_type, event=None, level=0): - """Recurse through node and parents for subscriptions. - - :param node: current node - :param event_type: Generally node_subscriptions_available - :param event: Particular event such a file_updated that has specific file subs - :param level: How deep the recursion is - :return: a dict of notification types with lists of users. - """ - subscriptions = check_node(node, event_type) - if event: - subscriptions = check_node(node, event) # Gets particular event subscriptions - parent_subscriptions = compile_subscriptions(node, event_type, level=level + 1) # get node and parent subs - elif getattr(node, 'parent_id', False): - parent_subscriptions = \ - compile_subscriptions(AbstractNode.load(node.parent_id), event_type, level=level + 1) - else: - parent_subscriptions = check_node(None, event_type) - for notification_type in parent_subscriptions: - p_sub_n = parent_subscriptions[notification_type] - p_sub_n.extend(subscriptions[notification_type]) - for nt in subscriptions: - if notification_type != nt: - p_sub_n = list(set(p_sub_n).difference(set(subscriptions[nt]))) - if level == 0: - p_sub_n, removed = utils.separate_users(node, p_sub_n) - parent_subscriptions[notification_type] = p_sub_n - return parent_subscriptions - - -def check_node(node, event): - """Return subscription for a particular node and event.""" - node_subscriptions = {key: [] for key in constants.NOTIFICATION_TYPES} - if node: - subscription = NotificationSubscription.load(utils.to_subscription_key(node._id, event)) - for notification_type in node_subscriptions: - users = getattr(subscription, notification_type, []) - if users: - for user in users.exclude(date_disabled__isnull=False): - if node.has_permission(user, READ): - node_subscriptions[notification_type].append(user._id) - return node_subscriptions - - -def get_user_subscriptions(user, event): - if user.is_disabled: - return {} - user_subscription = NotificationSubscription.load(utils.to_subscription_key(user._id, event)) - if user_subscription: - return {key: list(getattr(user_subscription, key).all().values_list('guids___id', flat=True)) for key in constants.NOTIFICATION_TYPES} - else: - return {key: [user._id] if (event in constants.USER_SUBSCRIPTIONS_AVAILABLE and key == 'email_transactional') else [] for key in constants.NOTIFICATION_TYPES} - - -def get_node_lineage(node): - """ Get a list of node ids in order from the node to top most project - e.g. [parent._id, node._id] - """ - from osf.models import Preprint - lineage = [node._id] - if isinstance(node, Preprint): - return lineage - - while node.parent_id: - node = node.parent_node - lineage = [node._id] + lineage - - return lineage - - -def get_settings_url(uid, user): - if uid == user._id: - return web_url_for('user_notifications', _absolute=True) - - node = AbstractNode.load(uid) - assert node, 'get_settings_url received an invalid Node id' - return node.web_url_for('node_setting', _guid=True, _absolute=True) - -def fix_locale(locale): - """Attempt to fix a locale to have the correct casing, e.g. de_de -> de_DE - - This is NOT guaranteed to return a valid locale identifier. - """ - try: - language, territory = locale.split('_', 1) - except ValueError: - return locale - else: - return '_'.join([language, territory.upper()]) - -def localize_timestamp(timestamp, user): - try: - user_timezone = dates.get_timezone(user.timezone) - except LookupError: - user_timezone = dates.get_timezone('Etc/UTC') - - try: - user_locale = Locale(user.locale) - except core.UnknownLocaleError: - user_locale = Locale('en') - - # Do our best to find a valid locale - try: - user_locale.date_formats - except OSError: # An IOError will be raised if locale's casing is incorrect, e.g. de_de vs. de_DE - # Attempt to fix the locale, e.g. de_de -> de_DE - try: - user_locale = Locale(fix_locale(user.locale)) - user_locale.date_formats - except (core.UnknownLocaleError, OSError): - user_locale = Locale('en') - - formatted_date = dates.format_date(timestamp, format='full', locale=user_locale) - formatted_time = dates.format_time(timestamp, format='short', tzinfo=user_timezone, locale=user_locale) - - return f'{formatted_time} on {formatted_date}' diff --git a/website/notifications/utils.py b/website/notifications/utils.py deleted file mode 100644 index b9a9b6d10e3..00000000000 --- a/website/notifications/utils.py +++ /dev/null @@ -1,479 +0,0 @@ -import collections - -from django.apps import apps -from django.db.models import Q - -from osf.utils.permissions import READ -from website.notifications import constants -from website.notifications.exceptions import InvalidSubscriptionError -from website.project import signals - -class NotificationsDict(dict): - def __init__(self): - super().__init__() - self.update(messages=[], children=collections.defaultdict(NotificationsDict)) - - def add_message(self, keys, messages): - """ - :param keys: ordered list of project ids from parent to node (e.g. ['parent._id', 'node._id']) - :param messages: built email message for an event that occurred on the node - :return: nested dict with project/component ids as the keys with the message at the appropriate level - """ - d_to_use = self - - for key in keys: - d_to_use = d_to_use['children'][key] - - if not isinstance(messages, list): - messages = [messages] - - d_to_use['messages'].extend(messages) - - -def find_subscription_type(subscription): - """Find subscription type string within specific subscription. - Essentially removes extraneous parts of the string to get the type. - """ - subs_available = list(constants.USER_SUBSCRIPTIONS_AVAILABLE.keys()) - subs_available.extend(list(constants.NODE_SUBSCRIPTIONS_AVAILABLE.keys())) - for available in subs_available: - if available in subscription: - return available - - -def to_subscription_key(uid, event): - """Build the Subscription primary key for the given guid and event""" - return f'{uid}_{event}' - - -def from_subscription_key(key): - parsed_key = key.split('_', 1) - return { - 'uid': parsed_key[0], - 'event': parsed_key[1] - } - - -@signals.contributor_removed.connect -def remove_contributor_from_subscriptions(node, user): - """ Remove contributor from node subscriptions unless the user is an - admin on any of node's parent projects. - """ - Preprint = apps.get_model('osf.Preprint') - DraftRegistration = apps.get_model('osf.DraftRegistration') - # Preprints don't have subscriptions at this time - if isinstance(node, Preprint): - return - if isinstance(node, DraftRegistration): - return - - # If user still has permissions through being a contributor or group member, or has - # admin perms on a parent, don't remove their subscription - if not (node.is_contributor_or_group_member(user)) and user._id not in node.admin_contributor_or_group_member_ids: - node_subscriptions = get_all_node_subscriptions(user, node) - for subscription in node_subscriptions: - subscription.objects.filter( - user=user, - ).delete() - -def separate_users(node, user_ids): - """Separates users into ones with permissions and ones without given a list. - :param node: Node to separate based on permissions - :param user_ids: List of ids, will also take and return User instances - :return: list of subbed, list of removed user ids - """ - OSFUser = apps.get_model('osf.OSFUser') - removed = [] - subbed = [] - for user_id in user_ids: - try: - user = OSFUser.load(user_id) - except TypeError: - user = user_id - if node.has_permission(user, READ): - subbed.append(user_id) - else: - removed.append(user_id) - return subbed, removed - - -def users_to_remove(source_event, source_node, new_node): - """Find users that do not have permissions on new_node. - :param source_event: such as _file_updated - :param source_node: Node instance where a subscription currently resides - :param new_node: Node instance where a sub or new sub will be. - :return: Dict of notification type lists with user_ids - """ - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - removed_users = {key: [] for key in constants.NOTIFICATION_TYPES} - if source_node == new_node: - return removed_users - old_sub = NotificationSubscription.load(to_subscription_key(source_node._id, source_event)) - old_node_sub = NotificationSubscription.load(to_subscription_key(source_node._id, - '_'.join(source_event.split('_')[-2:]))) - if not old_sub and not old_node_sub: - return removed_users - for notification_type in constants.NOTIFICATION_TYPES: - users = [] - if hasattr(old_sub, notification_type): - users += list(getattr(old_sub, notification_type).values_list('guids___id', flat=True)) - if hasattr(old_node_sub, notification_type): - users += list(getattr(old_node_sub, notification_type).values_list('guids___id', flat=True)) - subbed, removed_users[notification_type] = separate_users(new_node, users) - return removed_users - - -def move_subscription(remove_users, source_event, source_node, new_event, new_node): - """Moves subscription from old_node to new_node - :param remove_users: dictionary of lists of users to remove from the subscription - :param source_event: A specific guid event _file_updated - :param source_node: Instance of Node - :param new_event: A specific guid event - :param new_node: Instance of Node - :return: Returns a NOTIFICATION_TYPES list of removed users without permissions - """ - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - if source_node == new_node: - return - old_sub = NotificationSubscription.load(to_subscription_key(source_node._id, source_event)) - if not old_sub: - return - elif old_sub: - old_sub._id = to_subscription_key(new_node._id, new_event) - old_sub.event_name = new_event - old_sub.owner = new_node - new_sub = old_sub - new_sub.save() - # Remove users that don't have permission on the new node. - for notification_type in constants.NOTIFICATION_TYPES: - if new_sub: - for user_id in remove_users[notification_type]: - related_manager = getattr(new_sub, notification_type, None) - subscriptions = related_manager.all() if related_manager else [] - if user_id in subscriptions: - new_sub.delete() - - -def get_configured_projects(user): - """Filter all user subscriptions for ones that are on parent projects - and return the node objects. - :param user: OSFUser object - :return: list of node objects for projects with no parent - """ - configured_projects = set() - user_subscriptions = get_all_user_subscriptions(user, extra=( - ~Q(node__type='osf.collection') & - Q(node__is_deleted=False) - )) - - for subscription in user_subscriptions: - # If the user has opted out of emails skip - node = subscription.subscribed_object - - if subscription.message_frequency == 'none': - continue - - root = node.root - - if not root.is_deleted: - configured_projects.add(root) - - return sorted(configured_projects, key=lambda n: n.title.lower()) - - -def check_project_subscriptions_are_all_none(user, node): - node_subscriptions = get_all_node_subscriptions(user, node) - for s in node_subscriptions: - if not s.none.filter(id=user.id).exists(): - return False - return True - - -def get_all_user_subscriptions(user, extra=None): - """ Get all Subscription objects that the user is subscribed to""" - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - queryset = NotificationSubscription.objects.filter( - Q(none=user.pk) | - Q(email_digest=user.pk) | - Q(email_transactional=user.pk) - ).distinct() - return queryset.filter(extra) if extra else queryset - - -def get_all_node_subscriptions(user, node, user_subscriptions=None): - """ Get all Subscription objects for a node that the user is subscribed to - :param user: OSFUser object - :param node: Node object - :param user_subscriptions: all Subscription objects that the user is subscribed to - :return: list of Subscription objects for a node that the user is subscribed to - """ - if not user_subscriptions: - user_subscriptions = get_all_user_subscriptions(user) - return user_subscriptions.filter(user__isnull=True, node=node) - - -def format_data(user, nodes): - """ Format subscriptions data for project settings page - :param user: OSFUser object - :param nodes: list of parent project node objects - :return: treebeard-formatted data - """ - items = [] - - user_subscriptions = get_all_user_subscriptions(user) - for node in nodes: - assert node, f'{node._id} is not a valid Node.' - - can_read = node.has_permission(user, READ) - can_read_children = node.has_permission_on_children(user, READ) - - if not can_read and not can_read_children: - continue - - children = node.get_nodes(**{'is_deleted': False, 'is_node_link': False}) - children_tree = [] - # List project/node if user has at least READ permissions (contributor or admin viewer) or if - # user is contributor on a component of the project/node - - if can_read: - node_sub_available = list(constants.NODE_SUBSCRIPTIONS_AVAILABLE.keys()) - subscriptions = get_all_node_subscriptions( - user, - node, - user_subscriptions=user_subscriptions).filter(notification_type__name__in=node_sub_available) - - for subscription in subscriptions: - index = node_sub_available.index(getattr(subscription, 'event_name')) - children_tree.append(serialize_event(user, subscription=subscription, - node=node, event_description=node_sub_available.pop(index))) - for node_sub in node_sub_available: - children_tree.append(serialize_event(user, node=node, event_description=node_sub)) - children_tree.sort(key=lambda s: s['event']['title']) - - children_tree.extend(format_data(user, children)) - - item = { - 'node': { - 'id': node._id, - 'url': node.url if can_read else '', - 'title': node.title if can_read else 'Private Project', - }, - 'children': children_tree, - 'kind': 'folder' if not node.parent_node or not node.parent_node.has_permission(user, READ) else 'node', - 'nodeType': node.project_or_component, - 'category': node.category, - 'permissions': { - 'view': can_read, - }, - } - - items.append(item) - - return items - - -def format_user_subscriptions(user): - """ Format user-level subscriptions (e.g. comment replies across the OSF) for user settings page""" - user_subs_available = list(constants.USER_SUBSCRIPTIONS_AVAILABLE.keys()) - subscriptions = [ - serialize_event( - user, subscription, - event_description=user_subs_available.pop(user_subs_available.index(getattr(subscription, 'event_name'))) - ) - for subscription in get_all_user_subscriptions(user) - if subscription is not None and getattr(subscription, 'event_name') in user_subs_available - ] - subscriptions.extend([serialize_event(user, event_description=sub) for sub in user_subs_available]) - return subscriptions - - -def format_file_subscription(user, node_id, path, provider): - """Format a single file event""" - AbstractNode = apps.get_model('osf.AbstractNode') - node = AbstractNode.load(node_id) - wb_path = path.lstrip('/') - for subscription in get_all_node_subscriptions(user, node): - if wb_path in getattr(subscription, 'event_name'): - return serialize_event(user, subscription, node) - return serialize_event(user, node=node, event_description='file_updated') - - -all_subs = constants.NODE_SUBSCRIPTIONS_AVAILABLE.copy() -all_subs.update(constants.USER_SUBSCRIPTIONS_AVAILABLE) - -def serialize_event(user, subscription=None, node=None, event_description=None): - """ - :param user: OSFUser object - :param subscription: Subscription object, use if parsing particular subscription - :param node: Node object, use if node is known - :param event_description: use if specific subscription is known - :return: treebeard-formatted subscription event - """ - if not event_description: - event_description = getattr(subscription, 'event_name') - # Looks at only the types available. Deals with pre-pending file names. - for sub_type in all_subs: - if sub_type in event_description: - event_type = sub_type - else: - event_type = event_description - if node and node.parent_node: - notification_type = 'adopt_parent' - elif event_type.startswith('global_'): - notification_type = 'email_transactional' - else: - notification_type = 'none' - if subscription: - for n_type in constants.NOTIFICATION_TYPES: - if getattr(subscription, n_type).filter(id=user.id).exists(): - notification_type = n_type - return { - 'event': { - 'title': event_description, - 'description': all_subs[event_type], - 'notificationType': notification_type, - 'parent_notification_type': get_parent_notification_type(node, event_type, user) - }, - 'kind': 'event', - 'children': [] - } - - -def get_parent_notification_type(node, event, user): - """ - Given an event on a node (e.g. comment on node 'xyz'), find the user's notification - type on the parent project for the same event. - :param obj node: event owner (Node or User object) - :param str event: notification event (e.g. 'comment_replies') - :param obj user: OSFUser object - :return: str notification type (e.g. 'email_transactional') - """ - AbstractNode = apps.get_model('osf.AbstractNode') - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - - if node and isinstance(node, AbstractNode) and node.parent_node and node.parent_node.has_permission(user, READ): - parent = node.parent_node - key = to_subscription_key(parent._id, event) - try: - subscription = NotificationSubscription.objects.get(_id=key) - except NotificationSubscription.DoesNotExist: - return get_parent_notification_type(parent, event, user) - - for notification_type in constants.NOTIFICATION_TYPES: - if getattr(subscription, notification_type).filter(id=user.id).exists(): - return notification_type - else: - return get_parent_notification_type(parent, event, user) - else: - return None - - -def get_global_notification_type(global_subscription, user): - """ - Given a global subscription (e.g. NotificationSubscription object with event_type - 'global_file_updated'), find the user's notification type. - :param obj global_subscription: NotificationSubscription object - :param obj user: OSFUser object - :return: str notification type (e.g. 'email_transactional') - """ - for notification_type in constants.NOTIFICATION_TYPES: - # TODO Optimize me - if getattr(global_subscription, notification_type).filter(id=user.id).exists(): - return notification_type - - -def check_if_all_global_subscriptions_are_none(user): - # This function predates comment mentions, which is a global_ notification that cannot be disabled - # Therefore, an actual check would never return True. - # If this changes, an optimized query would look something like: - # not NotificationSubscription.objects.filter(Q(event_name__startswith='global_') & (Q(email_digest=user.pk)|Q(email_transactional=user.pk))).exists() - return False - - -def subscribe_user_to_global_notifications(user): - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - notification_type = 'email_transactional' - user_events = constants.USER_SUBSCRIPTIONS_AVAILABLE - for user_event in user_events: - user_event_id = to_subscription_key(user._id, user_event) - - # get_or_create saves on creation - subscription, created = NotificationSubscription.objects.get_or_create(_id=user_event_id, user=user, event_name=user_event) - subscription.add_user_to_subscription(user, notification_type) - subscription.save() - - -def subscribe_user_to_notifications(node, user): - """ Update the notification settings for the creator or contributors - :param user: User to subscribe to notifications - """ - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - Preprint = apps.get_model('osf.Preprint') - DraftRegistration = apps.get_model('osf.DraftRegistration') - if isinstance(node, Preprint): - raise InvalidSubscriptionError('Preprints are invalid targets for subscriptions at this time.') - - if isinstance(node, DraftRegistration): - raise InvalidSubscriptionError('DraftRegistrations are invalid targets for subscriptions at this time.') - - if node.is_collection: - raise InvalidSubscriptionError('Collections are invalid targets for subscriptions') - - if node.is_deleted: - raise InvalidSubscriptionError('Deleted Nodes are invalid targets for subscriptions') - - if getattr(node, 'is_registration', False): - raise InvalidSubscriptionError('Registrations are invalid targets for subscriptions') - - events = constants.NODE_SUBSCRIPTIONS_AVAILABLE - notification_type = 'email_transactional' - target_id = node._id - - if user.is_registered: - for event in events: - event_id = to_subscription_key(target_id, event) - global_event_id = to_subscription_key(user._id, 'global_' + event) - global_subscription = NotificationSubscription.load(global_event_id) - - subscription = NotificationSubscription.load(event_id) - - # If no subscription for component and creator is the user, do not create subscription - # If no subscription exists for the component, this means that it should adopt its - # parent's settings - if not (node and node.parent_node and not subscription and node.creator == user): - if not subscription: - subscription = NotificationSubscription(_id=event_id, owner=node, event_name=event) - # Need to save here in order to access m2m fields - subscription.save() - if global_subscription: - global_notification_type = get_global_notification_type(global_subscription, user) - subscription.add_user_to_subscription(user, global_notification_type) - else: - subscription.add_user_to_subscription(user, notification_type) - subscription.save() - - -def format_user_and_project_subscriptions(user): - """ Format subscriptions data for user settings page. """ - return [ - { - 'node': { - 'id': user._id, - 'title': 'Default Notification Settings', - 'help': 'These are default settings for new projects you create ' + - 'or are added to. Modifying these settings will not ' + - 'modify settings on existing projects.' - }, - 'kind': 'heading', - 'children': format_user_subscriptions(user) - }, - { - 'node': { - 'id': '', - 'title': 'Project Notifications', - 'help': 'These are settings for each of your projects. Modifying ' + - 'these settings will only modify the settings for the selected project.' - }, - 'kind': 'heading', - 'children': format_data(user, get_configured_projects(user)) - }] diff --git a/website/notifications/views.py b/website/notifications/views.py deleted file mode 100644 index 700594f69d6..00000000000 --- a/website/notifications/views.py +++ /dev/null @@ -1,540 +0,0 @@ -from django.contrib.contenttypes.models import ContentType -from rest_framework import status as http_status - -from flask import request - -from framework import sentry -from framework.auth.decorators import must_be_logged_in -from framework.exceptions import HTTPError - -from osf.models import AbstractNode, Registration, Node - -NOTIFICATION_TYPES = {} -USER_SUBSCRIPTIONS_AVAILABLE = {} -NODE_SUBSCRIPTIONS_AVAILABLE = {} -from website.project.decorators import must_be_valid_project -import collections - -from django.apps import apps -from django.db.models import Q - -from osf.models import NotificationSubscription -from osf.utils.permissions import READ - - -class NotificationsDict(dict): - def __init__(self): - super().__init__() - self.update(messages=[], children=collections.defaultdict(NotificationsDict)) - - def add_message(self, keys, messages): - """ - :param keys: ordered list of project ids from parent to node (e.g. ['parent._id', 'node._id']) - :param messages: built email message for an event that occurred on the node - :return: nested dict with project/component ids as the keys with the message at the appropriate level - """ - d_to_use = self - - for key in keys: - d_to_use = d_to_use['children'][key] - - if not isinstance(messages, list): - messages = [messages] - - d_to_use['messages'].extend(messages) - - -def find_subscription_type(subscription): - """Find subscription type string within specific subscription. - Essentially removes extraneous parts of the string to get the type. - """ - subs_available = list(USER_SUBSCRIPTIONS_AVAILABLE.keys()) - subs_available.extend(list(NODE_SUBSCRIPTIONS_AVAILABLE.keys())) - for available in subs_available: - if available in subscription: - return available - - -def to_subscription_key(uid, event): - """Build the Subscription primary key for the given guid and event""" - return f'{uid}_{event}' - - -def from_subscription_key(key): - parsed_key = key.split('_', 1) - return { - 'uid': parsed_key[0], - 'event': parsed_key[1] - } - - -def users_to_remove(source_event, source_node, new_node): - """Find users that do not have permissions on new_node. - :param source_event: such as _file_updated - :param source_node: Node instance where a subscription currently resides - :param new_node: Node instance where a sub or new sub will be. - :return: Dict of notification type lists with user_ids - """ - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - removed_users = {key: [] for key in NOTIFICATION_TYPES} - if source_node == new_node: - return removed_users - old_sub = NotificationSubscription.objects.get( - subscribed_object=source_node, - notification_type__name=source_event - ) - for notification_type in NOTIFICATION_TYPES: - users = [] - if hasattr(old_sub, notification_type): - users += list(getattr(old_sub, notification_type).values_list('guids___id', flat=True)) - return removed_users - - -def move_subscription(remove_users, source_event, source_node, new_event, new_node): - """Moves subscription from old_node to new_node - :param remove_users: dictionary of lists of users to remove from the subscription - :param source_event: A specific guid event _file_updated - :param source_node: Instance of Node - :param new_event: A specific guid event - :param new_node: Instance of Node - :return: Returns a NOTIFICATION_TYPES list of removed users without permissions - """ - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - OSFUser = apps.get_model('osf.OSFUser') - if source_node == new_node: - return - old_sub = NotificationSubscription.load(to_subscription_key(source_node._id, source_event)) - if not old_sub: - return - elif old_sub: - old_sub._id = to_subscription_key(new_node._id, new_event) - old_sub.event_name = new_event - old_sub.owner = new_node - new_sub = old_sub - new_sub.save() - # Remove users that don't have permission on the new node. - for notification_type in NOTIFICATION_TYPES: - if new_sub: - for user_id in remove_users[notification_type]: - related_manager = getattr(new_sub, notification_type, None) - subscriptions = related_manager.all() if related_manager else [] - if user_id in subscriptions: - user = OSFUser.load(user_id) - new_sub.remove_user_from_subscription(user) - - -def get_configured_projects(user): - """Filter all user subscriptions for ones that are on parent projects - and return the node objects. - :param user: OSFUser object - :return: list of node objects for projects with no parent - """ - configured_projects = set() - user_subscriptions = get_all_user_subscriptions(user, extra=( - ~Q(node__type='osf.collection') & - Q(node__is_deleted=False) - )) - - for subscription in user_subscriptions: - # If the user has opted out of emails skip - node = subscription.subscribed_object - - if subscription.message_frequency == 'none': - continue - if isinstance(node, Node): - root = node.root - - if not root.is_deleted: - configured_projects.add(root) - - return sorted(configured_projects, key=lambda n: n.title.lower()) - - -def check_project_subscriptions_are_all_none(user, node): - node_subscriptions = NotificationSubscription.objects.filter( - user=user, - object_id=node.id, - content_type=ContentType.objects.get_for_model(node).id, - ) - for s in node_subscriptions: - if not s.message_frequecy == 'none': - return False - return True - - -def get_all_user_subscriptions(user, extra=None): - """ Get all Subscription objects that the user is subscribed to""" - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - return NotificationSubscription.objects.filter( - user=user, - ) - - -def get_all_node_subscriptions(user, node, user_subscriptions=None): - """ Get all Subscription objects for a node that the user is subscribed to - :param user: OSFUser object - :param node: Node object - :param user_subscriptions: all Subscription objects that the user is subscribed to - :return: list of Subscription objects for a node that the user is subscribed to - """ - if not user_subscriptions: - user_subscriptions = get_all_user_subscriptions(user) - return user_subscriptions.filter( - object_id=node.id, - content_type=ContentType.objects.get_for_model(node).id, - ) - - -def format_data(user, nodes): - """ Format subscriptions data for project settings page - :param user: OSFUser object - :param nodes: list of parent project node objects - :return: treebeard-formatted data - """ - items = [] - - user_subscriptions = get_all_user_subscriptions(user) - for node in nodes: - assert node, f'{node._id} is not a valid Node.' - - can_read = node.has_permission(user, READ) - can_read_children = node.has_permission_on_children(user, READ) - - if not can_read and not can_read_children: - continue - - children = node.get_nodes(**{'is_deleted': False, 'is_node_link': False}) - children_tree = [] - # List project/node if user has at least READ permissions (contributor or admin viewer) or if - # user is contributor on a component of the project/node - - if can_read: - node_sub_available = list(NODE_SUBSCRIPTIONS_AVAILABLE.keys()) - subscriptions = get_all_node_subscriptions( - user, - node, - user_subscriptions=user_subscriptions - ).filter( - notification_type__name__in=node_sub_available - ) - - for subscription in subscriptions: - index = node_sub_available.index(getattr(subscription, 'event_name')) - children_tree.append(serialize_event(user, subscription=subscription, - node=node, event_description=node_sub_available.pop(index))) - for node_sub in node_sub_available: - children_tree.append(serialize_event(user, node=node, event_description=node_sub)) - children_tree.sort(key=lambda s: s['event']['title']) - - children_tree.extend(format_data(user, children)) - - item = { - 'node': { - 'id': node._id, - 'url': node.url if can_read else '', - 'title': node.title if can_read else 'Private Project', - }, - 'children': children_tree, - 'kind': 'folder' if not node.parent_node or not node.parent_node.has_permission(user, READ) else 'node', - 'nodeType': node.project_or_component, - 'category': node.category, - 'permissions': { - 'view': can_read, - }, - } - - items.append(item) - - return items - - -def format_user_subscriptions(user): - """ Format user-level subscriptions (e.g. comment replies across the OSF) for user settings page""" - user_subs_available = list(USER_SUBSCRIPTIONS_AVAILABLE.keys()) - subscriptions = [ - serialize_event( - user, subscription, - event_description=user_subs_available.pop(user_subs_available.index(getattr(subscription, 'event_name'))) - ) - for subscription in get_all_user_subscriptions(user) - if subscription is not None in user_subs_available - ] - subscriptions.extend([serialize_event(user, event_description=sub) for sub in user_subs_available]) - return subscriptions - - -def format_file_subscription(user, node_id, path, provider): - """Format a single file event""" - AbstractNode = apps.get_model('osf.AbstractNode') - node = AbstractNode.load(node_id) - wb_path = path.lstrip('/') - for subscription in get_all_node_subscriptions(user, node): - if wb_path in getattr(subscription, 'event_name'): - return serialize_event(user, subscription, node) - return serialize_event(user, node=node, event_description='file_updated') - -def serialize_event(user, subscription=None, node=None, event_description=None): - """ - :param user: OSFUser object - :param subscription: Subscription object, use if parsing particular subscription - :param node: Node object, use if node is known - :param event_description: use if specific subscription is known - :return: treebeard-formatted subscription event - """ - if not event_description: - event_description = getattr(subscription, 'event_name') - # Looks at only the types available. Deals with pre-pending file names. - for sub_type in {}: - if sub_type in event_description: - event_type = sub_type - else: - event_type = event_description - if node and node.parent_node: - notification_type = 'adopt_parent' - elif event_type.startswith('global_'): - notification_type = 'email_transactional' - else: - notification_type = 'none' - if subscription: - for n_type in {}: - if getattr(subscription, n_type).filter(id=user.id).exists(): - notification_type = n_type - return { - 'event': { - 'title': event_description, - 'description': {}[event_type], - 'notificationType': notification_type, - 'parent_notification_type': get_parent_notification_type(node, event_type, user) - }, - 'kind': 'event', - 'children': [] - } - - -def get_parent_notification_type(node, event, user): - """ - Given an event on a node (e.g. comment on node 'xyz'), find the user's notification - type on the parent project for the same event. - :param obj node: event owner (Node or User object) - :param str event: notification event (e.g. 'comment_replies') - :param obj user: OSFUser object - :return: str notification type (e.g. 'email_transactional') - """ - AbstractNode = apps.get_model('osf.AbstractNode') - NotificationSubscriptionLegacy = apps.get_model('osf.NotificationSubscriptionLegacy') - - if node and isinstance(node, AbstractNode) and node.parent_node and node.parent_node.has_permission(user, READ): - parent = node.parent_node - key = to_subscription_key(parent._id, event) - try: - subscription = NotificationSubscriptionLegacy.objects.get(_id=key) - except NotificationSubscriptionLegacy.DoesNotExist: - return get_parent_notification_type(parent, event, user) - - for notification_type in NOTIFICATION_TYPES: - if getattr(subscription, notification_type).filter(id=user.id).exists(): - return notification_type - else: - return get_parent_notification_type(parent, event, user) - else: - return None - - -def get_global_notification_type(global_subscription, user): - """ - Given a global subscription (e.g. NotificationSubscription object with event_type - 'global_file_updated'), find the user's notification type. - :param obj global_subscription: NotificationSubscription object - :param obj user: OSFUser object - :return: str notification type (e.g. 'email_transactional') - """ - for notification_type in NOTIFICATION_TYPES: - # TODO Optimize me - if getattr(global_subscription, notification_type).filter(id=user.id).exists(): - return notification_type - - -def check_if_all_global_subscriptions_are_none(user): - # This function predates comment mentions, which is a global_ notification that cannot be disabled - # Therefore, an actual check would never return True. - # If this changes, an optimized query would look something like: - # not NotificationSubscriptionLegacy.objects.filter(Q(event_name__startswith='global_') & (Q(email_digest=user.pk)|Q(email_transactional=user.pk))).exists() - return False - - -def subscribe_user_to_global_notifications(user): - NotificationSubscriptionLegacy = apps.get_model('osf.NotificationSubscriptionLegacy') - notification_type = 'email_transactional' - user_events = USER_SUBSCRIPTIONS_AVAILABLE - for user_event in user_events: - user_event_id = to_subscription_key(user._id, user_event) - - # get_or_create saves on creation - subscription, created = NotificationSubscriptionLegacy.objects.get_or_create(_id=user_event_id, user=user, event_name=user_event) - subscription.add_user_to_subscription(user, notification_type) - subscription.save() - - -class InvalidSubscriptionError: - pass - - -def subscribe_user_to_notifications(node, user): - """ Update the notification settings for the creator or contributors - :param user: User to subscribe to notifications - """ - NotificationSubscription = apps.get_model('osf.NotificationSubscription') - Preprint = apps.get_model('osf.Preprint') - DraftRegistration = apps.get_model('osf.DraftRegistration') - if isinstance(node, Preprint): - raise InvalidSubscriptionError('Preprints are invalid targets for subscriptions at this time.') - - if isinstance(node, DraftRegistration): - raise InvalidSubscriptionError('DraftRegistrations are invalid targets for subscriptions at this time.') - - if node.is_collection: - raise InvalidSubscriptionError('Collections are invalid targets for subscriptions') - - if node.is_deleted: - raise InvalidSubscriptionError('Deleted Nodes are invalid targets for subscriptions') - - if getattr(node, 'is_registration', False): - raise InvalidSubscriptionError('Registrations are invalid targets for subscriptions') - - events = NODE_SUBSCRIPTIONS_AVAILABLE - - if user.is_registered: - for event in events: - subscription, _ = NotificationSubscription.objects.get_or_create( - user=user, - notification_type__name=event - ) - - -def format_user_and_project_subscriptions(user): - """ Format subscriptions data for user settings page. """ - return [ - { - 'node': { - 'id': user._id, - 'title': 'Default Notification Settings', - 'help': 'These are default settings for new projects you create ' + - 'or are added to. Modifying these settings will not ' + - 'modify settings on existing projects.' - }, - 'kind': 'heading', - 'children': format_user_subscriptions(user) - }, - { - 'node': { - 'id': '', - 'title': 'Project Notifications', - 'help': 'These are settings for each of your projects. Modifying ' + - 'these settings will only modify the settings for the selected project.' - }, - 'kind': 'heading', - 'children': format_data(user, get_configured_projects(user)) - }] - - -@must_be_logged_in -def get_subscriptions(auth): - return format_user_and_project_subscriptions(auth.user) - - -@must_be_logged_in -@must_be_valid_project -def get_node_subscriptions(auth, **kwargs): - node = kwargs.get('node') or kwargs['project'] - return format_data(auth.user, [node]) - - -@must_be_logged_in -def get_file_subscriptions(auth, **kwargs): - node_id = request.args.get('node_id') - path = request.args.get('path') - provider = request.args.get('provider') - return format_file_subscription(auth.user, node_id, path, provider) - - -@must_be_logged_in -def configure_subscription(auth): - user = auth.user - json_data = request.get_json() - target_id = json_data.get('id') - event = json_data.get('event') - notification_type = json_data.get('notification_type') - path = json_data.get('path') - provider = json_data.get('provider') - - if not event or (notification_type not in NOTIFICATION_TYPES and notification_type != 'adopt_parent'): - raise HTTPError(http_status.HTTP_400_BAD_REQUEST, data=dict( - message_long='Must provide an event and notification type for subscription.') - ) - - node = AbstractNode.load(target_id) - if 'file_updated' in event and path is not None and provider is not None: - wb_path = path.lstrip('/') - event = wb_path + '_file_updated' - event_id = to_subscription_key(target_id, event) - - if not node: - # if target_id is not a node it currently must be the current user - if not target_id == user._id: - sentry.log_message( - '{!r} attempted to subscribe to either a bad ' - 'id or non-node non-self id, {}'.format(user, target_id) - ) - raise HTTPError(http_status.HTTP_404_NOT_FOUND) - - if notification_type == 'adopt_parent': - sentry.log_message( - f'{user!r} attempted to adopt_parent of a none node id, {target_id}' - ) - raise HTTPError(http_status.HTTP_400_BAD_REQUEST) - # owner = user - else: - if not node.has_permission(user, READ): - sentry.log_message(f'{user!r} attempted to subscribe to private node, {target_id}') - raise HTTPError(http_status.HTTP_403_FORBIDDEN) - - if isinstance(node, Registration): - sentry.log_message( - f'{user!r} attempted to subscribe to registration, {target_id}' - ) - raise HTTPError(http_status.HTTP_400_BAD_REQUEST) - - if notification_type != 'adopt_parent': - pass - # owner = node - else: - if 'file_updated' in event and len(event) > len('file_updated'): - pass - else: - parent = node.parent_node - if not parent: - sentry.log_message( - '{!r} attempted to adopt_parent of ' - 'the parentless project, {!r}'.format(user, node) - ) - raise HTTPError(http_status.HTTP_400_BAD_REQUEST) - - # If adopt_parent make sure that this subscription is None for the current User - subscription, _ = NotificationSubscription.objects.get_or_create( - user=user, - subscribed_object=node, - notification_type__name=event - ) - if not subscription: - return {} # We're done here - - subscription.delete() - return {} - - subscription, _ = NotificationSubscription.objects.get_or_create( - user=user, - notification_type__name=event - ) - subscription.save() - - return {'message': f'Successfully subscribed to {notification_type} list on {event_id}'} diff --git a/website/routes.py b/website/routes.py index 80b0d8bec92..d7d6cf3d9bc 100644 --- a/website/routes.py +++ b/website/routes.py @@ -56,7 +56,6 @@ from website.registries import views as registries_views from website.reviews import views as reviews_views from website.institutions import views as institution_views -from website.notifications import views as notification_views from website.ember_osf_web import views as ember_osf_web_views from website.closed_challenges import views as closed_challenges_views from website.identifiers import views as identifier_views @@ -1712,22 +1711,24 @@ def make_url_map(app): json_renderer, ), - Rule( - '/subscriptions/', - 'get', - notification_views.get_subscriptions, - json_renderer, - ), + # Legacy v1 API for notifications, which is no longer used by Angular/Post-NR + # Rule( + # '/subscriptions/', + # 'get', + # notification_views.get_subscriptions, + # json_renderer, + # ), - Rule( - [ - '/project//subscriptions/', - '/project//node//subscriptions/' - ], - 'get', - notification_views.get_node_subscriptions, - json_renderer, - ), + # Legacy v1 API for notifications, which is no longer used by Angular/Post-NR + # Rule( + # [ + # '/project//subscriptions/', + # '/project//node//subscriptions/' + # ], + # 'get', + # notification_views.get_node_subscriptions, + # json_renderer, + # ), Rule( [ @@ -1739,12 +1740,13 @@ def make_url_map(app): json_renderer, ), - Rule( - '/subscriptions/', - 'post', - notification_views.configure_subscription, - json_renderer, - ), + # Legacy v1 API for notifications, which is no longer used by Angular/Post-NR + # Rule( + # '/subscriptions/', + # 'post', + # notification_views.configure_subscription, + # json_renderer, + # ), Rule( [ diff --git a/website/settings/defaults.py b/website/settings/defaults.py index d09e583c181..e071b37d88e 100644 --- a/website/settings/defaults.py +++ b/website/settings/defaults.py @@ -186,6 +186,7 @@ def parent_dir(path): NO_ADDON_WAIT_TIME = timedelta(weeks=8) # 2 months for "Link an add-on to your OSF project" email NO_LOGIN_WAIT_TIME = timedelta(weeks=52) # 1 year for "We miss you at OSF" email NO_LOGIN_OSF4M_WAIT_TIME = timedelta(weeks=52) # 1 year for "We miss you at OSF" email to users created from OSF4M +NOTIFICATIONS_CLEANUP_AGE = timedelta(weeks=52) # 1 month to clean up old notifications and email tasks # Configuration for "We miss you at OSF" email (`NotificationType.Type.USER_NO_LOGIN`) # Note: 1) we can gradually increase `MAX_DAILY_NO_LOGIN_EMAILS` to 10000, 100000, etc. or set it to `None` after we @@ -435,6 +436,12 @@ class CeleryConfig: 'scripts.populate_new_and_noteworthy_projects', 'website.search.elastic_search', 'scripts.generate_sitemap', + 'scripts.remove_after_use.populate_notification_subscriptions_node_file_updated', + 'scripts.remove_after_use.update_notification_subscriptions_node_file_updated', + 'scripts.remove_after_use.populate_notification_subscriptions_user_global_file_updated', + 'scripts.remove_after_use.update_notification_subscriptions_user_global_file_updated', + 'scripts.remove_after_use.populate_notification_subscriptions_user_global_reviews', + 'scripts.remove_after_use.update_notification_subscriptions_user_global_reviews', 'osf.management.commands.clear_expired_sessions', 'osf.management.commands.delete_withdrawn_or_failed_registration_files', 'osf.management.commands.migrate_pagecounter_data', @@ -450,7 +457,7 @@ class CeleryConfig: 'osf.management.commands.monthly_reporters_go', 'osf.management.commands.ingest_cedar_metadata_templates', 'osf.metrics.reporters', - 'scripts.populate_notification_subscriptions', + 'scripts.remove_after_use.merge_notification_subscription_provider_ct', } med_pri_modules = { @@ -564,6 +571,12 @@ class CeleryConfig: 'scripts.approve_embargo_terminations', 'scripts.triggered_mails', 'scripts.generate_sitemap', + 'scripts.remove_after_use.populate_notification_subscriptions_node_file_updated', + 'scripts.remove_after_use.update_notification_subscriptions_node_file_updated', + 'scripts.remove_after_use.populate_notification_subscriptions_user_global_file_updated', + 'scripts.remove_after_use.update_notification_subscriptions_user_global_file_updated', + 'scripts.remove_after_use.populate_notification_subscriptions_user_global_reviews', + 'scripts.remove_after_use.update_notification_subscriptions_user_global_reviews', 'scripts.premigrate_created_modified', 'scripts.add_missing_identifiers_to_preprints', 'osf.management.commands.clear_expired_sessions', @@ -579,7 +592,7 @@ class CeleryConfig: 'osf.management.commands.monthly_reporters_go', 'osf.external.spam.tasks', 'api.share.utils', - 'scripts.populate_notification_subscriptions', + 'scripts.remove_after_use.merge_notification_subscription_provider_ct', ) # Modules that need metrics and release requirements @@ -648,6 +661,11 @@ class CeleryConfig: 'schedule': crontab(minute=0, hour=5), # Daily 12 a.m 'kwargs': {'dry_run': False}, }, + 'notifications_cleanup_task': { + 'task': 'notifications.tasks.notifications_cleanup_task', + 'schedule': crontab(minute=0, hour=7), # Daily 2 a.m + 'kwargs': {'dry_run': False}, + }, 'clear_expired_sessions': { 'task': 'osf.management.commands.clear_expired_sessions', 'schedule': crontab(minute=0, hour=5), # Daily 12 a.m diff --git a/website/settings/local-ci.py b/website/settings/local-ci.py index 022a973b35a..b1f3cab0566 100644 --- a/website/settings/local-ci.py +++ b/website/settings/local-ci.py @@ -85,6 +85,8 @@ class CeleryConfig(defaults.CeleryConfig): MAX_DAILY_NO_LOGIN_EMAILS = None NO_LOGIN_EMAIL_CUTOFF = None +NOTIFICATIONS_CLEANUP_AGE = timedelta(weeks=4) # 1 month to clean up old notifications and email tasks + USE_CDN_FOR_CLIENT_LIBS = False SENTRY_DSN = None