Lower memory usage in submissions and projects

hypha/apply/activity/models.py

@@ -236,7 +236,7 @@ class Meta:
     def get_absolute_url(self):
         # coverup for both submission and project as source.
         submission_id = (
-            self.source.submission.id
+            self.source.submission_id
             if hasattr(self.source, "submission")
             else self.source.id
         )

hypha/apply/dashboard/services.py

@@ -1,28 +1,28 @@
-from django.db.models import Count
+from django.db.models import Count, Exists, OuterRef
 
 from hypha.apply.projects.models import PAFApprovals
 
 
 def get_paf_for_review(user, is_paf_approval_sequential):
-    """Return a list of paf approvals ready for user's review"""
+    """Return a queryset of paf approvals ready for user's review"""
+    user_groups = list(user.groups.all())
 
     paf_approvals = PAFApprovals.objects.annotate(
         roles_count=Count("paf_reviewer_role__user_roles")
     ).filter(
-        roles_count=user.groups.count(),
+        roles_count=len(user_groups),
         approved=False,
     )
 
-    for role in user.groups.all():
+    for role in user_groups:
         paf_approvals = paf_approvals.filter(paf_reviewer_role__user_roles__id=role.id)
 
     if is_paf_approval_sequential:
-        all_matched_paf_approvals = list(paf_approvals)
-        for matched_paf_approval in all_matched_paf_approvals:
-            if matched_paf_approval.project.paf_approvals.filter(
-                paf_reviewer_role__sort_order__lt=matched_paf_approval.paf_reviewer_role.sort_order,
-                approved=False,
-            ).exists():
-                paf_approvals = paf_approvals.exclude(id=matched_paf_approval.id)
+        blocking_step = PAFApprovals.objects.filter(
+            project=OuterRef("project"),
+            approved=False,
+            paf_reviewer_role__sort_order__lt=OuterRef("paf_reviewer_role__sort_order"),
+        )
+        paf_approvals = paf_approvals.exclude(Exists(blocking_step))
 
     return paf_approvals

hypha/apply/dashboard/templates/dashboard/contracting_dashboard.html

@@ -41,7 +41,7 @@ <h2 class="section-header" id="paf-for-review">
         </section>
 
 
-        {% if projects_in_contracting.count %}
+        {% if PROJECTS_ENABLED and projects_in_contracting.count %}
             {% include "dashboard/includes/projects_in_contracting.html" with projects_in_contracting=projects_in_contracting %}
         {% endif %}
     </div>

hypha/apply/dashboard/templates/dashboard/finance_dashboard.html

@@ -92,7 +92,7 @@ <h2 class="section-header">{% trans 'Invoices' %} </h2>
         <section id="paf_for_review">
             <h2 class="section-header">{% trans "PAFs for review" %}</h2>
             <div class="overflow-x-auto border rounded-box">
-                {% if not paf_for_review.count %}
+                {% if PROJECTS_ENABLED and paf_for_review.count %}
                     {% render_table paf_for_review.table %}
                 {% else %}
                     <p class="p-4 text-sm text-fg-muted">

hypha/apply/dashboard/views.py

@@ -71,8 +71,32 @@ def my_flagged(self, submissions):
         )
 
 
-class AdminDashboardView(MyFlaggedMixin, TemplateView):
+class PAFReviewMixin:
+    paf_reviewer_role = None  # set to a user attribute e.g. "is_apply_staff"
+
+    def paf_for_review(self):
+        if not getattr(self.request.user, self.paf_reviewer_role, False):
+            return {"count": None, "table": None}
+        project_settings = ProjectSettings.for_request(self.request)
+        paf_approvals = list(
+            get_paf_for_review(
+                user=self.request.user,
+                is_paf_approval_sequential=project_settings.paf_approval_sequential,
+            ).select_related("project__submission__page", "paf_reviewer_role", "user")
+        )
+        paf_table = PAFForReviewDashboardTable(
+            paf_approvals, prefix="paf-review-", order_by="-date_requested"
+        )
+        RequestConfig(self.request, paginate=False).configure(paf_table)
+        return {
+            "count": len(paf_approvals),
+            "table": paf_table,
+        }
+
+
+class AdminDashboardView(PAFReviewMixin, MyFlaggedMixin, TemplateView):
     template_name = "dashboard/staff_dashboard.html"
+    paf_reviewer_role = "is_apply_staff"
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
@@ -97,25 +121,6 @@ def get_context_data(self, **kwargs):
 
         return context
 
-    def paf_for_review(self):
-        if not self.request.user.is_apply_staff:
-            return {"count": None, "table": None}
-        project_settings = ProjectSettings.for_request(self.request)
-
-        paf_approvals = get_paf_for_review(
-            user=self.request.user,
-            is_paf_approval_sequential=project_settings.paf_approval_sequential,
-        )
-        paf_table = PAFForReviewDashboardTable(
-            paf_approvals, prefix="paf-review-", order_by="-date_requested"
-        )
-        RequestConfig(self.request, paginate=False).configure(paf_table)
-
-        return {
-            "count": paf_approvals.count(),
-            "table": paf_table,
-        }
-
     def my_tasks(self):
         tasks = render_task_templates_for_user(self.request, self.request.user)
         return {
@@ -137,12 +142,16 @@ def awaiting_reviews(self, submissions):
         }
 
     def active_invoices(self):
-        invoices = Invoice.objects.filter(
-            project__lead=self.request.user,
-        ).in_progress()
+        invoices = list(
+            Invoice.objects.filter(
+                project__lead=self.request.user,
+            )
+            .in_progress()
+            .select_related("project")
+        )
 
         return {
-            "count": invoices.count(),
+            "count": len(invoices),
             "table": InvoiceDashboardTable(invoices),
         }
 
@@ -154,12 +163,13 @@ def projects(self):
         )
 
         limit = 10
+        projects_count = projects.count()
 
         return {
-            "count": projects.count(),
+            "count": projects_count,
             "filterset": filterset,
             "table": ProjectsDashboardTable(data=projects[:limit], prefix="project-"),
-            "display_more": projects.count() > limit,
+            "display_more": projects_count > limit,
             "url": reverse("apply:projects:all"),
         }
 
@@ -178,8 +188,9 @@ def rounds(self):
         }
 
 
-class FinanceDashboardView(MyFlaggedMixin, TemplateView):
+class FinanceDashboardView(PAFReviewMixin, MyFlaggedMixin, TemplateView):
     template_name = "dashboard/finance_dashboard.html"
+    paf_reviewer_role = "is_finance"
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
@@ -196,25 +207,6 @@ def get_context_data(self, **kwargs):
 
         return context
 
-    def paf_for_review(self):
-        if not self.request.user.is_finance:
-            return {"count": None, "table": None}
-        project_settings = ProjectSettings.for_request(self.request)
-
-        paf_approvals = get_paf_for_review(
-            user=self.request.user,
-            is_paf_approval_sequential=project_settings.paf_approval_sequential,
-        )
-        paf_table = PAFForReviewDashboardTable(
-            paf_approvals, prefix="paf-review-", order_by="-date_requested"
-        )
-        RequestConfig(self.request, paginate=False).configure(paf_table)
-
-        return {
-            "count": paf_approvals.count(),
-            "table": paf_table,
-        }
-
     def my_tasks(self):
         tasks = render_task_templates_for_user(self.request, self.request.user)
         return {
@@ -223,22 +215,20 @@ def my_tasks(self):
         }
 
     def active_invoices(self):
-        invoices = Invoice.objects.for_finance_1()
-
+        invoices = list(Invoice.objects.for_finance_1().select_related("project"))
         return {
-            "count": invoices.count(),
+            "count": len(invoices),
             "table": InvoiceDashboardTable(invoices),
         }
 
     def invoices_for_approval(self):
-        invoices = Invoice.objects.approved_by_staff()
-
-        return {"count": invoices.count(), "table": InvoiceDashboardTable(invoices)}
+        invoices = list(Invoice.objects.approved_by_staff().select_related("project"))
+        return {"count": len(invoices), "table": InvoiceDashboardTable(invoices)}
 
     def invoices_to_convert(self):
-        invoices = Invoice.objects.waiting_to_convert()
+        invoices = list(Invoice.objects.waiting_to_convert().select_related("project"))
         return {
-            "count": invoices.count(),
+            "count": len(invoices),
             "table": InvoiceDashboardTable(invoices),
         }
 
@@ -302,8 +292,9 @@ def awaiting_reviews(self, submissions):
         }
 
 
-class ContractingDashboardView(MyFlaggedMixin, TemplateView):
+class ContractingDashboardView(PAFReviewMixin, MyFlaggedMixin, TemplateView):
     template_name = "dashboard/contracting_dashboard.html"
+    paf_reviewer_role = "is_contracting"
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
@@ -317,25 +308,6 @@ def get_context_data(self, **kwargs):
 
         return context
 
-    def paf_for_review(self):
-        if not self.request.user.is_contracting:
-            return {"count": None, "table": None}
-        project_settings = ProjectSettings.for_request(self.request)
-
-        paf_approvals = get_paf_for_review(
-            user=self.request.user,
-            is_paf_approval_sequential=project_settings.paf_approval_sequential,
-        )
-        paf_table = PAFForReviewDashboardTable(
-            paf_approvals, prefix="paf-review-", order_by="-date_requested"
-        )
-        RequestConfig(self.request, paginate=False).configure(paf_table)
-
-        return {
-            "count": paf_approvals.count(),
-            "table": paf_table,
-        }
-
     def my_tasks(self):
         tasks = render_task_templates_for_user(self.request, self.request.user)
         return {
@@ -357,22 +329,22 @@ def projects_in_contracting(self):
                 },
             }
         projects_in_contracting = Project.objects.in_contracting()
-        waiting_for_contract = projects_in_contracting.filter(
-            contracts__isnull=True
-        ).for_table()
-        waiting_for_contract_approval = projects_in_contracting.filter(
-            contracts__isnull=False
-        ).for_table()
+        waiting_for_contract = list(
+            projects_in_contracting.filter(contracts__isnull=True).for_table()
+        )
+        waiting_for_contract_approval = list(
+            projects_in_contracting.filter(contracts__isnull=False).for_table()
+        )
         return {
-            "count": projects_in_contracting.count(),
+            "count": len(waiting_for_contract) + len(waiting_for_contract_approval),
             "waiting_for_contract": {
-                "count": waiting_for_contract.count(),
+                "count": len(waiting_for_contract),
                 "table": ProjectsDashboardTable(
                     data=waiting_for_contract, prefix="project-waiting-contract-"
                 ),
             },
             "waiting_for_contract_approval": {
-                "count": waiting_for_contract_approval.count(),
+                "count": len(waiting_for_contract_approval),
                 "table": ProjectsDashboardTable(
                     data=waiting_for_contract_approval,
                     prefix="project-waiting-approval-",
@@ -389,13 +361,13 @@ def get_context_data(self, **kwargs):
         submissions = ApplicationSubmission.objects.all().for_table(self.request.user)
 
         # Submissions in community review phase
-        my_community_review, my_community_review = self.my_community_review(
+        my_community_review_qs, my_community_review_table = self.my_community_review(
             self.request.user, submissions
         )
         context.update(
             {
-                "my_community_review": my_community_review,
-                "my_community_review_count": my_community_review.count(),
+                "my_community_review": my_community_review_table,
+                "my_community_review_count": my_community_review_qs.count(),
                 "my_reviewed": get_preview_context(
                     submissions.reviewed_by(self.request.user).order_by("-submit_time")
                 ),
@@ -444,26 +416,26 @@ def my_tasks(self):
         }
 
     def active_invoices(self):
-        active_invoices = (
+        active_invoices = list(
             Invoice.objects.filter(project__user=self.request.user)
             .exclude(status__in=[PAID, DECLINED])
             .order_by("-requested_at")
         )
-        return {"count": active_invoices.count(), "data": active_invoices}
+        return {"count": len(active_invoices), "data": active_invoices}
 
     def historical_project_data(self):
-        historical_projects = (
+        historical_projects = list(
             Project.objects.filter(user=self.request.user).complete().for_table()
         )
         return {
-            "count": historical_projects.count(),
+            "count": len(historical_projects),
             "table": ProjectsDashboardTable(
                 data=historical_projects, prefix="past-project-"
             ),
         }
 
     def historical_submission_data(self):
-        historical_submissions = (
+        historical_submissions = list(
             ApplicationSubmission.objects.filter(
                 user=self.request.user,
             )
@@ -472,8 +444,7 @@ def historical_submission_data(self):
             .for_table(self.request.user)
         )
         return {
-            "count": historical_submissions.count(),
-            "submissions": historical_submissions,
+            "count": len(historical_submissions),
             "table": SubmissionsTable(data=historical_submissions),
         }
 

hypha/apply/determinations/models.py

@@ -135,7 +135,7 @@ def clean_outcome(self):
     def get_absolute_url(self):
         return reverse(
             "apply:submissions:determinations:detail",
-            args=(self.submission.id, self.id),
+            args=(self.submission_id, self.id),
         )
 
     @property

hypha/apply/determinations/views.py

@@ -197,7 +197,7 @@ def form_valid(self, form):
         submissions = self.get_submissions()
         response = super().form_valid(form)
         determinations = {
-            determination.submission.id: determination
+            determination.submission_id: determination
             for determination in form.instances
         }
         sources = submissions.filter(id__in=list(determinations))

hypha/apply/funds/files.py

@@ -30,12 +30,12 @@ def get_entity_id(self):
         if isinstance(self.instance, ApplicationRevision) or isinstance(
             self.instance, Project
         ):
-            entity_id = self.instance.submission.pk
+            entity_id = self.instance.submission_id
         elif isinstance(self.instance, Report) or isinstance(self.instance, ProjectSOW):
-            entity_id = self.instance.project.submission.pk
+            entity_id = self.instance.project.submission_id
         elif isinstance(self.instance, ReportVersion):
             # Reports are project documents.
-            entity_id = self.instance.report.project.submission.pk
+            entity_id = self.instance.report.project.submission_id
         return entity_id
 
     def generate_filename(self):