Added static views to sitemap.

Author: sarahboyce

djangoproject/sitemaps.py

@@ -0,0 +1,76 @@
+from dataclasses import dataclass
+
+from django.contrib import sitemaps
+from django_hosts.resolvers import reverse
+
+
+@dataclass
+class URLObject:
+    name: str
+    host: str = "www"
+
+
+class LocationAbsoluteUrlMixin:
+    def get_urls(self, site=None, **kwargs):
+        """
+        Prevent the Django sitemap framework from prefixing the domain.
+        Use the absolute URL returned by location().
+        """
+        urls = []
+        for item in self.items():
+            loc = self.location(item)
+            urls.append(
+                {
+                    "location": loc,
+                    "lastmod": None,
+                    "changefreq": self.changefreq,
+                    "priority": self.priority,
+                }
+            )
+        return urls
+
+
+class StaticViewSitemap(LocationAbsoluteUrlMixin, sitemaps.Sitemap):
+    priority = 0.5
+    changefreq = "monthly"
+
+    def items(self):
+        return [
+            # accounts
+            URLObject("registration_register"),
+            # aggregator
+            URLObject("community-index"),
+            URLObject("community-ecosystem"),
+            URLObject("local-django-communities"),
+            # contact
+            URLObject("contact_foundation"),
+            # dashboard
+            URLObject("dashboard-index", host="dashboard"),
+            URLObject("metric-list", host="dashboard"),
+            # djangoproject
+            URLObject("homepage"),
+            URLObject("overview"),
+            URLObject("start"),
+            URLObject("code_of_conduct"),
+            URLObject("conduct_faq"),
+            URLObject("conduct_reporting"),
+            URLObject("conduct_enforcement"),
+            URLObject("conduct_changes"),
+            URLObject("diversity"),
+            URLObject("diversity_changes"),
+            # foundation
+            URLObject("foundation_meeting_archive_index"),
+            # fundraising
+            URLObject("fundraising:index"),
+            # members
+            URLObject("members:individual-members"),
+            URLObject("members:corporate-members"),
+            URLObject("members:corporate-members-join"),
+            URLObject("members:corporate-members-badges"),
+            URLObject("members:teams"),
+            # releases
+            URLObject("download"),
+        ]
+
+    def location(self, item):
+        return reverse(item.name, host=item.host)

djangoproject/tests.py

@@ -202,6 +202,7 @@ def test_single_h1_per_page(self):
             "styleguide/",  # Has multiple <h1> examples.
             "admin/",  # Admin templates are out of our control.
             "reset/done/",  # Uses an admin template.
+            "sitemap.xml",
         ]
         resolver = get_resolver()
         urls = self.extract_patterns(resolver.url_patterns)
@@ -211,3 +212,9 @@ def test_single_h1_per_page(self):
                     response = self.client.get(url)
                     self.assertEqual(response.status_code, 200)
                     self.assertContains(response, "<h1", count=1)
+
+
+class SiteMapTests(TestCase):
+    def test_sitemap_renders(self):
+        response = self.client.get(reverse("sitemap"))
+        self.assertEqual(response.status_code, 200)

djangoproject/urls/www.py

@@ -13,6 +13,7 @@
 from aggregator.feeds import CommunityAggregatorFeed, CommunityAggregatorFirehoseFeed
 from blog.feeds import WeblogEntryFeed
 from blog.sitemaps import WeblogSitemap
+from djangoproject.sitemaps import StaticViewSitemap
 from foundation.feeds import FoundationMinutesFeed
 from foundation.views import CoreDevelopers
 
@@ -21,6 +22,7 @@
 sitemaps = {
     "weblog": WeblogSitemap,
     "flatpages": FlatPageSitemap,
+    "static": StaticViewSitemap,
 }
 
 
@@ -135,6 +137,7 @@
         "sitemap.xml",
         cache_page(60 * 60 * 6)(sitemap_views.sitemap),
         {"sitemaps": sitemaps},
+        name="sitemap",
     ),
     path("weblog/", include("blog.urls")),
     path("download/", include("releases.urls")),

docs/sitemaps.py

@@ -1,10 +1,12 @@
 from django.contrib.sitemaps import Sitemap
 
+from djangoproject.sitemaps import LocationAbsoluteUrlMixin
+
 from .models import Document
 from .search import DocumentationCategory
 
 
-class DocsSitemap(Sitemap):
+class DocsSitemap(LocationAbsoluteUrlMixin, Sitemap):
     def __init__(self, lang):
         self.lang = lang
 
@@ -16,6 +18,9 @@ def items(self):
             .select_related("release__release")
         )
 
+    def location(self, item):
+        return item.get_absolute_url()
+
     def changefreq(self, obj):
         return "daily"
 
@@ -33,19 +38,3 @@ def priority(self, obj):
             return 1
         else:
             return 0.1
-
-    def _urls(self, page, site, protocol):
-        # XXX: To workaround bad interaction between contrib.sitemaps and
-        # django-hosts (scheme/domain would be repeated twice in URLs)
-        urls = []
-        for item in self.paginator.page(page).object_list:
-            loc = item.get_absolute_url()
-            priority = self.priority(item)
-            url_info = {
-                "item": item,
-                "location": loc,
-                "changefreq": self.changefreq(item),
-                "priority": str(priority if priority is not None else ""),
-            }
-            urls.append(url_info)
-        return urls

docs/tests/test_models.py

@@ -7,8 +7,10 @@
 from django.db import connection
 from django.test import TestCase
 from django.utils import timezone
+from django_hosts import reverse
 
 from blog.models import ContentFormat, Entry
+from djangoproject.sitemaps import StaticViewSitemap
 from releases.models import Release
 
 from ..models import Document, DocumentRelease
@@ -607,7 +609,7 @@ def test_sync_to_db_not_delete_website_docs(self):
 
     def test_sync_from_sitemap_skip_non_en_dev_release(self):
         release = Release.objects.create(version="5.2")
-        Entry.objects.create(
+        blog_entry = Entry.objects.create(
             pub_date=timezone.now() - datetime.timedelta(days=2),
             slug="a",
             body="<strong>test</strong>",
@@ -625,7 +627,16 @@ def test_sync_from_sitemap_skip_non_en_dev_release(self):
             )
             with self.subTest(lang=lang, release=release_obj):
                 doc_release.sync_from_sitemap()
-                self.assertFalse(Document.objects.exists())
+                self.assertFalse(
+                    Document.objects.filter(path=blog_entry.get_absolute_url()).exists()
+                )
+
+    @staticmethod
+    def _mock_static_sitemap_urls(mocker):
+        sitemap = StaticViewSitemap()
+        for item in sitemap.items():
+            url = reverse(item.name, host=item.host)
+            mocker.get(url, text="", headers={"Content-Type": "text/html"})
 
     @requests_mock.mock()
     def test_sync_from_sitemap(self, mocker):
@@ -636,17 +647,16 @@ def test_sync_from_sitemap(self, mocker):
             content_format=ContentFormat.HTML,
             is_active=True,
         )
+        self._mock_static_sitemap_urls(mocker)
         mocker.get(
             blog_entry.get_absolute_url(),
             text="<html><main>Main 1</main><h1>Title 1</h1></html>",
             headers={"Content-Type": "text/html"},
         )
         self.release.sync_from_sitemap()
 
-        document = Document.objects.get(release=self.release)
-        self.assertEqual(
-            document.path,
-            blog_entry.get_absolute_url(),
+        document = Document.objects.get(
+            release=self.release, path=blog_entry.get_absolute_url()
         )
         self.assertEqual(
             document.title,
@@ -683,12 +693,10 @@ def test_sync_from_sitemap_only_requests_non_existing(self, mock_fetch_html):
 
         mock_fetch_html.assert_not_called()
 
-        document = Document.objects.get(release=self.release)
-        # Confirm Document has not been updated.
-        self.assertEqual(
-            document.path,
-            blog_entry.get_absolute_url(),
+        document = Document.objects.get(
+            release=self.release, path=blog_entry.get_absolute_url()
         )
+        # Confirm Document has not been updated.
         self.assertEqual(
             document.metadata,
             {"parents": DocumentationCategory.WEBSITE},
@@ -714,6 +722,7 @@ def test_sync_from_sitemap_force(self, mocker):
             metadata={"parents": DocumentationCategory.WEBSITE},
             path=blog_url,
         )
+        self._mock_static_sitemap_urls(mocker)
         mocker.get(
             blog_url,
             text="<html><main>Main 1</main><h1>Title 1</h1></html>",