Synced webpages from sitemap to Document model for search.

sarahboyce · sarahboyce · commit fc43b710a6f2 · 2025-11-28T15:16:46.000+01:00
diff --git a/blog/admin.py b/blog/admin.py
@@ -16,10 +16,9 @@ class EntryAdmin(admin.ModelAdmin):
         "pub_date",
         "is_active",
         "is_published",
-        "is_searchable",
         "author",
     )
-    list_filter = ("is_active", "is_searchable")
+    list_filter = ("is_active",)
     exclude = ("summary_html", "body_html")
     prepopulated_fields = {"slug": ("headline",)}
     raw_id_fields = ["social_media_card"]
diff --git a/blog/models.py b/blog/models.py
@@ -37,9 +37,6 @@ def published(self):
     def active(self):
         return self.filter(is_active=True)
 
-    def searchable(self):
-        return self.filter(is_searchable=True)
-
 
 class ContentFormat(models.TextChoices):
     REST = "reST", "reStructuredText"
diff --git a/blog/tests.py b/blog/tests.py
@@ -70,26 +70,6 @@ def test_manager_published(self):
             transform=lambda entry: entry.headline,
         )
 
-    def test_manager_searchable(self):
-        """
-        Make sure that the Entry manager's `searchable` method works
-        """
-        Entry.objects.create(
-            pub_date=self.yesterday,
-            is_searchable=False,
-            headline="not searchable",
-            slug="a",
-        )
-        Entry.objects.create(
-            pub_date=self.yesterday, is_searchable=True, headline="searchable", slug="b"
-        )
-
-        self.assertQuerySetEqual(
-            Entry.objects.searchable(),
-            ["searchable"],
-            transform=lambda entry: entry.headline,
-        )
-
     def test_docutils_safe(self):
         """
         Make sure docutils' file inclusion directives are disabled by default.
diff --git a/djangoproject/scss/_style.scss b/djangoproject/scss/_style.scss
@@ -2650,7 +2650,7 @@ search.filters {
     position: relative;
 
     a {
-      padding: 10px 20px;
+      padding: 10px 15px;
       text-decoration: none;
       border-bottom: 3px solid transparent;
       transition: color 0.3s ease, border-bottom 0.3s ease;
diff --git a/docs/management/commands/update_docs.py b/docs/management/commands/update_docs.py
@@ -134,6 +134,8 @@ def build_doc_release(self, release, force=False, interactive=False):
         if self.verbosity >= 1:
             self.stdout.write(f"Starting update for {release} at {datetime.now()}...")
 
+        release.sync_from_sitemap(force=force)
+
         # checkout_dir is shared for all languages.
         checkout_dir = settings.DOCS_BUILD_ROOT / "sources" / release.version
         parent_build_dir = settings.DOCS_BUILD_ROOT / release.lang / release.version
diff --git a/docs/models.py b/docs/models.py
@@ -5,6 +5,7 @@
 from functools import partial, reduce
 from pathlib import Path
 
+import requests
 from django.conf import settings
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.postgres.search import (
@@ -26,19 +27,19 @@
 from django.utils.html import strip_tags
 from django_hosts.resolvers import reverse
 
-from blog.models import Entry
 from releases.models import Release
 
 from . import utils
 from .search import (
     DEFAULT_TEXT_SEARCH_CONFIG,
-    SEARCHABLE_VIEWS,
     START_SEL,
     STOP_SEL,
     TSEARCH_CONFIG_LANGUAGES,
     DocumentationCategory,
+    fetch_html,
     get_document_search_vector,
 )
+from .utils import extract_inner_html
 
 
 def get_search_config(lang):
@@ -185,7 +186,7 @@ def sync_to_db(self, decoded_documents):
         the database. Deletes all the release's documents first then
         reinserts them as needed.
         """
-        self.documents.all().delete()
+        self.documents.exclude(metadata__parents=DocumentationCategory.WEBSITE).delete()
 
         # Read excluded paths from robots.docs.txt.
         robots_path = settings.BASE_DIR / "djangoproject" / "static" / "robots.docs.txt"
@@ -216,81 +217,63 @@ def sync_to_db(self, decoded_documents):
                 metadata=document,
                 config=get_search_config(self.lang),
             )
-        for document in self.documents.all():
+        for document in self.documents.exclude(
+            metadata__parents=DocumentationCategory.WEBSITE
+        ):
             document.metadata["breadcrumbs"] = list(
                 Document.objects.breadcrumbs(document).values("title", "path")
             )
             document.save(update_fields=("metadata",))
 
-        self._sync_blog_to_db()
-        self._sync_views_to_db()
+    def sync_from_sitemap(self, force=False):
+        from djangoproject.urls.www import sitemaps
 
-    def _sync_blog_to_db(self):
-        """
-        Sync the blog entries into search based on the release documents
-        support end date.
-        """
-        if self.lang != "en":
-            return  # The blog is only written in English currently
+        if not self.is_dev:
+            return
 
-        entries = Entry.objects.published().searchable()
-        Document.objects.bulk_create(
-            [
-                Document(
-                    release=self,
-                    path=entry.get_absolute_url(),
-                    title=entry.headline,
-                    metadata={
-                        "body": entry.body_html,
-                        "breadcrumbs": [
-                            {
-                                "path": DocumentationCategory.WEBSITE,
-                                "title": "News",
-                            },
-                        ],
-                        "parents": DocumentationCategory.WEBSITE,
-                        "slug": entry.slug,
-                        "title": entry.headline,
-                        "toc": "",
-                    },
-                    config=get_search_config(self.lang),
-                )
-                for entry in entries
-            ]
-        )
+        if force:
+            Document.objects.filter(
+                metadata__parents=DocumentationCategory.WEBSITE
+            ).delete()
 
-    def _sync_views_to_db(self):
-        """
-        Sync the specific views into search based on the release documents
-        support end date.
-        """
-        if self.lang != "en":
-            return  # The searchable views are only written in English currently
+        doc_urls = set(
+            Document.objects.filter(
+                metadata__parents=DocumentationCategory.WEBSITE
+            ).values_list("path", flat=True)
+        )
 
-        Document.objects.bulk_create(
-            [
-                Document(
+        for sitemap in sitemaps.values():
+            for url in sitemap().get_urls():
+                path = url["location"]
+                if path in doc_urls:
+                    continue
+                try:
+                    page_html = fetch_html(path)
+                except requests.RequestException:
+                    continue
+                try:
+                    main_html = extract_inner_html(page_html, tag="main")
+                    title = extract_inner_html(page_html, tag="h1")
+                except ValueError:
+                    continue
+                Document.objects.create(
                     release=self,
-                    path=searchable_view.www_absolute_url,
-                    title=searchable_view.page_title,
+                    path=path,
+                    title=title,
                     metadata={
-                        "body": searchable_view.html,
+                        "body": main_html,
                         "breadcrumbs": [
                             {
                                 "path": DocumentationCategory.WEBSITE,
                                 "title": "Website",
                             },
                         ],
                         "parents": DocumentationCategory.WEBSITE,
-                        "slug": searchable_view.url_name,
-                        "title": searchable_view.page_title,
+                        "title": title,
                         "toc": "",
                     },
                     config=get_search_config(self.lang),
                 )
-                for searchable_view in SEARCHABLE_VIEWS
-            ]
-        )
 
 
 def _clean_document_path(path):
@@ -351,6 +334,14 @@ def search(self, query_text, release, document_category=None):
                 config=models.F("config"),
             )
             base_filter = Q(release_id=release.id)
+            if release.lang == settings.DEFAULT_LANGUAGE_CODE and not release.is_dev:
+                dev_release = DocumentRelease.objects.get_by_version_and_lang(
+                    version="dev", lang=settings.DEFAULT_LANGUAGE_CODE
+                )
+                base_filter |= Q(
+                    release_id=dev_release.id,
+                    metadata__parents=DocumentationCategory.WEBSITE,
+                )
             if document_category:
                 base_filter &= Q(metadata__parents__startswith=document_category)
             base_qs = (
diff --git a/docs/search.py b/docs/search.py
@@ -1,11 +1,8 @@
-from dataclasses import dataclass
-
+import requests
 from django.contrib.postgres.search import SearchVector
 from django.db.models import TextChoices
 from django.db.models.fields.json import KeyTextTransform
-from django.template.loader import get_template
 from django.utils.translation import gettext_lazy as _
-from django_hosts import reverse
 
 # Imported from
 # https://github.com/postgres/postgres/blob/REL_14_STABLE/src/bin/initdb/initdb.c#L659
@@ -81,25 +78,31 @@ def parse(cls, value, default=None):
             return None
 
 
-@dataclass
-class SearchableView:
-    page_title: str
-    url_name: str
-    template: str
-
-    @property
-    def html(self):
-        return get_template(self.template).render()
-
-    @property
-    def www_absolute_url(self):
-        return reverse(self.url_name, host="www")
-
+def fetch_html(url, timeout=10):
+    """
+    Fetch the HTML of a page if status code is 200.
+    Simulates a human browser and accepts only text/html.
+    """
 
-SEARCHABLE_VIEWS = [
-    SearchableView(
-        page_title="Django's Ecosystem",
-        url_name="community-ecosystem",
-        template="aggregator/ecosystem.html",
-    ),
-]
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"
+        ),
+        "Accept": "text/html",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    response = requests.get(url, headers=headers, timeout=timeout)
+
+    if response.status_code == 200:
+        content_type = response.headers.get("Content-Type", "")
+        if "text/html" in content_type:
+            return response.text
+        else:
+            raise requests.RequestException(f"Unexpected Content-Type: {content_type}")
+    else:
+        raise requests.RequestException(
+            f"Failed to fetch {url}, status code: {response.status_code}"
+        )
diff --git a/docs/tests/test_models.py b/docs/tests/test_models.py
diff --git a/docs/tests/test_utils.py b/docs/tests/test_utils.py
diff --git a/docs/utils.py b/docs/utils.py
