Synced webpages from sitemap to Document model for search.

Author: sarahboyce

djangoproject/scss/_style.scss

@@ -2650,7 +2650,7 @@ search.filters {
     position: relative;
 
     a {
-      padding: 10px 20px;
+      padding: 10px 15px;
       text-decoration: none;
       border-bottom: 3px solid transparent;
       transition: color 0.3s ease, border-bottom 0.3s ease;

docs/management/commands/update_docs.py

@@ -134,6 +134,8 @@ def build_doc_release(self, release, force=False, interactive=False):
         if self.verbosity >= 1:
             self.stdout.write(f"Starting update for {release} at {datetime.now()}...")
 
+        release.sync_from_sitemap(force=force)
+
         # checkout_dir is shared for all languages.
         checkout_dir = settings.DOCS_BUILD_ROOT.joinpath("sources", release.version)
         parent_build_dir = settings.DOCS_BUILD_ROOT.joinpath(

docs/models.py

@@ -5,6 +5,7 @@
 from functools import partial, reduce
 from pathlib import Path
 
+import requests
 from django.conf import settings
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.postgres.search import (
@@ -26,19 +27,19 @@
 from django.utils.html import strip_tags
 from django_hosts.resolvers import reverse
 
-from blog.models import Entry
 from releases.models import Release
 
 from . import utils
 from .search import (
     DEFAULT_TEXT_SEARCH_CONFIG,
-    SEARCHABLE_VIEWS,
     START_SEL,
     STOP_SEL,
     TSEARCH_CONFIG_LANGUAGES,
     DocumentationCategory,
+    fetch_html,
     get_document_search_vector,
 )
+from .utils import extract_inner_html
 
 
 def get_search_config(lang):
@@ -185,7 +186,7 @@ def sync_to_db(self, decoded_documents):
         the database. Deletes all the release's documents first then
         reinserts them as needed.
         """
-        self.documents.all().delete()
+        self.documents.exclude(metadata__parents=DocumentationCategory.WEBSITE).delete()
 
         # Read excluded paths from robots.docs.txt.
         robots_path = settings.BASE_DIR.joinpath(
@@ -218,81 +219,63 @@ def sync_to_db(self, decoded_documents):
                 metadata=document,
                 config=get_search_config(self.lang),
             )
-        for document in self.documents.all():
+        for document in self.documents.exclude(
+            metadata__parents=DocumentationCategory.WEBSITE
+        ):
             document.metadata["breadcrumbs"] = list(
                 Document.objects.breadcrumbs(document).values("title", "path")
             )
             document.save(update_fields=("metadata",))
 
-        self._sync_blog_to_db()
-        self._sync_views_to_db()
+    def sync_from_sitemap(self, force=False):
+        from djangoproject.urls.www import sitemaps
 
-    def _sync_blog_to_db(self):
-        """
-        Sync the blog entries into search based on the release documents
-        support end date.
-        """
-        if self.lang != "en":
-            return  # The blog is only written in English currently
+        if not self.is_dev:
+            return
 
-        entries = Entry.objects.published().searchable()
-        Document.objects.bulk_create(
-            [
-                Document(
-                    release=self,
-                    path=entry.get_absolute_url(),
-                    title=entry.headline,
-                    metadata={
-                        "body": entry.body_html,
-                        "breadcrumbs": [
-                            {
-                                "path": DocumentationCategory.WEBSITE,
-                                "title": "News",
-                            },
-                        ],
-                        "parents": DocumentationCategory.WEBSITE,
-                        "slug": entry.slug,
-                        "title": entry.headline,
-                        "toc": "",
-                    },
-                    config=get_search_config(self.lang),
-                )
-                for entry in entries
-            ]
-        )
+        if force:
+            Document.objects.filter(
+                metadata__parents=DocumentationCategory.WEBSITE
+            ).delete()
 
-    def _sync_views_to_db(self):
-        """
-        Sync the specific views into search based on the release documents
-        support end date.
-        """
-        if self.lang != "en":
-            return  # The searchable views are only written in English currently
+        doc_urls = set(
+            Document.objects.filter(
+                metadata__parents=DocumentationCategory.WEBSITE
+            ).values_list("path", flat=True)
+        )
 
-        Document.objects.bulk_create(
-            [
-                Document(
+        for sitemap in sitemaps.values():
+            for url in sitemap().get_urls():
+                path = url["location"]
+                if path in doc_urls:
+                    continue
+                try:
+                    page_html = fetch_html(path)
+                except requests.RequestException:
+                    continue
+                try:
+                    main_html = extract_inner_html(page_html, tag="main")
+                    title = extract_inner_html(page_html, tag="h1")
+                except ValueError:
+                    continue
+                Document.objects.create(
                     release=self,
-                    path=searchable_view.www_absolute_url,
-                    title=searchable_view.page_title,
+                    path=path,
+                    title=title,
                     metadata={
-                        "body": searchable_view.html,
+                        "body": main_html,
                         "breadcrumbs": [
                             {
                                 "path": DocumentationCategory.WEBSITE,
                                 "title": "Website",
                             },
                         ],
                         "parents": DocumentationCategory.WEBSITE,
-                        "slug": searchable_view.url_name,
-                        "title": searchable_view.page_title,
+                        "title": title,
                         "toc": "",
                     },
                     config=get_search_config(self.lang),
                 )
-                for searchable_view in SEARCHABLE_VIEWS
-            ]
-        )
 
 
 def _clean_document_path(path):
@@ -353,6 +336,14 @@ def search(self, query_text, release, document_category=None):
                 config=models.F("config"),
             )
             base_filter = Q(release_id=release.id)
+            if release.lang == settings.DEFAULT_LANGUAGE_CODE and not release.is_dev:
+                dev_release = DocumentRelease.objects.get_by_version_and_lang(
+                    version="dev", lang=settings.DEFAULT_LANGUAGE_CODE
+                )
+                base_filter |= Q(
+                    release_id=dev_release.id,
+                    metadata__parents=DocumentationCategory.WEBSITE,
+                )
             if document_category:
                 base_filter &= Q(metadata__parents__startswith=document_category)
             base_qs = (

docs/search.py

@@ -1,11 +1,8 @@
-from dataclasses import dataclass
-
+import requests
 from django.contrib.postgres.search import SearchVector
 from django.db.models import TextChoices
 from django.db.models.fields.json import KeyTextTransform
-from django.template.loader import get_template
 from django.utils.translation import gettext_lazy as _
-from django_hosts import reverse
 
 # Imported from
 # https://github.com/postgres/postgres/blob/REL_14_STABLE/src/bin/initdb/initdb.c#L659
@@ -81,25 +78,31 @@ def parse(cls, value, default=None):
             return None
 
 
-@dataclass
-class SearchableView:
-    page_title: str
-    url_name: str
-    template: str
-
-    @property
-    def html(self):
-        return get_template(self.template).render()
-
-    @property
-    def www_absolute_url(self):
-        return reverse(self.url_name, host="www")
-
+def fetch_html(url, timeout=10):
+    """
+    Fetch the HTML of a page if status code is 200.
+    Simulates a human browser and accepts only text/html.
+    """
 
-SEARCHABLE_VIEWS = [
-    SearchableView(
-        page_title="Django's Ecosystem",
-        url_name="community-ecosystem",
-        template="aggregator/ecosystem.html",
-    ),
-]
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"
+        ),
+        "Accept": "text/html",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    response = requests.get(url, headers=headers, timeout=timeout)
+
+    if response.status_code == 200:
+        content_type = response.headers.get("Content-Type", "")
+        if "text/html" in content_type:
+            return response.text
+        else:
+            raise requests.RequestException(f"Unexpected Content-Type: {content_type}")
+    else:
+        raise requests.RequestException(
+            f"Failed to fetch {url}, status code: {response.status_code}"
+        )

docs/templates/docs/search_results.html

@@ -1,7 +1,8 @@
 {% extends "docs/doc.html" %}
 {% load i18n docs %}
 
-{% block title %}{% translate "Search | Django documentation" %}{% endblock %}
+{% block title %}{% translate "Search" %}{% endblock %}
+{% block header %}{% endblock %}
 
 {% block toc-wrapper %}{% endblock %}
 {% block breadcrumbs-wrapper %}{% endblock %}