Merge pull request #64 from Mazizova/seo/indexing-basics

SEO: add robots.txt, noindex thin taxonomy pages, fix future sitemap lastmod

Author: Mazizova

hugo.yaml

@@ -36,6 +36,7 @@ params:
     category_id: "DIC_kwDOSeSzss4DFPF-"
 
 buildFuture: true
+enableRobotsTXT: true
 
 markup:
   goldmark:

layouts/_default/sitemap.xml

@@ -0,0 +1,19 @@
+{{/*
+  Same as Hugo's built-in sitemap, except it skips pages that are noindex
+  (see partials/seo/indexable.html) and drops <lastmod> values in the
+  future — upcoming events otherwise push list pages' lastmod past today,
+  which search engines treat as invalid.
+*/}}
+{{ printf "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?>" | safeHTML }}
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  {{- range where .Pages "Sitemap.Disable" "ne" true }}
+    {{- if and .Permalink (partial "seo/indexable.html" .) }}
+  <url>
+    <loc>{{ .Permalink }}</loc>
+    {{- if and (not .Lastmod.IsZero) (le .Lastmod.Unix now.Unix) }}
+    <lastmod>{{ .Lastmod.Format "2006-01-02T15:04:05-07:00" | safeHTML }}</lastmod>
+    {{- end }}
+  </url>
+    {{- end }}
+  {{- end }}
+</urlset>

layouts/partials/head.html

@@ -14,7 +14,7 @@
 {{ with .Params.author }}<meta name="author" content="{{ . }}">{{ end }}
 
 {{/* ── Robots ── */}}
-<meta name="robots" content="index, follow">
+<meta name="robots" content="{{ if partial "seo/indexable.html" . }}index{{ else }}noindex{{ end }}, follow">
 
 {{/* ── Feed auto-discovery (only present on pages that actually have one, e.g. /blog/) ── */}}
 {{ range .AlternativeOutputFormats }}

layouts/partials/seo/indexable.html

@@ -0,0 +1,19 @@
+{{/*
+  Whether a page should be indexed by search engines. Used by head.html
+  (meta robots) and the sitemap template so the two never disagree.
+
+  Taxonomy overview pages (/pricing/, /levels/, ...) and most facet term
+  pages (/pricing/free/, /countries/brazil/, ...) are thin filter views of
+  the same resources, so they are noindex (links on them are still
+  followed). Subcategory terms with at least 3 resources are kept — those
+  are real topical landing pages ("Monitoring & Observability", ...).
+  Any page can override with `robots: noindex` in front matter.
+*/}}
+{{ $indexable := true }}
+{{ if eq .Kind "taxonomy" }}
+  {{ $indexable = false }}
+{{ else if eq .Kind "term" }}
+  {{ $indexable = and (eq .Data.Plural "subcategories") (ge (len .Pages) 3) }}
+{{ end }}
+{{ with .Params.robots }}{{ if in . "noindex" }}{{ $indexable = false }}{{ end }}{{ end }}
+{{ return $indexable }}

layouts/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: {{ "sitemap.xml" | absURL }}