From 0453e509d306c96941d7ace4898014bc88c8f91a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=B2=8D=E8=B5=9F=E9=B9=8F?=
<87294463+baoypeng@users.noreply.github.com>
Date: Sat, 7 Nov 2026 17:08:04 +0800
Subject: [PATCH] fix(link-checker): inspect HTML bodies after successful HEAD
requests
---
.github/workflows/validate-pr.yml | 5 ++
scripts/check_readme.py | 10 ++-
scripts/test_check_readme.py | 115 ++++++++++++++++++++++++++++++
3 files changed, 128 insertions(+), 2 deletions(-)
create mode 100644 scripts/test_check_readme.py
diff --git a/.github/workflows/validate-pr.yml b/.github/workflows/validate-pr.yml
index 372d10f09..4742de58c 100644
--- a/.github/workflows/validate-pr.yml
+++ b/.github/workflows/validate-pr.yml
@@ -5,6 +5,8 @@ on:
paths:
- README.md
- CONTRIBUTING.md
+ - scripts/**
+ - .github/workflows/validate-pr.yml
permissions:
contents: read
@@ -23,6 +25,9 @@ jobs:
with:
fetch-depth: 0
+ - name: Run link-checker regression tests
+ run: python3 -m unittest discover -s scripts -v
+
- name: Lint README grammar, duplicates, and Table of Contents
run: python3 scripts/check_readme.py lint >> "$GITHUB_STEP_SUMMARY"
diff --git a/scripts/check_readme.py b/scripts/check_readme.py
index b3011684d..e4bf0bc4a 100644
--- a/scripts/check_readme.py
+++ b/scripts/check_readme.py
@@ -585,7 +585,11 @@ def _single_http_call(url, method, timeout):
def _attempt_with_fallback(url, timeout):
- """One HEAD (with GET fallback on certain statuses). Returns (status, headers, body, exc)."""
+ """Try HEAD, then GET for HTML bodies or unsupported HEAD requests.
+
+ A successful HEAD has no body, so HTML pages need a bounded GET as well
+ for the soft-404 title check. Returns (status, headers, body, exc).
+ """
try:
status, headers, body = _single_http_call(url, "HEAD", timeout)
except Exception as exc:
@@ -594,7 +598,9 @@ def _attempt_with_fallback(url, timeout):
except Exception as exc2:
return None, None, None, exc2
return status, headers, body, None
- if status in FALLBACK_TO_GET_STATUSES:
+ content_type = headers.get("content-type", "").split(";", 1)[0].strip().lower()
+ needs_body = 200 <= status < 300 and content_type in ("", "text/html", "application/xhtml+xml")
+ if status in FALLBACK_TO_GET_STATUSES or needs_body:
try:
status, headers, body = _single_http_call(url, "GET", timeout)
except Exception as exc2:
diff --git a/scripts/test_check_readme.py b/scripts/test_check_readme.py
new file mode 100644
index 000000000..ea0bcbab3
--- /dev/null
+++ b/scripts/test_check_readme.py
@@ -0,0 +1,115 @@
+"""Offline HTTP regression tests for README link checking.
+
+Run from the repository root: python3 -m unittest discover -s scripts -v
+"""
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from check_readme import check_url
+
+
+class LinkHandler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def do_HEAD(self):
+ self.respond(head=True)
+
+ def do_GET(self):
+ self.respond(head=False)
+
+ def respond(self, head):
+ self.server.requests.append((self.command, self.path))
+ status = 200
+ content_type = "text/html; charset=utf-8"
+ body = b"
A working tutorial"
+ if self.path == "/soft-404":
+ body = b"Page not found"
+ elif self.path == "/no-content-type":
+ content_type = None
+ body = b"Page not found"
+ elif self.path == "/xhtml":
+ content_type = "application/xhtml+xml"
+ body = b"Page not found"
+ elif self.path == "/binary":
+ content_type = "application/pdf"
+ body = b"%PDF-1.4"
+ elif self.path == "/head-unsupported" and head:
+ status = 405
+ elif self.path == "/redirect":
+ status = 302
+ elif self.path == "/gone":
+ status = 410
+ self.send_response(status)
+ if content_type is not None:
+ self.send_header("Content-Type", content_type)
+ self.send_header("Content-Length", str(len(body)))
+ if status == 302:
+ self.send_header("Location", "/working")
+ self.end_headers()
+ if not head:
+ self.wfile.write(body)
+
+
+class LinkBodyTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.server = ThreadingHTTPServer(("127.0.0.1", 0), LinkHandler)
+ cls.server.requests = []
+ cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
+ cls.thread.start()
+ cls.base_url = "http://127.0.0.1:{}".format(cls.server.server_port)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.server.shutdown()
+ cls.server.server_close()
+ cls.thread.join()
+
+ def setUp(self):
+ self.server.requests.clear()
+
+ def check(self, path, **kwargs):
+ return check_url(self.base_url + path, timeout=2, retries=0, **kwargs)
+
+ def test_successful_head_does_not_hide_soft_404(self):
+ result = self.check("/soft-404")
+ self.assertEqual(result["class"], "SUSPECT")
+ self.assertIn("soft-404", result["note"])
+
+ def test_working_html_is_checked_with_get(self):
+ self.assertEqual(self.check("/working")["class"], "OK")
+ self.assertEqual(self.server.requests, [("HEAD", "/working"), ("GET", "/working")])
+
+ def test_unspecified_content_type_is_checked(self):
+ self.assertEqual(self.check("/no-content-type")["class"], "SUSPECT")
+
+ def test_xhtml_is_checked(self):
+ self.assertEqual(self.check("/xhtml")["class"], "SUSPECT")
+
+ def test_binary_link_does_not_need_get(self):
+ self.assertEqual(self.check("/binary")["class"], "OK")
+ self.assertEqual(self.server.requests, [("HEAD", "/binary")])
+
+ def test_unsupported_head_falls_back_to_get_once(self):
+ self.assertEqual(self.check("/head-unsupported")["class"], "OK")
+ self.assertEqual(self.server.requests, [("HEAD", "/head-unsupported"), ("GET", "/head-unsupported")])
+
+ def test_redirect_is_followed_before_fetching_html(self):
+ result = self.check("/redirect")
+ self.assertEqual(result["class"], "OK")
+ self.assertEqual(result["redirect_to"], self.base_url + "/working")
+ self.assertEqual(self.server.requests, [("HEAD", "/redirect"), ("HEAD", "/working"), ("GET", "/working")])
+
+ def test_domain_policy_applies_to_soft_404(self):
+ host = "127.0.0.1:{}".format(self.server.server_port)
+ self.assertEqual(self.check("/soft-404", policy={host})["class"], "BLOCKED")
+
+ def test_gone_status_does_not_need_get(self):
+ self.assertEqual(self.check("/gone")["class"], "HARD_DEAD")
+ self.assertEqual(self.server.requests, [("HEAD", "/gone")])
+
+
+if __name__ == "__main__":
+ unittest.main()