From 34d728e92c6a8da93a41680ace617c354532d8ad Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 25 Oct 2026 17:23:30 -0700 Subject: [PATCH 1/2] Load the Search Page Only on a Run Over the Default List (#536) * Load the Search Page Only on a Run Over the Default List A run given another list with --urls, such as the family site's pages, no longer adds /search/, so it passes on its own pages. A list that wants the search page names it. Co-Authored-By: Claude Opus 5.5 * Say in --help That Another List Names the Search Page Co-Authored-By: Claude Opus 5.5 * Refuse a URL List That Names No Page Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- OPERATIONS.md | 2 +- checks/check-csp-browser.py | 24 ++++++++++++----- checks/tests/test_check_csp_browser.py | 37 +++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/OPERATIONS.md b/OPERATIONS.md index d225c6b5..c1799a1a 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -104,7 +104,7 @@ checks/check-csp-browser.py "$SITE_BASE_URL" - **It needs Docker and `uv` on the host, and no browser.** Chromium runs in the Playwright image [`checks/browser/Dockerfile`](./checks/browser/Dockerfile) pins, where Dependabot moves it. `uv` installs the Playwright package inside it at the image tag's own version, so the two cannot drift. Its cache is `UV_CACHE_DIR` where that is set, or a directory under the system temporary root. - **The container shares the host network**, so it reaches a local mirror by the name the host resolves. -- **A violation fails the run unless [`checks/csp-browser-allow.txt`](./checks/csp-browser-allow.txt) lists it.** Each line is a directive, a blocked origin, and the reason, tab separated. A blocked scheme is written as the browser reports it, such as `data` with no colon. An entry that nothing matched fails a run over the default list, so an entry goes when its cause does. `--urls` takes another list, and such a partial run leaves unused entries alone. +- **A violation fails the run unless [`checks/csp-browser-allow.txt`](./checks/csp-browser-allow.txt) lists it.** Each line is a directive, a blocked origin, and the reason, tab separated. A blocked scheme is written as the browser reports it, such as `data` with no colon. An entry that nothing matched fails a run over the default list, so an entry goes when its cause does. `--urls` takes another list, and such a partial run leaves unused entries alone. It also loads `/search/` only where the list names it, since a list can be for the family site, which has no search page. - **A page that does not answer `200` fails the run**, since a trial over error pages proves nothing. - **It runs only against the local mirrors.** VPS staging sits behind the auth gate, which the check does not present. A full run is hundreds of pages. Production takes only a handful of probes per pass, since a burst there can get the reviewing network banned. - **It runs by hand rather than in the release.** It needs Docker, a browser image, and a running server, and neither the release build nor the unit suite has all three. diff --git a/checks/check-csp-browser.py b/checks/check-csp-browser.py index 6af97fae..2117af22 100755 --- a/checks/check-csp-browser.py +++ b/checks/check-csp-browser.py @@ -8,7 +8,7 @@ event in the page and in each frame it holds, as it fires. The event fires for an enforcing policy and a Report-Only one alike. Each page is scrolled to its end so lazy images load, and its code-copy buttons are clicked. The search page also gets a typed query, so the index fetch and the -results run. +results run. A run over the default list always loads it, and another list only where it names it. The browser runs in the Playwright image `browser/Dockerfile` pins, where Dependabot moves it. The Playwright package is installed at the image tag's own version, so the two never disagree. The @@ -48,15 +48,19 @@ def pinned_image(dockerfile_text): return found.group(1), found.group(2) -def read_paths(texts): - """The site paths the lists name, in order, once each, the search page always among them.""" +def read_paths(texts, search): + """The site paths the lists name, in order, once each, and the search page where asked. + + Only a run over the default list asks, since another list can be for a site with no search + page. A list that wants it names it. + """ paths = [] for text in texts: for line in text.splitlines(): path = line.strip() if path and path not in paths: paths.append(path) - if SEARCH_PATH not in paths: + if search and SEARCH_PATH not in paths: paths.append(SEARCH_PATH) return paths @@ -234,7 +238,10 @@ def main(argv=None): "--urls", type=pathlib.Path, action="append", - help=f"a list of site paths, repeatable (default {URLS.name})", + help=( + f"a list of site paths, repeatable (default {URLS.name}, plus {SEARCH_PATH}," + f" which another list names where it wants it)" + ), ) parser.add_argument( "--probe", @@ -253,7 +260,12 @@ def main(argv=None): image, version = pinned_image(DOCKERFILE.read_text(encoding="utf-8")) allow = read_allow(ALLOW.read_text(encoding="utf-8")) - paths = read_paths(p.read_text(encoding="utf-8") for p in (args.urls or [URLS])) + paths = read_paths( + (p.read_text(encoding="utf-8") for p in (args.urls or [URLS])), + search=args.urls is None, + ) + if not paths: + parser.error("the --urls lists name no page, so there is nothing to load") cache = pathlib.Path( os.environ.get("UV_CACHE_DIR") or pathlib.Path(tempfile.gettempdir()) / "blog-csp-browser-uv" diff --git a/checks/tests/test_check_csp_browser.py b/checks/tests/test_check_csp_browser.py index b6503864..64586c6f 100644 --- a/checks/tests/test_check_csp_browser.py +++ b/checks/tests/test_check_csp_browser.py @@ -56,13 +56,17 @@ def test_the_committed_allow_list_parses(self) -> None: class ListTest(unittest.TestCase): def test_paths_keep_order_drop_repeats_and_add_search(self) -> None: - paths = gate.read_paths(["/\n/a/\n\n", "/a/\n/b/\n"]) + paths = gate.read_paths(["/\n/a/\n\n", "/a/\n/b/\n"], search=True) self.assertEqual(paths, ["/", "/a/", "/b/", gate.SEARCH_PATH]) def test_a_listed_search_page_is_not_added_twice(self) -> None: - paths = gate.read_paths([f"{gate.SEARCH_PATH}\n/\n"]) + paths = gate.read_paths([f"{gate.SEARCH_PATH}\n/\n"], search=True) self.assertEqual(paths, [gate.SEARCH_PATH, "/"]) + def test_paths_not_asking_for_search_keep_only_their_own(self) -> None: + paths = gate.read_paths(["/en/\n/af/\n"], search=False) + self.assertEqual(paths, ["/en/", "/af/"]) + def test_an_allow_line_without_a_reason_is_refused(self) -> None: with self.assertRaises(ValueError): gate.read_allow("img-src\thttps://images.example\n") @@ -135,7 +139,9 @@ def setUp(self) -> None: def run_main(self, records, argv=()): with ( - mock.patch.object(gate, "run_probe", return_value=(records, None)), + mock.patch.object( + gate, "run_probe", return_value=(records, None) + ) as self.probe, mock.patch.object(gate, "docker_command", return_value=["true"]), mock.patch.object(gate, "read_allow", return_value=[]), contextlib.redirect_stdout(io.StringIO()) as out, @@ -144,11 +150,21 @@ def run_main(self, records, argv=()): return code, out.getvalue() def test_every_page_loaded_clean_passes(self) -> None: - paths = gate.read_paths([gate.URLS.read_text(encoding="utf-8")]) + paths = gate.read_paths([gate.URLS.read_text(encoding="utf-8")], search=True) code, out = self.run_main([loaded(path) for path in paths]) self.assertEqual(code, 0, out) self.assertIn("PASS", out) + def test_a_run_over_another_list_loads_only_its_own_pages(self) -> None: + with tempfile.TemporaryDirectory() as folder: + urls = pathlib.Path(folder) / "family-urls.txt" + urls.write_text("/en/\n/af/\n", encoding="utf-8") + code, out = self.run_main( + [loaded("/en/"), loaded("/af/")], ["--urls", str(urls)] + ) + self.assertEqual(code, 0, out) + self.assertEqual(self.probe.call_args.args[1], ["/en/", "/af/"]) + def test_a_page_that_never_reported_fails(self) -> None: code, out = self.run_main([loaded("/")]) self.assertEqual(code, 1) @@ -181,6 +197,19 @@ def test_a_relative_base_url_is_a_usage_error(self) -> None: gate.main(["mirror.example"]) self.assertEqual(raised.exception.code, 2) + def test_a_list_naming_no_page_is_a_usage_error(self) -> None: + with tempfile.TemporaryDirectory() as folder: + urls = pathlib.Path(folder) / "empty-urls.txt" + urls.write_text("\n \n", encoding="utf-8") + with ( + mock.patch.object(gate, "run_probe") as probe, + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(SystemExit) as raised, + ): + gate.main([BASE, "--urls", str(urls)]) + self.assertEqual(raised.exception.code, 2) + probe.assert_not_called() + class CommandTest(unittest.TestCase): def test_the_container_installs_the_tag_version_and_reaches_the_host(self) -> None: From 9bd721a8de2422f03ce570723221584bae697d1e Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Sun, 25 Oct 2026 17:50:06 -0700 Subject: [PATCH 2/2] Rename Every Python File to the Fleet's snake_case Names (#537) * Load Every Hyphenated Sibling Check From One Shared Helper checks/sibling.py holds the one loader. The three helper copies in check-csp.py, fuzz-media-parsers.py, and test_check_media_metadata.py, and the four inline copies at the top of the other check tests, now call it by name. Co-Authored-By: Claude Opus 5.5 * Say Which Scripts Cannot Import the Sibling Loader Co-Authored-By: Claude Opus 5.5 * Rename Every Python File to the Fleet's snake_case Names A hyphenated module cannot be named in an import, which is why the tooling needed a by-path loader at all. Every .py under checks/, scripts/, and capture/ now takes a snake_case name, and a gate takes the _gate suffix, so siblings import by name and checks/sibling.py goes. A test puts its directory on sys.path and imports the module, the way the hub's tests do, so running a test file directly works again. Each directory's mypy config ignores the one module it imports from the other declared directory. Co-Authored-By: Claude Opus 5.5 * Give the checks/ mypy Override Its Real Reason Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .editorconfig | 2 +- .github/actions/validate/action.yml | 12 +++--- .secrets/example.env | 4 +- .vscode/tasks.json | 4 +- AUDIT.md | 4 +- CONTENT.md | 32 +++++++-------- ENVIRONMENT.md | 6 +-- GOVERNANCE.md | 2 +- OPERATIONS.md | 8 ++-- README.md | 18 ++++----- capture/README.md | 28 ++++++------- capture/{build-golden.py => build_golden.py} | 0 ...{build-redirects.py => build_redirects.py} | 0 .../{clean-content.py => clean_content.py} | 2 +- ...{enumerate-media.py => enumerate_media.py} | 0 ...alize-external.py => localize_external.py} | 2 +- ...ture-content.py => restructure_content.py} | 9 ++--- capture/run-wp2hugo.sh | 4 +- checks/README.md | 40 +++++++++---------- ...eck-csp-browser.py => csp_browser_gate.py} | 4 +- checks/{check-csp.py => csp_gate.py} | 16 ++------ .../{check-env-docs.py => env_docs_gate.py} | 0 ...check-github-env.py => github_env_gate.py} | 0 ...dia-metadata.py => media_metadata_gate.py} | 4 +- ...a-parsers.py => media_parser_fuzz_gate.py} | 18 ++------- ...redactions.py => media_redactions_gate.py} | 4 +- checks/pyproject.toml | 5 +++ ...sp_browser.py => test_csp_browser_gate.py} | 12 ++---- .../{test_check_csp.py => test_csp_gate.py} | 11 ++--- ..._github_env.py => test_github_env_gate.py} | 13 ++---- ...etadata.py => test_media_metadata_gate.py} | 31 +++++--------- ...heck_text_pii.py => test_text_pii_gate.py} | 13 ++---- .../{check-text-pii.py => text_pii_gate.py} | 0 ...check-url-parity.py => url_parity_gate.py} | 0 deploy/README.md | 26 ++++++------ deploy/make-release.sh | 4 +- ...{normalize-media.py => normalize_media.py} | 13 ++---- .../{normalize-text.py => normalize_text.py} | 0 scripts/pyproject.toml | 5 +++ scripts/{redact-media.py => redact_media.py} | 40 +++++-------------- scripts/tests/test_normalize_text.py | 13 ++---- scripts/tests/test_redact_media.py | 11 ++--- sites/viljoen.family/README.md | 4 +- themes/README.md | 2 +- 44 files changed, 177 insertions(+), 249 deletions(-) rename capture/{build-golden.py => build_golden.py} (100%) rename capture/{build-redirects.py => build_redirects.py} (100%) rename capture/{clean-content.py => clean_content.py} (98%) rename capture/{enumerate-media.py => enumerate_media.py} (100%) rename capture/{localize-external.py => localize_external.py} (99%) rename capture/{restructure-content.py => restructure_content.py} (94%) rename checks/{check-csp-browser.py => csp_browser_gate.py} (98%) rename checks/{check-csp.py => csp_gate.py} (97%) rename checks/{check-env-docs.py => env_docs_gate.py} (100%) rename checks/{check-github-env.py => github_env_gate.py} (100%) rename checks/{check-media-metadata.py => media_metadata_gate.py} (99%) rename checks/{fuzz-media-parsers.py => media_parser_fuzz_gate.py} (99%) rename checks/{check-media-redactions.py => media_redactions_gate.py} (96%) rename checks/tests/{test_check_csp_browser.py => test_csp_browser_gate.py} (96%) rename checks/tests/{test_check_csp.py => test_csp_gate.py} (97%) rename checks/tests/{test_check_github_env.py => test_github_env_gate.py} (95%) rename checks/tests/{test_check_media_metadata.py => test_media_metadata_gate.py} (97%) rename checks/tests/{test_check_text_pii.py => test_text_pii_gate.py} (96%) rename checks/{check-text-pii.py => text_pii_gate.py} (100%) rename checks/{check-url-parity.py => url_parity_gate.py} (100%) rename scripts/{normalize-media.py => normalize_media.py} (98%) rename scripts/{normalize-text.py => normalize_text.py} (100%) rename scripts/{redact-media.py => redact_media.py} (91%) diff --git a/.editorconfig b/.editorconfig index b797aa79..a63bbd99 100644 --- a/.editorconfig +++ b/.editorconfig @@ -238,7 +238,7 @@ end_of_line = unset insert_final_newline = false trim_trailing_whitespace = false -# Caddy map files are space-separated redirect tables generated by `capture/build-redirects.py`. +# Caddy map files are space-separated redirect tables generated by `capture/build_redirects.py`. # Trailing whitespace is significant to the parse, and the generator owns the formatting. [deploy/maps/*.map] trim_trailing_whitespace = false diff --git a/.github/actions/validate/action.yml b/.github/actions/validate/action.yml index 6b663346..08682b51 100644 --- a/.github/actions/validate/action.yml +++ b/.github/actions/validate/action.yml @@ -21,29 +21,29 @@ runs: # This check runs both directions: undocumented values, and rows describing nothing. - name: Check environment docs step shell: bash - run: python3 checks/check-env-docs.py + run: python3 checks/env_docs_gate.py # Hugo copies static/ byte for byte, so whatever a camera wrote is served with the image. # The build gate reads pages and the prose gate reads text, so nothing else looks inside a file. # This reads values rather than tag names, and opens archives, since both hid findings before. - name: Check media metadata step shell: bash - run: python3 checks/check-media-metadata.py + run: python3 checks/media_metadata_gate.py # The carried tree is all well formed, so this feeds the same parsers seeded malformed variants. - name: Fuzz media parsers step shell: bash - run: python3 checks/fuzz-media-parsers.py + run: python3 checks/media_parser_fuzz_gate.py # A redacted image replaced by its original passes every other check, so this compares hashes. - name: Check media redactions step shell: bash - run: python3 checks/check-media-redactions.py + run: python3 checks/media_redactions_gate.py # The prose gate reads characters, so this is what reads a post's text for personal data. - name: Check text personal data step shell: bash - run: python3 checks/check-text-pii.py + run: python3 checks/text_pii_gate.py # The hub's validate job checkout is shallow. # The mtime restore below needs full history to date each file from its last commit. @@ -65,7 +65,7 @@ runs: args: static # This is the build and URL-contract gate, not an extra step beside one. - # The script runs `hugo --panicOnWarning` and check-url-parity itself, so a separate build would repeat both. + # The script runs `hugo --panicOnWarning` and url_parity_gate.py itself, so a separate build would repeat both. # --panicOnWarning is the real gate, and the swap overrides in layouts/ let it stay on. # Every local asset reference must resolve, and every legacy image URL from the WordPress import must too. # diff --git a/.secrets/example.env b/.secrets/example.env index 77382c55..9f770f3b 100644 --- a/.secrets/example.env +++ b/.secrets/example.env @@ -2,7 +2,7 @@ # # WHAT EACH VALUE MEANS IS IN ENVIRONMENT.md, which is the one place it is described. # This file states the shape and a placeholder. Add a value here and describe it there, -# or checks/check-env-docs.py fails. +# or checks/env_docs_gate.py fails. # # One destination: ~/.secrets/blog...env on a workstation, one file # per environment. The off-host VPS backup pull, and the environment file it reads, are @@ -66,7 +66,7 @@ SITE_EXTRA_AUTH_TOKEN= # Environment-independent, so these belong in the default file only. # The provenance capture, holding the exports and the crawl of the old platform. -# Every script under capture/ reads beneath it, and all but build-redirects.py write there +# Every script under capture/ reads beneath it, and all but build_redirects.py write there # too. That one writes the committed maps under deploy/maps/ in the repository. CAPTURE_ROOT=/path/to/blog-capture diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 9fc2f6d2..ea4103d0 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -56,7 +56,7 @@ { "label": "Check: URL Parity", "type": "process", - "command": "checks/check-url-parity.py", + "command": "checks/url_parity_gate.py", "args": [ "public" ], @@ -66,7 +66,7 @@ { "label": "Check: CSP", "type": "process", - "command": "checks/check-csp.py", + "command": "checks/csp_gate.py", "args": [ "public" ], diff --git a/AUDIT.md b/AUDIT.md index 322859b4..51e1715f 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -61,7 +61,7 @@ One key covers both environments, a deliberate decision recorded in `OPERATIONS. `ENVIRONMENT.md`'s "The GitHub Environments" table declares every secret and variable name, its kind, and each store holding it. This repo's own check compares that table against GitHub: ```sh -python3 checks/check-github-env.py +python3 checks/github_env_gate.py ``` It compares in both directions and reports three kinds of finding. A store lacks a declared name, a store holds a name the table does not list, or a value is held as the wrong kind. The second is the one presence-checking misses. A Pangolin access token on `production` is reported as unlisted rather than passed as a harmless extra. Production answers unauthenticated, so a token there means a check could pass through a gate production is not supposed to have. @@ -76,7 +76,7 @@ This site's whole risk is silent URL loss, so the contract is audited like any o ```sh hugo --gc --minify --panicOnWarning -checks/check-url-parity.py public +checks/url_parity_gate.py public checks/check-live-urls.sh ``` diff --git a/CONTENT.md b/CONTENT.md index 5e043e0f..c06d6aeb 100644 --- a/CONTENT.md +++ b/CONTENT.md @@ -76,13 +76,13 @@ Embed with the `figure` shortcode, and wrap a set in `gallery`: **Alt text is required and describes the image**, since it is what a reader without the image gets. -**Every image is normalized before it is committed.** Run [`scripts/normalize-media.py`][normalize]. It keeps only what a picture needs in order to render, the dimensions and, on every format but GIF, the color profile, plus the orientation and the capture timestamp on a JPEG, the one format here whose Exif the gate admits. A JPEG keeps its timestamp only when its whole Exif segment already passes the gate. An Exif segment holding anything the gate does not admit is dropped whole, its timestamp with it, and only an orientation other than upright is written back. A JPEG whose Orientation is not a single SHORT is refused, since browsers ignore any other shape while other decoders honor it. So is one whose Orientation holds two different values, twice in its main directory, IFD0, or once there and once in a directory reached from it, since decoders differ on which of the two they read. So is one holding an Orientation in, or reached from, an IFD0 that sits inside the TIFF header or whose entries run past the end of its Exif segment, since decoders differ on whether they read that IFD0 at all. A directory past IFD0 whose entries run past the end is read as far as it goes, and the JPEG is refused where reading it changes the Orientation. An Orientation entry the end of the segment cuts off is refused wherever it sits. An IFD0 holding no Orientation counts as upright, since a browser reads IFD0, so a turn held only in a directory reached from it is refused too. A JPEG with two Exif segments is refused rather than cleaned. A PNG or WebP keeps no Exif at all, so it keeps no timestamp. The normalizer refuses one whose Orientation a JPEG would be refused for, or that turns the picture, since dropping that Exif could turn it back. A color profile stays only when it is byte for byte one of the standard sRGB or Display P3 profiles the gate pins by digest. The normalizer refuses any other, so convert that picture to sRGB first. A PNG keeps its gamma and chromaticity only at the sRGB values, and its pixel density only as square pixels, and the normalizer refuses any other. Its significant bits and background color stay only at full depth and zero, since no browser draws by them, and go otherwise. A palette in a truecolor PNG only suggests one, so it goes too. The normalizer refuses two palettes in any PNG, and a palette in a grayscale PNG. In a palette image, it refuses a palette that is empty, holds a partial entry, follows the picture data, or outruns the bit depth. It refuses an animated PNG too, since GIF and WebP carry the animation here. It refuses a PNG whose header is not the first chunk or states a value the format does not define, one with no picture data chunk, and a palette image with no palette, since no decoder draws them and every byte of one is free. It refuses a PNG whose header, palette, or picture data fails its checksum. It drops an ancillary chunk that does, as libpng drops it, except an Exif or animation chunk it would refuse with a good checksum, which it refuses either way. It refuses a PNG whose picture data is split by another chunk, is empty, or is not a valid compressed stream. It refuses one whose stream is cut off, or holds bytes after its end. It refuses one whose picture data inflates to more or fewer bytes than the header's rows need. It refuses one holding a row filter type the format does not define. It refuses one wider or taller than 1,000,000 pixels, which libpng refuses to draw. It refuses one whose rows need more than 256 MiB once inflated, the most the gate reads of any file. It writes a PNG end chunk holding bytes or a wrong checksum as empty. It writes a GIF's unused graphic control fields as zero. It does the same for a GIF's sort flags, reserved bits, color resolution, background index, aspect ratio, and table size where no table follows. It drops a GIF's global color table where every image carries its own, and writes its version as 89a. It writes a WebP animation's background color as zero, and drops fill bytes between JPEG segments. It refuses a JPEG holding fill bytes inside a scan, and a PNG whose gamma, chromaticity, profile, transparency, or density chunk sits where a decoder does not read it. Everything else goes, and it never asks what the removed thing was. Location, device, and authorship are not on the list, so they go without anyone thinking of them. That is the point. A rule that enumerates hiding places is only as good as its last revision. The removal is lossless on every format carried here, so an original is not degraded in order to clean it. [`checks/check-media-metadata.py`][metadata] fails the build on anything left over, and it reads inside a `.zip`. It names a symlink rather than reading through it, a symlinked tree included, and the normalizer checks for one again before it writes each file. Then inspect the image at full resolution. Redact a house number, a license plate, a face, a serial number, a rating plate, a barcode, and a hardware address. "What Identifies" below gives the full checklist and the narrower rule for an image already published. Cover each one with a flat opaque fill, never a blur or a mosaic. Both can be reversed for a short string drawn from a known alphabet. A fill also tells the reader that something was removed. Verify both halves. `python3 checks/check-media-metadata.py` passes, and a full-resolution crop of each redacted area is unreadable. +**Every image is normalized before it is committed.** Run [`scripts/normalize_media.py`][normalize]. It keeps only what a picture needs in order to render, the dimensions and, on every format but GIF, the color profile, plus the orientation and the capture timestamp on a JPEG, the one format here whose Exif the gate admits. A JPEG keeps its timestamp only when its whole Exif segment already passes the gate. An Exif segment holding anything the gate does not admit is dropped whole, its timestamp with it, and only an orientation other than upright is written back. A JPEG whose Orientation is not a single SHORT is refused, since browsers ignore any other shape while other decoders honor it. So is one whose Orientation holds two different values, twice in its main directory, IFD0, or once there and once in a directory reached from it, since decoders differ on which of the two they read. So is one holding an Orientation in, or reached from, an IFD0 that sits inside the TIFF header or whose entries run past the end of its Exif segment, since decoders differ on whether they read that IFD0 at all. A directory past IFD0 whose entries run past the end is read as far as it goes, and the JPEG is refused where reading it changes the Orientation. An Orientation entry the end of the segment cuts off is refused wherever it sits. An IFD0 holding no Orientation counts as upright, since a browser reads IFD0, so a turn held only in a directory reached from it is refused too. A JPEG with two Exif segments is refused rather than cleaned. A PNG or WebP keeps no Exif at all, so it keeps no timestamp. The normalizer refuses one whose Orientation a JPEG would be refused for, or that turns the picture, since dropping that Exif could turn it back. A color profile stays only when it is byte for byte one of the standard sRGB or Display P3 profiles the gate pins by digest. The normalizer refuses any other, so convert that picture to sRGB first. A PNG keeps its gamma and chromaticity only at the sRGB values, and its pixel density only as square pixels, and the normalizer refuses any other. Its significant bits and background color stay only at full depth and zero, since no browser draws by them, and go otherwise. A palette in a truecolor PNG only suggests one, so it goes too. The normalizer refuses two palettes in any PNG, and a palette in a grayscale PNG. In a palette image, it refuses a palette that is empty, holds a partial entry, follows the picture data, or outruns the bit depth. It refuses an animated PNG too, since GIF and WebP carry the animation here. It refuses a PNG whose header is not the first chunk or states a value the format does not define, one with no picture data chunk, and a palette image with no palette, since no decoder draws them and every byte of one is free. It refuses a PNG whose header, palette, or picture data fails its checksum. It drops an ancillary chunk that does, as libpng drops it, except an Exif or animation chunk it would refuse with a good checksum, which it refuses either way. It refuses a PNG whose picture data is split by another chunk, is empty, or is not a valid compressed stream. It refuses one whose stream is cut off, or holds bytes after its end. It refuses one whose picture data inflates to more or fewer bytes than the header's rows need. It refuses one holding a row filter type the format does not define. It refuses one wider or taller than 1,000,000 pixels, which libpng refuses to draw. It refuses one whose rows need more than 256 MiB once inflated, the most the gate reads of any file. It writes a PNG end chunk holding bytes or a wrong checksum as empty. It writes a GIF's unused graphic control fields as zero. It does the same for a GIF's sort flags, reserved bits, color resolution, background index, aspect ratio, and table size where no table follows. It drops a GIF's global color table where every image carries its own, and writes its version as 89a. It writes a WebP animation's background color as zero, and drops fill bytes between JPEG segments. It refuses a JPEG holding fill bytes inside a scan, and a PNG whose gamma, chromaticity, profile, transparency, or density chunk sits where a decoder does not read it. Everything else goes, and it never asks what the removed thing was. Location, device, and authorship are not on the list, so they go without anyone thinking of them. That is the point. A rule that enumerates hiding places is only as good as its last revision. The removal is lossless on every format carried here, so an original is not degraded in order to clean it. [`checks/media_metadata_gate.py`][metadata] fails the build on anything left over, and it reads inside a `.zip`. It names a symlink rather than reading through it, a symlinked tree included, and the normalizer checks for one again before it writes each file. Then inspect the image at full resolution. Redact a house number, a license plate, a face, a serial number, a rating plate, a barcode, and a hardware address. "What Identifies" below gives the full checklist and the narrower rule for an image already published. Cover each one with a flat opaque fill, never a blur or a mosaic. Both can be reversed for a short string drawn from a known alphabet. A fill also tells the reader that something was removed. Verify both halves. `python3 checks/media_metadata_gate.py` passes, and a full-resolution crop of each redacted area is unreadable. -**The text of a post is gated the same way.** [`checks/check-text-pii.py`][text-pii] reads every Markdown file under `content/` outside the imported archive years, the same boundary `.github/prose-gate-excludes` draws for the prose gate. It reads front matter and code blocks too, since a pasted configuration is where a hardware address usually hides. It reports an email address, a hardware address, a public IP address, a coordinate pair, a URL carrying coordinates or naming a sensor or station, a street address, and a phone number. Each finding names the file, the line, and the class, and never the value, so the CI log does not republish it. A documentation-range or private address is not public and is not reported. Voice still requires a placeholder for a private one. A hardware address is read in its colon, hyphen, and dotted forms. A bare run of twelve hex digits is not, since a short commit hash has the same shape. A four-part number directly after a word such as version, build, or firmware is read as a version rather than an address. Run `python3 checks/check-text-pii.py` locally, the same command the validate action runs. +**The text of a post is gated the same way.** [`checks/text_pii_gate.py`][text-pii] reads every Markdown file under `content/` outside the imported archive years, the same boundary `.github/prose-gate-excludes` draws for the prose gate. It reads front matter and code blocks too, since a pasted configuration is where a hardware address usually hides. It reports an email address, a hardware address, a public IP address, a coordinate pair, a URL carrying coordinates or naming a sensor or station, a street address, and a phone number. Each finding names the file, the line, and the class, and never the value, so the CI log does not republish it. A documentation-range or private address is not public and is not reported. Voice still requires a placeholder for a private one. A hardware address is read in its colon, hyphen, and dotted forms. A bare run of twelve hex digits is not, since a short commit hash has the same shape. A four-part number directly after a word such as version, build, or firmware is read as a version rather than an address. Run `python3 checks/text_pii_gate.py` locally, the same command the validate action runs. **A value a post prints on purpose is declared, not passed silently.** A vendor's support email or a public service's address goes in [`checks/text-pii-allow.json`][text-pii-allow] as an entry under `allow`, with a `class` naming what the gate reported, exactly one of `value` (the exact text) or `pattern` (a regular expression the whole value must match), and a `reason`. An entry is bound to its class, so a pattern declared for one class never silences another. An entry that matches nothing in scope fails the gate, so the list holds only values a post still prints. Any other false positive is declared the same way. The tests in `checks/tests/` run in the validator's unit-test job, under coverage, and every value in them is constructed. -**Every image added must be linked from a page.** `ORPHANED_MEDIA` in [`checks/check-url-parity.py`][parity] is an exact count rather than a ceiling, so an unlinked file moves it and fails the gate. That is deliberate: it is the only check that can see an image the site carries and no page shows. +**Every image added must be linked from a page.** `ORPHANED_MEDIA` in [`checks/url_parity_gate.py`][parity] is an exact count rather than a ceiling, so an unlinked file moves it and fails the gate. That is deliberate: it is the only check that can see an image the site carries and no page shows. ## What Identifies @@ -122,9 +122,9 @@ These are settled, in new media and old alike, and reopening one wastes a review Treat that as a judgment about the archive, never as a precedent for a new post. -**Every archive redaction is data, never a hand edit.** [`checks/media-redactions.json`][redactions] names each file, its fills or crop, and why. [`scripts/redact-media.py`][redact] applies it after the metadata normalizer, and records the hash each file starts from and lands on. So a run is deterministic, and a second run changes nothing. [`checks/check-media-redactions.py`][redactions-check] fails the build on a declared file that is not at its redacted result. To change an entry, restore the file's original from history, run the metadata normalizer on it, edit the entry, and run `uv run --no-project --with-requirements scripts/requirements.txt scripts/redact-media.py --record`. A restored file that any committed revision of the manifest records as a result is refused, since it already carries an earlier round's fills. So is one other than the original the entry's `source` hash records, and deleting that hash takes a deliberately different original. An unchanged entry can also stop reproducing, after a Pillow bump that moves the output. Re-record it the same way, deleting its `declared` digest rather than editing the entry. +**Every archive redaction is data, never a hand edit.** [`checks/media-redactions.json`][redactions] names each file, its fills or crop, and why. [`scripts/redact_media.py`][redact] applies it after the metadata normalizer, and records the hash each file starts from and lands on. So a run is deterministic, and a second run changes nothing. [`checks/media_redactions_gate.py`][redactions-check] fails the build on a declared file that is not at its redacted result. To change an entry, restore the file's original from history, run the metadata normalizer on it, edit the entry, and run `uv run --no-project --with-requirements scripts/requirements.txt scripts/redact_media.py --record`. A restored file that any committed revision of the manifest records as a result is refused, since it already carries an earlier round's fills. So is one other than the original the entry's `source` hash records, and deleting that hash takes a deliberately different original. An unchanged entry can also stop reproducing, after a Pillow bump that moves the output. Re-record it the same way, deleting its `declared` digest rather than editing the entry. -**The archive's text follows the same model.** [`scripts/normalize-text.py`][normalize-text] replaces typographic characters with ASCII outside fenced code, blockquotes, front matter, and lines holding a protected product name. It then applies the corrections listed one by one in [`scripts/text-corrections.json`][text-corrections]: doubled words, grammar slips, a title's HTML entity, and two links that located the home. Nothing else in an old post is restyled, its punctuation included. +**The archive's text follows the same model.** [`scripts/normalize_text.py`][normalize-text] replaces typographic characters with ASCII outside fenced code, blockquotes, front matter, and lines holding a protected product name. It then applies the corrections listed one by one in [`scripts/text-corrections.json`][text-corrections]: doubled words, grammar slips, a title's HTML entity, and two links that located the home. Nothing else in an old post is restyled, its punctuation included. ### Judging One @@ -191,13 +191,13 @@ This is the reason this file exists, so it is worth stating plainly. | Check | Reaches a post | | --- | --- | | `hugo --gc --minify --panicOnWarning` | yes, it must build | -| [`checks/check-url-parity.py`][parity] | yes, for URLs, asset references, and the orphan count | -| [`checks/check-csp.py`][csp] | yes, for an inline handler or `javascript:` URL a post's raw HTML adds, and for a script, image, stylesheet, font, frame, media file, or object it loads from another host | +| [`checks/url_parity_gate.py`][parity] | yes, for URLs, asset references, and the orphan count | +| [`checks/csp_gate.py`][csp] | yes, for an inline handler or `javascript:` URL a post's raw HTML adds, and for a script, image, stylesheet, font, frame, media file, or object it loads from another host | | `checks/check-live-urls.sh` | yes, against a running server | | markdownlint | no, `content/.markdownlint-cli2.jsonc` ignores the tree | | CSpell | no, `cspell.json` ignores the tree | | `prose_lint.py` | yes, for characters and punctuation on changed lines. `.github/prose-gate-excludes` leaves out the imported posts' years | -| [`checks/check-text-pii.py`][text-pii] | yes, for personal data in the text, outside the same archive years | +| [`checks/text_pii_gate.py`][text-pii] | yes, for personal data in the text, outside the same archive years | Nothing above judges the writing itself. A human does, before the post merges, and that read is the only gate for it. @@ -211,15 +211,15 @@ A fact in a published post that proves wrong is corrected in the post. It is not [checks]: ./checks/ [codestyle]: ./CODESTYLE.md -[csp]: ./checks/check-csp.py +[csp]: ./checks/csp_gate.py [operations]: ./OPERATIONS.md -[metadata]: ./checks/check-media-metadata.py -[normalize]: ./scripts/normalize-media.py -[text-pii]: ./checks/check-text-pii.py +[metadata]: ./checks/media_metadata_gate.py +[normalize]: ./scripts/normalize_media.py +[text-pii]: ./checks/text_pii_gate.py [text-pii-allow]: ./checks/text-pii-allow.json -[normalize-text]: ./scripts/normalize-text.py -[parity]: ./checks/check-url-parity.py -[redact]: ./scripts/redact-media.py +[normalize-text]: ./scripts/normalize_text.py +[parity]: ./checks/url_parity_gate.py +[redact]: ./scripts/redact_media.py [redactions]: ./checks/media-redactions.json -[redactions-check]: ./checks/check-media-redactions.py +[redactions-check]: ./checks/media_redactions_gate.py [text-corrections]: ./scripts/text-corrections.json diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md index 790ff705..85790e35 100644 --- a/ENVIRONMENT.md +++ b/ENVIRONMENT.md @@ -2,7 +2,7 @@ Every configuration value this repository reads or writes, described once. [`OPERATIONS.md`](./OPERATIONS.md) is the procedure and this is the reference the procedure points at, so a value is explained here and named elsewhere. -**A description lives here and nowhere else.** The `.example` files state the format, the scripts state the defaults, and this file states what a value means. [`checks/check-env-docs.py`](./checks/check-env-docs.py) fails if a value is declared or consumed anywhere and has no row below, or has a row and is declared nowhere, so the two stay in step without depending on anyone remembering. +**A description lives here and nowhere else.** The `.example` files state the format, the scripts state the defaults, and this file states what a value means. [`checks/env_docs_gate.py`](./checks/env_docs_gate.py) fails if a value is declared or consumed anywhere and has no row below, or has a row and is declared nowhere, so the two stay in step without depending on anyone remembering. **Values are grouped by where they live rather than by what reads them**, because where a value lives determines who can change it, what happens when it is wrong, and whether it reaches a public history. @@ -30,7 +30,7 @@ Held in `~/.secrets/blog...env`, one file per environment, | `SITE_EXTRA_BASE_URL` | the family site's base URL in this environment | Read by `check-live-urls.sh`, which checks the family page there when it is set. Leave it unset until the family hostname reaches this environment, since the check fails on any other answer. | | `SITE_EXTRA_AUTH_TOKEN_ID` | the family resource's access token id, for a family site behind the auth gate | Set both or neither, and only with `SITE_EXTRA_BASE_URL`. A token opens exactly one proxy resource, so the blog's pair cannot open the family host. | | `SITE_EXTRA_AUTH_TOKEN` | the token itself | Sent as `P-Access-Token` to the family host and nowhere else. | -| `CAPTURE_ROOT` | the provenance capture, holding the WordPress exports, the crawl of the old platform, and the inventories derived from it | Every script under [`capture/`](./capture/) reads beneath it, and all but one write there too. The exception is [`capture/build-redirects.py`](./capture/build-redirects.py), which writes the committed maps under `deploy/maps/` in this repository, and which also accepts the capture as a first argument that wins over this value. Environment-independent, so it belongs in the default file only. | +| `CAPTURE_ROOT` | the provenance capture, holding the WordPress exports, the crawl of the old platform, and the inventories derived from it | Every script under [`capture/`](./capture/) reads beneath it, and all but one write there too. The exception is [`capture/build_redirects.py`](./capture/build_redirects.py), which writes the committed maps under `deploy/maps/` in this repository, and which also accepts the capture as a first argument that wins over this value. Environment-independent, so it belongs in the default file only. | | `CAPTURE_SOURCE_URL` | the old platform's base URL, the site the crawl and the URL verification ran against | **Not `SITE_BASE_URL`.** The two hold the same string after the cutover and mean different things, so merging them points a verification run at the new site while every check still passes. Environment-independent. | | `CAPTURE_SOURCE_API` | the old platform's REST API for that site, carrying its numeric site id | Read for the post and page bodies in **rendered** form, which is what expands shortcodes so a media reference is seen the way a reader's browser sees it. Environment-independent. | | `CAPTURE_AUTHOR_SLUG` | the old platform's author slug, used to backfill the author archive and its pagination | Optional, and an account name rather than a site value, which is why it is a variable at all. Unset, [`capture/classify.py`](./capture/classify.py) skips the backfill and says so, rather than emitting a list that is silently short by the author URLs. Environment-independent. | @@ -60,7 +60,7 @@ Every variable and secret this repository stores for GitHub Actions, Dependabot, | `CODEGEN_APP_PRIVATE_KEY` | secret | `repository`, `dependabot` | the merge bot's GitHub App private key, read the same way | | `CODECOV_TOKEN` | secret | `repository`, `dependabot` | the Codecov upload token, passed by every workflow that calls the hub validator. Its coverage upload is best-effort, so an empty token skips the upload and fails nothing. The `dependabot` copy is what a Dependabot pull request's run reads | -**[`checks/check-github-env.py`](./checks/check-github-env.py) compares this table against GitHub.** It reads the names those stores hold, never a value, and leaves the Codespaces secret store unread. It reports a value missing from a store, held on a store the table does not name, or held as the wrong kind. Every environment GitHub has is read, so an environment the table never names must hold nothing. Run it after changing a value on GitHub or a row here. It needs a `gh` login that can administer the repository, since listing environment secrets requires that. A workflow's `GITHUB_TOKEN` cannot, so CI does not run it. +**[`checks/github_env_gate.py`](./checks/github_env_gate.py) compares this table against GitHub.** It reads the names those stores hold, never a value, and leaves the Codespaces secret store unread. It reports a value missing from a store, held on a store the table does not name, or held as the wrong kind. Every environment GitHub has is read, so an environment the table never names must hold nothing. Run it after changing a value on GitHub or a row here. It needs a `gh` login that can administer the repository, since listing environment secrets requires that. A workflow's `GITHUB_TOKEN` cannot, so CI does not run it. **The family values take the hub's names for a second site.** The hub's deploy task reads `SITE_EXTRA_BASE_URL` and the `SITE_EXTRA_AUTH_TOKEN_ID`/`SITE_EXTRA_AUTH_TOKEN` pair by those names from the environment. `check-live-urls.sh` and the `~/.secrets/blog.*.env` files use the same three names, so nothing translates between them. A deploy therefore checks the family host with its own token. diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 28349b57..68e53623 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -270,7 +270,7 @@ Every repo's GitHub repository details (the About panel) follow a fixed conventi - [`AGENTS.md`](./AGENTS.md): the agent entry point, carrying context and delegation rules plus the map to the sections below. - [`GOVERNANCE.md`](./GOVERNANCE.md), [`CODESTYLE.md`](./CODESTYLE.md), [`WORKFLOW.md`](./WORKFLOW.md), [`OPERATIONS.md`](./OPERATIONS.md), [`AUDIT.md`](./AUDIT.md): the governance, operations, and audit docs. This file is the cross-cutting-rules authority. -- [`content/`](./content/): the posts and pages, one file per URL. An imported archive that keeps its own voice. Its only text changes are what `scripts/normalize-text.py` applies: ASCII character substitutions, and a manifest of doubled-word and grammar corrections, a title's HTML entity, and two links that located the home. A blockquote, a fenced code block, and front matter keep the characters they were written with. [`CONTENT.md`](./CONTENT.md) is the contract for writing a new one. +- [`content/`](./content/): the posts and pages, one file per URL. An imported archive that keeps its own voice. Its only text changes are what `scripts/normalize_text.py` applies: ASCII character substitutions, and a manifest of doubled-word and grammar corrections, a title's HTML entity, and two links that located the home. A blockquote, a fenced code block, and front matter keep the characters they were written with. [`CONTENT.md`](./CONTENT.md) is the contract for writing a new one. - [`static/`](./static/): the media the posts reference, verified by content hash against the source export. - [`layouts/`](./layouts/), [`themes/`](./themes/), [`assets/`](./assets/): the site's templates and shortcodes, the vendored theme, and the site styles added to its stylesheet. - [`checks/`](./checks/): the URL contract and the gates that enforce it, covering both the built output and a running server. Its `pyproject.toml` holds the ruff, mypy, and coverage config for these standard-library-only gates. diff --git a/OPERATIONS.md b/OPERATIONS.md index c1799a1a..be974e5c 100644 --- a/OPERATIONS.md +++ b/OPERATIONS.md @@ -94,12 +94,12 @@ Expect a `PASS` naming the number of URLs honored. That is the three lists' comb A documentation-only or workflow-only change does not need this. A change to the four paths above does, because for those CI's green is not evidence. -**A change to a `Content-Security-Policy` in `security-headers.txt` is also tried in a browser**, on the local staging mirror, before its pull request opens. So is a theme or `layouts/` change that adds a style, a font, a fetch, or a script. [`checks/check-csp-browser.py`](./checks/check-csp-browser.py) loads every page of `golden-urls.txt` plus `/search/` in headless Chromium. It scrolls each page to its end, clicks its code-copy buttons, and types a search query. It reads each `securitypolicyviolation` event from the page and from each frame in it. It reports them by directive and blocked origin, with the first page each occurred on. The event fires for an enforcing policy and a `Content-Security-Policy-Report-Only` one alike, so a trial can run Report-Only first: +**A change to a `Content-Security-Policy` in `security-headers.txt` is also tried in a browser**, on the local staging mirror, before its pull request opens. So is a theme or `layouts/` change that adds a style, a font, a fetch, or a script. [`checks/csp_browser_gate.py`](./checks/csp_browser_gate.py) loads every page of `golden-urls.txt` plus `/search/` in headless Chromium. It scrolls each page to its end, clicks its code-copy buttons, and types a search query. It reads each `securitypolicyviolation` event from the page and from each frame in it. It reports them by directive and blocked origin, with the first page each occurred on. The event fires for an enforcing policy and a `Content-Security-Policy-Report-Only` one alike, so a trial can run Report-Only first: ```sh set -e set -a; . ~/.secrets/blog.local.staging.env; set +a -checks/check-csp-browser.py "$SITE_BASE_URL" +checks/csp_browser_gate.py "$SITE_BASE_URL" ``` - **It needs Docker and `uv` on the host, and no browser.** Chromium runs in the Playwright image [`checks/browser/Dockerfile`](./checks/browser/Dockerfile) pins, where Dependabot moves it. `uv` installs the Playwright package inside it at the image tag's own version, so the two cannot drift. Its cache is `UV_CACHE_DIR` where that is set, or a directory under the system temporary root. @@ -214,7 +214,7 @@ ssh "$VPS_SSH_HOST" true && echo reachable **The off-host copy, maintained outside this repository, writes the rotated access logs to `LOG_ARCHIVE_ROOT`, and this section covers nothing more about it.** Its installation, how the VPS itself is provisioned, and its trust model are the backup host's own configuration to document, not this repository's. What "Logs and Debugging" needs from its schedule and copy behavior, to read the logs correctly, is covered there instead. Read the unit and its last run on the backup host rather than trusting a schedule written down anywhere, including here. -**`LOG_ARCHIVE_ROOT` is spelled the same way on both sides, so there is nothing to reconcile.** The pull writes it and the log review reads it, under the one name. Every value this repository reads or writes is described once, in [`ENVIRONMENT.md`](./ENVIRONMENT.md), and [`checks/check-env-docs.py`](./checks/check-env-docs.py) fails if one is declared without a description or described without existing. +**`LOG_ARCHIVE_ROOT` is spelled the same way on both sides, so there is nothing to reconcile.** The pull writes it and the log review reads it, under the one name. Every value this repository reads or writes is described once, in [`ENVIRONMENT.md`](./ENVIRONMENT.md), and [`checks/env_docs_gate.py`](./checks/env_docs_gate.py) fails if one is declared without a description or described without existing. ```sh set -a; . ~/.secrets/blog.local.production.env; set +a @@ -604,7 +604,7 @@ Ordering is load-bearing, so every redirect lives in a single `route` block. Out | Term archives the generator does not build | `terms.map` | | A `slugs.map` or `terms.map` key without its trailing slash, on a path with no dot | A second lookup of both maps, keyed on the path with the slash added | -The maps are generated by [`capture/build-redirects.py`](./capture/build-redirects.py) and the generated files are committed, so a deploy never regenerates them. How it selects its input, and why that selection is the part to get right, is in [`capture/README.md`](./capture/README.md). +The maps are generated by [`capture/build_redirects.py`](./capture/build_redirects.py) and the generated files are committed, so a deploy never regenerates them. How it selects its input, and why that selection is the part to get right, is in [`capture/README.md`](./capture/README.md). ### The Family Site diff --git a/README.md b/README.md index 2ddb3066..ef304eb4 100644 --- a/README.md +++ b/README.md @@ -138,9 +138,9 @@ flowchart LR A post is public the moment it deploys, and a photo or a pasted configuration can carry more than its author meant to publish. Three gates keep personal data out. Each runs in this repository's validate action on every pull request, under the required status check, so a finding blocks the merge. -- **Image metadata.** Before an image is committed, its author runs [`scripts/normalize-media.py`][normalize-media], which strips it to an allowlist of what it needs to render. That is its dimensions, a pinned color profile, and on a JPEG its orientation. A JPEG keeps its capture time only when nothing else in its Exif fails the gate. Location, device, and authorship are not on the list, so they go without anyone having to name them. The removal is lossless, and a file it cannot clean unambiguously is refused rather than guessed at. [`checks/check-media-metadata.py`][check-media-metadata] is the gate, and it fails the build on anything left over. It reads inside a `.zip` and the family site's PNG and icon files too. -- **Redactions.** A house number, a face, or a serial number is covered with a flat opaque fill, never a blur. A fill on an image already published is declared in [`checks/media-redactions.json`][media-redactions] and applied by a script. [`checks/check-media-redactions.py`][check-media-redactions] fails when a redacted file no longer matches its recorded hash, which is how a quietly restored original is caught. -- **Text.** [`checks/check-text-pii.py`][check-text-pii] reads every page and post under `content/` outside the imported archive years, front matter and code blocks included. It reports email addresses, hardware addresses, public IP addresses, coordinates, street addresses, and phone numbers. It also reports a URL that carries coordinates or names one specific sensor or station. A finding names the file, the line, and the class, never the value, so the CI log does not republish it. A value a post prints on purpose is declared in an allow file with a reason, and an entry that stops matching fails the gate. +- **Image metadata.** Before an image is committed, its author runs [`scripts/normalize_media.py`][normalize-media], which strips it to an allowlist of what it needs to render. That is its dimensions, a pinned color profile, and on a JPEG its orientation. A JPEG keeps its capture time only when nothing else in its Exif fails the gate. Location, device, and authorship are not on the list, so they go without anyone having to name them. The removal is lossless, and a file it cannot clean unambiguously is refused rather than guessed at. [`checks/media_metadata_gate.py`][media-metadata-gate] is the gate, and it fails the build on anything left over. It reads inside a `.zip` and the family site's PNG and icon files too. +- **Redactions.** A house number, a face, or a serial number is covered with a flat opaque fill, never a blur. A fill on an image already published is declared in [`checks/media-redactions.json`][media-redactions] and applied by a script. [`checks/media_redactions_gate.py`][media-redactions-gate] fails when a redacted file no longer matches its recorded hash, which is how a quietly restored original is caught. +- **Text.** [`checks/text_pii_gate.py`][text-pii-gate] reads every page and post under `content/` outside the imported archive years, front matter and code blocks included. It reports email addresses, hardware addresses, public IP addresses, coordinates, street addresses, and phone numbers. It also reports a URL that carries coordinates or names one specific sensor or station. A finding names the file, the line, and the class, never the value, so the CI log does not republish it. A value a post prints on purpose is declared in an allow file with a reason, and an entry that stops matching fails the gate. A fuzzer feeds the metadata parsers malformed images in the same action, because the carried tree is all well formed and would never exercise a refusal. What no gate can judge, whether a picture locates the home or identifies a person, is the checklist in [CONTENT.md][content]. @@ -193,8 +193,8 @@ Build the site and verify it against the URL contract and both sites' Content-Se ```sh hugo --gc --minify --panicOnWarning -checks/check-url-parity.py public -checks/check-csp.py public +checks/url_parity_gate.py public +checks/csp_gate.py public ``` Build a release and verify it against a running server: @@ -243,9 +243,9 @@ Licensed under the [MIT License][license]\ [capture]: ./capture/ [capture-readme]: ./capture/README.md -[check-media-metadata]: ./checks/check-media-metadata.py -[check-media-redactions]: ./checks/check-media-redactions.py -[check-text-pii]: ./checks/check-text-pii.py +[media-metadata-gate]: ./checks/media_metadata_gate.py +[media-redactions-gate]: ./checks/media_redactions_gate.py +[text-pii-gate]: ./checks/text_pii_gate.py [checks]: ./checks/ [content]: ./CONTENT.md [deploy]: ./deploy/ @@ -260,7 +260,7 @@ Licensed under the [MIT License][license]\ [license]: ./LICENSE [media-redactions]: ./checks/media-redactions.json [migration-post]: ./content/posts/2026/08/01/moving-this-blog-from-wordpress-to-hugo.md -[normalize-media]: ./scripts/normalize-media.py +[normalize-media]: ./scripts/normalize_media.py [operations]: ./OPERATIONS.md [operations-family]: ./OPERATIONS.md#the-family-site [workflow]: ./WORKFLOW.md diff --git a/capture/README.md b/capture/README.md index 6edf0dbe..ff7edb34 100644 --- a/capture/README.md +++ b/capture/README.md @@ -43,33 +43,33 @@ Two downloads from the WordPress account, both from Tools then Export. In order. Each Python step is a dry run by default and takes `--apply` to write. -Everything here is standard library except `clean-content.py`, which needs PyYAML to read front matter. It names the package if it is missing rather than raising an import error. +Everything here is standard library except `clean_content.py`, which needs PyYAML to read front matter. It names the package if it is missing rather than raising an import error. ```sh set -a; . ~/.secrets/blog.local.production.env; set +a capture/run-wp2hugo.sh # convert, into $CAPTURE_ROOT/converted/ -capture/clean-content.py --apply # drop comments, reduce the front matter -capture/restructure-content.py --apply # reshape the tree to match the URLs -capture/localize-external.py --apply # pull the hotlinked images local -capture/build-redirects.py # regenerate deploy/maps/ +capture/clean_content.py --apply # drop comments, reduce the front matter +capture/restructure_content.py --apply # reshape the tree to match the URLs +capture/localize_external.py --apply # pull the hotlinked images local +capture/build_redirects.py # regenerate deploy/maps/ ``` -`build-redirects.py` is the only one that writes into this repository, and what it writes is committed. The other four write into the capture. +`build_redirects.py` is the only one that writes into this repository, and what it writes is committed. The other four write into the capture. -**The export is selected by content, never by filename.** An account holds several exports and a media-only one carries the attachments and no posts, so a glob would choose by filesystem order and converting the wrong one yields a site that builds and is empty. `build-redirects.py` makes the choice, and `run-wp2hugo.sh` asks it with `--print-export` rather than repeating the logic, so the conversion and the maps are provably built from the same file. +**The export is selected by content, never by filename.** An account holds several exports and a media-only one carries the attachments and no posts, so a glob would choose by filesystem order and converting the wrong one yields a site that builds and is empty. `build_redirects.py` makes the choice, and `run-wp2hugo.sh` asks it with `--print-export` rather than repeating the logic, so the conversion and the maps are provably built from the same file. ## What the conversion does not carry -**Hotlinked images are in no export.** Images embedded from other hosts during the Windows Live Writer and Blogger era were never in the media library, so they are absent from the WXR, and the converter skips them as non-relative links. Left alone they stay a permanent dependency on someone else serving fifteen-year-old URLs. `localize-external.py` downloads them, names each by a hash of its source URL, because the originals carry characters that do not survive a filesystem, then rewrites every reference and records the mapping in the capture's inventory. +**Hotlinked images are in no export.** Images embedded from other hosts during the Windows Live Writer and Blogger era were never in the media library, so they are absent from the WXR, and the converter skips them as non-relative links. Left alone they stay a permanent dependency on someone else serving fifteen-year-old URLs. `localize_external.py` downloads them, names each by a hash of its source URL, because the originals carry characters that do not survive a filesystem, then rewrites every reference and records the mapping in the capture's inventory. **Comments are dropped rather than scrubbed.** The blog is read-only going forward, so the data file is deleted outright. There is no partial scrub to get wrong and nothing to carry. -**Front matter is reduced to what drives the site.** The converter carries every WordPress custom field through, most of them platform internals that no template reads. `clean-content.py` keeps a short list and drops the rest. The list is in the script rather than restated here, so a reader can diff it against a new export. +**Front matter is reduced to what drives the site.** The converter carries every WordPress custom field through, most of them platform internals that no template reads. `clean_content.py` keeps a short list and drops the rest. The list is in the script rather than restated here, so a reader can diff it against a new export. ## Three results that look like success and are not -- **A Picasa or ggpht URL whose size segment ends in `-h` returns HTTP 200 with an HTML page**, not an image, and a naive fetch writes a few hundred bytes of markup with a successful status. `localize-external.py` parses the wrapper and follows the `` it names, which is the platform's own answer rather than a guess. +- **A Picasa or ggpht URL whose size segment ends in `-h` returns HTTP 200 with an HTML page**, not an image, and a naive fetch writes a few hundred bytes of markup with a successful status. `localize_external.py` parses the wrapper and follows the `` it names, which is the platform's own answer rather than a guess. - **Any non-image body is a failure regardless of status.** The magic bytes are checked, and anything that is not an image is reported rather than written. - **Selecting the export by filename yields empty maps that pass every check** until the redirects are live. See the note above on selection by content. @@ -79,8 +79,8 @@ The contract in [`checks/`](../checks/) was not derived from the content tree. I | Script | Did | | --- | --- | -| `enumerate-media.py` | pulled every post and page from the platform's REST API in **rendered** form, so shortcodes were expanded and a media reference was seen as a browser sees it, and split the results by host to separate library media from hotlinks | -| `build-golden.py` | unioned the crawl, the sitemap and a derivation of the archive and pagination shapes, then requested every candidate against the live site and recorded what it answered | +| `enumerate_media.py` | pulled every post and page from the platform's REST API in **rendered** form, so shortcodes were expanded and a media reference was seen as a browser sees it, and split the results by host to separate library media from hotlinks | +| `build_golden.py` | unioned the crawl, the sitemap and a derivation of the archive and pagination shapes, then requested every candidate against the live site and recorded what it answered | | `classify.py` | split the verified set into what must render and what must redirect, using the sitemap as the discriminator for bare one-segment URLs, which are ambiguous between a real page and an attachment page | **These three cannot run again once the old hosting ends.** They are carried as the record behind an append-only contract, not as a step anyone re-runs. An append to `golden-urls.txt` is only reviewable against the derivation that produced the original, which is why the derivation is here rather than lost with the platform. @@ -89,7 +89,7 @@ The contract in [`checks/`](../checks/) was not derived from the content tree. I ## Two properties of the maps that are easy to break -**The Blogger map holds more entries than there are Blogger-era posts.** That platform truncated an auto-generated slug at a fixed length on a whole-word boundary. A long title was therefore served at the truncated address, which is the one search indexes hold. The importer registered the full slug, both answer, and both are mapped. The limit is a named constant in `build-redirects.py`, and changing it changes how many entries the map has. A few addresses Blogger served, or the contract lists, match neither form. `BLOGGER_ALIASES`, beside the limit, maps each through the Blogger address it stands for. Each post's Blogger comment feed is mapped as well, keyed on the numeric post id the export carries. +**The Blogger map holds more entries than there are Blogger-era posts.** That platform truncated an auto-generated slug at a fixed length on a whole-word boundary. A long title was therefore served at the truncated address, which is the one search indexes hold. The importer registered the full slug, both answer, and both are mapped. The limit is a named constant in `build_redirects.py`, and changing it changes how many entries the map has. A few addresses Blogger served, or the contract lists, match neither form. `BLOGGER_ALIASES`, beside the limit, maps each through the Blogger address it stands for. Each post's Blogger comment feed is mapped as well, keyed on the numeric post id the export carries. **A term archive the old platform served is not always one Hugo builds.** Where it does not, the URL still has to answer, which is why the maps carry terms that no longer exist as pages. @@ -104,7 +104,7 @@ Seven scripts stayed behind, and each is named here so nobody goes looking for s | `inventory/fetch-at-risk.sh` | downloaded every hotlinked image with its size and SHA-256 | The manifest it wrote is the durable artifact. Re-verification reads the manifest and never re-fetches. | | `inventory/fetch-missing.sh` | fetched library media the crawl did not reach | Same shape, same reason. | | `inventory/fetch-unattached.sh` | fetched media listed in the export but never embedded in a post | Same shape, same reason. | -| `build-redirects.py` at the capture root | an older copy of the generator | **Stale.** Superseded by the copy here. | +| `build_redirects.py` at the capture root | an older copy of the generator | **Stale.** Superseded by the copy here. | | a second copy under the capture's own `checks/` | byte-identical to the one above | **Stale.** Superseded by the copy here. That directory belongs to the capture and is not this repository's `checks/`. | **The two stale copies are named by path on purpose**, so that finding one is not mistaken for finding a source. Neither has the fixes this copy carries. diff --git a/capture/build-golden.py b/capture/build_golden.py similarity index 100% rename from capture/build-golden.py rename to capture/build_golden.py diff --git a/capture/build-redirects.py b/capture/build_redirects.py similarity index 100% rename from capture/build-redirects.py rename to capture/build_redirects.py diff --git a/capture/clean-content.py b/capture/clean_content.py similarity index 98% rename from capture/clean-content.py rename to capture/clean_content.py index 29337a0d..54eea800 100755 --- a/capture/clean-content.py +++ b/capture/clean_content.py @@ -62,7 +62,7 @@ def capture_root() -> pathlib.Path: def converted_site(argv) -> pathlib.Path: """The converted site: a first argument, else the one generated-* under the capture. - Ambiguity aborts rather than picking, the same rule build-redirects.py applies to the + Ambiguity aborts rather than picking, the same rule build_redirects.py applies to the export. wp2hugo stamps the directory with a run timestamp, so there is no fixed name to default to and a glob that matched two would otherwise choose by filesystem order. """ diff --git a/capture/enumerate-media.py b/capture/enumerate_media.py similarity index 100% rename from capture/enumerate-media.py rename to capture/enumerate_media.py diff --git a/capture/localize-external.py b/capture/localize_external.py similarity index 99% rename from capture/localize-external.py rename to capture/localize_external.py index 25bf32e4..fc31d74f 100755 --- a/capture/localize-external.py +++ b/capture/localize_external.py @@ -61,7 +61,7 @@ def capture_root() -> pathlib.Path: def converted_site(argv) -> pathlib.Path: """The converted site: a first argument, else the one generated-* under the capture. - Ambiguity aborts rather than picking, the same rule build-redirects.py applies to the + Ambiguity aborts rather than picking, the same rule build_redirects.py applies to the export. wp2hugo stamps the directory with a run timestamp, so there is no fixed name to default to and a glob that matched two would otherwise choose by filesystem order. """ diff --git a/capture/restructure-content.py b/capture/restructure_content.py similarity index 94% rename from capture/restructure-content.py rename to capture/restructure_content.py index 51d6d725..e5fe39a7 100755 --- a/capture/restructure-content.py +++ b/capture/restructure_content.py @@ -46,7 +46,7 @@ def capture_root() -> pathlib.Path: def converted_site(argv) -> pathlib.Path: """The converted site: a first argument, else the one generated-* under the capture. - Ambiguity aborts rather than picking, the same rule build-redirects.py applies to the + Ambiguity aborts rather than picking, the same rule build_redirects.py applies to the export. wp2hugo stamps the directory with a run timestamp, so there is no fixed name to default to and a glob that matched two would otherwise choose by filesystem order. """ @@ -139,10 +139,9 @@ def main(site: pathlib.Path, apply: bool): else: print("\nDRY RUN - pass --apply") - # A problem is a file this cannot place: no url: front matter, or a url that is not the - # dated shape. The valid moves still happen, because leaving them undone helps nobody, - # but the run is incomplete and must not report success. Contrast clean-content.py, - # whose recovered titles are a repair rather than a skip and correctly exit zero. + # A problem is a file this cannot place: no url: front matter, or a url that is not the dated shape. + # The valid moves still happen, because leaving them undone helps nobody, but the run is incomplete and must not report success. + # Contrast clean_content.py, whose recovered titles are a repair rather than a skip and correctly exit zero. if problems: print(f"{len(problems)} file(s) could not be placed, listed above") return 1 diff --git a/capture/run-wp2hugo.sh b/capture/run-wp2hugo.sh index 451897b8..8c6109df 100755 --- a/capture/run-wp2hugo.sh +++ b/capture/run-wp2hugo.sh @@ -3,7 +3,7 @@ # Media is downloaded here because that is what drives absolute->relative URL rewriting, but the bytes are replaced afterwards from the official media tar. # WordPress.com serves optimized derivatives over HTTP for some images, one of them at a fraction of the original's dimensions. # -# The export is chosen by build-redirects.py --print-export rather than by a glob here, so the conversion and the redirect maps are provably built from the same file. +# The export is chosen by build_redirects.py --print-export rather than by a glob here, so the conversion and the redirect maps are provably built from the same file. # An account holds several exports and a media-only one carries the attachments and no posts, and converting that one yields a site that builds and is empty. # The selection cannot be reimplemented in shell: the test is that ONE item carries both post_type=post and status=publish, where two greps over a whole file would accept the media-only export. set -Eeuo pipefail @@ -22,7 +22,7 @@ CAPTURE_ROOT="$(cd "$CAPTURE_ROOT" && pwd)" export CAPTURE_ROOT here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -export_xml="$("$here/build-redirects.py" --print-export)" +export_xml="$("$here/build_redirects.py" --print-export)" echo "==> export: $export_xml" cd "$CAPTURE_ROOT" diff --git a/checks/README.md b/checks/README.md index 1546482b..8b5b3f00 100644 --- a/checks/README.md +++ b/checks/README.md @@ -6,14 +6,14 @@ This README covers the URL contract alone. The other checks in this directory ea | Check | Fails when | | --- | --- | -| [`check-csp-browser.py`][check-csp-browser] | a page on a running mirror makes the browser block something that `csp-browser-allow.txt` does not list | -| [`check-csp.py`][check-csp] | a page or stylesheet on either site fetches something its site's policy does not allow, or a policy names another host | -| [`check-env-docs.py`][check-env-docs] | a configuration value is declared without a description in `ENVIRONMENT.md`, or described there and declared nowhere | -| [`check-github-env.py`][check-github-env] | the variables and secrets GitHub holds differ from the ones `ENVIRONMENT.md` lists | -| [`check-media-metadata.py`][check-media-metadata] | carried media is not in a form the gate can vouch for | -| [`check-media-redactions.py`][check-media-redactions] | a file declared in `media-redactions.json` is not its redacted result | -| [`check-text-pii.py`][check-text-pii] | a Markdown file under `content/`, outside the imported archive years, carries personal data in its text | -| [`fuzz-media-parsers.py`][fuzz-media-parsers] | malformed input breaks the media gate or its normalizer | +| [`csp_browser_gate.py`][csp-browser-gate] | a page on a running mirror makes the browser block something that `csp-browser-allow.txt` does not list | +| [`csp_gate.py`][csp-gate] | a page or stylesheet on either site fetches something its site's policy does not allow, or a policy names another host | +| [`env_docs_gate.py`][env-docs-gate] | a configuration value is declared without a description in `ENVIRONMENT.md`, or described there and declared nowhere | +| [`github_env_gate.py`][github-env-gate] | the variables and secrets GitHub holds differ from the ones `ENVIRONMENT.md` lists | +| [`media_metadata_gate.py`][media-metadata-gate] | carried media is not in a form the gate can vouch for | +| [`media_redactions_gate.py`][media-redactions-gate] | a file declared in `media-redactions.json` is not its redacted result | +| [`text_pii_gate.py`][text-pii-gate] | a Markdown file under `content/`, outside the imported archive years, carries personal data in its text | +| [`media_parser_fuzz_gate.py`][media-parser-fuzz-gate] | malformed input breaks the media gate or its normalizer | ## Why this exists @@ -23,7 +23,7 @@ The contract is enforced by two gates, because one cannot cover both halves: | Gate | Proves | Runs | | --- | --- | --- | -| [`check-url-parity.py`][check-url-parity] | Every URL that must render exists as a built page, every legacy image URL resolves, every local asset reference points at a real file, and the count of carried media linked from no page still equals its recorded baseline | Against `public/`, in CI and before any release is installed | +| [`url_parity_gate.py`][url-parity-gate] | Every URL that must render exists as a built page, every legacy image URL resolves, every local asset reference points at a real file, and the count of carried media linked from no page still equals its recorded baseline | Against `public/`, in CI and before any release is installed | | [`check-live-urls.sh`][check-live-urls] | Every redirect resolves, and its destination answers | Against a running server, which is the only thing that exercises a redirect | **Production's deploy checks a sample of the contract rather than all of it.** The VPS edge bans a client that requests more than 40 distinct pages in a burst, and the full contract is over 1,200. The host side exempts the site's own names from that scenario, which is what lets staging's full run pass. Production stays under the limit without the exemption. Its deploy hook sets `SAMPLE_CONTRACT=1`, and `check-live-urls.sh` then requests the first URL of each class in file order. That covers every redirect class in [`deploy/README.md`][deploy-readme-redirects], every shape of page that must render, each media tree, and the family site. It also requests `/all/`, `/feed.xml`, and `/sitemap.xml`, the fixed redirect destinations, since no list names them and a sampled redirect is not followed to one. A sampled redirect is checked for its status and its `Location` and is not followed. The script asserts the sample's budget before its first request, and `checks/tests/test_check_live_urls_sample.py` holds the classes to the table's sizes and the sample to that budget. `checks/check-live-urls.sh --print-sample` prints the sample and its budget without requesting anything. Staging, the local mirrors, and any run without the variable check every URL, which is where the contract itself is proven. @@ -85,7 +85,7 @@ The other property worth knowing belongs to the generator rather than to the con **Adding a URL.** Real traffic finds what the lists missed. When a server log shows a 404 for an address that should work, append it to the appropriate list and add a redirect rule or map entry to cover it. The lists are append-only, per Directionality below. -**Regenerating the maps.** The generator is [`capture/build-redirects.py`][capture-build-redirects], and it lives there rather than here because it generates rather than gates. [`capture/README.md`][capture-readme] covers how to run it and how it chooses its input. +**Regenerating the maps.** The generator is [`capture/build_redirects.py`][capture-build-redirects], and it lives there rather than here because it generates rather than gates. [`capture/README.md`][capture-readme] covers how to run it and how it chooses its input. **Checking a count.** Every count above is derivable from the files, so check rather than trust: @@ -186,18 +186,18 @@ Three traps in that adjudication, each of which produced a wrong answer first an -[capture-build-redirects]: ../capture/build-redirects.py +[capture-build-redirects]: ../capture/build_redirects.py [caddy-dockerfile]: ./caddy/Dockerfile [capture-readme]: ../capture/README.md -[check-csp-browser]: ./check-csp-browser.py -[check-csp]: ./check-csp.py -[check-env-docs]: ./check-env-docs.py -[check-github-env]: ./check-github-env.py +[csp-browser-gate]: ./csp_browser_gate.py +[csp-gate]: ./csp_gate.py +[env-docs-gate]: ./env_docs_gate.py +[github-env-gate]: ./github_env_gate.py [check-live-urls]: ./check-live-urls.sh -[check-media-metadata]: ./check-media-metadata.py -[check-media-redactions]: ./check-media-redactions.py -[check-text-pii]: ./check-text-pii.py -[check-url-parity]: ./check-url-parity.py +[media-metadata-gate]: ./media_metadata_gate.py +[media-redactions-gate]: ./media_redactions_gate.py +[text-pii-gate]: ./text_pii_gate.py +[url-parity-gate]: ./url_parity_gate.py [deploy-readme-redirects]: ../deploy/README.md#how-the-redirects-are-expressed [deploy-readme-security-headers]: ../deploy/README.md#security-headers -[fuzz-media-parsers]: ./fuzz-media-parsers.py +[media-parser-fuzz-gate]: ./media_parser_fuzz_gate.py diff --git a/checks/check-csp-browser.py b/checks/csp_browser_gate.py similarity index 98% rename from checks/check-csp-browser.py rename to checks/csp_browser_gate.py index 2117af22..2715bcb1 100755 --- a/checks/check-csp-browser.py +++ b/checks/csp_browser_gate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Load a running site in headless Chromium and report what its Content-Security-Policy blocks. -`check-csp.py` proves the built scripts match the policy's hashes and that every fetch the pages +`csp_gate.py` proves the built scripts match the policy's hashes and that every fetch the pages and stylesheets name is one the policy allows. It cannot see what a browser blocks at run time: a fetch a script makes, or a script a proxy injects. This check loads each page in a URL list against a mirror. It records every `securitypolicyviolation` @@ -157,7 +157,7 @@ def docker_command(image, version, base_url, cache): "--with", f"playwright=={version}", "python", - "/checks/check-csp-browser.py", + "/checks/csp_browser_gate.py", "--probe", base_url, ] diff --git a/checks/check-csp.py b/checks/csp_gate.py similarity index 97% rename from checks/check-csp.py rename to checks/csp_gate.py index 99d04c10..172882b8 100755 --- a/checks/check-csp.py +++ b/checks/csp_gate.py @@ -34,13 +34,14 @@ import base64 import collections import hashlib -import importlib.util import pathlib import re import sys from html.parser import HTMLParser from urllib.parse import urlsplit +import url_parity_gate + CHECKS = pathlib.Path(__file__).resolve().parent HEADERS = CHECKS / "security-headers.txt" CADDYFILE = CHECKS.parent / "deploy" / "Caddyfile" @@ -66,15 +67,6 @@ } -def load_sibling(name, path): - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise ImportError(f"cannot load {path.name}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - def site_policy(headers_text, site): """A site's policy as security-headers.txt holds it, or None where the site sends none.""" for line in headers_text.splitlines(): @@ -372,9 +364,7 @@ def main(argv=None): args = parser.parse_args(argv) if not args.public.is_dir(): sys.exit(f"FAIL: {args.public} is not a directory - run hugo first") - site_origin = load_sibling( - "check_url_parity", CHECKS / "check-url-parity.py" - ).site_origin + site_origin = url_parity_gate.site_origin findings = [] for site, root in (("blog", args.public), ("family", FAMILY)): diff --git a/checks/check-env-docs.py b/checks/env_docs_gate.py similarity index 100% rename from checks/check-env-docs.py rename to checks/env_docs_gate.py diff --git a/checks/check-github-env.py b/checks/github_env_gate.py similarity index 100% rename from checks/check-github-env.py rename to checks/github_env_gate.py diff --git a/checks/check-media-metadata.py b/checks/media_metadata_gate.py similarity index 99% rename from checks/check-media-metadata.py rename to checks/media_metadata_gate.py index 2d99657c..a5f33ade 100755 --- a/checks/check-media-metadata.py +++ b/checks/media_metadata_gate.py @@ -21,7 +21,7 @@ so they fail. A file this reports is not accused of carrying anything. It is a file whose form cannot -be vouched for, and `scripts/normalize-media.py` is what resolves that, by decoding to +be vouched for, and `scripts/normalize_media.py` is what resolves that, by decoding to pixels and writing a fresh file that is clean by construction. """ @@ -1453,7 +1453,7 @@ def main() -> int: print(f"{name}: {', '.join(sorted(tags))}") print( f"\n{len(found)} file(s) hold something this cannot vouch for." - " Run scripts/normalize-media.py on each, see CONTENT.md." + " Run scripts/normalize_media.py on each, see CONTENT.md." " An icon is rendered again instead." ) return 1 diff --git a/checks/fuzz-media-parsers.py b/checks/media_parser_fuzz_gate.py similarity index 99% rename from checks/fuzz-media-parsers.py rename to checks/media_parser_fuzz_gate.py index a3e1009c..7bceba91 100755 --- a/checks/fuzz-media-parsers.py +++ b/checks/media_parser_fuzz_gate.py @@ -28,7 +28,6 @@ import argparse import contextlib import functools -import importlib.util import io import itertools import pathlib @@ -44,20 +43,11 @@ from collections.abc import Callable, Iterable, Iterator from typing import Any -REPO = pathlib.Path(__file__).resolve().parent.parent - - -def load(name: str, path: pathlib.Path): - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise ImportError(f"cannot load {path.name}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module +import media_metadata_gate as gate - -gate = load("gate", REPO / "checks" / "check-media-metadata.py") -normalizer = load("normalizer", REPO / "scripts" / "normalize-media.py") +REPO = pathlib.Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "scripts")) +import normalize_media as normalizer # A comment segment, chunk, block or atom holding nothing, so no allowlist can admit it. PLANT_TEXT = b"planted" diff --git a/checks/check-media-redactions.py b/checks/media_redactions_gate.py similarity index 96% rename from checks/check-media-redactions.py rename to checks/media_redactions_gate.py index 24abdd8b..0ecbbbce 100755 --- a/checks/check-media-redactions.py +++ b/checks/media_redactions_gate.py @@ -4,7 +4,7 @@ The manifest records the hash each redaction produces, so this needs no image decoder. A file replaced by its original, re-exported, or edited by hand fails here. So does an entry whose declared digest is missing or no longer matches its fills and crop. -`scripts/redact-media.py` is what brings either back. +`scripts/redact_media.py` is what brings either back. """ import hashlib @@ -39,7 +39,7 @@ def main() -> int: print(line) if failures: print( - f"\n{len(failures)} file(s) fail. Run scripts/redact-media.py" + f"\n{len(failures)} file(s) fail. Run scripts/redact_media.py" " under scripts/requirements.txt, see CONTENT.md." ) return 1 diff --git a/checks/pyproject.toml b/checks/pyproject.toml index 9e4c72ca..3c61e0f7 100644 --- a/checks/pyproject.toml +++ b/checks/pyproject.toml @@ -23,3 +23,8 @@ omit = ["tests/*"] [[tool.mypy.overrides]] module = ["playwright.*"] ignore_missing_imports = true + +# The normalizer sits in scripts/, a directory of its own that this one's type check does not read. +[[tool.mypy.overrides]] +module = ["normalize_media"] +ignore_missing_imports = true diff --git a/checks/tests/test_check_csp_browser.py b/checks/tests/test_csp_browser_gate.py similarity index 96% rename from checks/tests/test_check_csp_browser.py rename to checks/tests/test_csp_browser_gate.py index 64586c6f..6a914fbf 100644 --- a/checks/tests/test_check_csp_browser.py +++ b/checks/tests/test_csp_browser_gate.py @@ -1,7 +1,6 @@ -"""Tests for check-csp-browser.py's host half, every record in them constructed rather than observed.""" +"""Tests for csp_browser_gate.py's host half, every record in them constructed rather than observed.""" import contextlib -import importlib.util import io import os import pathlib @@ -11,13 +10,8 @@ from unittest import mock CHECKS = pathlib.Path(__file__).resolve().parent.parent -SPEC = importlib.util.spec_from_file_location( - "check_csp_browser", CHECKS / "check-csp-browser.py" -) -if SPEC is None or SPEC.loader is None: - raise ImportError("cannot load check-csp-browser.py") -gate = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(gate) +sys.path.insert(0, str(CHECKS)) +import csp_browser_gate as gate BASE = "https://mirror.example/" ALLOWED = ("img-src", "https://images.example") diff --git a/checks/tests/test_check_csp.py b/checks/tests/test_csp_gate.py similarity index 97% rename from checks/tests/test_check_csp.py rename to checks/tests/test_csp_gate.py index 20ff04e8..7a772992 100644 --- a/checks/tests/test_check_csp.py +++ b/checks/tests/test_csp_gate.py @@ -1,18 +1,15 @@ -"""Tests for check-csp.py, every page and stylesheet in them constructed rather than built.""" +"""Tests for csp_gate.py, every page and stylesheet in them constructed rather than built.""" import contextlib -import importlib.util import io import pathlib +import sys import tempfile import unittest CHECKS = pathlib.Path(__file__).resolve().parent.parent -SPEC = importlib.util.spec_from_file_location("check_csp", CHECKS / "check-csp.py") -if SPEC is None or SPEC.loader is None: - raise ImportError("cannot load check-csp.py") -gate = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(gate) +sys.path.insert(0, str(CHECKS)) +import csp_gate as gate THEME = "document.body.dataset.ready=1" PAGE = f"" diff --git a/checks/tests/test_check_github_env.py b/checks/tests/test_github_env_gate.py similarity index 95% rename from checks/tests/test_check_github_env.py rename to checks/tests/test_github_env_gate.py index a94e2164..ad6ce5f5 100644 --- a/checks/tests/test_check_github_env.py +++ b/checks/tests/test_github_env_gate.py @@ -1,20 +1,15 @@ -"""Tests for check-github-env.py, every value in them constructed rather than observed.""" +"""Tests for github_env_gate.py, every value in them constructed rather than observed.""" import contextlib -import importlib.util import io import pathlib +import sys import unittest from unittest import mock CHECKS = pathlib.Path(__file__).resolve().parent.parent -SPEC = importlib.util.spec_from_file_location( - "check_github_env", CHECKS / "check-github-env.py" -) -if SPEC is None or SPEC.loader is None: - raise ImportError("cannot load check-github-env.py") -gate = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(gate) +sys.path.insert(0, str(CHECKS)) +import github_env_gate as gate DOC = """# Environment diff --git a/checks/tests/test_check_media_metadata.py b/checks/tests/test_media_metadata_gate.py similarity index 97% rename from checks/tests/test_check_media_metadata.py rename to checks/tests/test_media_metadata_gate.py index 0d28ef71..bf1f85c1 100644 --- a/checks/tests/test_check_media_metadata.py +++ b/checks/tests/test_media_metadata_gate.py @@ -1,13 +1,13 @@ -"""Tests for the free values check-media-metadata.py pins, every value in them constructed.""" +"""Tests for the free values media_metadata_gate.py pins, every value in them constructed.""" import contextlib -import importlib.util import io import pathlib import random import shutil import struct import subprocess +import sys import tempfile import unittest import zipfile @@ -16,18 +16,12 @@ REPO = pathlib.Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "checks")) +sys.path.insert(0, str(REPO / "scripts")) +import normalize_media as normalizer -def load(name: str, path: pathlib.Path): - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise ImportError(f"cannot load {path.name}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module +import media_parser_fuzz_gate as fuzz - -normalizer = load("normalizer", REPO / "scripts" / "normalize-media.py") -fuzz = load("fuzz", REPO / "checks" / "fuzz-media-parsers.py") gate = normalizer.gate # A stand-in profile, admitted by patching its digest into the set for the test that needs it. @@ -571,7 +565,7 @@ def test_normalizer_never_sends_an_icon_to_ffmpeg(self) -> None: data = gate.ICO_HEADER + b"free" + bytes(20) icon = self.root / "favicon.ico" icon.write_bytes(data) - argv = ["normalize-media.py", str(icon)] + argv = ["normalize_media.py", str(icon)] with ( mock.patch.object(normalizer.shutil, "which", return_value="ffmpeg"), mock.patch.object(normalizer.sys, "argv", argv), @@ -588,7 +582,7 @@ def test_normalizer_skips_a_missing_default_root(self) -> None: held.write_bytes(png_with(text_chunk())) with ( mock.patch.object(normalizer, "REPO", self.root), - mock.patch.object(normalizer.sys, "argv", ["normalize-media.py"]), + mock.patch.object(normalizer.sys, "argv", ["normalize_media.py"]), contextlib.redirect_stdout(io.StringIO()) as out, ): self.assertEqual(normalizer.main(), 0) @@ -601,7 +595,7 @@ def test_normalizer_names_a_dangling_default_root(self) -> None: self.skipTest(f"symlinks need a privilege here: {exc}") with ( mock.patch.object(normalizer, "REPO", self.root), - mock.patch.object(normalizer.sys, "argv", ["normalize-media.py"]), + mock.patch.object(normalizer.sys, "argv", ["normalize_media.py"]), contextlib.redirect_stdout(io.StringIO()) as out, ): self.assertEqual(normalizer.main(), 0) @@ -626,7 +620,7 @@ def outside(self) -> pathlib.Path: return target def normalize(self, *argv: str) -> str: - args = ["normalize-media.py", *argv] + args = ["normalize_media.py", *argv] with ( mock.patch.object(normalizer, "REPO", self.root), mock.patch.object(normalizer.sys, "argv", args), @@ -657,10 +651,7 @@ def test_tree_under_a_symlinked_directory_is_named_and_not_read(self) -> None: def seeds(self) -> list[str]: """The carried seeds the fuzzer samples from the scratch repository.""" - with ( - mock.patch.object(fuzz, "REPO", self.root), - mock.patch.object(fuzz.gate, "REPO", self.root), - ): + with mock.patch.object(fuzz, "REPO", self.root): seeds = fuzz.carried_seeds(random.Random(0), 9, 1 << 20) return [name for name, _ in seeds] diff --git a/checks/tests/test_check_text_pii.py b/checks/tests/test_text_pii_gate.py similarity index 96% rename from checks/tests/test_check_text_pii.py rename to checks/tests/test_text_pii_gate.py index f9a8ceb7..ea8c54ce 100644 --- a/checks/tests/test_check_text_pii.py +++ b/checks/tests/test_text_pii_gate.py @@ -1,21 +1,16 @@ -"""Tests for check-text-pii.py, every value in them constructed rather than observed.""" +"""Tests for text_pii_gate.py, every value in them constructed rather than observed.""" import contextlib -import importlib.util import io import json import pathlib +import sys import tempfile import unittest CHECKS = pathlib.Path(__file__).resolve().parent.parent -SPEC = importlib.util.spec_from_file_location( - "check_text_pii", CHECKS / "check-text-pii.py" -) -if SPEC is None or SPEC.loader is None: - raise ImportError("cannot load check-text-pii.py") -gate = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(gate) +sys.path.insert(0, str(CHECKS)) +import text_pii_gate as gate POSITIVE = { "email": ["Write to someone@example.com for a copy."], diff --git a/checks/check-text-pii.py b/checks/text_pii_gate.py similarity index 100% rename from checks/check-text-pii.py rename to checks/text_pii_gate.py diff --git a/checks/check-url-parity.py b/checks/url_parity_gate.py similarity index 100% rename from checks/check-url-parity.py rename to checks/url_parity_gate.py diff --git a/deploy/README.md b/deploy/README.md index f4cd760c..991a54be 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -7,15 +7,15 @@ How the site is built, released, and served. The release is a **self-contained b | Tool | Used by | Needed for | Install | | --- | --- | --- | --- | | `hugo` (extended) | `make-release.sh` | building the site | see below | -| `python3` (3.10+) | `check-url-parity.py`, `check-csp.py`, `check-csp-browser.py`, `capture/build-redirects.py` | the build-time gates, the browser trial, regenerating maps | `apt install python3` | -| PyYAML | `capture/clean-content.py` | reading front matter during a conversion | `apt install python3-yaml` | +| `python3` (3.10+) | `url_parity_gate.py`, `csp_gate.py`, `csp_browser_gate.py`, `capture/build_redirects.py` | the build-time gates, the browser trial, regenerating maps | `apt install python3` | +| PyYAML | `capture/clean_content.py` | reading front matter during a conversion | `apt install python3-yaml` | | `rsync` | `make-release.sh` | installing a release, hard-linking unchanged files | `apt install rsync` | | `gzip` | `make-release.sh` | precompression | coreutils, already present | | `brotli` | `make-release.sh` | precompression | `apt install brotli` | | `curl` 7.84+ | `check-live-urls.sh` | verifying a running server, reading a header with `%header{}` | `apt install curl` | | `bash` 4.4+ | all scripts | arrays, `mapfile` | already present | -| `docker` | serving, `check-csp-browser.py` | running Caddy, the headless browser | orchestrated elsewhere | -| `uv` | `check-csp-browser.py` | installing Playwright in the browser container | the upstream installer | +| `docker` | serving, `csp_browser_gate.py` | running Caddy, the headless browser | orchestrated elsewhere | +| `uv` | `csp_browser_gate.py` | installing Playwright in the browser container | the upstream installer | **These scripts run on Linux, which is a dependency ceiling rather than an omission.** They use `mv -Tf`, `find -printf`, `mapfile`, and `sed -i` without an argument, all of which are GNU or @@ -27,7 +27,7 @@ macOS or Windows contributor runs them through a container or a remote Linux hos **`brotli` is easy to miss and degrades silently.** The Caddyfile serves `precompressed br gzip`, so without the binary every text response falls back to gzip while the site keeps working and nothing errors. `make-release.sh` warns loudly when it is absent, and `REQUIRE_BROTLI=1` turns that into a hard failure. **CI must set it.** Measured on the home page: 18,845 bytes raw, 6,395 gzip, 5,105 brotli. -[`capture/build-redirects.py`][capture-build-redirects] additionally needs the capture directory holding the source export, which is not in this repo. It runs by hand rather than in CI, and its outputs under `deploy/maps/` are committed. +[`capture/build_redirects.py`][capture-build-redirects] additionally needs the capture directory holding the source export, which is not in this repo. It runs by hand rather than in CI, and its outputs under `deploy/maps/` are committed. ## Container contract @@ -305,9 +305,9 @@ The HSTS value carries neither `includeSubDomains` nor `preload`. `includeSubDom **Each site sends its own `Content-Security-Policy`.** The family page loads few resources, so its policy names each kind: its own files and fonts, inline styles, and `data:` images. The blog policy allows its own files and fonts, and inline styles, which the theme and some archived posts carry. Both sites serve their fonts themselves, under `/fonts/`. Neither policy names another host, so every resource either site loads is its own. An embed from another host, such as a video or a map, is therefore not possible. The browser would block it, and the release build fails on it. A scheme such as `data:` names no host, so a directive may list one. Its `script-src` lists a hash for each inline script the theme writes, rather than allowing every inline script. -**A Hugo update, a theme update, or a theme setting in `hugo.yaml` can change those scripts.** [`checks/check-csp.py`][checks-csp] runs in the release build and fails when the hashes and the built scripts differ. For a hash, the fix is to copy each one it reports into both the Caddyfile and `security-headers.txt`. Drop each one it reports as unused. It also fails when a page or stylesheet on either site fetches something its policy does not allow, or when a policy names another host. The fix there is to serve the resource from the site itself, and to drop the host from the policy. +**A Hugo update, a theme update, or a theme setting in `hugo.yaml` can change those scripts.** [`checks/csp_gate.py`][checks-csp] runs in the release build and fails when the hashes and the built scripts differ. For a hash, the fix is to copy each one it reports into both the Caddyfile and `security-headers.txt`. Drop each one it reports as unused. It also fails when a page or stylesheet on either site fetches something its policy does not allow, or when a policy names another host. The fix there is to serve the resource from the site itself, and to drop the host from the policy. -**The gate reads only the files each site ships, so a policy change is also tried in a browser.** [`checks/check-csp-browser.py`][checks-csp-browser] loads every page of a running mirror in headless Chromium. It fails on anything the policy blocks that [`checks/csp-browser-allow.txt`][checks-csp-browser-allow] does not list. A run over the whole default list also fails on an entry there that nothing matched. The list is empty today, so any blocked load fails the run. [OPERATIONS.md "Local Verification"][operations-local-verification] says when it runs and how. +**The gate reads only the files each site ships, so a policy change is also tried in a browser.** [`checks/csp_browser_gate.py`][checks-csp-browser] loads every page of a running mirror in headless Chromium. It fails on anything the policy blocks that [`checks/csp-browser-allow.txt`][checks-csp-browser-allow] does not list. A run over the whole default list also fails on an entry there that nothing matched. The list is empty today, so any blocked load fails the run. [OPERATIONS.md "Local Verification"][operations-local-verification] says when it runs and how. **The policy also blocks a script the proxy injects into a page.** Cloudflare's Rocket Loader, Email Address Obfuscation, and Web Analytics each inject one, so each stays off. They are the host side's to set, and the gate cannot see them, since it reads only the files each site ships. @@ -427,7 +427,7 @@ Every class below is a legacy shape, closed by the migration, so no count here m Two orderings are load-bearing. `@post_child_feed` precedes `@post_child` because both match the same shape and the broader one would claim both. No golden URL is five segments under a date, so `@post_child` cannot swallow a page that must render, and `@uploads` rewrites under a prefix no rendered page occupies. -`blogger.map` carries more entries than there are Blogger-era posts, because that platform served a long title at a truncated address and both forms answer. A few addresses Blogger served, or the contract lists, match neither form. The mechanism is in [`capture/README.md`][capture-readme], beside the generator that implements it. `slugs.map` is generated by [`capture/build-redirects.py`][capture-build-redirects], which recovers each attachment's parent from the media inventory, since each has `post_parent = 0` in the export. 85 entries resolve to a real post, and 21 go to `/` because nothing on the site claims them. The generator gives the rest a fixed destination. +`blogger.map` carries more entries than there are Blogger-era posts, because that platform served a long title at a truncated address and both forms answer. A few addresses Blogger served, or the contract lists, match neither form. The mechanism is in [`capture/README.md`][capture-readme], beside the generator that implements it. `slugs.map` is generated by [`capture/build_redirects.py`][capture-build-redirects], which recovers each attachment's parent from the media inventory, since each has `post_parent = 0` in the export. 85 entries resolve to a real post, and 21 go to `/` because nothing on the site claims them. The generator gives the rest a fixed destination. **Two shapes in the Caddyfile look wrong and are correct.** The `map` on `{re.label.1}` sits outside `route`, before the matcher that sets that placeholder runs. A map resolves lazily, when a directive reads its output, so it sees the matched request. And `handle_errors` rewrites to `/404.html` and serves it with `file_server`, which keeps the error's `404` status rather than answering `200`. @@ -435,8 +435,8 @@ Two orderings are load-bearing. `@post_child_feed` precedes `@post_child` becaus Each of these fails silently when it fails, which is why it is checked rather than assumed: -- **The URL contract**, via `check-url-parity.py`, before anything is installed. -- **Both sites' script hashes and every fetch their pages and stylesheets make**, via `check-csp.py`, before anything is installed. +- **The URL contract**, via `url_parity_gate.py`, before anything is installed. +- **Both sites' script hashes and every fetch their pages and stylesheets make**, via `csp_gate.py`, before anything is installed. - **World-readability of the finished release.** `--chmod` and `--no-g` apply only to files rsync newly transfers. A file supplied by `--link-dest` is a hard link to the previous release's inode and keeps that inode's mode and ownership, so a badly moded file rides the @@ -451,11 +451,11 @@ Each of these fails silently when it fails, which is why it is checked rather th [caddyfile]: ./Caddyfile -[capture-build-redirects]: ../capture/build-redirects.py +[capture-build-redirects]: ../capture/build_redirects.py [capture-readme]: ../capture/README.md -[checks-csp-browser]: ../checks/check-csp-browser.py +[checks-csp-browser]: ../checks/csp_browser_gate.py [checks-csp-browser-allow]: ../checks/csp-browser-allow.txt -[checks-csp]: ../checks/check-csp.py +[checks-csp]: ../checks/csp_gate.py [checks-redirect-urls]: ../checks/redirect-urls.txt [checks-security-headers]: ../checks/security-headers.txt [example-env]: ../.secrets/example.env diff --git a/deploy/make-release.sh b/deploy/make-release.sh index c1e377f8..e6ecc93c 100755 --- a/deploy/make-release.sh +++ b/deploy/make-release.sh @@ -202,10 +202,10 @@ rm -rf public hugo --gc --minify --panicOnWarning echo "==> verifying the URL contract" -checks/check-url-parity.py public +checks/url_parity_gate.py public echo "==> verifying the Content-Security-Policy" -checks/check-csp.py public +checks/csp_gate.py public echo "==> precompressing" # Text only, because JPEG, PNG, and MP4 are already compressed. diff --git a/scripts/normalize-media.py b/scripts/normalize_media.py similarity index 98% rename from scripts/normalize-media.py rename to scripts/normalize_media.py index 1fbe8f63..b0cf69fd 100755 --- a/scripts/normalize-media.py +++ b/scripts/normalize_media.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Bring carried media into a form `checks/check-media-metadata.py` can vouch for. +"""Bring carried media into a form `checks/media_metadata_gate.py` can vouch for. Run on demand rather than in CI, because it rewrites files. @@ -25,7 +25,6 @@ import argparse import contextlib -import importlib.util import os import pathlib import shutil @@ -41,14 +40,8 @@ REPO = pathlib.Path(__file__).resolve().parent.parent -# The gate's filename carries a hyphen, so it loads by path rather than by import. -_spec = importlib.util.spec_from_file_location( - "gate", REPO / "checks" / "check-media-metadata.py" -) -if _spec is None or _spec.loader is None: - raise ImportError("cannot load check-media-metadata.py") -gate = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(gate) +sys.path.insert(0, str(REPO / "checks")) +import media_metadata_gate as gate def png_chunk(name: bytes, body: bytes) -> bytes: diff --git a/scripts/normalize-text.py b/scripts/normalize_text.py similarity index 100% rename from scripts/normalize-text.py rename to scripts/normalize_text.py diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index d74962c8..3ce07029 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -12,6 +12,11 @@ platform = "linux" files = ["."] exclude = ['^\.venv/'] +# The media gates sit in checks/, a directory of its own that this one's type check does not read. +[[tool.mypy.overrides]] +module = ["media_metadata_gate", "media_redactions_gate"] +ignore_missing_imports = true + [tool.pyright] pythonVersion = "3.12" pythonPlatform = "Linux" diff --git a/scripts/redact-media.py b/scripts/redact_media.py similarity index 91% rename from scripts/redact-media.py rename to scripts/redact_media.py index 36669bd4..3f9706fb 100755 --- a/scripts/redact-media.py +++ b/scripts/redact_media.py @@ -2,9 +2,9 @@ """Apply the redactions declared in `checks/media-redactions.json` to carried media. Run on demand with -`uv run --no-project --with-requirements scripts/requirements.txt scripts/redact-media.py`, +`uv run --no-project --with-requirements scripts/requirements.txt scripts/redact_media.py`, rather than in CI, because it rewrites files. -`checks/check-media-redactions.py` is the gate that proves the result. +`checks/media_redactions_gate.py` is the gate that proves the result. **The manifest is the whole of the decision.** Each entry names a file, the flat opaque fills and the crop it takes, and why, so a redaction is reviewed as data and @@ -20,7 +20,7 @@ **A redacted file no longer holds its source, so changing one starts from history.** Restore the file's original with `git checkout -- `, normalize it -with `scripts/normalize-media.py --apply`, edit its entry, and run with `--record`. +with `scripts/normalize_media.py --apply`, edit its entry, and run with `--record`. A restored file whose hash any committed revision of the manifest records as a result is refused, since it already carries an earlier round's fills. A restored file other than the one the entry's `source` hash records is refused too. @@ -42,12 +42,11 @@ One whose transparency is a tRNS color key is refused, since the key carries across and a fill in the key's color would come out transparent. An animated PNG is refused, since a fill would reach only its first frame. -The output then goes through the same lossless drop as `scripts/normalize-media.py`. +The output then goes through the same lossless drop as `scripts/normalize_media.py`. """ import argparse import hashlib -import importlib.util import io import json import os @@ -65,28 +64,11 @@ FILL = (0, 0, 0) PNG_COLOR_CHUNKS = (b"gAMA", b"cHRM", b"sRGB") -# The normalizer's filename carries a hyphen, so it loads by path rather than by import. -_spec = importlib.util.spec_from_file_location( - "normalize", REPO / "scripts" / "normalize-media.py" -) -if _spec is None or _spec.loader is None: - raise ImportError("cannot load normalize-media.py") -normalize = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(normalize) -_spec = importlib.util.spec_from_file_location( - "check", REPO / "checks" / "check-media-redactions.py" -) -if _spec is None or _spec.loader is None: - raise ImportError("cannot load check-media-redactions.py") -check = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(check) -_spec = importlib.util.spec_from_file_location( - "gate", REPO / "checks" / "check-media-metadata.py" -) -if _spec is None or _spec.loader is None: - raise ImportError("cannot load check-media-metadata.py") -gate = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(gate) +sys.path.insert(0, str(REPO / "checks")) +import media_metadata_gate as gate +import media_redactions_gate as check + +import normalize_media as normalize def sha256(data: bytes) -> str: @@ -249,7 +231,7 @@ def main() -> int: if f"pillow=={PIL.__version__}" not in pins: print(f"Pillow {PIL.__version__} is not the version {REQUIREMENTS.name} pins.") print( - "Run with `uv run --no-project --with-requirements scripts/requirements.txt scripts/redact-media.py`." + "Run with `uv run --no-project --with-requirements scripts/requirements.txt scripts/redact_media.py`." ) return 1 @@ -299,7 +281,7 @@ def main() -> int: continue if changed and (unclean := gate.scan(data)): errors.append( - f"{name}: not normalized ({', '.join(sorted(unclean))}), run scripts/normalize-media.py --apply" + f"{name}: not normalized ({', '.join(sorted(unclean))}), run scripts/normalize_media.py --apply" ) continue if changed and entry.get("source") not in (None, current): diff --git a/scripts/tests/test_normalize_text.py b/scripts/tests/test_normalize_text.py index 2d7dd449..f274250b 100644 --- a/scripts/tests/test_normalize_text.py +++ b/scripts/tests/test_normalize_text.py @@ -1,17 +1,12 @@ -"""Tests for `scripts/normalize-text.py`, on posts built here rather than taken from the archive.""" +"""Tests for `scripts/normalize_text.py`, on posts built here rather than taken from the archive.""" -import importlib.util import pathlib +import sys import unittest SCRIPTS = pathlib.Path(__file__).resolve().parent.parent -_spec = importlib.util.spec_from_file_location( - "normalize_text", SCRIPTS / "normalize-text.py" -) -if _spec is None or _spec.loader is None: - raise ImportError("cannot load normalize-text.py") -normalize_text = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(normalize_text) +sys.path.insert(0, str(SCRIPTS)) +import normalize_text CURLY = "\N{LEFT DOUBLE QUOTATION MARK}quoted\N{RIGHT DOUBLE QUOTATION MARK}" diff --git a/scripts/tests/test_redact_media.py b/scripts/tests/test_redact_media.py index 99d0df8d..642f51e3 100644 --- a/scripts/tests/test_redact_media.py +++ b/scripts/tests/test_redact_media.py @@ -1,16 +1,16 @@ -"""Tests for `scripts/redact-media.py`, on images built here rather than taken from the archive. +"""Tests for `scripts/redact_media.py`, on images built here rather than taken from the archive. Run with `uv run --no-project --with-requirements scripts/requirements.txt python -m pytest scripts`. """ import contextlib import hashlib -import importlib.util import io import json import os import pathlib import subprocess +import sys import tempfile import unittest from unittest import mock @@ -18,11 +18,8 @@ from PIL import Image SCRIPTS = pathlib.Path(__file__).resolve().parent.parent -_spec = importlib.util.spec_from_file_location("redact", SCRIPTS / "redact-media.py") -if _spec is None or _spec.loader is None: - raise ImportError("cannot load redact-media.py") -redact = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(redact) +sys.path.insert(0, str(SCRIPTS)) +import redact_media as redact def sha256(data: bytes) -> str: diff --git a/sites/viljoen.family/README.md b/sites/viljoen.family/README.md index 43f00d7e..82fcc942 100644 --- a/sites/viljoen.family/README.md +++ b/sites/viljoen.family/README.md @@ -50,7 +50,7 @@ The arms derive from the French family Villon de Varennes, whose crest was a gol - Each side of the mantling is one group mirrored with `translate(342 0) scale(-1 1)`. - The saltire is one filled path clipped to `#shield-clip`, which is built from `#shield-outline`. - The lettering is vector paths rather than ``, so it renders the same without the font. The page's `

` is visually hidden text, so the name stays a real heading. -- `favicon.svg` reuses the shield outline and saltire paths. `apple-touch-icon.png`, `og-image.png`, and the images inside `favicon.ico` are Chromium renders, and they carry no metadata chunks. [`check-media-metadata.py`][check-media-metadata] fails the build on one that does, reading each PNG inside `favicon.ico` as well. +- `favicon.svg` reuses the shield outline and saltire paths. `apple-touch-icon.png`, `og-image.png`, and the images inside `favicon.ico` are Chromium renders, and they carry no metadata chunks. [`media_metadata_gate.py`][media-metadata-gate] fails the build on one that does, reading each PNG inside `favicon.ico` as well. ## Page Design @@ -119,7 +119,7 @@ Serve the directory with any static server, for example `python3 -m http.server - The c generation on the tree, starting with the Henning and Johannes lines, in place of the stubs. -[check-media-metadata]: ../../checks/check-media-metadata.py +[media-metadata-gate]: ../../checks/media_metadata_gate.py [genealogy]: ../../GENEALOGY.md [operations]: ../../OPERATIONS.md diff --git a/themes/README.md b/themes/README.md index 28ec93dc..f08a02ef 100644 --- a/themes/README.md +++ b/themes/README.md @@ -73,7 +73,7 @@ The content width is unchanged from the theme default, deliberately. 720px at th ## Updating -Nothing is carried, so an update is a replace. Delete `PaperMod/`, drop the new upstream tree in its place, and update the upstream table at the top of the PaperMod section. Then confirm the site still builds under `--panicOnWarning`, which is the gate the theme has failed before. The release build's `checks/check-csp.py` also fails where an update changes the theme's inline scripts, and `deploy/README.md` "Security headers" says how to update the policy's hashes. Run the `diff -r` above afterwards, so the next reader inherits the same guarantee. +Nothing is carried, so an update is a replace. Delete `PaperMod/`, drop the new upstream tree in its place, and update the upstream table at the top of the PaperMod section. Then confirm the site still builds under `--panicOnWarning`, which is the gate the theme has failed before. The release build's `checks/csp_gate.py` also fails where an update changes the theme's inline scripts, and `deploy/README.md` "Security headers" says how to update the policy's hashes. Run the `diff -r` above afterwards, so the next reader inherits the same guarantee. Re-check the customization table before the replace, while the upstream table still names the old commit. Each theme hook it names is a comment-only file upstream today. If upstream adds content to one, the local override hides it without any build failure. The exception is `blank.css`, which the head partial bundles beside `custom.css` rather than shadowing. Added rules there still ship, ahead of `custom.css`. A rule duplicating one in `custom.css` needs no change, since the `custom.css` copy keeps the rule if upstream drops it. A conflicting upstream rule wins wherever its selector is more specific or its declaration is `!important`. Restate the site's value in `custom.css` with a selector at least as specific, since `custom.css` comes later, marked `!important` only where the upstream declaration is. Once `blank.css` carries any rule, correct each place calling it comment-only or rule-free: this paragraph, the customization table, and the `custom.css` header comment. Use the clone from the `diff -r` check above, or make a fresh one. Diff each row's theme hook between the recorded commit and the new commit. A comment-only diff needs no action. For a partial, added content means porting it into the override, or deciding it is unwanted.