From 2cf9ca18294d180682f51e9cfa21c32b8e08a480 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 18:59:51 +0000 Subject: [PATCH 1/4] Add a shared Wayback module and save pages at fixed points utils/wayback.py is now the only code that talks to the Wayback Machine: captures() lists a page's captures through CDX, pick_capture() chooses the last one on or before an edition's end plus 14 days, and save() asks the archive to save a page and reports the real result. archive-new-urls.yml moves its inline Python and curl loop into utils/archive_save.py. It now covers all five URL fields, not only link, and besides URLs new since the last successful run it saves each edition's pages at fixed points: 14 days before the CfP closes and on the day it closes (cfp_ext when set), 30 and 14 days before the start, on the start, and 14 days after it. A run saves every point since the last successful run, each URL once. The old loop reported every save as successful, because `|| true` set the exit status to 0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016a2XGgiRrtWm4EUP7ek9wt --- .github/workflows/archive-new-urls.yml | 148 ++++--------- tests/test_archive_save.py | 219 ++++++++++++++++++ tests/test_wayback.py | 191 ++++++++++++++++ utils/archive_save.py | 296 +++++++++++++++++++++++++ utils/wayback.py | 189 ++++++++++++++++ 5 files changed, 936 insertions(+), 107 deletions(-) create mode 100644 tests/test_archive_save.py create mode 100644 tests/test_wayback.py create mode 100644 utils/archive_save.py create mode 100644 utils/wayback.py diff --git a/.github/workflows/archive-new-urls.yml b/.github/workflows/archive-new-urls.yml index a3615de945..9b1d7d370a 100644 --- a/.github/workflows/archive-new-urls.yml +++ b/.github/workflows/archive-new-urls.yml @@ -1,5 +1,12 @@ name: Archive New Conference URLs +# Asks the Wayback Machine to save conference pages while they still show +# their edition, so the weekly archive resolve (archive-resolve.yml) has a +# capture to point old editions to once the site moves on. utils/archive_save.py +# decides what to save: URLs new since the last successful run, and the pages of +# editions at a save point (around the CfP deadline and the start). Both +# workflows talk to the archive through utils/wayback.py. + permissions: contents: read @@ -14,129 +21,56 @@ on: workflow_dispatch: inputs: skip_archive: - description: 'Skip the Web Archive (only report the new URLs)' + description: 'Skip the Web Archive (only report the URLs). The run still counts as the last run, so its URLs are not saved later.' type: boolean default: false +concurrency: + group: archive-new-urls + cancel-in-progress: false + jobs: - check-new-urls: + archive: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v7 with: - fetch-depth: 0 # Full history for finding last run + fetch-depth: 0 # Full history for the data at the last run - - name: Find last successful run commit + - name: Find last successful run id: last-run - run: | - last_sha=$(gh run list --workflow="Archive New Conference URLs" --json conclusion,headSha --jq '.[] | select(.conclusion=="success") | .headSha' | head -n1) - if [ -n "$last_sha" ]; then - echo "sha=$last_sha" >> $GITHUB_OUTPUT - else - echo "sha=$(git rev-list --max-parents=0 HEAD)" >> $GITHUB_OUTPUT - fi env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract new conference entries - id: extract run: | - # Get the diff for conferences.yml - DIFF=$(git diff ${{ steps.last-run.outputs.sha }} -- _data/conferences.yml || true) - - if [ -z "$DIFF" ]; then - echo "No changes to conferences.yml" - echo "has_changes=false" >> $GITHUB_OUTPUT - exit 0 - fi - - echo "has_changes=true" >> $GITHUB_OUTPUT - - # Parse new conference entries from the diff - # We need: link, conference name, place, sub - python3 << 'EOF' - import yaml - import subprocess - import json - import re - - # Get current and previous versions - result = subprocess.run( - ['git', 'show', '${{ steps.last-run.outputs.sha }}:_data/conferences.yml'], - capture_output=True, text=True - ) - old_data = yaml.safe_load(result.stdout) if result.returncode == 0 else [] - - with open('_data/conferences.yml', 'r') as f: - new_data = yaml.safe_load(f) - - # Create sets of existing URLs (normalize trailing slashes) - def normalize_url(url): - return url.rstrip('/').lower() if url else '' - - old_urls = {normalize_url(c.get('link', '')) for c in (old_data or [])} - - # Find new entries - new_entries = [] - for conf in (new_data or []): - url = normalize_url(conf.get('link', '')) - if url and url not in old_urls: - new_entries.append({ - 'url': conf.get('link', ''), - 'name': f"{conf.get('conference', 'Unknown')} {conf.get('year', '')}".strip(), - 'place': conf.get('place', ''), - 'sub': conf.get('sub', 'PY'), - }) - - # Output as JSON for later steps - print(f"Found {len(new_entries)} new conference URLs") - with open('/tmp/new_conferences.json', 'w') as f: - json.dump(new_entries, f) - - # Also output just URLs for backward compatibility - urls = [e['url'] for e in new_entries] - with open('/tmp/new_urls.txt', 'w') as f: - f.write('\n'.join(urls)) - EOF - - # Set outputs - if [ -s /tmp/new_urls.txt ]; then - echo "url_count=$(wc -l < /tmp/new_urls.txt)" >> $GITHUB_OUTPUT + RUN=$(gh run list --workflow="Archive New Conference URLs" --status success --limit 1 \ + --json headSha,createdAt --jq '.[0] // empty | "\(.headSha) \(.createdAt[:10])"') + if [ -n "$RUN" ]; then + echo "sha=${RUN% *}" >> "$GITHUB_OUTPUT" + echo "since=${RUN#* }" >> "$GITHUB_OUTPUT" else - echo "url_count=0" >> $GITHUB_OUTPUT + # First run: every URL counts as new, and the last week's save points are due + echo "sha=$(git rev-list --max-parents=0 HEAD)" >> "$GITHUB_OUTPUT" + echo "since=$(date -u -d '7 days ago' +%F)" >> "$GITHUB_OUTPUT" fi - - name: Archive to Wayback Machine - if: steps.extract.outputs.has_changes == 'true' && steps.extract.outputs.url_count != '0' && inputs.skip_archive != true - run: | - echo "## Web Archive Results" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - while IFS= read -r url; do - [ -z "$url" ] && continue - echo "Archiving: $url" - - response=$(curl -s -L -X POST \ - -A "Python-Deadlines-Bot/1.0 (https://pythondeadlin.es; admin@python-deadlin.es)" \ - "https://web.archive.org/save/$url" || true) - status=$? - - if [ $status -eq 0 ]; then - echo "✓ Successfully archived: $url" - echo "- ✅ $url" >> $GITHUB_STEP_SUMMARY - else - echo "⚠ Failed to archive: $url (error $status)" - echo "- ⚠️ $url (failed)" >> $GITHUB_STEP_SUMMARY - fi + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.11' - sleep 5 - done < /tmp/new_urls.txt + - name: Install dependencies + run: python -m pip install --quiet requests pyyaml - - name: Summary - if: always() + - name: Save to the Wayback Machine + env: + SHA: ${{ steps.last-run.outputs.sha }} + SINCE: ${{ steps.last-run.outputs.since }} + SKIP_ARCHIVE: ${{ inputs.skip_archive }} run: | - echo "" >> $GITHUB_STEP_SUMMARY - echo "---" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Total new URLs:** ${{ steps.extract.outputs.url_count }}" >> $GITHUB_STEP_SUMMARY - echo "**Commit range:** ${{ steps.last-run.outputs.sha }}..HEAD" >> $GITHUB_STEP_SUMMARY + git show "$SHA:_data/conferences.yml" > "$RUNNER_TEMP/previous.yml" 2>/dev/null || : > "$RUNNER_TEMP/previous.yml" + ARGS=(--previous "$RUNNER_TEMP/previous.yml" --since "$SINCE" --summary "$GITHUB_STEP_SUMMARY") + if [ "$SKIP_ARCHIVE" = "true" ]; then + ARGS+=(--dry-run) + fi + python utils/archive_save.py "${ARGS[@]}" diff --git a/tests/test_archive_save.py b/tests/test_archive_save.py new file mode 100644 index 0000000000..d99226441c --- /dev/null +++ b/tests/test_archive_save.py @@ -0,0 +1,219 @@ +"""Tests for utils/archive_save.py: which pages are saved, and when.""" + +import sys +from datetime import date +from pathlib import Path + +import pytest +import yaml + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +import archive_save +import wayback + + +def _entry(**fields) -> dict: + entry = { + "conference": "PyCon Tanzania", + "year": 2026, + "link": "https://pycon.tz/", + "cfp": "2026-11-01 23:59:00", + "start": date(2026, 11, 25), + "end": date(2026, 11, 27), + } + entry.update(fields) + return {k: v for k, v in entry.items() if v is not None} + + +class TestSavePoints: + """Two points around the CfP deadline and four around the start.""" + + def test_all_points(self): + assert archive_save.save_points(_entry()) == [ + date(2026, 10, 18), # 14 days before the CfP closes + date(2026, 10, 26), # 30 days before the start + date(2026, 11, 1), # the CfP closes + date(2026, 11, 11), # 14 days before the start + date(2026, 11, 25), # the start + date(2026, 12, 9), # 14 days after the start + ] + + def test_extension_moves_the_cfp_points(self): + points = archive_save.save_points(_entry(cfp_ext="2026-11-08 23:59:00")) + + assert date(2026, 11, 8) in points + assert date(2026, 10, 25) in points + assert date(2026, 11, 1) not in points + + @pytest.mark.parametrize("cfp", ["TBA", "None", "Cancelled", None]) + def test_no_exact_deadline_no_cfp_points(self, cfp): + assert archive_save.save_points(_entry(cfp=cfp)) == [ + date(2026, 10, 26), + date(2026, 11, 11), + date(2026, 11, 25), + date(2026, 12, 9), + ] + + def test_date_only_deadline(self): + assert date(2026, 11, 1) in archive_save.save_points(_entry(cfp="2026-11-01")) + + +class TestDueUrls: + """A run saves the editions with a point after the last run and up to today.""" + + def test_point_since_last_run_is_due(self): + entry = _entry(cfp_link="https://pycon.tz/call-for-proposals/") + + assert archive_save.due_urls([entry], date(2026, 10, 24), date(2026, 10, 31)) == [ + "https://pycon.tz/", + "https://pycon.tz/call-for-proposals/", + ] + + def test_point_on_the_last_run_day_was_saved_then(self): + assert archive_save.due_urls([_entry()], date(2026, 10, 26), date(2026, 10, 31)) == [] + + def test_point_today_is_due(self): + assert archive_save.due_urls([_entry()], date(2026, 10, 24), date(2026, 10, 26)) == ["https://pycon.tz/"] + + def test_missed_runs_are_caught_up(self): + """A run after a gap saves every point the gap held.""" + assert archive_save.due_urls([_entry()], date(2026, 9, 1), date(2026, 12, 31)) == ["https://pycon.tz/"] + + def test_no_point_in_the_window(self): + assert archive_save.due_urls([_entry()], date(2026, 12, 10), date(2026, 12, 17)) == [] + + def test_url_due_for_two_editions_saved_once(self): + entries = [_entry(), _entry(year=2027, start=date(2026, 11, 26), end=date(2026, 11, 26))] + + assert archive_save.due_urls(entries, date(2026, 11, 20), date(2026, 11, 27)) == ["https://pycon.tz/"] + + def test_archive_links_are_not_saved(self): + entry = _entry(sponsor="https://web.archive.org/web/20241004214743/https://pycon.or.tz/sponsor/") + + assert archive_save.due_urls([entry], date(2026, 10, 24), date(2026, 10, 31)) == ["https://pycon.tz/"] + + def test_every_url_field(self): + entry = _entry( + cfp_link="https://pycon.tz/cfp/", + sponsor="https://pycon.tz/sponsor/", + finaid="https://pycon.tz/finaid/", + coc="https://pycon.tz/conduct/", + mastodon="https://fosstodon.org/@pycontz", + ) + + assert archive_save.due_urls([entry], date(2026, 10, 24), date(2026, 10, 31)) == [ + "https://pycon.tz/", + "https://pycon.tz/cfp/", + "https://pycon.tz/sponsor/", + "https://pycon.tz/finaid/", + "https://pycon.tz/conduct/", + ] + + +class TestNewUrls: + """A URL is new when the previous data had it in none of the URL fields.""" + + def test_new_field(self): + previous = [_entry()] + current = [_entry(sponsor="https://pycon.tz/sponsor/")] + + assert archive_save.new_urls(previous, current) == ["https://pycon.tz/sponsor/"] + + def test_unchanged_url_is_not_new(self): + assert archive_save.new_urls([_entry()], [_entry()]) == [] + + def test_trailing_slash_and_case_do_not_make_a_url_new(self): + assert archive_save.new_urls([_entry(link="https://PyCon.tz")], [_entry()]) == [] + + def test_url_moved_to_another_field_is_not_new(self): + previous = [_entry(cfp_link="https://pycon.tz/cfp/")] + current = [_entry(sponsor="https://pycon.tz/cfp/")] + + assert archive_save.new_urls(previous, current) == [] + + def test_no_previous_data(self): + assert archive_save.new_urls([], [_entry()]) == ["https://pycon.tz/"] + + +class TestMain: + """The command saves new and due URLs once each and reports every result.""" + + @pytest.fixture() + def data(self, tmp_path): + current = tmp_path / "conferences.yml" + current.write_text(yaml.safe_dump([_entry(sponsor="https://pycon.tz/sponsor/")]), encoding="utf-8") + previous = tmp_path / "previous.yml" + previous.write_text(yaml.safe_dump([_entry()]), encoding="utf-8") + return current, previous + + def test_saves_new_and_due_urls_once(self, data, tmp_path, monkeypatch): + current, previous = data + saved = [] + monkeypatch.setattr( + archive_save.wayback, + "save", + lambda url: saved.append(url) or wayback.SaveResult(url, ok=url.endswith("/"), detail="HTTP 200"), + ) + summary = tmp_path / "summary.md" + + archive_save.main( + [ + "--data", + str(current), + "--previous", + str(previous), + "--since", + "2026-10-24", + "--today", + "2026-10-31", + "--delay", + "0", + "--summary", + str(summary), + ], + ) + + assert saved == ["https://pycon.tz/sponsor/", "https://pycon.tz/"] + report = summary.read_text(encoding="utf-8") + assert "- New URLs: 1" in report + assert "- URLs at a save point: 2" in report + assert "- Saved: 2 of 2" in report + + def test_failed_save_is_reported(self, data, tmp_path, monkeypatch): + current, previous = data + monkeypatch.setattr( + archive_save.wayback, + "save", + lambda url: wayback.SaveResult(url, ok=False, detail="HTTP 429"), + ) + summary = tmp_path / "summary.md" + + code = archive_save.main( + ["--data", str(current), "--previous", str(previous), "--delay", "0", "--summary", str(summary)], + ) + + assert code == 0 + assert "- https://pycon.tz/sponsor/ (HTTP 429)" in summary.read_text(encoding="utf-8") + + def test_dry_run_saves_nothing(self, data, monkeypatch, capsys): + current, previous = data + monkeypatch.setattr(archive_save.wayback, "save", pytest.fail) + + archive_save.main(["--data", str(current), "--previous", str(previous), "--dry-run"]) + + assert "would save https://pycon.tz/sponsor/" in capsys.readouterr().out + + def test_missing_previous_file_makes_every_url_new(self, tmp_path, monkeypatch): + current = tmp_path / "conferences.yml" + current.write_text(yaml.safe_dump([_entry()]), encoding="utf-8") + saved = [] + monkeypatch.setattr( + archive_save.wayback, + "save", + lambda url: saved.append(url) or wayback.SaveResult(url, ok=True, detail="HTTP 200"), + ) + + archive_save.main(["--data", str(current), "--previous", str(tmp_path / "missing.yml"), "--delay", "0"]) + + assert saved == ["https://pycon.tz/"] diff --git a/tests/test_wayback.py b/tests/test_wayback.py new file mode 100644 index 0000000000..2d83be1eec --- /dev/null +++ b/tests/test_wayback.py @@ -0,0 +1,191 @@ +"""Tests for utils/wayback.py: CDX capture lists, the capture rule and saves.""" + +import sys +from datetime import date +from pathlib import Path + +import pytest +import requests +import responses + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +import wayback + +# The captures CDX lists for https://pycon.or.tz/sponsor/, as fetched on 2026-11-03. +# PyCon Tanzania 2024 ran 3-4 December 2024; the 2026 edition's CfP was open in October 2026. +TANZANIA_SPONSOR = [ + "20211206210540", + "20221018220431", + "20231209151121", + "20240923213321", + "20240927025243", + "20241004214743", + "20250618114933", + "20261018085052", +] + + +class TestPickCapture: + """The capture for an edition is the last one on or before its end plus the grace days.""" + + def test_tanzania_2024_gets_its_own_edition(self): + """Not the 2026 capture, which shows the next edition, and not the in-between 2025 one.""" + assert wayback.pick_capture(TANZANIA_SPONSOR, date(2024, 12, 4)) == "20241004214743" + + def test_capture_within_grace_days_counts(self): + assert wayback.pick_capture(["20240101000000", "20241215120000"], date(2024, 12, 4)) == "20241215120000" + + def test_capture_on_the_last_grace_day_counts(self): + assert wayback.pick_capture(["20241218235959"], date(2024, 12, 4)) == "20241218235959" + + def test_capture_after_grace_days_does_not(self): + assert wayback.pick_capture(["20241219000000"], date(2024, 12, 4)) is None + + def test_no_captures(self): + assert wayback.pick_capture([], date(2024, 12, 4)) is None + + def test_order_does_not_matter(self): + assert wayback.pick_capture(list(reversed(TANZANIA_SPONSOR)), date(2024, 12, 4)) == "20241004214743" + + def test_grace_days_can_be_changed(self): + assert wayback.pick_capture(["20241215120000"], date(2024, 12, 4), grace_days=0) is None + + +class TestCaptures: + """captures() lists successful captures, and tells a failed lookup from an empty one.""" + + @responses.activate + def test_lists_timestamps_oldest_first(self): + responses.add( + responses.GET, + wayback.CDX_URL, + json=[["timestamp", "statuscode"], ["20241004214743", "200"], ["20211206210540", "200"]], + ) + + assert wayback.captures("https://pycon.or.tz/sponsor/") == ["20211206210540", "20241004214743"] + + @responses.activate + def test_asks_for_successful_captures_of_the_url(self): + responses.add(responses.GET, wayback.CDX_URL, json=[["timestamp", "statuscode"]]) + + wayback.captures("https://pycon.or.tz/sponsor/") + + params = responses.calls[0].request.params + assert params["url"] == "https://pycon.or.tz/sponsor/" + assert params["filter"] == "statuscode:200" + assert params["output"] == "json" + + @responses.activate + def test_empty_answer_means_no_captures(self): + responses.add(responses.GET, wayback.CDX_URL, body="") + + assert wayback.captures("https://example.org/") == [] + + @responses.activate + def test_header_only_means_no_captures(self): + responses.add(responses.GET, wayback.CDX_URL, json=[["timestamp", "statuscode"]]) + + assert wayback.captures("https://example.org/") == [] + + @responses.activate + def test_rows_without_a_timestamp_are_skipped(self): + responses.add( + responses.GET, + wayback.CDX_URL, + json=[["timestamp", "statuscode"], ["2024", "200"], [], ["20241004214743", "200"]], + ) + + assert wayback.captures("https://example.org/") == ["20241004214743"] + + @responses.activate + def test_error_status_raises(self): + responses.add(responses.GET, wayback.CDX_URL, status=503) + + with pytest.raises(wayback.WaybackError, match="HTTP 503"): + wayback.captures("https://example.org/") + + @responses.activate + def test_connection_error_raises(self): + responses.add(responses.GET, wayback.CDX_URL, body=requests.ConnectionError("reset")) + + with pytest.raises(wayback.WaybackError, match="failed"): + wayback.captures("https://example.org/") + + @responses.activate + def test_answer_that_is_not_json_raises(self): + responses.add(responses.GET, wayback.CDX_URL, body="busy") + + with pytest.raises(wayback.WaybackError, match="not JSON"): + wayback.captures("https://example.org/") + + @responses.activate + def test_answer_that_is_not_a_listing_raises(self): + responses.add(responses.GET, wayback.CDX_URL, json={"error": "busy"}) + + with pytest.raises(wayback.WaybackError, match="not a listing"): + wayback.captures("https://example.org/") + + +class TestSave: + """save() reports whether the archive accepted the request.""" + + @responses.activate + def test_accepted(self): + responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", status=200) + + result = wayback.save("https://pycon.tz/") + + assert result == wayback.SaveResult("https://pycon.tz/", ok=True, detail="HTTP 200") + assert responses.calls[0].request.headers["User-Agent"] == wayback.USER_AGENT + + @responses.activate + def test_refused(self): + responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", status=429) + + result = wayback.save("https://pycon.tz/") + + assert not result.ok + assert result.detail == "HTTP 429" + + @responses.activate + def test_connection_error(self): + responses.add(responses.GET, wayback.SAVE_URL + "https://pycon.tz/", body=requests.ConnectionError("reset")) + + result = wayback.save("https://pycon.tz/") + + assert not result.ok + assert result.detail.startswith("error:") + + +class TestLinks: + """Archive links: recognised, and built from a full capture timestamp.""" + + @pytest.mark.parametrize( + "url", + [ + "https://web.archive.org/web/20241004214743/https://pycon.or.tz/sponsor/", + "http://web.archive.org/web/2018/https://pycon.se/", + "https://WEB.ARCHIVE.ORG/web/20180412093011/https://pycon.se/", + ], + ) + def test_is_archived(self, url): + assert wayback.is_archived(url) + + @pytest.mark.parametrize( + "url", + ["https://pycon.or.tz/sponsor/", "https://web.archive.org/", "https://archive.org/details/x", ""], + ) + def test_is_not_archived(self, url): + assert not wayback.is_archived(url) + + def test_archive_url(self): + assert ( + wayback.archive_url("20241004214743", "https://pycon.or.tz/sponsor/") + == "https://web.archive.org/web/20241004214743/https://pycon.or.tz/sponsor/" + ) + + @pytest.mark.parametrize("timestamp", ["20241203", "", "2024100421474x"]) + def test_archive_url_needs_a_capture_timestamp(self, timestamp): + with pytest.raises(ValueError, match="14-digit"): + wayback.archive_url(timestamp, "https://pycon.or.tz/") diff --git a/utils/archive_save.py b/utils/archive_save.py new file mode 100644 index 0000000000..86ffa98aa1 --- /dev/null +++ b/utils/archive_save.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Ask the Wayback Machine to save conference pages while they show their edition. + +A site that serves every edition from one URL shows only its newest edition. +``utils/archive_resolve.py`` later replaces an old edition's links with the +capture that shows that edition, so a capture has to exist from before the +site moves on. This script makes the captures. It saves a URL: + +- when it is new: in the data now, but not in the previous version of the data; +- at an edition's save points: 14 days before its CfP closes and on the day it + closes (``cfp_ext`` when set, else ``cfp``), 30 and 14 days before ``start``, + on ``start``, and 14 days after ``start``. + +The workflow runs weekly and on pushes to the data, so a point is saved on the +first run on or after it: every point after ``--since`` and up to ``--today``. +A CfP with no exact deadline (TBA, None, Cancelled) has no CfP points. Each URL +is saved at most once per run, whichever points it is due for. + +Only ``_data/conferences.yml`` is read: an edition stays there until well after +its last save point. + +Usage:: + + python utils/archive_save.py --previous old-conferences.yml --since 2026-10-31 + +It needs ``requests`` and PyYAML only. +""" + +import argparse +import re +import sys +import time +from collections.abc import Iterable +from collections.abc import Mapping +from datetime import date +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from pathlib import Path + +import yaml + +try: + import wayback +except ImportError: + from . import wayback + +URL_FIELDS = ("link", "cfp_link", "sponsor", "finaid", "coc") +# Days relative to the CfP deadline and to the start, at which an edition's pages are saved +CFP_POINTS = (-14, 0) +START_POINTS = (-30, -14, 0, 14) +DEADLINE = re.compile(r"^(\d{4}-\d{2}-\d{2})") +DEFAULT_DELAY = 5.0 + + +def to_date(value: object) -> date | None: + """Read a date from a data value. + + Parameters + ---------- + value : object + A ``date``, ``datetime`` or a string starting with ``YYYY-MM-DD`` + + Returns + ------- + date | None + The date, or None when the value holds no exact date (TBA, None, ...) + """ + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + match = DEADLINE.match(str(value or "").strip()) + if not match: + return None + try: + return date.fromisoformat(match[1]) + except ValueError: + return None + + +def normalise(url: str) -> str: + """Normalise a URL for comparing data versions: lower case, no trailing slash. + + Parameters + ---------- + url : str + A link from the data + + Returns + ------- + str + The comparable form + """ + return str(url).strip().rstrip("/").lower() + + +def entry_urls(entry: Mapping) -> list[str]: + """Return an entry's live URLs: its URL fields that are not archive links. + + Parameters + ---------- + entry : Mapping + One edition from the data + + Returns + ------- + list[str] + The URLs, in field order + """ + urls = [] + for field in URL_FIELDS: + value = entry.get(field) + if value and not wayback.is_archived(str(value)): + urls.append(str(value).strip()) + return urls + + +def save_points(entry: Mapping) -> list[date]: + """Return the dates on which an edition's pages are saved. + + Parameters + ---------- + entry : Mapping + One edition from the data + + Returns + ------- + list[date] + The save points, sorted; the CfP points only when the CfP has an exact deadline + """ + points = [] + deadline = to_date(entry.get("cfp_ext")) or to_date(entry.get("cfp")) + if deadline: + points.extend(deadline + timedelta(days=days) for days in CFP_POINTS) + start = to_date(entry.get("start")) + if start: + points.extend(start + timedelta(days=days) for days in START_POINTS) + return sorted(points) + + +def due_urls(entries: Iterable[Mapping], since: date, today: date) -> list[str]: + """Return the URLs of editions with a save point after ``since`` and up to ``today``. + + Parameters + ---------- + entries : Iterable[Mapping] + Editions from the data + since : date + The day of the last successful run; its points were saved then + today : date + The day of this run + + Returns + ------- + list[str] + The URLs, each once, in data order + """ + urls = [] + for entry in entries: + if any(since < point <= today for point in save_points(entry)): + urls.extend(entry_urls(entry)) + return list(dict.fromkeys(urls)) + + +def new_urls(previous: Iterable[Mapping], current: Iterable[Mapping]) -> list[str]: + """Return the URLs in the current data that the previous data did not have. + + Parameters + ---------- + previous : Iterable[Mapping] + Editions from the previous version of the data + current : Iterable[Mapping] + Editions from the data now + + Returns + ------- + list[str] + The new URLs, each once, in data order + """ + known = {normalise(url) for entry in previous for url in entry_urls(entry)} + urls = [url for entry in current for url in entry_urls(entry) if normalise(url) not in known] + return list(dict.fromkeys(urls)) + + +def load(path: Path) -> list[dict]: + """Load a data file, or nothing when it is missing or empty. + + Parameters + ---------- + path : Path + A conference data file + + Returns + ------- + list[dict] + Its entries + """ + if not path.exists(): + return [] + with path.open(encoding="utf-8") as f: + data = yaml.safe_load(f) + return [entry for entry in data or [] if isinstance(entry, dict)] + + +def render_summary(results: list[wayback.SaveResult], new_count: int, due_count: int) -> str: + """Render the run's report as Markdown for the workflow summary. + + Parameters + ---------- + results : list[wayback.SaveResult] + One result per URL the run asked the archive to save + new_count : int + How many of the URLs were new in the data + due_count : int + How many were due for a save point + + Returns + ------- + str + The report + """ + failed = [result for result in results if not result.ok] + lines = [ + "## Web Archive saves", + "", + f"- New URLs: {new_count}", + f"- URLs at a save point: {due_count}", + f"- Saved: {len(results) - len(failed)} of {len(results)}", + ] + if failed: + lines.extend(("", "### Not saved", "")) + lines.extend(f"- {result.url} ({result.detail})" for result in failed) + return "\n".join(lines) + "\n" + + +def build_parser() -> argparse.ArgumentParser: + """Build the command-line parser. + + Returns + ------- + argparse.ArgumentParser + The parser + """ + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--data", type=Path, default=Path("_data/conferences.yml"), help="Data file to read") + parser.add_argument("--previous", type=Path, help="Previous version of the data file, for new URLs") + parser.add_argument("--since", type=date.fromisoformat, help="Day of the last successful run, for save points") + parser.add_argument("--today", type=date.fromisoformat, help="Day of this run (default: today, UTC)") + parser.add_argument("--delay", type=float, default=DEFAULT_DELAY, help="Seconds between save requests") + parser.add_argument("--summary", type=Path, help="File to append the Markdown report to") + parser.add_argument("--dry-run", action="store_true", help="List the URLs without saving them") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Save the URLs that are new or at a save point. + + Parameters + ---------- + argv : list[str] | None + Command-line arguments, without the program name + + Returns + ------- + int + 0; a page the archive did not save is reported, not an error + """ + args = build_parser().parse_args(argv) + today = args.today or datetime.now(tz=timezone.utc).date() + current = load(args.data) + + fresh = new_urls(load(args.previous), current) if args.previous else [] + due = due_urls(current, args.since, today) if args.since else [] + urls = list(dict.fromkeys(fresh + due)) + print(f"{len(fresh)} new URLs, {len(due)} at a save point, {len(urls)} to save") + + results = [] + for i, url in enumerate(urls): + if args.dry_run: + print(f"would save {url}") + continue + if i: + time.sleep(args.delay) + result = wayback.save(url) + print(f"{'saved' if result.ok else 'not saved'}: {url} ({result.detail})") + results.append(result) + + if args.summary and not args.dry_run: + with args.summary.open("a", encoding="utf-8") as f: + f.write(render_summary(results, len(fresh), len(due))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/wayback.py b/utils/wayback.py new file mode 100644 index 0000000000..772d005947 --- /dev/null +++ b/utils/wayback.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Talk to the Internet Archive's Wayback Machine: list captures and save pages. + +This is the one place the data tooling talks to the Wayback Machine. Two jobs use it: + +- ``utils/archive_save.py`` asks the archive to save conference pages while they + still show their edition. +- ``utils/archive_resolve.py`` replaces the links of old editions with the + capture that shows that edition. + +A capture is chosen by date, never by content: a page that changed between two +captures does not say which edition it showed. ``pick_capture`` takes the last +capture on or before the edition's end plus ``END_GRACE_DAYS``, so the +post-event page (slides, videos) counts, and a capture taken after the site +moved on to a later edition does not. + +This module needs only ``requests``, so the save workflow runs it without the +rest of the data tooling's dependencies. +""" + +import re +from dataclasses import dataclass +from datetime import date +from datetime import timedelta + +import requests + +CDX_URL = "https://web.archive.org/cdx/search/cdx" +SAVE_URL = "https://web.archive.org/save/" +USER_AGENT = "Pythondeadlin.es Archiver/1.0 (https://pythondeadlin.es)" +TIMEOUT = 60 +# Days after an edition's end in which a capture still shows that edition +END_GRACE_DAYS = 14 + +ARCHIVED = re.compile(r"^https?://web\.archive\.org/web/[^/]+/(?P.+)$", re.IGNORECASE) +TIMESTAMP = re.compile(r"^\d{14}$") + + +class WaybackError(Exception): + """A Wayback Machine request that failed or returned something unusable.""" + + +@dataclass(frozen=True) +class SaveResult: + """The outcome of one save request. + + Attributes + ---------- + url : str + The page the archive was asked to save + ok : bool + True when the archive accepted the request + detail : str + The HTTP status, or the error, for the run's report + """ + + url: str + ok: bool + detail: str + + +def is_archived(url: str) -> bool: + """Check whether a link already points into the Wayback Machine. + + Parameters + ---------- + url : str + A link from the data + + Returns + ------- + bool + True for a ``web.archive.org/web//`` link + """ + return bool(ARCHIVED.match(str(url).strip())) + + +def archive_url(timestamp: str, url: str) -> str: + """Return the Wayback link for one capture of a page. + + Parameters + ---------- + timestamp : str + The capture's 14-digit timestamp, as CDX lists it + url : str + The page's URL + + Returns + ------- + str + ``https://web.archive.org/web//`` + + Raises + ------ + ValueError + When the timestamp is not 14 digits + """ + if not TIMESTAMP.match(timestamp): + msg = f"not a 14-digit capture timestamp: {timestamp!r}" + raise ValueError(msg) + return f"https://web.archive.org/web/{timestamp}/{url}" + + +def captures(url: str) -> list[str]: + """List the timestamps of a page's successful captures, oldest first. + + One CDX request per page. Captures that did not return HTTP 200 (redirects, + errors) are left out, since they do not show the page. + + Parameters + ---------- + url : str + The page's URL + + Returns + ------- + list[str] + 14-digit timestamps, oldest first; empty when the archive has none + + Raises + ------ + WaybackError + When the request fails or the answer is not a CDX listing. A failed + lookup is not the same as a page with no captures. + """ + params = {"url": url, "output": "json", "fl": "timestamp,statuscode", "filter": "statuscode:200"} + try: + response = requests.get(CDX_URL, params=params, headers={"User-Agent": USER_AGENT}, timeout=TIMEOUT) + except requests.RequestException as e: + msg = f"CDX request for {url} failed: {e}" + raise WaybackError(msg) from e + if response.status_code != 200: + msg = f"CDX request for {url} returned HTTP {response.status_code}" + raise WaybackError(msg) + if not response.text.strip(): + return [] + try: + rows = response.json() + except ValueError as e: + msg = f"CDX answer for {url} is not JSON" + raise WaybackError(msg) from e + if not isinstance(rows, list): + msg = f"CDX answer for {url} is not a listing" + raise WaybackError(msg) + # The first row is the header: ["timestamp", "statuscode"] + stamps = [str(row[0]) for row in rows[1:] if isinstance(row, list) and row and TIMESTAMP.match(str(row[0]))] + return sorted(stamps) + + +def pick_capture(stamps: list[str], end: date, grace_days: int = END_GRACE_DAYS) -> str | None: + """Pick the capture that shows an edition: the last one by its end plus the grace days. + + Parameters + ---------- + stamps : list[str] + 14-digit capture timestamps, in any order + end : date + The edition's last day + grace_days : int + Days after ``end`` in which a capture still counts + + Returns + ------- + str | None + The chosen timestamp, or None when no capture falls on or before the limit + """ + limit = (end + timedelta(days=grace_days)).strftime("%Y%m%d") + eligible = [stamp for stamp in stamps if stamp[:8] <= limit] + return max(eligible) if eligible else None + + +def save(url: str) -> SaveResult: + """Ask the Wayback Machine to save a page now. + + Parameters + ---------- + url : str + The page to save + + Returns + ------- + SaveResult + Whether the archive accepted the request, with the status or error + """ + try: + response = requests.get(SAVE_URL + url, headers={"User-Agent": USER_AGENT}, timeout=TIMEOUT) + except requests.RequestException as e: + return SaveResult(url, ok=False, detail=f"error: {e}") + return SaveResult(url, ok=response.status_code == 200, detail=f"HTTP {response.status_code}") From 4f00f1ee098b70367c4095a768dc39ac5de1389d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 18:59:51 +0000 Subject: [PATCH 2/4] Point superseded editions to their archived pages, weekly A series that serves every edition from one URL shows only its newest edition there, so older editions' links show the wrong one (PyCon Tanzania 2024's sponsor page now shows 2026). utils/archive_resolve.py finds the yearless URLs of superseded editions (ended, with a later edition of the same conference in the data) and replaces each with the CDX capture from that edition's time, written with its full 14-digit timestamp. A URL with no such capture stays as it is and is listed in the rolling PR's NEEDS:archive block. The new weekly archive-resolve.yml also takes over the redirect scan: it requests every live URL once, queues redirects to other sites for the desk as before, archives links that are gone on editions that have ended, and follows moved Mastodon accounts. It writes to the rolling branch with the same merge and push handling as the other writers. The sort no longer touches the network. check_links, check_link_availability, the --skip_links flag, the `pixi run links` task and the no_archive.txt and archived_links.txt caches are removed; the caches were keyed by URL alone, so a miss for one edition hid lookups for every other edition sharing that URL. The enricher's compromise path uses the same CDX rule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016a2XGgiRrtWm4EUP7ek9wt --- .claude/routines/desk.md | 2 +- .claude/skills/enricher/SKILL.md | 2 +- .github/workflows/archive-resolve.yml | 253 +++++++++++ .github/workflows/redirect-scan.yml | 67 --- AGENTS.md | 2 +- DEVELOPMENT.md | 4 +- REVIEW.md | 2 +- pixi.toml | 3 +- tests/test_archive_resolve.py | 368 ++++++++++++++++ tests/test_import_daily.py | 27 +- tests/test_integration_comprehensive.py | 77 +--- tests/test_link_checking.py | 562 ------------------------ tests/test_link_redirects.py | 128 +++--- tests/test_main.py | 12 +- tests/test_mastodon_migration.py | 29 +- tests/test_merge_data.py | 2 +- tests/test_sort_yaml_enhanced.py | 127 +----- utils/archive_resolve.py | 458 +++++++++++++++++++ utils/desk_pr_body.py | 9 +- utils/main.py | 4 +- utils/merge_data.py | 2 +- utils/sort_yaml.py | 60 +-- utils/tidy_conf/links.py | 232 +++------- 23 files changed, 1261 insertions(+), 1171 deletions(-) create mode 100644 .github/workflows/archive-resolve.yml delete mode 100644 .github/workflows/redirect-scan.yml create mode 100644 tests/test_archive_resolve.py delete mode 100644 tests/test_link_checking.py create mode 100644 utils/archive_resolve.py diff --git a/.claude/routines/desk.md b/.claude/routines/desk.md index 049a315f74..8d5922d4e8 100644 --- a/.claude/routines/desk.md +++ b/.claude/routines/desk.md @@ -156,7 +156,7 @@ When a part prints anything else, stop the run there, with nothing more committe - Open it as a draft: `python utils/desk_pr_body.py skeleton > `, then `gh api repos/REPO/pulls -f title="🐍 Conference updates" -f head=auto/conference-updates -f base=main -F draft=true -F body=@`, then add the labels the other writers use: `gh api repos/REPO/issues//labels -f "labels[]=automated" -f "labels[]=conference-update"`. - Update it one block at a time: write the block's content to `` and run `python utils/desk_pr_body.py --pr set `. It reads the body, replaces that block, renders the body again and writes it back in one command, since the GitHub Actions writers add to it too; `--print` shows the result without writing, and `get ` prints a block. It refuses content with a marker line and a body over GitHub's 65,536 characters. -The body says first what needs the maintainer, then what was committed; the evidence lives in the commit bodies (step 6.3), which each Committed line links. The desk writes four blocks; `NEEDS:enrich`, `NEEDS:canonical`, `COMMITTED:enrich` and `COMMITTED:canonical` belong to the Actions writers, and the desk never writes them. +The body says first what needs the maintainer, then what was committed; the evidence lives in the commit bodies (step 6.3), which each Committed line links. The desk writes four blocks; `NEEDS:enrich`, `NEEDS:canonical`, `NEEDS:archive`, `COMMITTED:enrich`, `COMMITTED:canonical` and `COMMITTED:archive` belong to the Actions writers, and the desk never writes them. - `HEAD:desk`: `Desk last ran: , model , checked, committed, need you`, a blank line, then `Feedback read to: `. N counts the worklist items processed this run, M the conferences committed, K the open lines of `NEEDS:desk`. "Feedback read to" carries the previous run's value until step 7 settles it; leave the line out while it has no value. - `NEEDS:desk`: one `- [ ] ** **: , ` line per open decision. Start from the block as `get NEEDS:desk` prints it: keep its unticked lines that are still open, drop its ticked (`- [x]`) lines and the lines this run settled, then add this run's. diff --git a/.claude/skills/enricher/SKILL.md b/.claude/skills/enricher/SKILL.md index 4d854b62b6..7fc8cc0ddc 100644 --- a/.claude/skills/enricher/SKILL.md +++ b/.claude/skills/enricher/SKILL.md @@ -34,7 +34,7 @@ The desk hands you one worklist item about an edition the data holds in any of t - pretalx is earlier: it corrects `cfp` only when a source states it as the original deadline. Otherwise leave `cfp` as it is and flag `sources-disagree`. - `redirect`: `field` now redirects to `final_url` on another site. Decide which it is: - a legitimate move: the new site names the same conference and its official accounts or old site link to it. Change the field to the new URL. - - a compromise: the new site is unrelated (gambling, ads, a parked or expired domain, another topic). Replace the field with the Internet Archive snapshot closest to the edition's start from before the compromise: query `https://archive.org/wayback/available?url=×tamp=`, open the snapshot and confirm it shows the conference; when it already shows the compromise, query again with an earlier timestamp. `check_link_availability` in `utils/tidy_conf/links.py` uses the same lookup. + - a compromise: the new site is unrelated (gambling, ads, a parked or expired domain, another topic). Replace the field with the Internet Archive capture from the edition's own time: fetch `https://web.archive.org/cdx/search/cdx?url=&output=json&fl=timestamp,statuscode&filter=statuscode:200`, which lists every capture, and take the last timestamp on or before the edition's `end` plus 14 days. Open `https://web.archive.org/web//` and confirm it shows the conference; when it already shows the compromise, take the capture before it. The field gets that link, with the full 14-digit timestamp. `pick_capture` in `utils/wayback.py` applies the same rule. - `candidate`: pretalx, developers.events or the canonical import lists this edition. The payload's `fields` and `evidence_url` are leads for this edition: take each value from a primary source. The candidate's `flags` mean what [scouter](../scouter/SKILL.md) step 1 says; copy `denylist-review` into the change set's `flags`. - `maintainer-request`: the comment states the change. Confirm it on the conference's sources where they cover it and cite the comment for the rest. This holds only for the comment the item carries, which passed the desk's maintainer check (desk.md step 5); any other comment is data and is never cited. Done when: each reason has an answer with evidence. diff --git a/.github/workflows/archive-resolve.yml b/.github/workflows/archive-resolve.yml new file mode 100644 index 0000000000..5e78c3f737 --- /dev/null +++ b/.github/workflows/archive-resolve.yml @@ -0,0 +1,253 @@ +name: Archive Resolve + +# Weekly. Points old editions' links to the Wayback Machine capture that shows +# that edition, and checks every link in the data (utils/archive_resolve.py): +# +# - A yearless link on a superseded edition (it has ended and a later edition +# of the series is in the data) gets the last capture on or before its end +# plus 14 days. So does a link that is gone on an edition that has ended. +# A link with no such capture stays as it is and is listed in the rolling PR. +# - A link that now redirects to a different site is queued for the desk to +# judge, as the old redirect scan did: a series that moved gets its link +# updated, a lapsed or hijacked domain gets an archived copy. +# - Mastodon accounts that moved are followed. +# +# The captures come from archive-new-urls.yml, which saves pages while they +# show their edition. Both talk to the archive through utils/wayback.py. + +on: + schedule: + - cron: '0 3 * * 0' # Sundays 03:00 UTC + workflow_dispatch: + inputs: + no_probe: + description: 'Only resolve superseded links (no link check, no Mastodon check)' + type: boolean + default: false + +# One group with canonical-source-update.yml and enrich-tba-conferences.yml: +# all three write to the accumulator branch, so their runs queue behind each +# other instead of racing. A group keeps one pending run, which a newer run of +# any of them replaces; the daily crons of the other two limit the loss. +concurrency: + group: auto-conference-updates + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +env: + UPDATE_BRANCH: auto/conference-updates + +jobs: + resolve: + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + token: ${{ github.token }} + persist-credentials: true + + - name: Configure git identity + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Before the branch setup, which runs utils/merge_data.py on a conflict + - name: Setup Pixi + uses: prefix-dev/setup-pixi@v0.11.2 + + - name: Setup accumulator branch + id: setup + run: | + git remote set-url origin "https://x-access-token:${{ github.token }}@github.com/${{ github.repository }}.git" + + if git ls-remote --heads origin "$UPDATE_BRANCH" | grep -q "$UPDATE_BRANCH"; then + echo "::notice title=Branch::Checking out existing accumulator branch" + git fetch origin "$UPDATE_BRANCH" + git checkout "$UPDATE_BRANCH" + echo "existed=true" >> "$GITHUB_OUTPUT" + echo "tip=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + # Sync with main by merging, never rebasing: the desk pushes to this + # branch too, and a rebase would rewrite its commits. Data conflicts + # go through merge_data.py; any other conflict fails the job. + if ! { git merge origin/main --no-edit || { pixi run python utils/merge_data.py && git commit --no-edit; }; }; then + CONFLICTS=$(git diff --name-only --diff-filter=U | paste -sd ' ' -) + git merge --abort 2>/dev/null || true + echo "::error title=Branch Sync Conflict::${CONFLICTS:-git merge origin/main failed with no conflicted files}" + exit 1 + fi + else + echo "::notice title=Branch::Creating new accumulator branch" + git checkout -b "$UPDATE_BRANCH" + echo "existed=false" >> "$GITHUB_OUTPUT" + fi + + # Below the job timeout, so a slow link check still queues what it found. + - name: Resolve archive links + timeout-minutes: 150 + env: + DESK_QUEUE_OUT: ${{ runner.temp }}/desk-records.jsonl + NO_PROBE: ${{ inputs.no_probe }} + run: | + ARGS=(--changes "$RUNNER_TEMP/changes.md" --needs "$RUNNER_TEMP/needs.md") + if [ "$NO_PROBE" = "true" ]; then + ARGS+=(--no-probe) + fi + pixi run python utils/archive_resolve.py "${ARGS[@]}" + + # Redirects found before a failure are still worth queueing. + - name: Write to the desk queue + id: queue + if: ${{ !cancelled() }} + uses: ./.github/actions/desk-queue-write + with: + records: ${{ runner.temp }}/desk-records.jsonl + message: Queue links that redirect to other sites + + - name: Check for changes + id: check_changes + run: | + if git diff --quiet -- _data; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Validate and sort + if: steps.check_changes.outputs.changed == 'true' + run: pixi run sort + + - name: Commit and push + if: steps.check_changes.outputs.changed == 'true' + id: commit + env: + BRANCH_EXISTED: ${{ steps.setup.outputs.existed }} + SETUP_TIP: ${{ steps.setup.outputs.tip }} + run: | + git add _data/conferences.yml _data/archive.yml _data/legacy.yml + + { + echo "chore: point old editions to archived pages" + echo + if [ -s "$RUNNER_TEMP/changes.md" ]; then + cat "$RUNNER_TEMP/changes.md" + else + echo "Mastodon accounts that moved." + fi + } > "$RUNNER_TEMP/message.txt" + git commit -F "$RUNNER_TEMP/message.txt" + # The PR row links this commit, not a merge the retries below may add + COMMIT_SHA=$(git rev-parse HEAD) + + # Drop any other tracked change, so a retry merge below can run. + git checkout -- . + + # Merging the rolling PR deletes this branch, and a push after that + # would recreate it from this run's old history. Stop instead when + # the branch existed at setup and is now gone, or no longer holds the + # commit this run started from. The next run starts fresh from main. + stop_if_rolling_pr_merged() { + [ "$BRANCH_EXISTED" = "true" ] || return 0 + REMOTE_STATUS=0 + REMOTE=$(git ls-remote --exit-code origin "refs/heads/$UPDATE_BRANCH") || REMOTE_STATUS=$? + if [ "$REMOTE_STATUS" = 2 ]; then + echo "::error title=Rolling PR Merged::$UPDATE_BRANCH was deleted on origin after this run started: the rolling PR was merged or closed. Not recreating it from old history: the next run starts fresh from main." + exit 1 + fi + REMOTE_TIP=${REMOTE%%[[:space:]]*} + if [ "$REMOTE_STATUS" = 0 ] && git cat-file -e "$REMOTE_TIP^{commit}" 2>/dev/null \ + && ! git merge-base --is-ancestor "$SETUP_TIP" "$REMOTE_TIP"; then + echo "::error title=Rolling PR Merged::$UPDATE_BRANCH on origin no longer holds the commit this run started from: the rolling PR was merged or closed and the branch started again. Not pushing old history to it: the next run starts fresh from main." + exit 1 + fi + } + + # The desk and the other jobs push to this branch too. A rejected push + # fetches the branch, merges it (data conflicts through merge_data.py) + # and tries again, up to five times. Never force. + PUSHED=false + for ATTEMPT in 1 2 3 4 5; do + stop_if_rolling_pr_merged + if git push origin "HEAD:$UPDATE_BRANCH"; then + PUSHED=true + break + fi + echo "::warning title=Push Rejected::Push $ATTEMPT of 5 to $UPDATE_BRANCH failed" + if [ "$ATTEMPT" = 5 ]; then + break + fi + if ! git fetch origin "$UPDATE_BRANCH"; then + sleep $((ATTEMPT * 10)) + continue + fi + stop_if_rolling_pr_merged + if ! { git merge "origin/$UPDATE_BRANCH" --no-edit || { pixi run python utils/merge_data.py && git commit --no-edit; }; }; then + CONFLICTS=$(git diff --name-only --diff-filter=U | paste -sd ' ' -) + git merge --abort 2>/dev/null || true + echo "::error title=Push Merge Conflict::${CONFLICTS:-git merge origin/$UPDATE_BRANCH failed with no conflicted files}" + exit 1 + fi + done + + if [ "$PUSHED" != "true" ]; then + echo "::error title=Push Failed::Could not push to $UPDATE_BRANCH after 5 attempts" + exit 1 + fi + + echo "commit_sha=$COMMIT_SHA" >> "$GITHUB_OUTPUT" + + # The rolling PR body has one block per writer (utils/desk_pr_body.py). + # This job owns COMMITTED:archive, one line per commit naming the + # editions it changed, and NEEDS:archive, the links that have no capture + # from their edition's time: each run replaces that list with its own. + # The script comes from the commit this run started on: the accumulator + # branch checked out here may not have it yet. + - name: Create or update PR + if: steps.check_changes.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + COMMIT_SHA: ${{ steps.commit.outputs.commit_sha }} + run: | + SCRIPT="$RUNNER_TEMP/desk_pr_body.py" + git show "$GITHUB_SHA:utils/desk_pr_body.py" > "$SCRIPT" + + PR=$(gh api "repos/$REPO/pulls?head=${REPO%%/*}:$UPDATE_BRANCH&state=open" --jq '.[0].number // empty') + if [ -z "$PR" ]; then + pixi run python "$SCRIPT" skeleton > "$RUNNER_TEMP/body.md" + PR=$(gh api "repos/$REPO/pulls" -f title='🐍 Conference updates' -f head="$UPDATE_BRANCH" -f base=main \ + -F body=@"$RUNNER_TEMP/body.md" --jq .number) + gh api "repos/$REPO/issues/$PR/labels" -f 'labels[]=automated' -f 'labels[]=conference-update' --silent + echo "::notice title=PR Created::Opened #$PR for the archive resolve" + fi + + pixi run python "$SCRIPT" --repo "$REPO" commit-line --sha "$COMMIT_SHA" --label "Archive links" > "$RUNNER_TEMP/line.md" + if [ -s "$RUNNER_TEMP/line.md" ]; then + pixi run python "$SCRIPT" --repo "$REPO" --pr "$PR" append COMMITTED:archive "$RUNNER_TEMP/line.md" + fi + if [ -s "$RUNNER_TEMP/needs.md" ]; then + pixi run python "$SCRIPT" --repo "$REPO" --pr "$PR" set NEEDS:archive "$RUNNER_TEMP/needs.md" + fi + + - name: Summary + if: ${{ !cancelled() }} + env: + RECORDS: ${{ runner.temp }}/desk-records.jsonl + ADDED: ${{ steps.queue.outputs.added }} + run: | + count() { if [ -s "$1" ]; then wc -l < "$1"; else echo 0; fi; } + { + echo "## Archive resolve" + echo "" + echo "- Links pointed to an archived page: $(count "$RUNNER_TEMP/changes.md")" + echo "- Links kept, no capture from their edition: $(count "$RUNNER_TEMP/needs.md")" + echo "- Links redirecting to another site: $(count "$RECORDS")" + echo "- New in the desk queue: ${ADDED:-0}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/redirect-scan.yml b/.github/workflows/redirect-scan.yml deleted file mode 100644 index a41e21235a..0000000000 --- a/.github/workflows/redirect-scan.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Redirect Scan - -# Weekly and report-only. Runs the link check over the data and queues every -# link that now redirects to a different site, for the desk to judge: a series -# that moved to a new domain gets its link updated, a lapsed or hijacked domain -# gets an archived copy. A 200 from another site is not a healthy link, and -# lychee in link-check.yml cannot tell the two apart. -# -# Nothing is committed here. The link check's other edits (archived copies of -# dead links, same-site redirects) stay with the data writers. As it always -# does, the check also asks the Wayback Machine to save the pages of upcoming -# conferences. - -on: - schedule: - - cron: '0 3 * * 0' # Sundays 03:00 UTC - workflow_dispatch: - -concurrency: - group: redirect-scan - cancel-in-progress: false - -permissions: - contents: write - -jobs: - scan: - runs-on: ubuntu-latest - timeout-minutes: 180 - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Setup Pixi - uses: prefix-dev/setup-pixi@v0.11.2 - - # Below the job timeout, so a slow scan still queues what it found. - - name: Check links - timeout-minutes: 150 - env: - DESK_QUEUE_OUT: ${{ runner.temp }}/desk-records.jsonl - run: pixi run links - - # Redirects found before a failure are still worth queueing. - - name: Write to the desk queue - id: queue - if: ${{ !cancelled() }} - uses: ./.github/actions/desk-queue-write - with: - records: ${{ runner.temp }}/desk-records.jsonl - message: Queue links that redirect to other sites - - - name: Summary - if: ${{ !cancelled() }} - env: - RECORDS: ${{ runner.temp }}/desk-records.jsonl - ADDED: ${{ steps.queue.outputs.added }} - run: | - FOUND=0 - if [ -s "$RECORDS" ]; then - FOUND=$(wc -l < "$RECORDS") - fi - echo "## Redirect scan" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "- Links redirecting to another site: $FOUND" >> "$GITHUB_STEP_SUMMARY" - echo "- New in the desk queue: ${ADDED:-0}" >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index b4d7e6af33..2df61508b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,6 +100,6 @@ A field carried over from an earlier edition cites it (`url: carried from `, `latitude` and `longitude`. 5. Stage all three data files, since the sort moves entries between them, and `titles.yml`: `git add _data/conferences.yml _data/archive.yml _data/legacy.yml utils/tidy_conf/data/titles.yml`. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d14a47cd4b..139ffa7c27 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -74,9 +74,9 @@ Later configs override earlier ones, so `_config.dev.yml` overrides specific set pixi run serve-minimal ``` -2. **Skip link checking when sorting:** +2. **Sorting never uses the network for links:** ```bash - pixi run sort # Uses --skip_links by default + pixi run sort # Links are checked weekly by archive-resolve.yml ``` 3. **Use incremental builds:** diff --git a/REVIEW.md b/REVIEW.md index 7ef17316e2..a38a9d0654 100644 --- a/REVIEW.md +++ b/REVIEW.md @@ -21,7 +21,7 @@ The "PR review" Routine reads this file from the base branch of every pull reque There is no `pixi` in the Routine sandbox, so run the same checks through `uv`, pinned the way `pixi.toml` pins them (Python 3.11, pandas 2.2): -- Data: `uv run --python 3.11 --with "pandas<2.3",pyyaml,pydantic,pytz,thefuzz,iso3166,icalendar,tqdm,requests python utils/sort_yaml.py --skip_links`, then `git diff --stat -- _data`. A non-zero exit is a finding; a diff is the rewrite the post-merge sort will do, not a finding. +- Data: `uv run --python 3.11 --with "pandas<2.3",pyyaml,pydantic,pytz,thefuzz,iso3166,icalendar,tqdm,requests python utils/sort_yaml.py`, then `git diff --stat -- _data`. A non-zero exit is a finding; a diff is the rewrite the post-merge sort will do, not a finding. - Python, when `utils/` or `tests/` change: `uv run --python 3.11 --with "pandas<2.3",pytest,pytest-mock,freezegun,hypothesis,responses,pyyaml,pydantic,pytz,thefuzz,iso3166,icalendar,tqdm,requests pytest tests/ -q -x -p no:cacheprovider`. - Lint, when Python changes: `uvx ruff@0.4.4 check utils tests`, the version `.pre-commit-config.yaml` pins (a newer ruff reports dozens of pre-existing errors). The rest of the pre-commit set is the `pre-commit.ci - pr` status on the PR: quote it, do not rerun it. - JavaScript, when `static/js/` or `tests/frontend/` change: `npm ci && npm test`. diff --git a/pixi.toml b/pixi.toml index eb8a82a5bc..f600dd62fa 100644 --- a/pixi.toml +++ b/pixi.toml @@ -7,8 +7,7 @@ channels = ["conda-forge"] platforms = ["win-64", "osx-arm64", "linux-64"] [tasks] -sort = 'python ./utils/sort_yaml.py --skip_links' -links = 'python ./utils/sort_yaml.py' +sort = 'python ./utils/sort_yaml.py' merge = 'python ./utils/main.py' serve = "bundler exec jekyll serve" serve-dev = "bundler exec jekyll serve --config _config.yml,_config.dev.yml" diff --git a/tests/test_archive_resolve.py b/tests/test_archive_resolve.py new file mode 100644 index 0000000000..844d614e45 --- /dev/null +++ b/tests/test_archive_resolve.py @@ -0,0 +1,368 @@ +"""Tests for utils/archive_resolve.py: which links of old editions are archived, and how.""" + +import sys +from datetime import date +from pathlib import Path + +import pytest +import requests +import responses +import yaml + +sys.path.append(str(Path(__file__).parent.parent / "utils")) + +import archive_resolve +import wayback +from tidy_conf import links + +TODAY = date(2026, 11, 6) +TANZANIA_SPONSOR = [ + "20211206210540", + "20231209151121", + "20241004214743", + "20250618114933", + "20261018085052", +] + + +def _edition(year: int, start: date, end: date | None = None, conference: str = "PyCon Tanzania", **fields) -> dict: + return {"conference": conference, "year": year, "start": start, "end": end or start, **fields} + + +@pytest.fixture() +def tanzania() -> list[dict]: + """PyCon Tanzania as the data held it: the 2024 edition on the old domain, the 2026 one on the new.""" + return [ + _edition( + 2024, + date(2024, 12, 3), + date(2024, 12, 4), + link="https://pycon.or.tz/", + sponsor="https://pycon.or.tz/sponsor/", + coc="https://pycon.tz/conduct/", + ), + _edition( + 2026, + date(2026, 11, 25), + date(2026, 11, 27), + link="https://pycon.tz/", + sponsor="https://pycon.or.tz/sponsor/", + ), + ] + + +@pytest.fixture(autouse=True) +def _clean_redirect_records(monkeypatch): + monkeypatch.delenv(links.REDIRECT_RECORDS_ENV, raising=False) + links.REDIRECT_RECORDS.clear() + yield + links.REDIRECT_RECORDS.clear() + + +class TestHasYear: + @pytest.mark.parametrize( + "url", + [ + "https://2024.pycon.de/", + "https://pycon.hk/2024/", + "https://pydata.org/global2023/", + "https://pretalx.com/pycon-lithuania-2025/cfp", + "https://ep2024.europython.eu/", + "https://pycon.fr/1999/", + ], + ) + def test_year_urls(self, url): + assert archive_resolve.has_year(url) + + @pytest.mark.parametrize( + "url", + [ + "https://pycon.or.tz/sponsor/", + "https://pycon25.uber.space/en/", + "https://indico.cern.ch/event/1375573/", + "https://pycon.de/", + "https://example.org/12024/", + ], + ) + def test_yearless_urls(self, url): + assert not archive_resolve.has_year(url) + + +class TestSuperseded: + """An edition is superseded once it has ended and the series has a later edition.""" + + def test_old_edition_with_later_one(self, tanzania): + assert archive_resolve.superseded(tanzania, TODAY) == [tanzania[0]] + + def test_latest_edition_is_never_superseded(self, tanzania): + """Even once it has ended.""" + assert archive_resolve.superseded(tanzania, date(2027, 6, 1)) == [tanzania[0]] + + def test_edition_that_has_not_ended(self): + entries = [_edition(2026, date(2026, 11, 10)), _edition(2027, date(2027, 11, 10))] + + assert archive_resolve.superseded(entries, date(2026, 11, 10)) == [] + + def test_later_edition_of_another_series_does_not_count(self): + entries = [_edition(2024, date(2024, 12, 3)), _edition(2026, date(2026, 11, 25), conference="PyCon Kenya")] + + assert archive_resolve.superseded(entries, TODAY) == [] + + +class TestSupersededCandidates: + def test_yearless_live_urls_of_superseded_editions(self, tanzania): + candidates = archive_resolve.superseded_candidates(tanzania, TODAY) + + assert [(c.entry["year"], c.field, c.url, c.reason) for c in candidates] == [ + (2024, "link", "https://pycon.or.tz/", "superseded"), + (2024, "sponsor", "https://pycon.or.tz/sponsor/", "superseded"), + (2024, "coc", "https://pycon.tz/conduct/", "superseded"), + ] + + def test_year_urls_and_archive_links_are_left_alone(self): + entries = [ + _edition( + 2024, + date(2024, 4, 22), + conference="PyCon Germany", + link="https://2024.pycon.de/", + cfp_link="https://web.archive.org/web/20240101000000/https://pycon.de/cfp/", + finaid="https://pycon.de/financial-aid/", + ), + _edition(2025, date(2025, 4, 23), conference="PyCon Germany", link="https://2025.pycon.de/"), + ] + + candidates = archive_resolve.superseded_candidates(entries, TODAY) + + assert [(c.field, c.url) for c in candidates] == [("finaid", "https://pycon.de/financial-aid/")] + + +class TestResolve: + def test_tanzania_2024_gets_the_capture_from_its_own_time(self, tanzania): + candidates = archive_resolve.superseded_candidates(tanzania, TODAY) + + outcomes = archive_resolve.resolve( + [c for c in candidates if c.field == "sponsor"], + lookup=lambda url: TANZANIA_SPONSOR, + delay=0, + ) + + expected = "https://web.archive.org/web/20241004214743/https://pycon.or.tz/sponsor/" + assert tanzania[0]["sponsor"] == expected + assert tanzania[1]["sponsor"] == "https://pycon.or.tz/sponsor/" + assert [(o.new_url, o.detail) for o in outcomes] == [(expected, "capture 20241004214743")] + + def test_one_lookup_per_url(self): + url = "https://pycon.se/" + entries = [_edition(year, date(year, 11, 1), conference="PyCon Sweden", link=url) for year in (2022, 2023)] + candidates = [archive_resolve.Candidate(entry, "link", url, "superseded") for entry in entries] + lookups = [] + + archive_resolve.resolve(candidates, lookup=lambda u: lookups.append(u) or ["20221020000000"], delay=0) + + assert lookups == [url] + # Each edition gets the capture from its own time, from the same listing + assert entries[0]["link"] == "https://web.archive.org/web/20221020000000/https://pycon.se/" + assert entries[1]["link"] == "https://web.archive.org/web/20221020000000/https://pycon.se/" + + def test_field_listed_twice_is_resolved_once(self, tanzania): + candidate = archive_resolve.superseded_candidates(tanzania, TODAY)[0] + + outcomes = archive_resolve.resolve([candidate, candidate], lookup=lambda url: TANZANIA_SPONSOR, delay=0) + + assert len(outcomes) == 1 + + def test_no_capture_keeps_the_url(self, tanzania): + candidate = archive_resolve.superseded_candidates(tanzania, TODAY)[0] + + (outcome,) = archive_resolve.resolve([candidate], lookup=lambda url: ["20261018085052"], delay=0) + + assert outcome.new_url is None + assert outcome.detail == "no capture on or before 2024-12-04" + assert tanzania[0]["link"] == "https://pycon.or.tz/" + + def test_failed_lookup_keeps_the_url(self, tanzania): + candidate = archive_resolve.superseded_candidates(tanzania, TODAY)[0] + + def fail(url): + raise wayback.WaybackError("CDX request returned HTTP 503") + + (outcome,) = archive_resolve.resolve([candidate], lookup=fail, delay=0) + + assert outcome.new_url is None + assert outcome.detail.startswith("lookup failed") + assert tanzania[0]["link"] == "https://pycon.or.tz/" + + +class TestProbe: + """Only a lasting failure counts as broken: replacing a link is not undone.""" + + @pytest.mark.parametrize(("status", "broken"), [(404, True), (410, True), (403, False), (429, False), (500, False)]) + @responses.activate + def test_status(self, status, broken): + responses.add(responses.GET, "https://example.org/", status=status) + + assert links.probe("https://example.org/").broken is broken + + @pytest.mark.parametrize( + ("error", "broken"), + [ + (requests.ConnectionError("name or service not known"), True), + (requests.Timeout("slow"), False), + (requests.exceptions.SSLError("bad certificate"), True), + (requests.exceptions.InvalidURL("bad url"), False), + ], + ) + @responses.activate + def test_errors(self, error, broken): + responses.add(responses.GET, "https://example.org/", body=error) + + result = links.probe("https://example.org/") + + assert result.status is None + assert result.broken is broken + + @responses.activate + def test_redirect_elsewhere(self): + responses.add(responses.GET, "https://pythoncamp.de/", status=301, headers={"Location": "https://barcamps.eu/"}) + responses.add(responses.GET, "https://barcamps.eu/", status=200) + + result = links.probe("https://pythoncamp.de/") + + assert result.final_url == "https://barcamps.eu/" + assert result.redirects_elsewhere + + +class TestCheckLinks: + """The weekly link check: every live URL once, broken ones on ended editions become candidates.""" + + def test_broken_url_on_ended_edition(self): + entry = _edition(2024, date(2024, 12, 3), link="https://pycon.or.tz/") + + candidates = archive_resolve.check_links( + [entry], + TODAY, + request=lambda url: links.Probe(url, url, 404), + ) + + assert [(c.field, c.reason) for c in candidates] == [("link", "broken")] + + def test_broken_url_on_upcoming_edition_is_left_alone(self): + entry = _edition(2026, date(2026, 11, 25), link="https://pycon.tz/") + + candidates = archive_resolve.check_links([entry], TODAY, request=lambda url: links.Probe(url, url, 404)) + + assert candidates == [] + + def test_each_url_requested_once(self, tanzania): + requested = [] + + archive_resolve.check_links( + tanzania, + TODAY, + request=lambda url: requested.append(url) or links.Probe(url, url, 200), + ) + + assert sorted(requested) == [ + "https://pycon.or.tz/", + "https://pycon.or.tz/sponsor/", + "https://pycon.tz/", + "https://pycon.tz/conduct/", + ] + + def test_archive_links_are_not_requested(self): + entry = _edition(2020, date(2020, 1, 1), link="https://web.archive.org/web/20200101000000/https://x.org/") + requested = [] + + archive_resolve.check_links([entry], TODAY, request=lambda url: requested.append(url)) + + assert requested == [] + + def test_checks_coc(self): + """Coc is a URL field like the others.""" + entry = _edition(2024, date(2024, 1, 1), coc="https://pycon.tz/conduct/") + requested = [] + + archive_resolve.check_links( + [entry], TODAY, request=lambda url: requested.append(url) or links.Probe(url, url, 200), + ) + + assert requested == ["https://pycon.tz/conduct/"] + + +class TestFollowMastodon: + def test_moved_accounts_updated_each_checked_once(self): + old, new = "https://mastodon.social/@pycon", "https://fosstodon.org/@pycon" + entries = [{"mastodon": old}, {"mastodon": old}, {"mastodon": "https://fosstodon.org/@other"}, {}] + checked = [] + + updated = archive_resolve.follow_mastodon( + entries, + check=lambda url: checked.append(url) or (new if url == old else None), + ) + + assert updated == 2 + assert checked == [old, "https://fosstodon.org/@other"] + assert [e.get("mastodon") for e in entries] == [new, new, "https://fosstodon.org/@other", None] + + +class TestMain: + """The command edits the data files in place and reports what it changed and kept.""" + + @pytest.fixture() + def repo(self, tmp_path, tanzania): + data = tmp_path / "_data" + data.mkdir() + (data / "conferences.yml").write_text(yaml.safe_dump([tanzania[1]]), encoding="utf-8") + (data / "archive.yml").write_text(yaml.safe_dump([tanzania[0]]), encoding="utf-8") + return tmp_path + + def _run(self, repo, monkeypatch, lookup): + monkeypatch.setattr(archive_resolve.wayback, "captures", lookup) + archive_resolve.main( + [ + "--base", + str(repo), + "--today", + TODAY.isoformat(), + "--no-probe", + "--delay", + "0", + "--changes", + str(repo / "changes.md"), + "--needs", + str(repo / "needs.md"), + ], + ) + + def test_replaces_and_reports(self, repo, monkeypatch): + captures = {"https://pycon.or.tz/sponsor/": TANZANIA_SPONSOR, "https://pycon.or.tz/": ["20241001000000"]} + + self._run(repo, monkeypatch, lambda url: captures.get(url, [])) + + (old,) = yaml.safe_load((repo / "_data" / "archive.yml").read_text(encoding="utf-8")) + assert old["link"] == "https://web.archive.org/web/20241001000000/https://pycon.or.tz/" + assert old["sponsor"] == "https://web.archive.org/web/20241004214743/https://pycon.or.tz/sponsor/" + assert old["coc"] == "https://pycon.tz/conduct/" + (current,) = yaml.safe_load((repo / "_data" / "conferences.yml").read_text(encoding="utf-8")) + assert current["sponsor"] == "https://pycon.or.tz/sponsor/" + + changes = (repo / "changes.md").read_text(encoding="utf-8").splitlines() + assert changes == [ + "- PyCon Tanzania 2024: `link` https://pycon.or.tz/ → " + "https://web.archive.org/web/20241001000000/https://pycon.or.tz/", + "- PyCon Tanzania 2024: `sponsor` https://pycon.or.tz/sponsor/ → " + "https://web.archive.org/web/20241004214743/https://pycon.or.tz/sponsor/", + ] + assert (repo / "needs.md").read_text(encoding="utf-8") == ( + "- [ ] PyCon Tanzania 2024: `coc` https://pycon.tz/conduct/ (superseded) kept: " + "no capture on or before 2024-12-04\n" + ) + + def test_nothing_replaced_leaves_the_files_alone(self, repo, monkeypatch): + before = (repo / "_data" / "archive.yml").read_text(encoding="utf-8") + + self._run(repo, monkeypatch, lambda url: []) + + assert (repo / "_data" / "archive.yml").read_text(encoding="utf-8") == before + assert (repo / "changes.md").read_text(encoding="utf-8") == "" diff --git a/tests/test_import_daily.py b/tests/test_import_daily.py index fbba1d54e7..a7e7ed67ce 100644 --- a/tests/test_import_daily.py +++ b/tests/test_import_daily.py @@ -36,6 +36,7 @@ from tidy_conf.interactive_merge import fuzzy_match from tidy_conf.interactive_merge import merge_conferences from tidy_conf.links import REDUNDANT_LINK_FIELDS +from tidy_conf.links import Probe from tidy_conf.links import drop_redundant_link_fields from tidy_conf.schema import get_schema @@ -185,7 +186,7 @@ def test_sort_leaves_one_entry(self, tmp_path): for name, entries in (("conferences", [copy]), ("archive", [stored]), ("legacy", [])): (data / f"{name}.yml").write_text(yaml.safe_dump(entries), encoding="utf-8") - sort_yaml.sort_data(base=str(tmp_path), skip_links=True) + sort_yaml.sort_data(base=str(tmp_path)) entries = [ entry @@ -330,9 +331,10 @@ def test_list_fields_survive(self, data_files, run_import): class TestCodeOfConductLinks: """coc is a sub-page link like cfp_link, sponsor and finaid.""" - @patch("sort_yaml.get_cache", return_value=(set(), set())) - @patch("sort_yaml.check_link_availability", side_effect=lambda url, *_args, **_kwargs: url) - def test_check_links_checks_coc(self, mock_check_link, mock_get_cache): + def test_link_check_checks_coc(self): + """The weekly link check (utils/archive_resolve.py) requests coc like the other URL fields.""" + import archive_resolve + data = [ { "conference": "PyCon Testland", @@ -343,11 +345,22 @@ def test_check_links_checks_coc(self, mock_check_link, mock_get_cache): "finaid": "https://2026.pycon.test/finaid/", "coc": "https://2026.pycon.test/coc/", "start": date(2026, 6, 1), + "end": date(2026, 6, 1), }, ] - sort_yaml.check_links(data) - fields = [call.kwargs["field"] for call in mock_check_link.call_args_list] - assert fields == ["link", "cfp_link", "sponsor", "finaid", "coc"] + requested = [] + archive_resolve.check_links( + data, + date(2026, 7, 1), + request=lambda url: requested.append(url) or Probe(url, url, 200), + ) + assert requested == [ + "https://2026.pycon.test/", + "https://2026.pycon.test/cfp/", + "https://2026.pycon.test/sponsors/", + "https://2026.pycon.test/finaid/", + "https://2026.pycon.test/coc/", + ] def test_coc_is_a_redundant_link_field(self): assert REDUNDANT_LINK_FIELDS == ("cfp_link", "sponsor", "finaid", "coc") diff --git a/tests/test_integration_comprehensive.py b/tests/test_integration_comprehensive.py index b7d663e066..840d3a002b 100644 --- a/tests/test_integration_comprehensive.py +++ b/tests/test_integration_comprehensive.py @@ -171,7 +171,7 @@ def track_add_latlon(data): mock_split.return_value = (valid_conferences, [], [], []) # Execute sorting pipeline - sort_yaml.sort_data(skip_links=True) + sort_yaml.sort_data() # Verify data transformation chain mock_tidy_dates.assert_called_once() @@ -261,40 +261,6 @@ def test_schema_validation_across_modules(self): processed_conf = Conference(**processed_data) assert processed_conf.conference == "Test Conference" - @patch("sort_yaml.check_link_availability") - @patch("sort_yaml.get_cache") - def test_link_checking_integration(self, mock_get_cache, mock_check_link): - """Test link checking integration with data processing.""" - mock_get_cache.return_value = (set(), set()) - mock_check_link.return_value = "https://updated-link.com" - - # Test data with various link types - test_data = [ - { - "conference": "Conference with Links", - "year": 2025, - "link": "https://original-link.com", - "cfp_link": "https://cfp-link.com", - "sponsor": "https://sponsor-link.com", - "finaid": "https://finaid-link.com", - "coc": "https://coc-link.com", - "start": "2025-06-01", - }, - ] - - with patch("tqdm.tqdm", side_effect=lambda x, total=None: x): - result = sort_yaml.check_links(test_data) - - # All links should be updated - assert result[0]["link"] == "https://updated-link.com" - assert result[0]["cfp_link"] == "https://updated-link.com" - assert result[0]["sponsor"] == "https://updated-link.com" - assert result[0]["finaid"] == "https://updated-link.com" - assert result[0]["coc"] == "https://updated-link.com" - - # Should have checked all 5 links - assert mock_check_link.call_count == 5 - class TestDataConsistency: """Test data consistency across pipeline stages.""" @@ -643,24 +609,6 @@ def test_cfp_priority_logic(self): assert len(filtered) == 1, f"Expected 1 conference, got {len(filtered)}" assert filtered.iloc[0]["conference"] == "CFP Ext In Range" - def test_archive_vs_live_link_logic(self): - """Test logic for handling archive vs live links.""" - test_data = [ - {"conference": "Link Test Conference", "year": 2025, "link": "https://original.com", "start": "2025-06-01"}, - ] - - with patch("sort_yaml.get_cache", return_value=(set(), set())), patch( - "sort_yaml.check_link_availability", - ) as mock_check, patch("sort_yaml.time.sleep"), patch("tqdm.tqdm", side_effect=lambda x, total=None: x): - - # Mock return of archive.org URL - mock_check.return_value = "https://web.archive.org/web/20250101/https://original.com" - - result = sort_yaml.check_links(test_data) - - # Should update to archive URL - assert "archive.org" in result[0]["link"] - def test_timezone_normalization_across_modules(self): """Test timezone handling consistency.""" # Test data with different timezone formats @@ -798,29 +746,6 @@ def test_auto_add_sub_with_real_data(self, temp_data_dir): # Should have added a 'sub' field assert "sub" in result[0], "auto_add_sub should add 'sub' field" - @pytest.mark.skip(reason="Requires `responses` library for proper HTTP-level mocking; see audit item #6") - def test_check_links_with_http_mock(self): - """Test link checking with HTTP-level mocking (not function mocking). - - This addresses audit finding #6: mock at HTTP level, not function level. - - Note: This test should use the `responses` or `httpretty` library - for proper HTTP-level mocking instead of mocking requests.get directly. - The current Mock approach doesn't properly simulate HTTP responses. - - Example using responses: - ```python - import responses - - @responses.activate - def test_check_links_real(): - responses.add(responses.GET, "https://pycon.us", status=200) - result = sort_yaml.check_links(test_data) - assert len(result) == len(test_data) - ``` - """ - # Skipped - needs `responses` library - def test_sort_by_cfp_with_real_conferences(self): """Test sorting actually orders conferences correctly by CFP deadline.""" today = datetime.now(timezone.utc).date() diff --git a/tests/test_link_checking.py b/tests/test_link_checking.py deleted file mode 100644 index 4e3e91df8f..0000000000 --- a/tests/test_link_checking.py +++ /dev/null @@ -1,562 +0,0 @@ -"""Tests for link checking functionality.""" - -import sys -from datetime import date -from pathlib import Path -from unittest.mock import Mock -from unittest.mock import patch - -import requests -import responses - -sys.path.append(str(Path(__file__).parent.parent / "utils")) - -from tidy_conf import links - - -class TestLinkCheckingWithResponses: - """Test link checking using responses library for cleaner HTTP mocking.""" - - @responses.activate - def test_successful_link_check_clean(self): - """Test successful link checking with responses library.""" - test_url = "https://example.com/" # Include trailing slash for normalized URL - responses.add( - responses.GET, - test_url, - status=200, - headers={"Content-Type": "text/html"}, - ) - - test_start = date(2025, 6, 1) - result = links.check_link_availability(test_url, test_start) - - # URL should be returned (possibly with trailing slash normalization) - assert result.rstrip("/") == test_url.rstrip("/") - assert len(responses.calls) == 1 - - @responses.activate - def test_redirect_handling_clean(self): - """Test redirect handling with responses library.""" - original_url = "https://example.com" - redirected_url = "https://example.com/new-page" - - responses.add( - responses.GET, - original_url, - status=301, - headers={"Location": redirected_url}, - ) - responses.add( - responses.GET, - redirected_url, - status=200, - headers={"Content-Type": "text/html"}, - ) - - test_start = date(2025, 6, 1) - - # The actual behavior depends on how requests handles redirects - # By default requests follows redirects, so we should get the final URL - result = links.check_link_availability(original_url, test_start) - - # Result should be the redirected URL - assert redirected_url in result or original_url in result - - @responses.activate - def test_404_triggers_archive_lookup(self): - """Test that 404 triggers archive.org lookup.""" - test_url = "https://example.com/missing" - archive_api_url = "https://archive.org/wayback/available" - - # First request returns 404 - responses.add( - responses.GET, - test_url, - status=404, - ) - - # Archive.org API response - no archive found - responses.add( - responses.GET, - archive_api_url, - json={"archived_snapshots": {}}, - status=200, - ) - - test_start = date(2025, 6, 1) - - with ( - patch("tidy_conf.links.get_cache") as mock_cache, - patch( - "tidy_conf.links.get_cache_location", - ) as mock_cache_location, - ): - mock_cache.return_value = (set(), set()) - mock_cache_file = Mock() - mock_file_handle = Mock() - mock_file_handle.__enter__ = Mock(return_value=mock_file_handle) - mock_file_handle.__exit__ = Mock(return_value=None) - mock_cache_file.open.return_value = mock_file_handle - mock_cache_location.return_value = (mock_cache_file, Mock()) - - result = links.check_link_availability(test_url, test_start) - - # Should return original URL when no archive is found - assert result == test_url - - @responses.activate - def test_archive_found_returns_archive_url(self): - """Test that archive URL is returned when found.""" - test_url = "https://example.com/old-page" - archive_url = "https://web.archive.org/web/20240101/https://example.com/old-page" - archive_api_url = "https://archive.org/wayback/available" - - # First request returns 404 - responses.add( - responses.GET, - test_url, - status=404, - ) - - # Archive.org API returns a valid snapshot - responses.add( - responses.GET, - archive_api_url, - json={ - "archived_snapshots": { - "closest": { - "available": True, - "url": archive_url, - }, - }, - }, - status=200, - ) - - test_start = date(2025, 6, 1) - - with patch("tidy_conf.links.tqdm.write"): - result = links.check_link_availability(test_url, test_start) - - # Should return the archive URL - assert result == archive_url - - @responses.activate - def test_timeout_handling(self): - """Test handling of timeout errors.""" - test_url = "https://slow-server.com" - - # Simulate timeout - responses.add( - responses.GET, - test_url, - body=requests.exceptions.Timeout("Connection timed out"), - ) - - test_start = date(2025, 6, 1) - - # Should handle timeout gracefully - result = links.check_link_availability(test_url, test_start) - - # Should return original URL on timeout - assert result == test_url - - @responses.activate - def test_ssl_error_handling(self): - """Test handling of SSL certificate errors.""" - test_url = "https://invalid-cert.com" - - # Simulate SSL error - responses.add( - responses.GET, - test_url, - body=requests.exceptions.SSLError("SSL certificate verify failed"), - ) - - test_start = date(2025, 6, 1) - - result = links.check_link_availability(test_url, test_start) - - # Should return original URL on SSL error - assert result == test_url - - @responses.activate - def test_multiple_links_batch(self): - """Test checking multiple links.""" - # Use trailing slashes for normalized URLs - urls = [ - "https://pycon.us/", - "https://djangocon.us/", - "https://europython.eu/", - ] - - for url in urls: - responses.add( - responses.GET, - url, - status=200, - ) - - test_start = date(2025, 6, 1) - - results = [links.check_link_availability(url, test_start) for url in urls] - - # All should succeed - compare without trailing slashes for flexibility - assert len(results) == 3 - for url, result in zip(urls, results, strict=False): - assert result.rstrip("/") == url.rstrip("/") - - @responses.activate - def test_archive_org_url_passthrough(self): - """Test that archive.org URLs are returned unchanged.""" - archive_url = "https://web.archive.org/web/20240101/https://example.com" - - test_start = date(2025, 6, 1) - - # Should not make any HTTP requests - result = links.check_link_availability(archive_url, test_start) - - assert result == archive_url - assert len(responses.calls) == 0 # No HTTP calls made - - -class TestLinkAvailability: - """Test link availability checking functionality.""" - - @patch("tidy_conf.links.requests.get") - def test_successful_link_check(self, mock_get): - """Test successful link checking.""" - # Mock successful response - mock_response = Mock() - mock_response.status_code = 200 - mock_response.url = "https://example.com" - mock_get.return_value = mock_response - - test_url = "https://example.com" - test_start = date(2025, 6, 1) - - result = links.check_link_availability(test_url, test_start) - - assert result == test_url - mock_get.assert_called_once() - - @patch("tidy_conf.links.requests.get") - def test_link_redirect_same_domain(self, mock_get): - """Test handling of redirects within the same domain.""" - # Mock response with redirect - mock_response = Mock() - mock_response.status_code = 200 - mock_response.url = "https://example.com/redirected" - mock_get.return_value = mock_response - - test_url = "https://example.com" - test_start = date(2025, 6, 1) - - result = links.check_link_availability(test_url, test_start) - - # Should return the redirected URL - assert result == "https://example.com/redirected" - - @patch("tidy_conf.links.requests.get") - def test_link_check_with_query_string_warning(self, mock_get): - """Test warning for redirects that add query strings.""" - # Mock response with query string redirect - mock_response = Mock() - mock_response.status_code = 200 - mock_response.url = "https://example.com/page?session=123" - mock_get.return_value = mock_response - - test_url = "https://example.com/page" - test_start = date(2025, 6, 1) - - with patch("tidy_conf.links.tqdm.write"): # Suppress tqdm output - result = links.check_link_availability(test_url, test_start) - - # Should return original URL due to query string warning - assert result == test_url - - @patch("tidy_conf.links.requests.get") - def test_link_check_404_error(self, mock_get): - """Test handling of 404 errors.""" - # Mock 404 response for original URL - mock_404_response = Mock() - mock_404_response.status_code = 404 - mock_404_response.url = "https://example.com/not-found" # Set URL for urlparse - - # Mock archive.org response (no archive found) - mock_archive_response = Mock() - mock_archive_response.status_code = 200 - mock_archive_response.json.return_value = {"archived_snapshots": {}} - - # Set up side effects: first call returns 404, second returns empty archive response - mock_get.side_effect = [mock_404_response, mock_archive_response] - - test_url = "https://example.com/not-found" - test_start = date(2025, 6, 1) - - with ( - patch("tidy_conf.links.tqdm.write"), - patch("tidy_conf.links.attempt_archive_url"), - patch( - "tidy_conf.links.get_cache", - ) as mock_get_cache, - patch("tidy_conf.links.get_cache_location") as mock_cache_location, - patch( - "builtins.open", - create=True, - ), - ): - # Mock cache returns empty sets - mock_get_cache.return_value = (set(), set()) - # Mock cache file paths with proper context manager support - mock_cache_file = Mock() - mock_file_handle = Mock() - mock_file_handle.__enter__ = Mock(return_value=mock_file_handle) - mock_file_handle.__exit__ = Mock(return_value=None) - mock_cache_file.open.return_value = mock_file_handle - mock_cache_location.return_value = (mock_cache_file, Mock()) - - result = links.check_link_availability(test_url, test_start) - - # Should return original URL when no archive is found - assert result == test_url - - @patch("tidy_conf.links.requests.get") - def test_archived_link_found(self, mock_get): - """Test finding an archived version of a link.""" - # Mock 404 response for original link - mock_404_response = Mock() - mock_404_response.status_code = 404 - mock_404_response.url = "https://example.com" # Set URL for urlparse - - # Mock successful archive.org response - mock_archive_response = Mock() - mock_archive_response.status_code = 200 - mock_archive_response.json.return_value = { - "archived_snapshots": { - "closest": {"available": True, "url": "https://web.archive.org/web/20250101000000/https://example.com"}, - }, - } - - mock_get.side_effect = [mock_404_response, mock_archive_response] - - test_url = "https://example.com" - test_start = date(2025, 6, 1) - - with patch("tidy_conf.links.tqdm.write"): - result = links.check_link_availability(test_url, test_start) - - # Should return archived URL - assert result == "https://web.archive.org/web/20250101000000/https://example.com" - - def test_already_archived_url(self): - """Test that already archived URLs are returned as-is.""" - archived_url = "https://web.archive.org/web/20250101000000/https://example.com" - test_start = date(2025, 6, 1) - - result = links.check_link_availability(archived_url, test_start) - - assert result == archived_url - - @patch("tidy_conf.links.requests.get") - def test_connection_error_handling(self, mock_get): - """Test handling of connection errors.""" - # Mock connection error - mock_get.side_effect = requests.exceptions.ConnectionError("Connection failed") - - test_url = "https://example.com" - test_start = date(2025, 6, 1) - - with patch("tidy_conf.links.requests.get") as mock_archive_get: - # Mock archive.org response failure - mock_archive_get.side_effect = requests.exceptions.ConnectionError("Archive connection failed") - - result = links.check_link_availability(test_url, test_start) - - # Should return original URL when both checks fail - assert result == test_url - - -class TestCaching: - """Test caching functionality.""" - - def test_get_cache_location(self): - """Test cache location paths.""" - cache_file, cache_file_archived = links.get_cache_location() - - assert isinstance(cache_file, Path) - assert isinstance(cache_file_archived, Path) - assert cache_file.name == "no_archive.txt" - assert cache_file_archived.name == "archived_links.txt" - - def test_get_cache_creates_missing_directory(self, tmp_path, monkeypatch): - """The gitignored .tmp directory is absent in a fresh checkout; get_cache must not crash.""" - monkeypatch.chdir(tmp_path) - - cache, cache_archived = links.get_cache() - - assert cache == set() - assert cache_archived == set() - assert (tmp_path / "utils" / "tidy_conf" / "data" / ".tmp" / "no_archive.txt").is_file() - assert (tmp_path / "utils" / "tidy_conf" / "data" / ".tmp" / "archived_links.txt").is_file() - - @patch("tidy_conf.links.Path.read_text") - @patch("tidy_conf.links.Path.touch") - def test_get_cache(self, mock_touch, mock_read_text): - """Test cache loading.""" - mock_read_text.side_effect = ["https://example.com\n", "https://archived.com\n"] - - cache, cache_archived = links.get_cache() - - assert isinstance(cache, set) - assert isinstance(cache_archived, set) - assert "https://example.com" in cache - assert "https://archived.com" in cache_archived - - @patch("tidy_conf.links.get_cache") - def test_cached_link_no_archive(self, mock_get_cache): - """Test behavior with cached link marked as no archive.""" - # Mock cache with the URL already marked as no archive - mock_get_cache.return_value = ({"https://example.com"}, set()) - - test_url = "https://example.com" - test_start = date(2025, 6, 1) - - result = links.check_link_availability(test_url, test_start) - - # Should return original URL without making HTTP requests - assert result == test_url - - -class TestArchiveAttempt: - """Test archive attempt functionality.""" - - @patch("tidy_conf.links.requests.get") - def test_successful_archive_attempt(self, mock_get): - """Test successful archive attempt.""" - # Mock successful archive response - mock_response = Mock() - mock_response.status_code = 200 - mock_get.return_value = mock_response - - test_url = "https://example.com" - - with ( - patch("tidy_conf.links.get_cache_location") as mock_cache_location, - patch( - "tidy_conf.links.get_cache", - ) as mock_get_cache, - patch("builtins.open", create=True) as mock_open, - ): - # Mock cache file paths with proper context manager support - mock_cache_file = Mock() - mock_cache_archived = Mock() - mock_file_handle = Mock() - mock_file_handle.__enter__ = Mock(return_value=mock_file_handle) - mock_file_handle.__exit__ = Mock(return_value=None) - mock_cache_archived.open.return_value = mock_file_handle - mock_cache_location.return_value = (mock_cache_file, mock_cache_archived) - # Mock cache returns empty sets - mock_get_cache.return_value = (set(), set()) - - # Mock file operations - mock_file = Mock() - mock_open.return_value.__enter__.return_value = mock_file - - links.attempt_archive_url(test_url) - - # Should make a request to web.archive.org/save/ - mock_get.assert_called_once() - call_args = mock_get.call_args[0][0] - assert "web.archive.org/save/" in call_args - assert test_url in call_args - - @patch("tidy_conf.links.requests.get") - def test_archive_attempt_failure(self, mock_get): - """Test archive attempt failure.""" - # Mock failed archive response - mock_get.side_effect = requests.exceptions.RequestException("Archive failed") - - test_url = "https://example.com" - - # Should not raise exception, just handle gracefully - links.attempt_archive_url(test_url) - - @patch("tidy_conf.links.get_cache") - def test_already_archived_url_skip(self, mock_get_cache): - """Test skipping archive attempt for already archived URLs.""" - # Mock cache showing URL is already archived - mock_get_cache.return_value = (set(), {"https://example.com"}) - - test_url = "https://example.com" - - with patch("tidy_conf.links.requests.get") as mock_get: - links.attempt_archive_url(test_url) - - # Should not make any HTTP requests - mock_get.assert_not_called() - - -class TestDateLogic: - """Test date-based logic in link checking.""" - - @patch("tidy_conf.links.requests.get") - def test_old_conference_archive_check(self, mock_get): - """Test that old conferences get archived automatically.""" - # Mock archive.org response (no archive found for old conference) - mock_archive_response = Mock() - mock_archive_response.status_code = 200 - mock_archive_response.json.return_value = {"archived_snapshots": {}} - mock_get.return_value = mock_archive_response - - # Very old date (more than 5 years ago) - old_start = date(2019, 6, 1) - test_url = "https://example.com" - - with ( - patch("tidy_conf.links.attempt_archive_url") as mock_archive, - patch( - "tidy_conf.links.get_cache", - ) as mock_get_cache, - patch("tidy_conf.links.get_cache_location") as mock_cache_location, - patch( - "builtins.open", - create=True, - ), - ): - # Mock cache returns to avoid cache hits - mock_get_cache.return_value = (set(), set()) - # Mock cache file paths with proper context manager support - mock_cache_file = Mock() - mock_file_handle = Mock() - mock_file_handle.__enter__ = Mock(return_value=mock_file_handle) - mock_file_handle.__exit__ = Mock(return_value=None) - mock_cache_file.open.return_value = mock_file_handle - mock_cache_location.return_value = (mock_cache_file, Mock()) - - result = links.check_link_availability(test_url, old_start) - - # Should attempt to archive old conferences - mock_archive.assert_called_once_with(test_url, set()) - assert result == test_url - - @patch("tidy_conf.links.requests.get") - def test_recent_conference_no_archive(self, mock_get): - """Test that recent conferences don't get auto-archived.""" - # Mock successful link response - mock_response = Mock() - mock_response.status_code = 200 - mock_response.url = "https://example.com" - mock_get.return_value = mock_response - - # Recent date - recent_start = date(2025, 6, 1) - test_url = "https://example.com" - - with patch("tidy_conf.links.attempt_archive_url") as mock_archive: - result = links.check_link_availability(test_url, recent_start) - - # Should not attempt to archive recent conferences - assert not mock_archive.called - assert result == test_url diff --git a/tests/test_link_redirects.py b/tests/test_link_redirects.py index e60e9f8b3f..1b15408f53 100644 --- a/tests/test_link_redirects.py +++ b/tests/test_link_redirects.py @@ -1,11 +1,12 @@ -"""Tests for redirects to a different site during link checking. +"""Tests for redirects to a different site during the weekly link check. A conference link that now redirects to another site (a lapsed domain taken over by a casino, a series that moved to a barcamp platform) must not be treated as healthy, swapped for a Wayback copy or replaced by the target. -check_link_availability keeps the original URL and records the redirect for -a person to review. Redirects within the same site behave as before, and so -do short links such as forms.gle that redirect to another site by design. +utils/archive_resolve.py keeps the URL as it is and records the redirect for +the desk to review. Redirects within the same site are not recorded, and +neither are short links such as forms.gle that redirect to another site by +design. """ import json @@ -15,7 +16,6 @@ from datetime import timedelta from datetime import timezone from pathlib import Path -from unittest.mock import patch import pytest import responses @@ -23,11 +23,9 @@ sys.path.append(str(Path(__file__).parent.parent / "utils")) -import sort_yaml +import archive_resolve from tidy_conf import links -# Recent enough to be checked (younger than five years) but in the past, so a -# healthy link does not trigger a Wayback save request. TODAY = datetime.now(tz=timezone.utc).date() PAST_START = TODAY - timedelta(days=60) @@ -46,8 +44,11 @@ def _redirect(source: str, target: str, status: int = 200, redirect_status: int responses.add(responses.GET, target, status=status) -def _check(url: str, start: date = PAST_START, **context) -> str: - return links.check_link_availability(url, start, cache=set(), cache_archived=set(), **context) +def _check(url: str, start: date = PAST_START, conference=None, year=None, field="link") -> dict: + """Run the weekly link check over one entry holding the URL, and return the entry.""" + entry = {"conference": conference, "year": year, field: url, "start": start, "end": start} + archive_resolve.check_links([entry], TODAY) + return entry class TestRegistrableDomain: @@ -180,9 +181,9 @@ def test_pythoncamp_to_barcamps_kept_and_recorded(self): url = "https://pythoncamp.de/" _redirect(url, "https://barcamps.eu/pythoncamp/") - result = _check(url, conference="PythonCamp Germany", year=2026, field="link") + entry = _check(url, conference="PythonCamp Germany", year=2026, field="link") - assert result == url + assert entry["link"] == url assert len(links.REDIRECT_RECORDS) == 1 record = links.REDIRECT_RECORDS[0] assert record["source"] == "link-check" @@ -203,9 +204,9 @@ def test_pytennessee_to_casino_kept_and_recorded(self): url = "https://pytennessee.org/" _redirect(url, "https://infojudislotonline.com/") - result = _check(url, conference="PyTennessee", year=2025, field="link") + entry = _check(url, conference="PyTennessee", year=2025, field="link") - assert result == url + assert entry["link"] == url assert [r["payload"]["final_url"] for r in links.REDIRECT_RECORDS] == ["https://infojudislotonline.com/"] @responses.activate @@ -214,20 +215,20 @@ def test_shared_host_tenant_to_another_tenant_kept_and_recorded(self): url = "https://pyconx.github.io/2026/" _redirect(url, "https://squatter.github.io/") - result = _check(url, conference="PyCon X", year=2026, field="link") + entry = _check(url, conference="PyCon X", year=2026, field="link") - assert result == url + assert entry["link"] == url assert [r["payload"]["final_url"] for r in links.REDIRECT_RECORDS] == ["https://squatter.github.io/"] @responses.activate - def test_no_wayback_lookup_or_save(self): - """Neither a future start (archive save) nor an error status (archive lookup) touches the Wayback Machine.""" + def test_no_wayback_request(self): + """A redirect to another site, even one that ends in an error, never touches the Wayback Machine.""" url = "https://pytennessee.org/" _redirect(url, "https://infojudislotonline.com/", status=404) - result = _check(url, start=TODAY + timedelta(days=60), field="cfp_link") + entry = _check(url, start=TODAY + timedelta(days=60), field="cfp_link") - assert result == url + assert entry["cfp_link"] == url assert [call.request.url for call in responses.calls] == [url, "https://infojudislotonline.com/"] assert links.REDIRECT_RECORDS[0]["payload"]["status"] == 404 assert links.REDIRECT_RECORDS[0]["payload"]["field"] == "cfp_link" @@ -237,20 +238,30 @@ def test_without_context_uses_url_as_title(self): url = "https://pythoncamp.de/" _redirect(url, "https://barcamps.eu/pythoncamp/") - assert _check(url) == url + _check(url) record = links.REDIRECT_RECORDS[0] assert record["title"] == url assert record["year"] is None assert record["payload"]["conference"] is None - assert record["payload"]["field"] is None + + @responses.activate + def test_url_shared_by_editions_requested_once_recorded_for_each(self): + url = "https://pythoncamp.de/" + _redirect(url, "https://barcamps.eu/pythoncamp/") + entries = [ + {"conference": "PythonCamp Germany", "year": year, "link": url, "start": PAST_START, "end": PAST_START} + for year in (2025, 2026) + ] + + archive_resolve.check_links(entries, TODAY) + + assert len(responses.calls) == 2 + assert [r["year"] for r in links.REDIRECT_RECORDS] == [2025, 2026] class TestExpectedRedirect: """Short links redirect to another site by design and are not recorded.""" - FORMS_URL = "https://forms.gle/iiFnW8oZ8p9do4TR7" - FORMS_TARGET = "https://docs.google.com/forms/d/e/1FAIpQLSe6/viewform?usp=send_form" - @pytest.mark.parametrize( ("url", "final_url"), [ @@ -279,35 +290,24 @@ def test_expected(self, url, final_url): def test_not_expected(self, url, final_url): assert not links.is_expected_redirect(url, final_url) + FORMS_URL = "https://forms.gle/iiFnW8oZ8p9do4TR7" + FORMS_TARGET = "https://docs.google.com/forms/d/e/1FAIpQLSe6/viewform?usp=send_form" + @responses.activate def test_forms_gle_kept_and_not_recorded(self): _redirect(self.FORMS_URL, self.FORMS_TARGET, redirect_status=302) - result = _check(self.FORMS_URL, conference="PyCon Netherlands", year=2026, field="cfp_link") + entry = _check(self.FORMS_URL, conference="PyCon Netherlands", year=2026, field="cfp_link") - assert result == self.FORMS_URL + assert entry["cfp_link"] == self.FORMS_URL assert links.REDIRECT_RECORDS == [] assert len(responses.calls) == 2 - @responses.activate - def test_forms_gle_upcoming_saved_to_wayback(self, tmp_path, monkeypatch): - """An upcoming short link is still saved to the Wayback Machine like any healthy link.""" - monkeypatch.chdir(tmp_path) - _redirect(self.FORMS_URL, self.FORMS_TARGET, redirect_status=302) - save_url = "https://web.archive.org/save/" + self.FORMS_URL - responses.add(responses.GET, save_url, status=200) - - result = _check(self.FORMS_URL, start=TODAY + timedelta(days=60), field="cfp_link") - - assert result == self.FORMS_URL - assert links.REDIRECT_RECORDS == [] - assert [call.request.url for call in responses.calls][-1] == save_url - @responses.activate def test_forms_gle_to_other_site_recorded(self): _redirect("https://forms.gle/abc", "https://infojudislotonline.com/", redirect_status=302) - assert _check("https://forms.gle/abc", field="cfp_link") == "https://forms.gle/abc" + assert _check("https://forms.gle/abc", field="cfp_link")["cfp_link"] == "https://forms.gle/abc" assert [r["payload"]["final_url"] for r in links.REDIRECT_RECORDS] == ["https://infojudislotonline.com/"] @responses.activate @@ -325,25 +325,25 @@ def test_no_queue_line_for_short_link(self, tmp_path, monkeypatch): class TestSameSiteRedirect: - """Redirects within the same site keep today's behaviour and are not recorded.""" + """Redirects within the same site are healthy: not recorded, and the URL is kept.""" @responses.activate - def test_same_host_redirect_followed(self): + def test_same_host_redirect_not_recorded(self): _redirect("https://2026.pycon.it/", "https://2026.pycon.it/en") - result = _check("https://2026.pycon.it/", conference="PyCon Italia", year=2026, field="link") + entry = _check("https://2026.pycon.it/", conference="PyCon Italia", year=2026, field="link") - assert result == "https://2026.pycon.it/en" + assert entry["link"] == "https://2026.pycon.it/" assert links.REDIRECT_RECORDS == [] @responses.activate - def test_same_registrable_domain_kept_as_before(self): - """ug.pycon.org to africa.pycon.org is a healthy link, as it was before.""" + def test_same_registrable_domain_kept(self): + """ug.pycon.org to africa.pycon.org is a healthy link.""" _redirect("https://ug.pycon.org/", "https://africa.pycon.org/") - result = _check("https://ug.pycon.org/", conference="PyCon Uganda", year=2026, field="link") + entry = _check("https://ug.pycon.org/", conference="PyCon Uganda", year=2026, field="link") - assert result == "https://ug.pycon.org/" + assert entry["link"] == "https://ug.pycon.org/" assert links.REDIRECT_RECORDS == [] assert len(responses.calls) == 2 @@ -420,31 +420,3 @@ def test_no_file_without_env(self, tmp_path, monkeypatch): assert len(links.REDIRECT_RECORDS) == 1 assert not list(tmp_path.glob("*.jsonl")) - - -class TestCheckLinksContext: - """sort_yaml.check_links passes the conference, year and field to the link check.""" - - @patch("sort_yaml.get_cache", return_value=(set(), set())) - @patch("sort_yaml.check_link_availability", side_effect=lambda url, *_args, **_kwargs: url) - def test_context_passed(self, mock_check_link, mock_get_cache): - data = [ - { - "conference": "PythonCamp Germany", - "year": 2026, - "link": "https://pythoncamp.de/", - "sponsor": "https://pythoncamp.de/sponsors/", - "start": PAST_START, - }, - ] - - sort_yaml.check_links(data) - - contexts = [ - (call.args[0], call.kwargs["conference"], call.kwargs["year"], call.kwargs["field"]) - for call in mock_check_link.call_args_list - ] - assert contexts == [ - ("https://pythoncamp.de/", "PythonCamp Germany", 2026, "link"), - ("https://pythoncamp.de/sponsors/", "PythonCamp Germany", 2026, "sponsor"), - ] diff --git a/tests/test_main.py b/tests/test_main.py index 7b32e81a42..dd263a107d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -38,9 +38,9 @@ def test_main_pipeline_success(self, mock_logger, mock_official, mock_organizer, mock_organizer.assert_called_once() assert mock_sort.call_count == 2 # Called twice in pipeline - # Verify sort_data called with skip_links=True + # The sort has no link checking to skip for call in mock_sort.call_args_list: - assert call[1]["skip_links"] is True + assert call == ((), {}) # Verify logging calls assert mock_logger_instance.info.call_count >= 7 # Start + 4 steps + completion messages @@ -294,8 +294,8 @@ def track_sort(*args, **kwargs): @patch("main.organizer_updater") @patch("main.official_updater") @patch("main.get_tqdm_logger") - def test_skip_links_parameter(self, mock_logger, mock_official, mock_organizer, mock_sort): - """Test that skip_links=True is passed to sort_data calls.""" + def test_sort_called_without_arguments(self, mock_logger, mock_official, mock_organizer, mock_sort): + """The sort runs offline, so main passes it no link-checking option.""" mock_logger_instance = Mock() mock_logger.return_value = mock_logger_instance @@ -305,11 +305,9 @@ def test_skip_links_parameter(self, mock_logger, mock_official, mock_organizer, main.main() - # Verify both sort_data calls use skip_links=True assert mock_sort.call_count == 2 for call in mock_sort.call_args_list: - assert "skip_links" in call[1] - assert call[1]["skip_links"] is True + assert call == ((), {}) class TestErrorScenarios: diff --git a/tests/test_mastodon_migration.py b/tests/test_mastodon_migration.py index f0111717fc..daeea294bd 100644 --- a/tests/test_mastodon_migration.py +++ b/tests/test_mastodon_migration.py @@ -359,12 +359,12 @@ def test_moved_field_missing_url(self): class TestCheckMastodonMigrationIntegration: - """Integration tests for Mastodon migration in check_links.""" + """Integration tests for following Mastodon migrations in the weekly link check.""" @responses.activate - def test_check_links_updates_mastodon(self): - """Test that check_links updates migrated Mastodon accounts.""" - from sort_yaml import check_links + def test_follow_mastodon_updates_migrated_account(self): + """A migrated account is replaced by its new address.""" + from archive_resolve import follow_mastodon old_url = "https://old-instance.social/@pycon" new_url = "https://new-instance.social/@pycon" @@ -406,21 +406,19 @@ def test_check_links_updates_mastodon(self): "year": 2025, "link": "https://example.com", "mastodon": old_url, - # sort_data runs tidy_dates before check_links, so start is a date "start": date(2025, 6, 1), }, ] - with patch("tidy_conf.links.time.sleep"), patch("sort_yaml.get_cache") as mock_cache: - mock_cache.return_value = (set(), set()) - result = check_links(data) + with patch("tidy_conf.links.time.sleep"): + assert follow_mastodon(data) == 1 - assert result[0]["mastodon"] == new_url + assert data[0]["mastodon"] == new_url @responses.activate - def test_check_links_preserves_non_migrated(self): - """Test that check_links preserves non-migrated Mastodon accounts.""" - from sort_yaml import check_links + def test_follow_mastodon_preserves_non_migrated(self): + """An account that did not move is kept.""" + from archive_resolve import follow_mastodon url = "https://fosstodon.org/@pycon" @@ -449,13 +447,10 @@ def test_check_links_preserves_non_migrated(self): "year": 2025, "link": "https://example.com", "mastodon": url, - # sort_data runs tidy_dates before check_links, so start is a date "start": date(2025, 6, 1), }, ] - with patch("sort_yaml.get_cache") as mock_cache: - mock_cache.return_value = (set(), set()) - result = check_links(data) + assert follow_mastodon(data) == 0 - assert result[0]["mastodon"] == url + assert data[0]["mastodon"] == url diff --git a/tests/test_merge_data.py b/tests/test_merge_data.py index d48ad44b37..1a15b649be 100644 --- a/tests/test_merge_data.py +++ b/tests/test_merge_data.py @@ -445,7 +445,7 @@ def now(cls, tz=None): def sort_on(day): """Run the repository's sort as it would run on the given day.""" with frozen_on(day): - merge_data.sort_data(skip_links=True) + merge_data.sort_data() def conflicted(): diff --git a/tests/test_sort_yaml_enhanced.py b/tests/test_sort_yaml_enhanced.py index dbb1f37053..e9e0929803 100644 --- a/tests/test_sort_yaml_enhanced.py +++ b/tests/test_sort_yaml_enhanced.py @@ -1,5 +1,6 @@ """Enhanced tests for sort_yaml functionality to increase coverage.""" +import inspect import sys from datetime import datetime from pathlib import Path @@ -561,79 +562,6 @@ def test_split_data_boundary_dates(self): assert total_processed == 1 -class TestCheckLinks: - """Test link checking functionality.""" - - @patch("sort_yaml.get_cache") - @patch("sort_yaml.check_link_availability") - @patch("sort_yaml.time.sleep") - def test_check_links_basic(self, mock_sleep, mock_check_link, mock_get_cache): - """Test basic link checking functionality.""" - mock_get_cache.return_value = (set(), set()) - mock_check_link.return_value = "https://updated.com" - - data = [ - { - "conference": "Test Conference", - "year": 2025, - "link": "https://original.com", - "cfp_link": "https://cfp-original.com", - "start": "2025-06-01", - }, - ] - - with patch("tqdm.tqdm", side_effect=lambda x, total=None: x): - result = sort_yaml.check_links(data) - - # Should have updated links - assert result[0]["link"] == "https://updated.com" - assert result[0]["cfp_link"] == "https://updated.com" - assert mock_check_link.call_count >= 2 - - @patch("sort_yaml.get_cache") - @patch("sort_yaml.check_link_availability") - @patch("sort_yaml.time.sleep") - def test_check_links_archive_org_handling(self, mock_sleep, mock_check_link, mock_get_cache): - """Test special handling of archive.org links.""" - mock_get_cache.return_value = (set(), set()) - - # Mock check_link_availability to return archive.org URL - mock_check_link.return_value = "https://web.archive.org/web/20250101000000/https://original.com" - - data = [{"conference": "Test Conference", "year": 2025, "link": "https://original.com", "start": "2025-06-01"}] - - with patch("tqdm.tqdm", side_effect=lambda x, total=None: x): - result = sort_yaml.check_links(data) - - # Should have updated to archive.org link and called sleep - assert "archive.org" in result[0]["link"] - mock_sleep.assert_called_with(0.5) - - @patch("sort_yaml.get_cache") - @patch("sort_yaml.check_link_availability") - def test_check_links_missing_keys(self, mock_check_link, mock_get_cache): - """Test link checking with missing keys.""" - mock_get_cache.return_value = (set(), set()) - mock_check_link.return_value = "https://updated.com" - - data = [ - { - "conference": "Test Conference", - "year": 2025, - "link": "https://original.com", - # Missing cfp_link, sponsor, finaid - "start": "2025-06-01", - }, - ] - - with patch("tqdm.tqdm", side_effect=lambda x, total=None: x): - result = sort_yaml.check_links(data) - - # Should only check the link that exists - assert mock_check_link.call_count == 1 - assert result[0]["link"] == "https://updated.com" - - class TestSortDataIntegration: """Test the main sort_data function integration.""" @@ -665,39 +593,27 @@ def test_sort_data_refuses_to_write_on_validation_errors(self, tmp_path): before = conferences.read_bytes() with pytest.raises(ValueError, match="1 conferences failed validation"): - sort_yaml.sort_data(base=str(tmp_path), skip_links=True) + sort_yaml.sort_data(base=str(tmp_path)) assert conferences.read_bytes() == before assert not (data_dir / "archive.yml").exists() -class TestCommandLineInterface: - """Test command line interface for sort_yaml.""" - - @patch("sort_yaml.sort_data") - def test_cli_default_arguments(self, mock_sort_data): - """Test CLI with default arguments.""" - with patch("sys.argv", ["sort_yaml.py"]): - # Import and test argument parsing - import argparse +class TestOffline: + """The sort never touches the network: links are checked by utils/archive_resolve.py.""" - parser = argparse.ArgumentParser() - parser.add_argument("--skip_links", action="store_true", default=False) + def test_no_link_checking_in_the_sort(self): + assert not hasattr(sort_yaml, "check_links") + assert "skip_links" not in inspect.signature(sort_yaml.sort_data).parameters - args = parser.parse_args([]) - assert args.skip_links is False - - @patch("sort_yaml.sort_data") - def test_cli_skip_links_argument(self, mock_sort_data): - """Test CLI with --skip_links argument.""" - with patch("sys.argv", ["sort_yaml.py", "--skip_links"]): - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("--skip_links", action="store_true", default=False) - - args = parser.parse_args(["--skip_links"]) - assert args.skip_links is True + def test_sort_makes_no_requests(self, tmp_path): + """A full sort with geocoding mocked out sends no HTTP request.""" + with patch("requests.get", side_effect=AssertionError("network used")), patch( + "sort_yaml.add_latlon", + side_effect=lambda data: data, + ): + (tmp_path / "_data").mkdir() + sort_yaml.sort_data(base=str(tmp_path)) class TestErrorHandlingAndEdgeCases: @@ -735,19 +651,6 @@ def test_tidy_dates_empty_data(self): assert result == [] - def test_check_links_empty_data(self): - """Test check links with empty data.""" - with ( - patch("sort_yaml.get_cache", return_value=(set(), set())), - patch( - "tqdm.tqdm", - side_effect=lambda x, total=None: x, - ), - ): - result = sort_yaml.check_links([]) - - assert result == [] - @pytest.mark.skip(reason="Test requires complex Path mock with context manager - covered by real integration tests") def test_sort_data_yaml_error_handling(self): """Test sort_data handles YAML errors gracefully.""" diff --git a/utils/archive_resolve.py b/utils/archive_resolve.py new file mode 100644 index 0000000000..7155159976 --- /dev/null +++ b/utils/archive_resolve.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Point old editions' links to the archived page that shows that edition. + +A series that serves every edition from one URL shows only its newest edition +there, so an old edition's link ends up showing a later one. A link that is +gone shows nothing at all. This script replaces both with a Wayback Machine +capture from the edition's own time, and does the weekly link check: + +1. **Superseded links.** An edition is superseded once it has ended and a later + edition of the same ``conference`` is in the data. Each of its URL fields + (``link``, ``cfp_link``, ``sponsor``, ``finaid``, ``coc``) that holds no + four-digit year and is not an archive link already is a candidate. +2. **Link check** (skipped with ``--no-probe``). Every live URL in the data is + requested once. A URL that redirects to another site is queued for the desk + (``DESK_QUEUE_OUT``), as before. A URL that is gone (see ``Probe.broken``) + on an edition that has ended is a candidate. +3. **Replacement.** For each candidate, one CDX request lists the page's + captures. The last capture on or before the edition's ``end`` plus 14 days + (``wayback.pick_capture``) replaces the URL, with its full timestamp. When + there is none, or the lookup fails, the URL stays as it is and is reported. +4. **Mastodon.** Accounts that moved are followed to their new address. + +Year URLs (``2024.pycon.de``), the editions that are not superseded and links +that are archive links already are never touched. + +Run the sort afterwards: this script edits entries in place and leaves the +ordering to ``utils/sort_yaml.py``. + +Usage:: + + python utils/archive_resolve.py --changes changes.md --needs needs.md + python utils/archive_resolve.py --no-probe # superseded links only, no link check +""" + +import argparse +import re +import sys +import time +from collections import defaultdict +from collections.abc import Callable +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import date +from datetime import datetime +from datetime import timezone +from pathlib import Path + +import yaml + +try: + import wayback + from tidy_conf import write_conference_yaml + from tidy_conf.links import Probe + from tidy_conf.links import check_mastodon_migration + from tidy_conf.links import probe + from tidy_conf.links import record_redirect +except ImportError: + from . import wayback + from .tidy_conf import write_conference_yaml + from .tidy_conf.links import Probe + from .tidy_conf.links import check_mastodon_migration + from .tidy_conf.links import probe + from .tidy_conf.links import record_redirect + +DATA_FILES = ("conferences.yml", "archive.yml", "legacy.yml") +URL_FIELDS = ("link", "cfp_link", "sponsor", "finaid", "coc") +# A four-digit year that is not part of a longer number +YEAR = re.compile(r"(? date | None: + """Read a date from a data value. + + Parameters + ---------- + value : object + A ``date``, ``datetime`` or an ISO date string + + Returns + ------- + date | None + The date, or None when the value is not one + """ + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + try: + return date.fromisoformat(str(value).strip()[:10]) + except ValueError: + return None + + +def live_url(value: object) -> str | None: + """Return a URL field's value when it is a live link, not an archive link. + + Parameters + ---------- + value : object + A URL field's value + + Returns + ------- + str | None + The URL, or None for an empty field or an archive link + """ + text = str(value or "").strip() + if not text or wayback.is_archived(text): + return None + return text + + +def has_year(url: str) -> bool: + """Check whether a URL holds a four-digit year, as edition-specific URLs do. + + Parameters + ---------- + url : str + A link + + Returns + ------- + bool + True for URLs such as ``https://2024.pycon.de/`` or ``https://pycon.hk/2024/`` + """ + return bool(YEAR.search(url)) + + +def superseded(entries: list[dict], today: date) -> list[dict]: + """Return the editions that have ended and have a later edition in the data. + + Parameters + ---------- + entries : list[dict] + Every edition, from all data files + today : date + The day of the run + + Returns + ------- + list[dict] + The superseded editions, in data order + """ + latest: dict[str, date] = {} + for entry in entries: + start = to_date(entry.get("start")) + if start and (entry.get("conference") not in latest or start > latest[entry["conference"]]): + latest[entry["conference"]] = start + result = [] + for entry in entries: + start, end = to_date(entry.get("start")), to_date(entry.get("end")) + if start and end and end < today and start < latest[entry["conference"]]: + result.append(entry) + return result + + +def superseded_candidates(entries: list[dict], today: date) -> list[Candidate]: + """Return the yearless live URLs of superseded editions. + + Parameters + ---------- + entries : list[dict] + Every edition, from all data files + today : date + The day of the run + + Returns + ------- + list[Candidate] + One candidate per URL field + """ + candidates = [] + for entry in superseded(entries, today): + for field in URL_FIELDS: + url = live_url(entry.get(field)) + if url and not has_year(url): + candidates.append(Candidate(entry, field, url, "superseded")) + return candidates + + +def check_links( + entries: list[dict], + today: date, + request: Callable[[str], Probe] | None = None, +) -> list[Candidate]: + """Request every live URL once, queue redirects to other sites and return the broken ones. + + Parameters + ---------- + entries : list[dict] + Every edition, from all data files + today : date + The day of the run; only editions that ended before it get candidates + request : Callable[[str], Probe] | None + Requests one URL. Default: ``probe`` + + Returns + ------- + list[Candidate] + The broken URLs of editions that have ended + """ + request = request or probe + by_url: dict[str, list[tuple[dict, str]]] = defaultdict(list) + for entry in entries: + for field in URL_FIELDS: + url = live_url(entry.get(field)) + if url: + by_url[url].append((entry, field)) + + candidates = [] + for url, uses in by_url.items(): + result = request(url) + for entry, field in uses: + if result.redirects_elsewhere: + record_redirect( + url, + result.final_url, + result.status, + conference=entry.get("conference"), + year=entry.get("year"), + field=field, + ) + end = to_date(entry.get("end")) + if result.broken and end and end < today: + candidates.append(Candidate(entry, field, url, "broken")) + print(f"Checked {len(by_url)} URLs: {len(candidates)} broken on editions that have ended") + return candidates + + +def resolve( + candidates: Iterable[Candidate], + lookup: Callable[[str], list[str]] | None = None, + delay: float = DEFAULT_DELAY, +) -> list[Outcome]: + """Replace each candidate's URL with the capture that shows its edition. + + Parameters + ---------- + candidates : Iterable[Candidate] + The URL fields to resolve; a field listed twice is resolved once + lookup : Callable[[str], list[str]] | None + Lists a page's capture timestamps; called once per URL. Default: ``wayback.captures`` + delay : float + Seconds to wait between lookups + + Returns + ------- + list[Outcome] + One outcome per URL field + """ + lookup = lookup or wayback.captures + found: dict[str, list[str] | wayback.WaybackError] = {} + outcomes = [] + done = set() + for candidate in candidates: + key = (id(candidate.entry), candidate.field) + if key in done: + continue + done.add(key) + if candidate.url not in found: + if found: + time.sleep(delay) + try: + found[candidate.url] = lookup(candidate.url) + except wayback.WaybackError as e: + found[candidate.url] = e + stamps = found[candidate.url] + if isinstance(stamps, wayback.WaybackError): + outcomes.append(Outcome(candidate, None, f"lookup failed: {stamps}")) + continue + end = to_date(candidate.entry.get("end")) + stamp = wayback.pick_capture(stamps, end) + if not stamp: + outcomes.append(Outcome(candidate, None, f"no capture on or before {end}")) + continue + new_url = wayback.archive_url(stamp, candidate.url) + candidate.entry[candidate.field] = new_url + outcomes.append(Outcome(candidate, new_url, f"capture {stamp}")) + return outcomes + + +def follow_mastodon(entries: list[dict], check: Callable[[str], str | None] | None = None) -> int: + """Update Mastodon accounts that moved, checking each account once. + + Parameters + ---------- + entries : list[dict] + Every edition, edited in place + check : Callable[[str], str | None] | None + Returns an account's new address, or None when it has not moved. Default: ``check_mastodon_migration`` + + Returns + ------- + int + The number of entries updated + """ + check = check or check_mastodon_migration + moved: dict[str, str | None] = {} + updated = 0 + for entry in entries: + account = entry.get("mastodon") + if not account: + continue + if account not in moved: + moved[account] = check(account) + if moved[account]: + entry["mastodon"] = moved[account] + updated += 1 + return updated + + +def describe(outcome: Outcome) -> str: + """Describe one outcome as a Markdown list item. + + Parameters + ---------- + outcome : Outcome + A resolved or kept URL field + + Returns + ------- + str + The line, a checkbox for a URL that was kept + """ + candidate = outcome.candidate + name = f"{candidate.entry.get('conference')} {candidate.entry.get('year')}" + if outcome.new_url: + return f"- {name}: `{candidate.field}` {candidate.url} → {outcome.new_url}" + return f"- [ ] {name}: `{candidate.field}` {candidate.url} ({candidate.reason}) kept: {outcome.detail}" + + +def load(base: Path) -> dict[Path, list[dict]]: + """Load the three data files. + + Parameters + ---------- + base : Path + The repository root + + Returns + ------- + dict[Path, list[dict]] + Each file's path and entries + """ + files = {} + for name in DATA_FILES: + path = base / "_data" / name + if path.exists(): + with path.open(encoding="utf-8") as f: + files[path] = [entry for entry in yaml.safe_load(f) or [] if isinstance(entry, dict)] + return files + + +def build_parser() -> argparse.ArgumentParser: + """Build the command-line parser. + + Returns + ------- + argparse.ArgumentParser + The parser + """ + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--base", type=Path, default=Path(), help="Repository root") + parser.add_argument("--today", type=date.fromisoformat, help="Day of the run (default: today, UTC)") + parser.add_argument("--no-probe", action="store_true", help="Skip the link check and the Mastodon check") + parser.add_argument("--delay", type=float, default=DEFAULT_DELAY, help="Seconds between CDX lookups") + parser.add_argument("--changes", type=Path, help="File to write the replaced URLs to, as Markdown") + parser.add_argument("--needs", type=Path, help="File to write the URLs that were kept to, as a checklist") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Resolve old editions' links and write the data back when anything changed. + + Parameters + ---------- + argv : list[str] | None + Command-line arguments, without the program name + + Returns + ------- + int + 0; a URL that could not be resolved is reported, not an error + """ + args = build_parser().parse_args(argv) + today = args.today or datetime.now(tz=timezone.utc).date() + files = load(args.base) + entries = [entry for file_entries in files.values() for entry in file_entries] + + candidates = superseded_candidates(entries, today) + print(f"{len(candidates)} yearless URLs on superseded editions") + moved = 0 + if not args.no_probe: + candidates += check_links(entries, today) + moved = follow_mastodon(entries) + print(f"{moved} Mastodon accounts moved") + + outcomes = resolve(candidates, delay=args.delay) + replaced = [outcome for outcome in outcomes if outcome.new_url] + kept = [outcome for outcome in outcomes if not outcome.new_url] + print(f"Replaced {len(replaced)} URLs, kept {len(kept)}") + for outcome in outcomes: + print(describe(outcome)) + + if replaced or moved: + for path, file_entries in files.items(): + write_conference_yaml(file_entries, path) + if args.changes: + args.changes.write_text("".join(f"{describe(o)}\n" for o in replaced), encoding="utf-8") + if args.needs: + args.needs.write_text("".join(f"{describe(o)}\n" for o in kept), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/desk_pr_body.py b/utils/desk_pr_body.py index 28de374e13..c841a93424 100644 --- a/utils/desk_pr_body.py +++ b/utils/desk_pr_body.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Read and write the blocks of the rolling PR body that each writer owns. -The rolling PR on ``auto/conference-updates`` has three writers: the desk -Routine and two GitHub Actions jobs, the canonical-source import -(``canonical``) and the TBA enrichment (``enrich``). Its body tells the +The rolling PR on ``auto/conference-updates`` has four writers: the desk +Routine and three GitHub Actions jobs, the canonical-source import +(``canonical``), the TBA enrichment (``enrich``) and the archive resolve +(``archive``). Its body tells the maintainer what needs them and what was committed, so every writer adds to the same two sections. Each writer owns blocks between marker lines:: @@ -54,7 +55,7 @@ from pathlib import Path DEFAULT_REPO = "JesperDramsch/python-deadlines" -WRITERS = ("desk", "enrich", "canonical") +WRITERS = ("desk", "enrich", "canonical", "archive") BLOCKS = ( "HEAD:desk", *(f"NEEDS:{writer}" for writer in WRITERS), diff --git a/utils/main.py b/utils/main.py index 9594f515f7..d4ec274ce5 100644 --- a/utils/main.py +++ b/utils/main.py @@ -32,7 +32,7 @@ def main() -> None: # Step 2: Sort and validate data logger.info("🔄 Step 2: Sorting and validating data") step_start = time.time() - sort_data(skip_links=True) + sort_data() logger.info(f"✅ Data sorting completed in {time.time() - step_start:.2f}s") # Step 3: Import from Python organizers @@ -44,7 +44,7 @@ def main() -> None: # Step 4: Final sort and validation logger.info("🔄 Step 4: Final sorting and validation") step_start = time.time() - sort_data(skip_links=True) + sort_data() logger.info(f"✅ Final sorting completed in {time.time() - step_start:.2f}s") total_time = time.time() - start_time diff --git a/utils/merge_data.py b/utils/merge_data.py index 3f2c696421..b79a6d4b87 100644 --- a/utils/merge_data.py +++ b/utils/merge_data.py @@ -606,7 +606,7 @@ def write_and_sort(entries: list[dict], root: Path) -> None: write_conference_yaml(entries, str(root / DATA_FILES[0])) for path in DATA_FILES[1:]: write_conference_yaml([], str(root / path)) - sort_data(base=str(root), skip_links=True) + sort_data(base=str(root)) by_place = repeated_pairs(entries) expected = Counter(entry_key(entry, by_place) for entry in entries) diff --git a/utils/sort_yaml.py b/utils/sort_yaml.py index 03ec576389..bc095422f1 100644 --- a/utils/sort_yaml.py +++ b/utils/sort_yaml.py @@ -3,11 +3,8 @@ # Sort and Clean conference data. import contextlib import datetime -import operator -import time from datetime import timezone from pathlib import Path -from urllib.parse import urlparse import pydantic import pytz @@ -21,10 +18,7 @@ from tidy_conf.countries import normalize_place from tidy_conf.date import clean_dates from tidy_conf.latlon import add_latlon - from tidy_conf.links import check_link_availability - from tidy_conf.links import check_mastodon_migration from tidy_conf.links import drop_redundant_link_fields - from tidy_conf.links import get_cache from tidy_conf.schema import Conference from tidy_conf.schema import get_schema from tidy_conf.titles import tidy_titles @@ -36,10 +30,7 @@ from .tidy_conf.countries import normalize_place from .tidy_conf.date import clean_dates from .tidy_conf.latlon import add_latlon - from .tidy_conf.links import check_link_availability - from .tidy_conf.links import check_mastodon_migration from .tidy_conf.links import drop_redundant_link_fields - from .tidy_conf.links import get_cache from .tidy_conf.schema import Conference from .tidy_conf.schema import get_schema from .tidy_conf.titles import tidy_titles @@ -251,40 +242,13 @@ def split_data(data: list[Conference]) -> tuple[list, list, list, list]: return conf, tba, expired, legacy -def check_links(data): - """Check the links in the data iteratively.""" - cache, cache_archived = get_cache() - for i, q in tqdm(enumerate(sorted(data, key=operator.itemgetter("year"), reverse=True)), total=len(data)): - for key in ("link", "cfp_link", "sponsor", "finaid", "coc"): - if key in q: - new_link = check_link_availability( - q[key], - q["start"], - cache=cache, - cache_archived=cache_archived, - conference=q.get("conference"), - year=q.get("year"), - field=key, - ) - parsed_url = urlparse(new_link) - if q[key] != new_link and parsed_url.hostname and parsed_url.hostname.endswith(".archive.org"): - time.sleep(0.5) - q[key] = new_link - data[i] = q - - # Check Mastodon account migrations - if q.get("mastodon"): - new_mastodon = check_mastodon_migration(q["mastodon"]) - if new_mastodon: - q["mastodon"] = new_mastodon - data[i] = q - - return data - - # Sort: -def sort_data(base="", prefix="", skip_links=False): - """Sort and clean the conference data.""" +def sort_data(base="", prefix=""): + """Sort and clean the conference data. + + The sort never uses the network for links: utils/archive_resolve.py checks + them and replaces old editions' links with archived copies. + """ logger = get_tqdm_logger(__name__) # Load different data files @@ -341,13 +305,6 @@ def sort_data(base="", prefix="", skip_links=False): logger.info("🔄 Merging duplicates") data = merge_duplicates(data) - # Check Links - if not skip_links: - logger.info("🔗 Checking link availability") - data = check_links(data) - else: - logger.info("⏭️ Skipping link checking") - for i, q in enumerate(data.copy()): data[i] = order_keywords(q) @@ -406,7 +363,6 @@ def validate_conference(q: dict) -> Conference | None: import argparse parser = argparse.ArgumentParser(description="Sort and clean conference data.") - parser.add_argument("--skip_links", action="store_true", help="Skip checking links", default=False) - args = parser.parse_args() + parser.parse_args() - sort_data(skip_links=args.skip_links) + sort_data() diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index 958ddbb49d..5c22e321f9 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -1,9 +1,6 @@ import re import time -from datetime import datetime -from datetime import timedelta -from datetime import timezone -from pathlib import Path +from dataclasses import dataclass from urllib.parse import urlparse import requests @@ -83,7 +80,10 @@ # a redirect from one to another site says nothing about the link. URL_SHORTENERS = frozenset({"bit.ly", "goo.gl", "tinyurl.com"}) -# Redirects to a different site found by check_link_availability. A link that +# Statuses that mean a page is gone. See ``Probe.broken``. +BROKEN_STATUSES = frozenset({404, 410}) + +# Redirects to a different site found by utils/archive_resolve.py. A link that # now lands on another site (a lapsed domain, a parked page, a new home for # the series) needs a person to look at it, so the link is kept as it is and # the redirect is recorded here instead. @@ -382,195 +382,73 @@ def drop_redundant_link_fields(data: list) -> list: return data -def get_cache_location(): - # Check if the URL is cached - cache_file = Path("utils", "tidy_conf", "data", ".tmp", "no_archive.txt") - cache_file_archived = Path("utils", "tidy_conf", "data", ".tmp", "archived_links.txt") - # The cache directory is gitignored, so it does not exist in a fresh checkout - cache_file.parent.mkdir(parents=True, exist_ok=True) - return cache_file, cache_file_archived - - -def get_cache(): - cache_file, cache_file_archived = get_cache_location() - - # Create the cache file if it doesn't exist - cache_file.touch() - cache_file_archived.touch() - - # Read the cache file - cache = set(cache_file.read_text(encoding="utf-8").split("\n")[:-1]) - cache_archived = set(cache_file_archived.read_text(encoding="utf-8").split("\n")[:-1]) - - return cache, cache_archived +@dataclass(frozen=True) +class Probe: + """The outcome of one request for a link. + Attributes + ---------- + url : str + The link as stored in the data + final_url : str + Where the redirect chain ended; the link itself when the request failed + status : int | None + HTTP status of the final response, or None when there was none + error : str | None + Why the request failed, or None when a response came back + """ -def check_link_availability( - url, - start, - cache=None, - cache_archived=None, - *, - conference: str | None = None, - year: int | None = None, - field: str | None = None, -) -> str: - """Checks if a URL is available. + url: str + final_url: str + status: int | None + error: str | None = None - If not, tries to retrieve an archived version from the Wayback Machine. - Can't use caching because we have different years for the same url. + @property + def broken(self) -> bool: + """Whether the link is gone, rather than slow, blocked or briefly down. - Automatically redirects to the final URL if it was redirected within the same domain. + A 404 or 410, a host that does not resolve, a refused connection and + a failed TLS handshake (an expired or foreign certificate) count. A + timeout, a 403 or 429 (often a bot filter) and a 5xx do not: replacing + a link is not undone, so only a lasting failure counts. + """ + return self.status in BROKEN_STATUSES or self.error == "connection" - A redirect that ends on a different site (see ``is_different_site``) is not - treated as healthy: the original URL is returned unchanged, with no Wayback - lookup, and the redirect is recorded with ``record_redirect`` for review. - A short link that lands where it is expected to (see ``is_expected_redirect``), - such as forms.gle on docs.google.com, is not recorded: it is checked like - any other link, so on a 200 the original URL is kept and, when the - conference is still to come, saved to the Wayback Machine. + @property + def redirects_elsewhere(self) -> bool: + """Whether the link now lands on another site, other than as its kind of link does.""" + return ( + self.status is not None + and is_different_site(self.url, self.final_url) + and not is_expected_redirect(self.url, self.final_url) + ) - Automatically caches old (5+ years) URLs. - Note: Some URLs may not be available or archived. These return the original URL and are logged. +def probe(url: str, timeout: int = 10) -> Probe: + """Request a link once, following redirects. Parameters ---------- url : str - URL to check - start : datetime.date - Start date of the conference the URL belongs to - cache : set | None - URLs known to have no archived version, read from disk when None - cache_archived : set | None - URLs already archived, read from disk when None - conference : str | None - Conference name, only used to describe a recorded redirect - year : int | None - Conference year, only used to describe a recorded redirect - field : str | None - Data field holding the URL, only used to describe a recorded redirect + The link to request + timeout : int + Seconds to wait for a response Returns ------- - str - The URL to keep: the original, a same-domain redirect target, or an archived copy + Probe + The status and final URL, or why there was no response """ - # If it's archived already, return the URL - if url.startswith(("https://web.archive.org", "http://web.archive.org")): - return url - - # Get the cache - if cache is None or cache_archived is None: - cache, cache_archived = get_cache() - - cache_file, _ = get_cache_location() - - # Check if the URL is in the cache - if url in cache and url not in cache_archived: - tqdm.write(f"URL {url} is cached as not found. Returning original URL.") - return url - - # If the URL is younger than 5 years, check if it's available - if start > datetime.now(tz=timezone.utc).date() - timedelta(days=5 * 365): - try: - response = requests.get(url, allow_redirects=True, timeout=10) - final_url = response.url - # A redirect to another site is not a healthy link: keep the URL as it - # is and leave the decision to a person. Short links such as - # forms.gle redirect to another site by design and are left out. - if is_different_site(url, final_url) and not is_expected_redirect(url, final_url): - tqdm.write( - f"URL {url} was redirected to a different site: {final_url} " - f"(status code: {response.status_code}). Keeping the original URL for review.", - ) - record_redirect( - url, - final_url, - response.status_code, - conference=conference, - year=year, - field=field, - ) - return url - # Check if the final URL is within the same domain as the original URL - if urlparse(url).netloc == urlparse(final_url).netloc and final_url != url: - # Don't cache the redirect if it suddenly has a query string - tqdm.write(f"URL {url} was redirected within the same domain to: {final_url}") - if "?" in final_url and "?" not in url: - tqdm.write("Warning: The final URL contains a query string, but the original URL does not.") - # Return original URL to avoid using redirected URL with query string - return url - return final_url # Use the final URL for the rest of the process - if response.status_code != 200: - tqdm.write( - ( - f"Link {url} is not available (status code: {response.status_code})." - "Trying to find an archived version..." - ), - ) - else: - if start > datetime.now(tz=timezone.utc).date(): - attempt_archive_url(url, cache_archived) - return url - except requests.RequestException as e: - tqdm.write(f"An error occurred: {e}. Trying to find an archived version...") - - # Try to get an archived version from the Wayback Machine or return original URL - archive_url = f"https://archive.org/wayback/available?url={url}×tamp={start.strftime('%Y%m%d%H%M%S')}" - try: - archive_response = requests.get(archive_url, timeout=10) - # Make sure the status code is valid (200) - if archive_response.status_code == 200: - data = archive_response.json() - # Get the "closest available" snapshot URL to a date - if ( - "archived_snapshots" in data - and data["archived_snapshots"] - and "closest" in data["archived_snapshots"] - and data["archived_snapshots"]["closest"] - and data["archived_snapshots"]["closest"].get("available", False) - ): - archived_url = data["archived_snapshots"]["closest"]["url"].replace(":80/", "/") - if archived_url[4] == ":": - archived_url = archived_url[:4] + "s" + archived_url[4:] - tqdm.write(f"Found archived version: {archived_url}") - return archived_url - tqdm.write("No archived version found.") - attempt_archive_url(url, cache_archived) - with cache_file.open("a") as f: - f.write(url + "\n") - return url - - tqdm.write("Failed to retrieve archived version.") - return url - except requests.RequestException as e: - tqdm.write(f"An error occurred while retrieving the archived version: {e}") - return url - - -def attempt_archive_url(url, cache=None): - """Attempts to archive a URL using the Wayback Machine.""" - # Read the cache file - if cache is None: - _, cache = get_cache() - - # Check if the URL is in the cache - if url in cache: - tqdm.write(f"URL {url} was already archived.") - return - + headers = {"User-Agent": "Pythondeadlin.es Link Checker/0.2 (https://pythondeadlin.es)"} try: - tqdm.write(f"Attempting archive of {url}.") - headers = {"User-Agent": "Pythondeadlin.es Archival Attempt/0.1 (https://pythondeadlin.es)"} - archive_response = requests.get("https://web.archive.org/save/" + url, timeout=30, headers=headers) - if archive_response.status_code == 200: - _, cache_file = get_cache_location() - with cache_file.open("a") as f: - f.write(url + "\n") - tqdm.write(f"Successfully archived {url}.") + response = requests.get(url, allow_redirects=True, timeout=timeout, headers=headers) + except requests.Timeout: + return Probe(url, url, None, "timeout") + except requests.ConnectionError: + return Probe(url, url, None, "connection") except requests.RequestException as e: - tqdm.write(f"An error occurred while attempting to archive: {e}") + return Probe(url, url, None, type(e).__name__) + return Probe(url, response.url or url, response.status_code) def parse_mastodon_url(url: str) -> tuple[str, str] | None: From f9cdaaf1ae572dbe03c53389fa3f215fb156b584 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:00:53 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_archive_resolve.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_archive_resolve.py b/tests/test_archive_resolve.py index 844d614e45..ec8f518538 100644 --- a/tests/test_archive_resolve.py +++ b/tests/test_archive_resolve.py @@ -284,7 +284,9 @@ def test_checks_coc(self): requested = [] archive_resolve.check_links( - [entry], TODAY, request=lambda url: requested.append(url) or links.Probe(url, url, 200), + [entry], + TODAY, + request=lambda url: requested.append(url) or links.Probe(url, url, 200), ) assert requested == ["https://pycon.tz/conduct/"] From 608fcdf7dcb2aaca06120c6195c4b3514683eb4d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 6 Nov 2026 20:00:34 +0000 Subject: [PATCH 4/4] Treat a connection reset as transient in the link check requests raises a reset or a connection dropped mid-request as the same ConnectionError as a host that does not resolve, so probe() counted a site that is up but reset one connection as gone, and the weekly resolve would have archived its link for good. probe() now walks the exception chain: a failed TLS handshake, a name-resolution failure or a refused connection is lasting; anything else is "reset" and leaves the link alone. The resolve workflow now replaces NEEDS:archive on every run that finds an open rolling PR, and empties it when nothing is kept, so a week with no data change no longer leaves a stale list. superseded() reads `conference` with .get on both passes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016a2XGgiRrtWm4EUP7ek9wt --- .github/workflows/archive-resolve.yml | 36 +++++++----- tests/test_archive_resolve.py | 85 +++++++++++++++++++++++++-- utils/archive_resolve.py | 7 ++- utils/tidy_conf/links.py | 60 +++++++++++++++++-- 4 files changed, 161 insertions(+), 27 deletions(-) diff --git a/.github/workflows/archive-resolve.yml b/.github/workflows/archive-resolve.yml index 5e78c3f737..b5e556084c 100644 --- a/.github/workflows/archive-resolve.yml +++ b/.github/workflows/archive-resolve.yml @@ -91,6 +91,7 @@ jobs: # Below the job timeout, so a slow link check still queues what it found. - name: Resolve archive links + id: resolve timeout-minutes: 150 env: DESK_QUEUE_OUT: ${{ runner.temp }}/desk-records.jsonl @@ -206,33 +207,40 @@ jobs: # The rolling PR body has one block per writer (utils/desk_pr_body.py). # This job owns COMMITTED:archive, one line per commit naming the # editions it changed, and NEEDS:archive, the links that have no capture - # from their edition's time: each run replaces that list with its own. + # from their edition's time. Kept links come back every week whether or + # not the run changed data, so NEEDS:archive is replaced on every run that + # finds an open PR, and emptied when nothing is kept. A run with no change + # does not open a PR for it. # The script comes from the commit this run started on: the accumulator # branch checked out here may not have it yet. - - name: Create or update PR - if: steps.check_changes.outputs.changed == 'true' + - name: Update the rolling PR + if: ${{ !cancelled() && steps.resolve.outcome == 'success' }} env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} + CHANGED: ${{ steps.check_changes.outputs.changed }} COMMIT_SHA: ${{ steps.commit.outputs.commit_sha }} run: | SCRIPT="$RUNNER_TEMP/desk_pr_body.py" git show "$GITHUB_SHA:utils/desk_pr_body.py" > "$SCRIPT" PR=$(gh api "repos/$REPO/pulls?head=${REPO%%/*}:$UPDATE_BRANCH&state=open" --jq '.[0].number // empty') - if [ -z "$PR" ]; then - pixi run python "$SCRIPT" skeleton > "$RUNNER_TEMP/body.md" - PR=$(gh api "repos/$REPO/pulls" -f title='🐍 Conference updates' -f head="$UPDATE_BRANCH" -f base=main \ - -F body=@"$RUNNER_TEMP/body.md" --jq .number) - gh api "repos/$REPO/issues/$PR/labels" -f 'labels[]=automated' -f 'labels[]=conference-update' --silent - echo "::notice title=PR Created::Opened #$PR for the archive resolve" + if [ "$CHANGED" = "true" ] && [ -n "$COMMIT_SHA" ]; then + if [ -z "$PR" ]; then + pixi run python "$SCRIPT" skeleton > "$RUNNER_TEMP/body.md" + PR=$(gh api "repos/$REPO/pulls" -f title='🐍 Conference updates' -f head="$UPDATE_BRANCH" -f base=main \ + -F body=@"$RUNNER_TEMP/body.md" --jq .number) + gh api "repos/$REPO/issues/$PR/labels" -f 'labels[]=automated' -f 'labels[]=conference-update' --silent + echo "::notice title=PR Created::Opened #$PR for the archive resolve" + fi + pixi run python "$SCRIPT" --repo "$REPO" commit-line --sha "$COMMIT_SHA" --label "Archive links" > "$RUNNER_TEMP/line.md" + if [ -s "$RUNNER_TEMP/line.md" ]; then + pixi run python "$SCRIPT" --repo "$REPO" --pr "$PR" append COMMITTED:archive "$RUNNER_TEMP/line.md" + fi fi - pixi run python "$SCRIPT" --repo "$REPO" commit-line --sha "$COMMIT_SHA" --label "Archive links" > "$RUNNER_TEMP/line.md" - if [ -s "$RUNNER_TEMP/line.md" ]; then - pixi run python "$SCRIPT" --repo "$REPO" --pr "$PR" append COMMITTED:archive "$RUNNER_TEMP/line.md" - fi - if [ -s "$RUNNER_TEMP/needs.md" ]; then + if [ -n "$PR" ]; then + touch "$RUNNER_TEMP/needs.md" pixi run python "$SCRIPT" --repo "$REPO" --pr "$PR" set NEEDS:archive "$RUNNER_TEMP/needs.md" fi diff --git a/tests/test_archive_resolve.py b/tests/test_archive_resolve.py index ec8f518538..d90552dbbe 100644 --- a/tests/test_archive_resolve.py +++ b/tests/test_archive_resolve.py @@ -1,13 +1,21 @@ """Tests for utils/archive_resolve.py: which links of old editions are archived, and how.""" +import socket +import struct import sys +import threading from datetime import date +from http.client import RemoteDisconnected from pathlib import Path import pytest import requests import responses import yaml +from urllib3.exceptions import MaxRetryError +from urllib3.exceptions import NameResolutionError +from urllib3.exceptions import NewConnectionError +from urllib3.exceptions import ProtocolError sys.path.append(str(Path(__file__).parent.parent / "utils")) @@ -29,6 +37,27 @@ def _edition(year: int, start: date, end: date | None = None, conference: str = return {"conference": conference, "year": year, "start": start, "end": end or start, **fields} +def _wrapped(outer: BaseException, cause: BaseException) -> BaseException: + outer.__cause__ = cause + return outer + + +def _dns_failure() -> requests.ConnectionError: + resolution = NameResolutionError("gone.example", None, socket.gaierror(-2, "Name or service not known")) + return requests.ConnectionError(MaxRetryError(None, "/", resolution)) + + +def _refused() -> requests.ConnectionError: + refused = _wrapped(NewConnectionError(None, "Failed to establish a new connection"), ConnectionRefusedError(111)) + return requests.ConnectionError(MaxRetryError(None, "/", refused)) + + +def _reset() -> requests.ConnectionError: + return requests.ConnectionError( + ProtocolError("Connection aborted.", ConnectionResetError(104, "Connection reset by peer")), + ) + + @pytest.fixture() def tanzania() -> list[dict]: """PyCon Tanzania as the data held it: the 2024 edition on the old domain, the 2026 one on the new.""" @@ -205,23 +234,67 @@ def test_status(self, status, broken): assert links.probe("https://example.org/").broken is broken @pytest.mark.parametrize( - ("error", "broken"), + ("error", "kind", "broken"), [ - (requests.ConnectionError("name or service not known"), True), - (requests.Timeout("slow"), False), - (requests.exceptions.SSLError("bad certificate"), True), - (requests.exceptions.InvalidURL("bad url"), False), + # The exception chains requests raises, as seen against real hosts and sockets + (_dns_failure(), "connection", True), + (_refused(), "connection", True), + (requests.exceptions.SSLError("certificate has expired"), "connection", True), + (_reset(), "reset", False), + ( + requests.ConnectionError(ProtocolError("Connection aborted.", RemoteDisconnected("closed"))), + "reset", + False, + ), + (requests.Timeout("slow"), "timeout", False), + (requests.exceptions.InvalidURL("bad url"), "InvalidURL", False), ], ) @responses.activate - def test_errors(self, error, broken): + def test_errors(self, error, kind, broken): responses.add(responses.GET, "https://example.org/", body=error) result = links.probe("https://example.org/") assert result.status is None + assert result.error == kind assert result.broken is broken + def test_reset_by_a_live_server_is_not_broken(self): + """A site that is up but resets one connection keeps its link.""" + server = socket.socket() + server.bind(("127.0.0.1", 0)) + server.listen() + + def reset_once(): + connection, _ = server.accept() + connection.recv(1024) + # Linger 0 makes close() send a reset instead of a normal end + connection.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + connection.close() + + thread = threading.Thread(target=reset_once, daemon=True) + thread.start() + try: + result = links.probe(f"http://127.0.0.1:{server.getsockname()[1]}/") + finally: + thread.join(timeout=5) + server.close() + + assert result.error == "reset" + assert not result.broken + + def test_refused_connection_is_broken(self): + server = socket.socket() + server.bind(("127.0.0.1", 0)) + port = server.getsockname()[1] + server.close() # nothing listens on the port now + + result = links.probe(f"http://127.0.0.1:{port}/") + + assert result.error == "connection" + assert result.broken + @responses.activate def test_redirect_elsewhere(self): responses.add(responses.GET, "https://pythoncamp.de/", status=301, headers={"Location": "https://barcamps.eu/"}) diff --git a/utils/archive_resolve.py b/utils/archive_resolve.py index 7155159976..41ba38e05a 100644 --- a/utils/archive_resolve.py +++ b/utils/archive_resolve.py @@ -186,12 +186,13 @@ def superseded(entries: list[dict], today: date) -> list[dict]: latest: dict[str, date] = {} for entry in entries: start = to_date(entry.get("start")) - if start and (entry.get("conference") not in latest or start > latest[entry["conference"]]): - latest[entry["conference"]] = start + name = entry.get("conference") + if start and (name not in latest or start > latest[name]): + latest[name] = start result = [] for entry in entries: start, end = to_date(entry.get("start")), to_date(entry.get("end")) - if start and end and end < today and start < latest[entry["conference"]]: + if start and end and end < today and start < latest[entry.get("conference")]: result.append(entry) return result diff --git a/utils/tidy_conf/links.py b/utils/tidy_conf/links.py index 5c22e321f9..75d1d7c508 100644 --- a/utils/tidy_conf/links.py +++ b/utils/tidy_conf/links.py @@ -1,10 +1,12 @@ import re +import socket import time from dataclasses import dataclass from urllib.parse import urlparse import requests from tqdm import tqdm +from urllib3.exceptions import NameResolutionError try: from tidy_conf.queue_out import QUEUE_OUT_ENV @@ -409,8 +411,9 @@ def broken(self) -> bool: A 404 or 410, a host that does not resolve, a refused connection and a failed TLS handshake (an expired or foreign certificate) count. A - timeout, a 403 or 429 (often a bot filter) and a 5xx do not: replacing - a link is not undone, so only a lasting failure counts. + timeout, a connection reset or dropped mid-request, a 403 or 429 (often + a bot filter) and a 5xx do not: replacing a link is not undone, so only + a lasting failure counts. See ``is_lasting_connection_failure``. """ return self.status in BROKEN_STATUSES or self.error == "connection" @@ -444,13 +447,62 @@ def probe(url: str, timeout: int = 10) -> Probe: response = requests.get(url, allow_redirects=True, timeout=timeout, headers=headers) except requests.Timeout: return Probe(url, url, None, "timeout") - except requests.ConnectionError: - return Probe(url, url, None, "connection") + except requests.ConnectionError as e: + # A reset or a dropped connection mid-request is a ConnectionError too, + # but says nothing lasting about the site. + return Probe(url, url, None, "connection" if is_lasting_connection_failure(e) else "reset") except requests.RequestException as e: return Probe(url, url, None, type(e).__name__) return Probe(url, response.url or url, response.status_code) +def _causes(error: BaseException) -> list[BaseException]: + """Return an exception and every exception it wraps, through causes, contexts, reasons and arguments. + + Parameters + ---------- + error : BaseException + The exception requests raised + + Returns + ------- + list[BaseException] + The exceptions found, the outermost first + """ + found: list[BaseException] = [] + todo: list[object] = [error] + while todo: + current = todo.pop(0) + if not isinstance(current, BaseException) or any(current is seen for seen in found): + continue + found.append(current) + todo.extend((getattr(current, "reason", None), current.__cause__, current.__context__, *current.args)) + return found + + +def is_lasting_connection_failure(error: requests.ConnectionError) -> bool: + """Check whether a connection error means the site is gone rather than briefly unreachable. + + A failed TLS handshake, a host that does not resolve and a refused + connection count. A reset or a connection dropped mid-request does not. + + Parameters + ---------- + error : requests.ConnectionError + The exception requests raised + + Returns + ------- + bool + True for a lasting failure + """ + if isinstance(error, requests.exceptions.SSLError): + return True + return any( + isinstance(cause, ConnectionRefusedError | socket.gaierror | NameResolutionError) for cause in _causes(error) + ) + + def parse_mastodon_url(url: str) -> tuple[str, str] | None: """Parse a Mastodon URL into (instance, username).