sync-changedetection.yaml

name: Sync New URLs to Changedetection.io

on:
  push:
    branches: [main, master]
    paths:
      - '_data/**'
  workflow_dispatch:
    inputs:
      dry_run:
        description: 'Dry run (log actions without making API calls)'
        type: boolean
        default: false
      force_recheck:
        description: 'Check all URLs, not just new ones'
        type: boolean
        default: false

permissions:
  contents: read

concurrency:
  group: sync-changedetection
  cancel-in-progress: false

jobs:
  sync-urls:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v7
        with:
          python-version: '3.12'

      - name: Install dependencies
        run: pip install pyyaml

      - name: Find last successful run commit
        id: last-run
        run: |
          last_sha=$(gh run list --workflow="Sync New URLs to Changedetection.io" --json conclusion,headSha --jq '.[] | select(.conclusion=="success") | .headSha' | head -n1)
          if [ -n "$last_sha" ] && [ "${{ inputs.force_recheck }}" != "true" ]; then
            echo "sha=$last_sha" >> $GITHUB_OUTPUT
            echo "Found last successful run at $last_sha"
          else
            echo "sha=$(git rev-list --max-parents=0 HEAD)" >> $GITHUB_OUTPUT
            echo "Using initial commit (full scan mode)"
          fi
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Check changedetection.io configuration
        id: check_cd
        run: |
          if [ -z "${{ vars.CHANGEDETECTION_URL }}" ]; then
            echo "::error::CHANGEDETECTION_URL variable not set"
            exit 1
          fi
          if [ -z "${{ secrets.CHANGEDETECTION_KEY }}" ]; then
            echo "::error::CHANGEDETECTION_KEY secret not set"
            exit 1
          fi

          # Check if CF Access is required (common for changedetection.io instances)
          if [ -z "${{ secrets.CF_ACCESS_CLIENT_ID }}" ] || [ -z "${{ secrets.CF_ACCESS_CLIENT_SECRET }}" ]; then
            echo "::warning::Cloudflare Access credentials not configured"
            echo "::warning::If your instance requires CF Access, set CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET secrets"
            echo "::warning::Proceeding without CF Access headers - may cause HTTP 302 errors"
          else
            echo "::notice::Cloudflare Access credentials configured"
          fi

          echo "configured=true" >> $GITHUB_OUTPUT

      - name: Test API connectivity
        id: test-api
        if: inputs.dry_run != true
        run: |
          API_URL="${{ vars.CHANGEDETECTION_URL }}/api/v1/systeminfo"

          echo "Testing API connectivity..."

          CURL_OUTPUT=$(mktemp)
          CURL_HEADERS=$(mktemp)

          # Test with systeminfo endpoint
          if [ -n "${{ secrets.CF_ACCESS_CLIENT_ID }}" ]; then
            HTTP_CODE=$(curl -w '%{http_code}' -s -D "$CURL_HEADERS" \
              -H "x-api-key: ${{ secrets.CHANGEDETECTION_KEY }}" \
              -H "CF-Access-Client-Id: ${{ secrets.CF_ACCESS_CLIENT_ID }}" \
              -H "CF-Access-Client-Secret: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}" \
              "$API_URL" -o "$CURL_OUTPUT")
          else
            HTTP_CODE=$(curl -w '%{http_code}' -s -D "$CURL_HEADERS" \
              -H "x-api-key: ${{ secrets.CHANGEDETECTION_KEY }}" \
              "$API_URL" -o "$CURL_OUTPUT")
          fi

          echo "HTTP status code: $HTTP_CODE"

          if [ "$HTTP_CODE" = "302" ] || [ "$HTTP_CODE" = "301" ]; then
            echo "::error::❌ API connectivity test failed - HTTP $HTTP_CODE redirect"
            echo "::error::Redirect location:"
            grep -i "location:" "$CURL_HEADERS" || echo "No Location header found"
            echo "::error::This means CF Access credentials are not working"
            rm -f "$CURL_OUTPUT" "$CURL_HEADERS"
            exit 1
          fi

          if [ "$HTTP_CODE" != "200" ]; then
            echo "::error::❌ API connectivity test failed - HTTP $HTTP_CODE"
            echo "::error::Response body:"
            cat "$CURL_OUTPUT" | head -c 500
            rm -f "$CURL_OUTPUT" "$CURL_HEADERS"
            exit 1
          fi

          RESPONSE=$(cat "$CURL_OUTPUT")
          rm -f "$CURL_OUTPUT" "$CURL_HEADERS"

          # Validate response is valid JSON
          if ! echo "$RESPONSE" | jq empty 2>/dev/null; then
            echo "::error::❌ API returned invalid JSON (HTML login page)"
            echo "::error::Response preview:"
            echo "$RESPONSE" | head -c 500
            exit 1
          fi

          echo "✅ API connectivity test passed"
          echo "API version: $(echo "$RESPONSE" | jq -r '.version // "unknown"')"

      - name: Fetch existing watches from changedetection.io
        id: fetch-watches
        if: inputs.dry_run != true
        run: |
          API_URL="${{ vars.CHANGEDETECTION_URL }}/api/v1/watch"

          echo "Fetching existing watches from changedetection.io..."
          echo "Debug: API URL: $API_URL"

          # Debug: Show if CF Access credentials are being used
          if [ -n "${{ secrets.CF_ACCESS_CLIENT_ID }}" ]; then
            echo "Debug: CF Access credentials ARE configured"
          else
            echo "Debug: CF Access credentials NOT configured"
          fi

          # Test curl with verbose output to see what's happening
          echo "Debug: Testing curl with verbose output..."
          CURL_OUTPUT=$(mktemp)
          CURL_HEADERS=$(mktemp)

          # Build curl command with proper headers (no eval needed)
          if [ -n "${{ secrets.CF_ACCESS_CLIENT_ID }}" ]; then
            HTTP_CODE=$(curl -w '%{http_code}' -s -D "$CURL_HEADERS" \
              -H "x-api-key: ${{ secrets.CHANGEDETECTION_KEY }}" \
              -H "CF-Access-Client-Id: ${{ secrets.CF_ACCESS_CLIENT_ID }}" \
              -H "CF-Access-Client-Secret: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}" \
              "$API_URL" -o "$CURL_OUTPUT")
          else
            HTTP_CODE=$(curl -w '%{http_code}' -s -D "$CURL_HEADERS" \
              -H "x-api-key: ${{ secrets.CHANGEDETECTION_KEY }}" \
              "$API_URL" -o "$CURL_OUTPUT")
          fi

          echo "Debug: HTTP status code: $HTTP_CODE"
          echo "Debug: Response size: $(wc -c < "$CURL_OUTPUT") bytes"

          # Show redirect location if present
          if [ "$HTTP_CODE" = "302" ] || [ "$HTTP_CODE" = "301" ]; then
            echo "Debug: Redirect detected - response headers:"
            grep -i "location:" "$CURL_HEADERS" || echo "No Location header found"
            echo "Debug: All response headers:"
            cat "$CURL_HEADERS"
          fi

          if [ "$HTTP_CODE" != "200" ]; then
            echo "::error::Failed to fetch existing watches from changedetection.io"
            echo "::error::HTTP status code: $HTTP_CODE"
            echo "::error::Response body:"
            cat "$CURL_OUTPUT" | head -c 500
            echo ""
            echo "::error::"
            echo "::error::This usually indicates:"
            echo "::error::  1. Cloudflare Access credentials are missing or incorrect (HTTP 302)"
            echo "::error::  2. CHANGEDETECTION_KEY is invalid (HTTP 401/403)"
            echo "::error::  3. CHANGEDETECTION_URL is incorrect (HTTP 404)"
            rm -f "$CURL_OUTPUT" "$CURL_HEADERS"
            exit 1
          fi

          RESPONSE=$(cat "$CURL_OUTPUT")
          rm -f "$CURL_OUTPUT" "$CURL_HEADERS"

          # Validate response is valid JSON (not a CF Access login page)
          if ! echo "$RESPONSE" | jq empty 2>/dev/null; then
            echo "::error::API returned invalid JSON (possibly a Cloudflare Access login page)"
            echo "::error::Response preview:"
            echo "$RESPONSE" | head -c 500
            echo "::error::"
            echo "::error::This indicates Cloudflare Access authentication failed."
            echo "::error::Verify CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET secrets are set correctly."
            exit 1
          fi

          # Kept for the step that pauses watches on links replaced by an archived copy
          echo "$RESPONSE" > watches.json

          # Debug: Show response structure
          RESPONSE_TYPE=$(echo "$RESPONSE" | jq -r 'type')
          echo "Response type: $RESPONSE_TYPE"

          if [ "$RESPONSE_TYPE" = "object" ]; then
            # API returns object format: {"uuid": {watch_data}, ...}
            WATCH_COUNT=$(echo "$RESPONSE" | jq 'length')
            echo "API returned $WATCH_COUNT watches (object format)"

            if [ "$WATCH_COUNT" -gt 0 ]; then
              echo "First watch structure:"
              echo "$RESPONSE" | jq 'to_entries | .[0]'
            else
              echo "::warning::API returned empty object - no existing watches found"
            fi

            # Extract URLs from object format: convert to array of values, then get URLs
            echo "$RESPONSE" | jq -r 'to_entries[] | .value.url' | tr '[:upper:]' '[:lower:]' | sed 's:/*$::' > existing_urls.txt
          elif [ "$RESPONSE_TYPE" = "array" ]; then
            # Legacy: handle array format
            WATCH_COUNT=$(echo "$RESPONSE" | jq 'length')
            echo "API returned $WATCH_COUNT watches (array format)"

            if [ "$WATCH_COUNT" -gt 0 ]; then
              echo "First watch structure:"
              echo "$RESPONSE" | jq '.[0]'
            fi

            echo "$RESPONSE" | jq -r '.[] | .url' | tr '[:upper:]' '[:lower:]' | sed 's:/*$::' > existing_urls.txt
          else
            echo "::error::Unexpected response type: $RESPONSE_TYPE"
            echo "::error::Response preview:"
            echo "$RESPONSE" | head -c 500
            exit 1
          fi
          URL_COUNT=$(wc -l < existing_urls.txt)
          echo "Successfully fetched $URL_COUNT existing watch URLs"
          echo "count=$URL_COUNT" >> $GITHUB_OUTPUT

      - name: Create empty existing URLs file (dry run)
        if: inputs.dry_run == true
        run: touch existing_urls.txt

      - name: Process conference URLs
        id: process
        env:
          LAST_SHA: ${{ steps.last-run.outputs.sha }}
        run: |
          python3 << 'PYTHON_SCRIPT'
          import os
          import re
          import json
          import subprocess
          import sys
          import yaml
          from urllib.parse import urlparse
          from typing import Optional

          sys.path.insert(0, "utils")
          from watch_templates import rollover_offset

          LAST_SHA = os.environ.get('LAST_SHA', '')

          # Tag UUIDs
          BASE_TAGS = [
              "1ad0fbdf-482a-4296-b983-45311ccf2352",
              "aa19ce68-9d8f-4e86-8df2-342faccbccc8",
          ]

          CONTINENT_TAGS = {
              "africa": "70a6a80d-c13d-450f-b36e-cfa6cbd3fbd4",
              "europe": "23a96495-26fc-4fa6-bdda-78c3051748e5",
              "america": "b44ef3c3-7280-49bc-b71f-4d7c0ca62fb7",
              "asia": "b488faec-899a-4578-9560-7154999b8c9e",
              "oceania": "11742b59-eb4e-42b3-8069-64dfc1bced4e",
          }

          SUB_TO_TAG = {
              "WEB": "7054a0fb-69db-47fd-9c2e-3ee6c638bac7",
              "SCIPY": "571f57a6-258d-4101-9136-cd1643365e55",
              "DATA": "8e55732c-4d04-4457-bff8-2c0c851590db",
          }

          CFP_TAG = "5799f798-2b44-452e-aefa-43874f6c3e1e"

          # Hosts that carry many series, where only the path tells two series
          # apart (AGENTS.md "Names"): a watch on one series' page there says
          # nothing about another's, so these never match by domain.
          MULTI_SERIES_HOSTS = {
              "pydata.org",
              "events.linuxfoundation.org",
              "pretalx.com",
              "sessionize.com",
              "web.archive.org",
              "barcamps.eu",
          }

          URL_FIELDS = {'link': 'Main', 'cfp_link': 'CFP'}
          YEAR_PATTERN = re.compile(r'/(20(?:2[4-9]|3[0-5]))/')

          def get_continent(lat: float, lon: float) -> Optional[str]:
              if lat is None or lon is None:
                  return None
              if lon < -30:
                  return "america"
              if lat < 0 and lon > 100:
                  return "oceania"
              if lat < 35 and lon < 55:
                  return "africa"
              if lon > 40:
                  return "asia"
              return "europe"

          def get_location_coords(conf: dict) -> tuple[Optional[float], Optional[float]]:
              location = conf.get('location', [])
              if location and isinstance(location, list) and len(location) > 0:
                  loc = location[0]
                  return loc.get('latitude'), loc.get('longitude')
              return None, None

          def template_url(url: str, offset: Optional[int]) -> str:
              """Template years in URL (subdomain or path) with Jinja expressions.

              Examples:
                Subdomains:
                  https://2026.pycon.de -> https://{% now ... %}.pycon.de
                  https://ep2026.europython.eu -> https://ep{% now ... %}.europython.eu
                  https://scipy2026.scipy.org -> https://scipy{% now ... %}.scipy.org
                Paths:
                  https://pycon.de/2026/ -> https://pycon.de/{% now ... %}/
                  https://pretalx.com/conference-2026/ -> https://pretalx.com/conference-{% now ... %}/
                  https://pydata.org/london2026/ -> https://pydata.org/london{% now ... %}/

              The offset sets when the watch moves on to the next edition
              (utils/watch_templates.py): two weeks after the CfP closes, at
              the latest when the conference starts.
              """
              if offset is None:
                  return url

              template = f"{{% now 'utc' + 'months={offset}', '%Y' %}}"
              parsed = urlparse(url)
              netloc_parts = parsed.netloc.split('.')

              # Check subdomain for year patterns
              if len(netloc_parts) >= 2:
                  # Handle www.prefix2026.domain.com by checking both first and second part
                  for idx in [0, 1]:
                      if idx >= len(netloc_parts):
                          break

                      subdomain = netloc_parts[idx]

                      # Pattern 1: Pure year (2026)
                      if re.match(r'^20(?:2[4-9]|3[0-5])$', subdomain):
                          netloc_parts[idx] = template
                          new_netloc = '.'.join(netloc_parts)
                          return f"{parsed.scheme}://{new_netloc}{parsed.path or ''}"

                      # Pattern 2: Prefix + year (ep2026, scipy2026)
                      match = re.match(r'^([a-z]+)(20(?:2[4-9]|3[0-5]))$', subdomain, re.IGNORECASE)
                      if match:
                          prefix = match.group(1)
                          netloc_parts[idx] = f"{prefix}{template}"
                          new_netloc = '.'.join(netloc_parts)
                          return f"{parsed.scheme}://{new_netloc}{parsed.path or ''}"

                      # Only check second part if first part is 'www'
                      if idx == 0 and subdomain.lower() != 'www':
                          break

              # Check path for year patterns
              path = parsed.path or ''

              # Pattern 1: Standalone year in path (/2026/ or /2026)
              new_path = re.sub(
                  r'/(20(?:2[4-9]|3[0-5]))(/|$)',
                  rf'/{template}\2',
                  path
              )
              if new_path != path:
                  return f"{parsed.scheme}://{parsed.netloc}{new_path}"

              # Pattern 2: Word(s) + hyphen + year (/conference-2026/ or /conference-2026)
              new_path = re.sub(
                  r'/([a-z0-9-]+)-(20(?:2[4-9]|3[0-5]))(/|$)',
                  rf'/\1-{template}\3',
                  path,
                  flags=re.IGNORECASE
              )
              if new_path != path:
                  return f"{parsed.scheme}://{parsed.netloc}{new_path}"

              # Pattern 3: Word + year no hyphen (/london2026/ or /london2026)
              new_path = re.sub(
                  r'/([a-z]+)(20(?:2[4-9]|3[0-5]))(/|$)',
                  rf'/\1{template}\3',
                  path,
                  flags=re.IGNORECASE
              )
              if new_path != path:
                  return f"{parsed.scheme}://{parsed.netloc}{new_path}"

              # No year found, return original
              return url

          def build_tags(conf: dict, field: str = 'link') -> list[str]:
              tags = list(BASE_TAGS)
              lat, lon = get_location_coords(conf)
              continent = get_continent(lat, lon)
              if continent and continent in CONTINENT_TAGS:
                  tags.append(CONTINENT_TAGS[continent])
              sub = conf.get('sub', '').upper()
              if sub in SUB_TO_TAG:
                  tags.append(SUB_TO_TAG[sub])
              if field == 'cfp_link':
                  tags.append(CFP_TAG)
              return tags

          def normalize_url(url: str) -> str:
              return url.rstrip('/').lower() if url else ''

          def get_base_url(url: str) -> str:
              if not url:
                  return ''
              parsed = urlparse(url)
              return f"{parsed.scheme}://{parsed.netloc}"

          def get_root_domain(url: str) -> str:
              """Extract root domain, stripping year-based subdomains, templates, and 'www'.

              Year subdomains (2024-2035), Jinja templates, and 'www' are stripped.
              Meaningful subdomains are kept.

              Examples:
                https://2026.pycon.de/cfp -> https://pycon.de (year subdomain stripped)
                https://ep2026.europython.eu -> https://europython.eu (prefix+year stripped)
                https://{% now ... %}.pycon.de -> https://pycon.de (template stripped)
                https://ep{% now ... %}.europython.eu -> https://europython.eu (prefix+template stripped)
                https://www.pycon.de -> https://pycon.de (www stripped)
                https://us.pycon.org -> https://us.pycon.org (meaningful subdomain kept)
              """
              if not url:
                  return ''
              parsed = urlparse(url)
              netloc = parsed.netloc.lower()

              # Split by dots
              parts = netloc.split('.')

              # Strip 'www' if present
              if len(parts) >= 3 and parts[0] == 'www':
                  parts = parts[1:]

              # Strip year subdomain if present (2024-2035)
              if len(parts) >= 3 and re.match(r'^20(?:2[4-9]|3[0-5])$', parts[0]):
                  parts = parts[1:]

              # Strip prefix+year subdomain (ep2026, scipy2026)
              if len(parts) >= 3 and re.match(r'^[a-z]+20(?:2[4-9]|3[0-5])$', parts[0], re.IGNORECASE):
                  parts = parts[1:]

              # Strip Jinja template subdomain (starts with {%)
              if len(parts) >= 3 and parts[0].startswith('{%'):
                  parts = parts[1:]

              # Strip prefix+template subdomain (ep{% now ... %})
              if len(parts) >= 3 and re.search(r'{%.*%}', parts[0]):
                  parts = parts[1:]

              # Reconstruct domain
              root = '.'.join(parts)
              return f"{parsed.scheme}://{root}"

          def get_url_prefix_before_year(url: str) -> Optional[str]:
              match = YEAR_PATTERN.search(url)
              if not match:
                  return None
              return url[:match.start() + 1]

          # Load existing URLs
          existing_urls = set()
          existing_root_domains = set()
          try:
              with open('existing_urls.txt', 'r') as f:
                  for line in f:
                      url = line.strip()
                      if url:
                          existing_urls.add(url)
                          base = get_base_url(url)
                          if base:
                              existing_urls.add(base.lower())
                          # Also track root domains for subdomain matching
                          root = get_root_domain(url)
                          if root:
                              existing_root_domains.add(root.lower())
          except FileNotFoundError:
              print("::error::existing_urls.txt not found")
              sys.exit(1)

          print(f"Loaded {len(existing_urls)} existing URLs from changedetection.io")
          print(f"Tracking {len(existing_root_domains)} unique root domains")

          def url_exists(url: str, use_base_url: bool = True) -> tuple[bool, str]:
              if url.lower() in existing_urls:
                  return True, "exact URL match"
              host = urlparse(url).netloc.lower().removeprefix("www.")
              if use_base_url and host not in MULTI_SERIES_HOSTS:
                  base = get_base_url(url)
                  if base and base.lower() in existing_urls:
                      return True, "base URL match"
                  # Check root domain to catch subdomain variations (2026.pycon.de vs pycon.de)
                  root = get_root_domain(url)
                  if root and root.lower() in existing_root_domains:
                      return True, "root domain match (subdomain variation)"
              url_prefix = get_url_prefix_before_year(url)
              if url_prefix:
                  prefix_lower = url_prefix.lower()
                  for cached_url in existing_urls:
                      if cached_url.startswith(prefix_lower):
                          return True, "templated version exists"
              return False, ""

          # Load old conferences from git
          try:
              result = subprocess.run(
                  ['git', 'show', f'{LAST_SHA}:_data/conferences.yml'],
                  capture_output=True, text=True, check=True
              )
              old_conferences = yaml.safe_load(result.stdout) or []
          except subprocess.CalledProcessError:
              print(f"Warning: Could not load conferences from {LAST_SHA[:8]}")
              old_conferences = []
          except Exception as e:
              print(f"::error::Failed to load old conferences: {e}")
              sys.exit(1)

          # Load current conferences
          try:
              with open('_data/conferences.yml', 'r') as f:
                  new_conferences = yaml.safe_load(f) or []
          except Exception as e:
              print(f"::error::Failed to load current conferences: {e}")
              sys.exit(1)

          print(f"Old conferences: {len(old_conferences)}")
          print(f"Current conferences: {len(new_conferences)}")

          # Build set of old URLs
          old_urls = set()
          for conf in old_conferences:
              for field in URL_FIELDS.keys():
                  url = conf.get(field)
                  if url:
                      old_urls.add(normalize_url(url))
          print(f"URLs in previous version: {len(old_urls)}")

          # Find new URLs
          watches_to_add = []
          skipped = []

          for conf in new_conferences:
              conf_name = f"{conf.get('conference', 'Unknown')} {conf.get('year', '')}".strip()
              offset = rollover_offset(conf)

              for field, label in URL_FIELDS.items():
                  url = conf.get(field)
                  if not url:
                      continue

                  title = f"{conf_name} - {label}"

                  if normalize_url(url) in old_urls:
                      skipped.append(f"{title}: existed in previous version")
                      continue

                  # An archived copy never changes, so a watch on it would report nothing
                  if urlparse(url).netloc.lower() == "web.archive.org":
                      skipped.append(f"{title}: archived copy")
                      continue

                  use_base_url = (field == 'link')
                  exists, reason = url_exists(url, use_base_url=use_base_url)
                  if exists:
                      skipped.append(f"{title}: {reason}")
                      continue

                  final_url = template_url(url, offset)
                  watches_to_add.append({
                      'url': final_url,
                      'original_url': url if final_url != url else None,
                      'title': title,
                      'tags': build_tags(conf, field),
                      'fetch_backend': 'html_requests',  # CRITICAL: Required by changedetection.io API
                  })

          print(f"\nTo add: {len(watches_to_add)}")
          print(f"Skipped: {len(skipped)}")

          # Write outputs
          with open('watches_to_add.json', 'w') as f:
              json.dump(watches_to_add, f)
          with open('skipped.json', 'w') as f:
              json.dump(skipped, f)

          # Write to GITHUB_OUTPUT
          with open(os.environ.get('GITHUB_OUTPUT', '/dev/null'), 'a') as f:
              f.write(f"count={len(watches_to_add)}\n")

          for watch in watches_to_add:
              print(f"  ADD: {watch['title']}")
              if watch['original_url']:
                  print(f"       Templated: {watch['url']}")
          PYTHON_SCRIPT

      - name: Add watches to changedetection.io
        id: add-watches
        if: inputs.dry_run != true && steps.process.outputs.count != '0'
        run: |
          API_URL="${{ vars.CHANGEDETECTION_URL }}/api/v1/watch"

          ADDED=0
          FAILED=0
          TOTAL=$(jq length watches_to_add.json)

          echo "Adding $TOTAL watches to changedetection.io..."

          while read -r watch; do
            TITLE=$(echo "$watch" | jq -r '.title')

            # Use complete payload from Python (includes fetch_backend)
            PAYLOAD=$(echo "$watch" | jq -c '{url, title, tags, fetch_backend}')

            echo "Adding: $TITLE"

            # Build curl command with proper headers (no eval needed)
            if [ -n "${{ secrets.CF_ACCESS_CLIENT_ID }}" ]; then
              RESPONSE=$(curl -sf -X POST \
                -H "x-api-key: ${{ secrets.CHANGEDETECTION_KEY }}" \
                -H "CF-Access-Client-Id: ${{ secrets.CF_ACCESS_CLIENT_ID }}" \
                -H "CF-Access-Client-Secret: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}" \
                -H "Content-Type: application/json" \
                -d "$PAYLOAD" \
                "$API_URL")
            else
              RESPONSE=$(curl -sf -X POST \
                -H "x-api-key: ${{ secrets.CHANGEDETECTION_KEY }}" \
                -H "Content-Type: application/json" \
                -d "$PAYLOAD" \
                "$API_URL")
            fi

            if [ $? -eq 0 ]; then
              # Validate response has UUID
              UUID=$(echo "$RESPONSE" | jq -r '.uuid // empty')
              if [ -n "$UUID" ]; then
                echo "  ✅ Added successfully (UUID: $UUID)"
                ADDED=$((ADDED + 1))
              else
                echo "  ❌ No UUID in response"
                echo "  Response length: ${#RESPONSE}"
                echo "  Response: $RESPONSE"
                if [ -z "$RESPONSE" ]; then
                  echo "  Likely cause: Missing fetch_backend field in payload"
                fi
                FAILED=$((FAILED + 1))
              fi
            else
              echo "  ❌ Failed to add (check CF Access credentials)"
              FAILED=$((FAILED + 1))
            fi

            sleep 1
          done < <(jq -c '.[]' watches_to_add.json)

          echo ""
          echo "Results: $ADDED added, $FAILED failed out of $TOTAL"

          # Output results
          echo "added=$ADDED" >> $GITHUB_OUTPUT
          echo "failed=$FAILED" >> $GITHUB_OUTPUT

          # Fail the step if any additions failed
          if [ "$FAILED" -gt 0 ]; then
            echo "::error::$FAILED out of $TOTAL watches failed to add"
            echo "::error::Common causes:"
            echo "::error::  - Cloudflare Access authentication failed"
            echo "::error::  - API key is invalid or expired"
            echo "::error::  - Network connectivity issues"
            exit 1
          fi

      # A link the data replaced with its Internet Archive copy is dead or on a
      # domain someone else now owns: its watch would only report that page.
      # Pause it (reversible, history kept) rather than delete it.
      - name: Pause watches on links replaced by an archived copy
        id: pause-watches
        if: inputs.dry_run != true
        env:
          LAST_SHA: ${{ steps.last-run.outputs.sha }}
          CHANGEDETECTION_URL: ${{ vars.CHANGEDETECTION_URL }}
          CHANGEDETECTION_KEY: ${{ secrets.CHANGEDETECTION_KEY }}
          CF_ACCESS_CLIENT_ID: ${{ secrets.CF_ACCESS_CLIENT_ID }}
          CF_ACCESS_CLIENT_SECRET: ${{ secrets.CF_ACCESS_CLIENT_SECRET }}
        run: |
          python3 << 'PAUSE'
          import json
          import os
          import subprocess
          import sys

          import yaml

          sys.path.insert(0, "utils")
          from archived_watches import live_links
          from archived_watches import replaced_links
          from archived_watches import watches_to_pause

          files = ("_data/conferences.yml", "_data/archive.yml", "_data/legacy.yml")

          def entries(ref):
              found = []
              for path in files:
                  if ref:
                      result = subprocess.run(["git", "show", f"{ref}:{path}"], capture_output=True, text=True)
                      text = result.stdout if result.returncode == 0 else ""
                  else:
                      with open(path, encoding="utf-8") as f:
                          text = f.read()
                  found.extend(entry for entry in (yaml.safe_load(text) or []) if isinstance(entry, dict))
              return found

          current = entries(None)
          replaced = replaced_links(entries(os.environ["LAST_SHA"]), current)
          with open("watches.json", encoding="utf-8") as f:
              watches = json.load(f)
          if isinstance(watches, list):
              watches = {watch.get("uuid", str(i)): watch for i, watch in enumerate(watches)}
          pause = watches_to_pause(watches, [link for _, _, link in replaced], live_links(current))
          for conference, year, link in replaced:
              print(f"Replaced by an archived copy: {conference} {year}: {link}")
          with open("pause.txt", "w", encoding="utf-8") as f:
              f.write("".join(f"{uuid}\n" for uuid in pause))
          print(f"Watches to pause: {len(pause)}")
          PAUSE

          PAUSED=0
          FAILED=0
          while read -r UUID; do
            [ -z "$UUID" ] && continue
            ARGS=(-sf -X PUT -H "x-api-key: $CHANGEDETECTION_KEY" -H "Content-Type: application/json")
            if [ -n "$CF_ACCESS_CLIENT_ID" ]; then
              ARGS+=(-H "CF-Access-Client-Id: $CF_ACCESS_CLIENT_ID" -H "CF-Access-Client-Secret: $CF_ACCESS_CLIENT_SECRET")
            fi
            if curl "${ARGS[@]}" -d '{"paused": true}' "$CHANGEDETECTION_URL/api/v1/watch/$UUID" > /dev/null; then
              echo "  ⏸️ Paused $UUID"
              PAUSED=$((PAUSED + 1))
            else
              echo "::warning title=Pause Failed::Could not pause watch $UUID"
              FAILED=$((FAILED + 1))
            fi
          done < pause.txt
          echo "paused=$PAUSED" >> "$GITHUB_OUTPUT"
          echo "Paused $PAUSED, failed $FAILED"

      - name: Generate summary
        if: always()
        run: |
          echo "## Changedetection.io Sync Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          if [ "${{ inputs.dry_run }}" = "true" ]; then
            echo "### 🔍 Dry Run Mode" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            if [ -f watches_to_add.json ]; then
              COUNT=$(jq length watches_to_add.json)
              echo "Would add **$COUNT** watches:" >> $GITHUB_STEP_SUMMARY
              echo "" >> $GITHUB_STEP_SUMMARY
              jq -r '.[] | "- \(.title): \(.url)"' watches_to_add.json >> $GITHUB_STEP_SUMMARY
            fi
          elif [ -f watches_to_add.json ]; then
            TOTAL=$(jq length watches_to_add.json)
            ADDED="${{ steps.add-watches.outputs.added }}"
            FAILED="${{ steps.add-watches.outputs.failed }}"

            if [ "$TOTAL" = "0" ]; then
              echo "No new watches to add." >> $GITHUB_STEP_SUMMARY
            elif [ -n "$ADDED" ]; then
              echo "### Results" >> $GITHUB_STEP_SUMMARY
              echo "" >> $GITHUB_STEP_SUMMARY
              echo "- ✅ Added: **$ADDED**" >> $GITHUB_STEP_SUMMARY
              echo "- ❌ Failed: **$FAILED**" >> $GITHUB_STEP_SUMMARY
              echo "- Total: **$TOTAL**" >> $GITHUB_STEP_SUMMARY
            fi
          fi

          echo "" >> $GITHUB_STEP_SUMMARY

          if [ -f skipped.json ] && [ "$(jq length skipped.json)" -gt 0 ]; then
            echo "<details>" >> $GITHUB_STEP_SUMMARY
            echo "<summary>Skipped ($(jq length skipped.json) URLs)</summary>" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            jq -r '.[]' skipped.json | while read -r item; do
              echo "- $item" >> $GITHUB_STEP_SUMMARY
            done
            echo "</details>" >> $GITHUB_STEP_SUMMARY
          fi

          echo "" >> $GITHUB_STEP_SUMMARY
          echo "---" >> $GITHUB_STEP_SUMMARY
          echo "**Commit range:** \`${{ steps.last-run.outputs.sha }}\`..HEAD" >> $GITHUB_STEP_SUMMARY