Feat/crawler link check

.github/workflows/link-check-cron-crawler.yml

@@ -0,0 +1,38 @@
+name: link-check-cron-crawler
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  cron-crawler:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Run Crawler Link Checker
+        run: |
+          # Read ignore list and build linkinator skips
+          skips=()
+          if [ -f .github/workflows/link-check-ignore.txt ]; then
+            while IFS= read -r line || [ -n "$line" ]; do
+              # Skip empty lines and comments
+              if [[ ! "$line" =~ ^# ]] && [[ -n "$line" ]]; then
+                skips+=("--skip" "$line")
+              fi
+            done < .github/workflows/link-check-ignore.txt
+          fi
+
+          npx -y linkinator https://doris.apache.org --recurse --check-images --concurrency 5 --timeout 30000 --retry --retry-errors --retry-errors-count 3 --status-code "429:warn" --skip "^(?!https?://(www\.)?doris\.apache\.org)" "${skips[@]}" --format json > link_results.json || echo "HAS_BROKEN=true"
+
+      - name: Send Feishu Notification on failure
+        if: always()
+        env:
+          FEISHU_WEBHOOK: ${{ secrets.FEISHU_WEBHOOK_URL }}
+        run: |
+          node .github/workflows/scripts/feishu-reporter.js

.github/workflows/link-check-ignore.txt

@@ -0,0 +1,15 @@
+# Ignore Mozilla domains which often return 403 to crawlers
+^https?://([^/]+\.)?mozilla\.org
+
+# Ignore LinkedIn which blocks GitHub Action runners
+^https?://([^/]+\.)?linkedin\.com
+
+# Ignore Docker Hub / GitHub internal API URLs which might throttle/block automated requests
+^https?://hub\.docker\.com
+^https?://github\.com/apache/doris-website/pull/
+
+# Ignore Japanese routes until the ja site is published on doris.apache.org
+^https?://doris\.apache\.org/ja/
+
+# Ignore generated 404 pages picked up during crawling
+^https?://doris\.apache\.org/(zh-CN/)?404\.html/?$