docs: publish versioned docs from versions.json with a workflow and build script

.github/actions/setup-docs-builder/action.yaml

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Prepare Docs Builder
+description: 'Install the tools needed to build the documentation in docs'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Rust toolchain
+      uses: ./.github/actions/setup-rustup
+    - name: Setup uv
+      uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e  # v10.3.2
+    - name: Install Python dependencies
+      shell: bash
+      run: uv sync --package datafusion-docs
+    - name: Install Graphviz
+      shell: bash
+      run: |
+        set -x
+        sudo apt-get update
+        sudo apt-get install -y graphviz
+    - name: Install cargo-depgraph
+      uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91  # v2.87.31
+      with:
+        tool: cargo-depgraph@1.6

.github/workflows/docs.yaml

@@ -26,12 +26,17 @@ on:
 
 name: Deploy DataFusion site
 
+# run the publishing workflows one at a time.
+concurrency:
+  group: publish-asf-site
+  cancel-in-progress: false
+
 jobs:
   build-docs:
     permissions:
       contents: write
     name: Build docs
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-slim
     steps:
       - name: Checkout docs sources
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
@@ -42,20 +47,8 @@ jobs:
           ref: asf-site
           path: asf-site
 
-      - name: Setup uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e  # v10.3.2
-
-      - name: Install dependencies
-        run: uv sync --package datafusion-docs
-      - name: Install Graphviz
-        run: |
-          set -x
-          sudo apt-get update
-          sudo apt-get install -y graphviz
-      - name: Install cargo-depgraph
-        uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91  # v2.87.31
-        with:
-          tool: cargo-depgraph@1.6
+      - name: Setup docs builder
+        uses: ./.github/actions/setup-docs-builder
 
       - name: Build docs
         run: |

.github/workflows/docs_pr.yaml

@@ -51,20 +51,7 @@ jobs:
         with:
           submodules: true
           fetch-depth: 1
-      - name: Setup Rust toolchain
-        uses: ./.github/actions/setup-rustup
-      - name: Setup uv
-        uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e  # v10.3.2
-      - name: Install doc dependencies
-        run: uv sync --package datafusion-docs
-      - name: Install Graphviz
-        run: |
-          set -x
-          sudo apt-get update
-          sudo apt-get install -y graphviz
-      - name: Install cargo-depgraph
-        uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91  # v2.87.31
-        with:
-          tool: cargo-depgraph@1.6
+      - name: Setup docs builder
+        uses: ./.github/actions/setup-docs-builder
       - name: Build docs html and check for warnings
         run: ./ci/scripts/check_docs_html.sh # fails on errors

.github/workflows/docs_versions.yaml

@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Publishes the documentation for released versions to
+# https://datafusion.apache.org/versions/<version>/
+#
+# The list of versions is in docs/source/_static/versions.json
+#
+# The current development docs are published from `main` by docs.yaml.
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/docs_versions.yaml
+      - docs/scripts/build_versions.sh
+      - docs/scripts/release/conf.py
+      - docs/source/_static/versions.json
+      - docs/pyproject.toml
+      - uv.lock
+
+name: Deploy DataFusion versioned docs
+
+# run the publishing workflows one at a time.
+concurrency:
+  group: publish-asf-site
+  cancel-in-progress: false
+
+jobs:
+  build-versions:
+    permissions:
+      contents: write
+    name: Build versioned docs
+    runs-on: ubuntu-slim
+    steps:
+      - name: Checkout docs sources
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+
+      - name: Checkout asf-site branch
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          ref: asf-site
+          path: asf-site
+
+      - name: Setup docs builder
+        uses: ./.github/actions/setup-docs-builder
+
+      - name: Build versioned docs
+        run: |
+          set -x
+          docs/scripts/build_versions.sh --output docs/build/versions
+
+      - name: Copy & push the generated HTML
+        run: |
+          set -x
+          cd asf-site/
+          # versions/ mirrors the releases in versions.json; the current docs
+          # at the site root are published by docs.yaml
+          mkdir -p versions
+          # remove any old versions and replace with newly built ones
+          rsync -a --delete ../docs/build/versions/ versions/
+          git status --porcelain
+          if [ "$(git status --porcelain)" != "" ]; then
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git add --all
+            git commit -m 'Publish versioned docs triggered by ${{ github.sha }}'
+            git push
+          fi

dev/release/README.md

@@ -315,71 +315,27 @@ git push apache 55.2.0
 
 #### Publish the versioned documentation
 
-**1. Build the documentation from the release tag** using the configuration
-overlay (`docs/scripts/release/conf.py`) from the main checkout. For example,
-for `55.0.0`, from the repository root:
+Open a PR against `main` that adds the release to
+`docs/source/_static/versions.json` immediately after the `Development (main)`
+entry mark it with `"preferred": true`.
 
-```shell
-export DATAFUSION_DOCS_OVERLAY="$PWD/docs/scripts/release"
-git fetch apache tag 55.0.0
-git worktree add --detach /tmp/datafusion-55.0.0 55.0.0
-cd /tmp/datafusion-55.0.0/docs
-DATAFUSION_DOCS_SOURCE="$PWD/source" DATAFUSION_DOCS_VERSION=55.0.0 \
-  SPHINXOPTS="-W -c $DATAFUSION_DOCS_OVERLAY" \
-  uv run --package datafusion-docs --with sphinx-sitemap ./build.sh
-cd -
-```
-
-Notes:
-
-1. You need to use an absolute path to the overlay in your current checkout.
-2. The `--with sphinx-sitemap` option supplies the sitemap extension for older tags.
-
-To preview the result, build the docs from your main checkout
-(`uv run --package datafusion-docs ./build.sh` in `docs/`), copy the release
-build in under `versions/<version>/`, and serve the tree over HTTP (the version
-picker does not work when the pages are opened directly as files):
-
-```shell
-mkdir -p docs/build/html/versions/55.0.0
-rsync -a /tmp/datafusion-55.0.0/docs/build/html/ docs/build/html/versions/55.0.0/
-python3 -m http.server --directory docs/build/html 8000
-```
-
-Then open http://localhost:8000/ and http://localhost:8000/versions/55.0.0/
-and check the version picker in the navigation bar.
+Once the PR is merged, the
+[Deploy DataFusion versioned docs](../../.github/workflows/docs_versions.yaml)
+workflow builds the documentation for each release and
+pushes the result to the `asf-site` branch.
 
-**2. Add the built site to the `asf-site` branch.** Open a PR against
-`asf-site` that adds the release under `versions/<version>/` (for example
-[#25881](https://github.com/apache/datafusion/pull/25881)):
+To preview a release locally before opening the PR, build the current site
+and then the release with `docs/scripts/build_versions.sh <version>` and serve
+`docs/build/html` over HTTP:
 
 ```shell
-git fetch apache asf-site
-git worktree add -b docs/publish-55.0.0 /tmp/datafusion-asf-site apache/asf-site
-rsync -a --exclude '/.buildinfo' \
-  /tmp/datafusion-55.0.0/docs/build/html/ \
-  /tmp/datafusion-asf-site/versions/55.0.0/
-cd /tmp/datafusion-asf-site
-git add versions/55.0.0
-git commit -m "Publish documentation for 55.0.0"
-git push origin docs/publish-55.0.0
+cd docs
+uv run --package datafusion-docs ./build.sh
+scripts/build_versions.sh 55.2.0
+python3 -m http.server --directory build/html 8000
 ```
 
-The release is live at `https://datafusion.apache.org/versions/55.0.0/` once
-the PR is merged.
-
-**3. Add the release to the version picker.** Open a PR against `main` that
-adds the release to `docs/source/_static/versions.json`. Put the new release
-directly after the `Development (main)` entry and move `"preferred": true` to it, so that
-it is treated as the current stable release. Only add releases that are already
-published, since the picker links to them directly.
-
-Finally, remove the temporary worktrees:
-
-```shell
-git worktree remove /tmp/datafusion-55.0.0
-git worktree remove /tmp/datafusion-asf-site
-```
+Then open http://localhost:8000/ and check the version picker in the navigation bar.
 
 ### 10. Publish on Crates.io
 

docs/README.md

@@ -88,3 +88,8 @@ The Apache Software Foundation provides https://datafusion.apache.org/,
 which serves content based on the configuration in
 [.asf.yaml](https://github.com/apache/datafusion/blob/main/.asf.yaml),
 which specifies the target as https://datafusion.apache.org/.
+
+The documentation for previously released versions is served under
+`/versions/<version>/`. See the
+[release instructions](../dev/release/README.md#publish-the-versioned-documentation)
+for how it is published.