diff --git a/.github/actions/setup-docs-builder/action.yaml b/.github/actions/setup-docs-builder/action.yaml new file mode 100644 index 0000000000000..380109b722718 --- /dev/null +++ b/.github/actions/setup-docs-builder/action.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Prepare Docs Builder +description: 'Install the tools needed to build the documentation in docs' +runs: + using: "composite" + steps: + - name: Setup Rust toolchain + uses: ./.github/actions/setup-rustup + - name: Setup uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + - name: Install Python dependencies + shell: bash + run: uv sync --package datafusion-docs + - name: Install Graphviz + shell: bash + run: | + set -x + sudo apt-get update + sudo apt-get install -y graphviz + - name: Install cargo-depgraph + uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 + with: + tool: cargo-depgraph@1.6 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 292a71050842c..783ec8f26a9c7 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -26,12 +26,17 @@ on: name: Deploy DataFusion site +# run the publishing workflows one at a time. +concurrency: + group: publish-asf-site + cancel-in-progress: false + jobs: build-docs: permissions: contents: write name: Build docs - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Checkout docs sources uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -42,20 +47,8 @@ jobs: ref: asf-site path: asf-site - - name: Setup uv - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 - - - name: Install dependencies - run: uv sync --package datafusion-docs - - name: Install Graphviz - run: | - set -x - sudo apt-get update - sudo apt-get install -y graphviz - - name: Install cargo-depgraph - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 - with: - tool: cargo-depgraph@1.6 + - name: Setup docs builder + uses: ./.github/actions/setup-docs-builder - name: Build docs run: | diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml index 571995a108ca1..f62e6c2768583 100644 --- a/.github/workflows/docs_pr.yaml +++ b/.github/workflows/docs_pr.yaml @@ -51,20 +51,7 @@ jobs: with: submodules: true fetch-depth: 1 - - name: Setup Rust toolchain - uses: ./.github/actions/setup-rustup - - name: Setup uv - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 - - name: Install doc dependencies - run: uv sync --package datafusion-docs - - name: Install Graphviz - run: | - set -x - sudo apt-get update - sudo apt-get install -y graphviz - - name: Install cargo-depgraph - uses: taiki-e/install-action@1e29709c0ec63babbe8f266397a465f29a825f91 # v2.87.31 - with: - tool: cargo-depgraph@1.6 + - name: Setup docs builder + uses: ./.github/actions/setup-docs-builder - name: Build docs html and check for warnings run: ./ci/scripts/check_docs_html.sh # fails on errors diff --git a/.github/workflows/docs_versions.yaml b/.github/workflows/docs_versions.yaml new file mode 100644 index 0000000000000..a22727847ea99 --- /dev/null +++ b/.github/workflows/docs_versions.yaml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Publishes the documentation for released versions to +# https://datafusion.apache.org/versions// +# +# The list of versions is in docs/source/_static/versions.json +# +# The current development docs are published from `main` by docs.yaml. + +on: + push: + branches: + - main + paths: + - .github/workflows/docs_versions.yaml + - docs/scripts/build_versions.sh + - docs/scripts/release/conf.py + - docs/source/_static/versions.json + - docs/pyproject.toml + - uv.lock + +name: Deploy DataFusion versioned docs + +# run the publishing workflows one at a time. +concurrency: + group: publish-asf-site + cancel-in-progress: false + +jobs: + build-versions: + permissions: + contents: write + name: Build versioned docs + runs-on: ubuntu-slim + steps: + - name: Checkout docs sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Checkout asf-site branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: asf-site + path: asf-site + + - name: Setup docs builder + uses: ./.github/actions/setup-docs-builder + + - name: Build versioned docs + run: | + set -x + docs/scripts/build_versions.sh --output docs/build/versions + + - name: Copy & push the generated HTML + run: | + set -x + cd asf-site/ + # versions/ mirrors the releases in versions.json; the current docs + # at the site root are published by docs.yaml + mkdir -p versions + # remove any old versions and replace with newly built ones + rsync -a --delete ../docs/build/versions/ versions/ + git status --porcelain + if [ "$(git status --porcelain)" != "" ]; then + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add --all + git commit -m 'Publish versioned docs triggered by ${{ github.sha }}' + git push + fi diff --git a/dev/release/README.md b/dev/release/README.md index 477e01a068a33..e6a81e07bfcc6 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -315,71 +315,27 @@ git push apache 55.2.0 #### Publish the versioned documentation -**1. Build the documentation from the release tag** using the configuration -overlay (`docs/scripts/release/conf.py`) from the main checkout. For example, -for `55.0.0`, from the repository root: +Open a PR against `main` that adds the release to +`docs/source/_static/versions.json` immediately after the `Development (main)` +entry mark it with `"preferred": true`. -```shell -export DATAFUSION_DOCS_OVERLAY="$PWD/docs/scripts/release" -git fetch apache tag 55.0.0 -git worktree add --detach /tmp/datafusion-55.0.0 55.0.0 -cd /tmp/datafusion-55.0.0/docs -DATAFUSION_DOCS_SOURCE="$PWD/source" DATAFUSION_DOCS_VERSION=55.0.0 \ - SPHINXOPTS="-W -c $DATAFUSION_DOCS_OVERLAY" \ - uv run --package datafusion-docs --with sphinx-sitemap ./build.sh -cd - -``` - -Notes: - -1. You need to use an absolute path to the overlay in your current checkout. -2. The `--with sphinx-sitemap` option supplies the sitemap extension for older tags. - -To preview the result, build the docs from your main checkout -(`uv run --package datafusion-docs ./build.sh` in `docs/`), copy the release -build in under `versions//`, and serve the tree over HTTP (the version -picker does not work when the pages are opened directly as files): - -```shell -mkdir -p docs/build/html/versions/55.0.0 -rsync -a /tmp/datafusion-55.0.0/docs/build/html/ docs/build/html/versions/55.0.0/ -python3 -m http.server --directory docs/build/html 8000 -``` - -Then open http://localhost:8000/ and http://localhost:8000/versions/55.0.0/ -and check the version picker in the navigation bar. +Once the PR is merged, the +[Deploy DataFusion versioned docs](../../.github/workflows/docs_versions.yaml) +workflow builds the documentation for each release and +pushes the result to the `asf-site` branch. -**2. Add the built site to the `asf-site` branch.** Open a PR against -`asf-site` that adds the release under `versions//` (for example -[#25881](https://github.com/apache/datafusion/pull/25881)): +To preview a release locally before opening the PR, build the current site +and then the release with `docs/scripts/build_versions.sh ` and serve +`docs/build/html` over HTTP: ```shell -git fetch apache asf-site -git worktree add -b docs/publish-55.0.0 /tmp/datafusion-asf-site apache/asf-site -rsync -a --exclude '/.buildinfo' \ - /tmp/datafusion-55.0.0/docs/build/html/ \ - /tmp/datafusion-asf-site/versions/55.0.0/ -cd /tmp/datafusion-asf-site -git add versions/55.0.0 -git commit -m "Publish documentation for 55.0.0" -git push origin docs/publish-55.0.0 +cd docs +uv run --package datafusion-docs ./build.sh +scripts/build_versions.sh 55.2.0 +python3 -m http.server --directory build/html 8000 ``` -The release is live at `https://datafusion.apache.org/versions/55.0.0/` once -the PR is merged. - -**3. Add the release to the version picker.** Open a PR against `main` that -adds the release to `docs/source/_static/versions.json`. Put the new release -directly after the `Development (main)` entry and move `"preferred": true` to it, so that -it is treated as the current stable release. Only add releases that are already -published, since the picker links to them directly. - -Finally, remove the temporary worktrees: - -```shell -git worktree remove /tmp/datafusion-55.0.0 -git worktree remove /tmp/datafusion-asf-site -``` +Then open http://localhost:8000/ and check the version picker in the navigation bar. ### 10. Publish on Crates.io diff --git a/docs/README.md b/docs/README.md index b55d770cdc693..a94d2bc109f70 100644 --- a/docs/README.md +++ b/docs/README.md @@ -88,3 +88,8 @@ The Apache Software Foundation provides https://datafusion.apache.org/, which serves content based on the configuration in [.asf.yaml](https://github.com/apache/datafusion/blob/main/.asf.yaml), which specifies the target as https://datafusion.apache.org/. + +The documentation for previously released versions is served under +`/versions//`. See the +[release instructions](../dev/release/README.md#publish-the-versioned-documentation) +for how it is published. diff --git a/docs/scripts/build_versions.sh b/docs/scripts/build_versions.sh new file mode 100755 index 0000000000000..d71813045096a --- /dev/null +++ b/docs/scripts/build_versions.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Build the documentation for each release listed in docs/source/_static/versions.json +# +# The release is checked out into a temporary worktree, writing the result +# to //. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DOCS_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +REPO_DIR="$(cd "${DOCS_DIR}/.." && pwd)" +OVERLAY_DIR="${SCRIPT_DIR}/release" +MANIFEST="${DOCS_DIR}/source/_static/versions.json" + +# Where to fetch release tags from (forks usually do not contain release tags) +UPSTREAM_URL="${DATAFUSION_UPSTREAM_URL:-https://github.com/apache/datafusion.git}" + +OUTPUT_DIR="${DOCS_DIR}/build/html/versions" +WORK_DIR="" +VERSIONS=() + +usage() { + cat </ + (default: docs/build/html/versions, so that the versions + appear under a site built with docs/build.sh) + -w, --work DIR Check the tags out under DIR/ instead of a + temporary directory + -h, --help Show this help message + +Environment: + DATAFUSION_UPSTREAM_URL Repository to fetch missing release tags from + (default: ${UPSTREAM_URL}) + +Example: preview the site with the 55.0.0 docs under versions/55.0.0/ + cd docs + uv run --package datafusion-docs ./build.sh + scripts/build_versions.sh 55.0.0 + python3 -m http.server --directory build/html 8000 +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -o|--output) + OUTPUT_DIR="$2" + shift 2 + ;; + -w|--work) + WORK_DIR="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + *) + VERSIONS+=("$1") + shift + ;; + esac +done + +for tool in git uv python3; do + if ! command -v "${tool}" >/dev/null 2>&1; then + echo "${tool} is required." >&2 + exit 1 + fi +done + +# Releases listed in the version picker manifest, newest first. +list_versions() { + python3 - "${MANIFEST}" <<'PY' +import json +import sys + +with open(sys.argv[1]) as f: + for entry in json.load(f): + if entry["version"] != "main": + print(entry["version"]) +PY +} + +if [[ ${#VERSIONS[@]} -eq 0 ]]; then + while IFS= read -r version; do + VERSIONS+=("${version}") + done < <(list_versions) +fi + +if [[ ${#VERSIONS[@]} -eq 0 ]]; then + echo "No versions to build." >&2 + exit 1 +fi + +TEMP_WORK_DIR="" +if [[ -z "${WORK_DIR}" ]]; then + TEMP_WORK_DIR="$(mktemp -d -t datafusion-docs-versions.XXXXXX)" + WORK_DIR="${TEMP_WORK_DIR}" +fi +mkdir -p "${WORK_DIR}" "${OUTPUT_DIR}" +WORK_DIR="$(cd "${WORK_DIR}" && pwd)" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" + +cleanup() { + if [[ -n "${TEMP_WORK_DIR}" ]]; then + for worktree in "${TEMP_WORK_DIR}"/*/; do + [[ -d "${worktree}" ]] || continue + git -C "${REPO_DIR}" worktree remove --force "${worktree}" >/dev/null 2>&1 || true + done + rm -rf "${TEMP_WORK_DIR}" + git -C "${REPO_DIR}" worktree prune + fi +} +trap cleanup EXIT + +# Make sure this checkout has the release tag, fetching it from upstream +# otherwise (a shallow CI checkout stays shallow). +fetch_tag() { + local version="$1" + if git -C "${REPO_DIR}" rev-parse -q --verify "refs/tags/${version}^{commit}" >/dev/null; then + return + fi + echo "Fetching tag ${version} from ${UPSTREAM_URL}" + local depth=() + if [[ "$(git -C "${REPO_DIR}" rev-parse --is-shallow-repository)" == "true" ]]; then + depth=(--depth 1) + fi + git -C "${REPO_DIR}" fetch "${depth[@]}" "${UPSTREAM_URL}" "refs/tags/${version}:refs/tags/${version}" +} + +build_version() { + local version="$1" + local worktree="${WORK_DIR}/${version}" + local output="${OUTPUT_DIR}/${version}" + + echo "==> Building documentation for ${version}" + fetch_tag "${version}" + + if [[ -e "${worktree}" ]]; then + git -C "${REPO_DIR}" worktree remove --force "${worktree}" >/dev/null 2>&1 || rm -rf "${worktree}" + fi + git -C "${REPO_DIR}" worktree add --detach "${worktree}" "refs/tags/${version}" + + # The workspace dependency graph is generated at build time and embedded in + # the contributor guide (docs/build.sh does the same for the current docs). + if [[ -x "${worktree}/docs/scripts/generate_dependency_graph.sh" ]]; then + "${worktree}/docs/scripts/generate_dependency_graph.sh" + fi + + rm -rf "${output}" + ( + cd "${DOCS_DIR}" + DATAFUSION_DOCS_SOURCE="${worktree}/docs/source" \ + DATAFUSION_DOCS_VERSION="${version}" \ + uv run --package datafusion-docs \ + sphinx-build -b html -W -c "${OVERLAY_DIR}" "${worktree}/docs/source" "${output}" + ) + # Sphinx's incremental build state is not part of the published site. + rm -f "${output}/.buildinfo" + + git -C "${REPO_DIR}" worktree remove --force "${worktree}" + echo "==> Wrote ${output}" +} + +for version in "${VERSIONS[@]}"; do + build_version "${version}" +done diff --git a/docs/scripts/release/conf.py b/docs/scripts/release/conf.py index 8776449cb9711..9a403491c8ece 100644 --- a/docs/scripts/release/conf.py +++ b/docs/scripts/release/conf.py @@ -34,6 +34,8 @@ checkout: sphinx-build -c docs/scripts/release + +See docs/scripts/build_versions.sh for building past releases """ import os