diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..a04a038cf --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "docker" + directory: "/generator/build" + schedule: + interval: "weekly" + day: "sunday" + ignore: + # The build image stays on the Ubuntu release the hub package is built for + - dependency-name: "ubuntu" diff --git a/.github/workflows/bump-hugo-version.yml b/.github/workflows/bump-hugo-version.yml deleted file mode 100644 index 75ab2c260..000000000 --- a/.github/workflows/bump-hugo-version.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Hugo version update - -on: - schedule: - - cron: "0 0 * * 0" #perform every sunday's night - workflow_dispatch: - -jobs: - update_hugo_version: - if: github.repository_owner == 'cfengine' || github.repository_owner == 'mendersoftware' || github.repository_owner == 'NorthernTechHQ' - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@v4 - - name: Fetch latest Hugo release info - id: fetch_hugo_version - run: | - release_info=$(curl -s https://api.github.com/repos/gohugoio/hugo/releases/latest) - version=$(echo "$release_info" | jq -r '.tag_name' | sed 's/v//') - # download checksums and find checksum of hugo_${version}_Linux-64bit.tar.gz - checksum=$( curl -sSL https://github.com/gohugoio/hugo/releases/download/v${version}/hugo_${version}_checksums.txt | grep hugo_${version}_Linux-64bit.tar.gz | grep -oE "[a-z0-9]{64}") - echo "version=$version" >> $GITHUB_OUTPUT - echo "checksum=$checksum" >> $GITHUB_OUTPUT - - - name: Update install_hugo.sh - run: | - sed -i "s|https://github.com/gohugoio/hugo/releases/download/v.*_Linux-64bit.tar.gz|https://github.com/gohugoio/hugo/releases/download/v${{ steps.fetch_hugo_version.outputs.version }}/hugo_${{ steps.fetch_hugo_version.outputs.version }}_Linux-64bit.tar.gz|" generator/build/install_hugo.sh - sed -i "s/.*sha256sum.*/ echo \"${{ steps.fetch_hugo_version.outputs.checksum }} hugo.tar.gz\" | sha256sum -c/" generator/build/install_hugo.sh - - name: Create Pull Request - uses: cfengine/create-pull-request@v7 - with: - commit-message: Upgraded Hugo version to ${{ steps.fetch_hugo_version.outputs.version }} - title: Upgraded Hugo version to ${{ steps.fetch_hugo_version.outputs.version }} - body: | - This PR updates the version of Hugo used in the generator/build/install_hugo.sh to ${{ steps.fetch_hugo_version.outputs.version }}. - reviewers: | - aleksandrychev - - branch: hugo-version-update-${{ steps.fetch_hugo_version.outputs.version }} - sign-commits: true - delete-branch: true diff --git a/build-locally.sh b/build-locally.sh index c13bc9552..d6fce16c8 100755 --- a/build-locally.sh +++ b/build-locally.sh @@ -14,8 +14,6 @@ # PACKAGE_BUILD (default: n/a — unused with cf-remote) # LTS_VERSION (default: empty) # DOCKER docker binary to use (default: docker) -# IMAGE_NAME tag for the build image (default: cfengine-docs-hugo) -# SKIP_PUBLISH=1 skip the _publish.sh step (just build) # SKIP_SERVE=1 don't start the serve container (site + search) # SERVE_ONLY=1 skip the build; just (re)start the serve container # against the previously built site @@ -40,7 +38,6 @@ PACKAGE_UPLOAD_DIRECTORY="${PACKAGE_UPLOAD_DIRECTORY:-n/a}" PACKAGE_BUILD="${PACKAGE_BUILD:-n/a}" LTS_VERSION="${LTS_VERSION:-}" DOCKER="${DOCKER:-docker}" -IMAGE_NAME="${IMAGE_NAME:-cfengine-docs-hugo}" SITE_PORT="${SITE_PORT:-8000}" SERVE_IMAGE="${SERVE_IMAGE:-cfengine-docs-serve}" SERVE_CONTAINER="cfengine-docs-serve" @@ -113,10 +110,8 @@ if [ -n "${SERVE_ONLY:-}" ]; then fi # 1. Clone (or update) the sibling repos under tmp/cache/, then export a -# clean working copy (no .git) to tmp/work/. We mount the .git-free -# copy because the container does `chmod -R` over each repo, and on -# macOS Docker bind mounts can't chmod git pack files written by the -# host user. +# clean working copy (no .git) to tmp/work/, which is the docker build +# context. echo "==> Preparing sibling repos under $TMP_DIR" for entry in "${REPOS[@]}"; do # shellcheck disable=SC2086 @@ -169,51 +164,28 @@ rsync -a --delete \ "$SCRIPT_DIR/" "$DOC_WORK/" rm -f "$DOC_WORK/content/cfdoc_log.markdown" -# 2. Build the docker image (only if it's not already built). -if ! "$DOCKER" image inspect "$IMAGE_NAME" >/dev/null 2>&1; then - echo "==> Building docker image $IMAGE_NAME" - "$DOCKER" build --tag "$IMAGE_NAME" "$SCRIPT_DIR/generator/build" -else - echo "==> Reusing docker image $IMAGE_NAME (delete it to rebuild)" -fi - -# 3. Run the build inside the container. -# main.sh expects /nt/{documentation,core,nova,enterprise,masterfiles,nt-docs}. -# We bind-mount this checkout as /nt/documentation and each tmp/ as -# its sibling, so nothing outside this directory is touched. -echo "==> Running documentation build in container" -RUN_FLAGS=( - --rm - -v "$DOC_WORK:/nt/documentation" -) -for entry in "${REPOS[@]}"; do - # shellcheck disable=SC2086 - set -- $entry - RUN_FLAGS+=(-v "$WORK_DIR/$1:/nt/$1") -done - -"$DOCKER" run "${RUN_FLAGS[@]}" "$IMAGE_NAME" \ - bash -x documentation/generator/build/main.sh \ - "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" \ - "$PACKAGE_BUILD" "$LTS_VERSION" - -# 4. Optionally package the result (mirrors the Jenkins pipeline). -# _publish.sh mutates _site in place for the offline archive, so restore it from packed-for-shipping.tar.gz (what production deploys) afterwards. -if [ -z "${SKIP_PUBLISH:-}" ]; then - echo "==> Packaging output" - mkdir -p "$DOC_WORK/output" - "$DOCKER" run "${RUN_FLAGS[@]}" -v "$DOC_WORK/output:/nt/output" "$IMAGE_NAME" \ - bash -x documentation/generator/_scripts/_publish.sh "$BRANCH" - tar -xzf "$DOC_WORK/output/packed-for-shipping.tar.gz" -C "$DOC_WORK/generator" -fi - -# 5. Serve the site with a working search (see serve_site above). +# 2. Run the build (the generator/build/Dockerfile stages), the tarballs are exported to output/. +echo "==> Building documentation" +"$DOCKER" build \ + --file "$DOC_WORK/generator/build/Dockerfile" \ + --build-arg BRANCH="$BRANCH" \ + --build-arg PACKAGE_JOB="$PACKAGE_JOB" \ + --build-arg PACKAGE_UPLOAD_DIRECTORY="$PACKAGE_UPLOAD_DIRECTORY" \ + --build-arg PACKAGE_BUILD="$PACKAGE_BUILD" \ + --build-arg LTS_VERSION="$LTS_VERSION" \ + --output "type=local,dest=$DOC_WORK/output" \ + "$WORK_DIR" + +# 3. Unpack the site from packed-for-shipping.tar.gz (what production deploys). +tar -xzf "$DOC_WORK/output/packed-for-shipping.tar.gz" -C "$DOC_WORK/generator" + +# 4. Serve the site with a working search (see serve_site above). if [ -z "${SKIP_SERVE:-}" ]; then serve_site fi echo "==> Done. Generated site is in: $DOC_WORK/generator/_site" -echo " Tarballs (if packaged) are in: $DOC_WORK/output/" +echo " Tarballs are in: $DOC_WORK/output/" if [ -z "${SKIP_SERVE:-}" ]; then echo " Documentation is served at:" echo " http://127.0.0.1:$SITE_PORT/" diff --git a/generator/README.org b/generator/README.org index 556ca2b6e..2f3772014 100644 --- a/generator/README.org +++ b/generator/README.org @@ -25,7 +25,7 @@ example ={{< params "CFE_manuals_version" >}}=. :END: The documentation is rendered using [[https://gohugo.io/][Hugo]]. The Hugo site lives in [[../hugo][=hugo/=]] -and the version used by the build is installed by [[./build/install_hugo.sh][=build/install_hugo.sh=]]. +and it is built in the official Hugo image, pinned as the =hugo= stage of [[./build/Dockerfile][=build/Dockerfile=]]. ** Building Locally :PROPERTIES: @@ -102,6 +102,10 @@ For example: ./_scripts/cfdoc_preprocess.py master || exit 4 bash -x ./_scripts/_run.sh master || exit 5 + + (cd ../hugo && npm ci && npm run build:all) || exit 6 + + bash -x ./_scripts/_postprocess.sh master || exit 7 #+END_SRC @@ -117,8 +121,9 @@ Once the syntax map has been regenerated [[https://github.com/cfengine/documenta (whatever that means), resolving links, expanding our custom macros and creating printable (dead trees) versions of the documentation. -After pre-processing is complete [[https://github.com/cfengine/documentation/blob/master/generator/_scripts/_run.sh][_run.sh]] performs the Hugo build and -post processes the build with [[https://github.com/cfengine/documentation/blob/master/generator/_scripts/cfdoc_postprocess.py][cfdoc_postprocess.py]] +After pre-processing is complete [[https://github.com/cfengine/documentation/blob/master/generator/_scripts/_run.sh][_run.sh]] prepares the Hugo site, Hugo builds it and +[[https://github.com/cfengine/documentation/blob/master/generator/_scripts/_postprocess.sh][_postprocess.sh]] post processes the build with [[https://github.com/cfengine/documentation/blob/master/generator/_scripts/cfdoc_postprocess.py][cfdoc_postprocess.py]]. In CI these +steps run as the stages of [[./build/Dockerfile][=build/Dockerfile=]]. *** Publishing :PROPERTIES: diff --git a/generator/_scripts/_postprocess.sh b/generator/_scripts/_postprocess.sh new file mode 100755 index 000000000..00d9c4a49 --- /dev/null +++ b/generator/_scripts/_postprocess.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Post-processes the site built in the hugo stage of build/Dockerfile. + +if [ -z "$WRKDIR" ] +then + echo Environment WRKDIR is not set, setting it to current working directory + WRKDIR=`pwd` + export WRKDIR +fi + +export LC_ALL=C.UTF-8 + +cd $WRKDIR/documentation/generator +mkdir -p $WRKDIR/documentation/generator/_site/assets/searchIndex/documents +cp -rf $WRKDIR/documentation/hugo/scripts/search/index/searchIndex/* $WRKDIR/documentation/generator/_site/assets/searchIndex +cp -rf $WRKDIR/documentation/hugo/scripts/search/index/documents/* $WRKDIR/documentation/generator/_site/assets/searchIndex/documents/ +cp -f $WRKDIR/documentation/redirects.conf $WRKDIR/documentation/generator/_site/ + +$WRKDIR/documentation/generator/_scripts/cfdoc_postprocess.py "$@" +if [ "$?" -gt "0" ]; then + exit 2; +fi diff --git a/generator/_scripts/_run.sh b/generator/_scripts/_run.sh index 870e55370..752d934f5 100755 --- a/generator/_scripts/_run.sh +++ b/generator/_scripts/_run.sh @@ -23,19 +23,5 @@ find $WRKDIR/documentation/hugo/content -type f -name "*.markdown" -exec grep -l find $WRKDIR/documentation/hugo/content -name "*.include.markdown" -type f -delete -# Hugo build +# Hugo site (built in the hugo stage of build/Dockerfile) cp -rn $WRKDIR/nt-docs/* $WRKDIR/documentation/hugo/ -cd $WRKDIR/documentation/hugo -npm ci -npm run build:all || exit 2 - -cd $WRKDIR/documentation/generator -mkdir -p $WRKDIR/documentation/generator/_site/assets/searchIndex/documents -cp -rf $WRKDIR/documentation/hugo/scripts/search/index/searchIndex/* $WRKDIR/documentation/generator/_site/assets/searchIndex -cp -rf $WRKDIR/documentation/hugo/scripts/search/index/documents/* $WRKDIR/documentation/generator/_site/assets/searchIndex/documents/ -cp -f $WRKDIR/documentation/redirects.conf $WRKDIR/documentation/generator/_site/ - -$WRKDIR/documentation/generator/_scripts/cfdoc_postprocess.py "$@" -if [ "$?" -gt "0" ]; then - exit 2; -fi diff --git a/generator/build/Dockerfile b/generator/build/Dockerfile index 148bf5028..242ec8c67 100644 --- a/generator/build/Dockerfile +++ b/generator/build/Dockerfile @@ -1,4 +1,7 @@ -FROM ubuntu:22.04 +# Build context is the directory with all the repos (documentation, core, nova, enterprise, masterfiles, nt-docs) + +# Pre-processing +FROM ubuntu:22.04 AS tools # We need sudo before all RUN apt-get -qy update && \ @@ -7,16 +10,38 @@ RUN apt-get -qy update && \ # Add jenkins user RUN useradd -s /bin/bash jenkins RUN echo "jenkins ALL=(ALL) NOPASSWD: ALL">> /etc/sudoers -RUN mkdir -p /home/jenkins -RUN chown -R jenkins:jenkins /home/jenkins +RUN mkdir -p /home/jenkins /nt +RUN chown -R jenkins:jenkins /home/jenkins /nt # Run rest of setup as jenkins user -COPY install.sh / -COPY install_hugo.sh / +COPY documentation/generator/build/install.sh / USER jenkins WORKDIR /home/jenkins RUN sudo apt-get update -qy && bash -x /install.sh -RUN bash /install_hugo.sh -# This is where our repos will be WORKDIR /nt +COPY --chown=jenkins:jenkins . /nt + +ARG BRANCH +ARG PACKAGE_JOB +ARG PACKAGE_UPLOAD_DIRECTORY +ARG PACKAGE_BUILD +ARG LTS_VERSION +RUN bash -x documentation/generator/build/main.sh "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" "$PACKAGE_BUILD" "$LTS_VERSION" + +# Hugo build +FROM ghcr.io/gohugoio/hugo:v0.169.0@sha256:a146e0639230bbc73358e6563240f0a6103466f0390b80a457c775068ec802a9 AS hugo +COPY --from=tools --chown=hugo:hugo /nt/documentation /nt/documentation +WORKDIR /nt/documentation/hugo +RUN npm ci && npm run build:all + +# Post-processing and packing into /nt/output +FROM tools AS postprocess +COPY --from=hugo --chown=jenkins:jenkins /nt/documentation /nt/documentation +ARG BRANCH +RUN bash -x documentation/generator/_scripts/_postprocess.sh "$BRANCH" && \ + bash -x documentation/generator/_scripts/_publish.sh "$BRANCH" + +# Just the tarballs, exported with --output type=local +FROM scratch AS output +COPY --from=postprocess /nt/output / diff --git a/generator/build/Dockerfile.dockerignore b/generator/build/Dockerfile.dockerignore new file mode 100644 index 000000000..72bad44c2 --- /dev/null +++ b/generator/build/Dockerfile.dockerignore @@ -0,0 +1,3 @@ +# Build context is the directory with all the repos (see Dockerfile), none of their git history is needed +**/.git +output diff --git a/generator/build/README.md b/generator/build/README.md index bab9864a5..8d25be9fd 100644 --- a/generator/build/README.md +++ b/generator/build/README.md @@ -11,6 +11,7 @@ You will need to have the following repos checked out: - nova (used for changelog) - enterprise (used for changelog) - masterfiles (used to document masterfies) +- nt-docs (Hugo theme and build scripts) - documentation ## Usage @@ -33,26 +34,45 @@ If you have buildah installed: - `$PACKAGE_BUILD` - RELEASE of the build to be downloaded, usually 1 + - `$LTS_VERSION` - LTS version, only used when `$BRANCH` is `lts` + 3. `cd` to directory with all repos and run `run.sh` from this dir ## Details -To make a container, cd to this directory and run: - - buildah build-using-dockerfile -t docs . - -or - - docker build --tag docs . - -To run docs job in a container, run: - - c=$(buildah from -v $PWD:/nt docs) - buildah run $c bash -x documentation/generator/build/main.sh $BRANCH $PACKAGE_JOB $PACKAGE_UPLOAD_DIRECTORY $PACKAGE_BUILD - buildah run $c bash -x documentation/generator/_scripts/_publish.sh $BRANCH - buildah rm $c - -or - - docker run --rm -it -v $PWD:/nt docs bash -x documentation/generator/build/main.sh $BRANCH $PACKAGE_JOB $PACKAGE_UPLOAD_DIRECTORY $PACKAGE_BUILD - docker run --rm -it -v $PWD:/nt docs bash -x documentation/generator/_scripts/_publish.sh $BRANCH +The whole build runs in the stages of the [Dockerfile](Dockerfile), with the +directory holding all the repos as build context: + +1. `tools` - Ubuntu build environment, runs `main.sh` (hub package, syntax + JSON, pre-processing, `_scripts/_run.sh` to prepare the Hugo site) +2. `hugo` - official Hugo image, runs `npm ci && npm run build:all` +3. `postprocess` - runs `_scripts/_postprocess.sh` and `_scripts/_publish.sh` +4. `output` - just the tarballs from `/nt/output` + +`run.sh` runs the build with buildah. To run it by hand, from the directory with +all repos: + + buildah build-using-dockerfile \ + --file documentation/generator/build/Dockerfile \ + --ignorefile documentation/generator/build/Dockerfile.dockerignore \ + --build-arg BRANCH=$BRANCH \ + --build-arg PACKAGE_JOB=$PACKAGE_JOB \ + --build-arg PACKAGE_UPLOAD_DIRECTORY=$PACKAGE_UPLOAD_DIRECTORY \ + --build-arg PACKAGE_BUILD=$PACKAGE_BUILD \ + --output type=local,dest=output . + +or with docker, which picks up `Dockerfile.dockerignore` by itself: + + docker build \ + --file documentation/generator/build/Dockerfile \ + --build-arg BRANCH=$BRANCH \ + --build-arg PACKAGE_JOB=$PACKAGE_JOB \ + --build-arg PACKAGE_UPLOAD_DIRECTORY=$PACKAGE_UPLOAD_DIRECTORY \ + --build-arg PACKAGE_BUILD=$PACKAGE_BUILD \ + --output type=local,dest=output . + +The tarballs (`packed-for-shipping.tar.gz` and +`cfengine-documentation-$BRANCH.tar.gz`) end up in `./output`. + +To build and serve the docs locally with docker, including cloning the repos, +use `build-locally.sh` in the root of the documentation repo. diff --git a/generator/build/install.sh b/generator/build/install.sh index cc743d8b0..69a3cbc8e 100644 --- a/generator/build/install.sh +++ b/generator/build/install.sh @@ -4,7 +4,7 @@ set -ex # All of these commands are executed by the jenkins user. -sudo apt-get install -y -qq curl gnupg2 wget git +sudo apt-get install -y -qq wget ca-certificates # some of packages below depend on tzdata. Installing it without DEBIAN_FRONTEND=noninteractive causes it to ask where in the world are you, interrupting the build process DEBIAN_FRONTEND=noninteractive sudo --preserve-env=DEBIAN_FRONTEND apt-get install -y -qq tzdata @@ -12,9 +12,3 @@ DEBIAN_FRONTEND=noninteractive sudo --preserve-env=DEBIAN_FRONTEND apt-get insta # Python is needed for our pre and post processing scripts. sudo apt-get install -y -qq python3 - -curl -sL https://deb.nodesource.com/setup_21.x -o nodesource_setup.sh -sudo bash nodesource_setup.sh -sudo apt-get install -y -qq nodejs -node --version -npm --version diff --git a/generator/build/install_hugo.sh b/generator/build/install_hugo.sh deleted file mode 100644 index c67bf328a..000000000 --- a/generator/build/install_hugo.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# go and git are needed by Hugo to download modules (llms-txt) -sudo apt-get update -qq -sudo apt-get install -y -qq golang-1.26-go git -sudo ln -sf /usr/lib/go-1.26/bin/go /usr/local/bin/go - -wget -nv https://github.com/gohugoio/hugo/releases/download/v0.169.0/hugo_0.169.0_Linux-64bit.tar.gz -O hugo.tar.gz - echo "2d2f0bfe357d8cb4cec02f6bb28a3bff34978c8dda44b618029c5eb9d66765a7 hugo.tar.gz" | sha256sum -c -if [ $? -eq 1 ]; then - exit 2 -fi - -tar -zxvf hugo.tar.gz -sudo mv hugo /usr/local/bin/hugo -sudo chmod +x /usr/local/bin/hugo -rm hugo.tar.gz diff --git a/generator/build/main.sh b/generator/build/main.sh index c0418b078..a616dfb61 100644 --- a/generator/build/main.sh +++ b/generator/build/main.sh @@ -144,6 +144,5 @@ fi set +x export LC_ALL=C.UTF-8 -bash ./build/install_hugo.sh echo "+ bash -x ./_scripts/_run.sh $BRANCH || exit 6" bash -x ./_scripts/_run.sh "$BRANCH" || exit 6 diff --git a/generator/build/run.sh b/generator/build/run.sh index 391f2b7cc..f34261ceb 100644 --- a/generator/build/run.sh +++ b/generator/build/run.sh @@ -3,11 +3,6 @@ set -ex trap "echo FAILURE" ERR -image_name=docs-hugo -if ! buildah inspect ${image_name} >/dev/null 2>&1; then - buildah build-using-dockerfile -t ${image_name} documentation/generator/build -fi - # Current path must have the following repos cloned: # * core (used for changelog, examples) # * nova (used for changelog) @@ -31,7 +26,16 @@ elif [ -n "$BRANCH_NAME" ]; then BRANCH="$BRANCH_NAME" fi -c=$(buildah from -v "$PWD":/nt ${image_name}) -trap 'buildah run "$c" bash -c "sudo chown -R root:root /nt; sudo chmod -R a+rwX /nt"; buildah rm "$c" >/dev/null' EXIT -buildah run "$c" bash -x documentation/generator/build/main.sh "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" "$PACKAGE_BUILD" "$LTS_VERSION" -buildah run "$c" bash -x documentation/generator/_scripts/_publish.sh "$BRANCH" +# The whole build runs in the Dockerfile stages (pre-process, Hugo, post-process) with the +# current path as build context. The tarballs end up in ./output for the Jenkins publish step. +# No --layers: each build starts fresh, so results are never stale and no cached copies of the workspace pile up on the host. +buildah build-using-dockerfile \ + --file documentation/generator/build/Dockerfile \ + --ignorefile documentation/generator/build/Dockerfile.dockerignore \ + --build-arg BRANCH="$BRANCH" \ + --build-arg PACKAGE_JOB="$PACKAGE_JOB" \ + --build-arg PACKAGE_UPLOAD_DIRECTORY="$PACKAGE_UPLOAD_DIRECTORY" \ + --build-arg PACKAGE_BUILD="$PACKAGE_BUILD" \ + --build-arg LTS_VERSION="$LTS_VERSION" \ + --output type=local,dest=output \ + .