ENT-14913: Built docs in Dockerfile stages using the official Hugo image

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "docker"
+    directory: "/generator/build"
+    schedule:
+      interval: "weekly"
+      day: "sunday"
+    ignore:
+      # The build image stays on the Ubuntu release the hub package is built for
+      - dependency-name: "ubuntu"

build-locally.sh

@@ -14,8 +14,6 @@
 #   PACKAGE_BUILD             (default: n/a — unused with cf-remote)
 #   LTS_VERSION               (default: empty)
 #   DOCKER                  docker binary to use (default: docker)
-#   IMAGE_NAME              tag for the build image (default: cfengine-docs-hugo)
-#   SKIP_PUBLISH=1          skip the _publish.sh step (just build)
 #   SKIP_SERVE=1            don't start the serve container (site + search)
 #   SERVE_ONLY=1            skip the build; just (re)start the serve container
 #                           against the previously built site
@@ -40,7 +38,6 @@ PACKAGE_UPLOAD_DIRECTORY="${PACKAGE_UPLOAD_DIRECTORY:-n/a}"
 PACKAGE_BUILD="${PACKAGE_BUILD:-n/a}"
 LTS_VERSION="${LTS_VERSION:-}"
 DOCKER="${DOCKER:-docker}"
-IMAGE_NAME="${IMAGE_NAME:-cfengine-docs-hugo}"
 SITE_PORT="${SITE_PORT:-8000}"
 SERVE_IMAGE="${SERVE_IMAGE:-cfengine-docs-serve}"
 SERVE_CONTAINER="cfengine-docs-serve"
@@ -113,10 +110,8 @@ if [ -n "${SERVE_ONLY:-}" ]; then
 fi
 
 # 1. Clone (or update) the sibling repos under tmp/cache/, then export a
-#    clean working copy (no .git) to tmp/work/. We mount the .git-free
-#    copy because the container does `chmod -R` over each repo, and on
-#    macOS Docker bind mounts can't chmod git pack files written by the
-#    host user.
+#    clean working copy (no .git) to tmp/work/, which is the docker build
+#    context.
 echo "==> Preparing sibling repos under $TMP_DIR"
 for entry in "${REPOS[@]}"; do
     # shellcheck disable=SC2086
@@ -169,51 +164,28 @@ rsync -a --delete \
     "$SCRIPT_DIR/" "$DOC_WORK/"
 rm -f "$DOC_WORK/content/cfdoc_log.markdown"
 
-# 2. Build the docker image (only if it's not already built).
-if ! "$DOCKER" image inspect "$IMAGE_NAME" >/dev/null 2>&1; then
-    echo "==> Building docker image $IMAGE_NAME"
-    "$DOCKER" build --tag "$IMAGE_NAME" "$SCRIPT_DIR/generator/build"
-else
-    echo "==> Reusing docker image $IMAGE_NAME (delete it to rebuild)"
-fi
-
-# 3. Run the build inside the container.
-# main.sh expects /nt/{documentation,core,nova,enterprise,masterfiles,nt-docs}.
-# We bind-mount this checkout as /nt/documentation and each tmp/<repo> as
-# its sibling, so nothing outside this directory is touched.
-echo "==> Running documentation build in container"
-RUN_FLAGS=(
-    --rm
-    -v "$DOC_WORK:/nt/documentation"
-)
-for entry in "${REPOS[@]}"; do
-    # shellcheck disable=SC2086
-    set -- $entry
-    RUN_FLAGS+=(-v "$WORK_DIR/$1:/nt/$1")
-done
-
-"$DOCKER" run "${RUN_FLAGS[@]}" "$IMAGE_NAME" \
-    bash -x documentation/generator/build/main.sh \
-        "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" \
-        "$PACKAGE_BUILD" "$LTS_VERSION"
-
-# 4. Optionally package the result (mirrors the Jenkins pipeline).
-# _publish.sh mutates _site in place for the offline archive, so restore it from packed-for-shipping.tar.gz (what production deploys) afterwards.
-if [ -z "${SKIP_PUBLISH:-}" ]; then
-    echo "==> Packaging output"
-    mkdir -p "$DOC_WORK/output"
-    "$DOCKER" run "${RUN_FLAGS[@]}" -v "$DOC_WORK/output:/nt/output" "$IMAGE_NAME" \
-        bash -x documentation/generator/_scripts/_publish.sh "$BRANCH"
-    tar -xzf "$DOC_WORK/output/packed-for-shipping.tar.gz" -C "$DOC_WORK/generator"
-fi
-
-# 5. Serve the site with a working search (see serve_site above).
+# 2. Run the build (the generator/build/Dockerfile stages), the tarballs are exported to output/.
+echo "==> Building documentation"
+"$DOCKER" build \
+    --file "$DOC_WORK/generator/build/Dockerfile" \
+    --build-arg BRANCH="$BRANCH" \
+    --build-arg PACKAGE_JOB="$PACKAGE_JOB" \
+    --build-arg PACKAGE_UPLOAD_DIRECTORY="$PACKAGE_UPLOAD_DIRECTORY" \
+    --build-arg PACKAGE_BUILD="$PACKAGE_BUILD" \
+    --build-arg LTS_VERSION="$LTS_VERSION" \
+    --output "type=local,dest=$DOC_WORK/output" \
+    "$WORK_DIR"
+
+# 3. Unpack the site from packed-for-shipping.tar.gz (what production deploys).
+tar -xzf "$DOC_WORK/output/packed-for-shipping.tar.gz" -C "$DOC_WORK/generator"
+
+# 4. Serve the site with a working search (see serve_site above).
 if [ -z "${SKIP_SERVE:-}" ]; then
     serve_site
 fi
 
 echo "==> Done. Generated site is in: $DOC_WORK/generator/_site"
-echo "    Tarballs (if packaged) are in: $DOC_WORK/output/"
+echo "    Tarballs are in: $DOC_WORK/output/"
 if [ -z "${SKIP_SERVE:-}" ]; then
     echo "    Documentation is served at:"
     echo "    http://127.0.0.1:$SITE_PORT/"

generator/README.org

@@ -25,7 +25,7 @@ example ={{< params "CFE_manuals_version" >}}=.
 :END:
 
 The documentation is rendered using [[https://gohugo.io/][Hugo]]. The Hugo site lives in [[../hugo][=hugo/=]]
-and the version used by the build is installed by [[./build/install_hugo.sh][=build/install_hugo.sh=]].
+and it is built in the official Hugo image, pinned as the =hugo= stage of [[./build/Dockerfile][=build/Dockerfile=]].
 
 ** Building Locally
 :PROPERTIES:
@@ -102,6 +102,10 @@ For example:
   ./_scripts/cfdoc_preprocess.py master || exit 4
 
   bash -x ./_scripts/_run.sh master || exit 5
+
+  (cd ../hugo && npm ci && npm run build:all) || exit 6
+
+  bash -x ./_scripts/_postprocess.sh master || exit 7
 #+END_SRC
 
 
@@ -117,8 +121,9 @@ Once the syntax map has been regenerated [[https://github.com/cfengine/documenta
 (whatever that means), resolving links, expanding our custom macros and creating
 printable (dead trees) versions of the documentation.
 
-After pre-processing is complete [[https://github.com/cfengine/documentation/blob/master/generator/_scripts/_run.sh][_run.sh]] performs the Hugo build and
-post processes the build with [[https://github.com/cfengine/documentation/blob/master/generator/_scripts/cfdoc_postprocess.py][cfdoc_postprocess.py]]
+After pre-processing is complete [[https://github.com/cfengine/documentation/blob/master/generator/_scripts/_run.sh][_run.sh]] prepares the Hugo site, Hugo builds it and
+[[https://github.com/cfengine/documentation/blob/master/generator/_scripts/_postprocess.sh][_postprocess.sh]] post processes the build with [[https://github.com/cfengine/documentation/blob/master/generator/_scripts/cfdoc_postprocess.py][cfdoc_postprocess.py]]. In CI these
+steps run as the stages of [[./build/Dockerfile][=build/Dockerfile=]].
 
 *** Publishing
 :PROPERTIES:

generator/_scripts/_postprocess.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Post-processes the site built in the hugo stage of build/Dockerfile.
+
+if [ -z "$WRKDIR" ]
+then
+    echo Environment WRKDIR is not set, setting it to current working directory
+    WRKDIR=`pwd`
+    export WRKDIR
+fi
+
+export LC_ALL=C.UTF-8
+
+cd $WRKDIR/documentation/generator
+mkdir -p $WRKDIR/documentation/generator/_site/assets/searchIndex/documents
+cp -rf $WRKDIR/documentation/hugo/scripts/search/index/searchIndex/* $WRKDIR/documentation/generator/_site/assets/searchIndex
+cp -rf $WRKDIR/documentation/hugo/scripts/search/index/documents/* $WRKDIR/documentation/generator/_site/assets/searchIndex/documents/
+cp -f $WRKDIR/documentation/redirects.conf $WRKDIR/documentation/generator/_site/
+
+$WRKDIR/documentation/generator/_scripts/cfdoc_postprocess.py "$@"
+if [ "$?" -gt "0" ]; then
+    exit 2;
+fi

generator/_scripts/_run.sh

@@ -23,19 +23,5 @@ find $WRKDIR/documentation/hugo/content -type f -name "*.markdown" -exec grep -l
 find $WRKDIR/documentation/hugo/content -name "*.include.markdown" -type f -delete
 
 
-# Hugo build
+# Hugo site (built in the hugo stage of build/Dockerfile)
 cp -rn $WRKDIR/nt-docs/* $WRKDIR/documentation/hugo/
-cd $WRKDIR/documentation/hugo
-npm ci
-npm run build:all || exit 2
-
-cd $WRKDIR/documentation/generator
-mkdir -p $WRKDIR/documentation/generator/_site/assets/searchIndex/documents
-cp -rf $WRKDIR/documentation/hugo/scripts/search/index/searchIndex/* $WRKDIR/documentation/generator/_site/assets/searchIndex
-cp -rf $WRKDIR/documentation/hugo/scripts/search/index/documents/* $WRKDIR/documentation/generator/_site/assets/searchIndex/documents/
-cp -f $WRKDIR/documentation/redirects.conf $WRKDIR/documentation/generator/_site/
-
-$WRKDIR/documentation/generator/_scripts/cfdoc_postprocess.py "$@"
-if [ "$?" -gt "0" ]; then
-    exit 2;
-fi

generator/build/Dockerfile

@@ -1,4 +1,7 @@
-FROM ubuntu:22.04
+# Build context is the directory with all the repos (documentation, core, nova, enterprise, masterfiles, nt-docs)
+
+# Pre-processing
+FROM ubuntu:22.04 AS tools
 
 # We need sudo before all
 RUN apt-get -qy update && \
@@ -7,16 +10,38 @@ RUN apt-get -qy update && \
 # Add jenkins user
 RUN useradd -s /bin/bash jenkins
 RUN echo "jenkins    ALL=(ALL)       NOPASSWD: ALL">> /etc/sudoers
-RUN mkdir -p /home/jenkins
-RUN chown -R jenkins:jenkins /home/jenkins
+RUN mkdir -p /home/jenkins /nt
+RUN chown -R jenkins:jenkins /home/jenkins /nt
 
 # Run rest of setup as jenkins user
-COPY install.sh /
-COPY install_hugo.sh /
+COPY documentation/generator/build/install.sh /
 USER jenkins
 WORKDIR /home/jenkins
 RUN sudo apt-get update -qy && bash -x /install.sh
-RUN bash /install_hugo.sh
 
-# This is where our repos will be
 WORKDIR /nt
+COPY --chown=jenkins:jenkins . /nt
+
+ARG BRANCH
+ARG PACKAGE_JOB
+ARG PACKAGE_UPLOAD_DIRECTORY
+ARG PACKAGE_BUILD
+ARG LTS_VERSION
+RUN bash -x documentation/generator/build/main.sh "$BRANCH" "$PACKAGE_JOB" "$PACKAGE_UPLOAD_DIRECTORY" "$PACKAGE_BUILD" "$LTS_VERSION"
+
+# Hugo build
+FROM ghcr.io/gohugoio/hugo:v0.169.0@sha256:a146e0639230bbc73358e6563240f0a6103466f0390b80a457c775068ec802a9 AS hugo
+COPY --from=tools --chown=hugo:hugo /nt/documentation /nt/documentation
+WORKDIR /nt/documentation/hugo
+RUN npm ci && npm run build:all
+
+# Post-processing and packing into /nt/output
+FROM tools AS postprocess
+COPY --from=hugo --chown=jenkins:jenkins /nt/documentation /nt/documentation
+ARG BRANCH
+RUN bash -x documentation/generator/_scripts/_postprocess.sh "$BRANCH" && \
+    bash -x documentation/generator/_scripts/_publish.sh "$BRANCH"
+
+# Just the tarballs, exported with --output type=local
+FROM scratch AS output
+COPY --from=postprocess /nt/output /

generator/build/Dockerfile.dockerignore

@@ -0,0 +1,3 @@
+# Build context is the directory with all the repos (see Dockerfile), none of their git history is needed
+**/.git
+output

generator/build/README.md

@@ -11,6 +11,7 @@ You will need to have the following repos checked out:
 - nova (used for changelog)
 - enterprise (used for changelog)
 - masterfiles (used to document masterfies)
+- nt-docs (Hugo theme and build scripts)
 - documentation
 
 ## Usage
@@ -33,26 +34,45 @@ If you have buildah installed:
 
    - `$PACKAGE_BUILD` - RELEASE of the build to be downloaded, usually 1
 
+   - `$LTS_VERSION` - LTS version, only used when `$BRANCH` is `lts`
+
 3. `cd` to directory with all repos and run `run.sh` from this dir
 
 ## Details
 
-To make a container, cd to this directory and run:
-
-    buildah build-using-dockerfile -t docs .
-
-or
-
-    docker build --tag docs .
-
-To run docs job in a container, run:
-
-    c=$(buildah from -v $PWD:/nt docs)
-    buildah run $c bash -x documentation/generator/build/main.sh $BRANCH $PACKAGE_JOB $PACKAGE_UPLOAD_DIRECTORY $PACKAGE_BUILD
-    buildah run $c bash -x documentation/generator/_scripts/_publish.sh $BRANCH
-    buildah rm $c
-
-or
-
-    docker run --rm -it -v $PWD:/nt docs bash -x documentation/generator/build/main.sh $BRANCH $PACKAGE_JOB $PACKAGE_UPLOAD_DIRECTORY $PACKAGE_BUILD
-    docker run --rm -it -v $PWD:/nt docs bash -x documentation/generator/_scripts/_publish.sh $BRANCH
+The whole build runs in the stages of the [Dockerfile](Dockerfile), with the
+directory holding all the repos as build context:
+
+1. `tools` - Ubuntu build environment, runs `main.sh` (hub package, syntax
+   JSON, pre-processing, `_scripts/_run.sh` to prepare the Hugo site)
+2. `hugo` - official Hugo image, runs `npm ci && npm run build:all`
+3. `postprocess` - runs `_scripts/_postprocess.sh` and `_scripts/_publish.sh`
+4. `output` - just the tarballs from `/nt/output`
+
+`run.sh` runs the build with buildah. To run it by hand, from the directory with
+all repos:
+
+    buildah build-using-dockerfile \
+      --file documentation/generator/build/Dockerfile \
+      --ignorefile documentation/generator/build/Dockerfile.dockerignore \
+      --build-arg BRANCH=$BRANCH \
+      --build-arg PACKAGE_JOB=$PACKAGE_JOB \
+      --build-arg PACKAGE_UPLOAD_DIRECTORY=$PACKAGE_UPLOAD_DIRECTORY \
+      --build-arg PACKAGE_BUILD=$PACKAGE_BUILD \
+      --output type=local,dest=output .
+
+or with docker, which picks up `Dockerfile.dockerignore` by itself:
+
+    docker build \
+      --file documentation/generator/build/Dockerfile \
+      --build-arg BRANCH=$BRANCH \
+      --build-arg PACKAGE_JOB=$PACKAGE_JOB \
+      --build-arg PACKAGE_UPLOAD_DIRECTORY=$PACKAGE_UPLOAD_DIRECTORY \
+      --build-arg PACKAGE_BUILD=$PACKAGE_BUILD \
+      --output type=local,dest=output .
+
+The tarballs (`packed-for-shipping.tar.gz` and
+`cfengine-documentation-$BRANCH.tar.gz`) end up in `./output`.
+
+To build and serve the docs locally with docker, including cloning the repos,
+use `build-locally.sh` in the root of the documentation repo.

generator/build/install.sh

@@ -4,17 +4,11 @@ set -ex
 
 # All of these commands are executed by the jenkins user.
 
-sudo apt-get install -y -qq curl gnupg2 wget git
+sudo apt-get install -y -qq wget ca-certificates
 
 # some of packages below depend on tzdata. Installing it without DEBIAN_FRONTEND=noninteractive causes it to ask where in the world are you, interrupting the build process
 DEBIAN_FRONTEND=noninteractive sudo --preserve-env=DEBIAN_FRONTEND apt-get install -y -qq tzdata
 
 # Python is needed for our pre and post processing scripts.
 
 sudo apt-get install -y -qq python3
-
-curl -sL https://deb.nodesource.com/setup_21.x -o nodesource_setup.sh
-sudo bash nodesource_setup.sh
-sudo apt-get install -y -qq nodejs
-node --version
-npm --version