diff --git a/README.md b/README.md index 4da0389a..907d9dd6 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,20 @@ Unit/RSpec tests can also be debugged/stepped through when needed. See for examp ### Acceptance Tests See [acceptance-tests README](/acceptance-tests/README.md). + +### Local Dev Builds + +`scripts/dev-build.sh` builds release tarballs for any combination of the seven variants +(`openssl`, `openssl-patched`, `awslc`, `awslc-patched`, `awslc-fips`, `awslc-fips-patched`, `multi`) +and uploads them to the targeted BOSH director. Useful for iterating on packaging changes +without going through CI. + +```bash +./scripts/dev-build.sh # build all 7 variants, version=dev +./scripts/dev-build.sh multi # build only the multi release +./scripts/dev-build.sh awslc awslc-fips # build a subset +./scripts/dev-build.sh --version 1.0 multi # custom version tag +./scripts/dev-build.sh --upload-only # re-upload previously built tarballs +``` + +Tarballs are written to `./dev-releases/` and uploaded with `bosh upload-release --fix`. diff --git a/acceptance-tests/README.md b/acceptance-tests/README.md index c8ff20c3..ee4475d5 100644 --- a/acceptance-tests/README.md +++ b/acceptance-tests/README.md @@ -12,6 +12,23 @@ cd acceptance-tests ./run-local.sh ``` +### TLS Backend Variant + +By default the tests run against HAProxy linked to the system OpenSSL from the stemcell. To exercise a different TLS backend, set the `VARIANT` environment variable: + +```shell +VARIANT=awslc ./run-local.sh # AWS-LC, non-FIPS +VARIANT=awslc-fips ./run-local.sh # AWS-LC FIPS +VARIANT=patched ./run-local.sh # system OpenSSL + HAProxy patches +VARIANT=awslc-patched ./run-local.sh # AWS-LC + patches +VARIANT=awslc-fips-patched ./run-local.sh # AWS-LC FIPS + patches +VARIANT=multi ./run-local.sh # multi release (all binaries bundled) +``` + +A single invocation runs the suite **once** against the chosen variant — `run-local.sh` is not a matrix runner. To cover several variants, invoke it once per variant in a shell loop. Each run rebuilds the BOSH release from scratch, and AWS-LC variants compile the library inside the BOSH compilation VM, so a full sweep takes hours rather than minutes. + +When using `-k` (keep BOSH running, see below) the cached state belongs to whichever variant ran last — switching `VARIANT` inside a kept container is not supported. Stop the container between variants, or run without `-k`. + ### Running on Docker for Mac Acceptance tests cannot be run on Mac with arm64 architecture: diff --git a/acceptance-tests/run-local.sh b/acceptance-tests/run-local.sh index 1cb4b70a..4c4c3fc6 100755 --- a/acceptance-tests/run-local.sh +++ b/acceptance-tests/run-local.sh @@ -94,9 +94,9 @@ if [ -n "$KEEP_RUNNING" ] ; then echo echo "*** KEEP_RUNNING enabled. Please clean up docker scratch after removing containers: ${DOCKER_SCRATCH}" echo - docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="${FOCUS}" -e PARALLELISM="${PARALLELISM}" -e KEEP_RUNNING="${KEEP_RUNNING}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity" + docker run --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e FOCUS="${FOCUS}" -e PARALLELISM="${PARALLELISM}" -e KEEP_RUNNING="${KEEP_RUNNING}" -e VARIANT="${VARIANT:-}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests ; sleep infinity" else - docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" -e PARALLELISM="${PARALLELISM}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests" + docker run --rm --privileged -v "$REPO_DIR":/repo -v "${DOCKER_SCRATCH}":/scratch/docker -e REPO_ROOT=/repo -e KEEP_RUNNING="" -e PARALLELISM="${PARALLELISM}" -e VARIANT="${VARIANT:-}" haproxy-boshrelease-testflight bash -c "cd /repo/ci/scripts && ./acceptance-tests" echo "Cleaning up docker scratch: ${DOCKER_SCRATCH}" sudo rm -rf "${DOCKER_SCRATCH}" fi diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 9b805502..b46a4aef 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -6,6 +6,8 @@ groups: - unit-tests - unit-tests-pr - acceptance-tests + - acceptance-tests-awslc + - acceptance-tests-awslc-fips - acceptance-tests-pr - rc - shipit @@ -149,6 +151,74 @@ jobs: icon_url: "((slack.icon))" text: "((slack.fail_url)) haproxy-boshrelease : acceptance tests failed" + - name: acceptance-tests-awslc + public: true + serial: true + plan: + - do: + - in_parallel: + - { get: git, trigger: true, passed: [unit-tests] } + - { get: stemcell } + - { get: stemcell-jammy } + - get: haproxy-boshrelease-testflight + - task: acceptance-tests-awslc + privileged: true + timeout: 4h + image: haproxy-boshrelease-testflight + config: + platform: linux + inputs: + - { name: git } + - { name: stemcell } + - { name: stemcell-jammy } + run: + path: ./git/ci/scripts/acceptance-tests + args: [] + params: + REPO_ROOT: git + VARIANT: awslc + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : acceptance tests (AWS-LC) failed" + + - name: acceptance-tests-awslc-fips + public: true + serial: true + plan: + - do: + - in_parallel: + - { get: git, trigger: true, passed: [unit-tests] } + - { get: stemcell } + - { get: stemcell-jammy } + - get: haproxy-boshrelease-testflight + - task: acceptance-tests-awslc-fips + privileged: true + timeout: 4h + image: haproxy-boshrelease-testflight + config: + platform: linux + inputs: + - { name: git } + - { name: stemcell } + - { name: stemcell-jammy } + run: + path: ./git/ci/scripts/acceptance-tests + args: [] + params: + REPO_ROOT: git + VARIANT: awslc-fips + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : acceptance tests (AWS-LC FIPS) failed" + - name: acceptance-tests-pr public: true serial: true @@ -300,7 +370,7 @@ jobs: name: gh/name tag: gh/tag body: gh/notes.md - globs: [gh/artifacts/*, gh/artifacts-patched/*] + globs: [gh/artifacts/*, gh/artifacts-patched/*, gh/artifacts-awslc/*, gh/artifacts-awslc-patched/*, gh/artifacts-awslc-fips/*, gh/artifacts-awslc-fips-patched/*, gh/artifacts-multi/*] - put: notify params: channel: "#haproxy-boshrelease" diff --git a/ci/scripts/autobump-dependencies.py b/ci/scripts/autobump-dependencies.py index 3034eee1..e4aacd52 100755 --- a/ci/scripts/autobump-dependencies.py +++ b/ci/scripts/autobump-dependencies.py @@ -24,6 +24,10 @@ LUA_VERSION = "5.4" PCRE_VERSION = "10" HATOP_VERSION = "0" +AWS_LC_VERSION = "1" +CMAKE_VERSION = "3.31" +AWS_LC_FIPS_VERSION = "3" +GOLANG_VERSION = "1.26" # Required Environment Vars BLOBSTORE_SECRET_ACCESS_KEY = os.environ["GCP_SERVICE_KEY"] @@ -36,6 +40,7 @@ # Other Global Variables BLOBS_PATH = "config/blobs.yml" +VERSIONS_PATH = "src/haproxy-versions.sh" PACKAGING_PATH = "packages/{}/packaging" @@ -114,18 +119,23 @@ class Dependency: def pr_branch(self): return f"{self.name}-auto-bump-{PR_BASE}" + @property + def versions_file(self) -> str: + if self.package == "haproxy": + return VERSIONS_PATH + return PACKAGING_PATH.format(self.package) + @property def current_version(self) -> version.Version: """ - Fetches the current version of the release from the packaging file if not already known. + Fetches the current version of the release from the versions file if not already known. (Should always be identical to the version in blobs.yml) """ if self._current_version: return self._current_version - with open(PACKAGING_PATH.format(self.package), "r") as packaging_file: - for line in packaging_file.readlines(): + with open(self.versions_file, "r") as versions_file: + for line in versions_file.readlines(): if line.startswith(self.version_var_name): - # Regex: expecting e.g. "RELEASE_VERSION=1.2.3 # http://release.org/download". extracting Semver Group rgx = rf"{self.version_var_name}=((?:[0-9]+\.){{1,3}}[0-9]+)\s+#.*$" match = re.match(rgx, line) if match: @@ -155,8 +165,11 @@ def get_release_notes(self) -> str: """ raise NotImplementedError + def blob_filename(self, ver) -> str: + return f"{self.name}-{ver}.tar.gz" + def remove_current_blob(self): - current_blob_path = f"{self.package}/{self.name}-{self.current_version}.tar.gz" + current_blob_path = f"{self.package}/{self.blob_filename(self.current_version)}" if self._check_blob_exists(current_blob_path): BoshHelper.remove_blob(current_blob_path) else: @@ -172,17 +185,17 @@ def _check_blob_exists(self, blob_path) -> bool: def update_packaging_file(self): """ - Writes the new dependency version and download-url into packages/haproxy/packaging + Writes the new dependency version and download-url into the versions file. """ - with open(PACKAGING_PATH.format(self.package), "r") as packaging_file: + with open(self.versions_file, "r") as f: replacement = "" - for line in packaging_file.readlines(): + for line in f.readlines(): if line.startswith(self.version_var_name): line = f"{self.version_var_name}={self.latest_release.version} # {self.latest_release.url}\n" replacement += line - with open(PACKAGING_PATH.format(self.package), "w") as packaging_file_write: - packaging_file_write.write(replacement) + with open(self.versions_file, "w") as f: + f.write(replacement) def open_pr_exists(self) -> bool: prs_exist = False @@ -208,7 +221,7 @@ def create_pr(self): self._update_file( self.remote_repo, - PACKAGING_PATH.format(self.package), + self.versions_file, self.pr_branch, f"Bump {self.name} version to {self.latest_release.version}", ) @@ -255,6 +268,10 @@ class GithubDependency(Dependency): tagname_prefix: str = "" filename_suffix: str = ".tar.gz" + blob_version_prefix: str = "" + + def blob_filename(self, ver) -> str: + return f"{self.name}-{self.blob_version_prefix}{ver}{self.filename_suffix}" def fetch_latest_release(self) -> Release: repo_org_and_name = self.root_url.lstrip("https://github.com/") @@ -281,7 +298,7 @@ def get_release_download_url(rel): latest_release = Release( rel.title, get_release_download_url(rel), - f"{self.name}-{str(current_version)}{self.filename_suffix}", + self.blob_filename(current_version), current_version, ) @@ -333,6 +350,87 @@ def get_release_notes(self) -> str: return f"Make sure to check the [CHANGELOG]({self.changelog_url}) for any breaking changes." +@dataclass +class GolangDependency(Dependency): + """ + Handles Go toolchain downloads from go.dev/dl/. + Go binaries are named go1.X.Y.linux-amd64.tar.gz but stored as golang-X.Y.Z.tar.gz in blobs. + """ + + def fetch_latest_release(self) -> Release: + data = requests.get(self.root_url) + html = BeautifulSoup(data.text, "html.parser") + + versions = [] + links = [link for link in html.select("a") if "href" in link.attrs] + + pattern = rf"go({self.pinned_version}(?:\.[0-9]+)*)\.linux-amd64\.tar\.gz" + + for link in links: + match = re.search(pattern, link.attrs["href"]) + if match: + ver = version.parse(match.group(1)) + url = requests.compat.urljoin(self.root_url, link.attrs["href"]) + versions.append( + Release( + f"golang-{match.group(1)}", + url, + self.blob_filename(ver), + ver, + ) + ) + + if versions: + return sorted(versions, key=lambda r: r.version, reverse=True)[0] + + raise Exception(f"Failed to get latest {self.name} version from {self.root_url}") + + def get_release_notes(self) -> str: + return f"Make sure to check the [CHANGELOG](https://go.dev/doc/devel/release) for any breaking changes." + + +@dataclass +class GithubArchiveDependency(Dependency): + """ + For GitHub repos where releases don't have downloadable assets, + so we use the archive tarball URL instead. + """ + + tagname_prefix: str = "" + + def fetch_latest_release(self) -> Release: + repo_org_and_name = self.root_url.lstrip("https://github.com/") + repo = gh.get_repo(repo_org_and_name) + releases = repo.get_releases() + + latest_release = None + latest_version = version.parse("0.0.0") + + for rel in releases: + if rel.prerelease: + continue + current_raw = rel.tag_name.lstrip(self.tagname_prefix) + current_version = version.parse(current_raw) + if latest_version < current_version and current_raw.startswith(self.pinned_version): + latest_version = current_version + tag = rel.tag_name + url = f"{self.root_url}/archive/refs/tags/{tag}.tar.gz" + latest_release = Release( + rel.title, + url, + self.blob_filename(current_version), + current_version, + ) + + if latest_version == version.parse("0.0.0") or latest_release is None: + raise Exception(f"No release found for '{self.root_url}'") + + return latest_release + + def get_release_notes(self) -> str: + return f"Make sure to check the [CHANGELOG]({self.root_url}/releases) for any breaking changes." + + @dataclass class HaproxyDependency(Dependency): def __post_init__(self): @@ -501,6 +599,34 @@ def main() -> None: tagname_prefix="v", filename_suffix="", ), + GithubDependency( + "aws-lc", + "AWS_LC_VERSION", + AWS_LC_VERSION, + "https://github.com/aws/aws-lc", + tagname_prefix="v", + blob_version_prefix="v", + ), + GithubArchiveDependency( + "aws-lc-fips", + "AWS_LC_FIPS_VERSION", + AWS_LC_FIPS_VERSION, + "https://github.com/aws/aws-lc", + tagname_prefix="AWS-LC-FIPS-", + ), + GithubDependency( + "cmake", + "CMAKE_VERSION", + CMAKE_VERSION, + "https://github.com/Kitware/CMake", + tagname_prefix="v", + ), + GolangDependency( + "golang", + "GOLANG_VERSION", + GOLANG_VERSION, + "https://go.dev/dl/", + ), ] write_private_yaml() diff --git a/ci/scripts/functions-ci.sh b/ci/scripts/functions-ci.sh index fa8e5465..4d7eb356 100755 --- a/ci/scripts/functions-ci.sh +++ b/ci/scripts/functions-ci.sh @@ -54,6 +54,75 @@ function bosh_release() { echo "----- Creating candidate BOSH release..." bosh -n reset-release # in case dev_releases/ is in repo accidentally + local variant="${VARIANT:-}" + local add_patches=false + local base="" + + case "$variant" in + ""|openssl) + base="openssl" + ;; + patched) + base="openssl" + add_patches=true + ;; + awslc) + base="awslc" + ;; + awslc-patched) + base="awslc" + add_patches=true + ;; + awslc-fips) + base="awslc-fips" + ;; + awslc-fips-patched) + base="awslc-fips" + add_patches=true + ;; + multi) + base="multi" + ;; + *) + echo "ERROR: unknown VARIANT '$variant' (valid: '', patched, awslc, awslc-patched, awslc-fips, awslc-fips-patched, multi)" >&2 + return 1 + ;; + esac + + echo "----- VARIANT='${variant}' -> base='${base}', patched=${add_patches}" + + case "$base" in + openssl) + ;; + awslc) + echo "----- Adding AWS-LC blobs to haproxy package spec..." + echo "- haproxy/aws-lc-v*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/cmake-*.tar.gz" >> packages/haproxy/spec + ;; + awslc-fips) + echo "----- Adding AWS-LC FIPS blobs to haproxy package spec..." + echo "- haproxy/aws-lc-fips-*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/cmake-*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/golang-*.tar.gz" >> packages/haproxy/spec + ;; + multi) + echo "----- Building multi release (all variants, property-driven selection)..." + cp -r packages-multi/* packages/ + sed -i 's/^- haproxy$/- haproxy-deps\n- haproxy-openssl\n- haproxy-openssl-patched\n- haproxy-awslc\n- haproxy-awslc-patched\n- haproxy-awslc-fips\n- haproxy-awslc-fips-patched/' jobs/haproxy/spec + # multi always bundles the patched binaries, so the patches blob is required + add_patches=true + ;; + esac + + if [ "$add_patches" == "true" ]; then + echo "----- Adding HAProxy patches blob..." + tar -czvf haproxy-patches.tar.gz haproxy-patches + bosh add-blob haproxy-patches.tar.gz haproxy/patches.tar.gz + if [ "$base" != "multi" ]; then + echo "- haproxy/patches.tar.gz" >> packages/haproxy/spec + fi + fi + bosh create-release --force bosh upload-release --rebase release_final_version=$(spruce json dev_releases/*/index.yml | jq -r ".builds[].version" | sed -e "s%+.*%%") diff --git a/ci/scripts/shipit b/ci/scripts/shipit index 636b7b51..c395d1c4 100755 --- a/ci/scripts/shipit +++ b/ci/scripts/shipit @@ -63,12 +63,20 @@ header "Pulling in any git submodules..." git submodule update --init --recursive --force cd - +ensure_patches_blob() { + tar -czvf haproxy-patches.tar.gz haproxy-patches + bosh add-blob haproxy-patches.tar.gz haproxy/patches.tar.gz +} + version() { - # extract the version variable $1 from the packaging script $2 (default 'haproxy') pattern='s/VERSION=(.*)(\s?#.*)/\1/p' - package=${2:-haproxy} - # extract version and remove all spaces - sed -n -E "${pattern//VERSION/${1:?}}" "${REPO_ROOT}/packages/${package}/packaging" | sed 's/ *//g' + package=${2:-} + if [[ -n "${package}" ]]; then + source_file="${REPO_ROOT}/packages/${package}/packaging" + else + source_file="${REPO_ROOT}/src/haproxy-versions.sh" + fi + sed -n -E "${pattern//VERSION/${1:?}}" "${source_file}" | sed 's/ *//g' } HAPROXY_VERSION=$(version HAPROXY_VERSION) @@ -76,11 +84,18 @@ LUA_VERSION=$(version LUA_VERSION) SOCAT_VERSION=$(version SOCAT_VERSION) PCRE_VERSION=$(version PCRE_VERSION) KEEPALIVED_VERSION=$(version KEEPALIVED_VERSION keepalived) +AWS_LC_VERSION=$(version AWS_LC_VERSION) +CMAKE_VERSION=$(version CMAKE_VERSION) +AWS_LC_FIPS_VERSION=$(version AWS_LC_FIPS_VERSION) +GOLANG_VERSION=$(version GOLANG_VERSION) VERSION="${VERSION_TO_CREATE}+${HAPROXY_VERSION}" cd "${REPO_ROOT}" header "Create final release..." + +bosh upload-blobs + bosh -n create-release --final --version "${VERSION}" bosh -n create-release "releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}.yml" \ --tarball "releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}.tgz" @@ -133,6 +148,10 @@ The following versions of upstream components are included in this haproxy-boshr | Lua | \`${LUA_VERSION}\` | | PCRE | \`${PCRE_VERSION}\` | | socat | \`${SOCAT_VERSION}\` | +| AWS-LC | \`${AWS_LC_VERSION}\` | +| AWS-LC FIPS | \`${AWS_LC_FIPS_VERSION}\` | +| cmake | \`${CMAKE_VERSION}\` | +| Go | \`${GOLANG_VERSION}\` | ### Deployment \`\`\`yaml @@ -168,16 +187,71 @@ pushd "${REPO_ROOT}" git status git commit -m "release v${VERSION}" - # After creating a final release we will also create a dev release patches from haproxy-patches directory - echo "- haproxy/patches.tar.gz" >> packages/haproxy/spec - tar -czvf haproxy-patches.tar.gz haproxy-patches - bosh add-blob haproxy-patches.tar.gz haproxy/patches.tar.gz - bosh upload-blobs + # Create additional release variants. + # --- OpenSSL + Patched --- + ensure_patches_blob + echo "- haproxy/patches.tar.gz" >> packages/haproxy/spec bosh -n create-release --force --version "${VERSION}-patched" \ --tarball "../${RELEASE_NAME}-${VERSION}-patched.tgz" - # Undo changes to repo from creating dev release + git clean -df + git reset --hard + + # --- AWS-LC variant (unpatched) --- + echo "- haproxy/aws-lc-v*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/cmake-*.tar.gz" >> packages/haproxy/spec + + bosh -n create-release --force --version "${VERSION}-awslc" \ + --tarball "../${RELEASE_NAME}-${VERSION}-awslc.tgz" + + git clean -df + git reset --hard + + # --- AWS-LC + Patched variant --- + ensure_patches_blob + echo "- haproxy/patches.tar.gz" >> packages/haproxy/spec + echo "- haproxy/aws-lc-v*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/cmake-*.tar.gz" >> packages/haproxy/spec + + bosh -n create-release --force --version "${VERSION}-awslc-patched" \ + --tarball "../${RELEASE_NAME}-${VERSION}-awslc-patched.tgz" + + git clean -df + git reset --hard + + # --- AWS-LC FIPS variant (unpatched) --- + echo "- haproxy/aws-lc-fips-*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/cmake-*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/golang-*.tar.gz" >> packages/haproxy/spec + + bosh -n create-release --force --version "${VERSION}-awslc-fips" \ + --tarball "../${RELEASE_NAME}-${VERSION}-awslc-fips.tgz" + + git clean -df + git reset --hard + + # --- AWS-LC FIPS + Patched variant --- + ensure_patches_blob + echo "- haproxy/patches.tar.gz" >> packages/haproxy/spec + echo "- haproxy/aws-lc-fips-*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/cmake-*.tar.gz" >> packages/haproxy/spec + echo "- haproxy/golang-*.tar.gz" >> packages/haproxy/spec + + bosh -n create-release --force --version "${VERSION}-awslc-fips-patched" \ + --tarball "../${RELEASE_NAME}-${VERSION}-awslc-fips-patched.tgz" + + git clean -df + git reset --hard + + # --- Multi release (all variants, property-driven selection) --- + cp -r packages-multi/* packages/ + ensure_patches_blob + sed -i 's/^- haproxy$/- haproxy-deps\n- haproxy-openssl\n- haproxy-openssl-patched\n- haproxy-awslc\n- haproxy-awslc-patched\n- haproxy-awslc-fips\n- haproxy-awslc-fips-patched/' jobs/haproxy/spec + + bosh -n create-release --force --version "${VERSION}-multi" \ + --tarball "../${RELEASE_NAME}-${VERSION}-multi.tgz" + git clean -df git reset --hard popd @@ -208,6 +282,139 @@ releases: \`\`\` EOF +mkdir -p "${RELEASE_ROOT}/artifacts-awslc" +mv "${RELEASE_NAME}-${VERSION}-awslc.tgz" "${RELEASE_ROOT}/artifacts-awslc" + +AWSLC_RELEASE_TGZ=${RELEASE_ROOT}/artifacts-awslc/${RELEASE_NAME}-${VERSION}-awslc.tgz +# shellcheck disable=SC2155 +export AWSLC_SHA1=$(sha1sum "${AWSLC_RELEASE_TGZ}" | head -n1 | awk '{print $1}') +echo "AWSLC_SHA1=${AWSLC_SHA1}" +# shellcheck disable=SC2155 +export AWSLC_SHA256=$(sha256sum "${AWSLC_RELEASE_TGZ}" | head -n1 | awk '{print $1}') +echo "AWSLC_SHA256=${AWSLC_SHA256}" + +cat >> "${RELEASE_ROOT}/notes.md" <> "${RELEASE_ROOT}/notes.md" <> "${RELEASE_ROOT}/notes.md" <> "${RELEASE_ROOT}/notes.md" <> "${RELEASE_ROOT}/notes.md" < +if [ -d "/var/vcap/packages/haproxy-<%= ssl_variant %>" ]; then + HAPROXY_PKG="/var/vcap/packages/haproxy-<%= ssl_variant %>" +else + HAPROXY_PKG="/var/vcap/packages/haproxy" +fi +export PATH=$PATH:${HAPROXY_PKG}/bin:/var/vcap/packages/ttar/bin CONFIG=/var/vcap/jobs/haproxy/config/haproxy.config PID_FILE=/var/vcap/sys/run/haproxy/haproxy.pid DRAIN_LOCK=/var/vcap/sys/run/haproxy/drain.lock diff --git a/jobs/haproxy/templates/pre-start.erb b/jobs/haproxy/templates/pre-start.erb index b511c8c8..e532ed3f 100644 --- a/jobs/haproxy/templates/pre-start.erb +++ b/jobs/haproxy/templates/pre-start.erb @@ -13,13 +13,16 @@ if [ ! -e /usr/bin/python ] && [ -e /usr/bin/python3 ]; then sudo ln -s /usr/bin/python3 /usr/bin/python fi -if [ ! -e /usr/local/bin/hatop ]; then - sudo ln -s /var/vcap/packages/haproxy/hatop-wrapper /usr/local/bin/hatop -fi - -if [ ! -e /usr/local/bin/socat ]; then - sudo ln -s /var/vcap/packages/haproxy/bin/socat /usr/local/bin/socat +<% ssl_variant = p("ha_proxy.ssl_variant", "openssl") %> +if [ -d "/var/vcap/packages/haproxy-deps" ]; then + DEPS_PKG="/var/vcap/packages/haproxy-deps" +elif [ -d "/var/vcap/packages/haproxy-<%= ssl_variant %>" ]; then + DEPS_PKG="/var/vcap/packages/haproxy-<%= ssl_variant %>" +else + DEPS_PKG="/var/vcap/packages/haproxy" fi +sudo ln -sf ${DEPS_PKG}/hatop-wrapper /usr/local/bin/hatop +sudo ln -sf ${DEPS_PKG}/bin/socat /usr/local/bin/socat <%- if_p("ha_proxy.pre_start_script") do |script| -%> # ha_proxy.pre_start_script {{{ diff --git a/packages-multi/aws-lc-fips/packaging b/packages-multi/aws-lc-fips/packaging new file mode 100644 index 00000000..333f6b4a --- /dev/null +++ b/packages-multi/aws-lc-fips/packaging @@ -0,0 +1,28 @@ +set -euxo pipefail + +source haproxy-versions.sh + +export PATH=/var/vcap/packages/cmake/bin:$PATH + +# Install Go toolchain (required for FIPS delocate/inject_hash) +tar xzf haproxy/golang-${GOLANG_VERSION}.tar.gz -C ${BOSH_INSTALL_TARGET} +export GOROOT=${BOSH_INSTALL_TARGET}/go +export GOPATH=${BOSH_INSTALL_TARGET}/gopath +export GOCACHE=${BOSH_INSTALL_TARGET}/gocache +export PATH=${GOROOT}/bin:$PATH + +tar xzf haproxy/aws-lc-fips-${AWS_LC_FIPS_VERSION}.tar.gz +pushd aws-lc-AWS-LC-FIPS-${AWS_LC_FIPS_VERSION} + mkdir -p build && cd build + cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DFIPS=1 \ + -DBUILD_SHARED_LIBS=0 \ + -DBUILD_TESTING=0 \ + -DCMAKE_INSTALL_PREFIX=${BOSH_INSTALL_TARGET} \ + .. + make -j$(nproc) + make install +popd + +# Clean up Go toolchain to save disk space +rm -rf ${BOSH_INSTALL_TARGET}/go ${BOSH_INSTALL_TARGET}/gopath ${BOSH_INSTALL_TARGET}/gocache diff --git a/packages-multi/aws-lc-fips/spec b/packages-multi/aws-lc-fips/spec new file mode 100644 index 00000000..dfc376cc --- /dev/null +++ b/packages-multi/aws-lc-fips/spec @@ -0,0 +1,8 @@ +--- +name: aws-lc-fips +dependencies: +- cmake +files: +- haproxy/aws-lc-fips-*.tar.gz +- haproxy/golang-*.tar.gz +- haproxy-versions.sh diff --git a/packages-multi/aws-lc/packaging b/packages-multi/aws-lc/packaging new file mode 100644 index 00000000..8d213c98 --- /dev/null +++ b/packages-multi/aws-lc/packaging @@ -0,0 +1,19 @@ +set -euxo pipefail + +source haproxy-versions.sh + +export PATH=/var/vcap/packages/cmake/bin:$PATH + +tar xzf haproxy/aws-lc-v${AWS_LC_VERSION}.tar.gz +pushd aws-lc-${AWS_LC_VERSION} + mkdir -p build && cd build + cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_SHARED_LIBS=0 \ + -DDISABLE_GO=1 \ + -DDISABLE_PERL=1 \ + -DBUILD_TESTING=0 \ + -DCMAKE_INSTALL_PREFIX=${BOSH_INSTALL_TARGET} \ + .. + make -j$(nproc) + make install +popd diff --git a/packages-multi/aws-lc/spec b/packages-multi/aws-lc/spec new file mode 100644 index 00000000..70a17e5a --- /dev/null +++ b/packages-multi/aws-lc/spec @@ -0,0 +1,7 @@ +--- +name: aws-lc +dependencies: +- cmake +files: +- haproxy/aws-lc-v*.tar.gz +- haproxy-versions.sh diff --git a/packages-multi/cmake/packaging b/packages-multi/cmake/packaging new file mode 100644 index 00000000..9350b1f6 --- /dev/null +++ b/packages-multi/cmake/packaging @@ -0,0 +1,10 @@ +set -euxo pipefail + +source haproxy-versions.sh + +tar xzf haproxy/cmake-${CMAKE_VERSION}.tar.gz +pushd cmake-${CMAKE_VERSION} + ./bootstrap --prefix=${BOSH_INSTALL_TARGET} --parallel=$(nproc) + make -j$(nproc) + make install +popd diff --git a/packages-multi/cmake/spec b/packages-multi/cmake/spec new file mode 100644 index 00000000..667d68d0 --- /dev/null +++ b/packages-multi/cmake/spec @@ -0,0 +1,5 @@ +--- +name: cmake +files: +- haproxy/cmake-*.tar.gz +- haproxy-versions.sh diff --git a/packages-multi/haproxy-awslc-fips-patched/packaging b/packages-multi/haproxy-awslc-fips-patched/packaging new file mode 100644 index 00000000..fefdbdc1 --- /dev/null +++ b/packages-multi/haproxy-awslc-fips-patched/packaging @@ -0,0 +1,30 @@ +set -euxo pipefail + +source haproxy-versions.sh +DEPS_DIR=/var/vcap/packages/haproxy-deps + +mkdir ${BOSH_INSTALL_TARGET}/bin + +SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=/var/vcap/packages/aws-lc-fips/include SSL_LIB=/var/vcap/packages/aws-lc-fips/lib" + +echo "Unpacking HAproxy..." +tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz +pushd haproxy-${HAPROXY_VERSION} + mkdir -p ${BOSH_INSTALL_TARGET}/applied-patches + tar xf ../haproxy/patches.tar.gz + + for patchfile in haproxy-patches/*.patch; do + echo "Applying patch file ${patchfile}" + patch -F 0 -p0 < ${patchfile} + cp ${patchfile} ${BOSH_INSTALL_TARGET}/applied-patches + done + rm -r haproxy-patches + + echo "Installing HAproxy..." + make TARGET=linux-glibc USE_PROMEX=1 ${SSL_MAKE_FLAGS} \ + USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 \ + PCRE2DIR=${DEPS_DIR} \ + USE_LUA=1 LUA_LIB=${DEPS_DIR}/lib LUA_INC=${DEPS_DIR}/include + cp haproxy ${BOSH_INSTALL_TARGET}/bin/ + chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy +popd diff --git a/packages-multi/haproxy-awslc-fips-patched/spec b/packages-multi/haproxy-awslc-fips-patched/spec new file mode 100644 index 00000000..0ae6c9ac --- /dev/null +++ b/packages-multi/haproxy-awslc-fips-patched/spec @@ -0,0 +1,9 @@ +--- +name: haproxy-awslc-fips-patched +dependencies: +- haproxy-deps +- aws-lc-fips +files: +- haproxy/haproxy-*.tar.gz +- haproxy/patches.tar.gz +- haproxy-versions.sh diff --git a/packages-multi/haproxy-awslc-fips/packaging b/packages-multi/haproxy-awslc-fips/packaging new file mode 100644 index 00000000..c1696406 --- /dev/null +++ b/packages-multi/haproxy-awslc-fips/packaging @@ -0,0 +1,20 @@ +set -euxo pipefail + +source haproxy-versions.sh +DEPS_DIR=/var/vcap/packages/haproxy-deps + +mkdir ${BOSH_INSTALL_TARGET}/bin + +SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=/var/vcap/packages/aws-lc-fips/include SSL_LIB=/var/vcap/packages/aws-lc-fips/lib" + +echo "Unpacking HAproxy..." +tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz +pushd haproxy-${HAPROXY_VERSION} + echo "Installing HAproxy..." + make TARGET=linux-glibc USE_PROMEX=1 ${SSL_MAKE_FLAGS} \ + USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 \ + PCRE2DIR=${DEPS_DIR} \ + USE_LUA=1 LUA_LIB=${DEPS_DIR}/lib LUA_INC=${DEPS_DIR}/include + cp haproxy ${BOSH_INSTALL_TARGET}/bin/ + chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy +popd diff --git a/packages-multi/haproxy-awslc-fips/spec b/packages-multi/haproxy-awslc-fips/spec new file mode 100644 index 00000000..93f0b9a0 --- /dev/null +++ b/packages-multi/haproxy-awslc-fips/spec @@ -0,0 +1,8 @@ +--- +name: haproxy-awslc-fips +dependencies: +- haproxy-deps +- aws-lc-fips +files: +- haproxy/haproxy-*.tar.gz +- haproxy-versions.sh diff --git a/packages-multi/haproxy-awslc-patched/packaging b/packages-multi/haproxy-awslc-patched/packaging new file mode 100644 index 00000000..3d2d0c30 --- /dev/null +++ b/packages-multi/haproxy-awslc-patched/packaging @@ -0,0 +1,30 @@ +set -euxo pipefail + +source haproxy-versions.sh +DEPS_DIR=/var/vcap/packages/haproxy-deps + +mkdir ${BOSH_INSTALL_TARGET}/bin + +SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=/var/vcap/packages/aws-lc/include SSL_LIB=/var/vcap/packages/aws-lc/lib" + +echo "Unpacking HAproxy..." +tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz +pushd haproxy-${HAPROXY_VERSION} + mkdir -p ${BOSH_INSTALL_TARGET}/applied-patches + tar xf ../haproxy/patches.tar.gz + + for patchfile in haproxy-patches/*.patch; do + echo "Applying patch file ${patchfile}" + patch -F 0 -p0 < ${patchfile} + cp ${patchfile} ${BOSH_INSTALL_TARGET}/applied-patches + done + rm -r haproxy-patches + + echo "Installing HAproxy..." + make TARGET=linux-glibc USE_PROMEX=1 ${SSL_MAKE_FLAGS} \ + USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 \ + PCRE2DIR=${DEPS_DIR} \ + USE_LUA=1 LUA_LIB=${DEPS_DIR}/lib LUA_INC=${DEPS_DIR}/include + cp haproxy ${BOSH_INSTALL_TARGET}/bin/ + chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy +popd diff --git a/packages-multi/haproxy-awslc-patched/spec b/packages-multi/haproxy-awslc-patched/spec new file mode 100644 index 00000000..d2e14e89 --- /dev/null +++ b/packages-multi/haproxy-awslc-patched/spec @@ -0,0 +1,9 @@ +--- +name: haproxy-awslc-patched +dependencies: +- haproxy-deps +- aws-lc +files: +- haproxy/haproxy-*.tar.gz +- haproxy/patches.tar.gz +- haproxy-versions.sh diff --git a/packages-multi/haproxy-awslc/packaging b/packages-multi/haproxy-awslc/packaging new file mode 100644 index 00000000..52eb95a3 --- /dev/null +++ b/packages-multi/haproxy-awslc/packaging @@ -0,0 +1,20 @@ +set -euxo pipefail + +source haproxy-versions.sh +DEPS_DIR=/var/vcap/packages/haproxy-deps + +mkdir ${BOSH_INSTALL_TARGET}/bin + +SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=/var/vcap/packages/aws-lc/include SSL_LIB=/var/vcap/packages/aws-lc/lib" + +echo "Unpacking HAproxy..." +tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz +pushd haproxy-${HAPROXY_VERSION} + echo "Installing HAproxy..." + make TARGET=linux-glibc USE_PROMEX=1 ${SSL_MAKE_FLAGS} \ + USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 \ + PCRE2DIR=${DEPS_DIR} \ + USE_LUA=1 LUA_LIB=${DEPS_DIR}/lib LUA_INC=${DEPS_DIR}/include + cp haproxy ${BOSH_INSTALL_TARGET}/bin/ + chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy +popd diff --git a/packages-multi/haproxy-awslc/spec b/packages-multi/haproxy-awslc/spec new file mode 100644 index 00000000..8610fdae --- /dev/null +++ b/packages-multi/haproxy-awslc/spec @@ -0,0 +1,8 @@ +--- +name: haproxy-awslc +dependencies: +- haproxy-deps +- aws-lc +files: +- haproxy/haproxy-*.tar.gz +- haproxy-versions.sh diff --git a/packages-multi/haproxy-deps/packaging b/packages-multi/haproxy-deps/packaging new file mode 100644 index 00000000..6c2ac876 --- /dev/null +++ b/packages-multi/haproxy-deps/packaging @@ -0,0 +1,36 @@ +set -euxo pipefail + +source haproxy-versions.sh + +mkdir ${BOSH_INSTALL_TARGET}/bin + +echo "Extracting lua..." +tar xzf haproxy/lua-${LUA_VERSION}.tar.gz +pushd lua-${LUA_VERSION} + make linux install INSTALL_TOP=${BOSH_INSTALL_TARGET} +popd + +echo "Extracting pcre..." +tar xzf haproxy/pcre2-${PCRE_VERSION}.tar.gz +pushd pcre2-${PCRE_VERSION} + ./configure \ + --enable-jit \ + --prefix ${BOSH_INSTALL_TARGET} + make + make install +popd + +echo "Installing socat..." +tar xzf haproxy/socat-${SOCAT_VERSION}.tar.gz +pushd socat-${SOCAT_VERSION} + ./configure + make + cp socat ${BOSH_INSTALL_TARGET}/bin + chmod 755 ${BOSH_INSTALL_TARGET}/bin/socat +popd + +echo "Installing hatop..." +cp haproxy/hatop-${HATOP_VERSION} ${BOSH_INSTALL_TARGET}/bin/hatop +chmod 755 ${BOSH_INSTALL_TARGET}/bin/hatop +cp hatop-wrapper ${BOSH_INSTALL_TARGET}/ +chmod 755 ${BOSH_INSTALL_TARGET}/hatop-wrapper diff --git a/packages-multi/haproxy-deps/spec b/packages-multi/haproxy-deps/spec new file mode 100644 index 00000000..b1bf0b68 --- /dev/null +++ b/packages-multi/haproxy-deps/spec @@ -0,0 +1,9 @@ +--- +name: haproxy-deps +files: +- haproxy/pcre2-*.tar.gz +- haproxy/socat-*.tar.gz +- haproxy/lua-*.tar.gz +- haproxy/hatop-* +- hatop-wrapper +- haproxy-versions.sh diff --git a/packages-multi/haproxy-openssl-patched/packaging b/packages-multi/haproxy-openssl-patched/packaging new file mode 100644 index 00000000..e1ed3468 --- /dev/null +++ b/packages-multi/haproxy-openssl-patched/packaging @@ -0,0 +1,30 @@ +set -euxo pipefail + +source haproxy-versions.sh +DEPS_DIR=/var/vcap/packages/haproxy-deps + +mkdir ${BOSH_INSTALL_TARGET}/bin + +SSL_MAKE_FLAGS="USE_OPENSSL=1" + +echo "Unpacking HAproxy..." +tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz +pushd haproxy-${HAPROXY_VERSION} + mkdir -p ${BOSH_INSTALL_TARGET}/applied-patches + tar xf ../haproxy/patches.tar.gz + + for patchfile in haproxy-patches/*.patch; do + echo "Applying patch file ${patchfile}" + patch -F 0 -p0 < ${patchfile} + cp ${patchfile} ${BOSH_INSTALL_TARGET}/applied-patches + done + rm -r haproxy-patches + + echo "Installing HAproxy..." + make TARGET=linux-glibc USE_PROMEX=1 ${SSL_MAKE_FLAGS} \ + USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 \ + PCRE2DIR=${DEPS_DIR} \ + USE_LUA=1 LUA_LIB=${DEPS_DIR}/lib LUA_INC=${DEPS_DIR}/include + cp haproxy ${BOSH_INSTALL_TARGET}/bin/ + chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy +popd diff --git a/packages-multi/haproxy-openssl-patched/spec b/packages-multi/haproxy-openssl-patched/spec new file mode 100644 index 00000000..70daa39c --- /dev/null +++ b/packages-multi/haproxy-openssl-patched/spec @@ -0,0 +1,8 @@ +--- +name: haproxy-openssl-patched +dependencies: +- haproxy-deps +files: +- haproxy/haproxy-*.tar.gz +- haproxy/patches.tar.gz +- haproxy-versions.sh diff --git a/packages-multi/haproxy-openssl/packaging b/packages-multi/haproxy-openssl/packaging new file mode 100644 index 00000000..9034f50a --- /dev/null +++ b/packages-multi/haproxy-openssl/packaging @@ -0,0 +1,20 @@ +set -euxo pipefail + +source haproxy-versions.sh +DEPS_DIR=/var/vcap/packages/haproxy-deps + +mkdir ${BOSH_INSTALL_TARGET}/bin + +SSL_MAKE_FLAGS="USE_OPENSSL=1" + +echo "Unpacking HAproxy..." +tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz +pushd haproxy-${HAPROXY_VERSION} + echo "Installing HAproxy..." + make TARGET=linux-glibc USE_PROMEX=1 ${SSL_MAKE_FLAGS} \ + USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 \ + PCRE2DIR=${DEPS_DIR} \ + USE_LUA=1 LUA_LIB=${DEPS_DIR}/lib LUA_INC=${DEPS_DIR}/include + cp haproxy ${BOSH_INSTALL_TARGET}/bin/ + chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy +popd diff --git a/packages-multi/haproxy-openssl/spec b/packages-multi/haproxy-openssl/spec new file mode 100644 index 00000000..2fe7bd5a --- /dev/null +++ b/packages-multi/haproxy-openssl/spec @@ -0,0 +1,7 @@ +--- +name: haproxy-openssl +dependencies: +- haproxy-deps +files: +- haproxy/haproxy-*.tar.gz +- haproxy-versions.sh diff --git a/packages/haproxy/packaging b/packages/haproxy/packaging index f3daf999..d3b78ff4 100644 --- a/packages/haproxy/packaging +++ b/packages/haproxy/packaging @@ -1,16 +1,7 @@ # abort script on failures set -euxo pipefail - -LUA_VERSION=5.4.8 # https://www.lua.org/ftp/lua-5.4.8.tar.gz - -PCRE_VERSION=10.47 # https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.47/pcre2-10.47.tar.gz - -SOCAT_VERSION=1.8.1.1 # http://www.dest-unreach.org/socat/download/socat-1.8.1.1.tar.gz - -HAPROXY_VERSION=3.2.19 # https://www.haproxy.org/download/3.2/src/haproxy-3.2.19.tar.gz - -HATOP_VERSION=0.8.2 # https://github.com/jhunt/hatop/releases/download/v0.8.2/hatop +source haproxy-versions.sh mkdir ${BOSH_INSTALL_TARGET}/bin @@ -39,6 +30,77 @@ pushd socat-${SOCAT_VERSION} chmod 755 ${BOSH_INSTALL_TARGET}/bin/socat popd +# --- AWS-LC support (conditional) --- +# If an AWS-LC FIPS source tarball is present, build with FIPS support. +# If a non-FIPS AWS-LC source tarball is present, build without FIPS. +# Otherwise, use system OpenSSL. +if ls haproxy/aws-lc-fips-*.tar.gz 1>/dev/null 2>&1; then + # --- AWS-LC FIPS build --- + echo "Installing Go toolchain..." + tar xzf haproxy/golang-${GOLANG_VERSION}.tar.gz -C ${BOSH_INSTALL_TARGET} + export GOROOT=${BOSH_INSTALL_TARGET}/go + export GOPATH=${BOSH_INSTALL_TARGET}/gopath + export GOCACHE=${BOSH_INSTALL_TARGET}/gocache + export PATH=${GOROOT}/bin:$PATH + + echo "Building cmake..." + tar xzf haproxy/cmake-${CMAKE_VERSION}.tar.gz + pushd cmake-${CMAKE_VERSION} + ./bootstrap --prefix=${BOSH_INSTALL_TARGET} --parallel=$(nproc) + make -j$(nproc) + make install + popd + export PATH=${BOSH_INSTALL_TARGET}/bin:$PATH + + echo "Building AWS-LC (FIPS)..." + tar xzf haproxy/aws-lc-fips-${AWS_LC_FIPS_VERSION}.tar.gz + pushd aws-lc-AWS-LC-FIPS-${AWS_LC_FIPS_VERSION} + mkdir -p build && cd build + cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DFIPS=1 \ + -DBUILD_SHARED_LIBS=0 \ + -DBUILD_TESTING=0 \ + -DCMAKE_INSTALL_PREFIX=${BOSH_INSTALL_TARGET} \ + .. + make -j$(nproc) + make install + popd + + SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=${BOSH_INSTALL_TARGET}/include SSL_LIB=${BOSH_INSTALL_TARGET}/lib" + +elif ls haproxy/aws-lc-*.tar.gz 1>/dev/null 2>&1; then + # --- AWS-LC non-FIPS build --- + echo "Building cmake..." + tar xzf haproxy/cmake-${CMAKE_VERSION}.tar.gz + pushd cmake-${CMAKE_VERSION} + ./bootstrap --prefix=${BOSH_INSTALL_TARGET} --parallel=$(nproc) + make -j$(nproc) + make install + popd + export PATH=${BOSH_INSTALL_TARGET}/bin:$PATH + + echo "Building AWS-LC..." + tar xzf haproxy/aws-lc-v${AWS_LC_VERSION}.tar.gz + pushd aws-lc-${AWS_LC_VERSION} + mkdir -p build && cd build + cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DBUILD_SHARED_LIBS=0 \ + -DDISABLE_GO=1 \ + -DDISABLE_PERL=1 \ + -DBUILD_TESTING=0 \ + -DCMAKE_INSTALL_PREFIX=${BOSH_INSTALL_TARGET} \ + .. + make -j$(nproc) + make install + popd + + SSL_MAKE_FLAGS="USE_OPENSSL_AWSLC=1 SSL_INC=${BOSH_INSTALL_TARGET}/include SSL_LIB=${BOSH_INSTALL_TARGET}/lib" + +else + # --- System OpenSSL --- + SSL_MAKE_FLAGS="USE_OPENSSL=1" +fi + echo "Unpacking HAproxy..." tar xf haproxy/haproxy-${HAPROXY_VERSION}.tar.gz pushd haproxy-${HAPROXY_VERSION} @@ -60,7 +122,7 @@ pushd haproxy-${HAPROXY_VERSION} fi echo "Installing HAproxy..." - make TARGET=linux-glibc USE_PROMEX=1 USE_OPENSSL=1 USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 PCRE2DIR=${BOSH_INSTALL_TARGET} USE_LUA=1 LUA_LIB=${BOSH_INSTALL_TARGET}/lib LUA_INC=${BOSH_INSTALL_TARGET}/include + make TARGET=linux-glibc USE_PROMEX=1 ${SSL_MAKE_FLAGS} USE_PCRE2=1 USE_PCRE2_JIT=yes USE_STATIC_PCRE2=1 USE_ZLIB=1 PCRE2DIR=${BOSH_INSTALL_TARGET} USE_LUA=1 LUA_LIB=${BOSH_INSTALL_TARGET}/lib LUA_INC=${BOSH_INSTALL_TARGET}/include cp haproxy ${BOSH_INSTALL_TARGET}/bin/ chmod 755 ${BOSH_INSTALL_TARGET}/bin/haproxy popd diff --git a/packages/haproxy/spec b/packages/haproxy/spec index 65c27799..5609fbd5 100644 --- a/packages/haproxy/spec +++ b/packages/haproxy/spec @@ -7,3 +7,4 @@ files: - haproxy/lua-*.tar.gz - haproxy/hatop-* - hatop-wrapper +- haproxy-versions.sh diff --git a/scripts/dev-build.sh b/scripts/dev-build.sh new file mode 100755 index 00000000..b195b059 --- /dev/null +++ b/scripts/dev-build.sh @@ -0,0 +1,261 @@ +#!/usr/bin/env bash +# +# dev-build.sh +# +# Builds HAProxy release variants locally and uploads them to the BOSH director. +# +# Usage: ./scripts/dev-build.sh [--upload-only] [--version BASE] [--output-dir DIR] [variant...] +# +# Variants: +# openssl, openssl-patched, awslc, awslc-patched, awslc-fips, awslc-fips-patched, multi +# +# If no variants are specified, all 7 are built. +# +# Versioning: +# Each invocation produces release versions of the form +# +dev[-]. +# e.g. "16.9.0+dev.1779286286" (openssl), "16.9.0+dev-awslc.1779286286" (awslc). +# BASE defaults to the highest existing final release in releases/haproxy/ (or 0.0.0 if none), +# and can be overridden with --version. Each run uses a fresh timestamp, so previously +# built tarballs do not need to be deleted before rebuilding. +# +# Examples: +# ./scripts/dev-build.sh # build all 7, BASE=highest final release +# ./scripts/dev-build.sh multi # build only multi +# ./scripts/dev-build.sh --version 17.0.0 multi # override BASE +# ./scripts/dev-build.sh awslc awslc-fips # build awslc and awslc-fips +# ./scripts/dev-build.sh --upload-only # upload everything in dev-releases/ +# +# Prerequisites: +# - All blobs present locally (bosh add-blob done for aws-lc, cmake, golang, aws-lc-fips) +# - haproxy-patches/ directory exists with .patch files +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR/.." + +ALL_VARIANTS=(openssl openssl-patched awslc awslc-patched awslc-fips awslc-fips-patched multi) + +is_variant() { + local arg="$1" + for v in "${ALL_VARIANTS[@]}"; do + [[ "$v" == "$arg" ]] && return 0 + done + return 1 +} + +UPLOAD_ONLY=false +BASE_VERSION="" +OUTPUT_DIR="./dev-releases" +VARIANTS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --upload-only) + UPLOAD_ONLY=true + shift + ;; + --version) + BASE_VERSION="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + *) + if is_variant "$1"; then + VARIANTS+=("$1") + else + echo "Unknown argument: $1" >&2 + echo "Valid variants: ${ALL_VARIANTS[*]}" >&2 + exit 1 + fi + shift + ;; + esac +done + +if [[ ${#VARIANTS[@]} -eq 0 ]]; then + VARIANTS=("${ALL_VARIANTS[@]}") +fi + +# Derive base version from the highest existing final release if --version not given. +# Final releases live in releases/haproxy/haproxy-[+meta].yml +if [[ -z "$BASE_VERSION" ]]; then + BASE_VERSION=$( + ls releases/haproxy/haproxy-*.yml 2>/dev/null \ + | sed 's|.*/haproxy-||;s|\.yml$||;s|+.*||' \ + | sort -V \ + | tail -1 + ) + BASE_VERSION="${BASE_VERSION:-0.0.0}" +fi + +TIMESTAMP=$(date +%s) + +should_build() { + local variant="$1" + for v in "${VARIANTS[@]}"; do + [[ "$v" == "$variant" ]] && return 0 + done + return 1 +} + +mkdir -p "$OUTPUT_DIR" + +if [[ "$UPLOAD_ONLY" == false ]]; then + +SPEC_FILE="packages/haproxy/spec" +SPEC_ORIG=$(cat "$SPEC_FILE") +JOB_SPEC_FILE="jobs/haproxy/spec" +JOB_SPEC_ORIG=$(cat "$JOB_SPEC_FILE") + +copy_multi_packages() { + cp -r packages-multi/* packages/ +} + +remove_multi_packages() { + for dir in packages-multi/*/; do + rm -rf "packages/$(basename "$dir")" + done +} + +cleanup() { + echo "$SPEC_ORIG" > "$SPEC_FILE" + echo "$JOB_SPEC_ORIG" > "$JOB_SPEC_FILE" + remove_multi_packages + rm -f haproxy-patches.tar.gz +} +trap cleanup EXIT + +reset_spec() { + echo "$SPEC_ORIG" > "$SPEC_FILE" + echo "$JOB_SPEC_ORIG" > "$JOB_SPEC_FILE" +} + +add_patches_to_spec() { + echo "- haproxy/patches.tar.gz" >> "$SPEC_FILE" +} + +build_release() { + local variant="$1" + local suffix="dev" + [[ -n "$variant" ]] && suffix="dev-${variant}" + local version="${BASE_VERSION}+${suffix}.${TIMESTAMP}" + local tarball="$OUTPUT_DIR/haproxy-${version}.tgz" + + echo "" + echo "========================================" + echo " Building: haproxy (version: $version)" + echo "========================================" + echo "" + + bosh -n create-release --force \ + --name "haproxy" \ + --version "$version" \ + --tarball "$tarball" + + echo " -> $tarball" +} + +# --- 1. OpenSSL (base) --- +if should_build openssl; then + reset_spec + build_release "" +fi + +# --- 2. OpenSSL + Patched --- +if should_build openssl-patched; then + reset_spec + add_patches_to_spec + tar -czf haproxy-patches.tar.gz haproxy-patches + bosh add-blob haproxy-patches.tar.gz haproxy/patches.tar.gz + build_release "patched" +fi + +# --- 3. AWS-LC --- +if should_build awslc; then + reset_spec + echo "- haproxy/aws-lc-v*.tar.gz" >> "$SPEC_FILE" + echo "- haproxy/cmake-*.tar.gz" >> "$SPEC_FILE" + build_release "awslc" +fi + +# --- 4. AWS-LC + Patched --- +if should_build awslc-patched; then + reset_spec + add_patches_to_spec + echo "- haproxy/aws-lc-v*.tar.gz" >> "$SPEC_FILE" + echo "- haproxy/cmake-*.tar.gz" >> "$SPEC_FILE" + tar -czf haproxy-patches.tar.gz haproxy-patches + bosh add-blob haproxy-patches.tar.gz haproxy/patches.tar.gz + build_release "awslc-patched" +fi + +# --- 5. AWS-LC FIPS --- +if should_build awslc-fips; then + reset_spec + echo "- haproxy/aws-lc-fips-*.tar.gz" >> "$SPEC_FILE" + echo "- haproxy/cmake-*.tar.gz" >> "$SPEC_FILE" + echo "- haproxy/golang-*.tar.gz" >> "$SPEC_FILE" + build_release "awslc-fips" +fi + +# --- 6. AWS-LC FIPS + Patched --- +if should_build awslc-fips-patched; then + reset_spec + add_patches_to_spec + echo "- haproxy/aws-lc-fips-*.tar.gz" >> "$SPEC_FILE" + echo "- haproxy/cmake-*.tar.gz" >> "$SPEC_FILE" + echo "- haproxy/golang-*.tar.gz" >> "$SPEC_FILE" + tar -czf haproxy-patches.tar.gz haproxy-patches + bosh add-blob haproxy-patches.tar.gz haproxy/patches.tar.gz + build_release "awslc-fips-patched" +fi + +# --- 7. Multi (all variants, property-driven selection) --- +if should_build multi; then + reset_spec + copy_multi_packages + tar -czf haproxy-patches.tar.gz haproxy-patches + bosh add-blob haproxy-patches.tar.gz haproxy/patches.tar.gz + sed -i.bak 's/^- haproxy$/- haproxy-deps\n- haproxy-openssl\n- haproxy-openssl-patched\n- haproxy-awslc\n- haproxy-awslc-patched\n- haproxy-awslc-fips\n- haproxy-awslc-fips-patched/' "$JOB_SPEC_FILE" + rm -f "${JOB_SPEC_FILE}.bak" + build_release "multi" + remove_multi_packages +fi + +fi # UPLOAD_ONLY + +# --- Upload all releases --- +echo "" +echo "========================================" +echo " Uploading releases to BOSH director" +echo "========================================" +echo "" + +if [[ "$UPLOAD_ONLY" == true ]]; then + upload_glob="$OUTPUT_DIR/haproxy-*.tgz" +else + upload_glob="$OUTPUT_DIR/haproxy-*.${TIMESTAMP}.tgz" +fi + +shopt -s nullglob +uploaded=0 +for tgz in $upload_glob; do + echo "Uploading: $tgz" + bosh upload-release "$tgz" --fix + uploaded=$((uploaded + 1)) +done +shopt -u nullglob + +if [[ $uploaded -eq 0 ]]; then + echo "No tarballs matching '$upload_glob' to upload." >&2 + exit 1 +fi + +echo "" +echo "Done." diff --git a/src/haproxy-versions.sh b/src/haproxy-versions.sh new file mode 100644 index 00000000..50f1dc61 --- /dev/null +++ b/src/haproxy-versions.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Shared version definitions for all HAProxy packages. +# Sourced by packaging scripts in packages/ and packages-multi/. + +HAPROXY_VERSION=3.2.19 # https://www.haproxy.org/download/3.2/src/haproxy-3.2.19.tar.gz +LUA_VERSION=5.4.8 # https://www.lua.org/ftp/lua-5.4.8.tar.gz +PCRE_VERSION=10.47 # https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.47/pcre2-10.47.tar.gz +SOCAT_VERSION=1.8.1.1 # http://www.dest-unreach.org/socat/download/socat-1.8.1.1.tar.gz +HATOP_VERSION=0.8.2 # https://github.com/jhunt/hatop/releases/download/v0.8.2/hatop +AWS_LC_VERSION=1.72.0 # https://github.com/aws/aws-lc/archive/refs/tags/v1.72.0.tar.gz +AWS_LC_FIPS_VERSION=3.3.0 # https://github.com/aws/aws-lc/archive/refs/tags/AWS-LC-FIPS-3.3.0.tar.gz +CMAKE_VERSION=3.31.6 # https://github.com/Kitware/CMake/releases/download/v3.31.6/cmake-3.31.6.tar.gz +GOLANG_VERSION=1.26.2 # https://go.dev/dl/go1.26.2.linux-amd64.tar.gz