From cd0bfbc1ef10becce772022e40fec0196f28d974 Mon Sep 17 00:00:00 2001 From: Leonid Borchuk Date: Tue, 2 Sep 2025 13:28:14 +0000 Subject: [PATCH] Add ubuntu22.04 build/test docker containers Ubuntu is one of the most popular Linux distros and is widely used not only on workstations, but also on server-side platforms. We use Debian-based distributions as the main operating system platform in our environment. Greenplum works on Ubuntu without any issues, and our tests with Cloudberry show the same. Here I have added support for Ubuntu Jammy LTS in Cloudberry. As for Rocky Linux distros, we create a Docker container with all dependencies already resolved. Then, we build a Debian package using debhelper inside the container. The container can also be used to run tests, but that is outside the scope of this PR. Also there is some changes compared to RPM-based builds: 1. Debian packages (and docker containers) does not need to build libxerces since 22.04 repo contains libxerces-c3.2. libxerces-c3.2 fits perfectly for cloudberry. 2. Debian prohibits install something inside /usr/local/. So we need to build our binaries somewhere else, then list relative local build paths in control file and finally write install script to copy it in /usr/local/cloudberry-db. In reality we just build to /usr/cloudberry-db and then create symbolic link /usr/cloudberry-db -> /usr/local/cloudberry-db --- .../docker-cbdb-build-containers.yml | 7 +- .../workflows/docker-cbdb-test-containers.yml | 7 +- .../cloudberry/scripts/build-cloudberry.sh | 4 +- .../cloudberry/scripts/cloudberry-utils.sh | 29 ++- .../scripts/configure-cloudberry.sh | 27 ++- .../scripts/create-cloudberry-demo-cluster.sh | 4 +- .../cloudberry/scripts/unittest-cloudberry.sh | 2 +- devops/build/packaging/deb/build-deb.sh | 164 +++++++++++++ .../build/packaging/deb/ubuntu22.04/changelog | 5 + devops/build/packaging/deb/ubuntu22.04/compat | 1 + .../build/packaging/deb/ubuntu22.04/control | 101 ++++++++ .../build/packaging/deb/ubuntu22.04/install | 1 + .../build/packaging/deb/ubuntu22.04/postinst | 12 + .../build/packaging/deb/ubuntu22.04/preinst | 13 + devops/build/packaging/deb/ubuntu22.04/rules | 49 ++++ .../packaging/deb/ubuntu22.04/source/format | 1 + .../deb/ubuntu22.04/source/local-options | 2 + .../docker/build/ubuntu22.04/Dockerfile | 196 +++++++++++++++ .../build/ubuntu22.04/configs/90-cbdb-limits | 32 +++ .../ubuntu22.04/configs/gpinitsystem.conf | 89 +++++++ .../build/ubuntu22.04/configs/init_system.sh | 195 +++++++++++++++ .../build/ubuntu22.04/tests/requirements.txt | 3 + .../tests/testinfra/test_cloudberry_db_env.py | 128 ++++++++++ .../deploy/docker/test/ubuntu22.04/Dockerfile | 141 +++++++++++ .../test/ubuntu22.04/configs/90-cbdb-limits | 32 +++ .../ubuntu22.04/configs/gpinitsystem.conf | 87 +++++++ .../test/ubuntu22.04/configs/init_system.sh | 224 ++++++++++++++++++ pom.xml | 7 +- 28 files changed, 1542 insertions(+), 21 deletions(-) create mode 100755 devops/build/packaging/deb/build-deb.sh create mode 100644 devops/build/packaging/deb/ubuntu22.04/changelog create mode 100644 devops/build/packaging/deb/ubuntu22.04/compat create mode 100644 devops/build/packaging/deb/ubuntu22.04/control create mode 100644 devops/build/packaging/deb/ubuntu22.04/install create mode 100644 devops/build/packaging/deb/ubuntu22.04/postinst create mode 100644 devops/build/packaging/deb/ubuntu22.04/preinst create mode 100755 devops/build/packaging/deb/ubuntu22.04/rules create mode 100644 devops/build/packaging/deb/ubuntu22.04/source/format create mode 100644 devops/build/packaging/deb/ubuntu22.04/source/local-options create mode 100644 devops/deploy/docker/build/ubuntu22.04/Dockerfile create mode 100644 devops/deploy/docker/build/ubuntu22.04/configs/90-cbdb-limits create mode 100644 devops/deploy/docker/build/ubuntu22.04/configs/gpinitsystem.conf create mode 100755 devops/deploy/docker/build/ubuntu22.04/configs/init_system.sh create mode 100644 devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt create mode 100644 devops/deploy/docker/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py create mode 100644 devops/deploy/docker/test/ubuntu22.04/Dockerfile create mode 100644 devops/deploy/docker/test/ubuntu22.04/configs/90-cbdb-limits create mode 100644 devops/deploy/docker/test/ubuntu22.04/configs/gpinitsystem.conf create mode 100644 devops/deploy/docker/test/ubuntu22.04/configs/init_system.sh diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index f5f8676cd4c..62973613a48 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -60,6 +60,7 @@ on: paths: - 'devops/deploy/docker/build/rocky8/**' - 'devops/deploy/docker/build/rocky9/**' + - 'devops/deploy/docker/build/ubuntu22.04/**' workflow_dispatch: # Manual trigger # Prevent multiple workflow runs from interfering with each other @@ -73,10 +74,10 @@ jobs: timeout-minutes: 60 runs-on: ubuntu-latest - # Matrix strategy to build for both Rocky Linux 8 and 9 + # Matrix strategy to build for both Rocky Linux 8 and 9, Ubuntu 22.04 strategy: matrix: - platform: ['rocky8', 'rocky9'] + platform: ['rocky8', 'rocky9', 'ubuntu22.04'] steps: # Checkout repository code with full history @@ -103,6 +104,8 @@ jobs: - 'devops/deploy/docker/build/rocky8/**' rocky9: - 'devops/deploy/docker/build/rocky9/**' + ubuntu22.04: + - 'devops/deploy/docker/build/ubuntu22.04/**' # Set up QEMU for multi-architecture support # This allows building ARM64 images on AMD64 infrastructure and vice versa diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 87fcf245edc..57d8819bcd3 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -49,6 +49,7 @@ on: paths: - 'devops/deploy/docker/test/rocky8/**' - 'devops/deploy/docker/test/rocky9/**' + - 'devops/deploy/docker/test/ubuntu22.04/**' workflow_dispatch: # Manual trigger # Prevent multiple workflow runs from interfering with each other @@ -62,8 +63,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Build for both Rocky Linux 8 and 9 - platform: ['rocky8', 'rocky9'] + # Build for both Rocky Linux 8 and 9, Ubuntu 22.04 + platform: ['rocky8', 'rocky9', 'ubuntu22.04'] steps: # Checkout repository code @@ -87,6 +88,8 @@ jobs: - 'devops/deploy/docker/test/rocky8/**' rocky9: - 'devops/deploy/docker/test/rocky9/**' + ubuntu22.04: + - 'devops/deploy/docker/test/ubuntu22.04/**' # Skip if no changes for current platform - name: Skip if not relevant diff --git a/devops/build/automation/cloudberry/scripts/build-cloudberry.sh b/devops/build/automation/cloudberry/scripts/build-cloudberry.sh index db04f1b7592..efa061a0f83 100755 --- a/devops/build/automation/cloudberry/scripts/build-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/build-cloudberry.sh @@ -45,7 +45,7 @@ # Prerequisites: # - configure-cloudberry.sh must be run first # - Required build dependencies must be installed -# - /usr/local/cloudberry-db/lib must exist and be writable +# - ${BUILD_DESTINATION}/lib (by default /usr/local/cloudberry-db/lib) must exist and be writable # # Exit Codes: # 0 - Build and installation completed successfully @@ -71,7 +71,7 @@ init_environment "Cloudberry Build Script" "${BUILD_LOG}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Build process diff --git a/devops/build/automation/cloudberry/scripts/cloudberry-utils.sh b/devops/build/automation/cloudberry/scripts/cloudberry-utils.sh index 01b0b1b8381..c8d0f8cc44e 100755 --- a/devops/build/automation/cloudberry/scripts/cloudberry-utils.sh +++ b/devops/build/automation/cloudberry/scripts/cloudberry-utils.sh @@ -79,11 +79,15 @@ # # -------------------------------------------------------------------- +DEFAULT_BUILD_DESTINATION=/usr/local/cloudberry-db + # Initialize logging and environment init_environment() { local script_name=$1 local log_file=$2 + init_build_destination_var + echo "=== Initializing environment for ${script_name} ===" echo "${script_name} executed at $(date)" | tee -a "${log_file}" echo "Whoami: $(whoami)" | tee -a "${log_file}" @@ -91,6 +95,7 @@ init_environment() { echo "Working directory: $(pwd)" | tee -a "${log_file}" echo "Source directory: ${SRC_DIR}" | tee -a "${log_file}" echo "Log directory: ${LOG_DIR}" | tee -a "${log_file}" + echo "Build destination: ${BUILD_DESTINATION}" | tee -a "${log_file}" if [ -z "${SRC_DIR:-}" ]; then echo "Error: SRC_DIR environment variable is not set" | tee -a "${log_file}" @@ -121,7 +126,7 @@ run_psql_cmd() { # Function to source Cloudberry environment source_cloudberry_env() { echo "=== Sourcing Cloudberry environment ===" | tee -a "${LOG_DIR}/environment.log" - source /usr/local/cloudberry-db/cloudberry-env.sh + source ${BUILD_DESTINATION}/cloudberry-env.sh source ${SRC_DIR}/../cloudberry/gpAux/gpdemo/gpdemo-env.sh } @@ -146,3 +151,25 @@ log_completion() { local timestamp=$(date "+%Y.%m.%d-%H.%M.%S") echo "${script_name} execution completed successfully at ${timestamp}" | tee -a "${log_file}" } + +# Function to get OS identifier +detect_os() { + if [ -f /etc/os-release ]; then + . /etc/os-release + OS_ID=$ID + OS_VERSION=$VERSION_ID + else + echo "Unsupported system: cannot detect OS" >&2 + exit 99 + fi +} + +# Init BUILD_DESTINATION default value if not set +init_build_destination_var() { + + if [ -z ${BUILD_DESTINATION+x} ]; then + export BUILD_DESTINATION=${DEFAULT_BUILD_DESTINATION} + exec "$@" + fi + +} diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index 4d84c3bfc21..58d64922ac7 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -23,7 +23,7 @@ # Description: Configures Apache Cloudberry build environment and runs # ./configure with optimized settings. Performs the # following: -# 1. Prepares /usr/local/cloudberry-db directory +# 1. Prepares ${BUILD_DESTINATION} (by default /usr/local/cloudberry-db) directory # 2. Sets up library dependencies # 3. Configures build with required features enabled # @@ -49,6 +49,7 @@ # # Required Environment Variables: # SRC_DIR - Root source directory +# BUILD_DESTINATION - Directory to build binaries # # Optional Environment Variables: # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) @@ -92,6 +93,10 @@ set -euo pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "${SCRIPT_DIR}/cloudberry-utils.sh" +# Call it before conditional logic +detect_os +echo "Detected OS: $OS_ID $OS_VERSION" + # Define log directory and files export LOG_DIR="${SRC_DIR}/build-logs" CONFIGURE_LOG="${LOG_DIR}/configure.log" @@ -101,18 +106,20 @@ init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" # Initial setup log_section "Initial Setup" -execute_cmd sudo rm -rf /usr/local/cloudberry-db || exit 2 +execute_cmd sudo rm -rf ${BUILD_DESTINATION} || exit 2 execute_cmd sudo chmod a+w /usr/local || exit 2 -execute_cmd mkdir -p /usr/local/cloudberry-db/lib || exit 2 -execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ - /usr/local/xerces-c/lib/libxerces-c-3.3.so \ - /usr/local/cloudberry-db/lib || exit 3 -execute_cmd sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db || exit 2 +execute_cmd sudo mkdir -p ${BUILD_DESTINATION}/lib || exit 2 +if [[ "$OS_ID" == "rocky" && "$OS_VERSION" =~ ^(8|9) ]]; then + execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ + /usr/local/xerces-c/lib/libxerces-c-3.3.so \ + ${BUILD_DESTINATION}/lib || exit 3 +fi +execute_cmd sudo chown -R gpadmin:gpadmin ${BUILD_DESTINATION} || exit 2 log_section_end "Initial Setup" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Add debug options if ENABLE_DEBUG is set to "true" @@ -127,7 +134,7 @@ fi # Configure build log_section "Configure" -execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ +execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --disable-external-fts \ --enable-gpcloud \ --enable-ic-proxy \ @@ -152,7 +159,7 @@ execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ --with-openssl \ --with-uuid=e2fs \ --with-includes=/usr/local/xerces-c/include \ - --with-libraries=/usr/local/cloudberry-db/lib || exit 4 + --with-libraries=${BUILD_DESTINATION}/lib || exit 4 log_section_end "Configure" # Capture version information diff --git a/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh b/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh index e01d62fac9c..ec582e80402 100755 --- a/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh +++ b/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh @@ -35,7 +35,7 @@ # LOG_DIR - Directory for logs (defaults to ${SRC_DIR}/build-logs) # # Prerequisites: -# - Apache Cloudberry must be installed (/usr/local/cloudberry-db) +# - Apache Cloudberry must be installed (default location is /usr/local/cloudberry-db) # - SSH must be configured for passwordless access to localhost # - User must have permissions to create cluster directories # - PostgreSQL client tools (psql) must be available @@ -79,7 +79,7 @@ init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" # Setup environment log_section "Environment Setup" -source /usr/local/cloudberry-db/cloudberry-env.sh || exit 1 +source ${BUILD_DESTINATION}/cloudberry-env.sh || exit 1 log_section_end "Environment Setup" # Verify SSH access diff --git a/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh b/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh index f7bc120bd08..97107ea1a9f 100755 --- a/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh @@ -56,7 +56,7 @@ init_environment "Cloudberry Unittest Script" "${UNITTEST_LOG}" # Set environment log_section "Environment Setup" -export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${BUILD_DESTINATION}/lib:LD_LIBRARY_PATH log_section_end "Environment Setup" # Unittest process diff --git a/devops/build/packaging/deb/build-deb.sh b/devops/build/packaging/deb/build-deb.sh new file mode 100755 index 00000000000..2e7312be53f --- /dev/null +++ b/devops/build/packaging/deb/build-deb.sh @@ -0,0 +1,164 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Script Name: build-deb.sh +# +# Description: +# This script automates the process of building an DEB package using a specified +# version number. It ensures that the necessary tools are installed +# and that the control file exists before attempting to build the DEB. The script +# also includes error handling to provide meaningful feedback in case of failure. +# +# Usage: +# ./build-deb.sh [-v ] [-h] [--dry-run] +# +# Options: +# -v, --version : Specify the version (required) +# -h, --help : Display this help and exit +# -n, --dry-run : Show what would be done, without making any changes +# +# Example: +# ./build-deb.sh -v 1.5.5 # Build with version 1.5.5 +# +# Prerequisites: +# - The dpkg-buildpackage package must be installed (provides the dpkg-buildpackage command). +# - The control file must exist at debian/control. +# +# Error Handling: +# The script includes checks to ensure: +# - The version option (-v or --version) is provided. +# - The necessary commands are available. +# - The control file exists at the specified location. +# If any of these checks fail, the script exits with an appropriate error message. + +# Enable strict mode for better error handling +set -euo pipefail + +# Default values +VERSION="" +RELEASE="1" +DEBUG_BUILD=false + +# Function to display usage information +usage() { + echo "Usage: $0 [-v ] [-h] [--dry-run]" + echo " -v, --version : Specify the version (optional)" + echo " -h, --help : Display this help and exit" + echo " -n, --dry-run : Show what would be done, without making any changes" + exit 1 +} + +# Function to check if required commands are available +check_commands() { + local cmds=("dpkg-buildpackage") + for cmd in "${cmds[@]}"; do + if ! command -v "$cmd" &> /dev/null; then + echo "Error: Required command '$cmd' not found. Please install it before running the script." + exit 1 + fi + done +} + +function print_changelog() { +cat < $(date +'%a, %d %b %Y %H:%M:%S %z') +EOF +} + +# Parse options +while [[ "$#" -gt 0 ]]; do + case $1 in + -v|--version) + VERSION="$2" + shift 2 + ;; + -h|--help) + usage + ;; + -n|--dry-run) + DRY_RUN=true + shift + ;; + *) + echo "Unknown option: ($1)" + shift + ;; + esac +done + +export CBDB_FULL_VERSION=$VERSION + +# Set version if not provided +if [ -z "${VERSION}" ]; then + export CBDB_FULL_VERSION=$(./getversion | cut -d'-' -f 1 | cut -d'+' -f 1) +fi + +if [[ ! $CBDB_FULL_VERSION =~ ^[0-9] ]]; then + export CBDB_FULL_VERSION="0.$CBDB_FULL_VERSION" +fi + +if [ -z ${BUILD_NUMBER+x} ]; then + export BUILD_NUMBER=1 +fi + +if [ -z ${BUILD_USER+x} ]; then + export BUILD_USER=github +fi + +export CBDB_PKG_VERSION=${CBDB_FULL_VERSION}-${BUILD_NUMBER}-$(git --git-dir=.git rev-list HEAD --count).$(git --git-dir=.git rev-parse --short HEAD) + +# Check if required commands are available +check_commands + +# Define the control file path +CONTROL_FILE=debian/control + +# Check if the spec file exists +if [ ! -f "$CONTROL_FILE" ]; then + echo "Error: Control file not found at $CONTROL_FILE." + exit 1 +fi + +# Build the rpmbuild command based on options +DEBBUILD_CMD="dpkg-buildpackage -us -uc" + +# Dry-run mode +if [ "${DRY_RUN:-false}" = true ]; then + echo "Dry-run mode: This is what would be done:" + print_changelog + echo "" + echo "$DEBBUILD_CMD" + exit 0 +fi + +# Run debbuild with the provided options +echo "Building DEB with Version $CBDB_FULL_VERSION ..." + +print_changelog > debian/changelog + +if ! eval "$DEBBUILD_CMD"; then + echo "Error: deb build failed." + exit 1 +fi + +# Print completion message +echo "DEB build completed successfully with package $CBDB_PKG_VERSION" diff --git a/devops/build/packaging/deb/ubuntu22.04/changelog b/devops/build/packaging/deb/ubuntu22.04/changelog new file mode 100644 index 00000000000..211d271b88f --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/changelog @@ -0,0 +1,5 @@ +apache-cloudberry-db-incubating (2.0.0) jammy; urgency=medium + + * Initial release. + + -- Cloudberry Team Wed, 26 Mar 2025 11:10:44 +0000 diff --git a/devops/build/packaging/deb/ubuntu22.04/compat b/devops/build/packaging/deb/ubuntu22.04/compat new file mode 100644 index 00000000000..ec635144f60 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/compat @@ -0,0 +1 @@ +9 diff --git a/devops/build/packaging/deb/ubuntu22.04/control b/devops/build/packaging/deb/ubuntu22.04/control new file mode 100644 index 00000000000..c4bc2e6f84d --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/control @@ -0,0 +1,101 @@ +Source: apache-cloudberry-db-incubating +Maintainer: Apache Cloudberry (Incubating) +Section: database +Build-Depends: debhelper (>= 9), + bison, + ca-certificates-java, + ca-certificates, + cmake, + curl, + cgroup-tools, + flex, + gcc-11, + g++-11, + git, + krb5-multidev, + libapr1-dev, + libbz2-dev, + libcurl4-gnutls-dev, + libevent-dev, + libkrb5-dev, + libldap2-dev, + libperl-dev, + libreadline6-dev, + libssl-dev, + libxml2-dev, + libyaml-dev, + libzstd-dev, + libaprutil1-dev, + libpam0g-dev, + libpam0g, + libcgroup1, + libyaml-0-2, + libldap-2.5-0, + libssl3, + libxerces-c-dev, + libxerces-c3.2, + ninja-build, + quilt, + unzip, + wget, + zlib1g-dev, + libuv1-dev + +Package: apache-cloudberry-db-incubating +Provides: apache-cloudberry-db +Architecture: amd64 +Depends: curl, + cgroup-tools, + iputils-ping, + krb5-multidev, + less, + libapr1, + libbz2-1.0, + libcurl4, + libcurl3-gnutls, + libevent-2.1-7, + libreadline8, + libxml2, + libyaml-0-2, + libldap-2.5-0, + libzstd1, + libcgroup1, + libssl3, + libpam0g, + libxerces-c3.2, + locales, + net-tools, + openssh-client, + openssh-server, + openssl, + python-six, + python2.7, + python2.7-dev, + rsync, + wget, + zlib1g, + libuv1 +Description: Apache Cloudberry (incubating) is an advanced, open-source, massively + parallel processing (MPP) data warehouse developed from PostgreSQL and + Greenplum. It is designed for high-performance analytics on + large-scale data sets, offering powerful analytical capabilities and + enhanced security features. + Key Features: + - Massively parallel processing for optimized performance + - Advanced analytics for complex data processing + - Integration with ETL and BI tools + - Compatibility with multiple data sources and formats + - Enhanced security features + Apache Cloudberry supports both batch processing and real-time data + warehousing, making it a versatile solution for modern data + environments. + Apache Cloudberry (incubating) is an effort undergoing incubation at + the Apache Software Foundation (ASF), sponsored by the Apache + Incubator PMC. + Incubation is required of all newly accepted projects until a further + review indicates that the infrastructure, communications, and decision + making process have stabilized in a manner consistent with other + successful ASF projects. + While incubation status is not necessarily a reflection of the + completeness or stability of the code, it does indicate that the + project has yet to be fully endorsed by the ASF. diff --git a/devops/build/packaging/deb/ubuntu22.04/install b/devops/build/packaging/deb/ubuntu22.04/install new file mode 100644 index 00000000000..3e29bb0dd35 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/install @@ -0,0 +1 @@ +debian/build/* /usr/cloudberry-db diff --git a/devops/build/packaging/deb/ubuntu22.04/postinst b/devops/build/packaging/deb/ubuntu22.04/postinst new file mode 100644 index 00000000000..ccb33a54033 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/postinst @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin +GPHOME=/usr/cloudberry-db + +if [ "$1" = configure ]; then + + ln -sf ${GPHOME} /usr/local/cloudberry-db + +fi diff --git a/devops/build/packaging/deb/ubuntu22.04/preinst b/devops/build/packaging/deb/ubuntu22.04/preinst new file mode 100644 index 00000000000..29d6211fc2b --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/preinst @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin + +case $1 in + install|upgrade) + id "${GPADMIN}" >> /dev/null 2>&1 || \ + useradd -r -m -d /home/${GPADMIN} -s /bin/bash ${GPADMIN} + ;; +esac + diff --git a/devops/build/packaging/deb/ubuntu22.04/rules b/devops/build/packaging/deb/ubuntu22.04/rules new file mode 100755 index 00000000000..6213985b48c --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/rules @@ -0,0 +1,49 @@ +#!/usr/bin/make -f + +DH_VERBOSE = 1 +DPKG_EXPORT_BUILDFLAGS = 1 + +CBDB_BIN_PATH := /usr/local/cloudberry-db +DEBIAN_DESTINATION := $(shell pwd)/debian/build + +# assumes that CWD is root of cbdb source +CBDB_PKG_VERSION := $(CBDB_PKG_VERSION) +PACKAGE_CBDB := $(shell cat debian/control | egrep "^Package: " | cut -d " " -f 2) +PATH := ${DEBIAN_DESTINATION}/bin:${PATH} + +.PHONY: gpinstall + +include /usr/share/dpkg/default.mk + +%: + dh $@ --parallel + +gpinstall: + make install + +override_dh_auto_install: gpinstall + # the staging directory for creating a debian is NOT the right GPHOME. + # change GPHOME to point to the post-install target install directory. + sed -i "s#GPHOME=.*#GPHOME=${CBDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/cloudberry-env.sh + +override_dh_auto_build: + echo "Skipping build" + +override_dh_auto_clean: + echo "Skipping clean" + +override_dh_auto_configure: + echo "Skipping configure" + +override_dh_auto_test: + echo "Skipping auto test" + +override_dh_gencontrol: + echo "using version ${CBDB_PKG_VERSION} for binary Cloudberry" + dh_gencontrol -- -v${CBDB_PKG_VERSION} -p${PACKAGE_CBDB} + +override_dh_shlibdeps: + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/libfakeroot:${DEBIAN_DESTINATION}/lib dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info + +clean_dev_local: + rm -rf ${DEBIAN_DESTINATION} diff --git a/devops/build/packaging/deb/ubuntu22.04/source/format b/devops/build/packaging/deb/ubuntu22.04/source/format new file mode 100644 index 00000000000..89ae9db8f88 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/devops/build/packaging/deb/ubuntu22.04/source/local-options b/devops/build/packaging/deb/ubuntu22.04/source/local-options new file mode 100644 index 00000000000..00131ee8c41 --- /dev/null +++ b/devops/build/packaging/deb/ubuntu22.04/source/local-options @@ -0,0 +1,2 @@ +#abort-on-upstream-changes +#unapply-patches diff --git a/devops/deploy/docker/build/ubuntu22.04/Dockerfile b/devops/deploy/docker/build/ubuntu22.04/Dockerfile new file mode 100644 index 00000000000..7a3abd61c85 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu22.04/Dockerfile @@ -0,0 +1,196 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as +# a base environment for evaluating the Apache Cloudberry. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Apache Cloudberry +# DEB can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +FROM ubuntu:22.04 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="Europe/London" + +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin +ENV TZ=${TIMEZONE_VAR} +ENV DEBIAN_FRONTEND=noninteractive + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- + +RUN apt-get update && \ + apt-get install -y -qq \ + htop \ + bat \ + silversearcher-ag \ + vim \ + wget && \ + apt-get install -y -qq locales && \ + locale-gen "en_US.UTF-8" && \ + update-locale LANG="en_US.UTF-8" && \ + apt-get install -y -qq \ + bison \ + build-essential \ + cmake \ + dpkg-dev \ + fakeroot \ + flex \ + g++-11 \ + gcc-11 \ + git \ + iproute2 \ + iputils-ping \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libipc-run-perl \ + libkrb5-dev \ + libldap-dev \ + liblz4-dev \ + libpam0g-dev \ + libperl-dev \ + libprotobuf-dev \ + libreadline-dev \ + libssl-dev \ + libuv1-dev \ + libxerces-c-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + lsof \ + make \ + openssh-server \ + pkg-config \ + protobuf-compiler \ + python3-distutils \ + python3-pip \ + python3-setuptools \ + python3.10 \ + python3.10-dev \ + rsync \ + sudo \ + tzdata \ + zlib1g-dev && \ + apt-get install -y -qq \ + ca-certificates-java \ + cgroup-tools \ + curl \ + debhelper \ + libaprutil1-dev \ + libcgroup1 \ + ninja-build \ + quilt \ + unzip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + cd && GO_VERSION="go1.23.4" && \ + ARCH=$(uname -m) && \ + if [ "${ARCH}" = "aarch64" ]; then \ + GO_ARCH="arm64" && \ + GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + elif [ "${ARCH}" = "x86_64" ]; then \ + GO_ARCH="amd64" && \ + GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + else \ + echo "Unsupported architecture: ${ARCH}" && exit 1; \ + fi && \ + GO_URL="https://go.dev/dl/${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + wget -nv "${GO_URL}" && \ + echo "${GO_SHA256} ${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | sha256sum -c - && \ + tar xf "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + mv go "/usr/local/${GO_VERSION}" && \ + ln -s "/usr/local/${GO_VERSION}" /usr/local/go && \ + rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ + update-alternatives --install /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/gcc-11 100 && \ + update-alternatives --set gcc /usr/bin/gcc-11 && \ + update-alternatives --set g++ /usr/bin/g++-11 + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- + +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ + chmod 755 /tmp/init_system.sh && \ + /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd -m -g gpadmin gpadmin && \ + echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/90-gpadmin && \ + chmod 0440 /etc/sudoers.d/90-gpadmin && \ + ssh-keygen -A && \ + mkdir /var/run/sshd && chmod 0755 /var/run/sshd + +# Install testinfra via pip +RUN pip3 install pytest-testinfra + +# Example: Copying test files into the container +COPY tests /tests + +USER gpadmin +WORKDIR /home/gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/devops/deploy/docker/build/ubuntu22.04/configs/90-cbdb-limits b/devops/deploy/docker/build/ubuntu22.04/configs/90-cbdb-limits new file mode 100644 index 00000000000..8ea1d9ed988 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu22.04/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits + # -------------------------------------------------------------------- + # + # Licensed to the Apache Software Foundation (ASF) under one or more + # contributor license agreements. See the NOTICE file distributed + # with this work for additional information regarding copyright + # ownership. The ASF licenses this file to You under the Apache + # License, Version 2.0 (the "License"); you may not use this file + # except in compliance with the License. You may obtain a copy of the + # License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + # implied. See the License for the specific language governing + # permissions and limitations under the License. + # + # -------------------------------------------------------------------- + + # Core dump file size limits for gpadmin + gpadmin soft core unlimited + gpadmin hard core unlimited + + # Open file limits for gpadmin + gpadmin soft nofile 524288 + gpadmin hard nofile 524288 + + # Process limits for gpadmin + gpadmin soft nproc 131072 + gpadmin hard nproc 131072 diff --git a/devops/deploy/docker/build/ubuntu22.04/configs/gpinitsystem.conf b/devops/deploy/docker/build/ubuntu22.04/configs/gpinitsystem.conf new file mode 100644 index 00000000000..d4d312231c5 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu22.04/configs/gpinitsystem.conf @@ -0,0 +1,89 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# gpinitsystem Configuration File for Apache Cloudberry +# -------------------------------------------------------------------- +# This configuration file is used to initialize an Apache Cloudberry +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# -------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# -------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# -------------------------------------------------------------------- diff --git a/devops/deploy/docker/build/ubuntu22.04/configs/init_system.sh b/devops/deploy/docker/build/ubuntu22.04/configs/init_system.sh new file mode 100755 index 00000000000..52a44462c61 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu22.04/configs/init_system.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +## Container Initialization Script +# -------------------------------------------------------------------- +## This script sets up the environment inside the Docker container for +## the Apache Cloudberry Build Environment. It performs the following +## tasks: +## +## 1. Verifies that the container is running with the expected hostname. +## 2. Starts the SSH daemon to allow SSH access to the container. +## 3. Configures passwordless SSH access for the 'gpadmin' user. +## 4. Displays a welcome banner and system information. +## 5. Starts an interactive bash shell. +## +## This script is intended to be used as an entrypoint or initialization +## script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +cat <<-EOF +Welcome to the Apache Cloudberry Build Environment! + +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# Add go to PATH +source /etc/profile.d/go.sh + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt b/devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt new file mode 100644 index 00000000000..b9711eddac5 --- /dev/null +++ b/devops/deploy/docker/build/ubuntu22.04/tests/requirements.txt @@ -0,0 +1,3 @@ +testinfra +pytest-testinfra +paramiko diff --git a/devops/deploy/docker/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py b/devops/deploy/docker/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py new file mode 100644 index 00000000000..d7f018ab9df --- /dev/null +++ b/devops/deploy/docker/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py @@ -0,0 +1,128 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +import testinfra + +def test_installed_packages(host): + """ + Test if the essential packages are installed. + """ + packages = [ + "bat", + "bison", + "cmake", + "flex", + "g++-11", + "gcc-11", + "git", + "htop", + "iproute2", + "iputils-ping", + "libapr1-dev", + "libbz2-dev", + "libcurl4-gnutls-dev", + "libevent-dev", + "libipc-run-perl", + "libkrb5-dev", + "libldap-dev", + "liblz4-dev", + "libpam0g-dev", + "libperl-dev", + "libprotobuf-dev", + "libreadline-dev", + "libssl-dev", + "libuv1-dev", + "libxerces-c-dev", + "libxml2-dev", + "libyaml-dev", + "libzstd-dev", + "locales", + "lsof", + "make", + "openssh-server", + "pkg-config", + "protobuf-compiler", + "python3-distutils", + "python3-pip", + "python3-setuptools", + "python3.10", + "python3.10-dev", + "rsync", + "silversearcher-ag", + "sudo", + "tzdata", + "vim", + "wget", + "zlib1g-dev" + ] + for package in packages: + pkg = host.package(package) + assert pkg.is_installed + + +def test_user_gpadmin_exists(host): + """ + Test if the gpadmin user exists and is configured properly. + """ + user = host.user("gpadmin") + assert user.exists + assert "gpadmin" in user.groups + + +def test_ssh_service(host): + """ + Test if SSH service is configured correctly. + """ + sshd_config = host.file("/etc/ssh/sshd_config") + assert sshd_config.exists + + +def test_timezone(host): + """ + Test if the timezone is configured correctly. + """ + localtime = host.file("/etc/localtime") + assert localtime.exists + + +def test_system_limits_configured(host): + """ + Test if the custom system limits are applied. + """ + limits_file = host.file("/etc/security/limits.d/90-cbdb-limits") + assert limits_file.exists + + +def test_init_system_script(host): + """ + Test if the init_system.sh script is present and executable. + """ + script = host.file("/tmp/init_system.sh") + assert script.exists + assert script.mode == 0o755 + + +def test_locale_generated(host): + """ + Test if the en_US.UTF-8 locale is correctly generated. + """ + locale = host.run("locale -a | grep en_US.utf8") + assert locale.exit_status == 0 + assert "en_US.utf8" in locale.stdout diff --git a/devops/deploy/docker/test/ubuntu22.04/Dockerfile b/devops/deploy/docker/test/ubuntu22.04/Dockerfile new file mode 100644 index 00000000000..2a44c7bff1d --- /dev/null +++ b/devops/deploy/docker/test/ubuntu22.04/Dockerfile @@ -0,0 +1,141 @@ + +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as +# a base environment for evaluating the Apache Cloudberry. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Apache Cloudberry +# DEB can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +FROM ubuntu:22.04 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="Europe/London" + +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin +ENV TZ=${TIMEZONE_VAR} +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NOWARNINGS="yes" + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- + +RUN apt-get update && \ + apt-get install -y -qq \ + htop \ + bat \ + silversearcher-ag \ + vim \ + wget \ + git \ + iproute2 \ + iputils-ping \ + lsof \ + openssh-server \ + pkg-config \ + python3.10 \ + python3-distutils \ + python3-pip \ + python3-setuptools \ + rsync \ + sudo \ + tzdata && \ + apt-get install -y -qq locales && \ + locale-gen ${LANG} && \ + update-locale LANG=${LANG} && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/bin/python3.10 /usr/bin/python + +# -------------------------------------------------------------------- +# User Creation and Configuration +# -------------------------------------------------------------------- +# - Create the 'gpadmin' user and group. +# - Configure the 'gpadmin' user with passwordless sudo privileges. +# - Add Cloudberry-specific entries to the gpadmin's .bashrc. +# -------------------------------------------------------------------- +RUN /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd -m -g gpadmin gpadmin && \ + echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ + chmod 0440 /etc/sudoers.d/90-gpadmin && \ + echo '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/cloudberry-env.sh ]; then\n source /usr/local/cloudberry/cloudberry-env.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- +# - Copy custom configuration files from the build context to /tmp/. +# - Apply custom system limits and timezone. +# - Set up SSH for password-based authentication. +# - Generate locale and set the default locale to en_US.UTF-8. +# -------------------------------------------------------------------- +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ + chmod 755 /tmp/init_system.sh && \ + ssh-keygen -A + +USER gpadmin +WORKDIR /home/gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] + diff --git a/devops/deploy/docker/test/ubuntu22.04/configs/90-cbdb-limits b/devops/deploy/docker/test/ubuntu22.04/configs/90-cbdb-limits new file mode 100644 index 00000000000..474957c42f6 --- /dev/null +++ b/devops/deploy/docker/test/ubuntu22.04/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited + +# Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 + +# Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 diff --git a/devops/deploy/docker/test/ubuntu22.04/configs/gpinitsystem.conf b/devops/deploy/docker/test/ubuntu22.04/configs/gpinitsystem.conf new file mode 100644 index 00000000000..896c8c79e54 --- /dev/null +++ b/devops/deploy/docker/test/ubuntu22.04/configs/gpinitsystem.conf @@ -0,0 +1,87 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# ---------------------------------------------------------------------- +# gpinitsystem Configuration File for Cloudberry Database +# ---------------------------------------------------------------------- +# This configuration file is used to initialize a Cloudberry Database +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# ---------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# ---------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# ---------------------------------------------------------------------- diff --git a/devops/deploy/docker/test/ubuntu22.04/configs/init_system.sh b/devops/deploy/docker/test/ubuntu22.04/configs/init_system.sh new file mode 100644 index 00000000000..1928fe511d9 --- /dev/null +++ b/devops/deploy/docker/test/ubuntu22.04/configs/init_system.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# Container Initialization Script +# -------------------------------------------------------------------- +# This script sets up the environment inside the Docker container for +# the Apache Cloudberry Build Environment. It performs the following +# tasks: +# +# 1. Verifies that the container is running with the expected hostname. +# 2. Starts the SSH daemon to allow SSH access to the container. +# 3. Configures passwordless SSH access for the 'gpadmin' user. +# 4. Sets up the necessary directories and configuration files for +# Apache Cloudberry. +# 5. Displays a welcome banner and system information. +# 6. Starts an interactive bash shell. +# +# This script is intended to be used as an entrypoint or initialization +# script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +sudo mkdir -p /run/sshd +sudo chmod 755 /run/sshd + +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# -------------------------------------------------------------------- +# Cloudberry Data Directories Setup +# -------------------------------------------------------------------- +# The script sets up the necessary directories for Apache Cloudberry, +# including directories for the coordinator, standby coordinator, primary +# segments, and mirror segments. It also sets up the configuration files +# required for initializing the database. +# -------------------------------------------------------------------- +sudo rm -rf /data1/* +sudo mkdir -p /data1/coordinator /data1/standby_coordinator /data1/primary /data1/mirror +sudo chown -R gpadmin.gpadmin /data1 + +# Copy the gpinitsystem configuration file to the home directory +cp /tmp/gpinitsystem.conf /home/gpadmin + +# Set up the hostfile for cluster initialization +echo $(hostname) > /home/gpadmin/hostfile_gpinitsystem + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# - Cloudberry version (if installed) +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +# Check if Apache Cloudberry is installed and display its version +if dpkg -l apache-cloudberry-db-incubating > /dev/null 2>&1; then + CBDB_VERSION=$(/usr/local/cbdb/bin/postgres --gp-version) +else + CBDB_VERSION="Not installed" +fi + +cat <<-EOF +Welcome to the Apache Cloudberry Test Environment! + +Cloudberry version .. : $CBDB_VERSION +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/pom.xml b/pom.xml index a3102d4d882..d6913ec5053 100644 --- a/pom.xml +++ b/pom.xml @@ -1240,7 +1240,7 @@ code or new licensing patterns. Makefile aclocal.m4 python-dependencies.txt - + + devops/build/packaging/deb/ubuntu22.04/**