diff --git a/.github/workflows/docker-cbdb-build-containers.yml b/.github/workflows/docker-cbdb-build-containers.yml index 1ce1608..a3e4dab 100644 --- a/.github/workflows/docker-cbdb-build-containers.yml +++ b/.github/workflows/docker-cbdb-build-containers.yml @@ -60,6 +60,7 @@ on: paths: - 'images/docker/cbdb/build/rocky8/**' - 'images/docker/cbdb/build/rocky9/**' + - 'images/docker/cbdb/build/ubuntu22.04/**' workflow_dispatch: # Manual trigger # Prevent multiple workflow runs from interfering with each other @@ -76,7 +77,7 @@ jobs: # Matrix strategy to build for both Rocky Linux 8 and 9 strategy: matrix: - platform: ['rocky8', 'rocky9'] + platform: ['rocky8', 'rocky9', 'ubuntu22.04'] steps: # Checkout repository code with full history @@ -103,6 +104,8 @@ jobs: - 'images/docker/cbdb/build/rocky8/**' rocky9: - 'images/docker/cbdb/build/rocky9/**' + ubuntu22.04: + - 'images/docker/cbdb/build/ubuntu22.04/**' # Set up QEMU for multi-architecture support # This allows building ARM64 images on AMD64 infrastructure and vice versa diff --git a/.github/workflows/docker-cbdb-test-containers.yml b/.github/workflows/docker-cbdb-test-containers.yml index 9867f63..c385817 100644 --- a/.github/workflows/docker-cbdb-test-containers.yml +++ b/.github/workflows/docker-cbdb-test-containers.yml @@ -49,6 +49,7 @@ on: paths: - 'images/docker/cbdb/test/rocky8/**' - 'images/docker/cbdb/test/rocky9/**' + - 'images/docker/cbdb/test/ubuntu22.04/**' workflow_dispatch: # Manual trigger # Prevent multiple workflow runs from interfering with each other @@ -62,8 +63,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Build for both Rocky Linux 8 and 9 - platform: ['rocky8', 'rocky9'] + # Build for both Rocky Linux 8 and 9, ubuntu 22.04 + platform: ['rocky8', 'rocky9', 'ubuntu22.04'] steps: # Checkout repository code @@ -87,6 +88,8 @@ jobs: - 'images/docker/cbdb/test/rocky8/**' rocky9: - 'images/docker/cbdb/test/rocky9/**' + ubuntu22.04: + - 'images/docker/cbdb/test/ubuntu22.04/**' # Skip if no changes for current platform - name: Skip if not relevant diff --git a/build_automation/cloudberry/scripts/cloudberry-utils.sh b/build_automation/cloudberry/scripts/cloudberry-utils.sh index 01b0b1b..344b7fd 100755 --- a/build_automation/cloudberry/scripts/cloudberry-utils.sh +++ b/build_automation/cloudberry/scripts/cloudberry-utils.sh @@ -146,3 +146,14 @@ log_completion() { local timestamp=$(date "+%Y.%m.%d-%H.%M.%S") echo "${script_name} execution completed successfully at ${timestamp}" | tee -a "${log_file}" } + +detect_os() { + if [ -f /etc/os-release ]; then + . /etc/os-release + OS_ID=$ID + OS_VERSION=$VERSION_ID + else + echo "Unsupported system: cannot detect OS" >&2 + exit 99 + fi +} diff --git a/build_automation/cloudberry/scripts/configure-cloudberry.sh b/build_automation/cloudberry/scripts/configure-cloudberry.sh index 4d84c3b..58f6c97 100755 --- a/build_automation/cloudberry/scripts/configure-cloudberry.sh +++ b/build_automation/cloudberry/scripts/configure-cloudberry.sh @@ -33,7 +33,6 @@ # - MapReduce Processing # - Oracle Compatibility (orafce) # - ORCA Query Optimizer -# - PAX Access Method # - PXF External Table Access # - Test Automation Support (tap-tests) # @@ -92,6 +91,11 @@ set -euo pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "${SCRIPT_DIR}/cloudberry-utils.sh" +# Call it before conditional logic +detect_os + +echo "Detected OS: $OS_ID $OS_VERSION" + # Define log directory and files export LOG_DIR="${SRC_DIR}/build-logs" CONFIGURE_LOG="${LOG_DIR}/configure.log" @@ -103,11 +107,14 @@ init_environment "Cloudberry Configure Script" "${CONFIGURE_LOG}" log_section "Initial Setup" execute_cmd sudo rm -rf /usr/local/cloudberry-db || exit 2 execute_cmd sudo chmod a+w /usr/local || exit 2 -execute_cmd mkdir -p /usr/local/cloudberry-db/lib || exit 2 -execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ - /usr/local/xerces-c/lib/libxerces-c-3.3.so \ - /usr/local/cloudberry-db/lib || exit 3 -execute_cmd sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db || exit 2 +execute_cmd sudo mkdir -p ${BUILD_DESTINATION}/lib || exit 2 +if [[ "$OS_ID" == "rocky" && "$OS_VERSION" =~ ^(8|9) ]]; then + execute_cmd sudo cp /usr/local/xerces-c/lib/libxerces-c.so \ + /usr/local/xerces-c/lib/libxerces-c-3.3.so \ + /usr/local/cloudberry-db/lib || exit 3 +fi +execute_cmd sudo chown -R gpadmin:gpadmin ${BUILD_DESTINATION} || exit 2 + log_section_end "Initial Setup" # Set environment @@ -120,7 +127,6 @@ CONFIGURE_DEBUG_OPTS="" if [ "${ENABLE_DEBUG:-false}" = "true" ]; then CONFIGURE_DEBUG_OPTS="--enable-debug \ - --enable-profiling \ --enable-cassert \ --enable-debug-extensions" fi @@ -134,7 +140,7 @@ execute_cmd ./configure --prefix=/usr/local/cloudberry-db \ --enable-mapreduce \ --enable-orafce \ --enable-orca \ - --enable-pax \ + --enable-pax \ --enable-pxf \ --enable-tap-tests \ ${CONFIGURE_DEBUG_OPTS} \ diff --git a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh index e01d62f..61a72ee 100755 --- a/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh +++ b/build_automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh @@ -79,7 +79,7 @@ init_environment "Cloudberry Demo Cluster Script" "${CLUSTER_LOG}" # Setup environment log_section "Environment Setup" -source /usr/local/cloudberry-db/cloudberry-env.sh || exit 1 +source ${BUILD_DESTINATION}/cloudberry-env.sh || exit 1 log_section_end "Environment Setup" # Verify SSH access diff --git a/images/docker/cbdb/build/ubuntu22.04/Dockerfile b/images/docker/cbdb/build/ubuntu22.04/Dockerfile new file mode 100644 index 0000000..7a3abd6 --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/Dockerfile @@ -0,0 +1,196 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as +# a base environment for evaluating the Apache Cloudberry. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Apache Cloudberry +# DEB can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +FROM ubuntu:22.04 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="Europe/London" + +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin +ENV TZ=${TIMEZONE_VAR} +ENV DEBIAN_FRONTEND=noninteractive + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- + +RUN apt-get update && \ + apt-get install -y -qq \ + htop \ + bat \ + silversearcher-ag \ + vim \ + wget && \ + apt-get install -y -qq locales && \ + locale-gen "en_US.UTF-8" && \ + update-locale LANG="en_US.UTF-8" && \ + apt-get install -y -qq \ + bison \ + build-essential \ + cmake \ + dpkg-dev \ + fakeroot \ + flex \ + g++-11 \ + gcc-11 \ + git \ + iproute2 \ + iputils-ping \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libipc-run-perl \ + libkrb5-dev \ + libldap-dev \ + liblz4-dev \ + libpam0g-dev \ + libperl-dev \ + libprotobuf-dev \ + libreadline-dev \ + libssl-dev \ + libuv1-dev \ + libxerces-c-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + lsof \ + make \ + openssh-server \ + pkg-config \ + protobuf-compiler \ + python3-distutils \ + python3-pip \ + python3-setuptools \ + python3.10 \ + python3.10-dev \ + rsync \ + sudo \ + tzdata \ + zlib1g-dev && \ + apt-get install -y -qq \ + ca-certificates-java \ + cgroup-tools \ + curl \ + debhelper \ + libaprutil1-dev \ + libcgroup1 \ + ninja-build \ + quilt \ + unzip && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + cd && GO_VERSION="go1.23.4" && \ + ARCH=$(uname -m) && \ + if [ "${ARCH}" = "aarch64" ]; then \ + GO_ARCH="arm64" && \ + GO_SHA256="16e5017863a7f6071363782b1b8042eb12c6ca4f4cd71528b2123f0a1275b13e"; \ + elif [ "${ARCH}" = "x86_64" ]; then \ + GO_ARCH="amd64" && \ + GO_SHA256="6924efde5de86fe277676e929dc9917d466efa02fb934197bc2eba35d5680971"; \ + else \ + echo "Unsupported architecture: ${ARCH}" && exit 1; \ + fi && \ + GO_URL="https://go.dev/dl/${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + wget -nv "${GO_URL}" && \ + echo "${GO_SHA256} ${GO_VERSION}.linux-${GO_ARCH}.tar.gz" | sha256sum -c - && \ + tar xf "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + mv go "/usr/local/${GO_VERSION}" && \ + ln -s "/usr/local/${GO_VERSION}" /usr/local/go && \ + rm -f "${GO_VERSION}.linux-${GO_ARCH}.tar.gz" && \ + echo 'export PATH=$PATH:/usr/local/go/bin' | tee -a /etc/profile.d/go.sh > /dev/null + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 && \ + update-alternatives --install /usr/bin/x86_64-linux-gnu-gcc x86_64-linux-gnu-gcc /usr/bin/gcc-11 100 && \ + update-alternatives --set gcc /usr/bin/gcc-11 && \ + update-alternatives --set g++ /usr/bin/g++-11 + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- + +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ + chmod 755 /tmp/init_system.sh && \ + /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd -m -g gpadmin gpadmin && \ + echo 'gpadmin ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/90-gpadmin && \ + chmod 0440 /etc/sudoers.d/90-gpadmin && \ + ssh-keygen -A && \ + mkdir /var/run/sshd && chmod 0755 /var/run/sshd + +# Install testinfra via pip +RUN pip3 install pytest-testinfra + +# Example: Copying test files into the container +COPY tests /tests + +USER gpadmin +WORKDIR /home/gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] diff --git a/images/docker/cbdb/build/ubuntu22.04/configs/90-cbdb-limits b/images/docker/cbdb/build/ubuntu22.04/configs/90-cbdb-limits new file mode 100644 index 0000000..8ea1d9e --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits + # -------------------------------------------------------------------- + # + # Licensed to the Apache Software Foundation (ASF) under one or more + # contributor license agreements. See the NOTICE file distributed + # with this work for additional information regarding copyright + # ownership. The ASF licenses this file to You under the Apache + # License, Version 2.0 (the "License"); you may not use this file + # except in compliance with the License. You may obtain a copy of the + # License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + # implied. See the License for the specific language governing + # permissions and limitations under the License. + # + # -------------------------------------------------------------------- + + # Core dump file size limits for gpadmin + gpadmin soft core unlimited + gpadmin hard core unlimited + + # Open file limits for gpadmin + gpadmin soft nofile 524288 + gpadmin hard nofile 524288 + + # Process limits for gpadmin + gpadmin soft nproc 131072 + gpadmin hard nproc 131072 diff --git a/images/docker/cbdb/build/ubuntu22.04/configs/gpinitsystem.conf b/images/docker/cbdb/build/ubuntu22.04/configs/gpinitsystem.conf new file mode 100644 index 0000000..d4d3122 --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/configs/gpinitsystem.conf @@ -0,0 +1,89 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# gpinitsystem Configuration File for Apache Cloudberry +# -------------------------------------------------------------------- +# This configuration file is used to initialize an Apache Cloudberry +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# -------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# -------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# -------------------------------------------------------------------- diff --git a/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh b/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh new file mode 100755 index 0000000..52a4446 --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/configs/init_system.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +## Container Initialization Script +# -------------------------------------------------------------------- +## This script sets up the environment inside the Docker container for +## the Apache Cloudberry Build Environment. It performs the following +## tasks: +## +## 1. Verifies that the container is running with the expected hostname. +## 2. Starts the SSH daemon to allow SSH access to the container. +## 3. Configures passwordless SSH access for the 'gpadmin' user. +## 4. Displays a welcome banner and system information. +## 5. Starts an interactive bash shell. +## +## This script is intended to be used as an entrypoint or initialization +## script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +cat <<-EOF +Welcome to the Apache Cloudberry Build Environment! + +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# Add go to PATH +source /etc/profile.d/go.sh + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/images/docker/cbdb/build/ubuntu22.04/tests/requirements.txt b/images/docker/cbdb/build/ubuntu22.04/tests/requirements.txt new file mode 100644 index 0000000..b9711ed --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/tests/requirements.txt @@ -0,0 +1,3 @@ +testinfra +pytest-testinfra +paramiko diff --git a/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py b/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py new file mode 100644 index 0000000..d7f018a --- /dev/null +++ b/images/docker/cbdb/build/ubuntu22.04/tests/testinfra/test_cloudberry_db_env.py @@ -0,0 +1,128 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +import testinfra + +def test_installed_packages(host): + """ + Test if the essential packages are installed. + """ + packages = [ + "bat", + "bison", + "cmake", + "flex", + "g++-11", + "gcc-11", + "git", + "htop", + "iproute2", + "iputils-ping", + "libapr1-dev", + "libbz2-dev", + "libcurl4-gnutls-dev", + "libevent-dev", + "libipc-run-perl", + "libkrb5-dev", + "libldap-dev", + "liblz4-dev", + "libpam0g-dev", + "libperl-dev", + "libprotobuf-dev", + "libreadline-dev", + "libssl-dev", + "libuv1-dev", + "libxerces-c-dev", + "libxml2-dev", + "libyaml-dev", + "libzstd-dev", + "locales", + "lsof", + "make", + "openssh-server", + "pkg-config", + "protobuf-compiler", + "python3-distutils", + "python3-pip", + "python3-setuptools", + "python3.10", + "python3.10-dev", + "rsync", + "silversearcher-ag", + "sudo", + "tzdata", + "vim", + "wget", + "zlib1g-dev" + ] + for package in packages: + pkg = host.package(package) + assert pkg.is_installed + + +def test_user_gpadmin_exists(host): + """ + Test if the gpadmin user exists and is configured properly. + """ + user = host.user("gpadmin") + assert user.exists + assert "gpadmin" in user.groups + + +def test_ssh_service(host): + """ + Test if SSH service is configured correctly. + """ + sshd_config = host.file("/etc/ssh/sshd_config") + assert sshd_config.exists + + +def test_timezone(host): + """ + Test if the timezone is configured correctly. + """ + localtime = host.file("/etc/localtime") + assert localtime.exists + + +def test_system_limits_configured(host): + """ + Test if the custom system limits are applied. + """ + limits_file = host.file("/etc/security/limits.d/90-cbdb-limits") + assert limits_file.exists + + +def test_init_system_script(host): + """ + Test if the init_system.sh script is present and executable. + """ + script = host.file("/tmp/init_system.sh") + assert script.exists + assert script.mode == 0o755 + + +def test_locale_generated(host): + """ + Test if the en_US.UTF-8 locale is correctly generated. + """ + locale = host.run("locale -a | grep en_US.utf8") + assert locale.exit_status == 0 + assert "en_US.utf8" in locale.stdout diff --git a/images/docker/cbdb/test/ubuntu22.04/Dockerfile b/images/docker/cbdb/test/ubuntu22.04/Dockerfile new file mode 100644 index 0000000..2a44c7b --- /dev/null +++ b/images/docker/cbdb/test/ubuntu22.04/Dockerfile @@ -0,0 +1,141 @@ + +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# +# Apache Cloudberry (incubating) is an effort undergoing incubation at +# the Apache Software Foundation (ASF), sponsored by the Apache +# Incubator PMC. +# +# Incubation is required of all newly accepted projects until a +# further review indicates that the infrastructure, communications, +# and decision making process have stabilized in a manner consistent +# with other successful ASF projects. +# +# While incubation status is not necessarily a reflection of the +# completeness or stability of the code, it does indicate that the +# project has yet to be fully endorsed by the ASF. +# +# -------------------------------------------------------------------- +# Dockerfile for Apache Cloudberry Base Environment +# -------------------------------------------------------------------- +# This Dockerfile sets up a Ubuntu jammy 22.04 -based container to serve as +# a base environment for evaluating the Apache Cloudberry. It installs +# necessary system utilities, configures the environment for SSH access, +# and sets up a 'gpadmin' user with sudo privileges. The Apache Cloudberry +# DEB can be installed into this container for testing and +# functional verification. +# +# Key Features: +# - Locale setup for en_US.UTF-8 +# - SSH daemon setup for remote access +# - Essential system utilities installation +# - Separate user creation and configuration steps +# +# Security Considerations: +# - This Dockerfile prioritizes ease of use for functional testing and +# evaluation. It includes configurations such as passwordless sudo access +# for the 'gpadmin' user and SSH access with password authentication. +# - These configurations are suitable for testing and development but +# should NOT be used in a production environment due to potential security +# risks. +# +# Usage: +# docker build -t cloudberry-db-base-env . +# docker run -h cdw -it cloudberry-db-base-env +# -------------------------------------------------------------------- + +FROM ubuntu:22.04 + +# Argument for configuring the timezone +ARG TIMEZONE_VAR="Europe/London" + +# Environment variables for locale and user +ENV container=docker +ENV LANG=en_US.UTF-8 +ENV USER=gpadmin +ENV TZ=${TIMEZONE_VAR} +ENV DEBIAN_FRONTEND=noninteractive +ENV DEBCONF_NOWARNINGS="yes" + +# -------------------------------------------------------------------- +# Install Development Tools and Utilities +# -------------------------------------------------------------------- + +RUN apt-get update && \ + apt-get install -y -qq \ + htop \ + bat \ + silversearcher-ag \ + vim \ + wget \ + git \ + iproute2 \ + iputils-ping \ + lsof \ + openssh-server \ + pkg-config \ + python3.10 \ + python3-distutils \ + python3-pip \ + python3-setuptools \ + rsync \ + sudo \ + tzdata && \ + apt-get install -y -qq locales && \ + locale-gen ${LANG} && \ + update-locale LANG=${LANG} && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/bin/python3.10 /usr/bin/python + +# -------------------------------------------------------------------- +# User Creation and Configuration +# -------------------------------------------------------------------- +# - Create the 'gpadmin' user and group. +# - Configure the 'gpadmin' user with passwordless sudo privileges. +# - Add Cloudberry-specific entries to the gpadmin's .bashrc. +# -------------------------------------------------------------------- +RUN /usr/sbin/groupadd gpadmin && \ + /usr/sbin/useradd -m -g gpadmin gpadmin && \ + echo 'gpadmin ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/90-gpadmin && \ + chmod 0440 /etc/sudoers.d/90-gpadmin && \ + echo '\n# Add Cloudberry entries\nif [ -f /usr/local/cloudberry/cloudberry-env.sh ]; then\n source /usr/local/cloudberry/cloudberry-env.sh\n export COORDINATOR_DATA_DIRECTORY=/data1/coordinator/gpseg-1\nfi' >> /home/gpadmin/.bashrc + +# -------------------------------------------------------------------- +# Copy Configuration Files and Setup the Environment +# -------------------------------------------------------------------- +# - Copy custom configuration files from the build context to /tmp/. +# - Apply custom system limits and timezone. +# - Set up SSH for password-based authentication. +# - Generate locale and set the default locale to en_US.UTF-8. +# -------------------------------------------------------------------- +COPY ./configs/* /tmp/ + +RUN cp /tmp/90-cbdb-limits /etc/security/limits.d/90-cbdb-limits && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone && \ + chmod 755 /tmp/init_system.sh && \ + ssh-keygen -A + +USER gpadmin +WORKDIR /home/gpadmin + +CMD ["bash","-c","/tmp/init_system.sh"] + diff --git a/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits b/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits new file mode 100644 index 0000000..474957c --- /dev/null +++ b/images/docker/cbdb/test/ubuntu22.04/configs/90-cbdb-limits @@ -0,0 +1,32 @@ +# /etc/security/limits.d/90-db-limits +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- + +# Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited + +# Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 + +# Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 diff --git a/images/docker/cbdb/test/ubuntu22.04/configs/gpinitsystem.conf b/images/docker/cbdb/test/ubuntu22.04/configs/gpinitsystem.conf new file mode 100644 index 0000000..896c8c7 --- /dev/null +++ b/images/docker/cbdb/test/ubuntu22.04/configs/gpinitsystem.conf @@ -0,0 +1,87 @@ +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# ---------------------------------------------------------------------- +# gpinitsystem Configuration File for Cloudberry Database +# ---------------------------------------------------------------------- +# This configuration file is used to initialize a Cloudberry Database +# cluster. It defines the settings for the coordinator, primary segments, +# and mirrors, as well as other important configuration options. +# ---------------------------------------------------------------------- + +# Segment prefix - This prefix is used for naming the segment directories. +# For example, the primary segment directories will be named gpseg0, gpseg1, etc. +SEG_PREFIX=gpseg + +# Coordinator port - The port number where the coordinator will listen. +# This is the port used by clients to connect to the database. +COORDINATOR_PORT=5432 + +# Coordinator hostname - The hostname of the machine where the coordinator +# will be running. The $(hostname) command will automatically insert the +# hostname of the current machine. +COORDINATOR_HOSTNAME=$(hostname) + +# Coordinator data directory - The directory where the coordinator's data +# will be stored. This directory should have enough space to store metadata +# and system catalogs. +COORDINATOR_DIRECTORY=/data1/coordinator + +# Base port for primary segments - The starting port number for the primary +# segments. Each primary segment will use a unique port number starting from +# this base. +PORT_BASE=6000 + +# Primary segment data directories - An array specifying the directories where +# the primary segment data will be stored. Each directory corresponds to a +# primary segment. In this case, two primary segments will be created in the +# same directory. +declare -a DATA_DIRECTORY=(/data1/primary /data1/primary) + +# Base port for mirror segments - The starting port number for the mirror +# segments. Each mirror segment will use a unique port number starting from +# this base. +MIRROR_PORT_BASE=7000 + +# Mirror segment data directories - An array specifying the directories where +# the mirror segment data will be stored. Each directory corresponds to a +# mirror segment. In this case, two mirror segments will be created in the +# same directory. +declare -a MIRROR_DATA_DIRECTORY=(/data1/mirror /data1/mirror) + +# Trusted shell - The shell program used for remote execution. Cloudberry uses +# SSH to run commands on other machines in the cluster. 'ssh' is the default. +TRUSTED_SHELL=ssh + +# Database encoding - The character set encoding to be used by the database. +# 'UNICODE' is a common choice, especially for internationalization. +ENCODING=UNICODE + +# Default database name - The name of the default database to be created during +# initialization. This is also the default database that the gpadmin user will +# connect to. +DATABASE_NAME=gpadmin + +# Machine list file - A file containing the list of hostnames where the primary +# segments will be created. Each line in the file represents a different machine. +# This file is critical for setting up the cluster across multiple nodes. +MACHINE_LIST_FILE=/home/gpadmin/hostfile_gpinitsystem + +# ---------------------------------------------------------------------- +# End of gpinitsystem Configuration File +# ---------------------------------------------------------------------- diff --git a/images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh b/images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh new file mode 100644 index 0000000..1928fe5 --- /dev/null +++ b/images/docker/cbdb/test/ubuntu22.04/configs/init_system.sh @@ -0,0 +1,224 @@ +#!/bin/bash +# -------------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed +# with this work for additional information regarding copyright +# ownership. The ASF licenses this file to You under the Apache +# License, Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a copy of the +# License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. +# +# -------------------------------------------------------------------- +# Container Initialization Script +# -------------------------------------------------------------------- +# This script sets up the environment inside the Docker container for +# the Apache Cloudberry Build Environment. It performs the following +# tasks: +# +# 1. Verifies that the container is running with the expected hostname. +# 2. Starts the SSH daemon to allow SSH access to the container. +# 3. Configures passwordless SSH access for the 'gpadmin' user. +# 4. Sets up the necessary directories and configuration files for +# Apache Cloudberry. +# 5. Displays a welcome banner and system information. +# 6. Starts an interactive bash shell. +# +# This script is intended to be used as an entrypoint or initialization +# script for the Docker container. +# -------------------------------------------------------------------- + +# -------------------------------------------------------------------- +# Check if the hostname is 'cdw' +# -------------------------------------------------------------------- +# The script checks if the container's hostname is set to 'cdw'. This is +# a requirement for this environment, and if the hostname does not match, +# the script will exit with an error message. This ensures consistency +# across different environments. +# -------------------------------------------------------------------- +if [ "$(hostname)" != "cdw" ]; then + echo "Error: This container must be run with the hostname 'cdw'." + echo "Use the following command: docker run -h cdw ..." + exit 1 +fi + +# -------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# -------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# -------------------------------------------------------------------- +sudo mkdir -p /run/sshd +sudo chmod 755 /run/sshd + +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Remove /run/nologin to allow logins +# -------------------------------------------------------------------- +# The /run/nologin file, if present, prevents users from logging into +# the system. This file is removed to ensure that users can log in via SSH. +# -------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# -------------------------------------------------------------------- +# Configure passwordless SSH access for 'gpadmin' user +# -------------------------------------------------------------------- +# The script sets up SSH key-based authentication for the 'gpadmin' user, +# allowing passwordless SSH access. It generates a new SSH key pair if one +# does not already exist, and configures the necessary permissions. +# -------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh +chmod 700 /home/gpadmin/.ssh + +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 4096 -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" > /dev/null 2>&1 +fi + +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +chmod 600 /home/gpadmin/.ssh/authorized_keys + +# Add the container's hostname to the known_hosts file to avoid SSH warnings +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts 2>/dev/null + +# -------------------------------------------------------------------- +# Cloudberry Data Directories Setup +# -------------------------------------------------------------------- +# The script sets up the necessary directories for Apache Cloudberry, +# including directories for the coordinator, standby coordinator, primary +# segments, and mirror segments. It also sets up the configuration files +# required for initializing the database. +# -------------------------------------------------------------------- +sudo rm -rf /data1/* +sudo mkdir -p /data1/coordinator /data1/standby_coordinator /data1/primary /data1/mirror +sudo chown -R gpadmin.gpadmin /data1 + +# Copy the gpinitsystem configuration file to the home directory +cp /tmp/gpinitsystem.conf /home/gpadmin + +# Set up the hostfile for cluster initialization +echo $(hostname) > /home/gpadmin/hostfile_gpinitsystem + +# Change to the home directory of the current user +cd $HOME + +# -------------------------------------------------------------------- +# Display a Welcome Banner +# -------------------------------------------------------------------- +# The following ASCII art and welcome message are displayed when the +# container starts. This banner provides a visual indication that the +# container is running in the Apache Cloudberry Build Environment. +# -------------------------------------------------------------------- +cat <<-'EOF' + +====================================================================== + + ++++++++++ ++++++ + ++++++++++++++ +++++++ + ++++ +++++ ++++ + ++++ +++++++++ + =+==== =============+ + ======== =====+ ===== + ==== ==== ==== ==== + ==== === === ==== + ==== === === ==== + ==== === ==-- === + ===== ===== -- ==== + ===================== ====== + ============================ + =-----= + ____ _ _ _ + / ___|| | ___ _ _ __| || |__ ___ _ __ _ __ _ _ + | | | | / _ \ | | | | / _` || '_ \ / _ \| '__|| '__|| | | | + | |___ | || (_) || |_| || (_| || |_) || __/| | | | | |_| | + \____||_| \____ \__,_| \__,_||_.__/ \___||_| |_| \__, | + |___/ +---------------------------------------------------------------------- + +EOF + +# -------------------------------------------------------------------- +# Display System Information +# -------------------------------------------------------------------- +# The script sources the /etc/os-release file to retrieve the operating +# system name and version. It then displays the following information: +# - OS name and version +# - Current user +# - Container hostname +# - IP address +# - CPU model name and number of cores +# - Total memory available +# - Cloudberry version (if installed) +# This information is useful for users to understand the environment they +# are working in. +# -------------------------------------------------------------------- +source /etc/os-release + +# First, create the CPU info detection function +get_cpu_info() { + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + lscpu | grep 'Model name:' | awk '{print substr($0, index($0,$3))}' + elif [ "$ARCH" = "aarch64" ]; then + VENDOR=$(lscpu | grep 'Vendor ID:' | awk '{print $3}') + if [ "$VENDOR" = "Apple" ] || [ "$VENDOR" = "0x61" ]; then + echo "Apple Silicon ($ARCH)" + else + if [ -f /proc/cpuinfo ]; then + IMPL=$(grep "CPU implementer" /proc/cpuinfo | head -1 | awk '{print $3}') + PART=$(grep "CPU part" /proc/cpuinfo | head -1 | awk '{print $3}') + if [ ! -z "$IMPL" ] && [ ! -z "$PART" ]; then + echo "ARM $ARCH (Implementer: $IMPL, Part: $PART)" + else + echo "ARM $ARCH" + fi + else + echo "ARM $ARCH" + fi + fi + else + echo "Unknown architecture: $ARCH" + fi +} + +# Check if Apache Cloudberry is installed and display its version +if dpkg -l apache-cloudberry-db-incubating > /dev/null 2>&1; then + CBDB_VERSION=$(/usr/local/cbdb/bin/postgres --gp-version) +else + CBDB_VERSION="Not installed" +fi + +cat <<-EOF +Welcome to the Apache Cloudberry Test Environment! + +Cloudberry version .. : $CBDB_VERSION +Container OS ........ : $NAME $VERSION +User ................ : $(whoami) +Container hostname .. : $(hostname) +IP Address .......... : $(hostname -I | awk '{print $1}') +CPU Info ............ : $(get_cpu_info) +CPU(s) .............. : $(nproc) +Memory .............. : $(free -h | grep Mem: | awk '{print $2}') total +====================================================================== + +EOF + +# -------------------------------------------------------------------- +# Start an interactive bash shell +# -------------------------------------------------------------------- +# Finally, the script starts an interactive bash shell to keep the +# container running and allow the user to interact with the environment. +# -------------------------------------------------------------------- +/bin/bash diff --git a/packaging/deb/ubuntu22.04/debian/changelog b/packaging/deb/ubuntu22.04/debian/changelog new file mode 100644 index 0000000..211d271 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/changelog @@ -0,0 +1,5 @@ +apache-cloudberry-db-incubating (2.0.0) jammy; urgency=medium + + * Initial release. + + -- Cloudberry Team Wed, 26 Mar 2025 11:10:44 +0000 diff --git a/packaging/deb/ubuntu22.04/debian/compat b/packaging/deb/ubuntu22.04/debian/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/compat @@ -0,0 +1 @@ +9 diff --git a/packaging/deb/ubuntu22.04/debian/control b/packaging/deb/ubuntu22.04/debian/control new file mode 100644 index 0000000..c635c53 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/control @@ -0,0 +1,100 @@ +Source: apache-cloudberry-db-incubating +Maintainer: Apache Cloudberry (Incubating) +Section: database +Build-Depends: debhelper (>= 9), + bison, + ca-certificates-java, + ca-certificates, + cmake, + curl, + cgroup-tools, + flex, + gcc-11, + g++-11, + git, + krb5-multidev, + libapr1-dev, + libbz2-dev, + libcurl4-gnutls-dev, + libevent-dev, + libkrb5-dev, + libldap2-dev, + libperl-dev, + libreadline6-dev, + libssl-dev, + libxml2-dev, + libyaml-dev, + libzstd-dev, + libaprutil1-dev, + libpam0g-dev, + libpam0g, + libcgroup1, + libyaml-0-2, + libldap-2.5-0, + libssl3, + libxerces-c-dev, + libxerces-c3.2, + ninja-build, + quilt, + unzip, + wget, + zlib1g-dev, + libuv1-dev + +Package: apache-cloudberry-db-incubating +Architecture: amd64 +Depends: curl, + cgroup-tools, + iputils-ping, + krb5-multidev, + less, + libapr1, + libbz2-1.0, + libcurl4, + libcurl3-gnutls, + libevent-2.1-7, + libreadline8, + libxml2, + libyaml-0-2, + libldap-2.5-0, + libzstd1, + libcgroup1, + libssl3, + libpam0g, + libxerces-c3.2, + locales, + net-tools, + openssh-client, + openssh-server, + openssl, + python-six, + python2.7, + python2.7-dev, + rsync, + wget, + zlib1g, + libuv1 +Description: Apache Cloudberry (incubating) is an advanced, open-source, massively + parallel processing (MPP) data warehouse developed from PostgreSQL and + Greenplum. It is designed for high-performance analytics on + large-scale data sets, offering powerful analytical capabilities and + enhanced security features. + Key Features: + - Massively parallel processing for optimized performance + - Advanced analytics for complex data processing + - Integration with ETL and BI tools + - Compatibility with multiple data sources and formats + - Enhanced security features + Apache Cloudberry supports both batch processing and real-time data + warehousing, making it a versatile solution for modern data + environments. + Apache Cloudberry (incubating) is an effort undergoing incubation at + the Apache Software Foundation (ASF), sponsored by the Apache + Incubator PMC. + Incubation is required of all newly accepted projects until a further + review indicates that the infrastructure, communications, and decision + making process have stabilized in a manner consistent with other + successful ASF projects. + While incubation status is not necessarily a reflection of the + completeness or stability of the code, it does indicate that the + project has yet to be fully endorsed by the ASF. diff --git a/packaging/deb/ubuntu22.04/debian/install b/packaging/deb/ubuntu22.04/debian/install new file mode 100644 index 0000000..3e29bb0 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/install @@ -0,0 +1 @@ +debian/build/* /usr/cloudberry-db diff --git a/packaging/deb/ubuntu22.04/debian/postinst b/packaging/deb/ubuntu22.04/debian/postinst new file mode 100644 index 0000000..5382505 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/postinst @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin +GPHOME=/usr/cloudberry-db + +if [ "$1" = configure ]; then + + ln -s ${GPHOME} /usr/local/cloudberry-db + ln -s ${GPHOME} /usr/local/cloudberry + +fi diff --git a/packaging/deb/ubuntu22.04/debian/preinst b/packaging/deb/ubuntu22.04/debian/preinst new file mode 100644 index 0000000..29d6211 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/preinst @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +GPADMIN=gpadmin + +case $1 in + install|upgrade) + id "${GPADMIN}" >> /dev/null 2>&1 || \ + useradd -r -m -d /home/${GPADMIN} -s /bin/bash ${GPADMIN} + ;; +esac + diff --git a/packaging/deb/ubuntu22.04/debian/rules b/packaging/deb/ubuntu22.04/debian/rules new file mode 100755 index 0000000..6213985 --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/rules @@ -0,0 +1,49 @@ +#!/usr/bin/make -f + +DH_VERBOSE = 1 +DPKG_EXPORT_BUILDFLAGS = 1 + +CBDB_BIN_PATH := /usr/local/cloudberry-db +DEBIAN_DESTINATION := $(shell pwd)/debian/build + +# assumes that CWD is root of cbdb source +CBDB_PKG_VERSION := $(CBDB_PKG_VERSION) +PACKAGE_CBDB := $(shell cat debian/control | egrep "^Package: " | cut -d " " -f 2) +PATH := ${DEBIAN_DESTINATION}/bin:${PATH} + +.PHONY: gpinstall + +include /usr/share/dpkg/default.mk + +%: + dh $@ --parallel + +gpinstall: + make install + +override_dh_auto_install: gpinstall + # the staging directory for creating a debian is NOT the right GPHOME. + # change GPHOME to point to the post-install target install directory. + sed -i "s#GPHOME=.*#GPHOME=${CBDB_BIN_PATH}#g" ${DEBIAN_DESTINATION}/cloudberry-env.sh + +override_dh_auto_build: + echo "Skipping build" + +override_dh_auto_clean: + echo "Skipping clean" + +override_dh_auto_configure: + echo "Skipping configure" + +override_dh_auto_test: + echo "Skipping auto test" + +override_dh_gencontrol: + echo "using version ${CBDB_PKG_VERSION} for binary Cloudberry" + dh_gencontrol -- -v${CBDB_PKG_VERSION} -p${PACKAGE_CBDB} + +override_dh_shlibdeps: + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/libfakeroot:${DEBIAN_DESTINATION}/lib dh_shlibdeps --dpkg-shlibdeps-params=--ignore-missing-info + +clean_dev_local: + rm -rf ${DEBIAN_DESTINATION} diff --git a/packaging/deb/ubuntu22.04/debian/source/format b/packaging/deb/ubuntu22.04/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/packaging/deb/ubuntu22.04/debian/source/local-options b/packaging/deb/ubuntu22.04/debian/source/local-options new file mode 100644 index 0000000..00131ee --- /dev/null +++ b/packaging/deb/ubuntu22.04/debian/source/local-options @@ -0,0 +1,2 @@ +#abort-on-upstream-changes +#unapply-patches diff --git a/scripts/build-deb.sh b/scripts/build-deb.sh new file mode 100755 index 0000000..05b29d5 --- /dev/null +++ b/scripts/build-deb.sh @@ -0,0 +1,148 @@ +#!/bin/bash +# +# Script Name: build-deb.sh +# +# Description: +# This script automates the process of building an DEB package using a specified +# version number. It ensures that the necessary tools are installed +# and that the control file exists before attempting to build the DEB. The script +# also includes error handling to provide meaningful feedback in case of failure. +# +# Usage: +# ./build-deb.sh [-v ] [-h] [--dry-run] +# +# Options: +# -v, --version : Specify the version (required) +# -h, --help : Display this help and exit +# -n, --dry-run : Show what would be done, without making any changes +# +# Example: +# ./build-deb.sh -v 1.5.5 # Build with version 1.5.5 +# +# Prerequisites: +# - The dpkg-buildpackage package must be installed (provides the dpkg-buildpackage command). +# - The control file must exist at debian/control. +# +# Error Handling: +# The script includes checks to ensure: +# - The version option (-v or --version) is provided. +# - The necessary commands are available. +# - The control file exists at the specified location. +# If any of these checks fail, the script exits with an appropriate error message. + +# Enable strict mode for better error handling +set -euo pipefail + +# Default values +VERSION="" +RELEASE="1" +DEBUG_BUILD=false + +# Function to display usage information +usage() { + echo "Usage: $0 [-v ] [-h] [--dry-run]" + echo " -v, --version : Specify the version (optional)" + echo " -h, --help : Display this help and exit" + echo " -n, --dry-run : Show what would be done, without making any changes" + exit 1 +} + +# Function to check if required commands are available +check_commands() { + local cmds=("dpkg-buildpackage") + for cmd in "${cmds[@]}"; do + if ! command -v "$cmd" &> /dev/null; then + echo "Error: Required command '$cmd' not found. Please install it before running the script." + exit 1 + fi + done +} + +function print_changelog() { +cat < $(date +'%a, %d %b %Y %H:%M:%S %z') +EOF +} + +# Parse options +while [[ "$#" -gt 0 ]]; do + case $1 in + -v|--version) + VERSION="$2" + shift 2 + ;; + -h|--help) + usage + ;; + -n|--dry-run) + DRY_RUN=true + shift + ;; + *) + echo "Unknown option: ($1)" + shift + ;; + esac +done + +export CBDB_FULL_VERSION=$VERSION + +# Set version if not provided +if [ -z "${VERSION}" ]; then + export CBDB_FULL_VERSION=$(./getversion | cut -d'-' -f 1 | cut -d'+' -f 1) +fi + +if [[ ! $CBDB_FULL_VERSION =~ ^[0-9] ]]; then + export CBDB_FULL_VERSION="0.$CBDB_FULL_VERSION" +fi + +if [ -z ${BUILD_NUMBER+x} ]; then + export BUILD_NUMBER=1 +fi + +if [ -z ${BUILD_USER+x} ]; then + export BUILD_USER=github +fi + +export CBDB_PKG_VERSION=${CBDB_FULL_VERSION}-${BUILD_NUMBER}-$(git --git-dir=.git rev-list HEAD --count).$(git --git-dir=.git rev-parse --short HEAD) + +# Check if required commands are available +check_commands + +# Define the control file path +CONTROL_FILE=debian/control + +# Check if the spec file exists +if [ ! -f "$CONTROL_FILE" ]; then + echo "Error: Control file not found at $CONTROL_FILE." + exit 1 +fi + +# Build the rpmbuild command based on options +DEBBUILD_CMD="dpkg-buildpackage -us -uc" + +# Dry-run mode +if [ "${DRY_RUN:-false}" = true ]; then + echo "Dry-run mode: This is what would be done:" + print_changelog + echo "" + echo "$DEBBUILD_CMD" + exit 0 +fi + +# Run debbuild with the provided options +echo "Building DEB with Version $CBDB_FULL_VERSION ..." + +print_changelog > debian/changelog + +if ! eval "$DEBBUILD_CMD"; then + echo "Error: deb build failed." + exit 1 +fi + +# Print completion message +echo "DEB build completed successfully with package $CBDB_PKG_VERSION"